diff --git a/.agents/skills/triton-kernel-writing/SKILL.md b/.agents/skills/triton-kernel-writing/SKILL.md index 6dd6a8520b..a41de016b7 100644 --- a/.agents/skills/triton-kernel-writing/SKILL.md +++ b/.agents/skills/triton-kernel-writing/SKILL.md @@ -22,10 +22,14 @@ description: Write or review Triton kernels for vLLM, with practical guidance fo a small, legible heuristic when workloads need different choices. Use `triton.autotune` only when tuning is critical to performance, such as for a matrix multiplication. Otherwise prioritize simple code and fast startup. -- Be careful to avoid unintended runtime JIT compilation. For example, put - unimportant runtime integer scalars in `do_not_specialize`, especially those - that may alternate between values such as 0 and 1, which can produce - different specialization keys. +- Avoid unintended runtime JIT compilation. Put unimportant runtime integer + scalars in `do_not_specialize`, especially values that alternate between 0 + and 1. Pointer alignment also affects specialization: an offset tensor view + can trigger a new variant after warmup used an aligned allocation. For + scalar-only pointers whose alignment provides no useful optimization, use + `@triton.jit(do_not_specialize_on_alignment=["end_ptr"])`. Retain alignment + specialization when vectorized accesses benefit from it. See the + [Triton JIT options](https://triton-lang.org/main/python-api/generated/triton.jit.html). - The Triton compiler does not guarantee safe ordering when a kernel writes to a pointer and subsequently reads from the same pointer. This pattern must have a `tl.debug_barrier()` between the write and read. The barrier @@ -49,7 +53,9 @@ description: Write or review Triton kernels for vLLM, with practical guidance fo ## Validation - Check correctness at boundary shapes and at sizes that exercise masks and - large offsets. + large offsets. Include offset tensor views, not only fresh allocations. + Warm aligned buffers, then exercise offset views with vLLM's + `--jit-monitor-mode error --jit-monitor-verbose` to catch missing variants. - Choose accumulation and intermediate dtypes explicitly. Test numerically difficult inputs, not only random, well-scaled tensors. - Use `$kernel-microbenchmark` for benchmark construction, measurement, and diff --git a/.buildkite/intel_jobs/test-intel.yaml b/.buildkite/intel_jobs/test-intel.yaml index ef2b4ba073..458bef9518 100644 --- a/.buildkite/intel_jobs/test-intel.yaml +++ b/.buildkite/intel_jobs/test-intel.yaml @@ -98,7 +98,7 @@ steps: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && pip install dpctl && - pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py && + pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py --ignore=v1/core/prefix_cache/test_hybrid_prefix_cache_hit_rate.py && pytest -v -s v1/engine --ignore=v1/engine/test_output_processor.py && pytest -v -s v1/sample --ignore=v1/sample/test_logprobs.py --ignore=v1/sample/test_logprobs_e2e.py -k "not test_topk_only and not test_topp_only and not test_topk_and_topp" && pytest -v -s v1/worker --ignore=v1/worker/test_gpu_model_runner.py --ignore=v1/worker/test_worker_memory_snapshot.py && diff --git a/.buildkite/release-pipeline.yaml b/.buildkite/release-pipeline.yaml index 4af4eaf56e..2b90fef820 100644 --- a/.buildkite/release-pipeline.yaml +++ b/.buildkite/release-pipeline.yaml @@ -316,6 +316,66 @@ steps: - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129" - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129" release-images' + - label: "Build release image - x86_64 - Rubin CUDA 13.4" + depends_on: ~ + id: build-release-image-x86-rubin-cuda-13-4 + if: build.env("NIGHTLY") == "1" + agents: + queue: cpu_queue_release + commands: + - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" + - | + DOCKER_BUILDKIT=1 docker buildx build --load \ + $(bash .buildkite/scripts/docker-build-metadata-args.sh cu134) \ + --platform linux/amd64 \ + --build-arg max_jobs=16 \ + --build-arg USE_SCCACHE=1 \ + --build-arg GIT_REPO_CHECK=1 \ + --build-arg CUDA_VERSION=13.4 \ + --build-arg torch_cuda_arch_list="${CUDA_ARCH_X86}" \ + --build-arg INSTALL_KV_CONNECTORS=true \ + --build-arg INSTALL_RUBIN_PRERELEASE=true \ + --build-arg TRITON_INSTALL_FROM_SOURCE_REPO=https://github.com/triton-lang/triton.git \ + --build-arg TRITON_INSTALL_FROM_SOURCE_REVISION=3f6e41132b5edf639bfb872ad73d4688765e08b8 \ + --build-arg RUN_WHEEL_CHECK=false \ + --build-arg BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda13.4 \ + --build-arg FINAL_BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:26.08-cuda13.4-devel-ubuntu24.04 \ + --target vllm-openai \ + --progress plain \ + -f docker/Dockerfile . + - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu134" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu134" release-images' + + - label: "Build release image - aarch64 - Rubin CUDA 13.4" + depends_on: ~ + id: build-release-image-arm64-rubin-cuda-13-4 + if: build.env("NIGHTLY") == "1" + agents: + queue: arm64_cpu_queue_release + commands: + - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" + - | + DOCKER_BUILDKIT=1 docker buildx build --load \ + $(bash .buildkite/scripts/docker-build-metadata-args.sh cu134) \ + --platform linux/arm64 \ + --build-arg max_jobs=16 \ + --build-arg USE_SCCACHE=1 \ + --build-arg GIT_REPO_CHECK=1 \ + --build-arg CUDA_VERSION=13.4 \ + --build-arg torch_cuda_arch_list="${CUDA_ARCH_AARCH64}" \ + --build-arg INSTALL_KV_CONNECTORS=true \ + --build-arg INSTALL_RUBIN_PRERELEASE=true \ + --build-arg TRITON_INSTALL_FROM_SOURCE_REPO=https://github.com/triton-lang/triton.git \ + --build-arg TRITON_INSTALL_FROM_SOURCE_REVISION=3f6e41132b5edf639bfb872ad73d4688765e08b8 \ + --build-arg RUN_WHEEL_CHECK=false \ + --build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda13.4 \ + --build-arg FINAL_BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:26.08-cuda13.4-devel-ubuntu24.04 \ + --target vllm-openai \ + --progress plain \ + -f docker/Dockerfile . + - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu134" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu134" release-images' + - label: "Build release image - x86_64 - CUDA 13.0 - Ubuntu 24.04" depends_on: ~ id: build-release-image-x86-ubuntu2404 @@ -513,6 +573,20 @@ steps: - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129" - 'bash .buildkite/scripts/annotate-build-artifact.sh "Manifest: CUDA 12.9" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129" release-manifests' + - label: "Create multi-arch manifest - Rubin CUDA 13.4" + depends_on: + - build-release-image-x86-rubin-cuda-13-4 + - build-release-image-arm64-rubin-cuda-13-4 + id: create-multi-arch-manifest-rubin-cuda-13-4 + if: build.env("NIGHTLY") == "1" + agents: + queue: small_cpu_queue_release + commands: + - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" + - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu134 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-cu134 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-cu134 --amend" + - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu134" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "Manifest: Rubin CUDA 13.4" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu134" release-manifests' + - label: "Create multi-arch manifest - CUDA 13.0 - Ubuntu 24.04" depends_on: - build-release-image-x86-ubuntu2404 @@ -585,6 +659,24 @@ steps: DOCKER_BUILDKIT: "1" DOCKERHUB_USERNAME: "vllmbot" + - label: "Publish nightly multi-arch Rubin image to DockerHub - CUDA 13.4" + depends_on: + - create-multi-arch-manifest-rubin-cuda-13-4 + if: build.env("NIGHTLY") == "1" + agents: + queue: small_cpu_queue_release + commands: + - "bash .buildkite/scripts/push-nightly-builds.sh cu134" + # Clean up old nightly builds (keep only last 14) + - "bash .buildkite/scripts/cleanup-nightly-builds.sh cu134-nightly-" + plugins: + - docker-login#v3.0.0: + username: vllmbot + password-env: DOCKERHUB_TOKEN + env: + DOCKER_BUILDKIT: "1" + DOCKERHUB_USERNAME: "vllmbot" + - label: "Publish nightly CPU multi-arch image to DockerHub" depends_on: - build-cpu-release-image-x86 diff --git a/.buildkite/scripts/ci-bake-rocm.sh b/.buildkite/scripts/ci-bake-rocm.sh index 6c5f3308b7..0a9e6e7589 100644 --- a/.buildkite/scripts/ci-bake-rocm.sh +++ b/.buildkite/scripts/ci-bake-rocm.sh @@ -418,7 +418,7 @@ validate_ci_build_context_source() { describe_ci_revision() { git -C "$1" describe --tags --long --abbrev=10 \ - --match '*[0-9]*' "$2" 2>/dev/null + --match 'v[0-9]*' "$2" 2>/dev/null } write_ci_git_archival_metadata() { diff --git a/.buildkite/scripts/ci-clean-log.sh b/.buildkite/scripts/ci-clean-log.sh index e2e21483d5..2fe45f3b77 100644 --- a/.buildkite/scripts/ci-clean-log.sh +++ b/.buildkite/scripts/ci-clean-log.sh @@ -10,11 +10,20 @@ fi INPUT_FILE="$1" +# GNU sed uses "-i [-r]" while BSD/macOS sed needs "-i '' [-E]"; detect once. +if sed --version >/dev/null 2>&1; then + sedi() { sed -i "$@"; } + SED_ERE=-r +else + sedi() { sed -i '' "$@"; } + SED_ERE=-E +fi + # Strip timestamps -sed -i 's/^\[[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}T[0-9]\{2\}:[0-9]\{2\}:[0-9]\{2\}Z\] //' "$INPUT_FILE" +sedi 's/^\[[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}T[0-9]\{2\}:[0-9]\{2\}:[0-9]\{2\}Z\] //' "$INPUT_FILE" # Strip Buildkite inline timestamp markers (ESC _bk;t= BEL) -sed -i 's/\x1B_bk;t=[0-9]*\x07//g' "$INPUT_FILE" +sedi 's/\x1B_bk;t=[0-9]*\x07//g' "$INPUT_FILE" # Strip colorization -sed -i -r 's/\x1B\[[0-9;]*[mK]//g' "$INPUT_FILE" +sedi "$SED_ERE" 's/\x1B\[[0-9;]*[mK]//g' "$INPUT_FILE" diff --git a/.buildkite/scripts/cleanup-nightly-builds.sh b/.buildkite/scripts/cleanup-nightly-builds.sh index b324778b1f..41b04996e4 100755 --- a/.buildkite/scripts/cleanup-nightly-builds.sh +++ b/.buildkite/scripts/cleanup-nightly-builds.sh @@ -56,8 +56,8 @@ get_all_tags() { set -x # Get both last_updated timestamp and tag name, separated by | - # Exclude nightly-dev tags from cleanup - local tags=$(echo "$response" | jq -r --arg prefix "$TAG_PREFIX" '.results[] | select(.name | startswith($prefix)) | select(.name | startswith("nightly-dev") | not) | "\(.last_updated)|\(.name)"') + # Exclude mutable architecture tags and nightly-dev tags from cleanup + local tags=$(echo "$response" | jq -r --arg prefix "$TAG_PREFIX" '.results[] | select(.name | startswith($prefix)) | select(.name != ($prefix + "x86_64") and .name != ($prefix + "aarch64")) | select(.name | startswith("nightly-dev") | not) | "\(.last_updated)|\(.name)"') if [ -z "$tags" ]; then break diff --git a/.buildkite/scripts/hardware_ci/run-amd-test.sh b/.buildkite/scripts/hardware_ci/run-amd-test.sh index 541a8f5525..2705c3b87e 100755 --- a/.buildkite/scripts/hardware_ci/run-amd-test.sh +++ b/.buildkite/scripts/hardware_ci/run-amd-test.sh @@ -1612,6 +1612,13 @@ fi echo "Final commands: $commands" +# Match native CPU jobs even when the container can see AMD devices. +cpu_platform_env=() +if [[ "${VLLM_CI_EXPECTED_GPU_COUNT:-1}" == "0" \ + && "$commands" != *python_only_compile.sh* ]]; then + cpu_platform_env=(-e "VLLM_TARGET_DEVICE=cpu") +fi + standalone_merge_base_env=() if [[ "$commands" == *python_only_compile.sh* ]]; then # The ROCm test image often ships /vllm-workspace without .git. Resolve the @@ -1769,6 +1776,7 @@ else -e "VLLM_CACHE_ROOT=${CONTAINER_CACHE_ROOT}/vllm" \ -e "XDG_CACHE_HOME=${CONTAINER_CACHE_ROOT}/xdg" \ -e "PYTORCH_ROCM_ARCH=" \ + "${cpu_platform_env[@]}" \ "${standalone_merge_base_env[@]}" \ --name "${container_name}" \ "${image_name}" \ diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 4b72256a3f..fe2cfd18c8 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -114,6 +114,7 @@ steps: - label: ":computer: (CPU) Basic Models Other" # TBD timeout_in_minutes: 35 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 no_gpu: true optional: true @@ -124,12 +125,15 @@ steps: - tests/models/test_vision.py - tests/models/test_adapters.py - tests/models/transformers/fusers/ + - tests/model_executor/layers/test_activation.py + - tests/models/test_qwen3_5_mtp_config.py commands: - - pytest -v -s models/test_utils.py models/test_vision.py models/test_adapters.py models/transformers/fusers/ + - pytest -v -s models/test_utils.py models/test_vision.py models/test_adapters.py models/test_qwen3_5_mtp_config.py models/transformers/fusers/ model_executor/layers/test_activation.py - label: ":computer: (CPU) Multimodal Processor Shard %N" # TBD timeout_in_minutes: 90 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 no_gpu: true parallelism: 6 @@ -145,6 +149,7 @@ steps: - label: ":computer: (CPU) V1 Others" # TBD timeout_in_minutes: 45 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 no_gpu: true optional: true @@ -165,6 +170,7 @@ steps: - label: ":computer: (CPU) Async Engine, Inputs, Utils, Worker, Config" # TBD timeout_in_minutes: 80 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 no_gpu: true optional: true @@ -204,6 +210,7 @@ steps: - label: ":computer: (CPU) Rust Frontend Cargo Style + Clippy" # TBD timeout_in_minutes: 35 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 no_gpu: true optional: true @@ -214,11 +221,12 @@ steps: - .buildkite/test_areas/rust_frontend_cargo.yaml - .buildkite/scripts/run-rust-frontend-cargo-ci.sh commands: - - bash .buildkite/scripts/run-rust-frontend-cargo-ci.sh style-clippy + - .buildkite/scripts/run-rust-frontend-cargo-ci.sh style-clippy - label: ":computer: (CPU) Rust Frontend Cargo" # TBD timeout_in_minutes: 35 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 no_gpu: true optional: true @@ -229,11 +237,12 @@ steps: - .buildkite/test_areas/rust_frontend_cargo.yaml - .buildkite/scripts/run-rust-frontend-cargo-ci.sh commands: - - bash .buildkite/scripts/run-rust-frontend-cargo-ci.sh test + - .buildkite/scripts/run-rust-frontend-cargo-ci.sh test - label: ":computer: (CPU) Docker Build Metadata" # TBD timeout_in_minutes: 35 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 no_gpu: true optional: true @@ -264,6 +273,7 @@ steps: - label: ":amd: (MI250) PyTorch Fullgraph Smoke" # TBD timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -281,8 +291,9 @@ steps: #-------------------------------------------------------- mi250 · distributed --------------------------------------------------------# - label: ":amd: (MI250) Pipeline + Context Parallelism" # TBD - timeout_in_minutes: 80 + timeout_in_minutes: 90 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_4 num_gpus: 4 working_dir: "/vllm-workspace/tests" @@ -297,15 +308,20 @@ steps: - tests/distributed/ - vllm/_aiter_ops.py - vllm/platforms/rocm.py + - tests/v1/distributed/test_pp_dp_v2.py + optional: true commands: - pytest -v -s distributed/test_pp_cudagraph.py - pytest -v -s distributed/test_pipeline_parallel.py + - pytest -v -s v1/distributed/test_pp_dp_v2.py #---------------------------------------------------------- mi250 · kernels ----------------------------------------------------------# -- label: ":amd: (MI250) Helion Kernels" # TBD - timeout_in_minutes: 40 +- label: ":amd: (MI250) Helion Kernels Shard %N" + timeout_in_minutes: 45 + parallelism: 2 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -316,26 +332,46 @@ steps: - vllm/platforms/rocm.py commands: - pip install helion==1.4.0 - - pytest -v -s kernels/helion/ + - pytest -v -s kernels/helion/ --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT #----------------------------------------------------- mi250 · models / language -----------------------------------------------------# - label: ":amd: (MI250) Language Models (PPL)" # TBD - timeout_in_minutes: 45 + timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/ - tests/models/language/generation_ppl_test + - vllm/model_executor/models/qwen3_5.py + - vllm/model_executor/models/qwen3_5_mtp.py + - vllm/transformers_utils/configs/qwen3_5.py + - vllm/transformers_utils/configs/qwen3_5_moe.py + - vllm/model_executor/models/qwen2.py + - vllm/model_executor/models/qwen3.py + - vllm/model_executor/models/qwen3_next.py + - vllm/model_executor/models/qwen3_next_mtp.py + - vllm/model_executor/layers/fla/ops/ + - vllm/_aiter_ops.py + - vllm/v1/attention/backends/triton_attn.py + - vllm/v1/attention/backends/rocm_attn.py + - vllm/v1/attention/backends/rocm_aiter_unified_attn.py + - vllm/v1/attention/backends/rocm_aiter_fa.py + - vllm/v1/attention/backends/flex_attention.py + - vllm/v1/attention/ops/ + - vllm/platforms/rocm.py + optional: true commands: - pytest -v -s models/language/generation_ppl_test #---------------------------------------------------- mi250 · models / multimodal ----------------------------------------------------# - label: ":amd: (MI250) Multimodal Models (Standard) 3: llava + qwen2_vl" # TBD - timeout_in_minutes: 45 + timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -351,6 +387,7 @@ steps: - label: ":amd: (MI250) Batch Invariance" # TBD timeout_in_minutes: 35 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -367,8 +404,9 @@ steps: - pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py - label: ":amd: (MI250) CUDAGraph" # TBD - timeout_in_minutes: 45 + timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -378,13 +416,18 @@ steps: - vllm/config/compilation.py - vllm/compilation - vllm/platforms/rocm.py + - vllm/v1/worker/encoder_cudagraph.py + - vllm/v1/worker/encoder_cudagraph_defs.py commands: - pytest -v -s v1/cudagraph/test_cudagraph_dispatch.py - pytest -v -s v1/cudagraph/test_cudagraph_mode.py + - pytest -v -s v1/cudagraph/test_breakable_cudagraph.py + - pytest -v -s v1/cudagraph/test_encoder_cudagraph.py - label: ":amd: (MI250) E2E Core" # TBD timeout_in_minutes: 50 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -398,6 +441,7 @@ steps: - label: ":amd: (MI250) E2E Scheduling" # TBD timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -411,6 +455,7 @@ steps: - label: ":amd: (MI250) V1 Engine" # TBD timeout_in_minutes: 45 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -423,6 +468,7 @@ steps: - vllm/engine/arg_utils.py - vllm/transformers_utils/config.py - vllm/platforms/rocm.py + - vllm/v1/engine/ commands: - pytest -v -s v1/engine/test_preprocess_error_handling.py - pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py @@ -432,6 +478,7 @@ steps: - label: ":amd: (MI250) Spec Decode Draft Model" # TBD timeout_in_minutes: 60 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -447,8 +494,9 @@ steps: - pytest -v -s v1/e2e/spec_decode/draft_model/ - label: ":amd: (MI250) Spec Decode Speculators + MTP" # TBD - timeout_in_minutes: 65 + timeout_in_minutes: 75 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -461,6 +509,7 @@ steps: - vllm/transformers_utils/configs/speculators/ - tests/v1/e2e/spec_decode/ - vllm/platforms/rocm.py + - vllm/v1/attention/backends/ commands: - pytest -v -s v1/e2e/spec_decode/speculators/ - pytest -v -s v1/e2e/spec_decode/mtp/ @@ -468,7 +517,7 @@ steps: #------------------------------------------------------------- mi250 · misc ------------------------------------------------------------# - label: ":amd: (MI300) Python-only Installation" # TBD - timeout_in_minutes: 25 + timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -501,14 +550,16 @@ steps: - tests/basic_correctness/test_basic_correctness - tests/basic_correctness/test_cpu_offload - tests/basic_correctness/test_mem.py + optional: true commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s basic_correctness/test_mem.py - VLLM_TARGET_TEST_SUITE=MI300 pytest -v -s basic_correctness/test_basic_correctness.py - pytest -v -s basic_correctness/test_cpu_offload.py -- label: ":amd: (MI300) Distributed Models" # TBD +- label: ":amd: (MI300) Distributed Models Shard %N" timeout_in_minutes: 80 + parallelism: 3 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_2 @@ -531,21 +582,17 @@ steps: - tests/basic_correctness/ - tests/model_executor/model_loader/test_sharded_state_loader.py - tests/models/ + - vllm/v1/worker/ + - tests/conftest.py + - tests/utils.py + - tests/models/test_vision.py commands: - - TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)' - - HIP_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)' - - pytest models/transformers/test_backend.py -v -s -m 'distributed(num_gpus=2)' - - pytest models/language -v -s -m 'distributed(num_gpus=2)' - - pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py - - pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)' - - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)' - - pytest -v -s models/test_vision.py -m 'distributed(num_gpus=2)' - - VLLM_ROCM_USE_AITER=1 pytest -v -s models/test_hyv4_rocm.py -m 'distributed(num_gpus=2)' + - case "$$BUILDKITE_PARALLEL_JOB" in 0) TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)' && HIP_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)' ;; 1) pytest models/transformers/test_backend.py -v -s -m 'distributed(num_gpus=2)' && pytest models/language -v -s -m 'distributed(num_gpus=2)' && VLLM_ROCM_USE_AITER=1 pytest -v -s models/test_hyv4_rocm.py -m 'distributed(num_gpus=2)' ;; 2) pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py && pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)' && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)' && pytest -v -s models/test_vision.py -m 'distributed(num_gpus=2)' ;; *) exit 2 ;; esac #-------------------------------------------------------- mi300 · benchmarks ---------------------------------------------------------# - label: ":amd: (MI300) Benchmarks CLI" # TBD - timeout_in_minutes: 35 + timeout_in_minutes: 40 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -554,13 +601,14 @@ steps: source_file_dependencies: - vllm/ - tests/benchmarks/ + - rust/src/bench/tests/python_serve_flags.txt commands: - pytest -v -s benchmarks/ #---------------------------------------------------------- mi300 · compile ----------------------------------------------------------# - label: ":amd: (MI300) PyTorch Compilation" # TBD - timeout_in_minutes: 45 + timeout_in_minutes: 90 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -576,6 +624,29 @@ steps: - csrc/ - tests/compile - vllm/platforms/rocm.py + - vllm/__init__.py + - vllm/_aiter_ops.py + - vllm/_custom_ops.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/env_override.py + - vllm/envs.py + - vllm/forward_context.py + - vllm/inputs/ + - vllm/ir/ + - vllm/kernels/ + - vllm/logger.py + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/platforms/ + - vllm/plugins/ + - vllm/sampling_params.py + - vllm/sequence.py + - vllm/transformers_utils/ + - vllm/triton_utils/ + - vllm/utils/ + - vllm/v1/ commands: - "find compile/ -maxdepth 1 -name 'test_*.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'" @@ -596,6 +667,7 @@ steps: - tests/compile/fusions_e2e/ - vllm/_aiter_ops.py - vllm/platforms/rocm.py + optional: true commands: - rocm-smi - pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "llama-3" @@ -615,6 +687,7 @@ steps: - tests/compile/fusions_e2e/ - vllm/_aiter_ops.py - vllm/platforms/rocm.py + optional: true commands: - rocm-smi # Run all models and attn backends but only Inductor partition and native custom ops @@ -631,11 +704,37 @@ steps: source_file_dependencies: - vllm/ - tests/compile/passes + - vllm/platforms/rocm.py + - vllm/__init__.py + - vllm/_aiter_ops.py + - vllm/_custom_ops.py + - vllm/compilation/ + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/env_override.py + - vllm/envs.py + - vllm/forward_context.py + - vllm/inputs/ + - vllm/ir/ + - vllm/kernels/ + - vllm/logger.py + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/platforms/ + - vllm/plugins/ + - vllm/sampling_params.py + - vllm/sequence.py + - vllm/transformers_utils/ + - vllm/triton_utils/ + - vllm/utils/ + - vllm/v1/ + optional: true commands: - pytest -s -v compile/passes --ignore compile/passes/distributed - label: ":amd: (MI300) PyTorch Fullgraph" # TBD - timeout_in_minutes: 35 + timeout_in_minutes: 90 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -649,11 +748,33 @@ steps: - csrc/ - tests/compile - vllm/platforms/rocm.py + - vllm/__init__.py + - vllm/_aiter_ops.py + - vllm/_custom_ops.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/env_override.py + - vllm/envs.py + - vllm/forward_context.py + - vllm/inputs/ + - vllm/ir/ + - vllm/kernels/ + - vllm/logger.py + - vllm/multimodal/ + - vllm/platforms/ + - vllm/plugins/ + - vllm/sampling_params.py + - vllm/sequence.py + - vllm/transformers_utils/ + - vllm/triton_utils/ + - vllm/utils/ + - vllm/v1/ commands: - - pytest -v -s compile/fullgraph/test_full_graph.py + - find compile/fullgraph/ -name 'test_*.py' -not -name 'test_full_cudagraph.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}' - label: ":amd: (MI300) PyTorch Nightly Dependency Override Check" # TBD - timeout_in_minutes: 25 + timeout_in_minutes: 30 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -704,7 +825,7 @@ steps: - pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py::test_tp2_ar_rms_fusions -k "dynamo_partition" - label: ":amd: (MI300) Distributed Compile + RPC" # TBD - timeout_in_minutes: 45 + timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_2 @@ -722,9 +843,9 @@ steps: - tests/compile/test_wrapper.py - tests/entrypoints/llm/test_collective_rpc.py - vllm/platforms/rocm.py + optional: true commands: - pytest -v -s entrypoints/llm/test_collective_rpc.py - - pytest -v -s ./compile/fullgraph/test_basic_correctness.py - pytest -v -s ./compile/test_wrapper.py #----------------------------------------------------------- mi300 · cuda ------------------------------------------------------------# @@ -738,9 +859,16 @@ steps: source_file_dependencies: - vllm/ - tests/cuda + - vllm/envs.py + - vllm/logger.py + - vllm/platforms/ + - vllm/plugins/ + - vllm/utils/ + optional: true commands: - pytest -v -s cuda/test_cuda_context.py - pytest -v -s cuda/test_platform_no_cuda_init.py + - pytest -v -s cuda/test_cuda_compatibility_path.py #-------------------------------------------------------- mi300 · detokenizer --------------------------------------------------------# @@ -756,6 +884,19 @@ steps: - tests/detokenizer - tests/multimodal - tests/utils_ + - vllm/assets/ + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/inputs/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/tokenizers/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ commands: - pytest -v -s detokenizer - pytest -v -s -m 'not cpu_test' multimodal @@ -782,7 +923,7 @@ steps: - pytest -v -s distributed/test_shm_storage.py - label: ":amd: (MI300) EPLB Algorithm" # TBD - timeout_in_minutes: 25 + timeout_in_minutes: 30 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -814,7 +955,7 @@ steps: - pytest -v -s distributed/test_eplb_spec_decode.py - label: ":amd: (MI300) Distributed Features" # TBD - timeout_in_minutes: 45 + timeout_in_minutes: 90 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_2 @@ -831,6 +972,26 @@ steps: - examples/features/data_parallel/data_parallel_offline.py - vllm/_aiter_ops.py - vllm/platforms/rocm.py + - csrc/custom_quickreduce.cu + - csrc/ops.h + - csrc/torch_bindings.cpp + - vllm/model_executor/layers/ + - vllm/entrypoints/llm.py + - vllm/config/parallel.py + - vllm/v1/engine/ + - vllm/v1/executor/ + - vllm/v1/worker/ + - vllm/_custom_ops.py + - vllm/envs.py + - tests/distributed/test_rocm_aiter_custom_ar.py + - tests/distributed/test_rocm_quick_reduce.py + - tests/distributed/test_quick_all_reduce.py + - tests/v1/e2e/general/test_rocm_aiter_custom_ar.py + - tests/utils.py + - examples/rl/rlhf_async_new_apis.py + - tests/distributed/test_weight_transfer.py + - tests/distributed/test_packed_tensor.py + optional: true commands: - pytest -v -s tests/distributed/test_context_parallel.py - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_async_new_apis.py @@ -839,6 +1000,10 @@ steps: - pytest -v -s tests/v1/distributed/test_dbo.py - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py - pytest -v -s tests/distributed/test_packed_tensor.py + - pytest -v -s tests/distributed/test_rocm_aiter_custom_ar.py + - pytest -v -s tests/v1/e2e/general/test_rocm_aiter_custom_ar.py + - pytest -v -s tests/distributed/test_rocm_quick_reduce.py + - pytest -v -s tests/distributed/test_quick_all_reduce.py - label: ":amd: (MI300) Distributed" # TBD timeout_in_minutes: 40 @@ -993,7 +1158,7 @@ steps: #---------------------------------------------------------- mi300 · engine -----------------------------------------------------------# - label: ":amd: (MI300) Engine Core" # TBD - timeout_in_minutes: 35 + timeout_in_minutes: 40 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1008,6 +1173,17 @@ steps: - tests/test_vllm_port - tests/jit_monitor/test_hooks.py - tests/jit_monitor/test_hooks_gpu.py + - vllm/compilation/ + - vllm/config/ + - vllm/engine/ + - vllm/entrypoints/logger.py + - vllm/envs.py + - vllm/logger.py + - vllm/logging_utils/ + - vllm/platforms/ + - vllm/sequence.py + - vllm/triton_utils/ + - vllm/utils/ commands: - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py jit_monitor/test_hooks.py jit_monitor/test_hooks_gpu.py @@ -1049,8 +1225,9 @@ steps: - pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process - pytest -v -s entrypoints/llm/offline_mode # Needs to avoid interference with other tests -- label: ":amd: (MI300) Entrypoints Integration (API Server)" # TBD +- label: ":amd: (MI300) Entrypoints Integration (API Server) %N" timeout_in_minutes: 65 + parallelism: 4 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1063,12 +1240,12 @@ steps: - tests/entrypoints/scale_out commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc - - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc - - pytest -v -s entrypoints/scale_out + - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB + - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc; fi + - pytest -v -s entrypoints/scale_out --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - label: ":amd: (MI300) Entrypoints Integration (OpenAI API completion)" # TBD - timeout_in_minutes: 40 + timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1081,10 +1258,10 @@ steps: commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py - - pytest -v -s entrypoints/openai/ --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/correctness + - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/correctness - label: ":amd: (MI300) Entrypoints Integration (OpenAI API chat_completion)" # TBD - timeout_in_minutes: 55 + timeout_in_minutes: 70 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1114,7 +1291,6 @@ steps: - tests/entrypoints/cohere - tests/entrypoints/generate commands: - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s tool_use - pytest -v -s entrypoints/tool_parsers - pytest -v -s entrypoints/generate @@ -1122,7 +1298,7 @@ steps: - pytest -v -s entrypoints/cohere - label: ":amd: (MI300) Entrypoints Integration (Responses API)" # TBD - timeout_in_minutes: 45 + timeout_in_minutes: 50 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1133,7 +1309,6 @@ steps: - vllm/ - tests/entrypoints/openai/responses commands: - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/openai/responses - label: ":amd: (MI300) Entrypoints Integration (Speech to Text)" # TBD @@ -1146,6 +1321,7 @@ steps: source_file_dependencies: - vllm/ - tests/entrypoints/speech_to_text + optional: true commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/speech_to_text @@ -1165,23 +1341,8 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/multimodal -- label: ":amd: (MI300) Entrypoints Integration (Pooling)" # TBD - timeout_in_minutes: 65 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - optional: true - fast_check: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/entrypoints/pooling - commands: - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s entrypoints/pooling - - label: ":amd: (MI300) OpenAI API Correctness" # TBD - timeout_in_minutes: 25 + timeout_in_minutes: 30 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1220,11 +1381,12 @@ steps: - .buildkite/scripts/scheduled_integration_test/ - vllm/_aiter_ops.py - vllm/platforms/rocm.py + optional: true commands: - bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_prefetch_offload.sh 0.25 200 8030 - label: ":amd: (MI300) LM Eval Small Models" # TBD - timeout_in_minutes: 40 + timeout_in_minutes: 45 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1238,11 +1400,12 @@ steps: - vllm/v1/attention/selector.py - vllm/_aiter_ops.py - vllm/platforms/rocm.py + optional: true commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small.txt - label: ":amd: (MI300) MRCR Eval Small Models" # TBD - timeout_in_minutes: 35 + timeout_in_minutes: 40 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1275,11 +1438,12 @@ steps: - vllm/v1/attention/selector.py - vllm/_aiter_ops.py - vllm/platforms/rocm.py + - .buildkite/lm-eval-harness/ commands: - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-small-rocm.txt - label: ":amd: (MI300) Multimodal Accuracy Eval (Small Models)" # TBD - timeout_in_minutes: 30 + timeout_in_minutes: 35 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1295,7 +1459,7 @@ steps: - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt --tp-size=1 - label: ":amd: (MI300) GPQA Eval (GPT-OSS)" # TBD - timeout_in_minutes: 30 + timeout_in_minutes: 40 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_2 @@ -1317,7 +1481,7 @@ steps: - uv pip install --system 'gpt-oss[eval]==0.0.5' - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx942.txt -- label: ":amd: (MI300) LM Eval Small Models FP8 + Mixed" # TBD +- label: ":amd: (MI300) LM Eval Small Models Distributed" timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false @@ -1333,6 +1497,8 @@ steps: - vllm/v1/attention/selector.py - vllm/_aiter_ops.py - vllm/platforms/rocm.py + - tests/evals/gsm8k/configs/models-mi3xx-fp8-and-mixed.txt + optional: true commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx-fp8-and-mixed.txt @@ -1517,7 +1683,7 @@ steps: #--------------------------------------------------------- mi300 · examples ----------------------------------------------------------# - label: ":amd: (MI300) Examples" # TBD - timeout_in_minutes: 50 + timeout_in_minutes: 75 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1529,27 +1695,27 @@ steps: - examples/ - vllm/platforms/rocm.py commands: - - pip install tensorizer # Basic - - python3 basic/offline_inference/chat.py - - python3 basic/offline_inference/generate.py --model facebook/opt-125m - - python3 basic/offline_inference/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10 - - python3 basic/offline_inference/classify.py - - python3 basic/offline_inference/embed.py - - python3 basic/offline_inference/score.py # Multi-modal models - - python3 generate/multimodal/audio_language_offline.py --seed 0 - - python3 generate/multimodal/vision_language_offline.py --seed 0 - - python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0 - - python3 generate/multimodal/encoder_decoder_multimodal_offline.py --model-type whisper --seed 0 # Pooling models - - python3 pooling/embed/vision_embedding_offline.py --seed 0 # Features demo - - python3 features/automatic_prefix_caching/prefix_caching_offline.py - - python3 deployment/llm_engine_example.py - - python3 features/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 features/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors - - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048 - - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536 + - pip install --no-deps tensorizer + - python3 basic/offline_inference/chat.py + - python3 basic/offline_inference/generate.py --model facebook/opt-125m + - python3 basic/offline_inference/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10 + - python3 basic/offline_inference/classify.py + - python3 basic/offline_inference/embed.py + - python3 basic/offline_inference/score.py + - python3 generate/multimodal/audio_language_offline.py --seed 0 + - python3 generate/multimodal/vision_language_offline.py --seed 0 + - python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0 + - python3 generate/multimodal/encoder_decoder_multimodal_offline.py --model-type whisper --seed 0 + - python3 pooling/embed/vision_embedding_offline.py --seed 0 + - python3 features/automatic_prefix_caching/prefix_caching_offline.py + - python3 deployment/llm_engine_example.py + - python3 features/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 features/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors + - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048 + - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536 #---------------------------------------------------------- mi300 · kernels ----------------------------------------------------------# @@ -1565,6 +1731,8 @@ steps: - vllm/kernels - vllm/_aiter_ops.py - vllm/platforms/rocm.py + - tests/ir/ + - tests/kernels/ir/ commands: - pytest -v -s tests/ir - pytest -v -s tests/kernels/ir @@ -1584,15 +1752,16 @@ steps: - tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py - tests/kernels/attention/test_rocm_aiter_mla_fp8_support.py - tests/kernels/attention/test_rocm_aiter_mla_op_registration.py + optional: true commands: - - rocm-smi + - amd-smi - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode.py - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_fp8_support.py - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_op_registration.py - label: ":amd: (MI300) Attention Kernels Shard %N" # TBD - timeout_in_minutes: 70 + timeout_in_minutes: 90 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1607,27 +1776,10 @@ steps: - vllm/_aiter_ops.py - vllm/envs.py - vllm/platforms/rocm.py + - vllm/utils/flashinfer.py commands: - pytest -v -s kernels/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT -- label: ":amd: (MI300) Core Operation Kernels Shard %N" # TBD - timeout_in_minutes: 40 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - parallelism: 3 - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - csrc/ - - tests/kernels/core - - tests/kernels/test_top_k_per_row.py - - tests/kernels/test_concat_mla_q.py - - vllm/model_executor/layers/rotary_embedding/ - - vllm/_aiter_ops.py - - vllm/platforms/rocm.py - commands: - - pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_concat_mla_q.py kernels/test_top_k_per_row.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - - label: ":amd: (MI300) MiniMax Reduce RMS Kernels" # TBD timeout_in_minutes: 45 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] @@ -1643,6 +1795,12 @@ steps: - vllm/platforms/rocm.py - tests/kernels/core/test_minimax_reduce_rms.py - tests/utils.py + - csrc/libtorch_stable/minimax_reduce_rms_kernel.cu + - csrc/libtorch_stable/minimax_reduce_rms_kernel.h + - csrc/libtorch_stable/ops.h + - csrc/libtorch_stable/torch_bindings.cpp + - vllm/model_executor/layers/mamba/linear/minimax_linear_attn.py + optional: true commands: - pytest -v -s kernels/core/test_minimax_reduce_rms.py @@ -1667,7 +1825,7 @@ steps: - pytest -v -s models/kimi_k3/test_kda.py models/kimi_k3/test_kda_metadata.py - label: ":amd: (MI300) Mamba Kernels" # TBD - timeout_in_minutes: 50 + timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1676,13 +1834,14 @@ steps: source_file_dependencies: - csrc/mamba/ - tests/kernels/mamba + - vllm/model_executor/layers/mamba/gdn - vllm/model_executor/layers/mamba/ops - vllm/platforms/rocm.py commands: - pytest -v -s kernels/mamba - label: ":amd: (MI300) MoE Kernels Shard %N" # TBD - timeout_in_minutes: 50 + timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1708,8 +1867,9 @@ steps: - pytest -v -s kernels/moe/test_modular_oai_triton_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_MOE=1 pytest -v -s kernels/moe/test_modular_kernel_combinations.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT -- label: ":amd: (MI300) Quantization Kernels" # TBD - timeout_in_minutes: 55 +- label: ":amd: (MI300) Quantization Kernels Shard %N" + timeout_in_minutes: 120 + parallelism: 6 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1729,8 +1889,10 @@ steps: - vllm/platforms/rocm.py - vllm/model_executor/kernels/ - vllm/v1/attention/backends/rocm_aiter_fa.py + - vllm/config/ + - tests/kernels/quantization/test_rocm_skinny_gemms.py commands: - - pytest -v -s kernels/quantization + - pytest -v -s kernels/quantization --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - label: ":amd: (MI300) DeepEP FP8 MoE Kernels" # TBD timeout_in_minutes: 55 @@ -1754,7 +1916,7 @@ steps: #----------------------------------------------------------- mi300 · lora ------------------------------------------------------------# - label: ":amd: (MI300) LoRA Shard %N" # TBD - timeout_in_minutes: 40 + timeout_in_minutes: 85 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1768,8 +1930,9 @@ steps: commands: - pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_chatglm3_tp.py --ignore=lora/test_llama_tp.py --ignore=lora/test_qwen3_with_multi_loras.py --ignore=lora/test_olmoe_tp.py --ignore=lora/test_deepseekv2_tp.py --ignore=lora/test_gptoss_tp.py --ignore=lora/test_qwen3moe_tp.py --ignore=lora/test_qwen35_densemodel_lora.py -- label: ":amd: (MI300) LoRA TP (Distributed)" # TBD +- label: ":amd: (MI300) LoRA TP (Distributed) %N" timeout_in_minutes: 65 + parallelism: 4 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_4 @@ -1780,14 +1943,17 @@ steps: - vllm/model_executor/layers/fused_moe/ - tests/lora - vllm/platforms/rocm.py + optional: true commands: - - pytest -v -s -x lora/test_chatglm3_tp.py - - pytest -v -s -x lora/test_llama_tp.py - - pytest -v -s -x lora/test_qwen3_with_multi_loras.py - - pytest -v -s -x lora/test_olmoe_tp.py - - pytest -v -s -x lora/test_gptoss_tp.py - - pytest -v -s -x lora/test_qwen35_densemodel_lora.py - - pytest -v -s -x lora/test_gemma4_tp.py + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - case "$$BUILDKITE_PARALLEL_JOB" in 0|1|2|3) ;; *) echo "unexpected BUILDKITE_PARALLEL_JOB=$$BUILDKITE_PARALLEL_JOB" && exit 1;; esac + - if [ "$$BUILDKITE_PARALLEL_JOB" = "3" ]; then pytest -v -s -x lora/test_chatglm3_tp.py; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then pytest -v -s -x lora/test_llama_tp.py; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "3" ]; then pytest -v -s -x lora/test_qwen3_with_multi_loras.py; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "2" ]; then pytest -v -s -x lora/test_olmoe_tp.py; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s -x lora/test_gptoss_tp.py; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then pytest -v -s -x lora/test_qwen35_densemodel_lora.py; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "2" ]; then pytest -v -s -x lora/test_gemma4_tp.py; fi #------------------------------------------------------ mi300 · model_executor -------------------------------------------------------# @@ -1806,160 +1972,19 @@ steps: - tests/entrypoints/openai/completion/test_tensorizer_entrypoint.py - vllm/_aiter_ops.py - vllm/platforms/rocm.py + - vllm/model_executor/warmup + - tests/model_executor/test_jit_warmup.py + - tests/model_executor/test_jit_warmup_cutedsl_launcher.py + - tests/model_executor/test_jit_warmup_triton_launcher.py commands: - apt-get update && apt-get install -y curl libsodium23 - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s model_executor -m '(not slow_test)' - - pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py - -#---------------------------------------------------- mi300 · model_runner_v2 -------------------------------------------------------# - -- label: ":amd: (MI300) Model Runner V2 Core" # TBD - timeout_in_minutes: 55 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/v1/worker/gpu/ - - vllm/v1/worker/gpu_worker.py - - vllm/v1/core/sched/ - - vllm/v1/attention/ - - tests/v1/engine/test_llm_engine.py - - tests/v1/e2e/ - - tests/entrypoints/llm/test_struct_output_generate.py - - vllm/platforms/rocm.py - commands: - - set -x - - export VLLM_USE_V2_MODEL_RUNNER=1 - - pytest -v -s v1/engine/test_llm_engine.py -k "not test_engine_metrics" - - pytest -v -s v1/e2e/general/test_async_scheduling.py -k "not ngram" - - pytest -v -s v1/e2e/general/test_context_length.py - - pytest -v -s v1/e2e/general/test_min_tokens.py - - pytest -v -s entrypoints/llm/test_struct_output_generate.py -k "xgrammar and not speculative_config6 and not speculative_config7 and not speculative_config8 and not speculative_config0" - -- label: ":amd: (MI300) Model Runner V2 Examples" # TBD - timeout_in_minutes: 40 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - optional: true - working_dir: "/vllm-workspace/examples" - source_file_dependencies: - - vllm/v1/worker/gpu/ - - vllm/v1/core/sched/ - - vllm/v1/worker/gpu_worker.py - - examples/basic/offline_inference/ - - examples/generate/multimodal/ - - examples/features/ - - examples/pooling/embed/vision_embedding_offline.py - - examples/features/tensorize_vllm_model.py - - examples/deployment/llm_engine_example.py - - vllm/platforms/rocm.py - commands: - - set -x - - export VLLM_USE_V2_MODEL_RUNNER=1 - - pip install tensorizer - - python3 basic/offline_inference/chat.py - - python3 basic/offline_inference/generate.py --model facebook/opt-125m - - python3 generate/multimodal/audio_language_offline.py --seed 0 - - python3 generate/multimodal/vision_language_offline.py --seed 0 - - python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0 - - python3 generate/multimodal/encoder_decoder_multimodal_offline.py --model-type whisper --seed 0 - - python3 pooling/embed/vision_embedding_offline.py --seed 0 - - python3 features/automatic_prefix_caching/prefix_caching_offline.py - - python3 deployment/llm_engine_example.py - - python3 features/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 features/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors - - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048 - - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536 - -- label: ":amd: (MI300) Model Runner V2 Distributed" # TBD - timeout_in_minutes: 45 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_2 - num_gpus: 2 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/v1/worker/gpu/ - - vllm/v1/worker/gpu_worker.py - - tests/basic_correctness/test_basic_correctness.py - - tests/v1/distributed/test_async_llm_dp.py - - tests/v1/distributed/test_eagle_dp.py - - vllm/platforms/rocm.py - commands: - - set -x - - export VLLM_USE_V2_MODEL_RUNNER=1 - - TARGET_TEST_SUITE=MI300 pytest -v -s basic_correctness/test_basic_correctness.py -m 'distributed(num_gpus=2)' -k "not ray and not True" - - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py -k "not ray" - - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py - -- label: ":amd: (MI300) Model Runner V2 Pipeline Parallelism" # TBD - timeout_in_minutes: 65 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_4 - num_gpus: 4 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/v1/worker/gpu/ - - vllm/v1/worker/gpu_worker.py - - tests/distributed/test_pipeline_parallel.py - - tests/distributed/test_pp_cudagraph.py - - vllm/platforms/rocm.py - commands: - - set -x - - export VLLM_USE_V2_MODEL_RUNNER=1 - - pytest -v -s distributed/test_pipeline_parallel.py -k "not ray" - - pytest -v -s distributed/test_pp_cudagraph.py -k "not ray" - -- label: ":amd: (MI300) Model Runner V2 Spec Decode" # TBD - timeout_in_minutes: 90 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/v1/worker/gpu/ - - vllm/v1/worker/gpu_worker.py - - tests/v1/spec_decode/test_max_len.py - - tests/v1/spec_decode/test_rejection_sampler_utils.py - - tests/v1/spec_decode/test_synthetic_rejection_sampler_utils.py - - tests/v1/e2e/spec_decode/ - - vllm/platforms/rocm.py - commands: - - set -x - - export VLLM_USE_V2_MODEL_RUNNER=1 - - pytest -v -s v1/spec_decode/test_max_len.py -k "eagle or mtp" - - pytest -v -s v1/spec_decode/test_rejection_sampler_utils.py - - pytest -v -s v1/spec_decode/test_synthetic_rejection_sampler_utils.py - - pytest -v -s v1/e2e/spec_decode/eagle/ - - pytest -v -s v1/e2e/spec_decode/speculators/ - - pytest -v -s v1/e2e/spec_decode/mtp/ + - export PYTHONFAULTHANDLER=1 + - pytest -v -s model_executor -m '(not slow_test)' --timeout=900 --timeout-method=thread + - pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py --timeout=900 --timeout-method=thread #------------------------------------------------------ mi300 · models / basic -------------------------------------------------------# -- label: ":amd: (MI300) Basic Models (Extra Initialization) Shard %N" # TBD - timeout_in_minutes: 60 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - parallelism: 6 - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/model_executor/models/ - - vllm/model_executor/layers/ - - tests/models/test_initialization.py - - tests/models/registry.py - - vllm/_aiter_ops.py - - vllm/platforms/rocm.py - commands: - - pytest -v -s models/test_initialization.py -k 'not test_can_initialize_small_subset' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - - label: ":amd: (MI300) Basic Models (Initialization)" # TBD timeout_in_minutes: 30 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] @@ -1970,6 +1995,7 @@ steps: - vllm/ - tests/models/test_initialization.py - tests/models/registry.py + optional: true commands: - pytest -v -s models/test_initialization.py::test_can_initialize_small_subset @@ -1987,8 +2013,9 @@ steps: - tests/models/test_registry.py - tests/models/test_hyv4_rocm.py - tests/models/test_vision.py + - tests/models/test_deepseek_v4_vl_rocm.py commands: - - pytest -v -s models/test_terratorch.py models/transformers/test_backend.py models/test_registry.py + - pytest -v -s models/test_terratorch.py models/transformers/test_backend.py models/test_registry.py models/test_deepseek_v4_vl_rocm.py - VLLM_ROCM_USE_AITER=1 pytest -v -s models/test_hyv4_rocm.py -m 'not distributed' - pytest -v -s models/test_vision.py::test_simple_mrope_vision_model_spatial_merge @@ -2007,8 +2034,9 @@ steps: commands: - pytest -v -s models/language/pooling_mteb_test -- label: ":amd: (MI300) Language Models (Extended Pooling)" # TBD - timeout_in_minutes: 80 +- label: ":amd: (MI300) Language Models (Extended Pooling) Shard %N" + timeout_in_minutes: 95 + parallelism: 4 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -2018,10 +2046,10 @@ steps: - vllm/ - tests/models/language/pooling commands: - - pytest -v -s models/language/pooling -m 'not core_model' + - pytest -v -s models/language/pooling -m 'not core_model' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - label: ":amd: (MI300) Language Models (Standard)" # TBD - timeout_in_minutes: 45 + timeout_in_minutes: 60 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -2072,22 +2100,6 @@ steps: - CAUSAL_CONV1D_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0' - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)' -- label: ":amd: (MI300) Language Models (Hybrid) Shard %N" # TBD - timeout_in_minutes: 65 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - parallelism: 2 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/models/language/generation - commands: - - MAMBA_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@fix-rocm-7.0-warp-size-constexpr' - - CAUSAL_CONV1D_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0' - - pytest -v -s models/language/generation -m hybrid_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - #---------------------------------------------------- mi300 · models / multimodal ----------------------------------------------------# - label: ":amd: (MI300) Multimodal Models (Extended Generation 1)" # TBD @@ -2105,8 +2117,9 @@ steps: - pytest -v -s models/multimodal/generation -m 'not core_model' --ignore models/multimodal/generation/test_common.py - pytest -v -s models/multimodal/test_mapping.py -- label: ":amd: (MI300) Multimodal Models (Extended Generation 2)" # TBD - timeout_in_minutes: 65 +- label: ":amd: (MI300) Multimodal Models (Extended Generation 2) Shard %N" + timeout_in_minutes: 100 + parallelism: 4 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -2116,11 +2129,11 @@ steps: - vllm/ - tests/models/multimodal/generation commands: - - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model' + - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - label: ":amd: (MI300) Multimodal Models (Extended Generation 3)" # TBD - timeout_in_minutes: 55 + timeout_in_minutes: 90 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -2133,7 +2146,7 @@ steps: - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model' - label: ":amd: (MI300) Multimodal Models (Standard) 1: qwen2" # TBD - timeout_in_minutes: 45 + timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -2147,7 +2160,7 @@ steps: - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model - label: ":amd: (MI300) Multimodal Models (Standard) 3: llava + qwen2_vl" # TBD - timeout_in_minutes: 40 + timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -2157,27 +2170,13 @@ steps: - vllm/ - tests/models/multimodal/generation - tests/models/multimodal/test_mapping.py + - tests/models/multimodal commands: - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma" - pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model -- label: ":amd: (MI300) Multimodal Models (Standard) 4: other + whisper" # TBD - timeout_in_minutes: 70 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/models/multimodal - commands: - - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_mm_prefix_lm.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/generation/test_vit_cudagraph.py --ignore models/multimodal/processing - - pytest -v -s models/multimodal/generation/test_vit_cudagraph.py -m core_model - - pytest -v -s models/multimodal/generation/test_memory_leak.py -m core_model - - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model - - label: ":amd: (MI300) Multimodal Processor Shard %N" # TBD - timeout_in_minutes: 110 + timeout_in_minutes: 115 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -2205,7 +2204,7 @@ steps: - pytest -v -s models/multimodal/pooling -m 'not core_model' - label: ":amd: (MI300) Multimodal Models (Standard) 2: qwen3 + gemma" # TBD - timeout_in_minutes: 50 + timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -2233,7 +2232,9 @@ steps: - vllm/platforms/rocm.py - tests/models/quantization - vllm/model_executor/model_loader/ + optional: true commands: + - unset VLLM_USE_V2_MODEL_RUNNER - pytest -v -s models/quantization #-------------------------------------------------- mi300 · models / transformers ---------------------------------------------------# @@ -2311,7 +2312,7 @@ steps: #---------------------------------------------------------- mi300 · plugins ----------------------------------------------------------# - label: ":amd: (MI300) Plugin Integration" # TBD - timeout_in_minutes: 35 + timeout_in_minutes: 50 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_2 @@ -2324,43 +2325,43 @@ steps: - vllm/platforms/rocm.py commands: # BEGIN: platform plugin and general plugin tests, all the code in-between runs on dummy platform + # END: platform plugin tests + # BEGIN: `io_processor` plugins test, all the code in between uses the `prithvi_io_processor` plugin + # END: `io_processor` plugins test + # BEGIN: `bge_m3_sparse io_processor` test + # END: `bge_m3_sparse io_processor` test + # BEGIN: `colbert_query io_processor` test + # END: `colbert_query io_processor` test + # BEGIN: `stat_logger` plugins test + # END: `stat_logger` plugins test + # BEGIN: `endpoint` plugins test + # END: `endpoint` plugins test + # BEGIN: other tests - pip install -e ./plugins/vllm_add_dummy_platform - pytest -v -s plugins_tests/test_platform_plugins.py - pip uninstall vllm_add_dummy_platform -y - # END: platform plugin tests - # BEGIN: `io_processor` plugins test, all the code in between uses the `prithvi_io_processor` plugin + - pytest -v -s ./plugins_tests/test_io_processor_plugins.py - pip install -e ./plugins/prithvi_io_processor_plugin - - pytest -v -s plugins_tests/test_io_processor_plugins.py - pytest -v -s plugins_tests/test_terratorch_io_processor_plugins.py - pip uninstall prithvi_io_processor_plugin -y - # END: `io_processor` plugins test - # BEGIN: `bge_m3_sparse io_processor` test - pip install -e ./plugins/bge_m3_sparse_plugin - pytest -v -s plugins_tests/test_bge_m3_sparse_io_processor_plugins.py - pip uninstall bge_m3_sparse_plugin -y - # END: `bge_m3_sparse io_processor` test - # BEGIN: `colbert_query io_processor` test - pip install -e ./plugins/colbert_query_plugin - pytest -v -s plugins_tests/test_colbert_query_io_processor_plugins.py - pip uninstall colbert_query_plugin -y - # END: `colbert_query io_processor` test - # BEGIN: `stat_logger` plugins test - pip install -e ./plugins/vllm_add_dummy_stat_logger - pytest -v -s plugins_tests/test_stats_logger_plugins.py - pip uninstall dummy_stat_logger -y - # END: `stat_logger` plugins test - # BEGIN: `endpoint` plugins test - pip install -e ./plugins/vllm_add_dummy_endpoint_plugin - pytest -v -s plugins_tests/test_endpoint_plugins.py - pip uninstall vllm_add_dummy_endpoint_plugin -y - # END: `endpoint` plugins test - # BEGIN: other tests - pytest -v -s plugins_tests/test_scheduler_plugins.py - pip install -e ./plugins/vllm_add_dummy_model - pytest -v -s distributed/test_distributed_oot.py - - pytest -v -s plugins_tests/test_oot_registration_online.py # it needs a clean process - - pytest -v -s plugins_tests/test_oot_registration_offline.py # it needs a clean process - - pytest -v -s plugins_tests/lora_resolvers # unit tests for in-tree lora resolver plugins + - pytest -v -s plugins_tests/test_oot_registration_online.py + - pytest -v -s plugins_tests/test_oot_registration_offline.py + - pytest -v -s plugins_tests/lora_resolvers - label: ":amd: (MI300) GGUF Plugin" # TBD timeout_in_minutes: 45 @@ -2402,18 +2403,24 @@ steps: - tests/entrypoints/serve/instrumentator/test_uds.py - tests/v1/sample/test_logprobs_e2e.py - vllm/platforms/rocm.py + - tests/entrypoints/openai/chat_completion/test_include_reasoning.py + - tests/entrypoints/openai/chat_completion/test_serving_chat.py + - tests/entrypoints/openai/completion/test_shutdown.py + - tests/entrypoints/openai/test_uds.py commands: - export VLLM_USE_RUST_FRONTEND=1 - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s benchmarks/test_serve_cli.py -k "not insecure and not (test_bench_serve and not test_bench_serve_chat)" - pytest -v -s entrypoints/openai/chat_completion/test_chat_completion.py -k "not test_invalid_json_schema and not test_invalid_regex and not test_kv_transfer_prompt_token_ids_round_trip and not test_kv_transfer_prompt_token_ids_streaming" + - pytest -v -s entrypoints/openai/chat_completion/test_include_reasoning.py - pytest -v -s entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py -k "not multiple" + - pytest -v -s entrypoints/openai/chat_completion/test_serving_chat.py -k "test_chat_per_request_metrics_follow_server_flag or test_streaming_reasoning_usage_counts_across_deltas or test_completion_tokens_details" - pytest -v -s entrypoints/launchers/test_shutdown.py -k "not engine_failure and not test_abort_timeout_exits_quickly" - pytest -v -s entrypoints/openai/test_return_token_ids.py -k "not test_comparison" - pytest -v -s entrypoints/serve/instrumentator/test_uds.py - pytest -v -s v1/sample/test_logprobs_e2e.py -k "test_prompt_logprobs_e2e_server" -- label: ":amd: (MI300) Rust Frontend Serve + Admin Coverage" # TBD +- label: ":amd: (MI300) Rust Frontend Serve-Admin Coverage" timeout_in_minutes: 30 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false @@ -2509,8 +2516,9 @@ steps: #------------------------------------------------------- mi300 · quantization --------------------------------------------------------# -- label: ":amd: (MI300) Quantization" # TBD +- label: ":amd: (MI300) Quantization Shard %N" timeout_in_minutes: 115 + parallelism: 4 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -2524,19 +2532,19 @@ steps: - vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py - tests/quantization - tests/rocm/test_moe_weight_replay.py + optional: true commands: - # temporary install here since we need nightly, will move to requirements/test.in # after torchao 0.12 release, and pin a working version of torchao nightly here - # since torchao nightly is only compatible with torch nightly currently # https://github.com/pytorch/ao/issues/2919, we'll have to skip new torchao tests for now # we can only upgrade after this is resolved # TODO(jerryzh168): resolve the above comment - - uv pip install --system torchao==0.17.0 + - unset VLLM_USE_V2_MODEL_RUNNER + - uv pip install --system torchao==0.18.0 - uv pip install --system conch-triton-kernels - - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py - - pytest -v -s rocm/test_moe_weight_replay.py + - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s rocm/test_moe_weight_replay.py; fi #--------------------------------------------------------- mi300 · samplers ----------------------------------------------------------# @@ -2556,6 +2564,7 @@ steps: - tests/conftest.py - vllm/_aiter_ops.py - vllm/platforms/rocm.py + - vllm/entrypoints/generate/beam_search commands: - pytest -v -s samplers @@ -2570,6 +2579,18 @@ steps: source_file_dependencies: - vllm/ - tests/test_regression + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/inputs/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + optional: true commands: - pip install 'modelscope<1.38' - pytest -v -s test_regression.py @@ -2586,6 +2607,8 @@ steps: - requirements/ - setup.py - vllm/platforms/rocm.py + - .buildkite/scripts/check-ray-compatibility.sh + optional: true commands: - bash /vllm-workspace/.buildkite/scripts/check-ray-compatibility.sh @@ -2608,7 +2631,7 @@ steps: - pytest -v -s v1/spec_decode/test_acceptance_length.py -m slow_test - label: ":amd: (MI300) E2E Scheduling" # TBD - timeout_in_minutes: 45 + timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -2622,7 +2645,7 @@ steps: - pytest -v -s v1/e2e/general/test_async_scheduling.py - label: ":amd: (MI300) V1 Engine" # TBD - timeout_in_minutes: 40 + timeout_in_minutes: 45 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -2632,12 +2655,14 @@ steps: - vllm/v1/engine/ - tests/v1/engine/ - vllm/platforms/rocm.py + - tests/v1/test_tensor_ipc_queue.py commands: - pytest -v -s v1/engine/test_preprocess_error_handling.py - pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py + - pytest -v -s v1/test_tensor_ipc_queue.py - label: ":amd: (MI300) Spec Decode Draft Model" # TBD - timeout_in_minutes: 45 + timeout_in_minutes: 60 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -2654,7 +2679,7 @@ steps: commands: - pytest -v -s v1/e2e/spec_decode/draft_model/ -- label: ":amd: (MI300) Spec Decode Eagle" # TBD +- label: ":amd: (MI300) Spec Decode Eagle 1: DeepSeek + Qwen" timeout_in_minutes: 60 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false @@ -2670,7 +2695,26 @@ steps: - tests/v1/e2e/spec_decode/ - vllm/platforms/rocm.py commands: - - pytest -v -s v1/e2e/spec_decode/eagle/ + - pytest -v -s v1/e2e/spec_decode/eagle/ -k "deepseek_eagle or qwen3_eagle3" + +- label: ":amd: (MI300) Spec Decode Eagle 2: Llama 3 + Qwen VL + Other" + timeout_in_minutes: 60 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_1 + optional: true + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/model_executor/model_loader/ + - vllm/v1/sample/ + - vllm/model_executor/layers/ + - tests/v1/e2e/spec_decode/ + - vllm/platforms/rocm.py + commands: + - pytest -v -s v1/e2e/spec_decode/eagle/ -k "not deepseek_eagle and not qwen3_eagle3" + - label: ":amd: (MI300) Spec Decode N-Gram + Suffix" # TBD timeout_in_minutes: 35 @@ -2693,7 +2737,7 @@ steps: - python3 spec_decode/test_custom_proposer.py - label: ":amd: (MI300) Spec Decode Speculators + MTP" # TBD - timeout_in_minutes: 50 + timeout_in_minutes: 75 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -2708,12 +2752,13 @@ steps: - vllm/transformers_utils/configs/speculators/ - tests/v1/e2e/spec_decode/ - vllm/platforms/rocm.py + - vllm/v1/attention/backends/ commands: - pytest -v -s v1/e2e/spec_decode/speculators/ - pytest -v -s v1/e2e/spec_decode/mtp/ -- label: ":amd: (MI300) Speculators Correctness" # TBD - timeout_in_minutes: 35 +- label: ":amd: (MI300) Speculators Correctness Nightly" + timeout_in_minutes: 40 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -2740,6 +2785,7 @@ steps: - tests/evals/gsm8k/ - tests/v1/spec_decode/test_speculators_correctness.py - vllm/platforms/rocm.py + - vllm/v1/worker/gpu/spec_decode/dflash/ commands: - export VLLM_ALLOW_INSECURE_SERIALIZATION=1 - pytest -v -s v1/spec_decode/test_speculators_correctness.py -m slow_test @@ -2753,6 +2799,17 @@ steps: source_file_dependencies: - vllm/ - tests/v1/spec_decode + - vllm/config/ + - vllm/distributed/ + - vllm/inputs/ + - vllm/model_executor/ + - vllm/models/qwen4_exp/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + optional: true commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s -m 'not slow_test' v1/spec_decode @@ -2763,7 +2820,7 @@ steps: dind: false agent_pool: mi300_2 num_gpus: 2 - working_dir: "/vllm-workspace/" + working_dir: /vllm-workspace source_file_dependencies: - vllm/config/speculative.py - vllm/distributed/kv_transfer/kv_connector/ @@ -2785,12 +2842,15 @@ steps: - vllm/_aiter_ops.py - tests/v1/kv_connector/extract_hidden_states_integration/ - vllm/platforms/rocm.py + - vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py + - tests/v1/kv_connector/extract_hidden_states_integration + optional: true commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s tests/v1/kv_connector/extract_hidden_states_integration + - pytest -v -s -m 'distributed' tests/v1/kv_connector/extract_hidden_states_integration - label: ":amd: (MI300) V1 Attention Shard %N" # TBD - timeout_in_minutes: 35 + timeout_in_minutes: 125 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -2808,7 +2868,7 @@ steps: commands: - pytest -v -s v1/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT -- label: ":amd: (MI300) V1 Core + KV + Metrics" # TBD +- label: ":amd: (MI300) V1 Core" timeout_in_minutes: 60 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false @@ -2826,23 +2886,204 @@ steps: - tests/v1/ec_connector/unit - tests/v1/metrics - tests/entrypoints/openai/correctness/test_lmeval.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/entrypoints/pooling/ + - vllm/inputs/ + - vllm/lora/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/outputs.py + - vllm/platforms/ + - vllm/pooling_params.py + - vllm/profiler/ + - vllm/sampling_params.py + - vllm/tokenizers/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + optional: true commands: - - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + # - export HSA_NO_SCRATCH_RECLAIM=1 - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s -m 'not cpu_test' v1/core + +- label: ":amd: (MI300) V1 Executor + Worker" + timeout_in_minutes: 60 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_1 + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/ + - tests/v1/core + - tests/v1/executor + - tests/v1/kv_offload + - tests/v1/simple_kv_offload + - tests/v1/worker + - tests/v1/streaming_input + - tests/v1/kv_connector/unit + - tests/v1/ec_connector/unit + - tests/v1/metrics + - tests/entrypoints/openai/correctness/test_lmeval.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/entrypoints/pooling/ + - vllm/inputs/ + - vllm/lora/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/outputs.py + - vllm/platforms/ + - vllm/pooling_params.py + - vllm/profiler/ + - vllm/sampling_params.py + - vllm/tokenizers/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s v1/executor - - pytest -v -s v1/kv_offload - - pytest -v -s v1/simple_kv_offload - pytest -v -s v1/worker - pytest -v -s v1/streaming_input - - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit - - pytest -v -s -m 'not cpu_test' v1/ec_connector/unit + optional: true + +- label: ":amd: (MI300) V1 KV Offload" + timeout_in_minutes: 60 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_1 + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/ + - tests/v1/core + - tests/v1/executor + - tests/v1/kv_offload + - tests/v1/simple_kv_offload + - tests/v1/worker + - tests/v1/streaming_input + - tests/v1/kv_connector/unit + - tests/v1/ec_connector/unit + - tests/v1/metrics + - tests/entrypoints/openai/correctness/test_lmeval.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/entrypoints/pooling/ + - vllm/inputs/ + - vllm/lora/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/outputs.py + - vllm/platforms/ + - vllm/pooling_params.py + - vllm/profiler/ + - vllm/sampling_params.py + - vllm/tokenizers/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - bash /vllm-workspace/.buildkite/scripts/install-kv-offload.sh + - pytest -v -s v1/kv_offload + - pytest -v -s v1/simple_kv_offload + optional: true + +- label: ":amd: (MI300) V1 KV Connectors Shard %N" + timeout_in_minutes: 60 + parallelism: 4 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_1 + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/ + - tests/v1/core + - tests/v1/executor + - tests/v1/kv_offload + - tests/v1/simple_kv_offload + - tests/v1/worker + - tests/v1/streaming_input + - tests/v1/kv_connector/unit + - tests/v1/ec_connector/unit + - tests/v1/metrics + - tests/entrypoints/openai/correctness/test_lmeval.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/entrypoints/pooling/ + - vllm/inputs/ + - vllm/lora/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/outputs.py + - vllm/platforms/ + - vllm/pooling_params.py + - vllm/profiler/ + - vllm/sampling_params.py + - vllm/tokenizers/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + commands: + - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB + - pytest -v -s -m 'not cpu_test' v1/ec_connector/unit --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB + optional: true + +- label: ":amd: (MI300) V1 Metrics + LM Eval" + timeout_in_minutes: 65 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_1 + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/ + - tests/v1/core + - tests/v1/executor + - tests/v1/kv_offload + - tests/v1/simple_kv_offload + - tests/v1/worker + - tests/v1/streaming_input + - tests/v1/kv_connector/unit + - tests/v1/ec_connector/unit + - tests/v1/metrics + - tests/entrypoints/openai/correctness/test_lmeval.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/entrypoints/pooling/ + - vllm/inputs/ + - vllm/lora/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/outputs.py + - vllm/platforms/ + - vllm/pooling_params.py + - vllm/profiler/ + - vllm/sampling_params.py + - vllm/tokenizers/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + commands: + - export GIT_CONFIG_COUNT=1 + - export GIT_CONFIG_KEY_0=http.version + - export GIT_CONFIG_VALUE_0=HTTP/1.1 + - export GIT_TERMINAL_PROMPT=0 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s -m 'not cpu_test' v1/metrics - pip install -U git+https://github.com/vllm-project/lm-evaluation-harness.git@streaming-api - # - export HSA_NO_SCRATCH_RECLAIM=1 - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine + optional: true -- label: ":amd: (MI300) V1 Sample + Logits" # TBD + +- label: ":amd: (MI300) V1 Sample" timeout_in_minutes: 70 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false @@ -2856,41 +3097,56 @@ steps: - tests/v1/test_oracle.py - tests/v1/test_request.py - tests/v1/test_outputs.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/inputs/ + - vllm/logger.py + - vllm/model_executor/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s v1/sample + +- label: ":amd: (MI300) V1 Logits + Oracle" + timeout_in_minutes: 70 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_1 + optional: true + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/ + - tests/v1/sample + - tests/v1/logits_processors + - tests/v1/test_oracle.py + - tests/v1/test_request.py + - tests/v1/test_outputs.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/inputs/ + - vllm/logger.py + - vllm/model_executor/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s v1/logits_processors - pytest -v -s v1/test_oracle.py - pytest -v -s v1/test_request.py - pytest -v -s v1/test_outputs.py -- label: ":amd: (MI300) Distributed DP Basic" # TBD - timeout_in_minutes: 45 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_2 - optional: true - num_gpus: 2 - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/distributed/ - - vllm/engine/ - - vllm/executor/ - - vllm/worker/worker_base.py - - vllm/v1/engine/ - - vllm/v1/worker/ - - tests/v1/distributed - - tests/entrypoints/launchers/api_server/test_multi_api_servers.py - - vllm/platforms/rocm.py - commands: - - export NCCL_CUMEM_HOST_ENABLE=0 - - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py - - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py - - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py - - DP_SIZE=2 pytest -v -s entrypoints/launchers/api_server/test_multi_api_servers.py - label: ":amd: (MI300) Metrics, Tracing" # TBD - timeout_in_minutes: 25 + timeout_in_minutes: 30 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_2 @@ -2901,6 +3157,18 @@ steps: - vllm/ - tests/v1/tracing - tests/tracing/ + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/inputs/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/tracing/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ commands: - "pip install \ 'opentelemetry-sdk>=1.26.0' \ @@ -2911,7 +3179,7 @@ steps: - pytest -v -s tracing - label: ":amd: (MI300) V1 E2E" # TBD - timeout_in_minutes: 25 + timeout_in_minutes: 30 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_2 @@ -2921,31 +3189,32 @@ steps: source_file_dependencies: - vllm/ - tests/v1/e2e + - vllm/compilation/ + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/envs.py + - vllm/forward_context.py + - vllm/inputs/ + - vllm/logger.py + - vllm/logging_utils/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/transformers_utils/ + - vllm/triton_utils/ + - vllm/utils/ + - vllm/v1/ + - tests/v1/e2e/spec_decode/ commands: - >- pytest -v -s v1/e2e/spec_decode/draft_model/test_draft_model.py::test_draft_model_tensor_parallelism v1/e2e/spec_decode/draft_model/test_draft_model.py::test_draft_model_engine_args_tensor_parallelism -- label: ":amd: (MI300) NixlConnector PD + Spec Decode acceptance" # TBD - timeout_in_minutes: 45 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_2 - num_gpus: 2 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ - - vllm/v1/worker/kv_connector_model_runner_mixin.py - - tests/v1/kv_connector/nixl_integration/ - - vllm/platforms/rocm.py - commands: - - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt - - KV_CACHE_MEMORY_BYTES=8G ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh - - label: ":amd: (MI300) CrossLayer KV layout Distributed NixlConnector PD accuracy" # TBD - timeout_in_minutes: 45 + timeout_in_minutes: 60 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_4 @@ -2984,38 +3253,6 @@ steps: - pytest -v -s v1/engine/test_engine_core_client.py::test_kv_cache_events_dp - pytest -v -s distributed/test_utils.py -- label: ":amd: (MI300) Distributed NixlConnector PD accuracy" # TBD - timeout_in_minutes: 45 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_4 - num_gpus: 4 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ - - tests/v1/kv_connector/nixl_integration/ - - vllm/platforms/rocm.py - commands: - - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt - - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh - -- label: ":amd: (MI300) DP EP Distributed NixlConnector PD accuracy" # TBD - timeout_in_minutes: 30 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_4 - num_gpus: 4 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ - - tests/v1/kv_connector/nixl_integration/ - - vllm/platforms/rocm.py - commands: - - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt - - DP_EP=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh - - label: ":amd: (MI300) Hybrid SSM NixlConnector PD accuracy" # TBD timeout_in_minutes: 60 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] @@ -3089,48 +3326,21 @@ steps: - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_multi_connector_edge_case_test.sh - label: ":amd: (MI300) V1 E2E Hybrid Chunked Prefill" # TBD - timeout_in_minutes: 30 + timeout_in_minutes: 35 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_4 num_gpus: 4 optional: true source_file_dependencies: - - vllm/v1/attention/backends/utils.py - - vllm/v1/worker/gpu_model_runner.py - - tests/v1/e2e/test_hybrid_chunked_prefill.py + - vllm/v1/attention/backends/utils.py + - vllm/v1/worker/gpu_model_runner.py + - tests/v1/e2e/test_hybrid_chunked_prefill.py + - vllm/v1/core/ + working_dir: /vllm-workspace/tests commands: - pytest -v -s v1/e2e/test_hybrid_chunked_prefill.py -#------------------------------------------------------ mi300 · weight_loading -------------------------------------------------------# - -- label: ":amd: (MI300) Weight Loading Multi-GPU" # TBD - timeout_in_minutes: 70 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_2 - num_gpus: 2 - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/weight_loading - commands: - - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-amd.txt - -- label: ":amd: (MI300) Weight Loading Multi-GPU (Large Models)" # TBD - timeout_in_minutes: 35 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_2 - num_gpus: 2 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/weight_loading - commands: - - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-large-amd.txt - ######################################################################################################################################### # # # MI355 (gfx950) tests # @@ -3156,11 +3366,90 @@ steps: #-------------------------------------------------------- mi355 · distributed --------------------------------------------------------# -- label: ":amd: (MI355) Distributed Features" # TBD +- label: ":amd: (MI355) Distributed DP Basic" # TBD timeout_in_minutes: 45 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_2 + optional: true + num_gpus: 2 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/distributed/ + - vllm/engine/ + - vllm/executor/ + - vllm/worker/worker_base.py + - vllm/v1/engine/ + - vllm/v1/worker/ + - tests/v1/distributed + - tests/entrypoints/launchers/api_server/test_multi_api_servers.py + - vllm/platforms/rocm.py + - tests/v1/e2e/general/test_sharded_sampling.py + - tests/v1/e2e/spec_decode/test_sharded_sampling.py + - tests/v1/e2e/spec_decode/utils.py + commands: + - export NCCL_CUMEM_HOST_ENABLE=0 + - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py + - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py + - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py + - DP_SIZE=2 pytest -v -s entrypoints/launchers/api_server/test_multi_api_servers.py + - pytest -v -s v1/e2e/general/test_sharded_sampling.py + - pytest -v -s v1/e2e/spec_decode/test_sharded_sampling.py + +- label: ":amd: (MI355) NixlConnector PD + Spec Decode acceptance" # TBD + timeout_in_minutes: 45 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_2 + num_gpus: 2 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ + - vllm/v1/worker/kv_connector_model_runner_mixin.py + - tests/v1/kv_connector/nixl_integration/ + - vllm/platforms/rocm.py + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + - KV_CACHE_MEMORY_BYTES=8G ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh + +- label: ":amd: (MI355) Distributed NixlConnector PD accuracy" # TBD + timeout_in_minutes: 60 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_4 + num_gpus: 4 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ + - tests/v1/kv_connector/nixl_integration/ + - vllm/platforms/rocm.py + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh + +- label: ":amd: (MI355) DP EP Distributed NixlConnector PD accuracy" # TBD + timeout_in_minutes: 40 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_4 + num_gpus: 4 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ + - tests/v1/kv_connector/nixl_integration/ + - vllm/platforms/rocm.py + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + - DP_EP=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh + +- label: ":amd: (MI355) Distributed Features" # TBD + timeout_in_minutes: 90 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_2 num_gpus: 2 optional: true working_dir: "/vllm-workspace/" @@ -3191,18 +3480,28 @@ steps: - tests/v1/e2e/general/test_rocm_aiter_custom_ar.py - tests/v1/distributed/test_dbo.py - tests/utils.py + - examples/features/data_parallel/data_parallel_offline.py + - examples/rl/rlhf_async_new_apis.py + - tests/distributed/test_weight_transfer.py + - tests/distributed/test_packed_tensor.py commands: - pytest -v -s tests/distributed/test_context_parallel.py + - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_async_new_apis.py + - VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=deepep_high_throughput + - VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=allgather_reducescatter --disable-nccl-for-dp-synchronization + - pytest -v -s tests/v1/distributed/test_dbo.py + - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py + - pytest -v -s tests/distributed/test_packed_tensor.py - pytest -v -s tests/distributed/test_rocm_aiter_custom_ar.py - pytest -v -s tests/v1/e2e/general/test_rocm_aiter_custom_ar.py - pytest -v -s tests/distributed/test_rocm_quick_reduce.py - pytest -v -s tests/distributed/test_quick_all_reduce.py - - pytest -v -s tests/v1/distributed/test_dbo.py #-------------------------------------------------------- mi355 · entrypoints --------------------------------------------------------# -- label: ":amd: (MI355) Entrypoints Integration (API Server)" # TBD - timeout_in_minutes: 55 +- label: ":amd: (MI355) Entrypoints Integration (API Server) %N" + timeout_in_minutes: 65 + parallelism: 4 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3215,12 +3514,12 @@ steps: - tests/entrypoints/scale_out commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc - - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc - - pytest -v -s entrypoints/scale_out + - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB + - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc; fi + - pytest -v -s entrypoints/scale_out --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - label: ":amd: (MI355) Entrypoints Integration (OpenAI API completion)" # TBD - timeout_in_minutes: 35 + timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3236,7 +3535,7 @@ steps: - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/correctness - label: ":amd: (MI355) Entrypoints Integration (OpenAI API chat_completion)" # TBD - timeout_in_minutes: 50 + timeout_in_minutes: 70 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3251,7 +3550,7 @@ steps: - pytest -v -s entrypoints/openai/chat_completion - label: ":amd: (MI355) Entrypoints Integration (API Server Generate)" # TBD - timeout_in_minutes: 40 + timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3264,15 +3563,16 @@ steps: - tests/entrypoints/tool_parsers - tests/entrypoints/anthropic - tests/entrypoints/generate + - tests/entrypoints/cohere commands: - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s tool_use - pytest -v -s entrypoints/tool_parsers - pytest -v -s entrypoints/generate - pytest -v -s entrypoints/anthropic + - pytest -v -s entrypoints/cohere - label: ":amd: (MI355) Entrypoints Integration (Speech to Text)" # TBD - timeout_in_minutes: 50 + timeout_in_minutes: 60 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3281,12 +3581,13 @@ steps: source_file_dependencies: - vllm/ - tests/entrypoints/speech_to_text + optional: true commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/speech_to_text - label: ":amd: (MI355) Entrypoints Integration (Multimodal)" - timeout_in_minutes: 40 + timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3295,23 +3596,26 @@ steps: source_file_dependencies: - vllm/ - tests/entrypoints/multimodal + optional: true commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/multimodal -- label: ":amd: (MI355) Entrypoints Integration (Pooling)" # TBD - timeout_in_minutes: 55 +- label: ":amd: (MI355) Entrypoints Integration (Pooling) Shard %N" # TBD + timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 fast_check: true + parallelism: 4 working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/ - tests/entrypoints/pooling + optional: true commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s entrypoints/pooling + - pytest -v -s entrypoints/pooling --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB #----------------------------------------------------------- mi355 · evals -----------------------------------------------------------# @@ -3361,8 +3665,8 @@ steps: commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-mi355.txt -- label: ":amd: (MI355) LM Eval Small Models" # TBD - timeout_in_minutes: 45 +- label: ":amd: (MI355) LM Eval Small Models Distributed" + timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_2 @@ -3378,6 +3682,7 @@ steps: - vllm/v1/attention/selector.py - vllm/_aiter_ops.py - vllm/platforms/rocm.py + - tests/evals/gsm8k/configs/models-mi3xx-fp8-and-mixed.txt commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx-fp8-and-mixed.txt @@ -3424,7 +3729,7 @@ steps: - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm-tp4.txt --tp-size=4 - label: ":amd: (MI355) LM Eval Large Models" # TBD - timeout_in_minutes: 35 + timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_8 @@ -3449,7 +3754,7 @@ steps: #--------------------------------------------------------- mi355 · examples ----------------------------------------------------------# - label: ":amd: (MI355) Examples" # TBD - timeout_in_minutes: 40 + timeout_in_minutes: 75 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3459,23 +3764,24 @@ steps: - vllm/multimodal - examples/ - vllm/platforms/rocm.py + optional: true commands: - - pip install tensorizer # Basic + # Multi-modal models + # Pooling models + # Features demo + - pip install --no-deps tensorizer - python3 basic/offline_inference/chat.py - python3 basic/offline_inference/generate.py --model facebook/opt-125m - python3 basic/offline_inference/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10 - python3 basic/offline_inference/classify.py - python3 basic/offline_inference/embed.py - python3 basic/offline_inference/score.py - # Multi-modal models - python3 generate/multimodal/audio_language_offline.py --seed 0 - python3 generate/multimodal/vision_language_offline.py --seed 0 - python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0 - python3 generate/multimodal/encoder_decoder_multimodal_offline.py --model-type whisper --seed 0 - # Pooling models - python3 pooling/embed/vision_embedding_offline.py --seed 0 - # Features demo - python3 features/automatic_prefix_caching/prefix_caching_offline.py - python3 deployment/llm_engine_example.py - python3 features/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 features/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors @@ -3502,8 +3808,10 @@ steps: - vllm/model_executor/layers/rotary_embedding/ - vllm/_aiter_ops.py - vllm/platforms/rocm.py + - tests/kernels/test_fused_qk_norm_rope_gate.py + optional: true commands: - - pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_concat_mla_q.py kernels/test_top_k_per_row.py kernels/test_rocm_fp32_router_gemm.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_fused_qk_norm_rope_gate.py kernels/test_concat_mla_q.py kernels/test_top_k_per_row.py kernels/test_rocm_fp32_router_gemm.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - label: ":amd: (MI355) MiniMax Reduce RMS Kernels" # TBD timeout_in_minutes: 45 @@ -3520,6 +3828,12 @@ steps: - vllm/platforms/rocm.py - tests/kernels/core/test_minimax_reduce_rms.py - tests/utils.py + - csrc/libtorch_stable/minimax_reduce_rms_kernel.cu + - csrc/libtorch_stable/minimax_reduce_rms_kernel.h + - csrc/libtorch_stable/ops.h + - csrc/libtorch_stable/torch_bindings.cpp + - vllm/model_executor/layers/mamba/linear/minimax_linear_attn.py + optional: true commands: - pytest -v -s kernels/core/test_minimax_reduce_rms.py @@ -3543,8 +3857,12 @@ steps: - vllm/v1/attention/selector.py - vllm/platforms/rocm.py - vllm/_aiter_ops.py + - examples/basic/offline_inference/chat.py + - tests/kernels/attention/test_attention_selector.py + - tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py + optional: true commands: - - rocm-smi + - amd-smi - python3 examples/basic/offline_inference/chat.py --attention-backend TRITON_ATTN - pytest -v -s tests/kernels/attention/test_attention_selector.py - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py @@ -3564,15 +3882,16 @@ steps: - tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py - tests/kernels/attention/test_rocm_aiter_mla_fp8_support.py - tests/kernels/attention/test_rocm_aiter_mla_op_registration.py + optional: true commands: - - rocm-smi + - amd-smi - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode.py - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_fp8_support.py - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_op_registration.py - label: ":amd: (MI355) Attention Kernels Shard %N" # TBD - timeout_in_minutes: 40 + timeout_in_minutes: 90 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3586,11 +3905,13 @@ steps: - vllm/_aiter_ops.py - vllm/envs.py - vllm/platforms/rocm.py + - vllm/utils/flashinfer.py + optional: true commands: - pytest -v -s kernels/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - label: ":amd: (MI355) MoE Kernels Shard %N" # TBD - timeout_in_minutes: 45 + timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3610,6 +3931,7 @@ steps: - vllm/config - vllm/_aiter_ops.py - vllm/platforms/rocm.py + optional: true commands: - pytest -v -s kernels/moe --ignore=kernels/moe/test_modular_oai_triton_moe.py @@ -3646,11 +3968,15 @@ steps: - vllm/utils/torch_utils.py - vllm/platforms/ - vllm/_aiter_ops.py + - csrc/quantization/cutlass_w8a8/moe/ + - vllm/distributed/device_communicators/ + - vllm/config commands: - pytest -v -s kernels/moe/test_moe_layer.py -- label: ":amd: (MI355) Quantization Kernels" # TBD - timeout_in_minutes: 50 +- label: ":amd: (MI355) Quantization Kernels Shard %N" + timeout_in_minutes: 120 + parallelism: 6 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3669,8 +3995,11 @@ steps: - vllm/platforms/rocm.py - vllm/model_executor/kernels/ - vllm/v1/attention/backends/rocm_aiter_fa.py + - vllm/config/ + - tests/kernels/quantization/test_rocm_skinny_gemms.py + optional: true commands: - - pytest -v -s kernels/quantization + - pytest -v -s kernels/quantization --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - label: ":amd: (MI355) FP8 MoE Kernels" # TBD timeout_in_minutes: 40 @@ -3685,6 +4014,13 @@ steps: - vllm/_aiter_ops.py - vllm/platforms/rocm.py - vllm/envs.py + - tests/kernels/moe/test_gpt_oss_triton_kernels.py + - tests/kernels/moe/test_modular_oai_triton_moe.py + - tests/kernels/moe/test_moe.py + - tests/kernels/moe/test_block_int8.py + - tests/kernels/moe/test_triton_moe_no_act_mul.py + - tests/kernels/moe/test_triton_moe_ptpc_fp8.py + optional: true commands: - pytest -v -s kernels/moe/test_gpt_oss_triton_kernels.py - pytest -v -s kernels/moe/test_modular_oai_triton_moe.py @@ -3694,7 +4030,7 @@ steps: - pytest -v -s kernels/moe/test_triton_moe_ptpc_fp8.py - label: ":amd: (MI355) DeepEP FP8 MoE Kernels" # TBD - timeout_in_minutes: 45 + timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_2 @@ -3712,24 +4048,65 @@ steps: commands: - pytest -v -s kernels/moe/test_deepep_moe.py +#------------------------------------------------------ mi355 · models / basic -------------------------------------------------------# + +- label: ":amd: (MI355) Basic Models (Extra Initialization) Shard %N" # TBD + timeout_in_minutes: 60 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_1 + parallelism: 6 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/model_executor/models/ + - vllm/model_executor/layers/ + - tests/models/test_initialization.py + - tests/models/registry.py + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + optional: true + commands: + - pytest -v -s models/test_initialization.py -k 'not test_can_initialize_small_subset' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB + #----------------------------------------------------- mi355 · models / language -----------------------------------------------------# -- label: ":amd: (MI355) Language Models (Extended Generation)" # TBD +- label: ":amd: (MI355) Language Models (Hybrid) Shard %N" # TBD timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 + parallelism: 2 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/models/language/generation + commands: + - export GIT_CONFIG_COUNT=1 + - export GIT_CONFIG_KEY_0=http.version + - export GIT_CONFIG_VALUE_0=HTTP/1.1 + - MAMBA_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@fix-rocm-7.0-warp-size-constexpr' + - CAUSAL_CONV1D_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0' + - pytest -v -s models/language/generation -m hybrid_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB + +- label: ":amd: (MI355) Language Models (Extended Generation)" # TBD + timeout_in_minutes: 95 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/ - tests/models/language/generation + optional: true commands: - MAMBA_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@rocm-7.0-v2.3.0' - CAUSAL_CONV1D_FORCE_BUILD=TRUE uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0' - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)' -- label: ":amd: (MI355) Language Models (Extended Pooling)" # TBD - timeout_in_minutes: 65 +- label: ":amd: (MI355) Language Models (Extended Pooling) Shard %N" + timeout_in_minutes: 95 + parallelism: 4 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3739,10 +4116,10 @@ steps: - vllm/ - tests/models/language/pooling commands: - - pytest -v -s models/language/pooling -m 'not core_model' + - pytest -v -s models/language/pooling -m 'not core_model' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - label: ":amd: (MI355) Language Models (PPL)" # TBD - timeout_in_minutes: 35 + timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3767,6 +4144,8 @@ steps: - vllm/v1/attention/ops/ - vllm/platforms/rocm.py - tests/models/language/generation_ppl_test + - vllm/model_executor/layers/fla/ops/ + - vllm/ commands: - pytest -v -s models/language/generation_ppl_test @@ -3779,6 +4158,7 @@ steps: source_file_dependencies: - vllm/ - tests/models/language + optional: true commands: - pip freeze | grep -E 'torch' - pytest -v -s models/language -m 'core_model and (not slow_test)' @@ -3786,7 +4166,7 @@ steps: #---------------------------------------------------- mi355 · models / multimodal ----------------------------------------------------# - label: ":amd: (MI355) Multimodal Models (Extended Generation 1)" # TBD - timeout_in_minutes: 60 + timeout_in_minutes: 90 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3801,7 +4181,7 @@ steps: - pytest -v -s models/multimodal/test_mapping.py - label: ":amd: (MI355) Multimodal Models (Extended Generation 3)" # TBD - timeout_in_minutes: 40 + timeout_in_minutes: 90 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3814,7 +4194,7 @@ steps: - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model' - label: ":amd: (MI355) Multimodal Models (Extended Pooling)" # TBD - timeout_in_minutes: 40 + timeout_in_minutes: 75 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3827,7 +4207,7 @@ steps: - pytest -v -s models/multimodal/pooling -m 'not core_model' - label: ":amd: (MI355) Multimodal Models (Standard) 1: qwen2" # TBD - timeout_in_minutes: 35 + timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3841,7 +4221,7 @@ steps: - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model - label: ":amd: (MI355) Multimodal Models (Standard) 4: other + whisper" # TBD - timeout_in_minutes: 55 + timeout_in_minutes: 70 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3853,13 +4233,13 @@ steps: commands: - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_mm_prefix_lm.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/generation/test_vit_cudagraph.py --ignore models/multimodal/processing - pytest -v -s models/multimodal/generation/test_vit_cudagraph.py -m core_model - - pytest -v -s models/multimodal/generation/test_memory_leak.py -m core_model + - pytest models/multimodal/generation/test_memory_leak.py -m core_model - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model #----------------------------------------------------- mi355 · models / quantized -----------------------------------------------------# - label: ":amd: (MI355) Quantized Models" # TBD - timeout_in_minutes: 50 + timeout_in_minutes: 80 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3870,13 +4250,16 @@ steps: - vllm/_aiter_ops.py - vllm/platforms/rocm.py - vllm/model_executor/model_loader/ + optional: true commands: + - unset VLLM_USE_V2_MODEL_RUNNER - pytest -v -s models/quantization #------------------------------------------------------- mi355 · quantization --------------------------------------------------------# -- label: ":amd: (MI355) Quantization" # TBD - timeout_in_minutes: 80 +- label: ":amd: (MI355) Quantization Shard %N" + timeout_in_minutes: 115 + parallelism: 4 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3887,10 +4270,16 @@ steps: - tests/quantization - vllm/_aiter_ops.py - vllm/platforms/rocm.py + - vllm/model_executor/layers/fused_moe/oracle/unquantized.py + - vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py + - tests/rocm/test_moe_weight_replay.py + optional: true commands: + - unset VLLM_USE_V2_MODEL_RUNNER - uv pip install --system torchao==0.17.0 - uv pip install --system conch-triton-kernels - - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py + - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s rocm/test_moe_weight_replay.py; fi # - label: Quantized MoE Test (B200-MI355) # TBD # timeout_in_minutes: 180 @@ -3922,7 +4311,7 @@ steps: #------------------------------------------------------------ mi355 · v1 -------------------------------------------------------------# - label: ":amd: (MI355) V1 Attention Shard %N" # TBD - timeout_in_minutes: 35 + timeout_in_minutes: 125 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3936,11 +4325,12 @@ steps: - vllm/_aiter_ops.py - vllm/envs.py - vllm/platforms/rocm.py + optional: true commands: - pytest -v -s v1/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT -- label: ":amd: (MI355) V1 Core + KV + Metrics" # TBD - timeout_in_minutes: 55 +- label: ":amd: (MI355) V1 Core" + timeout_in_minutes: 60 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3956,24 +4346,243 @@ steps: - tests/v1/kv_connector/unit - tests/v1/metrics - tests/entrypoints/openai/correctness/test_lmeval.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/entrypoints/pooling/ + - vllm/inputs/ + - vllm/lora/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/outputs.py + - vllm/platforms/ + - vllm/pooling_params.py + - vllm/profiler/ + - vllm/sampling_params.py + - vllm/tokenizers/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + - tests/v1/simple_kv_offload + - tests/v1/streaming_input + - tests/v1/ec_connector/unit commands: - - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s -m 'not cpu_test' v1/core + +- label: ":amd: (MI355) V1 Executor + Worker" + timeout_in_minutes: 60 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_1 + optional: true + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/ + - tests/v1/core + - tests/v1/executor + - tests/v1/kv_offload + - tests/v1/worker + - tests/v1/cudagraph + - tests/v1/kv_connector/unit + - tests/v1/metrics + - tests/entrypoints/openai/correctness/test_lmeval.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/entrypoints/pooling/ + - vllm/inputs/ + - vllm/lora/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/outputs.py + - vllm/platforms/ + - vllm/pooling_params.py + - vllm/profiler/ + - vllm/sampling_params.py + - vllm/tokenizers/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + - tests/v1/simple_kv_offload + - tests/v1/streaming_input + - tests/v1/ec_connector/unit + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s v1/executor - - pytest -v -s v1/kv_offload - pytest -v -s v1/worker - - pytest -v -s v1/cudagraph/test_encoder_cudagraph.py - - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit + - pytest -v -s v1/streaming_input + +- label: ":amd: (MI355) V1 KV Offload" + timeout_in_minutes: 60 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_1 + optional: true + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/ + - tests/v1/core + - tests/v1/executor + - tests/v1/kv_offload + - tests/v1/worker + - tests/v1/cudagraph + - tests/v1/kv_connector/unit + - tests/v1/metrics + - tests/entrypoints/openai/correctness/test_lmeval.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/entrypoints/pooling/ + - vllm/inputs/ + - vllm/lora/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/outputs.py + - vllm/platforms/ + - vllm/pooling_params.py + - vllm/profiler/ + - vllm/sampling_params.py + - vllm/tokenizers/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + - tests/v1/simple_kv_offload + - tests/v1/streaming_input + - tests/v1/ec_connector/unit + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - bash /vllm-workspace/.buildkite/scripts/install-kv-offload.sh + - pytest -v -s v1/kv_offload + - pytest -v -s v1/simple_kv_offload + +- label: ":amd: (MI355) V1 KV Connectors Shard %N" + timeout_in_minutes: 60 + parallelism: 4 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_1 + optional: true + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/ + - tests/v1/core + - tests/v1/executor + - tests/v1/kv_offload + - tests/v1/worker + - tests/v1/cudagraph + - tests/v1/kv_connector/unit + - tests/v1/metrics + - tests/entrypoints/openai/correctness/test_lmeval.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/entrypoints/pooling/ + - vllm/inputs/ + - vllm/lora/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/outputs.py + - vllm/platforms/ + - vllm/pooling_params.py + - vllm/profiler/ + - vllm/sampling_params.py + - vllm/tokenizers/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + - tests/v1/simple_kv_offload + - tests/v1/streaming_input + - tests/v1/ec_connector/unit + commands: + - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB + - pytest -v -s -m 'not cpu_test' v1/ec_connector/unit --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB + +- label: ":amd: (MI355) V1 Metrics + LM Eval" + timeout_in_minutes: 65 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_1 + optional: true + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/ + - tests/v1/core + - tests/v1/executor + - tests/v1/kv_offload + - tests/v1/worker + - tests/v1/cudagraph + - tests/v1/kv_connector/unit + - tests/v1/metrics + - tests/entrypoints/openai/correctness/test_lmeval.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/entrypoints/pooling/ + - vllm/inputs/ + - vllm/lora/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/outputs.py + - vllm/platforms/ + - vllm/pooling_params.py + - vllm/profiler/ + - vllm/sampling_params.py + - vllm/tokenizers/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + - tests/v1/simple_kv_offload + - tests/v1/streaming_input + - tests/v1/ec_connector/unit + commands: + - export GIT_CONFIG_COUNT=1 + - export GIT_CONFIG_KEY_0=http.version + - export GIT_CONFIG_VALUE_0=HTTP/1.1 + - export GIT_TERMINAL_PROMPT=0 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s -m 'not cpu_test' v1/metrics - pip install -U git+https://github.com/vllm-project/lm-evaluation-harness.git@streaming-api - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine -- label: ":amd: (MI355) V1 Sample + Logits" # TBD +- label: ":amd: (MI355) CUDAGraph" timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 optional: true + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/ + - tests/v1/core + - tests/v1/executor + - tests/v1/kv_offload + - tests/v1/worker + - tests/v1/cudagraph + - tests/v1/kv_connector/unit + - tests/v1/metrics + - tests/entrypoints/openai/correctness/test_lmeval.py + - vllm/v1/cudagraph_dispatcher.py + - vllm/config/compilation.py + - vllm/compilation + - vllm/platforms/rocm.py + - vllm/v1/worker/encoder_cudagraph.py + - vllm/v1/worker/encoder_cudagraph_defs.py + commands: + - pytest -v -s v1/cudagraph/test_cudagraph_dispatch.py + - pytest -v -s v1/cudagraph/test_cudagraph_mode.py + - pytest -v -s v1/cudagraph/test_breakable_cudagraph.py + - pytest -v -s v1/cudagraph/test_encoder_cudagraph.py + + +- label: ":amd: (MI355) V1 Sample" + timeout_in_minutes: 70 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_1 + optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/ @@ -3982,15 +4591,56 @@ steps: - tests/v1/test_oracle.py - tests/v1/test_request.py - tests/v1/test_outputs.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/inputs/ + - vllm/logger.py + - vllm/model_executor/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s v1/sample + +- label: ":amd: (MI355) V1 Logits + Oracle" + timeout_in_minutes: 70 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_1 + optional: true + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/ + - tests/v1/sample + - tests/v1/logits_processors + - tests/v1/test_oracle.py + - tests/v1/test_request.py + - tests/v1/test_outputs.py + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/inputs/ + - vllm/logger.py + - vllm/model_executor/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s v1/logits_processors - pytest -v -s v1/test_oracle.py - pytest -v -s v1/test_request.py - pytest -v -s v1/test_outputs.py + - label: ":amd: (MI355) V1 Spec Decode" # TBD - timeout_in_minutes: 45 + timeout_in_minutes: 50 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -3998,11 +4648,24 @@ steps: source_file_dependencies: - vllm/ - tests/v1/spec_decode + - vllm/config/ + - vllm/distributed/ + - vllm/inputs/ + - vllm/model_executor/ + - vllm/models/qwen4_exp/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + optional: true commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s -m 'not slow_test' v1/spec_decode -- label: ":amd: (MI355) Spec Decode AL MTP + Other Acceptance" # TBD - timeout_in_minutes: 180 +- label: ":amd: (MI355) Spec Decode AL MTP + Other Acceptance Nightly" + timeout_in_minutes: 55 + parallelism: 3 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_1 @@ -4026,37 +4689,12 @@ steps: - tests/utils.py - tests/v1/e2e/spec_decode/ - vllm/platforms/rocm.py + - vllm/v1/worker/gpu/spec_decode/ + - tests/v1/e2e/spec_decode/conftest.py commands: - - pytest -v -s v1/e2e/spec_decode/acceptance_rates/mtp_other/ - -#------------------------------------------------------ mi355 · weight_loading -------------------------------------------------------# - -- label: ":amd: (MI355) Weight Loading Multi-GPU" # TBD - timeout_in_minutes: 40 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - dind: false - agent_pool: mi355_2 - num_gpus: 2 - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/weight_loading - commands: - - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-amd.txt - -- label: ":amd: (MI355) Weight Loading Multi-GPU (Large Models)" # TBD - timeout_in_minutes: 30 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - dind: false - agent_pool: mi355_2 - working_dir: "/vllm-workspace/tests" - num_gpus: 2 - optional: true - source_file_dependencies: - - vllm/ - - tests/weight_loading - commands: - - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-large-amd.txt + - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s v1/e2e/spec_decode/acceptance_rates/mtp_other/ --deselect "tests/v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[False]" --deselect "tests/v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[True]"; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then pytest -v -s "v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[False]"; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "2" ]; then pytest -v -s "v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[True]"; fi #----------------------------------------------------------- mi355 · misc ------------------------------------------------------------# @@ -4070,6 +4708,17 @@ steps: source_file_dependencies: - vllm/ - tests/test_regression + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/inputs/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ commands: - pip install 'modelscope<1.38' - pytest -v -s test_regression.py diff --git a/.buildkite/test_areas/compile.yaml b/.buildkite/test_areas/compile.yaml index 8eea800686..219a4ca018 100644 --- a/.buildkite/test_areas/compile.yaml +++ b/.buildkite/test_areas/compile.yaml @@ -92,7 +92,7 @@ steps: dind: false device: mi355_1 working_dir: "/vllm-workspace/" - timeout_in_minutes: 25 + timeout_in_minutes: 30 depends_on: - image-build-amd source_file_dependencies: @@ -133,7 +133,7 @@ steps: dind: false device: mi355_1 working_dir: "/vllm-workspace/" - timeout_in_minutes: 30 + timeout_in_minutes: 40 depends_on: - image-build-amd source_file_dependencies: diff --git a/.buildkite/test_areas/cuda.yaml b/.buildkite/test_areas/cuda.yaml index 230781cd5d..8ac8e1960a 100644 --- a/.buildkite/test_areas/cuda.yaml +++ b/.buildkite/test_areas/cuda.yaml @@ -23,7 +23,7 @@ steps: dind: false device: mi355_1 working_dir: "/vllm-workspace/tests" - timeout_in_minutes: 20 + timeout_in_minutes: 25 depends_on: - image-build-amd source_file_dependencies: @@ -49,8 +49,9 @@ steps: mirror: amd: label: ":amd: (MI250) CUDAGraph" + dind: false device: mi250_1 - timeout_in_minutes: 45 + timeout_in_minutes: 55 depends_on: - image-build-amd source_file_dependencies: diff --git a/.buildkite/test_areas/disaggregated.yaml b/.buildkite/test_areas/disaggregated.yaml index 226d7ac94a..b187169e76 100644 --- a/.buildkite/test_areas/disaggregated.yaml +++ b/.buildkite/test_areas/disaggregated.yaml @@ -16,9 +16,9 @@ steps: - bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh mirror: amd: - label: ":amd: (MI300) Distributed NixlConnector PD accuracy" + label: ":amd: (MI355) Distributed NixlConnector PD accuracy" dind: false - device: mi300_4 + device: mi355_4 timeout_in_minutes: 60 depends_on: - image-build-amd @@ -71,9 +71,9 @@ steps: - DP_EP=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh mirror: amd: - label: ":amd: (MI300) DP EP Distributed NixlConnector PD accuracy" + label: ":amd: (MI355) DP EP Distributed NixlConnector PD accuracy" dind: false - device: mi300_4 + device: mi355_4 timeout_in_minutes: 40 depends_on: - image-build-amd @@ -130,7 +130,7 @@ steps: label: ":amd: (MI300) Hybrid SSM NixlConnector PD accuracy" dind: false device: mi300_4 - timeout_in_minutes: 55 + timeout_in_minutes: 60 depends_on: - image-build-amd source_file_dependencies: @@ -221,9 +221,9 @@ steps: - bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh mirror: amd: - label: ":amd: (MI300) NixlConnector PD + Spec Decode acceptance" + label: ":amd: (MI355) NixlConnector PD + Spec Decode acceptance" dind: false - device: mi300_2 + device: mi355_2 timeout_in_minutes: 45 depends_on: - image-build-amd @@ -287,7 +287,7 @@ steps: PREFILL_BLOCK_SIZE: "256" DECODE_BLOCK_SIZE: "256" MODEL_NAMES: "deepseek-ai/DeepSeek-V4-Flash" - VLLM_ENGINE_READY_TIMEOUT_S: "900" + VLLM_ENGINE_READY_TIMEOUT_S: "1800" VLLM_SERVE_EXTRA_ARGS: "--trust-remote-code,--kv-cache-dtype,fp8" # The V4 indexer only supports block-outermost layouts; let it pick. PREFILLER_KV_LAYOUT: "" @@ -311,6 +311,9 @@ steps: PREFILLER_TP_SIZE: "2" DECODER_TP_SIZE: "2" MODEL_NAMES: "moonshotai/Kimi-Linear-48B-A3B-Instruct" + # Keep TP=2 prefill and TP=1 decode above both hybrid page-size minima. + PREFILL_BLOCK_SIZE: "2048" + DECODE_BLOCK_SIZE: "2048" VLLM_SERVE_EXTRA_ARGS: "--trust-remote-code,--mamba-cache-mode align,--max-model-len 32768" commands: - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh diff --git a/.buildkite/test_areas/disaggregated_mooncake.yaml b/.buildkite/test_areas/disaggregated_mooncake.yaml index 8c061a95ad..f6e944d705 100644 --- a/.buildkite/test_areas/disaggregated_mooncake.yaml +++ b/.buildkite/test_areas/disaggregated_mooncake.yaml @@ -40,7 +40,14 @@ steps: - vllm/v1/worker/gpu_worker.py - vllm/v1/worker/gpu/ec_connector.py - vllm/v1/worker/gpu/model_runner.py + - vllm/v1/worker/mm_encoder_model_runner.py + - vllm/v1/worker/encoder_cudagraph.py + - vllm/v1/worker/encoder_cudagraph_defs.py + - vllm/v1/attention/ops/vit_attn_wrappers.py + - vllm/v1/worker/gpu/model_states/ - vllm/v1/worker/gpu/mm/ + - tests/v1/cudagraph/test_encoder_cudagraph.py + - tests/models/multimodal/generation/test_vit_cudagraph.py - examples/disaggregated/disaggregated_encoder/ - tests/v1/ec_connector/ - requirements/kv_connectors.txt @@ -49,6 +56,9 @@ steps: PYTHONUNBUFFERED: "1" VLLM_HOST_IP: "127.0.0.1" VLLM_USE_V2_MODEL_RUNNER: "1" + E_CUDAGRAPH_MM_ENCODER: "1" + # FA padding NaNs are tracked separately in #57136. + E_MM_ENCODER_ATTN_BACKEND: "TRITON_ATTN" MOONCAKE_EC_PROTOCOL: "tcp" USE_MM_PROMPTS: "1" SKIP_BASELINE: "0" diff --git a/.buildkite/test_areas/distributed.yaml b/.buildkite/test_areas/distributed.yaml index 3ea26ce89c..a108246563 100644 --- a/.buildkite/test_areas/distributed.yaml +++ b/.buildkite/test_areas/distributed.yaml @@ -65,9 +65,9 @@ steps: - pytest -v -s v1/e2e/spec_decode/test_sharded_sampling.py mirror: amd: - label: ":amd: (MI300) Distributed DP Basic" + label: ":amd: (MI355) Distributed DP Basic" dind: false - device: mi300_2 + device: mi355_2 timeout_in_minutes: 45 depends_on: - image-build-amd @@ -125,6 +125,9 @@ steps: - tests/compile/test_wrapper.py - tests/entrypoints/llm/test_collective_rpc.py - vllm/platforms/rocm.py + commands: + - pytest -v -s entrypoints/llm/test_collective_rpc.py + - pytest -v -s ./compile/test_wrapper.py - label: ":nvidia: (L4) Distributed Torchrun + Shutdown" key: distributed-torchrun-shutdown-tests-2-gpus @@ -296,6 +299,57 @@ steps: - pytest -v -s tests/v1/distributed/test_dbo.py - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py - pytest -v -s tests/distributed/test_packed_tensor.py + mirror: + amd: + label: ':amd: (MI355) Distributed Features' + dind: false + device: mi355_2 + timeout_in_minutes: 90 + working_dir: /vllm-workspace/ + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/distributed/ + - vllm/v1/distributed/ + - vllm/model_executor/layers/fused_moe/ + - vllm/v1/attention/backends/ + - vllm/v1/attention/selector.py + - tests/v1/distributed/test_dbo.py + - tests/distributed/test_context_parallel.py + - examples/features/data_parallel/data_parallel_offline.py + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - csrc/custom_quickreduce.cu + - csrc/ops.h + - csrc/torch_bindings.cpp + - vllm/model_executor/layers/ + - vllm/entrypoints/llm.py + - vllm/config/parallel.py + - vllm/v1/engine/ + - vllm/v1/executor/ + - vllm/v1/worker/ + - vllm/_custom_ops.py + - vllm/envs.py + - tests/distributed/test_rocm_aiter_custom_ar.py + - tests/distributed/test_rocm_quick_reduce.py + - tests/distributed/test_quick_all_reduce.py + - tests/v1/e2e/general/test_rocm_aiter_custom_ar.py + - tests/utils.py + - examples/rl/rlhf_async_new_apis.py + - tests/distributed/test_weight_transfer.py + - tests/distributed/test_packed_tensor.py + commands: + - pytest -v -s tests/distributed/test_context_parallel.py + - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_async_new_apis.py + - VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=deepep_high_throughput + - VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=allgather_reducescatter --disable-nccl-for-dp-synchronization + - pytest -v -s tests/v1/distributed/test_dbo.py + - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py + - pytest -v -s tests/distributed/test_packed_tensor.py + - pytest -v -s tests/distributed/test_rocm_aiter_custom_ar.py + - pytest -v -s tests/v1/e2e/general/test_rocm_aiter_custom_ar.py + - pytest -v -s tests/distributed/test_rocm_quick_reduce.py + - pytest -v -s tests/distributed/test_quick_all_reduce.py - label: ":nvidia: (B200) Distributed" key: distributed-tests-2xb200 @@ -362,6 +416,7 @@ steps: mirror: amd: label: ":amd: (MI250) Pipeline + Context Parallelism" + dind: false device: mi250_4 timeout_in_minutes: 90 depends_on: diff --git a/.buildkite/test_areas/docker.yaml b/.buildkite/test_areas/docker.yaml index ef50062a7f..4c80f26ca5 100644 --- a/.buildkite/test_areas/docker.yaml +++ b/.buildkite/test_areas/docker.yaml @@ -18,6 +18,7 @@ steps: mirror: amd: label: ":amd: (MI250) Docker Build Metadata" + dind: false device: mi250_1 no_gpu: true timeout_in_minutes: 35 diff --git a/.buildkite/test_areas/e2e_integration.yaml b/.buildkite/test_areas/e2e_integration.yaml index 02ba409e34..20bec50b5e 100644 --- a/.buildkite/test_areas/e2e_integration.yaml +++ b/.buildkite/test_areas/e2e_integration.yaml @@ -57,7 +57,7 @@ steps: dind: false device: mi355_1 working_dir: "/vllm-workspace" - timeout_in_minutes: 20 + timeout_in_minutes: 30 depends_on: - image-build-amd source_file_dependencies: diff --git a/.buildkite/test_areas/engine.yaml b/.buildkite/test_areas/engine.yaml index bb848cd9e5..8e47a84e18 100644 --- a/.buildkite/test_areas/engine.yaml +++ b/.buildkite/test_areas/engine.yaml @@ -52,10 +52,13 @@ steps: mirror: amd: label: ":amd: (MI250) V1 Engine" + dind: false device: mi250_1 timeout_in_minutes: 45 depends_on: - image-build-amd + source_file_dependencies: + - vllm/platforms/rocm.py - label: ":nvidia: (H200 MIG 18GB) E2E Scheduling" key: e2e-scheduling-1-gpu @@ -69,15 +72,18 @@ steps: mirror: amd: label: ":amd: (MI250) E2E Scheduling" + dind: false device: mi250_1 timeout_in_minutes: 55 depends_on: - image-build-amd + source_file_dependencies: + - vllm/platforms/rocm.py - label: ":nvidia: (H200 MIG 18GB) E2E Core" device: h200_18gb key: e2e-core-1-gpu - timeout_in_minutes: 50 + timeout_in_minutes: 65 source_file_dependencies: - vllm/v1/ - tests/v1/e2e/general/ @@ -86,6 +92,7 @@ steps: mirror: amd: label: ":amd: (MI250) E2E Core" + dind: false commands: - pytest -v -s v1/e2e/general --ignore v1/e2e/general/test_async_scheduling.py device: mi250_1 diff --git a/.buildkite/test_areas/entrypoints.yaml b/.buildkite/test_areas/entrypoints.yaml index 00091707d5..60ac5d932c 100644 --- a/.buildkite/test_areas/entrypoints.yaml +++ b/.buildkite/test_areas/entrypoints.yaml @@ -21,9 +21,11 @@ steps: label: ":amd: (MI355) Entrypoints Unit" dind: false device: mi355_1 - timeout_in_minutes: 35 + timeout_in_minutes: 50 depends_on: - image-build-amd + source_file_dependencies: + - vllm/platforms/rocm.py - label: ":nvidia: (H200 MIG 35GB) Entrypoints Integration (LLM)" device: h200_35gb @@ -44,7 +46,7 @@ steps: label: ":amd: (MI300) Entrypoints Integration (LLM)" dind: false device: mi300_1 - timeout_in_minutes: 55 + timeout_in_minutes: 90 depends_on: - image-build-amd @@ -176,6 +178,15 @@ steps: commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/speech_to_text + mirror: + amd: + label: ":amd: (MI355) Entrypoints Integration (Speech to Text)" + dind: false + device: mi355_1 + timeout_in_minutes: 60 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd - label: ":nvidia: (H200 MIG 35GB) Entrypoints Integration (Multimodal)" device: h200_35gb @@ -210,6 +221,15 @@ steps: commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/pooling + mirror: + amd: + label: ":amd: (MI355) Entrypoints Integration (Pooling)" + dind: false + device: mi355_1 + timeout_in_minutes: 65 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd - label: ":nvidia: (H200 MIG 18GB) OpenAI API Correctness" key: openai-api-correctness diff --git a/.buildkite/test_areas/expert_parallelism.yaml b/.buildkite/test_areas/expert_parallelism.yaml index f74b1adf2e..485d4f3d31 100644 --- a/.buildkite/test_areas/expert_parallelism.yaml +++ b/.buildkite/test_areas/expert_parallelism.yaml @@ -43,7 +43,7 @@ steps: - label: ":nvidia: (H100) Elastic EP Scaling" key: elastic-ep-scaling-test - timeout_in_minutes: 30 + timeout_in_minutes: 40 device: h100 working_dir: "/vllm-workspace/tests" num_devices: 4 diff --git a/.buildkite/test_areas/kernels.yaml b/.buildkite/test_areas/kernels.yaml index ed4ea67f93..aab6e52b8e 100644 --- a/.buildkite/test_areas/kernels.yaml +++ b/.buildkite/test_areas/kernels.yaml @@ -13,6 +13,22 @@ steps: commands: - pytest -v -s tests/ir - pytest -v -s tests/kernels/ir + mirror: + amd: + label: ":amd: (MI355) vLLM IR" + dind: false + device: mi355_1 + timeout_in_minutes: 50 + working_dir: "/vllm-workspace/" + depends_on: + - image-build-amd + source_file_dependencies: + - tests/ir/ + - tests/kernels/ir/ + - vllm/ir + - vllm/kernels + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py - label: ":nvidia: (H200 MIG 35GB) Core Operation Kernels Shard %N" device: h200_35gb @@ -26,6 +42,29 @@ steps: commands: - pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_concat_mla_q.py kernels/test_fused_qk_norm_rope_gate.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT parallelism: 3 + mirror: + amd: + label: ":amd: (MI355) Core Operation Kernels Shard %N" + dind: false + device: mi355_1 + timeout_in_minutes: 40 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd + source_file_dependencies: + - csrc/ + - tests/kernels/core + - tests/kernels/test_top_k_per_row.py + - tests/kernels/test_concat_mla_q.py + - tests/kernels/test_rocm_fp32_router_gemm.py + - vllm/model_executor/layers/fused_moe/router/gate_linear.py + - vllm/model_executor/layers/fused_moe/router/rocm_fp32_router_gemm.py + - vllm/model_executor/layers/rotary_embedding/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - tests/kernels/test_fused_qk_norm_rope_gate.py + commands: + - pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_fused_qk_norm_rope_gate.py kernels/test_concat_mla_q.py kernels/test_top_k_per_row.py kernels/test_rocm_fp32_router_gemm.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - label: ":nvidia: (H100) MiniMax Reduce RMS Kernels" key: kernels-minimax-reduce-rms-test-2-gpus @@ -81,6 +120,7 @@ steps: - csrc/fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu - vllm/models/deepseek_v4/common/ops/ - vllm/models/deepseek_v4/nvidia/ + - vllm/utils/deep_gemm.py - tests/kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py - tests/models/test_deepseek_v4_mega_moe.py commands: @@ -145,6 +185,7 @@ steps: dind: false device: mi300_1 timeout_in_minutes: 90 + parallelism: 6 depends_on: - image-build-amd source_file_dependencies: @@ -208,9 +249,6 @@ steps: label: ":amd: (MI300) Quantization Kernels Shard %N" dind: false device: mi300_1 - # Pin the AMD mirror at its current 2 shards so raising NVIDIA - # parallelism does not multiply AMD copies (needs ci-infra #473). - parallelism: 2 timeout_in_minutes: 120 source_file_dependencies: - csrc/quantization/ @@ -222,6 +260,12 @@ steps: - vllm/kernels/aiter_ops.py - vllm/platforms/rocm.py - vllm/model_executor/kernels/ + - csrc/rocm/ + - vllm/v1/attention/backends/rocm_aiter_fa.py + - tests/kernels/quant_utils.py + - tests/kernels/utils.py + - vllm/_custom_ops.py + - vllm/envs.py depends_on: - image-build-amd @@ -258,6 +302,10 @@ steps: - vllm/config - vllm/_aiter_ops.py - vllm/platforms/rocm.py + - csrc/rocm/ + - tests/kernels/utils.py + - vllm/model_executor/layers/quantization/mxfp4.py + - vllm/model_executor/layers/quantization/utils/quant_utils.py depends_on: - image-build-amd @@ -268,16 +316,20 @@ steps: source_file_dependencies: - csrc/mamba/ - tests/kernels/mamba + - vllm/model_executor/layers/mamba/gdn - vllm/model_executor/layers/mamba/ops commands: - pytest -v -s kernels/mamba mirror: amd: label: ":amd: (MI250) Mamba Kernels" + dind: false device: mi250_1 timeout_in_minutes: 55 depends_on: - image-build-amd + source_file_dependencies: + - vllm/platforms/rocm.py - label: ":nvidia: (H100) DeepGEMM Kernels" key: kernels-deepgemm-test-h100 @@ -317,6 +369,7 @@ steps: - csrc/quantization/fp4/ - csrc/libtorch_stable/nvfp4_kv_cache_kernels.cu - csrc/libtorch_stable/quantization/fp4/ + - csrc/libtorch_stable/quantization/w8a8/cutlass/ - csrc/attention/mla/ - csrc/quantization/cutlass_w8a8/moe/ - vllm/model_executor/layers/fusion/ @@ -358,6 +411,7 @@ steps: - tests/kernels/test_ll_bf16_gemm.py - tests/kernels/test_top_k_per_row.py - tests/kernels/quantization/test_silu_mul_nvfp4_quant.py + - tests/kernels/quantization/test_block_fp8.py - tests/models/quantization/test_nvfp4.py commands: - nvidia-smi @@ -392,6 +446,7 @@ steps: pytest -v -s tests/kernels/attention/test_flashinfer.py -k 'not num_heads2' pytest -v -s tests/kernels/attention/test_cutlass_mla_decode.py pytest -v -s tests/kernels/quantization/test_cutlass_scaled_mm.py -k 'fp8' + pytest -v -s tests/kernels/quantization/test_block_fp8.py::test_w8a8_block_fp8_cutlass_matmul pytest -v -s tests/kernels/quantization/test_nvfp4_quant.py pytest -v -s tests/kernels/quantization/test_nvfp4_scaled_mm.py pytest -v -s tests/kernels/quantization/test_mxfp4_qutlass.py @@ -459,10 +514,14 @@ steps: mirror: amd: label: ":amd: (MI250) Helion Kernels Shard %N" + dind: false device: mi250_1 timeout_in_minutes: 45 depends_on: - image-build-amd + source_file_dependencies: + - vllm/platforms/rocm.py + - scripts/benchmark_helion_kernels.py - label: ":nvidia: (H100) FP8 MoE Kernels" @@ -491,6 +550,25 @@ steps: commands: - pytest -v -s kernels/moe/test_deepep_deepgemm_moe.py - pytest -v -s kernels/moe/test_deepep_moe.py + mirror: + amd: + label: ":amd: (MI355) DeepEP FP8 MoE Kernels" + dind: false + device: mi355_2 + timeout_in_minutes: 55 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd + source_file_dependencies: + - csrc/moe/ + - csrc/quantization/w8a8/cutlass/moe/ + - vllm/model_executor/layers/fused_moe/ + - tests/kernels/moe/test_deepep_moe.py + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/envs.py + commands: + - pytest -v -s kernels/moe/test_deepep_moe.py - label: ":nvidia: (B200) FP4 MoE Kernels" key: kernels-fp4-moe-test-b200 @@ -520,7 +598,30 @@ steps: - vllm/config commands: - pytest -v -s kernels/moe/test_moe_layer.py - + mirror: + amd: + label: ":amd: (MI355) FusedMoE Layer Kernels" + dind: false + device: mi355_2 + timeout_in_minutes: 35 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd + source_file_dependencies: + - csrc/moe/ + - csrc/rocm/ + - tests/kernels/moe + - vllm/model_executor/layers/fused_moe/ + - vllm/model_executor/layers/quantization/ + - vllm/distributed/ + - vllm/config/ + - vllm/forward_context.py + - vllm/v1/worker/workspace.py + - vllm/utils/import_utils.py + - vllm/utils/math_utils.py + - vllm/utils/torch_utils.py + - vllm/platforms/ + - vllm/_aiter_ops.py - label: ":nvidia: (B200) FusedMoE Layer Kernels" key: kernels-fusedmoe-layer-test-2-b200s @@ -538,3 +639,90 @@ steps: commands: - pytest -v -s kernels/moe/test_moe_layer.py - pytest -v -s kernels/moe/test_deepep_v2_moe.py + +- label: ":amd: (MI355) MLA Kernels" + key: amd-mla-kernels-mi355 + dind: false + device: mi355_1 + num_devices: 1 + timeout_in_minutes: 25 + working_dir: /vllm-workspace/ + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/v1/attention/backends/mla/rocm_aiter_mla.py + - vllm/v1/attention/selector.py + - vllm/platforms/rocm.py + - vllm/_aiter_ops.py + - tests/kernels/attention/test_rocm_aiter_mla_decode.py + - tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py + - tests/kernels/attention/test_rocm_aiter_mla_fp8_support.py + - tests/kernels/attention/test_rocm_aiter_mla_op_registration.py + commands: + - amd-smi + - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py + - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode.py + - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_fp8_support.py + - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_op_registration.py + +- label: ":amd: (MI355) FP8 MoE Kernels" + key: amd-fp8-moe-kernels-mi355 + dind: false + device: mi355_1 + num_devices: 1 + timeout_in_minutes: 40 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - csrc/moe/ + - vllm/model_executor/layers/fused_moe/ + - tests/kernels/moe/test_deepep_moe.py + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/envs.py + - tests/kernels/moe/test_gpt_oss_triton_kernels.py + - tests/kernels/moe/test_modular_oai_triton_moe.py + - tests/kernels/moe/test_moe.py + - tests/kernels/moe/test_block_int8.py + - tests/kernels/moe/test_triton_moe_no_act_mul.py + - tests/kernels/moe/test_triton_moe_ptpc_fp8.py + commands: + - pytest -v -s kernels/moe/test_gpt_oss_triton_kernels.py + - pytest -v -s kernels/moe/test_modular_oai_triton_moe.py + - pytest -v -s kernels/moe/test_moe.py + - pytest -v -s kernels/moe/test_block_int8.py + - pytest -v -s kernels/moe/test_triton_moe_no_act_mul.py + - pytest -v -s kernels/moe/test_triton_moe_ptpc_fp8.py + +- label: ":amd: (MI355) Kernels" + key: amd-kernels-mi355 + dind: false + device: mi355_1 + num_devices: 1 + timeout_in_minutes: 25 + working_dir: /vllm-workspace/ + depends_on: + - image-build-amd + source_file_dependencies: + - csrc/quantization/fp4/ + - csrc/attention/mla/ + - csrc/quantization/cutlass_w8a8/moe/ + - vllm/model_executor/layers/fused_moe/cutlass_moe.py + - vllm/v1/attention/backends/triton_attn.py + - vllm/v1/attention/backends/rocm_attn.py + - vllm/v1/attention/backends/rocm_aiter_fa.py + - vllm/v1/attention/backends/rocm_aiter_unified_attn.py + - vllm/v1/attention/backends/mla/aiter_triton_mla.py + - vllm/v1/attention/backends/mla/rocm_aiter_mla.py + - vllm/v1/attention/selector.py + - vllm/platforms/rocm.py + - vllm/_aiter_ops.py + - examples/basic/offline_inference/chat.py + - tests/kernels/attention/test_attention_selector.py + - tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py + commands: + - amd-smi + - python3 examples/basic/offline_inference/chat.py --attention-backend TRITON_ATTN + - pytest -v -s tests/kernels/attention/test_attention_selector.py + - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index ec45f7e69e..3eb7f0a444 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -30,6 +30,27 @@ steps: - vllm/_aiter_ops.py - vllm/platforms/rocm.py +- label: ":amd: (MI300) LM Eval Small Models Harness" + key: amd-lm-eval-small-models-harness + dind: false + device: mi300_1 + timeout_in_minutes: 35 + depends_on: + - image-build-amd + working_dir: /vllm-workspace/.buildkite/lm-eval-harness + source_file_dependencies: + - csrc/ + - vllm/model_executor/layers/quantization + - vllm/model_executor/models/ + - vllm/model_executor/model_loader/ + - vllm/v1/attention/backends/ + - vllm/v1/attention/selector.py + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - .buildkite/lm-eval-harness/ + commands: + - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-small-rocm.txt + - label: ":nvidia: (H200 MIG 35GB) LM Eval Watermarking" device: h200_35gb key: lm-eval-watermarking @@ -100,7 +121,7 @@ steps: label: ":amd: (MI355) LM Eval Small Models Distributed" dind: false device: mi355_2 - timeout_in_minutes: 45 + timeout_in_minutes: 55 depends_on: - image-build-amd source_file_dependencies: @@ -275,7 +296,7 @@ steps: - label: ":nvidia: (H200) LM Eval Large Models" key: lm-eval-large-models-8xh200 - timeout_in_minutes: 50 + timeout_in_minutes: 120 device: h200 optional: true num_devices: 8 @@ -286,13 +307,16 @@ steps: label: ":amd: (MI300) LM Eval Large Models" dind: false device: mi300_8 - timeout_in_minutes: 40 + timeout_in_minutes: 65 depends_on: - image-build-amd commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - export PYTORCH_ROCM_ARCH=gfx942 # Limit Quark compilation to save time - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx.txt + source_file_dependencies: + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py - label: ":nvidia: (H100) MoE Refactor Integration TEMPORARY" key: moe-refactor-integration-test-h100-temporary diff --git a/.buildkite/test_areas/lora.yaml b/.buildkite/test_areas/lora.yaml index 93fb5f5b81..28f695f391 100644 --- a/.buildkite/test_areas/lora.yaml +++ b/.buildkite/test_areas/lora.yaml @@ -61,7 +61,7 @@ steps: dind: false device: mi300_4 working_dir: "/vllm-workspace/tests" - timeout_in_minutes: 25 + timeout_in_minutes: 65 depends_on: - image-build-amd source_file_dependencies: @@ -73,11 +73,11 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn # expandable_segments breaks custom all-reduce IPC on ROCm. - case "$$BUILDKITE_PARALLEL_JOB" in 0|1|2|3) ;; *) echo "unexpected BUILDKITE_PARALLEL_JOB=$$BUILDKITE_PARALLEL_JOB" && exit 1;; esac - - test "$$BUILDKITE_PARALLEL_JOB" != "3" || pytest -v -s -x lora/test_chatglm3_tp.py - - test "$$BUILDKITE_PARALLEL_JOB" != "1" || pytest -v -s -x lora/test_llama_tp.py - - test "$$BUILDKITE_PARALLEL_JOB" != "3" || pytest -v -s -x lora/test_qwen3_with_multi_loras.py - - test "$$BUILDKITE_PARALLEL_JOB" != "2" || pytest -v -s -x lora/test_olmoe_tp.py - - test "$$BUILDKITE_PARALLEL_JOB" != "0" || pytest -v -s -x lora/test_gptoss_tp.py - - test "$$BUILDKITE_PARALLEL_JOB" != "1" || pytest -v -s -x lora/test_qwen35_densemodel_lora.py - - test "$$BUILDKITE_PARALLEL_JOB" != "2" || pytest -v -s -x lora/test_gemma4_tp.py - - test "$$BUILDKITE_PARALLEL_JOB" != "0" || pytest -v -s -x lora/test_sequence_classification.py::test_batched_loras_tp + - if [ "$$BUILDKITE_PARALLEL_JOB" = "3" ]; then pytest -v -s -x lora/test_chatglm3_tp.py; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then pytest -v -s -x lora/test_llama_tp.py; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "3" ]; then pytest -v -s -x lora/test_qwen3_with_multi_loras.py; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "2" ]; then pytest -v -s -x lora/test_olmoe_tp.py; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s -x lora/test_gptoss_tp.py; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then pytest -v -s -x lora/test_qwen35_densemodel_lora.py; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "2" ]; then pytest -v -s -x lora/test_gemma4_tp.py; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s -x lora/test_sequence_classification.py::test_batched_loras_tp; fi diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index 940447bf1b..1bff8e1850 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -99,7 +99,7 @@ steps: label: ":amd: (MI300) V1 Core" dind: false device: mi300_1 - timeout_in_minutes: 45 + timeout_in_minutes: 60 depends_on: - image-build-amd @@ -119,7 +119,7 @@ steps: label: ":amd: (MI300) V1 Executor + Worker" dind: false device: mi300_1 - timeout_in_minutes: 45 + timeout_in_minutes: 60 depends_on: - image-build-amd @@ -138,7 +138,7 @@ steps: label: ":amd: (MI300) V1 KV Offload" dind: false device: mi300_1 - timeout_in_minutes: 45 + timeout_in_minutes: 60 depends_on: - image-build-amd @@ -158,7 +158,7 @@ steps: label: ":amd: (MI300) V1 KV Connectors Shard %N" dind: false device: mi300_1 - timeout_in_minutes: 45 + timeout_in_minutes: 60 depends_on: - image-build-amd @@ -258,7 +258,7 @@ steps: label: ":amd: (MI355) Extract Hidden States Integration" dind: false device: mi355_2 - timeout_in_minutes: 20 + timeout_in_minutes: 30 depends_on: - image-build-amd source_file_dependencies: @@ -305,6 +305,15 @@ steps: - pip install 'modelscope<1.38' - pytest -v -s test_regression.py working_dir: "/vllm-workspace/tests" # optional + mirror: + amd: + label: ":amd: (MI355) Regression" + dind: false + device: mi355_1 + timeout_in_minutes: 25 + depends_on: + - image-build-amd + working_dir: /vllm-workspace/tests - label: ":nvidia: (H200 MIG 35GB) Examples" device: h200_35gb @@ -446,9 +455,15 @@ steps: label: ":amd: (MI355) Async Engine, Inputs, Utils, Worker" dind: false device: mi355_1 - timeout_in_minutes: 40 + timeout_in_minutes: 50 depends_on: - image-build-amd + commands: + - pytest -v -s detokenizer + - pytest -v -s -m 'not cpu_test' multimodal + # test_cache.py marks every test cpu_test, including this GPU regression. + - pytest -v -s multimodal/test_cache.py::test_sleep_wake_preserves_mm_cache_consistency + - pytest -v -s utils_ - label: ":computer: (CPU) Params, Env, Tokenizers, Parser" key: cpu-params-env-tokenizers-parser @@ -571,8 +586,9 @@ steps: mirror: amd: label: ":amd: (MI250) Batch Invariance" + dind: false device: mi250_1 - timeout_in_minutes: 30 + timeout_in_minutes: 35 depends_on: - image-build-amd source_file_dependencies: @@ -623,3 +639,17 @@ steps: commands: - export VLLM_ALLOW_INSECURE_SERIALIZATION=1 - pytest -v -s v1/spec_decode/test_acceptance_length.py -m slow_test + mirror: + amd: + label: ":amd: (MI355) Acceptance Length (Large Models)" + dind: false + device: mi355_1 + timeout_in_minutes: 45 + depends_on: + - image-build-amd + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/model_executor/models/mlp_speculator.py + - tests/v1/spec_decode/test_acceptance_length.py + - vllm/platforms/rocm.py diff --git a/.buildkite/test_areas/models_basic.yaml b/.buildkite/test_areas/models_basic.yaml index 870cc9b393..6702eb6dd3 100644 --- a/.buildkite/test_areas/models_basic.yaml +++ b/.buildkite/test_areas/models_basic.yaml @@ -19,7 +19,7 @@ steps: label: ":amd: (MI355) Basic Models (Initialization)" dind: false device: mi355_1 - timeout_in_minutes: 25 + timeout_in_minutes: 30 depends_on: - image-build-amd @@ -37,6 +37,23 @@ steps: # test.) Also run if model initialization test file is modified - pytest -v -s models/test_initialization.py -k 'not test_can_initialize_small_subset' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB parallelism: 14 + mirror: + amd: + label: ":amd: (MI355) Basic Models (Extra Initialization) Shard %N" + dind: false + device: mi355_1 + timeout_in_minutes: 60 + parallelism: 10 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/model_executor/models/ + - vllm/model_executor/layers/ + - tests/models/test_initialization.py + - tests/models/registry.py + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py - label: ":nvidia: (H200 MIG 35GB) Basic Models (Other)" device: h200_35gb @@ -153,15 +170,17 @@ steps: - tests/models/test_adapters.py - tests/models/test_qwen3_5_mtp_config.py - tests/models/transformers/fusers/ + - tests/model_executor/layers/test_activation.py device: cpu-small commands: - - pytest -v -s models/test_utils.py models/test_vision.py models/test_adapters.py models/test_qwen3_5_mtp_config.py models/transformers/fusers/ + - pytest -v -s models/test_utils.py models/test_vision.py models/test_adapters.py models/test_qwen3_5_mtp_config.py models/transformers/fusers/ model_executor/layers/test_activation.py mirror: amd: label: ":amd: (MI250) Basic Models Other" + dind: false device: mi250_1 no_gpu: true working_dir: "/vllm-workspace/tests" - timeout_in_minutes: 25 + timeout_in_minutes: 35 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/models_distributed.yaml b/.buildkite/test_areas/models_distributed.yaml index f735a9b018..a054c00db8 100644 --- a/.buildkite/test_areas/models_distributed.yaml +++ b/.buildkite/test_areas/models_distributed.yaml @@ -38,7 +38,7 @@ steps: label: ":amd: (MI300) Distributed Models Shard %N" dind: false device: mi300_2 - timeout_in_minutes: 35 + timeout_in_minutes: 80 depends_on: - image-build-amd source_file_dependencies: @@ -63,7 +63,7 @@ steps: - | case "$$BUILDKITE_PARALLEL_JOB" in 0) - TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)' + TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)' && HIP_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)' ;; 1) @@ -72,8 +72,8 @@ steps: VLLM_ROCM_USE_AITER=1 pytest -v -s models/test_hyv4_rocm.py -m 'distributed(num_gpus=2)' ;; 2) - pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py - pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)' + pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py && + pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)' && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)' ;; *) exit 2 ;; diff --git a/.buildkite/test_areas/models_language.yaml b/.buildkite/test_areas/models_language.yaml index 68f1cdd429..3f80dfdfa3 100644 --- a/.buildkite/test_areas/models_language.yaml +++ b/.buildkite/test_areas/models_language.yaml @@ -19,7 +19,7 @@ steps: label: ":amd: (MI300) Language Models (Standard)" dind: false device: mi300_1 - timeout_in_minutes: 45 + timeout_in_minutes: 60 depends_on: - image-build-amd @@ -82,10 +82,10 @@ steps: parallelism: 2 mirror: amd: - label: ":amd: (MI300) Language Models (Hybrid) Shard %N" + label: ":amd: (MI355) Language Models (Hybrid) Shard %N" dind: false - device: mi300_1 - timeout_in_minutes: 60 + device: mi355_1 + timeout_in_minutes: 65 depends_on: - image-build-amd commands: @@ -133,7 +133,7 @@ steps: label: ":amd: (MI355) Language Models (Extended Generation)" dind: false device: mi355_1 - timeout_in_minutes: 70 + timeout_in_minutes: 95 depends_on: - image-build-amd commands: @@ -155,6 +155,7 @@ steps: mirror: amd: label: ":amd: (MI250) Language Models (PPL)" + dind: false device: mi250_1 timeout_in_minutes: 65 depends_on: @@ -197,11 +198,8 @@ steps: dind: false device: mi300_1 timeout_in_minutes: 95 - parallelism: 2 depends_on: - image-build-amd - commands: - - pytest -v -s models/language/pooling -m 'not core_model' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - label: ":nvidia: (H200 MIG 18GB) Language Models (MTEB)" key: language-models-test-mteb @@ -217,6 +215,7 @@ steps: mirror: amd: label: ":amd: (MI250) Language Models (MTEB)" + dind: false device: mi250_1 timeout_in_minutes: 60 depends_on: diff --git a/.buildkite/test_areas/models_multimodal.yaml b/.buildkite/test_areas/models_multimodal.yaml index b09e8cc2d9..779801a2ea 100644 --- a/.buildkite/test_areas/models_multimodal.yaml +++ b/.buildkite/test_areas/models_multimodal.yaml @@ -57,6 +57,7 @@ steps: mirror: amd: label: ":amd: (MI250) Multimodal Models (Standard) 3: llava + qwen2_vl" + dind: false device: mi250_1 timeout_in_minutes: 55 depends_on: @@ -77,9 +78,9 @@ steps: - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model # Otherwise, mp_method="spawn" doesn't work mirror: amd: - label: ":amd: (MI300) Multimodal Models (Standard) 4: other + whisper" + label: ":amd: (MI355) Multimodal Models (Standard) 4: other + whisper" dind: false - device: mi300_1 + device: mi355_1 timeout_in_minutes: 70 depends_on: - image-build-amd @@ -101,29 +102,33 @@ steps: mirror: amd: label: ":amd: (MI250) Multimodal Processor Shard %N" + dind: false device: mi250_1 no_gpu: true working_dir: "/vllm-workspace/tests" - timeout_in_minutes: 80 + timeout_in_minutes: 90 + parallelism: 6 depends_on: - image-build-amd -- label: ":nvidia: (H200 MIG 18GB) Multimodal Processor" +- label: ":nvidia: (H200 MIG 18GB) Multimodal Processor Shard %N" key: multi-modal-processor timeout_in_minutes: 98 device: h200_18gb + parallelism: 4 source_file_dependencies: - vllm/ - "!vllm/distributed/kv_transfer/" - tests/models/multimodal - tests/models/registry.py commands: - - pytest -v -s models/multimodal/processing/test_tensor_schema.py + - pytest -v -s models/multimodal/processing/test_tensor_schema.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB mirror: amd: - label: ":amd: (MI355) Multimodal Processor" + label: ":amd: (MI355) Multimodal Processor Shard %N" dind: false device: mi355_1 + parallelism: 4 timeout_in_minutes: 115 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/plugins.yaml b/.buildkite/test_areas/plugins.yaml index 65e7a0b258..6b901d8f49 100644 --- a/.buildkite/test_areas/plugins.yaml +++ b/.buildkite/test_areas/plugins.yaml @@ -53,6 +53,7 @@ steps: mirror: amd: label: ":amd: (MI250) Plugin Integration" + dind: false device: mi250_2 timeout_in_minutes: 50 depends_on: @@ -75,6 +76,20 @@ steps: commands: - pip install "vllm-gguf-plugin >= 0.0.2" - pytest -v -s plugins_tests/gguf + mirror: + amd: + label: ":amd: (MI355) GGUF Plugin" + dind: false + device: mi355_1 + timeout_in_minutes: 45 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/model_executor/layers/quantization + - tests/plugins_tests/test_gguf_plugin.py + - tests/plugins_tests/gguf + - vllm/platforms/rocm.py - label: ":nvidia: (H200 MIG 18GB) BitsAndBytes Plugin" key: bitsandbytes-plugin diff --git a/.buildkite/test_areas/pytorch.yaml b/.buildkite/test_areas/pytorch.yaml index f34ff50e67..459f7faeb4 100644 --- a/.buildkite/test_areas/pytorch.yaml +++ b/.buildkite/test_areas/pytorch.yaml @@ -122,6 +122,21 @@ steps: - tests/compile/passes commands: - pytest -s -v compile/passes --ignore compile/passes/distributed + mirror: + amd: + label: ':amd: (MI300) PyTorch Compilation Passes' + dind: false + device: mi300_1 + timeout_in_minutes: 80 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/ + - tests/compile/passes + - vllm/platforms/rocm.py + commands: + - pytest -s -v compile/passes --ignore compile/passes/distributed - label: ":nvidia: (H200 MIG 35GB) PyTorch Fullgraph" device: h200_35gb @@ -154,6 +169,7 @@ steps: - vllm/utils/ - vllm/v1/ - tests/compile + - csrc/ commands: # Run smoke tests under fullgraph directory, except test_full_cudagraph.py # as it is a heavy test that is covered in other steps. diff --git a/.buildkite/test_areas/quantization.yaml b/.buildkite/test_areas/quantization.yaml index 1d895e1f2f..702bbe7024 100644 --- a/.buildkite/test_areas/quantization.yaml +++ b/.buildkite/test_areas/quantization.yaml @@ -20,6 +20,31 @@ steps: # The SM90-only checkpoint currently contains a removed weight_chan_scale # parameter. It was not exercised by the previous L4 job. - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py -k 'not test_compressed_tensors_w4a8_fp8' --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + mirror: + amd: + label: ":amd: (MI355) Quantization Shard %N" + dind: false + device: mi355_1 + timeout_in_minutes: 115 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd + source_file_dependencies: + - csrc/ + - vllm/model_executor/layers/quantization + - tests/quantization + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/model_executor/layers/fused_moe/oracle/unquantized.py + - vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py + - tests/rocm/test_moe_weight_replay.py + commands: + # Use the ROCm default instead of the NVIDIA runner override. + - unset VLLM_USE_V2_MODEL_RUNNER + - uv pip install --system torchao==0.17.0 + - uv pip install --system conch-triton-kernels + - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s rocm/test_moe_weight_replay.py; fi - label: ":nvidia: (H200 MIG 35GB) Quantized Fusions" device: h200_35gb @@ -63,3 +88,22 @@ steps: - tests/models/quantization commands: - pytest -v -s models/quantization + mirror: + amd: + label: ":amd: (MI355) Quantized Models" + dind: false + device: mi355_1 + timeout_in_minutes: 80 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/model_executor/layers/quantization + - tests/models/quantization + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/model_executor/model_loader/ + commands: + # Use the ROCm default instead of the NVIDIA runner override. + - unset VLLM_USE_V2_MODEL_RUNNER + - pytest -v -s models/quantization diff --git a/.buildkite/test_areas/ray_compat.yaml b/.buildkite/test_areas/ray_compat.yaml index d24c38430b..dc8571b260 100644 --- a/.buildkite/test_areas/ray_compat.yaml +++ b/.buildkite/test_areas/ray_compat.yaml @@ -16,3 +16,19 @@ steps: - setup.py commands: - bash /vllm-workspace/.buildkite/scripts/check-ray-compatibility.sh + mirror: + amd: + label: ":amd: (MI355) Ray Dependency Compatibility Check" + dind: false + device: mi355_1 + timeout_in_minutes: 25 + working_dir: "/" + depends_on: + - image-build-amd + source_file_dependencies: + - .buildkite/scripts/check-ray-compatibility.sh + - requirements/ + - setup.py + - vllm/platforms/rocm.py + commands: + - bash /vllm-workspace/.buildkite/scripts/check-ray-compatibility.sh diff --git a/.buildkite/test_areas/rust_frontend.yaml b/.buildkite/test_areas/rust_frontend.yaml index 770d6b64df..a679b405ed 100644 --- a/.buildkite/test_areas/rust_frontend.yaml +++ b/.buildkite/test_areas/rust_frontend.yaml @@ -136,7 +136,7 @@ steps: label: ":amd: (MI355) Rust Frontend Core Correctness" dind: false device: mi355_1 - timeout_in_minutes: 20 + timeout_in_minutes: 25 depends_on: - image-build-amd source_file_dependencies: @@ -166,7 +166,7 @@ steps: label: ":amd: (MI355) Rust Frontend Tool Use" dind: false device: mi355_1 - timeout_in_minutes: 25 + timeout_in_minutes: 30 depends_on: - image-build-amd source_file_dependencies: diff --git a/.buildkite/test_areas/rust_frontend_cargo.yaml b/.buildkite/test_areas/rust_frontend_cargo.yaml index 0b49787d5d..53d1da7edf 100644 --- a/.buildkite/test_areas/rust_frontend_cargo.yaml +++ b/.buildkite/test_areas/rust_frontend_cargo.yaml @@ -17,6 +17,7 @@ steps: mirror: amd: label: ":amd: (MI250) Rust Frontend Cargo Style + Clippy" + dind: false device: mi250_1 no_gpu: true working_dir: "/vllm-workspace" @@ -40,6 +41,7 @@ steps: mirror: amd: label: ":amd: (MI250) Rust Frontend Cargo" + dind: false device: mi250_1 no_gpu: true working_dir: "/vllm-workspace" diff --git a/.buildkite/test_areas/samplers.yaml b/.buildkite/test_areas/samplers.yaml index 18f857f65a..e1bc4c08d8 100644 --- a/.buildkite/test_areas/samplers.yaml +++ b/.buildkite/test_areas/samplers.yaml @@ -20,6 +20,7 @@ steps: mirror: amd: label: ":amd: (MI250) Samplers" + dind: false device: mi250_1 timeout_in_minutes: 40 depends_on: diff --git a/.buildkite/test_areas/spec_decode.yaml b/.buildkite/test_areas/spec_decode.yaml index 8387270359..f60d745df3 100644 --- a/.buildkite/test_areas/spec_decode.yaml +++ b/.buildkite/test_areas/spec_decode.yaml @@ -47,7 +47,7 @@ steps: label: ":amd: (MI300) Spec Decode Eagle 1: DeepSeek + Qwen" dind: false device: mi300_1 - timeout_in_minutes: 35 + timeout_in_minutes: 60 depends_on: - image-build-amd source_file_dependencies: @@ -74,7 +74,7 @@ steps: label: ":amd: (MI300) Spec Decode Eagle 2: Llama 3 + Qwen VL + Other" dind: false device: mi300_1 - timeout_in_minutes: 25 + timeout_in_minutes: 60 depends_on: - image-build-amd source_file_dependencies: @@ -174,7 +174,7 @@ steps: - label: ":nvidia: (H200 MIG 18GB) Spec Decode Draft Model" key: spec-decode-draft-model - timeout_in_minutes: 45 + timeout_in_minutes: 60 device: h200_18gb source_file_dependencies: - vllm/v1/spec_decode/ @@ -187,7 +187,7 @@ steps: label: ":amd: (MI300) Spec Decode Draft Model" dind: false device: mi300_1 - timeout_in_minutes: 55 + timeout_in_minutes: 60 depends_on: - image-build-amd source_file_dependencies: @@ -233,6 +233,9 @@ steps: timeout_in_minutes: 40 depends_on: - image-build-amd + source_file_dependencies: + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py - label: ":nvidia: (B200) Spec Decode DeepSeek MTP Parallel Load" key: spec-decode-deepseek-mtp-parallel-load-2xb200-2xmi300 @@ -280,6 +283,8 @@ steps: - vllm/v1/spec_decode/ - vllm/v1/worker/gpu/spec_decode/ - vllm/model_executor/models/qwen3_dflash.py + - vllm/model_executor/models/qwen3_dflash2.py + - vllm/model_executor/models/qwen3_5.py - vllm/model_executor/models/laguna_dflash.py - tests/v1/e2e/spec_decode/ commands: @@ -290,6 +295,8 @@ steps: dind: false device: mi300_1 timeout_in_minutes: 120 + commands: + - pytest -v -s v1/e2e/spec_decode/acceptance_rates/dflash/ -k "not dflash2" depends_on: - image-build-amd source_file_dependencies: @@ -362,6 +369,33 @@ steps: # Each Gemma4 parametrization is an indivisible ~23m floor. Keep them in # separate shards and put the remaining tests in shard 0, where new tests # land by default. - - test "$$BUILDKITE_PARALLEL_JOB" != "0" || pytest -v -s v1/e2e/spec_decode/acceptance_rates/mtp_other/ --deselect "tests/v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[False]" --deselect "tests/v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[True]" - - test "$$BUILDKITE_PARALLEL_JOB" != "1" || pytest -v -s "v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[False]" - - test "$$BUILDKITE_PARALLEL_JOB" != "2" || pytest -v -s "v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[True]" + - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s v1/e2e/spec_decode/acceptance_rates/mtp_other/ --deselect "tests/v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[False]" --deselect "tests/v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[True]"; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then pytest -v -s "v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[False]"; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "2" ]; then pytest -v -s "v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[True]"; fi + mirror: + amd: + label: ":amd: (MI355) Spec Decode AL MTP + Other Acceptance Nightly" + dind: false + device: mi355_1 + timeout_in_minutes: 55 + depends_on: + - image-build-amd + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/sample/ + - vllm/v1/worker/gpu/ + - vllm/v1/worker/gpu_model_runner.py + - vllm/v1/worker/gpu_worker.py + - vllm/model_executor/model_loader/ + - vllm/model_executor/models/gemma4.py + - vllm/model_executor/models/gemma4_mm.py + - vllm/model_executor/models/gemma4_mtp.py + - vllm/model_executor/models/llama.py + - vllm/model_executor/models/llama_eagle3.py + - vllm/model_executor/models/medusa.py + - vllm/model_executor/models/registry.py + - tests/evals/gsm8k/ + - tests/utils.py + - tests/v1/e2e/spec_decode/ + - vllm/platforms/rocm.py diff --git a/.buildkite/test_areas/weight_loading.yaml b/.buildkite/test_areas/weight_loading.yaml deleted file mode 100644 index 154b355778..0000000000 --- a/.buildkite/test_areas/weight_loading.yaml +++ /dev/null @@ -1,38 +0,0 @@ -group: Weight Loading -depends_on: - - image-build -steps: -- label: ":nvidia: (L4) Weight Loading Multi-GPU" - key: weight-loading-multiple-gpu - timeout_in_minutes: 50 - working_dir: "/vllm-workspace/tests" - device: l4 - num_devices: 2 - optional: true - source_file_dependencies: - - vllm/ - - "!vllm/distributed/kv_transfer/" - - tests/weight_loading - commands: - - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models.txt - mirror: - amd: - label: ":amd: (MI355) Weight Loading Multi-GPU" - dind: false - device: mi355_2 - timeout_in_minutes: 40 - depends_on: - - image-build-amd - commands: - - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-amd.txt - -# - label: Weight Loading Multiple GPU - Large Models # optional -# working_dir: "/vllm-workspace/tests" -# num_devices: 2 -# device: a100 -# optional: true -# source_file_dependencies: -# - vllm/ -# - tests/weight_loading -# commands: -# - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-large.txt diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index aa5b2a9fc9..f64cf5f561 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -110,7 +110,6 @@ /tests/entrypoints/llm/test_struct_output_generate.py @mgoin @aarnphm @arpera /tests/v1/structured_output @mgoin @aarnphm @arpera /tests/v1/core @WoosukKwon @robertgshaw2-redhat @njhill @ywang96 @alexm-redhat @heheda12345 @ApostaC @orozery @ivanium -/tests/weight_loading @mgoin @youkaichao @yewentao256 /tests/lora @jeejeelee /tests/models/language/generation/test_hybrid.py @tdoublep @tomeras91 /tests/v1/kv_connector/nixl_integration @NickLucche diff --git a/cmake/cpu_extension.cmake b/cmake/cpu_extension.cmake index ff2911dc11..cca2a98cee 100644 --- a/cmake/cpu_extension.cmake +++ b/cmake/cpu_extension.cmake @@ -488,6 +488,7 @@ set(VLLM_EXT_SRC "csrc/moe/dynamic_4bit_int_moe_cpu.cpp" "csrc/cpu/cpu_fused_moe.cpp" "csrc/cpu/cpu_attn.cpp" + "csrc/cpu/cpu_isa.cpp" "csrc/cpu/torch_bindings.cpp") if (CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64" AND VLLM_RVV_VLEN AND @@ -522,6 +523,12 @@ if (POWER9_FOUND OR POWER10_FOUND OR POWER11_FOUND) ${VLLM_EXT_SRC}) endif() +if (POWER10_FOUND OR POWER11_FOUND) + set(VLLM_EXT_SRC + "csrc/cpu/cpu_fused_moe_int8.cpp" + ${VLLM_EXT_SRC}) +endif() + if(USE_ONEDNN) set(VLLM_EXT_SRC "csrc/cpu/dnnl_kernels.cpp" @@ -540,11 +547,13 @@ if (ENABLE_X86_ISA) "csrc/cpu/sgl-kernels/gemm.cpp" "csrc/cpu/sgl-kernels/gemm_int8.cpp" "csrc/cpu/sgl-kernels/gemm_fp8.cpp" + "csrc/cpu/sgl-kernels/gemm_fp8_w8a8.cpp" "csrc/cpu/sgl-kernels/gemm_int4.cpp" "csrc/cpu/sgl-kernels/moe.cpp" "csrc/cpu/sgl-kernels/moe_int8.cpp" "csrc/cpu/sgl-kernels/moe_int4.cpp" "csrc/cpu/sgl-kernels/moe_fp8.cpp" + "csrc/cpu/sgl-kernels/moe_fp8_w8a8.cpp" "csrc/cpu/sgl-kernels/bmm.cpp" "csrc/cpu/sgl-kernels/decode.cpp" "csrc/cpu/sgl-kernels/extend.cpp" @@ -565,6 +574,7 @@ if (ENABLE_X86_ISA) "csrc/cpu/utils.cpp" "csrc/cpu/spec_decode_utils.cpp" "csrc/cpu/cpu_attn.cpp" + "csrc/cpu/cpu_isa.cpp" "csrc/cpu/dnnl_kernels.cpp" "csrc/cpu/mamba_cpu.cpp" "csrc/cpu/torch_bindings.cpp" @@ -581,6 +591,7 @@ if (ENABLE_X86_ISA) "csrc/cpu/utils.cpp" "csrc/cpu/spec_decode_utils.cpp" "csrc/cpu/cpu_attn.cpp" + "csrc/cpu/cpu_isa.cpp" "csrc/cpu/mamba_cpu.cpp" "csrc/cpu/dnnl_kernels.cpp" "csrc/cpu/torch_bindings.cpp" diff --git a/csrc/cpu/cpu_attn.cpp b/csrc/cpu/cpu_attn.cpp index b31a6faa01..ea0b4e49b2 100644 --- a/csrc/cpu/cpu_attn.cpp +++ b/csrc/cpu/cpu_attn.cpp @@ -1,16 +1,7 @@ #include "cpu_attn_dispatch_generated.h" -#if defined(__x86_64__) || defined(_M_X64) - #include -// Runtime check for AMX-FP8 (Diamond Rapids): CPUID leaf 7, subleaf 1, EAX[21]. -static bool runtime_has_amx_fp8() { - unsigned int eax = 0, ebx = 0, ecx = 0, edx = 0; - if (__get_cpuid_count(7, 1, &eax, &ebx, &ecx, &edx)) { - return (eax >> 21) & 1u; - } - return false; -} -#endif // x86_64 +// Runtime check for AMX-FP8, implemented in cpu_isa.cpp. +extern bool runtime_has_amx_fp8(); // Maps kv_cache_dtype string to Fp8KVCacheDataType enum. // "auto" -> kAuto(0); "fp8"/"fp8_e4m3" -> kFp8E4M3; "fp8_e5m2" -> kFp8E5M2. diff --git a/csrc/cpu/cpu_fused_moe_int8.cpp b/csrc/cpu/cpu_fused_moe_int8.cpp index 649317b478..27d0f6a7d5 100644 --- a/csrc/cpu/cpu_fused_moe_int8.cpp +++ b/csrc/cpu/cpu_fused_moe_int8.cpp @@ -26,10 +26,23 @@ #define NEON_DISPATCH(SCALAR_TYPE, ...) case cpu_utils::ISA::NEON: #endif +#if defined(__powerpc64__) + #include "cpu/micro_gemm/cpu_micro_gemm_int8_vsx.hpp" + #define VSX_DISPATCH(SCALAR_TYPE, ...) \ + case cpu_utils::ISA::VSX: { \ + using gemm_t = \ + cpu_micro_gemm::MicroGemmINT8; \ + return __VA_ARGS__(); \ + } +#else + #define VSX_DISPATCH(SCALAR_TYPE, ...) case cpu_utils::ISA::VSX: +#endif + #define CPU_INT8_ISA_DISPATCH_IMPL(ISA_TYPE, SCALAR_TYPE, ...) \ [&] { \ switch (ISA_TYPE) { \ NEON_DISPATCH(SCALAR_TYPE, __VA_ARGS__) \ + VSX_DISPATCH(SCALAR_TYPE, __VA_ARGS__) \ default: { \ TORCH_CHECK(false, "Invalid CPU ISA type."); \ } \ @@ -81,9 +94,9 @@ void fused_moe_int8_impl( const int32_t thread_num = cpu_utils::get_max_threads(); const int32_t w13_input_buffer_size = cpu_utils::round_up<64>( - gemm_m_tile_size * input_size_13 * sizeof(int8_t)); - const int32_t w2_input_buffer_size = - cpu_utils::round_up<64>(gemm_m_tile_size * input_size_2 * sizeof(int8_t)); + (gemm_m_tile_size * input_size_13 + 64) * sizeof(int8_t)); + const int32_t w2_input_buffer_size = cpu_utils::round_up<64>( + (gemm_m_tile_size * input_size_2 + 64) * sizeof(int8_t)); const int32_t w13_n_tile_size = [&]() { const int64_t cache_size = cpu_utils::get_available_l2_size(); diff --git a/csrc/cpu/cpu_isa.cpp b/csrc/cpu/cpu_isa.cpp new file mode 100644 index 0000000000..1c4a8faf19 --- /dev/null +++ b/csrc/cpu/cpu_isa.cpp @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +// AMX-FP8 is reported at CPUID leaf 0x1E, subleaf 1, EAX[4]. +#if defined(__x86_64__) || defined(_M_X64) + #include + +bool runtime_has_amx_fp8() { + unsigned int eax = 0, ebx = 0, ecx = 0, edx = 0; + if (__get_cpuid_max(0, nullptr) < 0x1e) return false; + if (__get_cpuid_count(0x1e, 1, &eax, &ebx, &ecx, &edx)) { + return (eax >> 4) & 1u; + } + return false; +} + +bool cpu_has_amx_fp8() { return runtime_has_amx_fp8(); } +#else +bool cpu_has_amx_fp8() { return false; } +#endif diff --git a/csrc/cpu/cpu_types_vsx.hpp b/csrc/cpu/cpu_types_vsx.hpp index d46906130e..eaea921d98 100644 --- a/csrc/cpu/cpu_types_vsx.hpp +++ b/csrc/cpu/cpu_types_vsx.hpp @@ -13,10 +13,10 @@ namespace vec_op { struct fp8_e4m3_tag {}; struct fp8_e5m2_tag {}; -#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \ - AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \ - AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__) \ - AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) +#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \ + AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \ + AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__) +// Note: FP16 (Half) is not supported on POWER VSX — no FP16Vec16 type. #define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \ AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__)) @@ -416,6 +416,7 @@ struct FP32Vec8 : public Vec { tmp.val[0] = two_x; tmp.val[1] = two_x; FP32Vec8 temp_vec(tmp); + FP32Vec8 exp_vec = temp_vec.exp(); vector float e = temp_vec.exp().reg.val[0]; vector float num = vec_sub(e, one); @@ -634,6 +635,16 @@ struct FP32Vec16 : public Vec { reg.val[3] = vec_ctf(v.reg.val[3], 0); } + explicit FP32Vec16(const float* __restrict__ base, const INT32Vec16& index) { + INT32Vec16::AliasReg idx; + idx.reg = index.reg; + AliasReg ar; + for (int i = 0; i < VEC_ELEM_NUM; ++i) { + ar.values[i] = base[idx.values[i]]; + } + reg = ar.reg; + } + FP32Vec16 operator*(const FP32Vec16& b) const { return FP32Vec16(f32x4x4_t({vec_mul(reg.val[0], b.reg.val[0]), vec_mul(reg.val[1], b.reg.val[1]), @@ -648,6 +659,13 @@ struct FP32Vec16 : public Vec { vec_add(reg.val[3], b.reg.val[3])})); } + FP32Vec16 operator-() const { + const __vector float zero = vec_splats(0.0f); + return FP32Vec16( + f32x4x4_t({vec_sub(zero, reg.val[0]), vec_sub(zero, reg.val[1]), + vec_sub(zero, reg.val[2]), vec_sub(zero, reg.val[3])})); + } + FP32Vec16 operator-(const FP32Vec16& b) const { return FP32Vec16(f32x4x4_t({vec_sub(reg.val[0], b.reg.val[0]), vec_sub(reg.val[1], b.reg.val[1]), @@ -655,11 +673,6 @@ struct FP32Vec16 : public Vec { vec_sub(reg.val[3], b.reg.val[3])})); } - FP32Vec16 operator-() const { - return FP32Vec16( - f32x4x4_t({-reg.val[0], -reg.val[1], -reg.val[2], -reg.val[3]})); - } - FP32Vec16 operator/(const FP32Vec16& b) const { return FP32Vec16(f32x4x4_t({vec_div(reg.val[0], b.reg.val[0]), vec_div(reg.val[1], b.reg.val[1]), diff --git a/csrc/cpu/micro_gemm/cpu_micro_gemm_int8_vsx.hpp b/csrc/cpu/micro_gemm/cpu_micro_gemm_int8_vsx.hpp new file mode 100644 index 0000000000..6033353728 --- /dev/null +++ b/csrc/cpu/micro_gemm/cpu_micro_gemm_int8_vsx.hpp @@ -0,0 +1,287 @@ +#ifndef CPU_MICRO_GEMM_INT8_VSX_HPP +#define CPU_MICRO_GEMM_INT8_VSX_HPP + +#include +#include +#include +#include +#include + +#include "cpu/micro_gemm/cpu_micro_gemm_impl.hpp" + +#include +#include +#include + +namespace cpu_micro_gemm { + +namespace vsx_int8 { + +constexpr int32_t K = 4; +constexpr int32_t Mr = 4; +constexpr int32_t Nr = 32; + +// --------------------------------------------------------------------------- +// 8-Accumulator MMA 4x32 micro-GEMM (POWER10 xvi8ger4pp). +// +// Computes 4 rows x 32 columns (128 outputs) using all 8 hardware accumulators. +// Operates on pre-biased uint8 weights (B + 128) and subtracts 128 * a_sums. +// Works for all m in [1, 4] with zero scalar loop overhead. +// --------------------------------------------------------------------------- +__attribute__((target("cpu=power10"))) FORCE_INLINE void gemm_mma_4x32( + const int8_t* __restrict__ a_packed, const uint8_t* __restrict__ b_panel, + const int32_t* __restrict__ a_sums, int32_t* __restrict__ c_ptr, + const int32_t m, const int32_t k_size, const int64_t ldc) { + TORCH_CHECK(m > 0 && m <= Mr); + TORCH_CHECK(k_size % K == 0); + + __vector_quad acc0, acc1, acc2, acc3, acc4, acc5, acc6, acc7; + __builtin_mma_xxsetaccz(&acc0); + __builtin_mma_xxsetaccz(&acc1); + __builtin_mma_xxsetaccz(&acc2); + __builtin_mma_xxsetaccz(&acc3); + __builtin_mma_xxsetaccz(&acc4); + __builtin_mma_xxsetaccz(&acc5); + __builtin_mma_xxsetaccz(&acc6); + __builtin_mma_xxsetaccz(&acc7); + + const int8_t* __restrict__ ap = a_packed; + const uint8_t* __restrict__ bp = b_panel; + +#pragma GCC unroll 4 + for (int32_t k = 0; k < k_size; k += 4) { + const __vector unsigned char avec = + vec_xl(0, reinterpret_cast(ap)); + __builtin_mma_xvi8ger4pp(&acc0, avec, vec_xl(0, bp)); + __builtin_mma_xvi8ger4pp(&acc1, avec, vec_xl(16, bp)); + __builtin_mma_xvi8ger4pp(&acc2, avec, vec_xl(32, bp)); + __builtin_mma_xvi8ger4pp(&acc3, avec, vec_xl(48, bp)); + __builtin_mma_xvi8ger4pp(&acc4, avec, vec_xl(64, bp)); + __builtin_mma_xvi8ger4pp(&acc5, avec, vec_xl(80, bp)); + __builtin_mma_xvi8ger4pp(&acc6, avec, vec_xl(96, bp)); + __builtin_mma_xvi8ger4pp(&acc7, avec, vec_xl(112, bp)); + ap += 16; + bp += 128; + } + + __vector signed int res[4]; +#define STORE_ACC(ACC, CG_IDX) \ + { \ + __builtin_mma_disassemble_acc(res, &ACC); \ + for (int r = 0; r < m; ++r) { \ + const __vector signed int vcorr = vec_splats(128 * a_sums[r]); \ + const __vector signed int val = vec_sub(res[r], vcorr); \ + vec_xst(val, 0, \ + reinterpret_cast(c_ptr + r * ldc + (CG_IDX) * 4)); \ + } \ + } + STORE_ACC(acc0, 0); + STORE_ACC(acc1, 1); + STORE_ACC(acc2, 2); + STORE_ACC(acc3, 3); + STORE_ACC(acc4, 4); + STORE_ACC(acc5, 5); + STORE_ACC(acc6, 6); + STORE_ACC(acc7, 7); +#undef STORE_ACC +} + +} // namespace vsx_int8 + +// ============================================================================= +// MicroGemmINT8 specialisation for VSX (POWER10+) +// ============================================================================= +template +class MicroGemmINT8 { + public: + static constexpr int32_t K = vsx_int8::K; // 4 + static constexpr int32_t Mr = vsx_int8::Mr; // 4 + static constexpr int32_t Nr = vsx_int8::Nr; // 32 + static constexpr int32_t NrGemv = Nr; + static constexpr int32_t MaxMSize = 8; + static constexpr int32_t NSize = Nr; + static constexpr int32_t WeightOCGroupSize = Nr; + static_assert(MaxMSize % Mr == 0); + + // ------------------------------------------------------------------------- + // quantize_row: scalar_t -> INT8 + per-token scale + // Single-pass 16-element vector conversion with aligned 16-byte stores. + // ------------------------------------------------------------------------- + static FORCE_INLINE void quantize_row(const scalar_t* input, int8_t* output, + float& scale, const int32_t size) { + TORCH_CHECK_EQ(size % 16, 0); + + const __vector signed short z = vec_splats((signed short)0); + + auto load_lo_hi = + [&](int32_t i) -> std::pair<__vector float, __vector float> { + if constexpr (std::is_same_v) { + const float* p = reinterpret_cast(input) + i; + return {vec_xl(0, p), vec_xl(16, p)}; + } else { + const signed short* p = + reinterpret_cast(input + i); + const __vector signed short raw = (__vector signed short)vec_xl(0, p); + return {(__vector float)vec_mergeh(z, raw), + (__vector float)vec_mergel(z, raw)}; + } + }; + + __vector float max_v = vec_splats(0.0f); + for (int32_t i = 0; i < size; i += 8) { + auto [lo, hi] = load_lo_hi(i); + max_v = vec_max(max_v, vec_abs(lo)); + max_v = vec_max(max_v, vec_abs(hi)); + } + __vector float t = vec_max(max_v, vec_sld(max_v, max_v, 8)); + t = vec_max(t, vec_sld(t, t, 4)); + const float abs_max = std::max(vec_extract(t, 0), 1.0e-7f); + scale = abs_max / 127.0f; + const __vector float inv_scale = vec_splats(127.0f / abs_max); + + for (int32_t i = 0; i < size; i += 16) { + auto [lo0, hi0] = load_lo_hi(i); + auto [lo1, hi1] = load_lo_hi(i + 8); + + const __vector signed int i0 = + vec_cts(vec_round(vec_mul(lo0, inv_scale)), 0); + const __vector signed int i1 = + vec_cts(vec_round(vec_mul(hi0, inv_scale)), 0); + const __vector signed int i2 = + vec_cts(vec_round(vec_mul(lo1, inv_scale)), 0); + const __vector signed int i3 = + vec_cts(vec_round(vec_mul(hi1, inv_scale)), 0); + + const __vector signed short s0 = vec_packs(i0, i1); + const __vector signed short s1 = vec_packs(i2, i3); + const __vector signed char s8 = vec_packs(s0, s1); + + vec_xst(s8, 0, reinterpret_cast(output + i)); + } + } + + // ------------------------------------------------------------------------- + // dequantize_tile: INT32 accumulator -> FP32 scaled output + // ------------------------------------------------------------------------- + static FORCE_INLINE void dequantize_tile( + int32_t* input, float* output, const float* __restrict__ input_scales, + const float* __restrict__ weight_scales, const int32_t m, const int32_t n, + const int32_t stride) { + TORCH_CHECK_EQ(n % 4, 0); + for (int32_t mi = 0; mi < m; ++mi) { + const __vector float a_scale = vec_splats(input_scales[mi]); + for (int32_t ni = 0; ni < n; ni += 4) { + const __vector signed int i32 = vec_xl(0, input + mi * stride + ni); + const __vector float wscale = vec_xl(0, weight_scales + ni); + vec_xst(vec_mul(vec_ctf(i32, 0), vec_mul(a_scale, wscale)), 0, + output + mi * stride + ni); + } + } + } + + // ------------------------------------------------------------------------- + // pack_input_from_rows: pack rows into 4-row MMA tiles and compute row sums + // Header: 64 bytes (contains int32 a_sums[0..7]) + // Data: a_packed + 64 (contains 4x4 tiles of A) + // ------------------------------------------------------------------------- + static void pack_input_from_rows(const int8_t* const* __restrict__ rows, + int8_t* __restrict__ a_packed, + const int32_t m, const int32_t k) { + TORCH_CHECK(m > 0 && m <= MaxMSize); + TORCH_CHECK_EQ(k % K, 0); + + int32_t* a_sums = reinterpret_cast(a_packed); + for (int32_t r = 0; r < 8; ++r) a_sums[r] = 0; + + int8_t* a_data = a_packed + 64; + + for (int32_t row_base = 0; row_base < m; row_base += Mr) { + const int32_t panel_m = std::min(Mr, m - row_base); + const int8_t* const* panel_rows = rows + row_base; + int32_t* panel_sums = a_sums + row_base; + int8_t* dst_panel = a_data + (row_base / Mr) * (k * Mr); + + for (int32_t ki = 0; ki < k; ki += 4) { + int8_t* dst = dst_panel + (ki / 4) * 16; + for (int32_t r = 0; r < 4; ++r) { + if (r < panel_m) { + const int8_t* src = panel_rows[r] + ki; + dst[r * 4 + 0] = src[0]; + dst[r * 4 + 1] = src[1]; + dst[r * 4 + 2] = src[2]; + dst[r * 4 + 3] = src[3]; + panel_sums[r] += + static_cast(src[0]) + src[1] + src[2] + src[3]; + } else { + dst[r * 4 + 0] = 0; + dst[r * 4 + 1] = 0; + dst[r * 4 + 2] = 0; + dst[r * 4 + 3] = 0; + } + } + } + } + } + + // ------------------------------------------------------------------------- + // pack_weight: [output_size, input_size] INT8 -> packed layout + // Pre-biases with +128 in [N/32][K/4][8 col-groups][4 cols x 4 k] layout + // ------------------------------------------------------------------------- + static void pack_weight(const int8_t* __restrict__ weight, + int8_t* __restrict__ packed_weight, + const int32_t output_size, const int32_t input_size) { + TORCH_CHECK_EQ(output_size % NSize, 0); + TORCH_CHECK_EQ(input_size % K, 0); + + uint8_t* dst_base = reinterpret_cast(packed_weight); + + for (int32_t o = 0; o < output_size; o += Nr) { + uint8_t* dst_panel = dst_base + o * input_size; + for (int32_t k = 0; k < input_size; k += 4) { + for (int32_t cg = 0; cg < 8; ++cg) { + uint8_t* dst = dst_panel + (k / 4) * 128 + cg * 16; + for (int32_t col = 0; col < 4; ++col) { + for (int32_t ki = 0; ki < 4; ++ki) { + const int8_t w = weight[(o + cg * 4 + col) * input_size + k + ki]; + dst[col * 4 + ki] = + static_cast(static_cast(w) + 128); + } + } + } + } + } + } + + // ------------------------------------------------------------------------- + // gemm: 8-accumulator MMA GEMM over 32-column tiles + // ------------------------------------------------------------------------- + void gemm(const int8_t* __restrict__ a_packed, + const int8_t* __restrict__ b_packed, int32_t* __restrict__ c, + const int32_t m, const int32_t k, const int64_t b_n_group_stride, + const int64_t ldc) const { + TORCH_CHECK(m > 0 && m <= MaxMSize); + TORCH_CHECK_EQ(k % K, 0); + (void)b_n_group_stride; + + const int32_t* a_sums = reinterpret_cast(a_packed); + const int8_t* a_data = a_packed + 64; + const uint8_t* b_u8 = reinterpret_cast(b_packed); + + for (int32_t n_idx = 0; n_idx < NSize; n_idx += Nr) { + const uint8_t* b_panel = b_u8 + n_idx * k; + + for (int32_t row_base = 0; row_base < m; row_base += Mr) { + const int32_t panel_m = std::min(Mr, m - row_base); + const int8_t* a_panel = a_data + (row_base / Mr) * (k * Mr); + const int32_t* panel_sums = a_sums + row_base; + + vsx_int8::gemm_mma_4x32(a_panel, b_panel, panel_sums, + c + row_base * ldc + n_idx, panel_m, k, ldc); + } + } + } +}; + +} // namespace cpu_micro_gemm + +#endif // CPU_MICRO_GEMM_INT8_VSX_HPP diff --git a/csrc/cpu/sgl-kernels/common.h b/csrc/cpu/sgl-kernels/common.h index 6f27ba2d5c..de1260bb33 100644 --- a/csrc/cpu/sgl-kernels/common.h +++ b/csrc/cpu/sgl-kernels/common.h @@ -12,11 +12,45 @@ #include #include #include +#include #if defined(_OPENMP) #include #endif +// Adapted from sglang: dispatch utilities for FP8 W8A8 kernels +#define AT_DISPATCH_OUT_TYPES(TYPE, NAME, ...) \ + AT_DISPATCH_SWITCH( \ + TYPE, \ + NAME, \ + AT_PRIVATE_CASE_TYPE_USING_HINT(at::ScalarType::Float, out_t, __VA_ARGS__) \ + AT_PRIVATE_CASE_TYPE_USING_HINT(at::ScalarType::BFloat16, out_t, __VA_ARGS__) \ + AT_PRIVATE_CASE_TYPE_USING_HINT(at::ScalarType::Half, out_t, __VA_ARGS__)) + +#define AT_DISPATCH_CASE_ENUM_NO_RETURN(VALUE, TYPE, HINT, ...) \ + case VALUE: { \ + constexpr TYPE HINT = VALUE; \ + __VA_ARGS__; \ + break; \ + } + +#define AT_DISPATCH_BOOL_NO_RETURN(VALUE, NAME, HINT, ...) \ + [&]() { \ + switch (VALUE) { \ + AT_DISPATCH_CASE_ENUM_NO_RETURN(true, bool, HINT, __VA_ARGS__) \ + AT_DISPATCH_CASE_ENUM_NO_RETURN(false, bool, HINT, __VA_ARGS__) \ + } \ + }() + +#define AT_DISPATCH_QUANT_MODE_NO_RETURN(MODE, NAME, HINT, ...) \ + [&]() { \ + switch (MODE) { \ + AT_DISPATCH_CASE_ENUM_NO_RETURN(PER_TENSOR, int, HINT, __VA_ARGS__) \ + AT_DISPATCH_CASE_ENUM_NO_RETURN(PER_ROW, int, HINT, __VA_ARGS__) \ + AT_DISPATCH_CASE_ENUM_NO_RETURN(PER_GROUP, int, HINT, __VA_ARGS__) \ + } \ + }() + namespace { // dispatch bool @@ -472,4 +506,53 @@ inline T* conditional_data_ptr(const std::optional& opt) { return opt.has_value() ? opt.value().data_ptr() : nullptr; } +// Adapted from sglang: M blocking utilities for FP8 W8A8 kernels +inline int64_t get_m_block(int64_t M) { + if (M <= 48) { + return M; + } else if (M < 64) { + return 32; + } else if (M < 96) { + return 64; + } else { + return 128; + } +} + +inline std::tuple get_m_blocking(int64_t M) { + bool parallel_on_M = M > 128; + int64_t block_m = get_m_block(M); + int64_t Mc = (M + block_m - 1) / block_m; + int64_t Mc_parallel = parallel_on_M ? Mc : 1; + return std::make_tuple(parallel_on_M, block_m, Mc, Mc_parallel); +} + +#if defined(CPU_CAPABILITY_AVX512) +template +void zero_buffer(T* data, int64_t size) { + const int32_t vec_size = at::vec::Vectorized::size(); + auto zero_vec = at::vec::Vectorized(0); + int64_t d = 0; + for (; d < size - (size % vec_size); d += vec_size) { + zero_vec.store(data + d); + } + if (d < size) { + zero_vec.store(data + d, size - d); + } +} +#else +template +void zero_buffer(T* data, int64_t size) { + memset(data, 0, sizeof(T) * size); +} +#endif + +#if defined(__x86_64__) +bool avx10_2_available() { + // __builtin_cpu_supports returns the masked feature bit rather than 0/1. + static const bool available = __builtin_cpu_supports("avx10.2"); + return available; +} +#endif + } // anonymous namespace diff --git a/csrc/cpu/sgl-kernels/gemm.h b/csrc/cpu/sgl-kernels/gemm.h index db7ad884d0..dc5f9c585d 100644 --- a/csrc/cpu/sgl-kernels/gemm.h +++ b/csrc/cpu/sgl-kernels/gemm.h @@ -94,7 +94,7 @@ enum class CPUActMethod : int { gelu_and_mul = 2, }; -enum class CPUQuantMethod : int64_t { BF16 = 0, INT8_W8A8 = 1, FP8_W8A16 = 2, INT4_W4A8 = 3, MXFP4 = 4 }; +enum class CPUQuantMethod : int64_t { BF16 = 0, INT8_W8A8 = 1, FP8_W8A16 = 2, INT4_W4A8 = 3, MXFP4 = 4, FP8_W8A8 = 5 }; constexpr bool operator==(CPUQuantMethod a, int64_t b) { return static_cast(a) == b; @@ -395,3 +395,55 @@ void tinygemm_kernel( bool brg, int64_t block_size_K, bool do_unpack = true); + +// FP8 W8A8 tinygemm kernel (defined in gemm_fp8_w8a8.cpp) +void tinygemm_kernel( + float* C, + const at::Float8_e4m3fn* A, + const float* scales_a, + const at::Float8_e4m3fn* B, + const float* scales_b, + int64_t M, + int64_t K, + int64_t lda, + int64_t ldc, + int64_t ldsa, + float* ukernel_buf, + at::BFloat16* dqA_buf, + at::BFloat16* dqB_buf); + +// FP8 W8A8 quantize activation (defined in gemm_fp8_w8a8.cpp) +std::tuple quantize_fp8e4m3_vec( + const at::Tensor& t, + bool channelwise, + c10::optional scale_opt); + +// FP8 W8A8 fused MoE kernel (defined in moe_fp8_w8a8.cpp) +template +void fused_experts_fp8_a8_kernel_impl( + scalar_t* __restrict__ output, + scalar_t* __restrict__ ic0, + scalar_t* __restrict__ ic1, + scalar_t* __restrict__ ic2, + at::Float8_e4m3fn* __restrict__ A_tmp, + scalar_t* __restrict__ B_tmp, + float* __restrict__ C_tmp, + float* __restrict__ Ukernel_tmp, + const at::Float8_e4m3fn* __restrict__ input, + const at::Float8_e4m3fn* __restrict__ packed_w1, + const at::Float8_e4m3fn* __restrict__ packed_w2, + const float* __restrict__ As, + const float* __restrict__ w1s, + const float* __restrict__ w2s, + int64_t block_size_N, + int64_t block_size_K, + const float* __restrict__ topk_weights, + const int32_t* __restrict__ sorted_ids, + const int32_t* __restrict__ expert_ids, + const int32_t* __restrict__ offsets, + int64_t M, + int64_t N, + int64_t K, + int64_t E, + int64_t topk, + int64_t num_tokens_post_pad); diff --git a/csrc/cpu/sgl-kernels/gemm_fp8_w8a8.cpp b/csrc/cpu/sgl-kernels/gemm_fp8_w8a8.cpp new file mode 100644 index 0000000000..854566d682 --- /dev/null +++ b/csrc/cpu/sgl-kernels/gemm_fp8_w8a8.cpp @@ -0,0 +1,1093 @@ +// Adapted from +// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu + +// clang-format off + +#include +#include +#include + +#include "common.h" +#include "vec.h" + +// Runtime check for AMX-FP8, implemented in cpu_isa.cpp. +extern bool cpu_has_amx_fp8(); + +namespace { + +#define BLOCK_N 32 + +#define PER_TENSOR 1 +#define PER_ROW 2 +#define PER_GROUP 3 + +static bool cpublas_checked = false; +static bool cpublas_can_pack = false; + +bool cpublas_could_pack() { + // the could_pack check requires AMX support implicitly + if (cpublas_checked) { + return cpublas_can_pack; + } + cpublas_can_pack = cpu_has_amx_fp8() && at::native::cpublas::could_pack(at::kFloat8_e4m3fn); + cpublas_checked = true; + return cpublas_can_pack; +} + +#if defined(CPU_CAPABILITY_AVX512) +static void cvt_f8e4m3_to_bf16( + const at::Float8_e4m3fn* __restrict__ in, at::BFloat16* out, int64_t rows, int64_t cols, int64_t stride) { + if (stride == cols) { + // A contiguous buffer + size_t len = rows * cols; + size_t i = 0; + for (; i < len; i += 32) { + __m256i fp8_vec = _mm256_loadu_si256((__m256i*)&in[i]); + __m512bh bf16_vec = cvt_e4m3_bf16_intrinsic_no_nan(fp8_vec); + _mm512_storeu_si512((__m512i*)(out + i), (__m512i)bf16_vec); + } + for (; i < len; ++i) { + out[i] = (at::BFloat16)in[i]; + } + } else { + // Non-contiguous. Access each row with stride + TORCH_CHECK(stride > cols); + for (int r = 0; r < rows; ++r) { + size_t i = 0; + size_t vec_len = cols / 32 * 32; + for (; i < vec_len; i += 32) { + __m256i fp8_vec = _mm256_loadu_si256((__m256i*)&in[r * stride + i]); + __m512bh bf16_vec = cvt_e4m3_bf16_intrinsic_no_nan(fp8_vec); + _mm512_storeu_si512((__m512i*)(out + r * cols + i), (__m512i)bf16_vec); + } + for (; i < cols; ++i) { + out[r * cols + i] = (at::BFloat16)in[r * stride + i]; + } + } + } +} + +// accumulate and store result to buffer +// if act/wei are per_group quantized, apply scales +template +static void _accumulate_result( + float* __restrict__ output, + const float* __restrict__ input, + const float* __restrict__ scale_a, + const float* __restrict__ scale_b, + int M, + int ldi, + int ldo, + int ldsa = 1) { + float a_scale, b_scale; + __m512 va_scale; + __m512 vb_scale; + for (int m = 0; m < M; ++m) { + if constexpr (act_quant_mode == PER_GROUP) { + a_scale = *(scale_a + m * ldsa); + va_scale = _mm512_set1_ps(a_scale); + } + constexpr int N_UNROLL = N / 16; + c10::ForcedUnroll{}([&](auto i) { + constexpr int n = i * 16; + __m512 vc_f = _mm512_loadu_ps(input + m * ldi + n); + if constexpr (act_quant_mode == PER_GROUP) { + vc_f = _mm512_mul_ps(vc_f, va_scale); + } + if constexpr (wei_quant_mode == PER_GROUP) { + vb_scale = _mm512_loadu_ps(scale_b + n); + vc_f = _mm512_mul_ps(vc_f, vb_scale); + } + if constexpr (accum) { + __m512 vo = _mm512_loadu_ps(output + m * ldo + n); + _mm512_storeu_ps(output + m * ldo + n, _mm512_add_ps(vo, vc_f)); + } else { + _mm512_storeu_ps(output + m * ldo + n, vc_f); + } + }); + constexpr int tail_start = N / 16 * 16; + for (int n = tail_start; n < N; ++n) { + float dq_val = input[m * ldi + n]; + if constexpr (act_quant_mode == PER_GROUP) { + dq_val = dq_val * a_scale; + } + if constexpr (wei_quant_mode == PER_GROUP) { + b_scale = scale_b[n]; + dq_val = dq_val * b_scale; + } + if constexpr (accum) { + output[m * ldo + n] += dq_val; + } else { + output[m * ldo + n] = dq_val; + } + } + } +} + +// Store result to output buffer with dtype conversion +// If act/wei are per_row or per_tensor quantized, apply scales +// If bias is not null, add bias +template +inline void store_out( + const float* y_buf, + out_dtype* c_ptr, + int64_t M, + int64_t lda, + const float* scales_a, + const float* scales_b, + const float* bias) { + float a_scale = 1.0, b_scale = 1.0; + __m512 va_scale, vb_scale; + if constexpr (act_quant_mode == PER_TENSOR) { + a_scale = *scales_a; + } + if constexpr (wei_quant_mode == PER_TENSOR) { + b_scale = *scales_b; + vb_scale = _mm512_set1_ps(b_scale); + } + for (int i = 0; i < M; ++i) { + if constexpr (act_quant_mode == PER_ROW) { + a_scale = *(scales_a + i); + } + if constexpr (act_quant_mode != PER_GROUP) { + va_scale = _mm512_set1_ps(a_scale); + } + constexpr int N_UNROLL = N / 16; + c10::ForcedUnroll{}([&](auto idx) { + constexpr int j = idx * 16; + __m512 y_vec = _mm512_loadu_ps(y_buf + i * N + j); + __m512 bias_vec = bias ? _mm512_loadu_ps(bias + j) : _mm512_setzero_ps(); + if constexpr (act_quant_mode != PER_GROUP) { + y_vec = _mm512_mul_ps(y_vec, va_scale); + } + if constexpr (wei_quant_mode == PER_ROW) { + vb_scale = _mm512_loadu_ps(scales_b + j); + } + if constexpr (wei_quant_mode != PER_GROUP) { + y_vec = _mm512_mul_ps(y_vec, vb_scale); + } + y_vec = _mm512_add_ps(y_vec, bias_vec); + if constexpr (std::is_same::value) { + _mm512_storeu_ps(c_ptr + i * lda + j, y_vec); + } else if constexpr (std::is_same::value) { + __m256i y_bf16_vec = at::vec::cvtfp32_bf16(y_vec); + _mm256_storeu_si256(reinterpret_cast<__m256i*>(c_ptr + i * lda + j), y_bf16_vec); + } else if constexpr (std::is_same::value) { + __m256i y_fp16_vec = at::vec::cvtfp32_fp16(y_vec); + _mm256_storeu_si256(reinterpret_cast<__m256i*>(c_ptr + i * lda + j), y_fp16_vec); + } else { + TORCH_CHECK(false, "Unsupported output dtype"); + } + }); + constexpr int tail_start = N / 16 * 16; + for (int j = tail_start; j < N; ++j) { + if constexpr (wei_quant_mode == PER_ROW) { + b_scale = scales_b[j]; + } + c_ptr[i * lda + j] = static_cast(y_buf[i * N + j] * a_scale * b_scale); + } + } // for M +} + +#else // no AVX512 + +static void cvt_f8e4m3_to_bf16( + const at::Float8_e4m3fn* __restrict__ in, at::BFloat16* out, int64_t rows, int64_t cols, int64_t stride) { + for (int r = 0; r < rows; ++r) { + for (int c = 0; c < cols; ++c) { + out[r * cols + c] = (at::BFloat16)in[r * stride + c]; + } + } +} + +// Store result to output buffer with dtype conversion +// If act/wei are per_row or per_tensor quantized, apply scales +// If bias is not null, add bias +template +inline void store_out( + const float* y_buf, + out_dtype* c_ptr, + int64_t M, + int64_t lda, + const float* scales_a, + const float* scales_b, + const float* bias) { + float a_scale = 1.0, b_scale = 1.0; + if constexpr (act_quant_mode == PER_TENSOR) { + a_scale = *scales_a; + } + if constexpr (wei_quant_mode == PER_TENSOR) { + b_scale = *scales_b; + } + for (int i = 0; i < M; ++i) { + if constexpr (act_quant_mode == PER_ROW) { + a_scale = *(scales_a + i); + } + for (int j = 0; j < N; ++j) { + if constexpr (wei_quant_mode == PER_ROW) { + b_scale = scales_b[j]; + } + c_ptr[i * lda + j] = static_cast(y_buf[i * N + j] * a_scale * b_scale); + } + } // for M +} + +#endif // CPU_CAPABILITY_AVX512 + +template +void _micro_gemm( + float* C, + const at::Float8_e4m3fn* A, + const float* scales_a, + const at::Float8_e4m3fn* B, + const float* scales_b, + int64_t M, + int64_t K, + int64_t lda, + int64_t ldc, + int64_t ldsa, + float* ukernel_buf, + at::BFloat16* dqA_buf, + at::BFloat16* dqB_buf) { + // If FP8 brgemm is not available, convert A/B to bf16 for computation + // Compute GEMM fp8 * fp8 -> fp32 (or bf16 * bf16 -> fp32) + // If per_group quant, apply scales. Otherwise, don't apply scales here + // Finally accumulate and store results +#if defined(CPU_CAPABILITY_AVX512) + if constexpr (cpublas_can_pack) { +#ifdef CPUBLAS_BRGEMM_F8F8F32 + at::native::cpublas::brgemm( + M, N, K, lda /*lda*/, N /*ldb*/, N /*ldc*/, false /* add_C */, A, B, ukernel_buf, true /* is_vnni */); +#else + cvt_f8e4m3_to_bf16(A, dqA_buf, M, K, lda); + cvt_f8e4m3_to_bf16(B, dqB_buf, K, N, N); + at::native::cpublas::brgemm( + M, N, K, K /*lda*/, N /*ldb*/, N /*ldc*/, false /* add_C */, dqA_buf, dqB_buf, ukernel_buf, true /* is_vnni */); +#endif + _mm_prefetch(B + N * (K + 128), _MM_HINT_T0); + _mm_prefetch(A + K + 128, _MM_HINT_T0); + _accumulate_result( + C, ukernel_buf, scales_a, scales_b, M, N /*ldi*/, ldc, ldsa); + } else +#endif + { + for (int64_t i = 0; i < M; ++i) { + for (int64_t j = 0; j < N; ++j) { + float sum = 0; + for (int64_t k = 0; k < K; ++k) { + sum += ((float)A[i * lda + k] * (float)B[k * N + j]); + } + if constexpr (act_quant_mode == PER_GROUP) { + sum *= scales_a[i * ldsa]; + } + if constexpr (wei_quant_mode == PER_GROUP) { + sum *= scales_b[j]; + } + C[i * ldc + j] += sum; + } + } + } +} + +template +void _float8_linear_impl( + const at::Tensor& input, + const at::Tensor& input_scales, + const at::Tensor& weight, + const at::Tensor& weight_scales, + const std::optional& bias, + at::Tensor& output) { + // input shape = [..., K] + // input is per token quantized + int64_t K = input.size(-1); + auto input_view = input.view({-1, K}); + int64_t M = input_view.size(0); + + // weight shape = [Nc, Kc, block_k, block_n] + // scales shape = [Nc, G, block_n] + int64_t Nc = weight.size(0); + int64_t Kc = wei_quant_mode != PER_GROUP ? 1 : weight.size(1); + int64_t block_k = wei_quant_mode != PER_GROUP ? weight.size(1) * weight.size(2) : weight.size(2); + constexpr int64_t block_n = BLOCK_N; + TORCH_CHECK(weight.size(3) == block_n, "Float8 linear: unexpected weight shape"); + int64_t N = Nc * block_n; + TORCH_CHECK(K == Kc * block_k, "Float8 linear: weight and input shapes mismatch"); + auto [parallel_on_M, block_m, Mc, Mc_parallel] = get_m_blocking(M); + + // scales shape = [Nc, G, block_n] + int64_t num_groups = wei_quant_mode == PER_TENSOR ? 1 : weight_scales.size(1); + TORCH_CHECK(K % num_groups == 0, "K should be divisible by num_groups"); + int64_t group_size = K / num_groups; + TORCH_CHECK(group_size % block_k == 0, "Float8 linear: group_size should be divisible by block_k"); + int64_t block_per_group = group_size / block_k; + TORCH_CHECK( + input_scales.numel() == 1 || input_scales.numel() == M || input_scales.numel() == M * num_groups, + "Float8 linear: unexpected input scales shape"); + auto ldsa = act_quant_mode == PER_TENSOR ? 0 : act_quant_mode == PER_ROW ? 1 : num_groups; + + const at::Float8_e4m3fn* a_ptr = input_view.data_ptr(); + const float* a_scales_ptr = input_scales.data_ptr(); + const at::Float8_e4m3fn* b_ptr = weight.data_ptr(); + const float* b_scales_ptr = weight_scales.data_ptr(); + out_dtype* c_ptr = output.data_ptr(); + const float* bias_ptr = bias.has_value() ? bias.value().data_ptr() : nullptr; + + int64_t block_size = block_m * block_n; + int64_t num_thread = at::get_num_threads(); + at::Tensor y_buffer = at::empty({num_thread, block_size}, output.options().dtype(at::kFloat)); + // Create buffer for brgemm output and dqA/dqB (optional) +#if defined(CPU_CAPABILITY_AVX512) + // buffer for brgemm output in float32 + int64_t buffer_size = block_size * 2; // float32 = bfloat16 * 2 +#ifndef CPUBLAS_BRGEMM_F8F8F32 + // buffers for dqA & dqB in bf16 + buffer_size += (block_k * block_n + block_m * block_k); +#endif + at::Tensor micro_gemm_buffer = at::empty({num_thread, buffer_size}, output.options().dtype(at::kBFloat16)); +#endif + + parallel_2d(Mc, Nc, [&](int64_t mc0, int64_t mc1, int64_t nc0, int64_t nc1) { + int tid = get_thread_num(); + float* y_buf = y_buffer.data_ptr() + tid * block_size; + at::BFloat16 *dqA_buffer = nullptr, *dqB_buffer = nullptr; + float* ukernel_buf = nullptr; +#if defined(CPU_CAPABILITY_AVX512) + at::BFloat16* micro_gemm_buf = micro_gemm_buffer.data_ptr() + tid * buffer_size; + ukernel_buf = reinterpret_cast(micro_gemm_buf); +#ifndef CPUBLAS_BRGEMM_F8F8F32 + dqA_buffer = micro_gemm_buf; + dqB_buffer = micro_gemm_buf + block_m * block_k; + ukernel_buf = reinterpret_cast(micro_gemm_buf + block_m * block_k + block_k * block_n); +#endif +#endif + + loop_2d(mc0, mc1, nc0, nc1, block_n * K, [&](int64_t mci, int64_t nc, int64_t) { + int64_t m_size = mci * block_m + block_m > M ? M - mci * block_m : block_m; + zero_buffer(y_buf, m_size * block_n); + for (int kci = 0; kci < Kc; ++kci) { + auto scales_a = a_scales_ptr + mci * block_m * num_groups + kci / block_per_group; + auto scales_b = b_scales_ptr + nc * block_n * num_groups + kci / block_per_group * block_n; + _micro_gemm( + /* C */ y_buf, + /* A */ a_ptr + mci * block_m * K + kci * block_k, + /* scales_a */ scales_a, + /* B */ b_ptr + (nc * Kc + kci) * block_n * block_k, + /* scales_b */ scales_b, + /* M */ m_size, + /* K */ block_k, + /* lda */ K, + /* ldc */ block_n, + /* ldsa */ ldsa, + /* ukernel_buf */ ukernel_buf, + /* dqA_buf */ dqA_buffer, + /* dqB_buf */ dqB_buffer); + } + auto scales_a = act_quant_mode == PER_TENSOR ? a_scales_ptr + : act_quant_mode == PER_ROW ? a_scales_ptr + mci * block_m + : nullptr; + auto scales_b = wei_quant_mode == PER_TENSOR ? b_scales_ptr + : wei_quant_mode == PER_ROW ? b_scales_ptr + nc * block_n + : nullptr; + auto bias_data = bias_ptr ? bias_ptr + nc * block_n : nullptr; + store_out( + y_buf, c_ptr + mci * block_m * N + nc * block_n, m_size, N /*lda*/, scales_a, scales_b, bias_data); + }); + + if constexpr (cpublas_can_pack) { + at::native::cpublas::brgemm_release(); + } + }); +} + +} // anonymous namespace + +void tinygemm_kernel( + float* C, + const at::Float8_e4m3fn* A, + const float* scales_a, + const at::Float8_e4m3fn* B, + const float* scales_b, + int64_t M, + int64_t K, + int64_t lda, + int64_t ldc, + int64_t ldsa, + float* ukernel_buf, + at::BFloat16* dqA_buf, + at::BFloat16* dqB_buf) { + // act_quant_mode = per row (2), wei_quant_mode = per group (3) + if (cpublas_could_pack()) { + _micro_gemm(C, A, scales_a, B, scales_b, M, K, lda, ldc, ldsa, ukernel_buf, dqA_buf, dqB_buf); + } else { + _micro_gemm(C, A, scales_a, B, scales_b, M, K, lda, ldc, ldsa, ukernel_buf, dqA_buf, dqB_buf); + } +} + +#define INSTANTIATE_TINYGEMM_TEMPLATE() \ + void tinygemm_kernel( \ + float* C, \ + const at::Float8_e4m3fn* A, \ + const float* scales_a, \ + const at::Float8_e4m3fn* B, \ + const float* scales_b, \ + int64_t M, \ + int64_t K, \ + int64_t lda, \ + int64_t ldc, \ + int64_t ldsa, \ + float* ukernel_buf, \ + at::BFloat16* dqA_buf, \ + at::BFloat16* dqB_buf) + +INSTANTIATE_TINYGEMM_TEMPLATE(); +/* +return: packed_weight, packed_scales +*/ +std::tuple float8_linear_prepack_impl(const at::Tensor& weight, const at::Tensor& scales) { + // weight shape = [N, K] + // scales shape = [N, G] + TORCH_CHECK(weight.dim() == 2, "Float8 linear CPU: Weight should be a 2D tensor for packing"); + TORCH_CHECK(weight.size(1) % 2 == 0, "Float8 linear CPU: Weight should have even number of columns for packing"); + + auto new_scales = scales; + if (new_scales.dim() == 1) { + new_scales.unsqueeze_(1); + } + new_scales = new_scales.to(at::kFloat); + int N = weight.size(0); + int K = weight.size(1); + int G = scales.size(1); + int group_size = K / G; + int block_k = group_size > 128 ? 128 : group_size; + while (K % block_k != 0) { + block_k /= 2; + } + TORCH_CHECK( + block_k > 0 && block_k <= group_size, "Float8 linear CPU: Invalid block_k size, should be in (0, group_size]"); + constexpr int block_n = BLOCK_N; + int Nc = N / block_n; + int Kc = K / block_k; + + // Reorder weight to [N/block_n, K/block_k, block_k, block_n] + // Reorder scales to [N/block_n, G, block_n] + auto weight_view = weight.view({Nc, block_n, Kc, block_k}); + at::Tensor weight_reordered = weight_view.permute({0, 2, 3, 1}).contiguous(); + at::Tensor blocked_weight; + at::Tensor blocked_scales = new_scales.view({Nc, block_n, G}).permute({0, 2, 1}).contiguous(); + +#if defined(CPU_CAPABILITY_AVX512) + if (cpublas_could_pack()) { +#ifdef CPUBLAS_BRGEMM_F8F8F32 + constexpr int vnni_size = 4; // for fp8 +#else + constexpr int vnni_size = 2; // for float16 +#endif + blocked_weight = at::empty({Nc, Kc, block_k, block_n}, weight.options()); + auto weight_ptr = reinterpret_cast(weight_reordered.data_ptr()); + auto blocked_weight_ptr = reinterpret_cast(blocked_weight.data_ptr()); + int64_t num_blocks = Nc * Kc; + at::parallel_for(0, num_blocks, 1, [&](int64_t begin, int64_t end) { + for (const auto i : c10::irange(begin, end)) { + auto in_ptr = weight_ptr + i * block_k * block_n; + auto out_ptr = blocked_weight_ptr + i * block_k * block_n; + + // Reorder weight block to VNNI + // plain shape = [block_k, block_n] + // packed shape = [block_k / VNNI_SIZE, block_n, VNNI_SIZE] viewed as [block_k, block_n] + constexpr int n_group_size = 8; + constexpr int n_group = block_n / n_group_size; // 4 + for (int nb = 0; nb < n_group; ++nb) { + for (int k = 0; k < block_k; k += vnni_size) { + for (int ni = 0; ni < n_group_size; ++ni) { + for (int ki = 0; ki < vnni_size; ++ki) { + int src_idx = nb * n_group_size + ni + (k + ki) * block_n; + int dst_idx = (nb * n_group_size + ni) * vnni_size + k * block_n + ki; + *(out_ptr + dst_idx) = *(in_ptr + src_idx); + } + } + } + } + } + }); + } else +#endif + { + blocked_weight = weight_reordered; + } + + return std::make_tuple(std::move(blocked_weight), std::move(blocked_scales)); +} + +// AVX512 optimized channel-wise max computation +inline void +compute_channel_max_avx512(const float* data, float* max_vals, int64_t num_channels, int64_t elements_per_channel) { + at::parallel_for(0, num_channels, 1, [&](int64_t start, int64_t end) { + for (int64_t c = start; c < end; ++c) { + const float* channel_data = data + c * elements_per_channel; + float max_val = 0.0f; + + int64_t i = 0; + // Process 16 elements at a time + __m512 max_vec = _mm512_setzero_ps(); + + for (; i <= elements_per_channel - 16; i += 16) { + __m512 data_vec = _mm512_loadu_ps(&channel_data[i]); + __m512 abs_vec = _mm512_abs_ps(data_vec); + max_vec = _mm512_max_ps(max_vec, abs_vec); + } + + // Horizontal max of the vector + float hmax = _mm512_reduce_max_ps(max_vec); + max_val = std::max(max_val, hmax); + + // Handle remaining elements + for (; i < elements_per_channel; ++i) { + max_val = std::max(max_val, std::abs(channel_data[i])); + } + + max_vals[c] = max_val; + } + }); +} + +// AVX512 optimized scaling and clamping for channel-wise quantization +inline void scale_clamp_channelwise_avx512( + const float* src, + float* dst, + const float* scales, + int64_t num_channels, + int64_t elements_per_channel, + float quant_max, + float neg_quant_max) { + at::parallel_for(0, num_channels, 1, [&](int64_t start, int64_t end) { + for (int64_t c = start; c < end; ++c) { + float scale_val = scales[c]; + float scale_reciprocal = 1.0f / scale_val; + + const float* channel_src = src + c * elements_per_channel; + float* channel_dst = dst + c * elements_per_channel; + + int64_t i = 0; + const __m512 scale_recip_vec = _mm512_set1_ps(scale_reciprocal); + const __m512 quant_max_vec = _mm512_set1_ps(quant_max); + const __m512 neg_quant_max_vec = _mm512_set1_ps(neg_quant_max); + + for (; i <= elements_per_channel - 16; i += 16) { + __m512 src_vec = _mm512_loadu_ps(&channel_src[i]); + __m512 scaled_vec = _mm512_mul_ps(src_vec, scale_recip_vec); + __m512 clamped_vec = _mm512_min_ps(_mm512_max_ps(scaled_vec, neg_quant_max_vec), quant_max_vec); + _mm512_storeu_ps(&channel_dst[i], clamped_vec); + } + + // Handle remaining elements + for (; i < elements_per_channel; ++i) { + float scaled = channel_src[i] * scale_reciprocal; + channel_dst[i] = std::clamp(scaled, neg_quant_max, quant_max); + } + } + }); +} + +// AVX512 optimized scaling and clamping for global quantization +inline void scale_clamp_global_avx512( + const float* src, float* dst, int64_t size, float scale_reciprocal, float quant_max, float neg_quant_max) { + if (size <= 0) return; + + const __m512 scale_recip_vec = _mm512_set1_ps(scale_reciprocal); + const __m512 quant_max_vec = _mm512_set1_ps(quant_max); + const __m512 neg_quant_max_vec = _mm512_set1_ps(neg_quant_max); + + at::parallel_for(0, size, 4096, [&](int64_t start, int64_t end) { + int64_t block_size = end - start; + const float* block_src = src + start; + float* block_dst = dst + start; + + int64_t i = 0; + for (; i <= block_size - 16; i += 16) { + __m512 src_vec = _mm512_loadu_ps(&block_src[i]); + __m512 scaled_vec = _mm512_mul_ps(src_vec, scale_recip_vec); + __m512 clamped_vec = _mm512_min_ps(_mm512_max_ps(scaled_vec, neg_quant_max_vec), quant_max_vec); + _mm512_storeu_ps(&block_dst[i], clamped_vec); + } + + // Handle remaining elements in block + for (; i < block_size; ++i) { + float scaled = block_src[i] * scale_reciprocal; + block_dst[i] = std::clamp(scaled, neg_quant_max, quant_max); + } + }); +} + +std::tuple +_quantize_fp8e4m3(const at::Tensor& t, bool channelwise, c10::optional scale_opt = c10::nullopt) { + constexpr float quant_max = 448.0f; // torch.finfo(torch.float8_e4m3fn).max + constexpr float eps = std::numeric_limits::epsilon(); + + // Ensure input is contiguous and in float32 + auto t_float = t.to(at::ScalarType::Float).contiguous(); + at::Tensor qt; + at::Tensor scale_tensor; + + if (channelwise) { + // Channel-wise quantization with AVX512 optimization + int64_t num_channels = t_float.size(0); + int64_t elements_per_channel = t_float.numel() / num_channels; + + if (elements_per_channel * num_channels != t_float.numel()) { + throw std::runtime_error("Tensor must be divisible by number of channels for channel-wise quantization"); + } + + // Allocate scale tensor + scale_tensor = at::empty({num_channels}, t_float.options()); + float* scale_data = scale_tensor.data_ptr(); + + // Compute channel-wise max using AVX512 + compute_channel_max_avx512(t_float.data_ptr(), scale_data, num_channels, elements_per_channel); + + // Apply quant_max and EPS + at::parallel_for(0, num_channels, 1, [&](int64_t start, int64_t end) { + for (int64_t c = start; c < end; ++c) { + scale_data[c] = std::max(scale_data[c] / quant_max, eps); + } + }); + + // Create output tensor for quantized values + qt = at::empty_like(t_float); + + // Scale and clamp using AVX512 + scale_clamp_channelwise_avx512( + t_float.data_ptr(), + qt.data_ptr(), + scale_data, + num_channels, + elements_per_channel, + quant_max, + -quant_max); + } else { + // Global quantization with AVX512 optimization + if (!scale_opt.has_value()) { + throw std::runtime_error("Scale must be provided for non-channelwise quantization in AVX512 version"); + } + + scale_tensor = scale_opt.value().to(at::ScalarType::Float).contiguous(); + + // Handle scalar scale case + float scale_val; + if (scale_tensor.numel() == 1) { + scale_val = std::max(scale_tensor.item(), eps); + } else { + // Compute max of scale tensor if it's not scalar + float* scale_data = scale_tensor.data_ptr(); + __m512 max_vec = _mm512_set1_ps(eps); + + int64_t i = 0; + for (; i <= scale_tensor.numel() - 16; i += 16) { + __m512 scale_vec = _mm512_loadu_ps(&scale_data[i]); + max_vec = _mm512_max_ps(max_vec, scale_vec); + } + + float hmax = _mm512_reduce_max_ps(max_vec); + for (; i < scale_tensor.numel(); ++i) { + hmax = std::max(hmax, scale_data[i]); + } + scale_val = std::max(hmax, eps); + } + + scale_tensor = at::tensor({scale_val}, scale_tensor.options()); + float scale_reciprocal = 1.0f / scale_val; + + // Create output tensor + qt = at::empty_like(t_float); + + // Scale and clamp using AVX512 + scale_clamp_global_avx512( + t_float.data_ptr(), qt.data_ptr(), t_float.numel(), scale_reciprocal, quant_max, -quant_max); + } + + // Final conversion to FP8 E4M3FN using ATen's native conversion + qt = at::_to_copy(qt, at::ScalarType::Float8_e4m3fn); + + return std::make_tuple(qt, scale_tensor); +} + +inline __m128i cvtfp32_fp8e4m3(__m512& src) { + // cvt 16x32 from fp32 to fp8 e4m3 + const __m512i sign_mask = _mm512_set1_epi32(0x80000000); + const __m512i fp8_max = _mm512_set1_epi32(UINT32_C(1087) << 20); + const __m512i denorm_thresh = _mm512_set1_epi32(UINT32_C(121) << 23); + const __m512i denorm_mask = _mm512_set1_epi32(UINT32_C(141) << 23); + const __m512i bias_part1 = _mm512_set1_epi32((uint32_t)(7 - 127) << 23); + const __m512i rounding_bias = _mm512_set1_epi32(0x7FFFF); + __m512i f_bits = _mm512_castps_si512(src); + // Extract and save sign + __m512i sign = _mm512_and_epi32(f_bits, sign_mask); + f_bits = _mm512_xor_epi32(f_bits, sign); + + // Prepare result containers + __m512i result = _mm512_setzero_si512(); + + // Step 1: Handle case of overflow + // (f_bits >= fp8_max): set result = 0x7f + __mmask16 overflow_mask = _mm512_cmpge_epu32_mask(f_bits, fp8_max); + if (overflow_mask) { + result = _mm512_mask_set1_epi32(result, overflow_mask, 0x7f); + } + + // Step 2: Handle small numbers (denormals) + // Small numbers (f_bits < denorm_thresh) + __mmask16 denorm_thresh_mask = _mm512_cmplt_epu32_mask(f_bits, denorm_thresh); + + if (denorm_thresh_mask) { + __m512 small_input = _mm512_castsi512_ps(f_bits); + __m512 small_denorm = _mm512_add_ps(small_input, _mm512_castsi512_ps(denorm_mask)); + __m512i small_denorm_bits = _mm512_castps_si512(small_denorm); + __m512i small_result = _mm512_sub_epi32(small_denorm_bits, denorm_mask); + result = _mm512_mask_mov_epi32(result, denorm_thresh_mask, small_result); + } + + // Step 3: Handle normal numbers + __mmask16 normal_mask = ~(overflow_mask | denorm_thresh_mask); + + if (normal_mask) { + // mant_odd = (f_bits >> 20) & 1 + __m512i mant_odd = _mm512_and_epi32(_mm512_srli_epi32(f_bits, 20), _mm512_set1_epi32(1)); + // f_bits += bias_part1 + rounding_bias + __m512i rounded = _mm512_add_epi32(f_bits, bias_part1); + rounded = _mm512_add_epi32(rounded, rounding_bias); + // Add mant_odd + rounded = _mm512_add_epi32(rounded, mant_odd); + // Shift right by 20 bits + __m512i normal_result = _mm512_srli_epi32(rounded, 20); + result = _mm512_mask_mov_epi32(result, normal_mask, normal_result); + } + + // Merge back the sign + __m512i sign_shifted = _mm512_srli_epi32(sign, 24); + result = _mm512_or_epi32(result, sign_shifted); + + // Now result is 16 x 32-bit integers, but we only need 8-bit for each + __m512i packed = _mm512_and_si512(result, _mm512_set1_epi32(0xFF)); + + // Narrow 32-bit integers to 8-bit + return _mm512_cvtepi32_epi8(packed); +} + +__attribute__((target("avx10.2"))) inline __m128i cvtfp32_fp8e4m3_avx10_2(__m512& src) { + __m256i f16_vec = _mm512_cvt_roundps_ph(src, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + return _mm256_cvtph_hf8(_mm256_castsi256_ph(f16_vec)); +} + +// Carries the whole loop rather than just the conversion, as GCC cannot inline an avx10.2 +// function into a caller lacking that target, so branching per vector is costly. +__attribute__((target("avx10.2"))) void quantize_bf16_block_avx10_2( + const at::BFloat16* __restrict__ src, at::Float8_e4m3fn* __restrict__ dst, int64_t count, float scale_reciprocal) { + constexpr float quant_max = 448.0f; // torch.finfo(torch.float8_e4m3fn).max + const __m512 scale_recip_vec = _mm512_set1_ps(scale_reciprocal); + const __m512 quant_max_vec = _mm512_set1_ps(quant_max); + const __m512 neg_quant_max_vec = _mm512_set1_ps(-quant_max); + + for (int64_t i = 0; i <= count - 32; i += 32) { + __m256i src_vec1 = _mm256_loadu_si256(reinterpret_cast(&src[i])); + __m256i src_vec2 = _mm256_loadu_si256(reinterpret_cast(&src[i + 16])); + + __m512 fp32_vec1 = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(src_vec1), 16)); + __m512 fp32_vec2 = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(src_vec2), 16)); + + __m512 scaled_vec1 = _mm512_mul_ps(fp32_vec1, scale_recip_vec); + __m512 clamped_vec1 = _mm512_min_ps(_mm512_max_ps(scaled_vec1, neg_quant_max_vec), quant_max_vec); + + __m512 scaled_vec2 = _mm512_mul_ps(fp32_vec2, scale_recip_vec); + __m512 clamped_vec2 = _mm512_min_ps(_mm512_max_ps(scaled_vec2, neg_quant_max_vec), quant_max_vec); + + __m128i fp8_vec1 = cvtfp32_fp8e4m3_avx10_2(clamped_vec1); + __m128i fp8_vec2 = cvtfp32_fp8e4m3_avx10_2(clamped_vec2); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i), fp8_vec1); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + i + 16), fp8_vec2); + } +} + +std::tuple _quantize_fp8e4m3_bf16_per_tensor_no_scale(const at::Tensor& t) { + constexpr float quant_max = 448.0f; // torch.finfo(torch.float8_e4m3fn).max + constexpr float eps = std::numeric_limits::epsilon(); + + // Input validation and preparation + assert(t.scalar_type() == at::ScalarType::BFloat16); + auto t_bf16 = t.contiguous(); + int64_t num_channels = t_bf16.size(0); + int64_t elements_per_channel = t_bf16.numel() / num_channels; + assert(elements_per_channel % 32 == 0); // do not consider tile currently + at::Tensor quant_t = at::empty(t_bf16.sizes(), t_bf16.options().dtype(at::kFloat8_e4m3fn)); + + // Allocate output tensors + at::Tensor scale_tensor = at::empty({num_channels}, t_bf16.options().dtype(at::ScalarType::Float)); + float* scale_data = scale_tensor.data_ptr(); + + // Unified processing: compute max, apply scale, and quantize in single pass + at::parallel_for(0, num_channels, 1, [&](int64_t start, int64_t end) { + for (int64_t c = start; c < end; ++c) { + const at::BFloat16* channel_src = t_bf16.data_ptr() + c * elements_per_channel; + at::Float8_e4m3fn* quant_dst = quant_t.data_ptr() + c * elements_per_channel; + + // Step 1: Compute channel-wise max using AVX512 + float channel_max = 0.0f; + int64_t i = 0; + + // Process 32 elements at a time for max computation + for (; i <= elements_per_channel - 32; i += 32) { + // Load 32 BF16 values (2x 256-bit vectors) + __m256i src_vec1 = _mm256_loadu_si256(reinterpret_cast(&channel_src[i])); + __m256i src_vec2 = _mm256_loadu_si256(reinterpret_cast(&channel_src[i + 16])); + + // Convert BF16 to FP32 + __m512i fp32_int1 = _mm512_cvtepu16_epi32(src_vec1); + __m512 fp32_vec1 = _mm512_castsi512_ps(_mm512_slli_epi32(fp32_int1, 16)); + + __m512i fp32_int2 = _mm512_cvtepu16_epi32(src_vec2); + __m512 fp32_vec2 = _mm512_castsi512_ps(_mm512_slli_epi32(fp32_int2, 16)); + + // Compute absolute values + __m512 abs_vec1 = _mm512_abs_ps(fp32_vec1); + __m512 abs_vec2 = _mm512_abs_ps(fp32_vec2); + + // Find max in each vector + float max1 = _mm512_reduce_max_ps(abs_vec1); + float max2 = _mm512_reduce_max_ps(abs_vec2); + channel_max = std::max(channel_max, std::max(max1, max2)); + } + + // Step 2: Apply quant_max and EPS to compute scale + float scale_val = std::max(channel_max / quant_max, eps); + float scale_reciprocal = 1.0f / scale_val; + scale_data[c] = scale_val; + + // Step 3: Scale and clamp using AVX512 (reuse the same loop structure) + if (avx10_2_available()) { + quantize_bf16_block_avx10_2(channel_src, quant_dst, elements_per_channel, scale_reciprocal); + continue; + } + i = 0; + const __m512 scale_recip_vec = _mm512_set1_ps(scale_reciprocal); + const __m512 quant_max_vec = _mm512_set1_ps(quant_max); + const __m512 neg_quant_max_vec = _mm512_set1_ps(-quant_max); + + for (; i <= elements_per_channel - 32; i += 32) { + // Load 32 BF16 values + __m256i src_vec1 = _mm256_loadu_si256(reinterpret_cast(&channel_src[i])); + __m256i src_vec2 = _mm256_loadu_si256(reinterpret_cast(&channel_src[i + 16])); + + // Convert BF16 to FP32 + __m512 fp32_vec1 = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(src_vec1), 16)); + __m512 fp32_vec2 = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(src_vec2), 16)); + + // Scale and clamp + __m512 scaled_vec1 = _mm512_mul_ps(fp32_vec1, scale_recip_vec); + __m512 clamped_vec1 = _mm512_min_ps(_mm512_max_ps(scaled_vec1, neg_quant_max_vec), quant_max_vec); + + __m512 scaled_vec2 = _mm512_mul_ps(fp32_vec2, scale_recip_vec); + __m512 clamped_vec2 = _mm512_min_ps(_mm512_max_ps(scaled_vec2, neg_quant_max_vec), quant_max_vec); + + __m128i fp8_vec1 = cvtfp32_fp8e4m3(clamped_vec1); + __m128i fp8_vec2 = cvtfp32_fp8e4m3(clamped_vec2); + _mm_storeu_si128(reinterpret_cast<__m128i*>(quant_dst + i), fp8_vec1); + _mm_storeu_si128(reinterpret_cast<__m128i*>(quant_dst + i + 16), fp8_vec2); + } + } + }); + + return std::make_tuple(quant_t, scale_tensor); +} + +std::tuple +_quantize_fp8e4m3_bf16_per_tensor_with_scale(const at::Tensor& t, at::Tensor& scale_tensor) { + constexpr float quant_max = 448.0f; // torch.finfo(torch.float8_e4m3fn).max + constexpr float eps = std::numeric_limits::epsilon(); + + // Input validation + assert(t.scalar_type() == at::ScalarType::BFloat16); + + auto t_bf16 = t.contiguous(); + auto scale_tensor_contig = scale_tensor.contiguous().to(at::ScalarType::Float); + + // Get scale value (handle scalar or tensor) + float scale_val; + if (scale_tensor_contig.numel() == 1) { + scale_val = scale_tensor_contig.item(); + } else { + // Take max if scale is a tensor (though should be scalar for global quantization) + scale_val = scale_tensor_contig.max().item(); + } + + // Apply EPS to scale + scale_val = std::max(scale_val, eps); + float scale_reciprocal = 1.0f / scale_val; + + at::Tensor scale_output = at::tensor({scale_val}, scale_tensor_contig.options()); + + // Apply scale and clamp using AVX512 + const at::BFloat16* src_data = t_bf16.data_ptr(); + at::Tensor quant_t = at::empty(t_bf16.sizes(), t_bf16.options().dtype(at::kFloat8_e4m3fn)); + int64_t total_elements = t_bf16.numel(); + const __m512 scale_recip_vec = _mm512_set1_ps(scale_reciprocal); + const __m512 quant_max_vec = _mm512_set1_ps(quant_max); + const __m512 neg_quant_max_vec = _mm512_set1_ps(-quant_max); + int64_t num_channels = t_bf16.size(0); + int64_t elements_per_channel = total_elements / num_channels; + assert(elements_per_channel % 32 == 0); // do not consider tile currently + // Process in parallel blocks + at::parallel_for(0, num_channels, 1, [&](int64_t start, int64_t end) { + for (int64_t c = start; c < end; ++c) { + const at::BFloat16* src_data = t_bf16.data_ptr() + c * elements_per_channel; + at::Float8_e4m3fn* quant_t_data = quant_t.data_ptr() + c * elements_per_channel; + + if (avx10_2_available()) { + quantize_bf16_block_avx10_2(src_data, quant_t_data, elements_per_channel, scale_reciprocal); + continue; + } + + int64_t i = 0; + // Process 32 elements at a time using AVX512 + for (; i <= elements_per_channel - 32; i += 32) { + // Load 32 BF16 values (2x 256-bit vectors) + __m256i src_vec1 = _mm256_loadu_si256(reinterpret_cast(&src_data[i])); + __m256i src_vec2 = _mm256_loadu_si256(reinterpret_cast(&src_data[i + 16])); + + // Convert BF16 to FP32 + __m512 fp32_vec1 = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(src_vec1), 16)); + __m512 fp32_vec2 = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(src_vec2), 16)); + + // Scale and clamp + __m512 scaled_vec1 = _mm512_mul_ps(fp32_vec1, scale_recip_vec); + __m512 clamped_vec1 = _mm512_min_ps(_mm512_max_ps(scaled_vec1, neg_quant_max_vec), quant_max_vec); + + __m512 scaled_vec2 = _mm512_mul_ps(fp32_vec2, scale_recip_vec); + __m512 clamped_vec2 = _mm512_min_ps(_mm512_max_ps(scaled_vec2, neg_quant_max_vec), quant_max_vec); + + __m128i fp8_vec1 = cvtfp32_fp8e4m3(clamped_vec1); + __m128i fp8_vec2 = cvtfp32_fp8e4m3(clamped_vec2); + _mm_storeu_si128(reinterpret_cast<__m128i*>(quant_t_data + i), fp8_vec1); + _mm_storeu_si128(reinterpret_cast<__m128i*>(quant_t_data + i + 16), fp8_vec2); + } + } + }); + + return std::make_tuple(quant_t, scale_output); +} + +std::tuple +_quantize_fp8e4m3_vec(const at::Tensor& t, bool channelwise, c10::optional scale_opt) { + if (channelwise) { + return _quantize_fp8e4m3_bf16_per_tensor_no_scale(t); + } else { + assert(scale_opt.has_value()); + return _quantize_fp8e4m3_bf16_per_tensor_with_scale(t, scale_opt.value()); + } +} + +// Public wrapper for torch op registration +std::tuple +quantize_fp8e4m3_vec(const at::Tensor& t, bool channelwise, c10::optional scale_opt) { + return _quantize_fp8e4m3_vec(t, channelwise, scale_opt); +} + +at::Tensor fp8_scaled_mm_with_quant( + const at::Tensor& act, + const std::optional& act_scales, + bool channelwise, + const at::Tensor& weight, + const at::Tensor& weight_scales, + const std::optional& bias, + at::ScalarType output_dtype) { + // TODO: refine here to use tensor ptr and in below dispatch api. + std::tuple quant_act = act.scalar_type() == at::ScalarType::BFloat16 + ? _quantize_fp8e4m3_vec(act, channelwise, act_scales) + : _quantize_fp8e4m3(act, channelwise, act_scales); + auto input = std::get<0>(quant_act); + auto input_scales = std::get<1>(quant_act); + int64_t N = weight.dim() == 4 ? weight.size(0) * weight.size(-1) : weight.size(0); + int act_quant_mode = input_scales.numel() == 1 ? PER_TENSOR + : input_scales.numel() == input.numel() / input.size(-1) ? PER_ROW + : PER_GROUP; + int wei_quant_mode = weight_scales.numel() == 1 ? PER_TENSOR : weight_scales.numel() == N ? PER_ROW : PER_GROUP; + // Case to fall back + if (weight.dim() == 2) { + TORCH_CHECK( + act_quant_mode != PER_GROUP && wei_quant_mode != PER_GROUP, + "FP8 linear: Per-group quantization is not supported in the fallback path"); + auto y_fp32 = at::linear(input.to(at::kFloat).mul_(input_scales), weight.to(at::kFloat).mul_(weight_scales), bias); + return y_fp32.to(output_dtype); + } + + static bool cpublas_can_pack = cpublas_could_pack(); + auto out_sizes = input.sizes().vec(); + out_sizes.back() = N; + auto output = at::empty(out_sizes, input.options().dtype(output_dtype)); + +#define AT_DISPATCH_FP8_LINEAR_KERNEL(OUT_DTYPE, CAN_PACK, A_QUANT_MODE, B_QUANT_MODE, ...) \ + AT_DISPATCH_BOOL_NO_RETURN( \ + CAN_PACK, \ + "cpublas_can_pack", \ + can_pack, \ + AT_DISPATCH_QUANT_MODE_NO_RETURN( \ + A_QUANT_MODE, \ + "act_quant_mode", \ + a_quant_mode, \ + AT_DISPATCH_QUANT_MODE_NO_RETURN( \ + B_QUANT_MODE, \ + "wei_quant_mode", \ + b_quant_mode, \ + AT_DISPATCH_OUT_TYPES(OUT_DTYPE, "out_dtype", __VA_ARGS__)))) + + AT_DISPATCH_FP8_LINEAR_KERNEL(output_dtype, cpublas_can_pack, act_quant_mode, wei_quant_mode, [&]() { + _float8_linear_impl( + input, input_scales, weight, weight_scales, bias, output); + }); + return output; +} + +at::Tensor float8_linear_impl( + const at::Tensor& input, + const at::Tensor& input_scales, + const at::Tensor& weight, + const at::Tensor& weight_scales, + const std::optional& bias, + at::ScalarType output_dtype) { + int64_t N = weight.dim() == 4 ? weight.size(0) * weight.size(-1) : weight.size(0); + int act_quant_mode = input_scales.numel() == 1 ? PER_TENSOR + : input_scales.numel() == input.numel() / input.size(-1) ? PER_ROW + : PER_GROUP; + int wei_quant_mode = weight_scales.numel() == 1 ? PER_TENSOR : weight_scales.numel() == N ? PER_ROW : PER_GROUP; + // Case to fall back + if (weight.dim() == 2) { + TORCH_CHECK( + act_quant_mode != PER_GROUP && wei_quant_mode != PER_GROUP, + "FP8 linear: Per-group quantization is not supported in the fallback path"); + auto y_fp32 = at::linear(input.to(at::kFloat).mul_(input_scales), weight.to(at::kFloat).mul_(weight_scales), bias); + return y_fp32.to(output_dtype); + } + + static bool cpublas_can_pack = cpublas_could_pack(); + auto out_sizes = input.sizes().vec(); + out_sizes.back() = N; + auto output = at::empty(out_sizes, input.options().dtype(output_dtype)); + +#define AT_DISPATCH_FP8_LINEAR_KERNEL(OUT_DTYPE, CAN_PACK, A_QUANT_MODE, B_QUANT_MODE, ...) \ + AT_DISPATCH_BOOL_NO_RETURN( \ + CAN_PACK, \ + "cpublas_can_pack", \ + can_pack, \ + AT_DISPATCH_QUANT_MODE_NO_RETURN( \ + A_QUANT_MODE, \ + "act_quant_mode", \ + a_quant_mode, \ + AT_DISPATCH_QUANT_MODE_NO_RETURN( \ + B_QUANT_MODE, \ + "wei_quant_mode", \ + b_quant_mode, \ + AT_DISPATCH_OUT_TYPES(OUT_DTYPE, "out_dtype", __VA_ARGS__)))) + + AT_DISPATCH_FP8_LINEAR_KERNEL(output_dtype, cpublas_can_pack, act_quant_mode, wei_quant_mode, [&]() { + _float8_linear_impl( + input, input_scales, weight, weight_scales, bias, output); + }); + return output; +} \ No newline at end of file diff --git a/csrc/cpu/sgl-kernels/moe.cpp b/csrc/cpu/sgl-kernels/moe.cpp index 3afdc1bd67..09a170eb6d 100644 --- a/csrc/cpu/sgl-kernels/moe.cpp +++ b/csrc/cpu/sgl-kernels/moe.cpp @@ -836,6 +836,11 @@ static inline void check_moe_scales( TORCH_CHECK(w2_scale.has_value(), "missing w2_scale for mxfp4."); TORCH_CHECK(w1_scale.value().scalar_type() == at::kByte, "expect w1_scale to be uint8."); TORCH_CHECK(w2_scale.value().scalar_type() == at::kByte, "expect w2_scale to be uint8."); + } else if constexpr (quant == CPUQuantMethod::FP8_W8A8) { + TORCH_CHECK(w1_scale.has_value(), "missing w1_scale for fp8 w8a8."); + TORCH_CHECK(w2_scale.has_value(), "missing w2_scale for fp8 w8a8."); + TORCH_CHECK(block_size.has_value(), "missing block_size for fp8 w8a8."); + TORCH_CHECK(block_size.value().size() == 2, "expect block_size.size() to be 2."); } } @@ -850,9 +855,14 @@ static inline void check_moe_scales( check_moe_scales(w1_scale, w2_scale, block_size); } else if (moe_comp_method == CPUQuantMethod::MXFP4) { check_moe_scales(w1_scale, w2_scale, block_size); + } else if (moe_comp_method == CPUQuantMethod::FP8_W8A8) { + check_moe_scales(w1_scale, w2_scale, block_size); } } +// CHECK_MOE_SCALES_FP8: validate scale shapes for FP8_W8A16. +// w1s shape [E, 2N/block_N, K/block_K, block_N]: DIM0=1, DIM1=2 +// w2s shape [E, K/block_N, N/block_K]: DIM0=1, DIM1=2 #define CHECK_MOE_SCALES_FP8(DIM0, DIM1) \ auto w1s = w1_scale.value(); \ auto w2s = w2_scale.value(); \ @@ -864,6 +874,17 @@ static inline void check_moe_scales( TORCH_CHECK(w2s.size(DIM0) == div_up(K, block_size_N)); \ TORCH_CHECK(w2s.size(DIM1) == div_up(N, block_size_K)) +// CHECK_MOE_SCALES_FP8_W8A8: extract scale info for FP8_W8A8 without shape +// validation, because w1s is pre-packed by float8_linear_prepack_cpu with +// shape [E, 2N/BLOCK_N, G, BLOCK_N] where BLOCK_N=32 != block_size_N(128). +// This matches the sglang upstream behavior (checks are commented out there). +#define CHECK_MOE_SCALES_FP8_W8A8(DIM0, DIM1) \ + auto w1s = w1_scale.value(); \ + auto w2s = w2_scale.value(); \ + auto block_size_val = block_size.value(); \ + int64_t block_size_N = block_size_val[0]; \ + int64_t block_size_K = block_size_val[1] + // hidden_states: [M, K] // w1: [E, 2N, K] or [E, 2N, K / 2] for uint8 // w2: [E, K, N] or [E, K, N / 2] for uint8 @@ -883,6 +904,7 @@ void fused_experts_cpu( const std::optional& w2_scale, const std::optional& w1_zero, const std::optional& w2_zero, + const std::optional& a1_scale, const std::optional> block_size, const std::optional& w1_bias, const std::optional& w2_bias, @@ -895,7 +917,10 @@ void fused_experts_cpu( constexpr int64_t BLOCK_M = block_size_m(); constexpr int64_t BLOCK_N = block_size_n(); - const auto st = hidden_states.scalar_type(); + // FP8_W8A8 hidden_states input is FP8; scalar_t should be BF16 for output + const auto st = (moe_comp_method == CPUQuantMethod::FP8_W8A8) + ? at::kBFloat16 + : hidden_states.scalar_type(); // TODO: fused_topk_torch_native (CPU fallback for models like MiniMax) // returns int64 topk_ids; fused_experts_cpu requires int32. Remove the typecast after topk kernel is provided auto topk_ids_ = topk_ids.scalar_type() == at::kInt ? topk_ids : topk_ids.to(at::kInt); @@ -908,6 +933,9 @@ void fused_experts_cpu( if (moe_comp_method == CPUQuantMethod::INT4_W4A8 && is_vnni) { CHECK_DIM(4, w1); CHECK_DIM(4, w2); + } else if (moe_comp_method == CPUQuantMethod::FP8_W8A8) { + // w1 is prepackaged by float8_linear_prepack_cpu (5D), skip w1 dim check + CHECK_DIM(3, w2); } else { CHECK_DIM(3, w1); CHECK_DIM(3, w2); @@ -924,8 +952,9 @@ void fused_experts_cpu( int64_t M = hidden_states.size(0); int64_t K = hidden_states.size(1); - int64_t N = moe_comp_method == CPUQuantMethod::INT4_W4A8 ? w1_scale.value().size(1) * w1_scale.value().size(3) / 2 - : w1.size(1) / 2; + int64_t N = moe_comp_method == CPUQuantMethod::INT4_W4A8 + ? w1_scale.value().size(1) * w1_scale.value().size(3) / 2 + : (moe_comp_method == CPUQuantMethod::FP8_W8A8 ? w2.size(2) : w1.size(1) / 2); int64_t E = w1.size(0); int64_t topk = topk_weights_.size(1); @@ -935,7 +964,10 @@ void fused_experts_cpu( // check weight shapes CHECK_EQ(w2.size(0), E); - if (!(moe_comp_method == CPUQuantMethod::INT4_W4A8)) { + if (moe_comp_method == CPUQuantMethod::FP8_W8A8) { + CHECK_EQ(w2.size(1), K); + CHECK_EQ(packed_w2.size(2), packed_N); + } else if (!(moe_comp_method == CPUQuantMethod::INT4_W4A8)) { CHECK_EQ(w2.size(1), K); CHECK_EQ(packed_w1.size(2), packed_K / (moe_comp_method == CPUQuantMethod::INT4_W4A8 ? 2 : 1)); CHECK_EQ(packed_w2.size(2), packed_N / (moe_comp_method == CPUQuantMethod::INT4_W4A8 ? 2 : 1)); @@ -993,22 +1025,35 @@ void fused_experts_cpu( // 5. Aq_tmp : [M, K] or [M * topk, N] // 6. As_tmp : [M * topk] // - // for fp8 w8a16 and mxfp4: + // for fp8 w8a16, mxfp4, and fp8 w8a8: // 7. intermediate_cache0 : [M * topk, 2N] // 8. B_tmp : [T, MAX_CACHE_BLOCK_SIZE, BLOCK_N, std::max(K, N)] // + // for fp8 w8a8 additionally: + // 9. Ukernel_tmp : [T, 2 * BLOCK_M * BLOCK_N] (float32 for FP8×FP8 brgemm) + // int64_t buffer_size_nbytes = M * topk * N * 2 + M * topk * K * 2 + num_threads * BLOCK_M * K * - (moe_comp_method == CPUQuantMethod::INT8_W8A8 | moe_comp_method == CPUQuantMethod::INT4_W4A8 ? 1 : 2) + + (moe_comp_method == CPUQuantMethod::INT8_W8A8 | moe_comp_method == CPUQuantMethod::INT4_W4A8 + | moe_comp_method == CPUQuantMethod::FP8_W8A8 ? 1 : 2) + num_threads * 2 * BLOCK_M * BLOCK_N * sizeof(float); if (moe_comp_method == CPUQuantMethod::INT8_W8A8) { buffer_size_nbytes += std::max(M * K, M * topk * N) + M * topk * sizeof(float); } - if (moe_comp_method == CPUQuantMethod::FP8_W8A16 || moe_comp_method == CPUQuantMethod::MXFP4) { + if (moe_comp_method == CPUQuantMethod::FP8_W8A16 || moe_comp_method == CPUQuantMethod::MXFP4 + || moe_comp_method == CPUQuantMethod::FP8_W8A8) { buffer_size_nbytes += M * topk * 2 * N * 2 + num_threads * MAX_CACHE_BLOCK_SIZE * BLOCK_N * std::max(K, N) * 2; } + if (moe_comp_method == CPUQuantMethod::FP8_W8A8) { + // Ukernel buffer: float32 workspace per thread for FP8×FP8 brgemm + buffer_size_nbytes += num_threads * 2 * BLOCK_M * BLOCK_N * sizeof(float); +#ifndef CPUBLAS_BRGEMM_F8F8F32 + // BF16 fallback: also need dqA [T, BLOCK_M, BLOCK_K] + dqB [T, BLOCK_K, BLOCK_N] + buffer_size_nbytes += num_threads * (BLOCK_M * BLOCK_K + BLOCK_K * BLOCK_N) * 2; +#endif + } if (moe_comp_method == CPUQuantMethod::INT4_W4A8) { buffer_size_nbytes += M * topk * 2 * N * 2 + std::max(M * K, M * topk * N) + M * topk * sizeof(float) + num_threads * 2 * get_4bit_block_k_size(K / w1_scale.value().size(2)) * BLOCK_N; @@ -1181,6 +1226,47 @@ void fused_experts_cpu( E, topk, num_tokens_post_pad); + } else if (moe_comp_method == CPUQuantMethod::FP8_W8A8) { + // Stage 1: FP8×FP8 GEMM (hidden_states is FP8 input) + // Stage 2: BF16×FP8 GEMM (W8A16 path for down-projection) + at::Float8_e4m3fn* __restrict__ A_tmp = + (at::Float8_e4m3fn*)((void*)(intermediate_cache2 + M * topk * K)); + float* __restrict__ C_tmp = (float*)((void*)(A_tmp + num_threads * BLOCK_M * K)); + scalar_t* __restrict__ intermediate_cache0 = + (scalar_t*)((void*)(C_tmp + num_threads * 2 * BLOCK_M * BLOCK_N)); + scalar_t* __restrict__ B_tmp = + (scalar_t*)((void*)(intermediate_cache0 + M * topk * 2 * N)); + float* __restrict__ Ukernel_tmp = + (float*)((void*)(B_tmp + num_threads * MAX_CACHE_BLOCK_SIZE * BLOCK_N * std::max(K, N))); + + CHECK_MOE_SCALES_FP8_W8A8(1, 2); + fused_experts_fp8_a8_kernel_impl( + out_hidden_states.data_ptr(), + intermediate_cache0, + intermediate_cache1, + intermediate_cache2, + A_tmp, + B_tmp, + C_tmp, + Ukernel_tmp, + hidden_states.data_ptr(), + packed_w1.data_ptr(), + packed_w2.data_ptr(), + a1_scale.has_value() ? a1_scale.value().data_ptr() : nullptr, + w1s.data_ptr(), + w2s.data_ptr(), + block_size_N, + block_size_K, + topk_weights_.data_ptr(), + sorted_ids, + expert_ids, + offsets, + M, + N, + K, + E, + topk, + num_tokens_post_pad); } else { scalar_t* __restrict__ A_tmp = intermediate_cache2 + M * topk * K; float* __restrict__ C_tmp = (float*)((void*)(A_tmp + num_threads * BLOCK_M * K)); diff --git a/csrc/cpu/sgl-kernels/moe_fp8_w8a8.cpp b/csrc/cpu/sgl-kernels/moe_fp8_w8a8.cpp new file mode 100644 index 0000000000..0f393a34a8 --- /dev/null +++ b/csrc/cpu/sgl-kernels/moe_fp8_w8a8.cpp @@ -0,0 +1,432 @@ +// Adapted from +// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu + +// clang-format off + +#include + +#include "common.h" +#include "gemm.h" +#include "moe.h" + +namespace { + +// Byte-wise copy for FP8 elements (no remainder assumption) +inline void copy_stub2( + at::Float8_e4m3fn* __restrict__ out, + const at::Float8_e4m3fn* __restrict__ input, + int64_t size) { + for (int64_t d = 0; d < size; d++) { + out[d] = input[d]; + } +} + +#define PER_TENSOR 1 +#define PER_ROW 2 +#define PER_GROUP 3 + +// Store float32 GEMM result to output buffer with dequantization and dtype +// conversion. Activations are PER_ROW quantized; weights are PER_GROUP quantized. +// Uses AVX512 on capable platforms. +#if defined(CPU_CAPABILITY_AVX512) + +template +inline void store_out( + const float* y_buf, + out_dtype* c_ptr, + int64_t M, + int64_t lda, + const float* scales_a, + const float* scales_b, + const float* bias) { + float a_scale = 1.0, b_scale = 1.0; + __m512 va_scale, vb_scale; + if constexpr (act_quant_mode == PER_TENSOR) { + a_scale = *scales_a; + } + if constexpr (wei_quant_mode == PER_TENSOR) { + b_scale = *scales_b; + vb_scale = _mm512_set1_ps(b_scale); + } + for (int i = 0; i < M; ++i) { + if constexpr (act_quant_mode == PER_ROW) { + a_scale = *(scales_a + i); + } + if constexpr (act_quant_mode != PER_GROUP) { + va_scale = _mm512_set1_ps(a_scale); + } + constexpr int N_UNROLL = N / 16; + c10::ForcedUnroll{}([&](auto idx) { + constexpr int j = idx * 16; + __m512 y_vec = _mm512_loadu_ps(y_buf + i * N + j); + __m512 bias_vec = bias ? _mm512_loadu_ps(bias + j) : _mm512_setzero_ps(); + if constexpr (act_quant_mode != PER_GROUP) { + y_vec = _mm512_mul_ps(y_vec, va_scale); + } + if constexpr (wei_quant_mode == PER_ROW) { + vb_scale = _mm512_loadu_ps(scales_b + j); + } + if constexpr (wei_quant_mode != PER_GROUP) { + y_vec = _mm512_mul_ps(y_vec, vb_scale); + } + y_vec = _mm512_add_ps(y_vec, bias_vec); + if constexpr (std::is_same::value) { + _mm512_storeu_ps(c_ptr + i * lda + j, y_vec); + } else if constexpr (std::is_same::value) { + __m256i y_bf16_vec = at::vec::cvtfp32_bf16(y_vec); + _mm256_storeu_si256(reinterpret_cast<__m256i*>(c_ptr + i * lda + j), y_bf16_vec); + } else if constexpr (std::is_same::value) { + __m256i y_fp16_vec = at::vec::cvtfp32_fp16(y_vec); + _mm256_storeu_si256(reinterpret_cast<__m256i*>(c_ptr + i * lda + j), y_fp16_vec); + } else { + TORCH_CHECK(false, "Unsupported output dtype"); + } + }); + constexpr int tail_start = N / 16 * 16; + for (int j = tail_start; j < N; ++j) { + if constexpr (wei_quant_mode == PER_ROW) { + b_scale = scales_b[j]; + } + c_ptr[i * lda + j] = static_cast(y_buf[i * N + j] * a_scale * b_scale); + } + } // for M +} + +#else // no AVX512 + +template +inline void store_out( + const float* y_buf, + out_dtype* c_ptr, + int64_t M, + int64_t lda, + const float* scales_a, + const float* scales_b, + const float* bias) { + float a_scale = 1.0, b_scale = 1.0; + if constexpr (act_quant_mode == PER_TENSOR) { + a_scale = *scales_a; + } + if constexpr (wei_quant_mode == PER_TENSOR) { + b_scale = *scales_b; + } + for (int i = 0; i < M; ++i) { + if constexpr (act_quant_mode == PER_ROW) { + a_scale = *(scales_a + i); + } + for (int j = 0; j < N; ++j) { + if constexpr (wei_quant_mode == PER_ROW) { + b_scale = scales_b[j]; + } + float val = y_buf[i * N + j] * a_scale * b_scale; + if (bias) val += bias[j]; + c_ptr[i * lda + j] = static_cast(val); + } + } +} + +#endif // CPU_CAPABILITY_AVX512 + +} // anonymous namespace + +// FP8 W8A8 fused MoE kernel. +// +// hidden_states: [M, K] FP8 E4M3, pre-quantized activation +// packed_w1: [E, 2*Nc, Kc, BLOCK_K, BLOCK_N] FP8 W8A8 prepack (float8_linear_prepack_cpu) +// packed_w2: [E, K, N_packed] FP8 W8A16 VNNI pack (convert_weight_packed) +// As: [M] float32 per-token activation scales +// w1s: [E, num_groups, 2N] float32 per-group weight scales for w1 +// w2s: [E, ...] float32 per-group weight scales for w2 +// +// Stage 1: hidden_states(FP8) @ w1(FP8) → ic0(BF16) [M*topk, 2N] +// Uses FP8×FP8 GEMM (tinygemm_kernel with FP8 inputs) +// Stage 1.5: ic1 = silu(ic0[:,:N]) * ic0[:,N:] [M*topk, N] +// Stage 2: ic1(BF16) @ w2(FP8) → ic2(BF16) [M*topk, K] +// Uses BF16×FP8 GEMM (W8A16 path, tinygemm_kernel) +// Stage 3: out = weighted_sum(ic2, topk_weights) [M, K] + +template +void fused_experts_fp8_a8_kernel_impl( + scalar_t* __restrict__ output, + scalar_t* __restrict__ ic0, + scalar_t* __restrict__ ic1, + scalar_t* __restrict__ ic2, + at::Float8_e4m3fn* __restrict__ A_tmp, + scalar_t* __restrict__ B_tmp, + float* __restrict__ C_tmp, + float* __restrict__ Ukernel_tmp, + const at::Float8_e4m3fn* __restrict__ input, + const at::Float8_e4m3fn* __restrict__ packed_w1, + const at::Float8_e4m3fn* __restrict__ packed_w2, + const float* __restrict__ As, + const float* __restrict__ w1s, + const float* __restrict__ w2s, + int64_t block_size_N, + int64_t block_size_K, + const float* __restrict__ topk_weights, + const int32_t* __restrict__ sorted_ids, + const int32_t* __restrict__ expert_ids, + const int32_t* __restrict__ offsets, + int64_t M, + int64_t N, + int64_t K, + int64_t E, + int64_t topk, + int64_t num_tokens_post_pad) { + // 1. intermediate_cache1 : [M * topk, N] + // 2. intermediate_cache2 : [M * topk, K] + // 3. A_tmp : [T, BLOCK_M * K] FP8 + // 4. C_tmp : [T, 2 * BLOCK_M * BLOCK_N] float32 + // 5. intermediate_cache0 : [M * topk, 2N] + // 6. B_tmp : [T, MAX_CACHE_BLOCK_SIZE, BLOCK_N, max(K, N)] scalar_t + // 7. Ukernel_tmp: [T, 2 * BLOCK_M * BLOCK_N] float32 + // 8. (non-FP8-brgemm only) dqA [T, BLOCK_M * BLOCK_K] + dqB [T, BLOCK_K * BLOCK_N] in BF16 + // appended after Ukernel_tmp; see moe.cpp FP8_W8A8 buffer allocation + + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N_VAL = block_size_n(); // = 32 + const int64_t KB = K / BLOCK_K; // K blocks + + int64_t B_tmp_size_per_thread = MAX_CACHE_BLOCK_SIZE * BLOCK_N_VAL * std::max(K, N); + + // stage 1: intermediate_cache0 = hidden_states @ w1 + const int64_t MB = div_up(num_tokens_post_pad, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N_VAL); + + int64_t scale_size_N = div_up(2 * N, block_size_N); + int64_t scale_size_K = div_up(K, block_size_K); + int64_t blocks_n_per_group = block_size_N / BLOCK_N_VAL; + int64_t num_groups = div_up(K, block_size_K); // G + int64_t blocks_k_per_group = block_size_K / BLOCK_K; + const int64_t stride_e = 2 * N * K; // per-expert stride for w1 (total elements) + bool use_brgemm = true; + int64_t num_thread = at::get_num_threads(); + + // For the BF16 fallback path (no native FP8 brgemm), tinygemm_kernel needs temporary + // BF16 buffers to dequantize FP8 inputs before running AMX-BF16 brgemm. + // These are placed after the Ukernel_tmp region (see moe.cpp FP8_W8A8 buffer allocation). +#ifndef CPUBLAS_BRGEMM_F8F8F32 + // dqA: [T, BLOCK_M, BLOCK_K] BF16 + // dqB: [T, BLOCK_K, BLOCK_N] BF16 + // Layout: immediately after Ukernel_tmp[num_thread * 2 * BLOCK_M * BLOCK_N_VAL] + at::BFloat16* __restrict__ dq_base = reinterpret_cast( + Ukernel_tmp + num_thread * 2 * BLOCK_M * BLOCK_N_VAL); + const int64_t dq_per_thread = BLOCK_M * BLOCK_K + BLOCK_K * BLOCK_N_VAL; +#endif + + at::parallel_for(0, MB * NB, 1, [&](int64_t begin, int64_t end) { + int tid = get_thread_num(); + // A_tmp: FP8 activation buffer for this thread [BLOCK_M, K] + at::Float8_e4m3fn* __restrict__ A = A_tmp + tid * BLOCK_M * K; + float* __restrict__ C0 = C_tmp + tid * 2 * BLOCK_M * BLOCK_N_VAL; + float* __restrict__ C1 = C0 + BLOCK_M * BLOCK_N_VAL; + alignas(64) float As_[BLOCK_M]; + float* __restrict__ ukernel_buf_1 = Ukernel_tmp + tid * 2 * BLOCK_M * BLOCK_N_VAL; +#ifndef CPUBLAS_BRGEMM_F8F8F32 + at::BFloat16* __restrict__ dqA_buf = dq_base + tid * dq_per_thread; + at::BFloat16* __restrict__ dqB_buf = dqA_buf + BLOCK_M * BLOCK_K; +#else + at::BFloat16* __restrict__ dqA_buf = nullptr; + at::BFloat16* __restrict__ dqB_buf = nullptr; +#endif + float* __restrict__ ukernel_buf_2 = ukernel_buf_1 + BLOCK_M * BLOCK_N_VAL; + auto ldsa = 1; // activation per-row quantized + + for (int64_t i = begin; i < end; ++i) { + int64_t mb = i / NB; + int64_t nb = i % NB; + int64_t nb1 = nb + NB; // gate is first N columns, up is second N columns + + int64_t n_size = std::min(N - nb * BLOCK_N_VAL, BLOCK_N_VAL); + + // B (weight) for expert expert_id: gate+up fused [2N, K] prepackaged as FP8 + int32_t expert_id = expert_ids[mb]; + const at::Float8_e4m3fn* __restrict__ B = packed_w1 + expert_id * stride_e; + const float* __restrict__ Bs = w1s + expert_id * num_groups * (2 * N); + + // Load activation tile for this M-block + const int32_t* A_ids = sorted_ids + mb * BLOCK_M; + int64_t m_size = offsets[mb + 1] - offsets[mb]; + + for (int64_t m = 0; m < m_size; ++m) { + int32_t index = A_ids[m] / topk; + copy_stub2(A + m * K, input + index * K, K); + As_[m] = As[index]; + } + + const int64_t offset = offsets[mb]; + + // Gate path (nb): w1[:N, :] + zero_buffer(C0, BLOCK_M * BLOCK_N_VAL); + for (int kci = 0; kci < KB; ++kci) { + // FP8×FP8 tinygemm: act_quant=PER_ROW(2), wei_quant=PER_GROUP(3) + tinygemm_kernel( + /* C */ C0, + /* A */ A + kci * BLOCK_K, + /* scales_a*/ As_ + kci / blocks_k_per_group, + /* B */ B + (nb * KB + kci) * BLOCK_K * BLOCK_N_VAL, + /* scales_b*/ Bs + nb * BLOCK_N_VAL * num_groups + kci / blocks_k_per_group * BLOCK_N_VAL, + /* M */ m_size, + /* K */ BLOCK_K, + /* lda */ K, + /* ldc */ BLOCK_N_VAL, + /* ldsa */ ldsa, + /* ukernel */ ukernel_buf_1, + /* dqA_buf */ dqA_buf, + /* dqB_buf */ dqB_buf); + } + store_out( + C0, + ic0 + offset * 2 * N + nb * BLOCK_N_VAL, + m_size, + 2 * N, // lda + As_, + nullptr, + nullptr); + + // Up path (nb1): w1[N:, :] + zero_buffer(C1, BLOCK_M * BLOCK_N_VAL); + for (int kci = 0; kci < KB; ++kci) { + tinygemm_kernel( + /* C */ C1, + /* A */ A + kci * BLOCK_K, + /* scales_a*/ As_ + kci / blocks_k_per_group, + /* B */ B + (nb1 * KB + kci) * BLOCK_K * BLOCK_N_VAL, + /* scales_b*/ Bs + nb1 * BLOCK_N_VAL * num_groups + kci / blocks_k_per_group * BLOCK_N_VAL, + /* M */ m_size, + /* K */ BLOCK_K, + /* lda */ K, + /* ldc */ BLOCK_N_VAL, + /* ldsa */ ldsa, + /* ukernel */ ukernel_buf_2, + /* dqA_buf */ dqA_buf, + /* dqB_buf */ dqB_buf); + } + store_out( + C1, + ic0 + offset * 2 * N + nb1 * BLOCK_N_VAL, + m_size, + 2 * N, // lda + As_, + nullptr, + nullptr); + } + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); + + // stage 1.5: intermediate_cache1 = silu(ic0[:, :N]) * ic0[:, N:] + at::parallel_for(0, M * topk, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + silu_and_mul_stub(ic1 + m * N, ic0 + m * 2 * N, ic0 + m * 2 * N + N, N); + } + }); + + // stage 2: intermediate_cache2 = intermediate_cache1 @ w2 + // w2: [E, K, N] FP8, VNNI packed; treated as W8A16 (BF16 act × FP8 weight) + const int64_t OC = K; // output channels (hidden dim) + const int64_t IC = N; // input channels (expert intermediate dim) + const int64_t MB2 = MB; + const int64_t NB2 = div_up(OC, BLOCK_N_VAL); + scale_size_N = div_up(K, block_size_N); + scale_size_K = div_up(N, block_size_K); + const int64_t stride_e2 = OC * IC; // per-expert stride for w2 + const int64_t stride_oc = IC; + + int64_t avg_M = std::max(int64_t(1), M * topk / E); + bool use_brgemm2 = can_use_brgemm(avg_M); + + parallel_2d(MB2, NB2, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + int tid = get_thread_num(); + alignas(64) scalar_t C[BLOCK_M * BLOCK_K]; + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N_VAL * IC, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + int64_t m_size = offsets[mb + 1] - offsets[mb]; + int64_t n_size = std::min(OC - nb * BLOCK_N_VAL, BLOCK_N_VAL); + + // A from ic1: [M*topk, N] in sorted order (contiguous after stage 1.5) + const scalar_t* __restrict__ A2 = ic1 + offsets[mb] * N; + const int32_t* A_ids = sorted_ids + mb * BLOCK_M; + + // B from w2: [E, K, N] VNNI packed, offset to expert + output block + int32_t expert_id = expert_ids[mb]; + const at::Float8_e4m3fn* __restrict__ B2 = + packed_w2 + expert_id * stride_e2 + nb * BLOCK_N_VAL * stride_oc; + const float* __restrict__ Bs2 = + w2s + expert_id * scale_size_N * scale_size_K + (nb / blocks_n_per_group) * scale_size_K; + + // Only unpack B on first visit or new expert + int32_t pre_expert_id = mb == 0 ? -1 : expert_ids[mb - 1]; + bool do_unpack = (mb == mb0) || (expert_id != pre_expert_id); + + tinygemm_kernel( + /* A */ A2, + /* B */ B2, + /* C */ C, + /* Btmp */ B_tmp + tid * B_tmp_size_per_thread + nb_offset * BLOCK_N_VAL * IC, + /* Ctmp */ C_tmp + tid * 2 * BLOCK_M * BLOCK_N_VAL, + /* Bbias */ nullptr, + /* scale */ Bs2, + /* M */ m_size, + /* N */ n_size, + /* K */ IC, + /* lda */ IC, + /* ldb */ n_size, + /* ldc */ BLOCK_N_VAL, + /* brg */ use_brgemm2, + /* block_sz_K */ block_size_K, + /* do_unpack */ do_unpack); + + // Scatter ic2 in original token order, weighting by topk_weights + for (int64_t m = 0; m < m_size; ++m) { + int32_t index = A_ids[m]; + float weight = topk_weights[index]; + copy_mul_stub(ic2 + index * K + nb * BLOCK_N_VAL, C + m * BLOCK_N_VAL, weight, n_size); + } + }); + + if (use_brgemm2) { + at::native::cpublas::brgemm_release(); + } + }); + + // stage 3: output = sum(ic2, dim=topk) [M, K] + at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + sum_stub(output + m * K, ic2 + m * topk * K, topk, K); + } + }); +} + +#define INSTANTIATE_MOE_FP8_A8_TEMPLATE(TYPE) \ + template void fused_experts_fp8_a8_kernel_impl( \ + TYPE* __restrict__ output, \ + TYPE* __restrict__ ic0, \ + TYPE* __restrict__ ic1, \ + TYPE* __restrict__ ic2, \ + at::Float8_e4m3fn* __restrict__ A_tmp, \ + TYPE* __restrict__ B_tmp, \ + float* __restrict__ C_tmp, \ + float* __restrict__ Ukernel_tmp, \ + const at::Float8_e4m3fn* __restrict__ input, \ + const at::Float8_e4m3fn* __restrict__ packed_w1, \ + const at::Float8_e4m3fn* __restrict__ packed_w2, \ + const float* __restrict__ As, \ + const float* __restrict__ w1s, \ + const float* __restrict__ w2s, \ + int64_t block_size_N, \ + int64_t block_size_K, \ + const float* __restrict__ topk_weights, \ + const int32_t* __restrict__ sorted_ids, \ + const int32_t* __restrict__ expert_ids, \ + const int32_t* __restrict__ offsets, \ + int64_t M, \ + int64_t N, \ + int64_t K, \ + int64_t E, \ + int64_t topk, \ + int64_t num_tokens_post_pad) + +INSTANTIATE_MOE_FP8_A8_TEMPLATE(at::BFloat16); +INSTANTIATE_MOE_FP8_A8_TEMPLATE(at::Half); diff --git a/csrc/cpu/torch_bindings.cpp b/csrc/cpu/torch_bindings.cpp index 8f663a93c3..accd5fcec3 100644 --- a/csrc/cpu/torch_bindings.cpp +++ b/csrc/cpu/torch_bindings.cpp @@ -73,6 +73,7 @@ void fused_experts_cpu(at::Tensor& out, at::Tensor& hidden_states, const std::optional& w2_scale, const std::optional& w1_zero, const std::optional& w2_zero, + const std::optional& a1_scale, const std::optional> block_size, const std::optional& w1_bias, const std::optional& w2_bias, @@ -91,6 +92,18 @@ at::Tensor fp8_scaled_mm_cpu(at::Tensor& mat1, at::Tensor& mat2, const std::optional& bias, at::ScalarType out_dtype, bool is_vnni); +// Adapted from sglang: FP8 W8A8 kernels +std::tuple float8_linear_prepack_impl( + const at::Tensor& weight, const at::Tensor& scales); +at::Tensor fp8_scaled_mm_with_quant(const at::Tensor& act, + const std::optional& act_scales, + bool channelwise, const at::Tensor& weight, + const at::Tensor& weight_scales, + const std::optional& bias, + at::ScalarType output_dtype); +std::tuple quantize_fp8e4m3_vec( + const at::Tensor& t, bool channelwise, c10::optional scale_opt); + // Adapted from sglang: MLA CPU kernels (AMX-only) void decode_attention_cpu(at::Tensor& query, at::Tensor& k_buffer, at::Tensor& v_buffer, at::Tensor& output, @@ -281,6 +294,7 @@ void activation_lut_bf16(torch::Tensor& out, torch::Tensor& input, const std::string& activation); bool cpu_attn_has_isa(const std::string& isa); +bool cpu_has_amx_fp8(); torch::Tensor get_scheduler_metadata( const int64_t num_req, const int64_t num_heads_q, @@ -611,7 +625,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) { "fused_experts_cpu(Tensor(a0!) out, Tensor hidden_states, Tensor w1, " "Tensor w2, Tensor topk_weights, Tensor topk_ids, " "int moe_comp_method, Tensor? w1_scale, Tensor? w2_scale, " - "Tensor? w1_zero, Tensor? w2_zero, int[]? block_size, " + "Tensor? w1_zero, Tensor? w2_zero, Tensor? a1_scale, int[]? block_size, " "Tensor? w1_bias, Tensor? w2_bias, float? alpha, float? limit, " "bool is_vnni) -> " "()"); @@ -629,7 +643,23 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) { "bool is_vnni) -> Tensor"); ops.impl("fp8_scaled_mm_cpu", torch::kCPU, &fp8_scaled_mm_cpu); - // Adapted from sglang: casual_conv1d kernels + // Adapted from sglang: FP8 W8A8 kernels + ops.def( + "float8_linear_prepack_cpu(Tensor weight, Tensor scales) -> (Tensor, " + "Tensor)"); + ops.impl("float8_linear_prepack_cpu", torch::kCPU, + &float8_linear_prepack_impl); + ops.def( + "fp8_scaled_mm_with_quant(Tensor act, Tensor? act_scales, bool " + "channelwise, " + "Tensor weight, Tensor weight_scales, Tensor? bias, ScalarType " + "output_dtype) -> Tensor"); + ops.impl("fp8_scaled_mm_with_quant", torch::kCPU, &fp8_scaled_mm_with_quant); + ops.def( + "quantize_fp8e4m3_vec(Tensor input, bool channelwise, Tensor? scale_opt) " + "-> (Tensor, Tensor)"); + ops.impl("quantize_fp8e4m3_vec", torch::kCPU, &quantize_fp8e4m3_vec); + ops.def("causal_conv1d_weight_pack(Tensor weight) -> Tensor"); ops.impl("causal_conv1d_weight_pack", torch::kCPU, &causal_conv1d_weight_pack); @@ -832,6 +862,9 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) { "-> (Tensor, Tensor)"); ops.impl("fused_gdn_gating_cpu", torch::kCPU, &fused_gdn_gating_cpu); + // Pure runtime CPU ISA capability check. + ops.def("cpu_has_amx_fp8() -> bool", &cpu_has_amx_fp8); + // CPU attention kernels ops.def("cpu_attn_has_isa(str isa) -> bool", &cpu_attn_has_isa); ops.def( @@ -883,8 +916,10 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) { "bool skip_weighted, " "str act, str isa) -> ()"); ops.impl("cpu_fused_moe", torch::kCPU, &cpu_fused_moe); -#if defined(ARM_I8MM_SUPPORT) && defined(ARM_BF16_SUPPORT) && \ - !defined(__APPLE__) + +#if (defined(ARM_I8MM_SUPPORT) && defined(ARM_BF16_SUPPORT) && \ + !defined(__APPLE__)) || \ + defined(__powerpc64__) ops.def( "prepack_moe_weight_int8(Tensor weight, Tensor(a1!) packed_weight, " "str isa) -> ()"); @@ -895,8 +930,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) { "Tensor? w2_bias, Tensor topk_weights, Tensor topk_id, bool " "skip_weighted, str act, str isa) -> ()"); ops.impl("cpu_fused_moe_int8", torch::kCPU, &cpu_fused_moe_int8); -#endif // #if defined(ARM_I8MM_SUPPORT) && defined(ARM_BF16_SUPPORT) && - // !defined(__APPLE__) +#endif // ARM_I8MM+BF16 or __powerpc64__ ops.def( "mla_decode_kvcache(" " Tensor! out, Tensor query, Tensor kv_cache," diff --git a/csrc/libtorch_stable/cache_kernels.cu b/csrc/libtorch_stable/cache_kernels.cu index 887c83f3af..4b8f4c23a5 100644 --- a/csrc/libtorch_stable/cache_kernels.cu +++ b/csrc/libtorch_stable/cache_kernels.cu @@ -19,7 +19,6 @@ #include #include -#include #include #ifdef USE_ROCM @@ -84,7 +83,9 @@ void swap_blocks(torch::stable::Tensor& src, torch::stable::Tensor& dst, namespace { // ROCm hipMemcpyBatchAsync faults for count > 8192 (MI350X/gfx950), so chunk // at that ceiling. +#if defined(USE_ROCM) constexpr int64_t kRocmDefaultMaxBatchDescriptors = 8192; +#endif constexpr const char* kMaxBatchDescriptorsEnv = "VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS"; @@ -587,8 +588,9 @@ __global__ void concat_and_cache_ds_mla_kernel( max_abs = fmaxf(max_abs, VLLM_SHFL_XOR_SYNC_WIDTH(max_abs, offset, 16)); } - // Compute the scale for the tile - float tile_scale = fmaxf(max_abs / kFp8ScaleDivisor, FLT_MIN); + // Both SM90 and SM100 readers preserve power-of-two fp32 scales exactly. + float tile_scale = fmaxf(max_abs / kFp8ScaleDivisor, 1e-4f); + tile_scale = exp2f(ceilf(log2f(tile_scale))); // The first lane of each half-warp writes the scale to kv_cache if ((lane_idx == 0) || (lane_idx == 16)) { diff --git a/csrc/libtorch_stable/fused_kimi_k3_mla_key_concat_kv_cache_kernel.cu b/csrc/libtorch_stable/fused_kimi_k3_mla_key_concat_kv_cache_kernel.cu index 05c7bfc6d1..3892790c40 100644 --- a/csrc/libtorch_stable/fused_kimi_k3_mla_key_concat_kv_cache_kernel.cu +++ b/csrc/libtorch_stable/fused_kimi_k3_mla_key_concat_kv_cache_kernel.cu @@ -35,8 +35,9 @@ * * fp8_ds_mla (fused_kimi_k3_mla_key_concat_ds_mla_insert): * - full key concat (bf16), and cache insert in DeepSeek's 656-byte - * block-scaled layout (NoPE fp8 in 4 tiles of 128 with per-tile dynamic - * scales, RoPE bf16), bit-compatible with concat_and_cache_ds_mla_kernel. + * block-scaled layout (NoPE fp8 in 4 tiles of 128 with per-tile + * power-of-two scales, RoPE bf16), bit-compatible with + * concat_and_cache_ds_mla_kernel. * * Both use Programmatic Dependent Launch (PDL) to overlap the tail of the * producing GEMMs on sm_90+, and are structured after @@ -68,7 +69,6 @@ #include "../quantization/w8a8/fp8/amd/quant_utils.cuh" #endif #include -#include #include #ifdef USE_ROCM @@ -111,9 +111,9 @@ constexpr float kFp8Max = 224.0f; #else constexpr float kFp8Max = 448.0f; #endif -// Divisor for fp8_ds_mla per-tile dynamic scales (matches cache_kernels.cu). -// fp8_ds_mla 656B entry: [0,512) NoPE fp8 (4 tiles of 128), [512,528) 4 fp32 -// tile scales, [528,656) RoPE 64 bf16. +// Divisor for fp8_ds_mla per-tile power-of-two scales (matches +// cache_kernels.cu). fp8_ds_mla 656B entry: [0,512) NoPE fp8 (4 tiles of 128), +// [512,528) 4 fp32 tile scales, [528,656) RoPE 64 bf16. constexpr float kFp8ScaleDivisor = kFp8Max; // Copy 8 source elements (one uint4 of bf16/fp16) to `dst`. FP8=false stores a @@ -325,7 +325,7 @@ __device__ __forceinline__ void writeLatent576( } // Write [kv_c | k_pe] into the fp8_ds_mla 656B entry using one warp: NoPE 512 -// as fp8 in 4 tiles of 128 (per-tile dynamic absmax scale, 4 fp32 scales at +// as fp8 in 4 tiles of 128 (per-tile power-of-two scale, 4 fp32 scales at // [512,528)), RoPE 64 as bf16 at [528,656). Bit-compatible with // concat_and_cache_ds_mla_kernel. template @@ -349,7 +349,8 @@ __device__ __forceinline__ void writeDsMlaCache( for (int offset = 4; offset > 0; offset /= 2) { max_abs = fmaxf(max_abs, VLLM_SHFL_XOR_SYNC_WIDTH(max_abs, offset, 8)); } - float const tile_scale = fmaxf(max_abs / kFp8ScaleDivisor, FLT_MIN); + float tile_scale = fmaxf(max_abs / kFp8ScaleDivisor, 1e-4f); + tile_scale = exp2f(ceilf(log2f(tile_scale))); if ((laneId & 7) == 0) { reinterpret_cast(row)[kKvLoraRank / 4 + tile] = tile_scale; } @@ -598,7 +599,7 @@ __global__ void fusedKimiK3MLAKVConcatPackKernel( // ds_mla variant: concat full key (bf16) + fp8_ds_mla latent cache insert // // Cache entry (656 bytes), matching concat_and_cache_ds_mla_kernel: -// [0, 512) NoPE 512 vals as fp8, 4 tiles of 128, each dynamically scaled +// [0, 512) NoPE 512 vals as fp8, 4 tiles of 128, power-of-two scaled // [512, 528) 4 fp32 per-tile scales // [528, 656) RoPE 64 vals as bf16 (unquantized) // The cache slot uses one warp: lane L quantizes NoPE elems [L*16, L*16+16) diff --git a/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_blockwise_sm100_fp8_dispatch.cuh b/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_blockwise_sm100_fp8_dispatch.cuh index 8f74f29914..fa2add9182 100644 --- a/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_blockwise_sm100_fp8_dispatch.cuh +++ b/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_blockwise_sm100_fp8_dispatch.cuh @@ -152,6 +152,29 @@ void cutlass_gemm_caller_blockwise(torch::stable::Tensor& out, torch::stable::Te int32_t m = a.size(0), n = b.size(1), k = a.size(1); + // The SM100 blockwise SF copy requires the SFA M extent to be a multiple + // of 4 (4-row copy atom). For misaligned M on the non-swapAB path, pad only + // the per-token activation scales (column-major (m, k/128), 1/64 the size + // of the bf16 activation) and build layout_SFA from the padded extent. + int32_t m_sf = m; + std::optional a_scales_pad; + void const* a_scales_data = a_scales.data_ptr(); + if constexpr (!swap_ab) { + if (m % 4 != 0) { + m_sf = (m + 3) & ~3; + int64_t num_groups = a_scales.numel() / m; + a_scales_pad = torch::stable::new_empty( + a_scales, {num_groups * m_sf}, + torch::headeronly::ScalarType::Float); + cudaMemcpy2DAsync(a_scales_pad->data_ptr(), m_sf * sizeof(float), + a_scales.data_ptr(), m * sizeof(float), + m * sizeof(float), num_groups, + cudaMemcpyDeviceToDevice, + get_current_cuda_stream(a.get_device())); + a_scales_data = a_scales_pad->data_ptr(); + } + } + StrideA a_stride; StrideB b_stride; StrideC c_stride; @@ -162,16 +185,16 @@ void cutlass_gemm_caller_blockwise(torch::stable::Tensor& out, torch::stable::Te c_stride = cutlass::make_cute_packed_stride(StrideC{}, swap_ab ? cute::make_shape(n, m, 1) : cute::make_shape(m, n, 1)); - LayoutSFA layout_SFA = swap_ab ? + LayoutSFA layout_SFA = swap_ab ? ScaleConfig::tile_atom_to_shape_SFA(make_shape(n, m, k, 1)) : - ScaleConfig::tile_atom_to_shape_SFA(make_shape(m, n, k, 1)); + ScaleConfig::tile_atom_to_shape_SFA(make_shape(m_sf, n, k, 1)); LayoutSFB layout_SFB = swap_ab ? ScaleConfig::tile_atom_to_shape_SFB(make_shape(n, m, k, 1)) : ScaleConfig::tile_atom_to_shape_SFB(make_shape(m, n, k, 1)); auto a_ptr = static_cast(a.data_ptr()); auto b_ptr = static_cast(b.data_ptr()); - auto a_scales_ptr = static_cast(a_scales.data_ptr()); + auto a_scales_ptr = static_cast(a_scales_data); auto b_scales_ptr = static_cast(b_scales.data_ptr()); typename GemmKernel::MainloopArguments mainloop_args{}; @@ -212,7 +235,9 @@ void cutlass_gemm_blockwise_sm100_fp8_dispatch(torch::stable::Tensor& out, constexpr int TILE_K = 128; // TODO: better heuristics - bool swap_ab = (m < 16) || (m % 4 != 0); + // swapAB only wins for small M; misaligned M >= 65 is faster on the + // non-swapAB path with padded activation scales (measured on GB300). + bool swap_ab = (m <= 64); bool use_tma_epilogue = (m * n) % 4 == 0; if (!swap_ab) { constexpr int TILE_N = 128; diff --git a/docker/Dockerfile b/docker/Dockerfile index d1a7afcbfb..ad010285b1 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -231,10 +231,12 @@ RUN --mount=type=cache,target=/opt/uv/cache \ echo "Installing Rubin prerelease requirements..." \ && /opt/venv/bin/python3 use_existing_torch.py --prefix \ && uv pip install --python /opt/venv/bin/python3 \ + --prerelease=allow --index-strategy unsafe-first-match \ -r /tmp/vllm-requirements/rubin-prerelease.txt \ && uv pip install --python /opt/venv/bin/python3 \ -r requirements/cuda.txt \ && uv pip install --python /opt/venv/bin/python3 \ + --prerelease=allow --index-strategy unsafe-first-match \ -r /tmp/vllm-requirements/rubin-prerelease.txt; \ elif [ "${PYTORCH_NIGHTLY}" = "1" ]; then \ echo "Installing torch nightly..." \ @@ -941,11 +943,13 @@ RUN --mount=type=cache,target=/opt/uv/cache \ fi && \ if [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ]; then \ uv pip install --system \ + --prerelease=allow --index-strategy unsafe-first-match \ -r /tmp/vllm-requirements/rubin-prerelease.txt \ && sed -Ei '/^[[:space:]]*(torch|torchvision|torchaudio)[[:space:]]*==/Id' \ /tmp/requirements-cuda.txt \ && uv pip install --system -r /tmp/requirements-cuda.txt \ && uv pip install --system \ + --prerelease=allow --index-strategy unsafe-first-match \ -r /tmp/vllm-requirements/rubin-prerelease.txt \ && printf '%s\n' \ "/usr/local/lib/python${PYTHON_VERSION}/dist-packages/nvpl/lib" \ @@ -1262,6 +1266,7 @@ RUN --mount=type=cache,target=/opt/uv/cache \ if [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ]; then \ echo "Installing Rubin prerelease requirements..." \ && uv pip install --system \ + --prerelease=allow --index-strategy unsafe-first-match \ -r /tmp/vllm-requirements/rubin-prerelease.txt; \ fi diff --git a/docker/Dockerfile.rock_base b/docker/Dockerfile.rock_base index 80b6aaca29..adde16af3c 100644 --- a/docker/Dockerfile.rock_base +++ b/docker/Dockerfile.rock_base @@ -6,7 +6,7 @@ ARG TORCHAUDIO_VERSION=2.11.0+rocm10.0.0 ARG TRITON_VERSION=3.8.0+git4cff872c.rocm10.0.0 ARG FA_BRANCH="0e60e394" ARG FA_REPO="https://github.com/Dao-AILab/flash-attention.git" -ARG AITER_BRANCH="v0.1.21.post1" +ARG AITER_BRANCH="v0.1.22.post1" ARG AITER_REPO="https://github.com/ROCm/aiter.git" ARG MORI_BRANCH="f95cd5e0ca7b730f92f9ffdfb358484d6965d9f7" # TheRock compatible v1.1.0 ARG MORI_REPO="https://github.com/ROCm/mori.git" diff --git a/docker/Dockerfile.rocm_base b/docker/Dockerfile.rocm_base index d842a7b07e..eb4bb5cc97 100644 --- a/docker/Dockerfile.rocm_base +++ b/docker/Dockerfile.rocm_base @@ -9,13 +9,13 @@ ARG PYTORCH_AUDIO_BRANCH="v2.11.0" ARG PYTORCH_AUDIO_REPO="https://github.com/pytorch/audio.git" ARG FA_BRANCH="0e60e394" ARG FA_REPO="https://github.com/Dao-AILab/flash-attention.git" -ARG AITER_BRANCH="v0.1.21.post2" +ARG AITER_BRANCH="v0.1.22.post1" ARG AITER_REPO="https://github.com/ROCm/aiter.git" ARG MORI_BRANCH="v1.1.0" ARG MORI_REPO="https://github.com/ROCm/mori.git" ARG ROCM_SYSTEMS_REPO="https://github.com/ROCm/rocm-systems.git" ARG ROCPROFILER_SDK_COMMIT="2b22ab0195cc1461cd9abf3b969e9dd7c10af350" -ARG ROCM_RUNTIME_COMMIT="b539bf7eebfd99ad0a69668caa1f4037034d501f" +ARG ROCM_RUNTIME_COMMIT="af70edc91f1dc5d228fd4c076df99e65e49c4521" # Sccache configuration (only used in release pipeline) ARG USE_SCCACHE diff --git a/docker/Dockerfile.s390x b/docker/Dockerfile.s390x index 9517ff3718..6faba540a9 100644 --- a/docker/Dockerfile.s390x +++ b/docker/Dockerfile.s390x @@ -169,8 +169,6 @@ ENV PKG_CONFIG_PATH="/opt/rh/gcc-toolset-14/root/usr/lib64/pkgconfig:/usr/local/ ENV PATH="${VIRTUAL_ENV:+${VIRTUAL_ENV}/bin}:/opt/rh/gcc-toolset-14/root/usr/bin:/usr/local/bin:$CARGO_HOME/bin:$RUSTUP_HOME/bin:$PATH" ENV PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL} ENV UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL} -# Force pure Python protobuf to avoid s390x C++ extension crashes -ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python COPY . /workspace/vllm WORKDIR /workspace/vllm @@ -214,10 +212,6 @@ RUN --mount=type=cache,target=/root/.cache/uv \ VLLM_TARGET_DEVICE=cpu VLLM_CPU_MOE_PREPACK=0 python setup.py bdist_wheel && \ uv pip install "$(echo dist/*.whl)[tensorizer]" -# Remove protobuf C++ extension that crashes on s390x -RUN rm -rf /opt/vllm/lib64/python${PYTHON_VERSION}/site-packages/google/_upb/*.so \ - /opt/vllm/lib64/python${PYTHON_VERSION}/site-packages/google/protobuf/pyext/*.so 2>/dev/null || true - # setup non-root user for vllm RUN umask 002 && \ /usr/sbin/useradd --uid 2000 --gid 0 vllm && \ diff --git a/docs/benchmarking/cli.md b/docs/benchmarking/cli.md index d46ab04b0e..e7ee794da6 100644 --- a/docs/benchmarking/cli.md +++ b/docs/benchmarking/cli.md @@ -147,7 +147,7 @@ With standard decoding, each streamed output usually contains one token, so ITL and TPOT are typically similar. With speculative decoding, one streamed output can contain multiple tokens, -such as several accepted draft tokens within a single engine tstep. ITL records +such as several accepted draft tokens within a single engine step. ITL records only the gaps between streamed outputs; it does not add zero-duration gaps for tokens in the same output. TPOT instead amortizes the request's decoding time over every output token. @@ -451,7 +451,7 @@ vllm bench serve \ --num-prompts -1 ``` -Available categories include `[high_entropy, mixed, low_entropy]`, where high entropy data contains unstructued data such as creative writing while low entropy data contains more structured data such as coding, more details are in the dataset card. +Available categories include `[high_entropy, mixed, low_entropy]`, where high entropy data contains unstructured data such as creative writing while low entropy data contains more structured data such as coding, more details are in the dataset card. #### BFCL (Tool-Calling) Benchmark diff --git a/docs/benchmarking/dashboard.md b/docs/benchmarking/dashboard.md index 44effc078e..4c1132c17f 100644 --- a/docs/benchmarking/dashboard.md +++ b/docs/benchmarking/dashboard.md @@ -36,7 +36,7 @@ When run, benchmark script generates results under **benchmark/results** folder, - `ON_CPU`: set the value to '1' on Intel® Xeon® and Arm® Neoverse™ Processors. Default value is 0. - `SERVING_JSON`: JSON file to use for the serving tests. Default value is empty string (use default file). - `LATENCY_JSON`: JSON file to use for the latency tests. Default value is empty string (use default file). -- `THROUGHPUT_JSON`: JSON file to use for the throughout tests. Default value is empty string (use default file). +- `THROUGHPUT_JSON`: JSON file to use for the throughput tests. Default value is empty string (use default file). - `REMOTE_HOST`: IP for the remote vLLM service to benchmark. Default value is empty string. - `REMOTE_PORT`: Port for the remote vLLM service to benchmark. Default value is empty string. - `PROMPTS_PER_CONCURRENCY`: Multiplier to compute `num_prompts` for serving tests (`num_prompts = max_concurrency × value`). Overrides JSON `num_prompts`. Default is NULL. diff --git a/docs/contributing/README.md b/docs/contributing/README.md index 7a1af9c1a7..e4b86f99ae 100644 --- a/docs/contributing/README.md +++ b/docs/contributing/README.md @@ -50,7 +50,7 @@ To rebuild only the Rust frontend binary: ./build_rust.sh --debug # faster build for development ``` -If you are developing vLLM's Python and CUDA/C++ code, install Pytorch first: +If you are developing vLLM's Python and CUDA/C++ code, install PyTorch first: ```bash uv pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu129 diff --git a/docs/design/metrics.md b/docs/design/metrics.md index 3d108890d7..3e070f5888 100644 --- a/docs/design/metrics.md +++ b/docs/design/metrics.md @@ -670,7 +670,7 @@ fall under the more general heading of "Observability". vLLM has support for OpenTelemetry tracing: - Added by and reinstated by -- Configured with `--oltp-traces-endpoint` and `--collect-detailed-traces` +- Configured with `--otlp-traces-endpoint` and `--collect-detailed-traces` - [OpenTelemetry blog post](https://opentelemetry.io/blog/2024/llm-observability/) - [User-facing docs](../../examples/observability/opentelemetry/README.md) - [Blog post](https://medium.com/@ronen.schaffer/follow-the-trail-supercharging-vllm-with-opentelemetry-distributed-tracing-aa655229b46f) diff --git a/docs/features/custom_logitsprocs.md b/docs/features/custom_logitsprocs.md index 232f4363ef..8c1c259399 100644 --- a/docs/features/custom_logitsprocs.md +++ b/docs/features/custom_logitsprocs.md @@ -8,84 +8,40 @@ A "custom" logits processor is written by a user of vLLM and is loaded into vLLM This document shows how to write, load and use a custom logits processor. +!!! note + This document covers the Model Runner V2 (MRV2) interface. The legacy V1 + model runner (`VLLM_USE_V2_MODEL_RUNNER=0`) uses a different interface, + `vllm.v1.sample.logits_processor.LogitsProcessor`, which is not covered + here. + ## Logits Processors Background A logits processor adjusts the next-token probability distribution, usually with the intention of steering the model towards a desired type of behavior. -In vLLM, logits processors operate at batch granularity. During a given engine step, the logits processor consumes a `(num_requests) x (vocab_size)` tensor of raw logits output by the model. For all requests which enable the logits processor, the logits processor applies a transformation to the corresponding row of the logits tensor, while leaving other rows unmodified. The transformed logits tensor is then passed to softmax. +In vLLM, logits processors operate at batch granularity. During a given engine step, the logits processor consumes a `(num_logits_rows) x (vocab_size)` tensor of raw logits output by the model. Note that a logits row is not a request: rows are reordered every step, and under speculative decoding a request owns one row per draft token. The logits processor applies a transformation to the rows of the logits tensor, while leaving other rows unmodified. The transformed logits tensor is then used for sampling. ## Creating a Custom Logits Processor -Custom logits processors must subclass `vllm.v1.sample.logits_processor.LogitsProcessor` and define (at minimum) the following methods: +Custom logits processors must subclass `vllm.v1.worker.gpu.sample.logits_processor.LogitsProcessor` and define (at minimum) the following methods: -* `validate_params(cls, sampling_params: SamplingParams)`: - * Raise `ValueError` if `SamplingParams` has invalid arguments (especially custom arguments) used by logits processor. - * When request is sent to entrypoint, `validate_params()` will validate `SamplingParams` and refuse request with invalid arguments. - * **Note:** it's important to implement `validate_params()` to prevent invalid parameters for custom logits processor. Otherwise requests with invalid parameters can cause unexpected behaviour in custom logits processor. - -* `__init__(self, vllm_config: VllmConfig, device: torch.device, is_pin_memory: bool)` +* `__init__(self, vllm_config: VllmConfig, req_states: LogitsProcRequestState)`: * `vllm_config`: engine configuration data structure - * `device`: hardware accelerator device info - * `is_pin_memory`: flag indicating whether pin memory is available to support logits processor implementation + * `req_states`: a narrow, read-only view of the persistent batch, exposing the on-device token history (`all_token_ids`, `prompt_len`, `prefill_len`, `total_len`) plus `device`, `max_num_reqs` and `vocab_size` -* `apply(self, logits: torch.Tensor) -> torch.Tensor`: - * Consume a `(num_requests) x (vocab_size)` logits tensor (`logits`) +* `add_request(self, req_idx, sampling_params) -> bool`: + * Initialize per-slot state for a request entering the batch. Slots are recycled through a free list, so per-slot state must be fully (re)initialized here; there is no removal hook, since freed slots are never read + * Return whether this processor modifies logits for the request. The sampler skips the logits-processing pipeline for batches in which no request needs it + +* `apply_staged_writes(self) -> None` (optional): + * Flush host-side writes staged by `add_request()` to the device; called once per step before the forward pass + +* `validate_params(cls, sampling_params) -> None` (optional classmethod): + * Raise `ValueError` for invalid per-request arguments (especially custom arguments); runs at request admission, so invalid arguments fail the request instead of reaching the sampler + +* `apply(self, logits: torch.Tensor, ctx: LogitsContext) -> torch.Tensor`: + * Consume a `(num_logits_rows) x (vocab_size)` logits tensor and the step's batch layout (`ctx`: row-to-slot mappings, `input_ids`, positions) * Apply logits processor transformation at batch granularity - * Return a transformed `(num_requests) x (vocab_size)` logits tensor - * You can modify the input logits processors in-place or out-of-place; in-place is more memory-efficient - -* `is_argmax_invariant(self) -> bool`: - * Return `True` if the logits processor is argmax invariant (never changes what is the highest-logit-value token ID for a given request), `False` if the logits processor may modify argmax - * `is_argmax_invariant()` is evaluated once at startup; if `True`, vLLM will skip applying this logits processor in a given step when all requests use greedy sampling - -* `update_state(self, batch_update: Optional["BatchUpdate"]) -> None`: - * Consume a `BatchUpdate` data structure representing persistent batch state changes at the beginning of the current engine step - * Use the `BatchUpdate` members to update logits processor internal state - * **Note:** batch update data structure may be `None`, signaling no change to the batch constituents. In this case, the LogitsProcessor might still want to update its state based on the updated `output_token_ids` lists that it could have retained when they were added. - -### How the vLLM engine builds the `BatchUpdate` data structure - -!!! important - Some logits processors design changes are still in progress. We expect - that in the future you will not need to account for batch state changes - when implementing a logits processor, and the information in this section - will become irrelevant. - -Logits processor `update_state()` implementations should assume the following model for how the model runner updates persistent batch state (expressed here in terms of the `BatchUpdate` abstraction): - -1. Identify indices of requests which finished in the current engine step - -2. Identify new requests introduced in the current step - -3. Use Add operations to replace as many finished requests with new requests, in order of increasing index of the replaced request starting with the lowest index - -4. Based on the relative number of new and finished requests: - - 1. If the numbers of new and finished requests are the same, proceed to next step - - 2. *If there are more new requests than finished requests:* apply Add operations to extend the batch with the remaining new requests which did not replace finished requests. Assign consecutive indices to these new requests, starting with `current_max_batch_index + 1` - - 3. *If there are fewer new requests than finished requests:* - - * Apply Remove operations to finished requests which were not replaced with new requests. These removed request indices will necessarily be greater than the greatest index of the finished requests which were replaced in the previous step. The Removes may leave the batch in a non-contiguous state - - * **"Condense" the batch to be contiguous:** starting with the lowest-index empty slot (which was caused by a Remove), apply a Unidirectional Move from the current highest non-empty slot in the batch to fill the empty slot. Proceed with additional Unidirectional Move operations in order of increasing empty slot destination index and decreasing non-empty slot source index until the batch is contiguous - - * **Shrink the batch:** a side effect of condensing the batch is that empty slots resulting from Remove operations are grouped in a contiguous block at the end of the batch array. Thus, after condensing, update `BatchUpdate.batch_size` to reflect the number of non-empty slots - -5. Reorder the batch for improved efficiency. Depending on the attention backend implementation and the current characteristics of the batch, zero or more Swap Move operations may be applied to reorder the batch - -Notes: - -* A logits processor `update_state()` method must process batch update operations in the following order: removes, adds, moves - -* The index argument for Add operations refers to the index *at the time the Add occurred*, i.e. before any Move operations - * Example: if a request is Added at index 5 and then swapped with index 3, the Add operation in `BatchUpdate.added` will be associated with index 5 not 3 - * In other words Move operations can be assumed to be applied after Adds and Removes - -* Move operations can be assumed to be applied in the order in which they appear in `BatchUpdate.moved` - -* If there are no new/finished requests and there is no batch reordering, then the batch update for the logits processors will be `None` + * Return a transformed logits tensor. You can modify the input logits in-place or out-of-place; in-place is more memory-efficient ### Passing Custom Argument to a Custom Logits Processor @@ -93,7 +49,7 @@ Unlike built-in logits processors, custom logits processors may require configur ### Example Custom Logits Processor Implementation -The contrived example below implements a custom logits processor which consumes a `(num\_requests) \times (vocab\_size)` logits tensor and masks out all tokens except for one (`target_token`) with `float(-inf)`. The logits processor is disabled for any request that does not specify `target_token`. To determine whether the logits processor is enabled and which token to leave unmasked, the logits processor checks `SamplingParams.extra_args` for a `target_token` custom argument associated with each request: +The contrived example below implements a custom logits processor which masks out all tokens except for one (`target_token`) for the requests that enable it. The processor is disabled for any request that does not specify `target_token`; `add_request()` reports this via its return value, so the sampler can skip the logits-processing pipeline entirely when no request enables the processor. To determine whether the logits processor is enabled and which token to leave unmasked, the processor checks `SamplingParams.extra_args` for a `target_token` custom argument associated with each request: ??? code "Example custom logits processor definition" @@ -101,193 +57,55 @@ The contrived example below implements a custom logits processor which consumes import torch from vllm.config import VllmConfig from vllm.sampling_params import SamplingParams - from vllm.v1.sample.logits_processor import (BatchUpdate, - LogitsProcessor, - MoveDirectionality) - - class DummyLogitsProcessor(LogitsProcessor): - """Fake logit processor to support unit testing and examples""" - - @classmethod - def validate_params(cls, params: SamplingParams): - target_token: int | None = params.extra_args and params.extra_args.get( - "target_token" - ) - if target_token is not None and not isinstance(target_token, int): - raise ValueError(f"target_token value {target_token} is not int") - - def __init__(self, vllm_config: "VllmConfig", device: torch.device, - is_pin_memory: bool): - self.req_info: dict[int, int] = {} - - def is_argmax_invariant(self) -> bool: - """Never impacts greedy sampling""" - return False - - def update_state(self, batch_update: BatchUpdate | None): - if not batch_update: - return - - # Process added requests. - for index, params, _, _ in batch_update.added: - assert params is not None - self.validate_params(params) - if params.extra_args and (target_token := - params.extra_args.get("target_token")): - self.req_info[index] = target_token - else: - self.req_info.pop(index, None) - - if self.req_info: - # Process removed requests. - for index in batch_update.removed: - self.req_info.pop(index, None) - - # Process moved requests, unidirectional move (a->b) and swap - # (a<->b) - for adx, bdx, direct in batch_update.moved: - a_val = self.req_info.pop(adx, None) - b_val = self.req_info.pop(bdx, None) - if a_val is not None: - self.req_info[bdx] = a_val - if direct == MoveDirectionality.SWAP and b_val is not None: - self.req_info[adx] = b_val - - def apply(self, logits: torch.Tensor) -> torch.Tensor: - if not self.req_info: - return logits - - # Save target values before modification - cols = torch.tensor( - list(self.req_info.values()), dtype=torch.long, device=logits.device - ) - rows = torch.tensor( - list(self.req_info.keys()), dtype=torch.long, device=logits.device - ) - values_to_keep = logits[rows, cols].clone() - - # Mask all but target tokens - logits[rows] = float('-inf') - logits[rows, cols] = values_to_keep - - return logits - - ``` - -In the rest of this document, we will use `DummyLogitsProcessor` as an example of a custom logits processor. - -The `DummyLogitsProcessor.update_state()` implementation maintains a "sparse" representation of the batched requests in the `self.req_info` dictionary: only those requests which specify a `target_token` value have a key in the dictionary. `update_state()` adjusts the stored request indices and `target_token` values (keys and values respectively in `self.req_info`) in response to Add, Remove and Move operations against the persistent batch. - -### Wrapping an Existing Request-Level Logits Processor - -Although the vLLM engine applies logits processors at batch granularity, some users may want to use vLLM with a "request-level" logits processor implementation - an implementation which operates on individual requests. This will be especially true if your logits processor was developed for vLLM version 0, which required it to be a `Callable` (as described [here][vllm.logits_process]) conforming to the following type annotation: - -``` python -RequestLogitsProcessor = Union[ - - # (output token ids, logits tensor) -> logits tensor - Callable[[list[int], Tensor], Tensor], - - # (prompt token ids, output token ids, logits tensor) -> logits tensor - Callable[[list[int], list[int], Tensor], Tensor], -] -``` - -While request-level logits processors are explicitly *not* supported in the vLLM engine, vLLM *does* provide a convenient process to wrap an existing `Callable` request-level logits processor and create a batch-level logits processor that is compatible with vLLM. The `Callable` must conform to the type annotation above; if your request-level logits processor has a different interface, then in order to wrap it, you may need to modify it or implement an additional wrapper layer to comply with the interface specification above. - -You can wrap the request-level logits processor by subclassing `AdapterLogitsProcessor` as shown in the example below (in this example, `DummyPerReqLogitsProcessor` is a stand-in for your request-level logits processor which needs to be wrapped.): - -* Override `AdapterLogitsProcessor.validate_params(cls,params)` to validate request's sampling parameters. - -* Override `AdapterLogitsProcessor.is_argmax_invariant(self)` to accurately reflect whether your request-level logits processor may impact which token has the highest-value logit. - -* Override `AdapterLogitsProcessor.new_req_logits_processor(self,params)` to create a new request-level logits processor instance from a `SamplingParams` instance: - -??? code "Example of Wrapping a Request-Level Logits Processor" - - ``` python - ... - - from vllm.v1.sample.logits_processor import ( - AdapterLogitsProcessor, # Wrapper base-class - RequestLogitsProcessor, # Request-level logitsproc type annotation + from vllm.v1.worker.gpu.sample.logits_processor import ( + LogitsProcRequestState, + LogitsContext, + LogitsProcessor, ) - ... - # Stand-in for your request-level logits processor: - class DummyPerReqLogitsProcessor: - """The request-level logits processor masks out all logits except the - token id identified by `target_token`""" + class TargetTokenLogitsProcessor(LogitsProcessor): + """Masks out all tokens except `target_token` (a per-request custom + argument); requests without it are left alone.""" - def __init__(self, target_token: int) -> None: - """Specify `target_token`""" - self.target_token = target_token + def __init__(self, vllm_config: "VllmConfig", req_states: LogitsProcRequestState): + # Per-slot target; -1 means disabled. Staged on the host and + # flushed to the device once per step in apply_staged_writes(). + self.target_token = torch.full( + (req_states.max_num_reqs,), -1, dtype=torch.int64 + ) + self.target_token_dev = torch.full( + (req_states.max_num_reqs,), -1, dtype=torch.int64, device=req_states.device + ) - def __call__( - self, - output_ids: list[int], - logits: torch.Tensor, - ) -> torch.Tensor: - val_to_keep = logits[self.target_token].item() - logits[:] = float("-inf") - logits[self.target_token] = val_to_keep + def add_request(self, req_idx: int, sampling_params: SamplingParams) -> bool: + target = (sampling_params.extra_args or {}).get("target_token") + self.target_token[req_idx] = target if target is not None else -1 + return target is not None + + def apply_staged_writes(self) -> None: + self.target_token_dev.copy_(self.target_token, non_blocking=True) + + def apply(self, logits: torch.Tensor, ctx: LogitsContext) -> torch.Tensor: + # ctx.expanded_idx_mapping maps each logits row to its request slot. + cols = self.target_token_dev[ctx.expanded_idx_mapping.long()] + rows = torch.nonzero(cols >= 0).squeeze(1) + if rows.numel() == 0: + return logits + kept = logits[rows, cols[rows]].clone() + logits[rows] = float("-inf") + logits[rows, cols[rows]] = kept return logits - - ... - - # Example of wrapping the request-level logits processor: - class WrappedPerReqLogitsProcessor(AdapterLogitsProcessor): - """Example of wrapping a fake request-level logit processor to create a - batch-level logits processor""" - - @classmethod - def validate_params(cls, params: SamplingParams): - target_token: Any | None = params.extra_args and params.extra_args.get( - "target_token" - ) - if target_token is not None and not isinstance(target_token, int): - raise ValueError( - f"target_token value {target_token} is not int" - ) - - def is_argmax_invariant(self) -> bool: - return False - - def new_req_logits_processor( - self, - params: SamplingParams, - ) -> Optional[RequestLogitsProcessor]: - """This method returns a new request-level logits processor, customized - to the `target_token` value associated with a particular request. - - Returns None if the logits processor should not be applied to the - particular request. To use the logits processor the request must have - a "target_token" custom argument with an integer value. - - Args: - params: per-request sampling params - - Returns: - `Callable` request logits processor, or None - """ - target_token: Any | None = params.extra_args and params.extra_args.get( - "target_token" - ) - if target_token is None: - return None - return DummyPerReqLogitsProcessor(target_token) ``` -!!! note - Your `new_req_logits_processor()` override can return `None` to signal that the wrapped logits processor should not be applied to the request in question. - -Once you have created a custom subclass (like `WrappedPerReqLogitsProcessor`) which wraps your request level logits processor, you can pass the custom subclass to vLLM via any of the methods described in the following section. +Per-request state is keyed by the request slot index, and slots are recycled through a free list. The example keeps a per-slot state where `-1` means disabled; note that `add_request()` overwrites the slot's entry unconditionally, which is what keeps recycled slots from leaking state between requests. ## Ways to Load Your Custom Logits Processor in vLLM Logits processors are loaded at initialization. Critically, the set of loaded logits processors cannot be modified after the vLLM engine finishes loading, and new logits processors cannot be loaded on-demand for individual requests. +Loaded classes are validated against the Model Runner V2 interface; a class that does not subclass `vllm.v1.worker.gpu.sample.logits_processor.LogitsProcessor` is rejected with a clear error. + This section details different ways of making your logits processor visible to vLLM and triggering vLLM to load your logits processor. ### Method 1: Pass the Custom Logits Processor Fully-Qualified Class Name (FQCN) to vLLM at Initialization Time @@ -314,7 +132,7 @@ See examples below: # Pass in FQCN llm = LLM( model="facebook/opt-125m", - logits_processors=["your.module.path:DummyLogitsProcessor"], + logits_processors=["your.module.path:TargetTokenLogitsProcessor"], ) ``` @@ -323,14 +141,14 @@ See examples below: ``` python # Pass in FQCN engine_args = AsyncEngineArgs(model="facebook/opt-125m", - logits_processors=["your.module.path:DummyLogitsProcessor"]) + logits_processors=["your.module.path:TargetTokenLogitsProcessor"]) async_llm = AsyncLLM.from_engine_args(engine_args) ``` ??? code "Passing custom logits processor FQCN to vLLM server via CLI" ```bash - vllm serve facebook/opt-125m --logits_processors your.module.path:DummyLogitsProcessor + vllm serve facebook/opt-125m --logits_processors your.module.path:TargetTokenLogitsProcessor ``` ### Method 2: Automatically Detect Custom Logits Processors Installed in Your Python Environment As Entry Points @@ -345,7 +163,7 @@ Suppose that you have developed a Python package that holds your custom logits p ``` toml [project.entry-points."vllm.logits_processors"] - dummy_logits_processor = "your.module.path:DummyLogitsProcessor" + target_token_logits_processor = "your.module.path:TargetTokenLogitsProcessor" ``` Once your package is installed, your custom logits processor will be loaded automatically whenever vLLM is initialized. You do *not* need to pass the custom logits processor to the `LLM` or `AsyncLLM` constructors or to the vLLM server explicitly at initialization time if your logits processor is exposed as an entry point. @@ -361,26 +179,26 @@ You can pass one or more custom logits processor class objects to the `LLM` and ``` python # Import custom logits processor - from some.module import DummyLogitsProcessor + from some.module import TargetTokenLogitsProcessor # ...or... # Define custom logits processor locally - from vllm.v1.sample.logits_processor import LogitsProcessor + from vllm.v1.worker.gpu.sample.logits_processor import LogitsProcessor - class DummyLogitsProcessor(LogitsProcessor): - # See DummyLogitsProcessor implementation above + class TargetTokenLogitsProcessor(LogitsProcessor): + # See TargetTokenLogitsProcessor implementation above ... # Pass class object to LLM constructor llm = LLM( model="facebook/opt-125m", - logits_processors=[DummyLogitsProcessor], + logits_processors=[TargetTokenLogitsProcessor], ) # Pass class object to AsyncLLM constructor engine_args = AsyncEngineArgs(model="facebook/opt-125m", - logits_processors=[DummyLogitsProcessor]) + logits_processors=[TargetTokenLogitsProcessor]) async_llm = AsyncLLM.from_engine_args(engine_args) ``` @@ -388,7 +206,7 @@ You can pass one or more custom logits processor class objects to the `LLM` and The design of the custom logits processor determines whether the logits processor must be enabled/disabled for a given request, and what arguments must be provided to configure the logits processor. -The examples below show how a user would pass a custom argument (`target_token`) to `DummyLogitsProcessor` in order to (1) enable the logits processor for that particular request and (2) control the logits processor's behavior. +The examples below show how a user would pass a custom argument (`target_token`) to `TargetTokenLogitsProcessor` in order to (1) enable the logits processor for that particular request and (2) control the logits processor's behavior. ??? code "vLLM REST API: configure custom logits processor for a request" @@ -419,7 +237,7 @@ The examples below show how a user would pass a custom argument (`target_token`) ??? code "Offline: configure custom logits processor for an `LLM` request" ``` python - outputs_logitproc = llm.generate("your prompt", + outputs_logitproc = llm.generate("your prompt", SamplingParams(..., extra_args={"target_token": 67})) ``` @@ -438,31 +256,14 @@ The examples below show how a user would pass a custom argument (`target_token`) ## Best Practices for Writing Custom Logits Processors -Once vLLM loads a logits processor during initialization, then vLLM will invoke `update_state()` and `apply()` against that logits processor in every engine step. Both methods operate on all requests which currently reside in the vLLM persistent batch. Thus, it is important to implement these methods efficiently. - -* Write efficient `apply()` and `update_state()` implementations in light of the fact that logits processors operate at batch granularity - * For example, you may be able to use efficient vectorized operations to implement `apply()` or update internal state vectors in `update_state()` - * However, if you think that a logits processor may be used infrequently, it may be appropriate to use a "sparse" representation of request state i.e. the class can represent request configuration using a dictionary which only stores metadata about requests that enable the logits processor - * **Note:** wrapped request-level logits processors do not need to implement `apply()` and `update_state()`; the default `AdapterLogitsProcessor.update_state()` implementation maintains a sparse representation of request state, wherein requests for which `new_req_logits_processor()` returns `None` are not represented in the base-class state dictionary. The default implementation of `AdapterLogitsProcessor.apply()` applies the request-level logits processor to each row of input logits sequentially and assembles the output logits tensor. If the performance of this `AdapterLogitsProcessor` default implementation is insufficient, then avoid wrapping your request-level logits processor and instead re-implement it as a `LogitsProcessor` subclass with optimized `apply()` and `update_state()` implementations that operate at batch granularity +* Write an efficient `apply()` implementation in light of the fact that logits processors operate at batch granularity. For example, you may be able to use efficient vectorized operations to implement `apply()`, and to stage per-request state into contiguous tensors in `add_request()` instead of per-row Python loops * It is up to the logits processor author to determine: - 1. **The per-request attributes which configure the logits processor's behavior against that request.** Your custom logits processor's `update_state()` override determines how `SamplingParams` fields are mapped into logits processor state + 1. **The per-request attributes which configure the logits processor's behavior against that request.** Your custom logits processor's `add_request()` override determines how `SamplingParams` fields are mapped into logits processor state - * **Note:** for wrapped request-level logits processors, `new_req_logits_processor()` determines how `SamplingParams` fields are used to initialize a request-level logits processor instance. + 2. **The conditions under which the logits processor is or is not enabled on a per-request basis.** Unless your intention is for the custom logits processor to act on all requests all the time, you should write your logits processor in such a way that it is possible to disable the logits processor for a given request, i.e. by defaulting an argument to `None` or by passing in a specific do-nothing argument value i.e. `0.0`. Return `False` from `add_request()` for such requests so that the sampler can skip the logits-processing pipeline for batches in which no request needs it - 2. **The conditions under which the logits processor is or is not enabled on a per-request basis.** Unless your intention is for the custom logits processor to act on all requests all the time, you should write your logits processor in such a way that it is possible to disable the logits processor for a given request, i.e. by defaulting an argument to `None` or by passing in a specific do-nothing argument value i.e. `0.0`. Try to save compute and memory for requests which disable the logits processor +* Stage host-side writes in `add_request()` and flush them once per step in `apply_staged_writes()`, instead of writing device tensors directly; this is the discipline every built-in sampler state follows - * **Note:** for wrapped per-request logits processors, the default `AdapterLogitsProcessor.update_state()` implementation ensures that the request-level logits processor is disabled when `new_req_logits_processor()` returns `None` for that request - - 3. **The conditions under which the logits processor is short-circuited at the batch level.** Even if you have defined a way to disable the custom logits processor at the request level, it may be difficult to translate this into compute savings i.e. if your `update_state()` and `apply()` implementations use efficient vectorized implementations that operate on the whole persistent batch in a single command. For example, you cannot skip an entire vectorized operation in `apply()` just because one request disabled the logits processor. To save compute in the edge-case where no running requests utilize the custom logits processor, we recommend designing `apply()` to return the unmodified input tensor if all requests have the logits processor disabled. Similarly, consider whether steps can be skipped in `update_state()` if no requests enable the logits processor - - * Additionally, an easy way to save compute in `update_state()` is to exit early when the `batch_update` is `None` - - * **Note:** for wrapped per-request logits processors, the `AdapterLogitsProcessor` base-class implements the above optimizations by default - -* Ensure that the logits processor `update_state` method discards information about finished requests (i.e. requests which are replaced by an Add or which are subject to a Remove) - - * **Note:** for wrapped per-request logits processors, the `AdapterLogitsProcessor` base-class handles this by default - -* `is_argmax_invariant()` can be hard-coded to `True` or `False` if the logits processor has consistent behavior. However, the argmax invariance may also be determined programmatically (i.e. if your logits processor is user-customizable in some way that impacts whether the logits processor is argmax invariant). For this reason, `is_argmax_invariant()` is not a class method +* Since there is no removal hook, `add_request()` must fully (re)initialize the slot's state; do not rely on state left behind by the slot's previous occupant diff --git a/docs/features/nixl_connector_usage.md b/docs/features/nixl_connector_usage.md index 9799eb9cf1..0d1f16e2aa 100644 --- a/docs/features/nixl_connector_usage.md +++ b/docs/features/nixl_connector_usage.md @@ -418,12 +418,13 @@ activity for the last reporting interval. Example output: ```text KV Transfer metrics: Num successful transfers=4, Avg xfer time (ms)=1.381, P90 xfer time (ms)=2.601, Avg post time (ms)=0.672, P90 post time (ms)=0.801, -Avg MB per transfer=2.25, Throughput (MB/s)=1629.549, Avg number of descriptors=72.0 +Avg MB per transfer=2.25, Throughput (MB/s)=1629.549, Avg number of descriptors=72.0, +Num failed transfers=0, Num KV expired reqs=0 ``` The table below describes each field. All timing values cover only the -successful transfers recorded in the current interval; failed transfers are -counted separately via Prometheus (see +successful transfers recorded in the current interval; failures are reported +as interval counts in the same line and via Prometheus (see [Prometheus metrics](#prometheus-metrics) below). | Metric | Unit | Description | @@ -436,6 +437,8 @@ counted separately via Prometheus (see | `Avg MB per transfer` | MB | Mean payload size per transfer, computed as `total bytes transferred / number of transfers`. Reflects the average KV cache footprint of a single request (sequence length × layers × head dimension × dtype bytes). | | `Throughput (MB/s)` | MB/s | Effective bandwidth over the interval: `total MB transferred / total xfer time (s)` across all successful transfers. This is aggregate throughput, not per-request bandwidth. | | `Avg number of descriptors` | count | Mean number of NIXL memory descriptors (scatter-gather segments) submitted per transfer. More descriptors indicate more fragmented or larger KV cache allocations; very high counts can increase descriptor-registration overhead. | +| `Num failed transfers` | count | Failed NIXL transfers, handshakes, and completion notifications (`send_notif`) in the interval, grouped because all are sporadic lower-transport-layer events. | +| `Num KV expired reqs` | count | Requests whose KV blocks expired before being read. Reported separately from the failure counts above because it is an autoscaler/lease-tuning signal rather than a transport issue. | ### Prometheus metrics @@ -448,9 +451,9 @@ exported when NixlConnector is active: | `vllm:nixl_post_time_seconds` | Histogram | Time to submit the transfer request to the RDMA backend (seconds). | | `vllm:nixl_bytes_transferred` | Histogram | Bytes moved per transfer. | | `vllm:nixl_num_descriptors` | Histogram | Descriptor count per transfer. | -| `vllm:nixl_num_failed_transfers` | Counter | Cumulative count of failed NIXL KV-block transfers. | -| `vllm:nixl_num_failed_notifications` | Counter | Cumulative count of failed completion notifications (`send_notif`). | -| `vllm:nixl_num_kv_expired_reqs` | Counter | Requests whose KV blocks expired on the prefiller before the decoder read them (tracked on the P instance). | +| `vllm:nixl_num_failed_transfers` | Counter | Cumulative count of failed NIXL KV-block transfers, including handshake and notification (`send_notif`) failures. These are grouped because all are sporadic lower-transport-layer events. | +| `vllm:nixl_num_failed_notifications` | Counter | Cumulative count of failed notifications (`send_notif`), retained for compatibility. These failures are also included in `vllm:nixl_num_failed_transfers`; do not sum the two counters. | +| `vllm:nixl_num_kv_expired_reqs` | Counter | Requests whose KV blocks expired on the prefiller before the decoder read them (tracked on the P instance). Kept separate from the failure counters above: KV expiry is an autoscaler/lease-tuning signal, not a transport issue. | !!! tip High `vllm:nixl_num_kv_expired_reqs` indicates that the prefiller's lease diff --git a/docs/features/quantization/quark.md b/docs/features/quantization/quark.md index 8445288ff9..e5cfb8653b 100644 --- a/docs/features/quantization/quark.md +++ b/docs/features/quantization/quark.md @@ -100,7 +100,7 @@ kv-cache and the quantization algorithm is AutoSmoothQuant. !!! note Note the quantization algorithm needs a JSON config file and the config file is located in - [Quark Pytorch examples](https://quark.docs.amd.com/latest/pytorch/pytorch_examples.html), + [Quark PyTorch examples](https://quark.docs.amd.com/latest/pytorch/pytorch_examples.html), under the directory `examples/torch/language_modeling/llm_ptq/models`. For example, AutoSmoothQuant config file for Llama is `examples/torch/language_modeling/llm_ptq/models/llama/autosmoothquant_config.json`. diff --git a/docs/features/sleep_mode.md b/docs/features/sleep_mode.md index dcb6ad8c94..2616f374ea 100644 --- a/docs/features/sleep_mode.md +++ b/docs/features/sleep_mode.md @@ -74,6 +74,18 @@ llm.wake_up(tags=["weights"]) llm.wake_up(tags=["kv_cache"]) ``` +#### Release only KV cache memory + +`LLM.release_kv_cache_memory()` discards KV cache physical memory while keeping model weights resident. It requires a completed pause and all executor memory to be resident: full sleep, partial wake-up, and repeated release without restoring memory are rejected. Requests retained with `mode="keep"` are recomputed after wake-up. + +Use the default `cumem` backend with managed allocations: `enable_cumem_allocator=True` on CUDA/ROCm (also enabled by `enable_sleep_mode=True`), or `enable_sleep_mode=True` on XPU. Other backends must implement selective discard; CPU and unmanaged allocations are unsupported. + +```python +llm.sleep(level=0, mode="keep") # Wait for the pause to complete. +llm.release_kv_cache_memory() +llm.wake_up(tags=["kv_cache"]) # Reallocate KV cache and resume scheduling. +``` + ### Online Serving To enable sleep mode in a vLLM server you need to initialize it with the flag `VLLM_SERVER_DEV_MODE=1` and pass `--enable-sleep-mode` to the vLLM server. @@ -107,9 +119,18 @@ curl -X POST 'http://localhost:8000/collective_rpc' -H 'Content-Type: applicatio curl -X POST 'http://localhost:8000/wake_up?tags=kv_cache' ``` +To release only KV cache memory, wait for the level 0 pause to complete before calling the release endpoint. The same resident-memory and backend requirements as the Python API apply. + +```bash +curl -X POST 'http://localhost:8000/sleep?level=0&mode=keep' +curl -X POST 'http://localhost:8000/release_kv_cache_memory' +curl -X POST 'http://localhost:8000/wake_up?tags=kv_cache' +``` + #### HTTP endpoints - `POST /sleep?level=1` — Put the model to sleep (`level=1`). +- `POST /release_kv_cache_memory` — Discard KV cache memory after a completed pause, while all executor memory is resident. - `POST /wake_up` — Wake up the model. Supports optional `tags` query parameters for partial wake-up (e.g., `?tags=weights`). - `POST /collective_rpc` — Perform a collective remote procedure call (RPC). - `GET /is_sleeping` — Check if the model is sleeping. diff --git a/docs/features/speculative_decoding/draft_model.md b/docs/features/speculative_decoding/draft_model.md index 5ab4629409..880ea1df59 100644 --- a/docs/features/speculative_decoding/draft_model.md +++ b/docs/features/speculative_decoding/draft_model.md @@ -96,7 +96,7 @@ The code used to request completions as a client remains unchanged: }, gpu_memory_utilization=0.5, ) -outputs = llm.generate(prompts,sampling_params) +outputs = llm.generate(prompts, sampling_params) for output in outputs: prompt = output.prompt diff --git a/docs/features/tool_calling.md b/docs/features/tool_calling.md index 9628bc431a..fb70c50248 100644 --- a/docs/features/tool_calling.md +++ b/docs/features/tool_calling.md @@ -109,18 +109,22 @@ vLLM supports the `tool_choice='none'` option in the chat completion API. When t ## Constrained Decoding Behavior -Whether vLLM enforces the tool parameter schema during generation depends on the `tool_choice` mode and the per-tool `strict` field: +Structural-tag parsers resolve call obligation, grammar activation, and argument-schema enforcement separately. With structural-tag enforcement enabled: -| `tool_choice` value | Schema-constrained decoding | Behavior | +| `tool_choice` value | Call obligation | Structural-tag activation | | --- | --- | --- | -| Named function | Yes (via structured outputs backend) | Arguments are guaranteed to be valid JSON conforming to the function's parameter schema. | -| `"required"` | Yes (via structured outputs backend) | Same as named function. The model must produce at least one tool call. | -| `"auto"` | Only when `strict: true` is set on at least one tool | Structural-tag parsers constrain tool-call arguments when a tool opts in with `strict: true`. Without it, the model generates freely and tool calls are extracted from raw text. | -| `"none"` | N/A | No tool calls are produced. | +| Named function | Call the selected function | Always | +| `"required"` | Produce at least one tool call | Always | +| `"auto"` | Tool calls are optional | When at least one tool sets `strict: true`, or `--tool-strict-level` is `function` or `parameter` | +| `"none"` | Tool calling is disabled | Disabled | + +When a structural tag applies, each tool's declared parameter schema is enforced only when that tool sets `strict: true` or the server uses `--tool-strict-level parameter`. Tools with omitted or false `strict` receive broad argument-syntax constraints at levels `auto` and `function`, including for required and named calls. + +For parsers using schema-derived JSON constraints, required and named calls continue to enforce the declared parameter schemas. ### Strict Mode -For `tool_choice="required"` or named function calling, structural-tag constraints are always applied regardless of the `strict` field. For `tool_choice="auto"`, setting `strict: true` on at least one tool opts in to structural-tag constraints; without it, the model generates freely and tool calls are extracted from raw text. The `strict` field is supported across all three API surfaces: Chat Completion, Responses, and Anthropic Messages. +Structural-tag constraints have two layers. The tool-call *envelope* (markup and function name) is constrained whenever a structural tag applies: always for `tool_choice="required"` and named function calling, and for `tool_choice="auto"` once at least one tool sets `strict: true` or the server raises the floor. The *argument schema* of an individual tool is pinned only when that tool sets `strict: true` (or under `--tool-strict-level parameter`); a tool that omits `strict` keeps its arguments unconstrained, even when another tool in the same request is strict. The `strict` field is supported across all three API surfaces: Chat Completion, Responses, and Anthropic Messages. For best compatibility with strict schema enforcement, define tool parameter schemas in the OpenAI strict-schema style: @@ -134,6 +138,22 @@ vLLM also provides a global toggle via the `VLLM_ENFORCE_STRICT_TOOL_CALLING` en VLLM_ENFORCE_STRICT_TOOL_CALLING=false vllm serve ... ``` +### Server-Side Strictness Floor + +Most OpenAI-compatible clients and agent frameworks never set `strict` on their tools, so with `tool_choice="auto"` the model generates tool calls without any grammar and malformed markup can leak into the response. The `--tool-strict-level` option lets the server operator raise the floor for every request that carries tools, independently of what the client declares: + +| Value | Behavior | +| --- | --- | +| `auto` (default) | Follow the request's tool choice and per-tool strictness. Required/named choices activate structural tags; `tool_choice="auto"` activates them when at least one tool sets `strict: true`. | +| `function` | Constrain the tool-call envelope (markup and the function name) for every request with tools, leaving argument contents free unless the client marked the tool `strict: true`. | +| `parameter` | Additionally pin argument schemas for every tool, as if every tool had `strict: true`. | + +```bash +vllm serve ... --tool-strict-level function +``` + +The floor never relaxes a constraint the request would already receive: tools the client marked `strict: true` keep their schemas at every level. With `tool_choice="auto"`, the grammar does not force a tool call; a plain text response stays valid. `VLLM_ENFORCE_STRICT_TOOL_CALLING=false` disables structural tags entirely and takes precedence over this option. + ## Automatic Function Calling To enable this feature, you should set the following flags: @@ -152,7 +172,7 @@ from HuggingFace; and you can find an example of this in a `tokenizer_config.jso If your favorite tool-calling model is not supported, please feel free to contribute a parser & tool use chat template! !!! note - With `tool_choice="auto"`, schema-level constraint requires both `VLLM_ENFORCE_STRICT_TOOL_CALLING=true` (the default) and at least one tool with `strict: true`. When these conditions are met and the selected parser supports structural tags, vLLM constrains tool-call arguments. Otherwise, vLLM extracts tool calls from raw text, so arguments may occasionally be malformed or violate the function's parameter schema. + With `tool_choice="auto"`, structural-tag constraints require both `VLLM_ENFORCE_STRICT_TOOL_CALLING=true` (the default) and at least one tool with `strict: true`, or a server-side floor set via `--tool-strict-level`. When these conditions are met and the selected parser supports structural tags, vLLM constrains the tool-call envelope and pins the argument schema of each tool that sets `strict: true` (or of every tool under `--tool-strict-level parameter`). Otherwise, vLLM extracts tool calls from raw text, so arguments may occasionally be malformed or violate the function's parameter schema. ### Hermes Models (`hermes`) @@ -162,8 +182,8 @@ All Nous Research Hermes-series models newer than Hermes 2 Pro should be support * `NousResearch/Hermes-2-Theta-*` * `NousResearch/Hermes-3-*` -_Note that the Hermes 2 **Theta** models are known to have degraded tool call quality and capabilities due to the merge -step in their creation_. +*Note that the Hermes 2 **Theta** models are known to have degraded tool call quality and capabilities due to the merge +step in their creation*. Flags: `--tool-call-parser hermes` diff --git a/docs/getting_started/installation/cpu.md b/docs/getting_started/installation/cpu.md index 539c28d709..53b8eb596d 100644 --- a/docs/getting_started/installation/cpu.md +++ b/docs/getting_started/installation/cpu.md @@ -150,7 +150,7 @@ VLLM_USE_PRECOMPILED=1 VLLM_PRECOMPILED_WHEEL_VARIANT=cpu VLLM_TARGET_DEVICE=cpu - `VLLM_CPU_KVCACHE_SPACE`: specify the KV Cache size (e.g, `VLLM_CPU_KVCACHE_SPACE=40` means 40 GiB space for KV cache), larger setting will allow vLLM to run more requests in parallel. This parameter should be set based on the hardware configuration and memory management pattern of users. Default value is `0`. - `VLLM_CPU_OMP_THREADS_BIND`: specify the CPU cores dedicated to the OpenMP threads, can be set as CPU id lists, `auto` (by default), or `nobind` (to disable binding to individual CPU cores and to inherit user-defined OpenMP variables). For example, `VLLM_CPU_OMP_THREADS_BIND=0-31` means there will be 32 OpenMP threads bound on 0-31 CPU cores. `VLLM_CPU_OMP_THREADS_BIND=0-31|32-63` means there will be 2 tensor parallel processes, 32 OpenMP threads of rank0 are bound on 0-31 CPU cores, and the OpenMP threads of rank1 are bound on 32-63 CPU cores. By setting to `auto`, the OpenMP threads of each rank are bound to the CPU cores in each NUMA node respectively. If set to `nobind`, the number of OpenMP threads is determined by the standard `OMP_NUM_THREADS` environment variable. -- `VLLM_CPU_NUM_OF_RESERVED_CPU`: specify the number of CPU cores which are not dedicated to the OpenMP threads for each rank. The variable only takes effect when VLLM_CPU_OMP_THREADS_BIND is set to `auto`. Default value is `None`. If the value is not set and use `auto` thread binding, no CPU will be reserved for `world_size == 1`, 1 CPU per rank will be reserved for `world_size > 1`. +- `VLLM_CPU_NUM_OF_RESERVED_CPU`: specify the total number of CPU cores which are not dedicated to OpenMP threads. The variable only takes effect when `VLLM_CPU_OMP_THREADS_BIND` is set to `auto`. By default, x86, ARM, and RISC-V reserve `local_world_size` CPUs in total, while PowerPC and S390X retain a one-CPU default. When KV transfer is enabled, one additional CPU is reserved per local rank. Reserved CPUs are distributed across the local ranks. - `CPU_VISIBLE_MEMORY_NODES`: specify visible NUMA memory nodes for vLLM CPU workers, similar to ```CUDA_VISIBLE_DEVICES```. The variable only takes effect when VLLM_CPU_OMP_THREADS_BIND is set to `auto`. The variable provides more control for the auto thread-binding feature, such as masking nodes and changing nodes binding sequence. - `VLLM_ZENTORCH_WEIGHT_PREPACK` (AMD Zen only): when `ZenCpuPlatform` is active, eagerly prepack linear weights into ZenDNN's blocked layout at model load time, eliminating per-inference layout conversion overhead. Default is `1` (enabled). See [AMD Zen optimizations](#amd-zen-optimizations). @@ -253,7 +253,7 @@ See [AMD Zen optimizations](#amd-zen-optimizations) for detection rules, support ### How to decide `VLLM_CPU_OMP_THREADS_BIND`? -- Default `auto` thread-binding is recommended for most cases. Ideally, each OpenMP thread will be bound to a dedicated physical core respectively, threads of each rank will be bound to the same NUMA node respectively, and 1 CPU per rank will be reserved for other vLLM components when `world_size > 1`. If you have any performance problems or unexpected binding behaviours, please try to bind threads as following. +- Default `auto` thread-binding is recommended for most cases. Ideally, each OpenMP thread will be bound to a dedicated physical core respectively, threads of each rank will be bound to the same NUMA node respectively, and the default reserved CPUs will be kept for other vLLM components. On x86, ARM, and RISC-V, `local_world_size` CPUs are reserved in total, plus one additional CPU per local rank when KV transfer is enabled. If you have any performance problems or unexpected binding behaviours, please try to bind threads as following. - On a hyper-threading enabled platform with 16 logical CPU cores / 8 physical CPU cores: diff --git a/docs/getting_started/installation/cpu.s390x.inc.md b/docs/getting_started/installation/cpu.s390x.inc.md index 88767e4c6e..e1ff4ee3cb 100644 --- a/docs/getting_started/installation/cpu.s390x.inc.md +++ b/docs/getting_started/installation/cpu.s390x.inc.md @@ -138,19 +138,6 @@ VLLM_TARGET_DEVICE=cpu VLLM_CPU_MOE_PREPACK=0 python setup.py bdist_wheel && \ The Docker image (`Dockerfile.s390x`) already includes TCMalloc and sets `LD_PRELOAD` automatically. -!!! warning "Protobuf workaround for s390x" - The C++ protobuf extension crashes on s390x. After installation, set the - following environment variable and remove the C++ extensions: - - ```bash - export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python - - # Remove C++ protobuf extensions that crash on s390x - SITE_PKGS=$(python -c "import site; print(site.getsitepackages()[0])") - rm -rf "$SITE_PKGS/google/_upb/"*.so \ - "$SITE_PKGS/google/protobuf/pyext/"*.so 2>/dev/null || true - ``` - --8<-- [end:build-wheel-from-source] --8<-- [start:pre-built-images] diff --git a/docs/getting_started/installation/cpu.x86.inc.md b/docs/getting_started/installation/cpu.x86.inc.md index a64d0e5703..260ffe2863 100644 --- a/docs/getting_started/installation/cpu.x86.inc.md +++ b/docs/getting_started/installation/cpu.x86.inc.md @@ -33,7 +33,7 @@ uv pip install "https://github.com/vllm-project/vllm/releases/download/v${VLLM_V pip install "https://github.com/vllm-project/vllm/releases/download/v${VLLM_VERSION}/vllm-${VLLM_VERSION}+cpu-cp38-abi3-manylinux_2_34_x86_64.whl" --extra-index-url https://download.pytorch.org/whl/cpu ``` !!! warning "set `LD_PRELOAD`" - Before use vLLM CPU installed via wheels, make Intel OpenMP is added to `LD_PRELOAD`: + Before use vLLM CPU installed via wheels, make sure Intel OpenMP is added to `LD_PRELOAD`: ```bash # manually find the path sudo find / -iname *libiomp5.so diff --git a/docs/governance/committers.md b/docs/governance/committers.md index bd36759122..48ed4b2607 100644 --- a/docs/governance/committers.md +++ b/docs/governance/committers.md @@ -2,7 +2,7 @@ This document lists the current committers of the vLLM project and the core areas they maintain. Committers have write access to the vLLM repository and are responsible for reviewing and merging PRs. -You can also refer to the [CODEOWNERS](https://github.com/vllm-project/vllm/blob/main/.github/CODEOWNERS) file for concrete file-level ownership and reviewers. Both this documents and the CODEOWNERS file are living documents and they complement each other. +You can also refer to the [CODEOWNERS](https://github.com/vllm-project/vllm/blob/main/.github/CODEOWNERS) file for concrete file-level ownership and reviewers. Both these documents and the CODEOWNERS file are living documents and they complement each other. ## Active Committers diff --git a/docs/serving/online_serving/derenderer.md b/docs/serving/online_serving/derenderer.md index 298510062a..773131d033 100644 --- a/docs/serving/online_serving/derenderer.md +++ b/docs/serving/online_serving/derenderer.md @@ -6,7 +6,7 @@ This closes the loop for a token-in / token-out engine in disaggregated serving: - **GPU less post processing**: Detokenization, reasoning parsing, and tool call parsing run on the same GPU less frontend that hosts `/render` - **Parser parity**: The derenderer reuses vLLM's tool and reasoning parsers, so a disaggregated deployment produces the same `content`/`reasoning`/ `tool_calls` split as a standard `vllm serve` server -- **Non-streaming**: The endpoints expect a complete `GenerateResponse` with all token IDs present and perform one-shot parsing. Streaming derender would require a separate endpoint design and is not currently supported but is in the pipeline +- **Streaming and one-shot parsing**: Non-streaming calls send a complete `GenerateResponse` with all token IDs present and get one-shot parsing. Both endpoints also accept `stream: true`, taking one `GenerateStreamResponse` delta plus a client carried `stream_state` and returning `{chunk, stream_state}`. The chat endpoint's streaming path supports reasoning and tool call parsing while emitting the same `reasoning`/`content`/`tool_calls` deltas the generate streaming path would for the same token IDs Both endpoints are hosted by the GPU less rendering server started with [`vllm launch render`](../../cli/launch/render.md), alongside the `/render` endpoints. @@ -21,7 +21,7 @@ endpoints. └─────────────── request + prompt_tokens ──┘ ``` -The derender step needs more than the engine's `token_ids`. It also consumes the original `chat_request`/`completion_request` and `prompt_tokens` carried over from the render step (see [Request format](#request-format)) so the tool and reasoning parsers have the context they need. +The derender step needs more than the engine's `token_ids`. It also consumes the original `chat_request`/`completion_request` and `prompt_tokens` carried over from the render step (see [Request format](#request-format)) so the tool and reasoning parsers have the context they need. The chat streaming path additionally needs `prompt_token_ids` (see [Streaming cost](#streaming-cost)) when a parser is configured. ## API Reference @@ -52,6 +52,21 @@ Each request wraps the engine's `GenerateResponse`(s) together with the caller m Oversized payloads are rejected with a `400` before any `tokenizer.decode()` or parser runs. +## Streaming cost + +Streaming derender threads a client carried `stream_state` across per-chunk calls instead of keeping session state on the server. + +Plain detokenization (no parser configured) carries only a small, bounded incremental decode window in `stream_state` independent of generation length. + +When a tool or reasoning parser is configured, parser internal state (buffered markup, reasoning/tool phase) can't be serialized. `stream_state` therefore instead carries the full `output_token_ids` seen so far plus `output_chunk_lens`, the token count of each chunk they arrived in. Each chunk rebuilds a fresh parser and replays that history through `parse_delta`, one call per original chunk, before processing the new tokens for real. For the parser path only, this means: + +- **Transport**: `output_token_ids` and `output_chunk_lens` round-trip in full in both directions on every call. `output_chunk_lens` has one entry per chunk, which is one per token without speculative decoding. This means O(n) bytes per chunk, O(n²) bytes over a full generation. Bounded by `max_model_len`. `prompt_token_ids` is sent in full on every call too and it isn't trimmed as `output_token_ids` grows. This means that for most of a stream it dominates the per chunk payload. A 100k token prompt with 1k tokens of output means `prompt_token_ids` is ~99% of the request body on every chunk. +- **Compute**: replay is O(n) `parse_delta` calls per chunk (O(n²) per generation). `parse_delta` itself is O(n) for parsers that re-scan accumulated text (e.g. Hermes tool-call JSON, DeepSeek-R1 reasoning). The per-generation cost is O(n³) character work, not O(n²). This is a deliberately minimal first implementation with no caching layer. +- The parser path also requires `prompt_token_ids` so `parse_delta` can settle whether the prompt left reasoning open or not. Since parser state can't be carried across calls, it re-scans the full prompt once per chunk. +- Replay runs off the event loop on the renderer's executor (`renderer_num_workers`, default `1`). Size it for the expected number of concurrent parser configured streams. + +`output_token_ids` and `prompt_token_ids` are both bounded by `max_model_len` but callers streaming long reasoning traces through a parser configured model should expect materially more state transport and CPU cost than the plain detokenization path. + ## Example The example below drives the full `render → generate → derender` round trip for a chat request against a GPU less render server (`/render`, `/derender`) and a token-in / token-out engine (`/inference/v1/generate`). @@ -102,4 +117,4 @@ with httpx.Client(timeout=60.0) as client: print(response["choices"][0]["message"]["content"]) ``` -Passing `chat_request` lets the derenderer run the configured tool and reasoning parsers. This means `response["choices"][0]["message"]` carries the same `content` / `reasoning` / `tool_calls` split a `vllm serve` server would produce. Omit `chat_request` for plain detokenization only. +Passing `chat_request` lets the derenderer run the configured tool and reasoning parsers. This means `response["choices"][0]["message"]` carries the same `content` / `reasoning` / `tool_calls` split a `vllm serve` server would produce. `chat_request` can only be omitted for a model with no tool or reasoning parser configured. A parser configured model rejects a missing `chat_request` with a 400 rather than silently falling back to plain detokenization. diff --git a/docs/training/weight_transfer/base.md b/docs/training/weight_transfer/base.md index d73c88436a..759c55ef6f 100644 --- a/docs/training/weight_transfer/base.md +++ b/docs/training/weight_transfer/base.md @@ -233,7 +233,7 @@ Two implementations ship with vLLM: | `RayVLLMWeightSyncClient(handle)` | One or more `AsyncLLM`/`LLM` Ray actors. Accepts a list and fans each call out to every handle, blocking on all of them, so a multi-actor (e.g. multi-DP) deployment is driven as one unit | | `HTTPVLLMWeightSyncClient(base_url, timeout=300)` | A vLLM server over the RLHF HTTP routes | -Custom weight sync clients can be implement like so: +Custom weight sync clients can be implemented like so: ```python class MyFrameworkWeightSyncClient: diff --git a/examples/disaggregated/disaggregated_encoder/README.md b/examples/disaggregated/disaggregated_encoder/README.md index bd2a46abbe..71a0d0e72e 100644 --- a/examples/disaggregated/disaggregated_encoder/README.md +++ b/examples/disaggregated/disaggregated_encoder/README.md @@ -106,6 +106,22 @@ If you enable prefill instance (`--prefill-servers-urls` not disabled), you will | `--decode-servers-urls` | Comma-separated list of decode endpoints. Non-stream and stream paths both round-robin over this list. | | `--host`, `--port` | Bind address for the proxy itself (defaults: `0.0.0.0:8000`). | +The proxy batches images from the same user request assigned to the same encoder. +Set the proxy environment variable `ENCODER_MAX_BATCH_SIZE` to limit the number +of images per encoder subrequest. It defaults to `0` (unlimited); `1` sends each +image separately. Audio and video remain separate subrequests. + +For encoders with a smaller `--limit-mm-per-prompt` image limit than P/PD, set +`ENCODER_MAX_BATCH_SIZE` no higher than the smallest encoder image limit. +For example, for encoders configured with `--limit-mm-per-prompt '{"image": 2}'`: + +```bash +ENCODER_MAX_BATCH_SIZE=2 python disagg_epd_proxy.py \ + --encode-servers-urls "http://e1:8001,http://e2:8002" \ + --prefill-servers-urls disable \ + --decode-servers-urls "http://pd1:8003" +``` + Example usage: For E + PD setup: diff --git a/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py b/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py index c44c902690..3317b091fb 100644 --- a/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py +++ b/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py @@ -10,8 +10,8 @@ clusters: For MM input we: 1. Extract *every* image/audio/video item. - 2. Fire N concurrent requests to the encoder cluster - (one request per item, with **all text removed**). + 2. Send concurrent encoder requests with all text removed, + grouping images assigned to the same encoder. 3. Wait for all of them to succeed. 4. Forward the *original* request to a decode server. """ @@ -99,6 +99,11 @@ def validate_ec_consumer_routing( # Diagnostic switch: forward the original request to the decoder so the # only difference from the rewrite path is the rewrite itself. NO_REWRITE = False +# Maximum images per encoder subrequest; 0 leaves batches unlimited. +ENCODER_MAX_BATCH_SIZE = int(os.getenv("ENCODER_MAX_BATCH_SIZE", "0")) +if ENCODER_MAX_BATCH_SIZE < 0: + raise ValueError("ENCODER_MAX_BATCH_SIZE must be non-negative") + # Decode-side retries for a retryable internal error (`finish_reason="error"`, # e.g. an encoder embedding the connector could not deliver). Re-issuing runs # the encode again, which produces a fresh transfer. @@ -222,7 +227,7 @@ async def fanout_encoder_primer( req_id: str, consumer_zmq: str | None = None, ) -> tuple[dict[int, dict], dict[str, Any]]: - """1. Build one request *per MM item* with all text removed. + """1. Group images by encoder, retaining per-item round-robin assignment. 2. Send them concurrently to the encode cluster. 3. Raise if any of them fails. @@ -248,6 +253,7 @@ async def fanout_encoder_primer( item_uuids: dict[int, str] = {} item_transfer_ids: dict[int, str] = {} item_meta: dict[int, dict] = {} + transfer_items: dict[int, dict[str, str]] = {} ec_params: dict[str, Any] = {} # Round-robin over encode servers to distribute load a bit. The cursor @@ -259,29 +265,45 @@ async def fanout_encoder_primer( e_urls, encoder_rr_idx, len(mm_items) ) + groups: list[tuple[str, list[int]]] = [] + image_groups: dict[str, list[int]] = {} for idx, (item, target_url) in enumerate(zip(mm_items, url_cycle)): + if item["type"] == "image_url": + indices = image_groups.get(target_url) + if indices is None or ( + ENCODER_MAX_BATCH_SIZE and len(indices) >= ENCODER_MAX_BATCH_SIZE + ): + indices = [] + image_groups[target_url] = indices + groups.append((target_url, indices)) + indices.append(idx) + else: + groups.append((target_url, [idx])) + + for target_url, indices in groups: # Derive a *child* request id: :: - child_req_id = f"{req_id}:{idx}:{uuid.uuid4().hex[:6]}" + child_req_id = f"{req_id}:{indices[0]}:{uuid.uuid4().hex[:6]}" headers = {"x-request-id": child_req_id, "Content-Type": "application/json"} # With --no-rewrite the decoder still receives the raw image and derives # the cache key by hashing it, so the encoder must do the same -- passing # a uuid here would make the two disagree and silently defeat the EC # transfer, leaving the decoder to encode the image itself. - item_uuid = None if NO_REWRITE else content_uuid(item) - if item_uuid is not None: - item_uuids[idx] = item_uuid - transfer_id = uuid.uuid4().hex - item_transfer_ids[idx] = transfer_id + content = [] + for idx in indices: + item = mm_items[idx] + item_uuid = None if NO_REWRITE else content_uuid(item) + if item_uuid is not None: + item_uuids[idx] = item_uuid + item_transfer_ids[idx] = uuid.uuid4().hex + content.append(item if item_uuid is None else {**item, "uuid": item_uuid}) encoder_req = { "model": orig_request.get("model"), "messages": [ { "role": "user", - "content": [ - item if item_uuid is None else {**item, "uuid": item_uuid} - ], + "content": content, }, ], # No max_tokens cap: the encoder instance never samples, it finishes @@ -297,17 +319,13 @@ async def fanout_encoder_primer( if key in orig_request: encoder_req[key] = orig_request[key] if consumer_zmq is not None: - # No mm_hash here on purpose. The encoder's own - # `mm_features[i].identifier` is derived from the uuid *and* the - # engine's media_io_kwargs / mm_processor_kwargs, so this proxy - # cannot know it before the encoder runs. Sending the bare uuid - # would fail the connector's hash match and make the producer - # invent its own transfer id, which the consumer could then never - # cancel. Omitting it lets the connector match by position, which - # is exact: one encoder request carries exactly one item. + # The engine may rehash UUIDs with processing options. Match by + # position instead; batches contain only images in input order. encoder_req["ec_transfer_params"] = { "consumer_zmq": consumer_zmq, - "ec_items": [{"transfer_id": transfer_id}], + "ec_items": [ + {"transfer_id": item_transfer_ids[idx]} for idx in indices + ], } tasks.append( encode_session.post( @@ -320,7 +338,8 @@ async def fanout_encoder_primer( results = await asyncio.gather(*tasks, return_exceptions=True) # Fail fast if any sub-request failed - for idx, r in enumerate(results): + for (_, indices), r in zip(groups, results): + idx = indices[0] if isinstance(r, Exception): logger.error( "[%s] Encoder request #%d raised exception: %s", @@ -357,18 +376,45 @@ async def fanout_encoder_primer( logger.warning("[%s] Could not read encoder metadata #%d", req_id, idx) params = {} if params: - # One encoder request carries exactly one item, so there is a - # single reported entry. Do not key it by this proxy's uuid: when - # media_io_kwargs or mm_processor_kwargs are set the engine - # re-hashes the uuid together with them, so the encoder's own - # `mm_features[i].identifier` is a derived value this proxy cannot - # predict. Fall back to the sole entry, and carry the key the - # encoder actually used through as `ec_mm_hash`. - ec_mm_hash = item_uuids.get(idx) - reported = params.get(ec_mm_hash) - if reported is None and len(params) == 1: - ((ec_mm_hash, reported),) = params.items() - if reported: + by_index = {} + for mm_hash, reported in params.items(): + if len(indices) == 1: + break + for local_index in reported.get("item_indices", []): + if type(local_index) is not int or not 0 <= local_index < len( + indices + ): + raise HTTPException(502, "Invalid encoder item index") + if local_index in by_index: + raise HTTPException(502, "Duplicate encoder item index") + by_index[local_index] = (mm_hash, reported) + for local_index, idx in enumerate(indices): + matched = by_index.get(local_index) + if matched is None: + # Compatibility with encoders without position metadata. + mm_hash = item_uuids.get(idx) + if mm_hash in params: + matched = (mm_hash, params[mm_hash]) + elif len(indices) == 1 and len(params) == 1: + matched = next(iter(params.items())) + elif ( + mm_items[idx]["type"] == "video_url" + and (orig_request.get("mm_processor_kwargs") or {}).get( + "use_audio_in_video" + ) + and len(params) == 2 + and consumer_zmq is None + and all( + entry.keys() <= {"metadata", "item_indices"} + for entry in params.values() + ) + ): + # Keep raw video for its audio/video features when no transfer + # handles need to be forwarded. + continue + else: + raise HTTPException(502, "Encoder metadata cannot be matched") + ec_mm_hash, reported = matched metadata = reported.get("metadata") or {} if metadata: item_meta[idx] = { @@ -384,12 +430,16 @@ async def fanout_encoder_primer( # looks it up by mm_hash on the request, so carry it through. ec_params[ec_mm_hash] = reported if NO_REWRITE and consumer_zmq is not None: - ec_params.setdefault("ec_items", []).append( - {"mm_hash": ec_mm_hash, "transfer_id": item_transfer_ids[idx]} - ) + transfer_items[idx] = { + "mm_hash": ec_mm_hash, + "transfer_id": item_transfer_ids[idx], + } + + if transfer_items: + ec_params["ec_items"] = [transfer_items[idx] for idx in sorted(transfer_items)] logger.info( - "[%s] All %d encoder requests completed successfully", req_id, len(mm_items) + "[%s] All %d encoder requests completed successfully", req_id, len(groups) ) return item_meta, ec_params diff --git a/requirements/common.txt b/requirements/common.txt index acfc986c75..b55f680b76 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -11,7 +11,8 @@ transformers >= 5.10.4 huggingface_hub >= 1.31.0 # Required for the `huggingface_hub.utils.httpx` re-export tokenizers >= 0.21.1 # Required for fast incremental detokenization. safetensors >= 0.6.2 # MXFP4/MXFP6 dtype support (F8_E8M0, F4) added in 0.6.0: https://github.com/huggingface/safetensors/pull/611 -protobuf >= 5.29.6, !=6.30.*, !=6.31.*, !=6.32.*, !=6.33.0.*, !=6.33.1.*, !=6.33.2.*, !=6.33.3.*, !=6.33.4.* # Required by LlamaTokenizer, gRPC. CVE-2026-0994 +protobuf >= 5.29.6, !=6.30.*, !=6.31.*, !=6.32.*, !=6.33.0.*, !=6.33.1.*, !=6.33.2.*, !=6.33.3.*, !=6.33.4.*; platform_machine != "s390x" +protobuf == 7.36.1; platform_machine == "s390x" fastapi[standard] >= 0.133.0, < 0.137.0 # First version supporting Starlette 1.0; < 0.137.0 avoids route-tree change that breaks model-hosting-container-standards handler overrides. starlette >= 1.0.1 # CVE-2026-48710: Host header injection in < 1.0.1 aiohttp >= 3.13.3 @@ -25,7 +26,7 @@ lm-format-enforcer == 0.11.3 llguidance >= 1.7.0, < 1.8.0; platform_machine == "x86_64" or platform_machine == "arm64" or platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "s390x" outlines_core == 0.2.14 lark == 1.2.2 -xgrammar >= 0.2.1, < 1.0.0; platform_machine == "x86_64" or platform_machine == "aarch64" or platform_machine == "arm64" or platform_machine == "s390x" or platform_machine == "ppc64le" +xgrammar == 0.2.7; platform_machine == "x86_64" or platform_machine == "aarch64" or platform_machine == "arm64" or platform_machine == "s390x" or platform_machine == "ppc64le" typing_extensions >= 4.10 filelock >= 3.16.1 # need to contain https://github.com/tox-dev/filelock/pull/317 partial-json-parser # used for parsing partial JSON outputs diff --git a/requirements/test/cpu.txt b/requirements/test/cpu.txt index 996caab96a..c1d4913953 100644 --- a/requirements/test/cpu.txt +++ b/requirements/test/cpu.txt @@ -43,7 +43,7 @@ anyio==4.14.1 # sse-starlette # starlette # watchfiles -apache-tvm-ffi==0.1.9 +apache-tvm-ffi==0.1.14.post0 # via xgrammar arctic-inference==0.1.1 # via -r requirements/test/cuda.in @@ -1298,7 +1298,7 @@ word2number==1.1 # via lm-eval wrapt==1.17.2 # via smart-open -xgrammar==0.2.3 +xgrammar==0.2.7 # via -r requirements/test/../common.txt xxhash==3.5.0 # via diff --git a/requirements/test/cuda.txt b/requirements/test/cuda.txt index ea94558000..3207d395aa 100644 --- a/requirements/test/cuda.txt +++ b/requirements/test/cuda.txt @@ -1409,7 +1409,7 @@ word2number==1.1 # via lm-eval wrapt==1.17.2 # via smart-open -xgrammar==0.2.3 +xgrammar==0.2.7 # via # -c requirements/common.txt # -r requirements/test/../common.txt diff --git a/requirements/test/rocm.txt b/requirements/test/rocm.txt index 631c3952fd..c961b6294a 100644 --- a/requirements/test/rocm.txt +++ b/requirements/test/rocm.txt @@ -1360,7 +1360,7 @@ word2number==1.1 # via lm-eval wrapt==2.1.2 # via smart-open -xgrammar==0.2.1 +xgrammar==0.2.7 # via # -c requirements/common.txt # -r requirements/test/../common.txt diff --git a/requirements/test/xpu.txt b/requirements/test/xpu.txt index 4e4fd628b6..d9572389eb 100644 --- a/requirements/test/xpu.txt +++ b/requirements/test/xpu.txt @@ -1197,7 +1197,7 @@ werkzeug==3.1.8 # via schemathesis word2number==1.1 # via lm-eval -xgrammar==0.2.3 +xgrammar==0.2.7 # via # -c requirements/common.txt # -r requirements/test/../common.txt diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 601e11c054..b4917a303b 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -5941,7 +5941,7 @@ dependencies = [ [[package]] name = "vllm-proto" -version = "0.3.0" +version = "0.4.0" dependencies = [ "prost", "prost-types", @@ -6596,9 +6596,9 @@ checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" [[package]] name = "xgrammar-structural-tag" -version = "0.2.0+xgrammar.0.2.4.dd729e7" +version = "0.3.0+xgrammar.0.2.7.82505d0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4d24c842efc3c24e9756aa426d530cbdac0980e49af223cb384e276e981ca0a" +checksum = "3864005575c2ab76057f789898f59fb9fa1378d07c5ceabc24764e012d0e90bb" dependencies = [ "auto_impl", "serde", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index dde065e790..ea8b813619 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -153,7 +153,7 @@ vllm-text = { path = "src/text" } vllm-tokenizer = { path = "src/tokenizer" } vllm-tracing = { path = "src/tracing" } winnow = { version = "1.0.2", features = ["simd"] } -xgrammar-structural-tag = "0.2.0" +xgrammar-structural-tag = "0.3.0" zeromq = { version = "0.6.0", default-features = false, features = [ "tokio-runtime", "all-transport", diff --git a/rust/proto/Cargo.toml b/rust/proto/Cargo.toml index cac268bc74..492cb5a06d 100644 --- a/rust/proto/Cargo.toml +++ b/rust/proto/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vllm-proto" -version = "0.3.0" +version = "0.4.0" edition.workspace = true license.workspace = true description = "Protobuf messages and gRPC bindings for the vLLM Rust frontend" diff --git a/rust/proto/inference.proto b/rust/proto/inference.proto index 46313f915a..4eaf6f92f4 100644 --- a/rust/proto/inference.proto +++ b/rust/proto/inference.proto @@ -160,6 +160,8 @@ message SequenceOutput { // Only present in final output for this sequence optional FinishInfo finish_info = 8; + // Only present in final output when sampling-mask capture is enabled. + repeated TokenIds sampling_mask = 10; } // Prompt info, returned in the first response diff --git a/rust/src/chat/src/backend/hf.rs b/rust/src/chat/src/backend/hf.rs index af9af8b51c..07f0b2dce8 100644 --- a/rust/src/chat/src/backend/hf.rs +++ b/rust/src/chat/src/backend/hf.rs @@ -134,6 +134,7 @@ impl ChatBackend for HfChatBackend { self.tokenizer.clone(), options.tool_call_parser, options.reasoning_parser, + options.tool_strict_level, )?)) } } @@ -343,6 +344,7 @@ mod tests { let error = match backend.new_chat_output_processor( &mut request, NewChatOutputProcessorOptions { + tool_strict_level: crate::ToolStrictLevel::Auto, tool_call_parser: &ParserSelection::Explicit("json".to_string()), reasoning_parser: &ParserSelection::Auto, }, diff --git a/rust/src/chat/src/backend/mod.rs b/rust/src/chat/src/backend/mod.rs index af4579159e..a0343e53f3 100644 --- a/rust/src/chat/src/backend/mod.rs +++ b/rust/src/chat/src/backend/mod.rs @@ -13,7 +13,7 @@ use crate::multimodal::{MmLimitPerPrompt, MultimodalModelInfo}; use crate::output::DynChatOutputProcessor; use crate::renderer::DynChatRenderer; use crate::request::ChatRequest; -use crate::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection}; +use crate::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection, ToolStrictLevel}; pub mod hf; @@ -21,6 +21,7 @@ pub mod hf; pub struct NewChatOutputProcessorOptions<'a> { pub tool_call_parser: &'a ParserSelection, pub reasoning_parser: &'a ParserSelection, + pub tool_strict_level: ToolStrictLevel, } /// Minimal prompt-processing backend needed by `vllm-chat`. diff --git a/rust/src/chat/src/lib.rs b/rust/src/chat/src/lib.rs index 69d39ca78e..0273396ebd 100644 --- a/rust/src/chat/src/lib.rs +++ b/rust/src/chat/src/lib.rs @@ -30,7 +30,7 @@ pub use parser::reasoning::{ ReasoningDelta, ReasoningError, ReasoningParser, ReasoningParserFactory, }; pub use parser::tool::{ToolParser, ToolParserError, ToolParserFactory}; -pub use parser::{ParserSelection, validate_parser_overrides}; +pub use parser::{ParserSelection, ToolStrictLevel, validate_parser_overrides}; pub use reasoning::EffortValue; pub use renderer::hf::ChatTemplateContentFormatOption; pub use renderer::{ @@ -74,6 +74,8 @@ pub struct ChatRequestProcessor { tool_call_parser: ParserSelection, /// Reasoning parser selection used when preparing generation requests. reasoning_parser: ParserSelection, + /// Server-side floor for tool-call structural tags. + tool_strict_level: ToolStrictLevel, } impl ChatRequestProcessor { @@ -85,6 +87,7 @@ impl ChatRequestProcessor { model_dtype: Some(model_dtype), tool_call_parser: ParserSelection::Auto, reasoning_parser: ParserSelection::Auto, + tool_strict_level: ToolStrictLevel::Auto, } } @@ -95,6 +98,7 @@ impl ChatRequestProcessor { model_dtype: None, tool_call_parser: ParserSelection::Auto, reasoning_parser: ParserSelection::Auto, + tool_strict_level: ToolStrictLevel::Auto, } } @@ -109,6 +113,12 @@ impl ChatRequestProcessor { self } + /// Configure the server-side floor for tool-call structural tags. + pub fn with_tool_strict_level(mut self, tool_strict_level: ToolStrictLevel) -> Self { + self.tool_strict_level = tool_strict_level; + self + } + async fn finalize_rendered_prompt( &self, request: &ChatRequest, @@ -207,6 +217,7 @@ impl ChatRequestProcessor { NewChatOutputProcessorOptions { tool_call_parser: &self.tool_call_parser, reasoning_parser: &self.reasoning_parser, + tool_strict_level: self.tool_strict_level, }, )?; let text_request = self.prepare_text_request(request).await?; @@ -255,6 +266,12 @@ impl ChatLlm { self } + /// Set the server-side floor for tool-call structural tags. + pub fn with_tool_strict_level(mut self, tool_strict_level: ToolStrictLevel) -> Self { + self.processor.tool_strict_level = tool_strict_level; + self + } + /// Override the effective model dtype used for multimodal tensor encoding. pub fn with_model_dtype(mut self, model_dtype: ModelDtype) -> Self { self.processor.model_dtype = Some(model_dtype); diff --git a/rust/src/chat/src/output/default/mod.rs b/rust/src/chat/src/output/default/mod.rs index a731153063..9fb3d02b11 100644 --- a/rust/src/chat/src/output/default/mod.rs +++ b/rust/src/chat/src/output/default/mod.rs @@ -18,10 +18,10 @@ use self::unified::unified_event_stream; use super::structured::structured_chat_event_stream; use crate::error::Result; use crate::output::{ChatOutputProcessor, DynChatEventStream, DynDecodedTextEventStream}; -use crate::parser::ParserSelection; use crate::parser::reasoning::{ReasoningParser, ReasoningParserFactory}; use crate::parser::tool::{ToolParser, ToolParserFactory}; use crate::parser::unified::UnifiedParserFactory; +use crate::parser::{ParserSelection, ToolStrictLevel}; use crate::request::{ChatRequest, ChatTool}; use crate::{Error, Result as ChatResult}; @@ -49,6 +49,7 @@ impl DefaultChatOutputProcessor { tokenizer: DynTokenizer, tool_call_parser: &ParserSelection, reasoning_parser: &ParserSelection, + tool_strict_level: ToolStrictLevel, ) -> ChatResult { let parser = if let Some(parser) = Self::resolve_optional_unified_parser( request.tools(), @@ -73,7 +74,11 @@ impl DefaultChatOutputProcessor { Box::new(CombinedParser::new(reasoning_parser, tool_parser)) as Box }; - apply_structural_tag_constraint(request, parser.structural_tag_builder())?; + apply_structural_tag_constraint( + request, + parser.structural_tag_builder(), + tool_strict_level, + )?; if parser.preserve_special_tokens() { request.decode_options.skip_special_tokens = false; @@ -191,7 +196,7 @@ mod tests { use vllm_tokenizer::test_utils::TestTokenizer; use super::DefaultChatOutputProcessor; - use crate::parser::ParserSelection; + use crate::parser::{ParserSelection, ToolStrictLevel}; use crate::request::ChatRequest; fn tokenizer() -> Arc { @@ -213,6 +218,7 @@ mod tests { tokenizer(), &selection, &selection, + ToolStrictLevel::Auto, ) .unwrap(); } @@ -227,6 +233,7 @@ mod tests { tokenizer(), &ParserSelection::Auto, &ParserSelection::Auto, + ToolStrictLevel::Auto, ) .unwrap(); } @@ -244,6 +251,7 @@ mod tests { tokenizer(), tool, reasoning, + ToolStrictLevel::Auto, ) .unwrap(); } @@ -258,6 +266,7 @@ mod tests { tokenizer(), &ParserSelection::Auto, &ParserSelection::Explicit("gemma4".to_string()), + ToolStrictLevel::Auto, ) { Ok(_) => panic!("expected mixed Gemma4 parser selection to fail"), Err(error) => error, diff --git a/rust/src/chat/src/output/default/structural_tag.rs b/rust/src/chat/src/output/default/structural_tag.rs index 2ef6f08685..116cd53a4a 100644 --- a/rust/src/chat/src/output/default/structural_tag.rs +++ b/rust/src/chat/src/output/default/structural_tag.rs @@ -14,19 +14,21 @@ use xgrammar_structural_tag::{ build_structural_tag, }; +use crate::parser::ToolStrictLevel; use crate::request::{ChatRequest, ChatToolChoice}; use crate::{Error, Result as ChatResult}; /// Apply structural tag constraints to the request based on the tool parser's structural tag -/// support and the request's tool choice. +/// support, the request's tool choice, and the server-side strictness floor. pub(super) fn apply_structural_tag_constraint( request: &mut ChatRequest, builder: Option<&dyn StructuralTagBuilder>, + strict_level: ToolStrictLevel, ) -> ChatResult<()> { let Some(builder) = builder else { return Ok(()); }; - let Some(tool_choice) = structural_tag_tool_choice(request) else { + let Some(tool_choice) = structural_tag_tool_choice(request, strict_level) else { return Ok(()); }; @@ -34,11 +36,14 @@ pub(super) fn apply_structural_tag_constraint( .tools() .iter() .map(|tool| { + // A tool without an explicit `strict` is non-strict: its call envelope is + // constrained, but its arguments stay free unless the server level is `parameter`. + let strict = strict_level >= ToolStrictLevel::Parameter || tool.strict == Some(true); ToolParam::Function(FunctionToolParam::new(FunctionDefinition { name: tool.name.clone(), description: tool.description.clone(), parameters: Some(tool.parameters.clone()), - strict: tool.strict, + strict: Some(strict), })) }) .collect::>(); @@ -66,17 +71,21 @@ pub(super) fn apply_structural_tag_constraint( /// Resolve the tool choice used for [`xgrammar_structural_tag`] based on the request. /// /// Returns `None` if no structural tag constraints should be applied. -fn structural_tag_tool_choice(request: &ChatRequest) -> Option { +fn structural_tag_tool_choice( + request: &ChatRequest, + strict_level: ToolStrictLevel, +) -> Option { if request.tools().is_empty() { return None; } match request.tool_choice() { - // For `Auto`, only apply the structural tag if there's at least one strict tool. - ChatToolChoice::Auto if request.tools().iter().any(|tool| tool.strict == Some(true)) => { - Some(StructuralTagToolChoice::auto()) - } - ChatToolChoice::Auto | ChatToolChoice::None => None, + ChatToolChoice::None => None, + // For `Auto`, apply the structural tag only when the server raises the floor or at + // least one tool opts in with `strict: true`. + ChatToolChoice::Auto => (strict_level >= ToolStrictLevel::Function + || request.tools().iter().any(|tool| tool.strict == Some(true))) + .then(StructuralTagToolChoice::auto), ChatToolChoice::Required => Some(StructuralTagToolChoice::required()), ChatToolChoice::Function { name } => Some(StructuralTagToolChoice::function(name.clone())), @@ -148,8 +157,12 @@ mod tests { let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", Some(true))]); let parser = qwen3_coder_parser(request.tools()); - apply_structural_tag_constraint(&mut request, parser.structural_tag_builder()) - .expect("structural tag should build"); + apply_structural_tag_constraint( + &mut request, + parser.structural_tag_builder(), + ToolStrictLevel::Auto, + ) + .expect("structural tag should build"); let tag = structural_tag_value(&request); assert_eq!(tag["type"], "structural_tag"); @@ -172,8 +185,12 @@ mod tests { }; let parser = qwen3_coder_parser(request.tools()); - apply_structural_tag_constraint(&mut request, parser.structural_tag_builder()) - .expect("structural tag should build"); + apply_structural_tag_constraint( + &mut request, + parser.structural_tag_builder(), + ToolStrictLevel::Auto, + ) + .expect("structural tag should build"); let tag = structural_tag_value(&request); assert!(tag.to_string().contains("lookup")); @@ -199,8 +216,12 @@ mod tests { }; let parser = qwen3_coder_parser(request.tools()); - apply_structural_tag_constraint(&mut request, parser.structural_tag_builder()) - .expect("structural tag should build"); + apply_structural_tag_constraint( + &mut request, + parser.structural_tag_builder(), + ToolStrictLevel::Auto, + ) + .expect("structural tag should build"); let tag = structural_tag_value(&request).to_string(); assert!(tag.contains("search")); @@ -212,8 +233,12 @@ mod tests { let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", None)]); let parser = qwen3_coder_parser(request.tools()); - apply_structural_tag_constraint(&mut request, parser.structural_tag_builder()) - .expect("structural tag decision should succeed"); + apply_structural_tag_constraint( + &mut request, + parser.structural_tag_builder(), + ToolStrictLevel::Auto, + ) + .expect("structural tag decision should succeed"); assert!(request.sampling_params.structured_outputs.is_none()); } @@ -227,8 +252,12 @@ mod tests { }); let parser = qwen3_coder_parser(request.tools()); - apply_structural_tag_constraint(&mut request, parser.structural_tag_builder()) - .expect("structural tag should build"); + apply_structural_tag_constraint( + &mut request, + parser.structural_tag_builder(), + ToolStrictLevel::Auto, + ) + .expect("structural tag should build"); let params = structured_outputs(&request); assert!(params.constraint.is_structural_tag()); @@ -242,8 +271,12 @@ mod tests { let mut request = request(ChatToolChoice::Required, vec![chat_tool("search", None)]); let parser = qwen3_coder_parser(request.tools()); - apply_structural_tag_constraint(&mut request, parser.structural_tag_builder()) - .expect("structural tag should build"); + apply_structural_tag_constraint( + &mut request, + parser.structural_tag_builder(), + ToolStrictLevel::Auto, + ) + .expect("structural tag should build"); let tag = structural_tag_value(&request); assert_eq!(tag["type"], "structural_tag"); @@ -259,8 +292,12 @@ mod tests { }); let parser = qwen3_coder_parser(request.tools()); - apply_structural_tag_constraint(&mut request, parser.structural_tag_builder()) - .expect("structural tag should build"); + apply_structural_tag_constraint( + &mut request, + parser.structural_tag_builder(), + ToolStrictLevel::Auto, + ) + .expect("structural tag should build"); let params = structured_outputs(&request); assert!(params.constraint.is_structural_tag()); @@ -279,8 +316,12 @@ mod tests { ); let parser = qwen3_coder_parser(request.tools()); - apply_structural_tag_constraint(&mut request, parser.structural_tag_builder()) - .expect("structural tag should build"); + apply_structural_tag_constraint( + &mut request, + parser.structural_tag_builder(), + ToolStrictLevel::Auto, + ) + .expect("structural tag should build"); let tag = structural_tag_value(&request).to_string(); assert!(tag.contains("lookup")); @@ -292,8 +333,12 @@ mod tests { let mut request = request(ChatToolChoice::None, vec![chat_tool("search", Some(true))]); let parser = qwen3_coder_parser(request.tools()); - apply_structural_tag_constraint(&mut request, parser.structural_tag_builder()) - .expect("structural tag decision should succeed"); + apply_structural_tag_constraint( + &mut request, + parser.structural_tag_builder(), + ToolStrictLevel::Auto, + ) + .expect("structural tag decision should succeed"); assert!(request.sampling_params.structured_outputs.is_none()); } @@ -307,10 +352,127 @@ mod tests { }); let parser = qwen3_coder_parser(request.tools()); - apply_structural_tag_constraint(&mut request, parser.structural_tag_builder()) - .expect("structural tag decision should succeed"); + apply_structural_tag_constraint( + &mut request, + parser.structural_tag_builder(), + ToolStrictLevel::Auto, + ) + .expect("structural tag decision should succeed"); let params = structured_outputs(&request); assert!(params.constraint.is_json_object()); } + + fn build(tool_choice: ChatToolChoice, tools: Vec, level: ToolStrictLevel) -> ChatRequest { + let mut request = request(tool_choice, tools); + let parser = qwen3_coder_parser(request.tools()); + + apply_structural_tag_constraint(&mut request, parser.structural_tag_builder(), level) + .expect("structural tag decision should succeed"); + request + } + + fn tag_string(request: &ChatRequest) -> String { + structural_tag_value(request).to_string() + } + + #[test] + fn function_level_constrains_auto_without_strict_tools() { + let request = build( + ChatToolChoice::Auto, + vec![chat_tool("search", None)], + ToolStrictLevel::Function, + ); + + let tag = structural_tag_value(&request); + assert_eq!(tag["type"], "structural_tag"); + assert!(tag.to_string().contains("search")); + } + + #[test] + fn absent_strict_is_non_strict() { + let unset = build( + ChatToolChoice::Required, + vec![chat_tool("search", None)], + ToolStrictLevel::Auto, + ); + let non_strict = build( + ChatToolChoice::Required, + vec![chat_tool("search", Some(false))], + ToolStrictLevel::Auto, + ); + let strict = build( + ChatToolChoice::Required, + vec![chat_tool("search", Some(true))], + ToolStrictLevel::Auto, + ); + + assert_eq!(tag_string(&unset), tag_string(&non_strict)); + assert_ne!(tag_string(&unset), tag_string(&strict)); + } + + #[test] + fn strict_tool_does_not_pin_its_neighbour() { + let mixed = build( + ChatToolChoice::Auto, + vec![chat_tool("weather", Some(true)), chat_tool("search", None)], + ToolStrictLevel::Auto, + ); + let explicit = build( + ChatToolChoice::Auto, + vec![ + chat_tool("weather", Some(true)), + chat_tool("search", Some(false)), + ], + ToolStrictLevel::Auto, + ); + + assert_eq!(tag_string(&mixed), tag_string(&explicit)); + } + + #[test] + fn parameter_level_pins_argument_schemas() { + let pinned = build( + ChatToolChoice::Auto, + vec![chat_tool("search", None)], + ToolStrictLevel::Parameter, + ); + let strict = build( + ChatToolChoice::Auto, + vec![chat_tool("search", Some(true))], + ToolStrictLevel::Auto, + ); + let envelope_only = build( + ChatToolChoice::Auto, + vec![chat_tool("search", None)], + ToolStrictLevel::Function, + ); + + assert_eq!(tag_string(&pinned), tag_string(&strict)); + assert_ne!(tag_string(&pinned), tag_string(&envelope_only)); + } + + #[test] + fn tool_strict_level_parses_case_insensitively() { + assert_eq!("AUTO".parse::(), Ok(ToolStrictLevel::Auto)); + assert_eq!( + "PARAMETER".parse::(), + Ok(ToolStrictLevel::Parameter) + ); + assert_eq!( + "function".parse::(), + Ok(ToolStrictLevel::Function) + ); + assert_eq!(ToolStrictLevel::default(), ToolStrictLevel::Auto); + assert_eq!( + serde_json::to_value(ToolStrictLevel::Auto).unwrap(), + json!("auto") + ); + assert_eq!( + serde_json::from_value::(json!("auto")).unwrap(), + ToolStrictLevel::Auto + ); + assert!("off".parse::().is_err()); + assert!("strict".parse::().is_err()); + } } diff --git a/rust/src/chat/src/output/default/unified.rs b/rust/src/chat/src/output/default/unified.rs index 964f0bf35a..023e3c3ffa 100644 --- a/rust/src/chat/src/output/default/unified.rs +++ b/rust/src/chat/src/output/default/unified.rs @@ -438,6 +438,7 @@ mod tests { finish_reason: crate::FinishReason::Stop(None), kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, })), } } diff --git a/rust/src/chat/src/output/harmony/tests.rs b/rust/src/chat/src/output/harmony/tests.rs index 8dcf84dd98..035aef3cd4 100644 --- a/rust/src/chat/src/output/harmony/tests.rs +++ b/rust/src/chat/src/output/harmony/tests.rs @@ -58,6 +58,7 @@ fn finished() -> Finished { finish_reason: FinishReason::stop_eos(), kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, } } diff --git a/rust/src/chat/src/parser/mod.rs b/rust/src/chat/src/parser/mod.rs index c10a4f11bf..5a86b24934 100644 --- a/rust/src/chat/src/parser/mod.rs +++ b/rust/src/chat/src/parser/mod.rs @@ -57,6 +57,66 @@ impl ParserSelection { } } +/// Server-side floor for tool-call structural tags (`--tool-strict-level`). +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Default, + DeserializeFromStr, + SerializeDisplay, +)] +pub enum ToolStrictLevel { + /// Derive tool constraints from the request's tool choice and per-tool strictness. + #[default] + Auto, + /// Constrain the tool-call envelope for every request with tools. + Function, + /// Additionally pin argument schemas, as if every tool were `strict: true`. + Parameter, +} + +impl ToolStrictLevel { + pub const AUTO_LITERAL: &str = "auto"; + pub const FUNCTION_LITERAL: &str = "function"; + pub const PARAMETER_LITERAL: &str = "parameter"; +} + +impl FromStr for ToolStrictLevel { + type Err = String; + + fn from_str(value: &str) -> Result { + if value.eq_ignore_ascii_case(Self::AUTO_LITERAL) { + Ok(Self::Auto) + } else if value.eq_ignore_ascii_case(Self::FUNCTION_LITERAL) { + Ok(Self::Function) + } else if value.eq_ignore_ascii_case(Self::PARAMETER_LITERAL) { + Ok(Self::Parameter) + } else { + Err(format!( + "unknown tool strict level {value:?}; expected one of {}, {}, {}", + Self::AUTO_LITERAL, + Self::FUNCTION_LITERAL, + Self::PARAMETER_LITERAL + )) + } + } +} + +impl fmt::Display for ToolStrictLevel { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::Auto => Self::AUTO_LITERAL, + Self::Function => Self::FUNCTION_LITERAL, + Self::Parameter => Self::PARAMETER_LITERAL, + }) + } +} + /// Validate explicit parser override names without starting request processing. pub fn validate_parser_overrides( tool_call_parser: &ParserSelection, diff --git a/rust/src/chat/tests/chat.rs b/rust/src/chat/tests/chat.rs index ed96134cdc..ab366b8827 100644 --- a/rust/src/chat/tests/chat.rs +++ b/rust/src/chat/tests/chat.rs @@ -227,6 +227,7 @@ impl ChatBackend for FakeChatBackend { self.tokenizer(), options.tool_call_parser, options.reasoning_parser, + options.tool_strict_level, )?)) } } diff --git a/rust/src/chat/tests/roundtrip.rs b/rust/src/chat/tests/roundtrip.rs index 5669e255e0..9cba9f0aa2 100644 --- a/rust/src/chat/tests/roundtrip.rs +++ b/rust/src/chat/tests/roundtrip.rs @@ -717,6 +717,7 @@ async fn parse_completion( let processor = backends.chat_backend.new_chat_output_processor( &mut request, NewChatOutputProcessorOptions { + tool_strict_level: vllm_chat::ToolStrictLevel::Auto, tool_call_parser: &case.tool_call_parser, reasoning_parser: &case.reasoning_parser, }, @@ -805,6 +806,7 @@ fn decoded_completion_stream( finish_reason: FinishReason::stop_eos(), kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, })), } }); @@ -816,6 +818,7 @@ fn decoded_completion_stream( finish_reason: FinishReason::stop_eos(), kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, }); events.push(DecodedTextEvent::TextDelta { decoded: vllm_text::DecodedText::unattributed(chunk.delta), diff --git a/rust/src/cmd/src/cli.rs b/rust/src/cmd/src/cli.rs index cda2e6c1a8..5bdf66da57 100644 --- a/rust/src/cmd/src/cli.rs +++ b/rust/src/cmd/src/cli.rs @@ -24,6 +24,7 @@ use serde_with::{DefaultOnNull, OneOrMany, serde_as}; use thiserror_ext::AsReport as _; use uuid::Uuid; use vllm_chat::GenerationConfigMode; +use vllm_chat::ToolStrictLevel; use vllm_chat::multimodal::MmLimitPerPrompt; use vllm_engine_core_client::TransportMode; use vllm_managed_engine::ManagedEngineConfig; @@ -134,6 +135,10 @@ pub struct RenderArgs { /// `none` to disable parsing. #[arg(long, default_value_t)] reasoning_parser: ParserSelection, + /// Server-side floor for structural-tag based tool calling: `auto`, + /// `function`, or `parameter`. + #[arg(long, default_value_t)] + tool_strict_level: ToolStrictLevel, /// Select the native chat renderer implementation. #[arg(long = "tokenizer-mode", default_value_t)] renderer: RendererSelection, @@ -171,6 +176,7 @@ impl RenderArgs { port: self.port, tool_call_parser: self.tool_call_parser, reasoning_parser: self.reasoning_parser, + tool_strict_level: self.tool_strict_level, renderer: self.renderer, chat_template: self.chat_template, default_chat_template_kwargs: self.default_chat_template_kwargs.unwrap_or_default(), @@ -241,6 +247,14 @@ pub struct SharedRuntimeArgs { #[arg(long, default_value_t)] #[serde(default = "default_py_bootstrap_parser_selection")] pub reasoning_parser: ParserSelection, + /// Server-side floor for structural-tag based tool calling, applied on + /// top of the per-tool `strict` field: `auto` follows the request's + /// tool choice and per-tool strictness, + /// `function` constrains the tool-call envelope for every request with + /// tools, `parameter` additionally pins argument schemas. + #[arg(long, default_value_t)] + #[serde(default)] + pub tool_strict_level: ToolStrictLevel, /// Select the chat renderer implementation. #[arg(long = "tokenizer-mode", default_value_t)] #[serde(default, rename = "tokenizer_mode")] @@ -517,6 +531,7 @@ impl SharedRuntimeArgs { listener_mode: HttpListenerMode::InheritedFd { fd: listen_fd }, tool_call_parser: self.tool_call_parser, reasoning_parser: self.reasoning_parser, + tool_strict_level: self.tool_strict_level, renderer: self.renderer, language_model_only: self.language_model_only, chat_template: self.chat_template, @@ -574,6 +589,7 @@ impl SharedRuntimeArgs { listener_mode, tool_call_parser: self.tool_call_parser, reasoning_parser: self.reasoning_parser, + tool_strict_level: self.tool_strict_level, renderer: self.renderer, language_model_only: self.language_model_only, chat_template: self.chat_template, diff --git a/rust/src/cmd/src/cli/tests.rs b/rust/src/cmd/src/cli/tests.rs index 5cc6b19214..f7f6d0a658 100644 --- a/rust/src/cmd/src/cli/tests.rs +++ b/rust/src/cmd/src/cli/tests.rs @@ -184,6 +184,7 @@ fn serve_args_forward_python_flags_with_separator() { engine_ready_timeout_secs: 600, tool_call_parser: Auto, reasoning_parser: Auto, + tool_strict_level: Auto, renderer: Auto, language_model_only: false, max_logprobs: None, @@ -1023,6 +1024,7 @@ fn frontend_args_accept_json() { engine_ready_timeout_secs: 600, tool_call_parser: None, reasoning_parser: None, + tool_strict_level: Auto, renderer: Auto, language_model_only: false, max_logprobs: None, @@ -1693,6 +1695,7 @@ fn serve_args_accept_handshake_aliases() { engine_ready_timeout_secs: 600, tool_call_parser: Auto, reasoning_parser: Auto, + tool_strict_level: Auto, renderer: Auto, language_model_only: false, max_logprobs: None, @@ -1850,6 +1853,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() { }, tool_call_parser: Auto, reasoning_parser: Auto, + tool_strict_level: Auto, renderer: Auto, language_model_only: false, chat_template: None, @@ -1942,6 +1946,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() { }, tool_call_parser: Auto, reasoning_parser: Auto, + tool_strict_level: Auto, renderer: Auto, language_model_only: false, chat_template: None, @@ -2055,6 +2060,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present }, tool_call_parser: None, reasoning_parser: None, + tool_strict_level: Auto, renderer: Auto, language_model_only: false, chat_template: None, diff --git a/rust/src/engine-core-client/src/client.rs b/rust/src/engine-core-client/src/client.rs index 4cd7420e56..aa6aacf0e0 100644 --- a/rust/src/engine-core-client/src/client.rs +++ b/rust/src/engine-core-client/src/client.rs @@ -911,13 +911,22 @@ impl EngineCoreClient { Ok(()) } - /// Wake the engine from sleep, optionally limiting the wake-up to specific - /// tags. - pub async fn wake_up(&self, tags: Option>) -> Result<()> { - self.call_utility::<(), _>("wake_up", (tags,)).await?; + /// Release KV cache memory while keeping model weights resident. + pub async fn release_kv_cache_memory(&self) -> Result<()> { + self.call_utility::<(), _>("release_kv_cache_memory", ()).await?; Ok(()) } + /// Wake the engine from sleep, optionally limiting the wake-up to specific + /// tags, and return whether every engine is fully awake. + pub async fn wake_up(&self, tags: Option>) -> Result { + Ok(self + .call_utility::("wake_up", (tags,)) + .await? + .into_iter() + .all(|fully_awake| fully_awake)) + } + /// Pause the scheduler so generation can be halted pub async fn pause_scheduler(&self, mode: PauseMode, clear_cache: bool) -> Result<()> { self.call_utility::<(), _>("pause_scheduler", (mode, clear_cache)).await?; diff --git a/rust/src/engine-core-client/src/protocol/logprobs.rs b/rust/src/engine-core-client/src/protocol/logprobs.rs index 646c106a04..72d852ba2c 100644 --- a/rust/src/engine-core-client/src/protocol/logprobs.rs +++ b/rust/src/engine-core-client/src/protocol/logprobs.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project -mod array; +pub(crate) mod array; #[cfg(test)] mod tests; mod wire; diff --git a/rust/src/engine-core-client/src/protocol/logprobs/array.rs b/rust/src/engine-core-client/src/protocol/logprobs/array.rs index 4adf1ae975..919ef80f46 100644 --- a/rust/src/engine-core-client/src/protocol/logprobs/array.rs +++ b/rust/src/engine-core-client/src/protocol/logprobs/array.rs @@ -50,7 +50,7 @@ pub(super) fn decode_array2_u32( }) } -pub(super) fn decode_array1_u32( +pub(crate) fn decode_array1_u32( value: WireNdArray, field: &str, frames: &[Bytes], diff --git a/rust/src/engine-core-client/src/protocol/mod.rs b/rust/src/engine-core-client/src/protocol/mod.rs index 8f4ee06127..742fed59bd 100644 --- a/rust/src/engine-core-client/src/protocol/mod.rs +++ b/rust/src/engine-core-client/src/protocol/mod.rs @@ -30,6 +30,7 @@ pub mod multimodal; pub mod output; pub mod request; pub mod sampling; +pub mod sampling_mask; mod serde_utils; pub mod stats; pub mod structured_outputs; diff --git a/rust/src/engine-core-client/src/protocol/output.rs b/rust/src/engine-core-client/src/protocol/output.rs index c8a8705435..a45cbe3731 100644 --- a/rust/src/engine-core-client/src/protocol/output.rs +++ b/rust/src/engine-core-client/src/protocol/output.rs @@ -15,6 +15,7 @@ use super::serde_utils::AllowTrailingFields; use super::utility::UtilityOutput; use crate::error::{Error, Result, ext_value_decode}; use crate::protocol::logprobs::MaybeWireLogprobs; +use crate::protocol::sampling_mask::MaybeWireSamplingMask; use crate::protocol::stats::{PrefillStats, SchedulerStats}; use crate::protocol::{OpaqueValue, decode_msgpack}; @@ -127,7 +128,7 @@ pub struct EngineCoreOutput { #[serde(default)] pub mm_cache_miss_hashes: Option>, #[serde(default)] - pub new_sampling_mask: Option, + pub new_sampling_mask: Option, /// Per-request speculative-decoding acceptance metrics, set on the final /// output when `--per-request-spec-decode-metrics` is enabled. Opaque here; /// the Rust frontend does not yet surface it in responses. @@ -150,6 +151,9 @@ impl EngineCoreOutput { self.new_prompt_logprobs_tensors = (self.new_prompt_logprobs_tensors.take()) .map(|value| value.resolve(frames, "new_prompt_logprobs_tensors")) .transpose()?; + self.new_sampling_mask = (self.new_sampling_mask.take()) + .map(|value| value.resolve(frames, "new_sampling_mask")) + .transpose()?; Ok(()) } } diff --git a/rust/src/engine-core-client/src/protocol/sampling_mask.rs b/rust/src/engine-core-client/src/protocol/sampling_mask.rs new file mode 100644 index 0000000000..7f5c4592a7 --- /dev/null +++ b/rust/src/engine-core-client/src/protocol/sampling_mask.rs @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +use std::ops::{Deref, DerefMut}; + +use bytes::Bytes; +use enum_as_inner::EnumAsInner; +use serde::{Deserialize, Deserializer, Serialize}; +use serde_tuple::{Deserialize_tuple, Serialize_tuple}; + +use crate::error::{Error, Result, bail_ext_value_decode}; +use crate::protocol::logprobs::array::decode_array1_u32; +use crate::protocol::tensor::WireNdArray; + +/// Decoded sampling-mask support sets, one row per generated token position. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct SamplingMask { + pub rows: Vec>, +} + +/// Python `SamplingMaskLists` tuple before ndarray raw views are resolved. +#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple)] +pub struct WireSamplingMask { + token_ids: WireNdArray, + #[serde(default)] + offsets: Option, + #[serde(default)] + cu_num_generated_tokens: Option>, +} + +/// Sampling-mask field while it transitions from Python wire data to rows. +#[derive(Debug, Clone, PartialEq, EnumAsInner)] +pub enum MaybeWireSamplingMask { + Wire(Box), + Direct(SamplingMask), +} + +impl Deref for MaybeWireSamplingMask { + type Target = SamplingMask; + + fn deref(&self) -> &Self::Target { + match self { + Self::Wire(_) => panic!("sampling mask is still in wire format"), + Self::Direct(value) => value, + } + } +} + +impl DerefMut for MaybeWireSamplingMask { + fn deref_mut(&mut self) -> &mut Self::Target { + match self { + Self::Wire(_) => panic!("sampling mask is still in wire format"), + Self::Direct(value) => value, + } + } +} + +impl<'de> Deserialize<'de> for MaybeWireSamplingMask { + fn deserialize(deserializer: D) -> std::result::Result + where + D: Deserializer<'de>, + { + WireSamplingMask::deserialize(deserializer).map(|value| Self::Wire(Box::new(value))) + } +} + +impl Serialize for MaybeWireSamplingMask { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + match self { + Self::Wire(value) => value.serialize(serializer), + Self::Direct(value) => WireSamplingMask::from_direct(value) + .map_err(serde::ser::Error::custom)? + .serialize(serializer), + } + } +} + +impl MaybeWireSamplingMask { + pub(super) fn resolve(self, frames: &[Bytes], field_prefix: &str) -> Result { + match self { + Self::Direct(value) => Ok(Self::Direct(value)), + Self::Wire(value) => value.resolve(frames, field_prefix).map(Self::Direct), + } + } +} + +impl WireSamplingMask { + fn from_direct(value: &SamplingMask) -> std::result::Result { + let [row] = value.rows.as_slice() else { + return Err(format!( + "per-request engine-core sampling mask must contain exactly one row, got {}", + value.rows.len() + )); + }; + let token_ids = row.iter().map(|&token_id| i64::from(token_id)).collect::>(); + Ok(Self { + token_ids: WireNdArray::from_i64(vec![token_ids.len()], token_ids)?, + offsets: None, + cu_num_generated_tokens: None, + }) + } + + fn resolve(self, frames: &[Bytes], field_prefix: &str) -> Result { + if let Some(indices) = self.cu_num_generated_tokens { + bail_ext_value_decode!( + "{field_prefix}.cu_num_generated_tokens: expected None for per-request engine-core sampling-mask payload, got {indices:?}" + ); + } + + let token_ids = + decode_array1_u32(self.token_ids, &format!("{field_prefix}.token_ids"), frames)?; + let Some(offsets) = self.offsets else { + return Ok(SamplingMask { + rows: vec![token_ids], + }); + }; + let offsets = decode_array1_u32(offsets, &format!("{field_prefix}.offsets"), frames)?; + if offsets.first().copied() != Some(0) { + bail_ext_value_decode!("{field_prefix}.offsets: first offset must be zero"); + } + if offsets.windows(2).any(|pair| pair[0] > pair[1]) { + bail_ext_value_decode!("{field_prefix}.offsets: offsets must be nondecreasing"); + } + if offsets.last().copied().map(|value| value as usize) != Some(token_ids.len()) { + bail_ext_value_decode!( + "{field_prefix}.offsets: final offset {:?} does not match token_ids length {}", + offsets.last(), + token_ids.len() + ); + } + + let rows = offsets + .windows(2) + .map(|pair| token_ids[pair[0] as usize..pair[1] as usize].to_vec()) + .collect(); + Ok(SamplingMask { rows }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::protocol::dtype::{NumpyDtype, TensorDtype}; + use crate::protocol::output::{ + EngineCoreOutput, RequestBatchOutputs, decode_engine_core_outputs, + }; + use crate::protocol::tensor::WireArrayData; + use crate::protocol::{decode_msgpack, encode_msgpack}; + + #[test] + fn decodes_single_position_without_offsets() { + let wire = WireSamplingMask { + token_ids: WireNdArray::from_i64(vec![3], vec![2, 12, 16]).unwrap(), + offsets: None, + cu_num_generated_tokens: None, + }; + let encoded = encode_msgpack(&wire).unwrap(); + let decoded: MaybeWireSamplingMask = decode_msgpack(&encoded).unwrap(); + let decoded = decoded.resolve(&[], "new_sampling_mask").unwrap(); + + assert_eq!(decoded.into_direct().unwrap().rows, vec![vec![2, 12, 16]]); + } + + #[test] + fn decodes_csr_rows_and_rejects_bad_terminal_offset() { + let wire = WireSamplingMask { + token_ids: WireNdArray::from_i64(vec![4], vec![2, 12, 16, 18]).unwrap(), + offsets: Some(WireNdArray::from_i64(vec![3], vec![0, 2, 4]).unwrap()), + cu_num_generated_tokens: None, + }; + let decoded = MaybeWireSamplingMask::Wire(Box::new(wire)) + .resolve(&[], "new_sampling_mask") + .unwrap(); + assert_eq!( + decoded.into_direct().unwrap().rows, + vec![vec![2, 12], vec![16, 18]] + ); + + let malformed = WireSamplingMask { + token_ids: WireNdArray::from_i64(vec![2], vec![2, 12]).unwrap(), + offsets: Some(WireNdArray::from_i64(vec![2], vec![0, 1]).unwrap()), + cu_num_generated_tokens: None, + }; + let error = MaybeWireSamplingMask::Wire(Box::new(malformed)) + .resolve(&[], "new_sampling_mask") + .unwrap_err(); + assert!(error.to_string().contains("final offset")); + } + + #[test] + fn decodes_sampling_mask_from_multipart_aux_frames() { + let output = EngineCoreOutput { + request_id: "req-multipart-mask".to_string(), + new_token_ids: vec![16, 18], + new_sampling_mask: Some(MaybeWireSamplingMask::Wire(Box::new(WireSamplingMask { + token_ids: WireNdArray { + dtype: NumpyDtype::little(TensorDtype::I32), + shape: vec![4], + data: WireArrayData::AuxIndex(1), + }, + offsets: Some(WireNdArray { + dtype: NumpyDtype::little(TensorDtype::I64), + shape: vec![3], + data: WireArrayData::AuxIndex(2), + }), + cu_num_generated_tokens: None, + }))), + ..Default::default() + }; + let primary = encode_msgpack(&crate::protocol::output::EngineCoreOutputs::from( + RequestBatchOutputs { + outputs: vec![output], + ..Default::default() + }, + )) + .unwrap(); + let token_ids = + [2_i32, 12, 16, 18].into_iter().flat_map(i32::to_le_bytes).collect::>(); + let offsets = [0_i64, 2, 4].into_iter().flat_map(i64::to_le_bytes).collect::>(); + + let decoded = decode_engine_core_outputs(&[ + Bytes::from(primary), + Bytes::from(token_ids), + Bytes::from(offsets), + ]) + .unwrap(); + let mask = decoded.as_request_batch().unwrap().outputs[0] + .new_sampling_mask + .as_ref() + .unwrap(); + + assert_eq!(mask.rows, vec![vec![2, 12], vec![16, 18]]); + } +} diff --git a/rust/src/engine-core-client/src/tests/client.rs b/rust/src/engine-core-client/src/tests/client.rs index 739c34d6b7..eb8e713fb4 100644 --- a/rust/src/engine-core-client/src/tests/client.rs +++ b/rust/src/engine-core-client/src/tests/client.rs @@ -2712,15 +2712,15 @@ fn python_msgpack_fixtures_match_rust_encoding() { decode_value(&rmp_serde::to_vec_named(&expected_multimodal_request.mm_features).unwrap()); assert_eq!(python_mm_features, rust_mm_features); - let decoded_sampling_mask_outputs: EngineCoreOutputs = - rmp_serde::from_slice(&sampling_mask_outputs_bytes).unwrap(); + let decoded_sampling_mask_outputs = + decode_engine_core_outputs(&[bytes::Bytes::from(sampling_mask_outputs_bytes)]).unwrap(); let sampling_mask_output = &decoded_sampling_mask_outputs.as_request_batch().unwrap().outputs[0]; assert!(sampling_mask_output.mm_cache_miss_hashes.is_none()); - assert!(matches!( - sampling_mask_output.new_sampling_mask.as_ref(), - Some(Value::Array(fields)) if fields.len() == 3 - )); + assert_eq!( + sampling_mask_output.new_sampling_mask.as_ref().unwrap().rows, + vec![vec![2, 12, 16, 17, 18]] + ); let decoded_outputs: EngineCoreOutputs = rmp_serde::from_slice(&outputs_bytes).unwrap(); // Match msgspec's base-schema result for the same extended Python message. diff --git a/rust/src/engine-core-client/src/tests/python_compat.py b/rust/src/engine-core-client/src/tests/python_compat.py index 5013f9e2a5..a19dc9fdaa 100755 --- a/rust/src/engine-core-client/src/tests/python_compat.py +++ b/rust/src/engine-core-client/src/tests/python_compat.py @@ -234,14 +234,10 @@ sampling_mask_wire = [ [ ", + /// Sampling support sets aligned one-to-one with generated token positions. + pub sampling_mask: Option, } /// Prompt-scoped metadata emitted only once on the first [`GenerateOutput`] for @@ -155,6 +158,8 @@ pub struct GenerateOutput { /// Connector-specific encoder cache transfer parameters for disaggregated /// serving. pub ec_transfer_params: Option, + /// Sampling support sets aligned one-to-one with `token_ids`. + pub sampling_mask: Option, } impl GenerateOutput { @@ -202,6 +207,7 @@ impl GenerateOutput { cached_token_count: 0, kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, } } } @@ -271,6 +277,16 @@ impl Stream for GenerateOutputStream { } let logprobs = raw.new_logprobs.map(|value| value.into_direct().unwrap()); + let sampling_mask = raw.new_sampling_mask.map(|value| value.into_direct().unwrap()); + if let Some(mask) = sampling_mask.as_ref() + && mask.rows.len() != raw.new_token_ids.len() + { + return Poll::Ready(Some(Err(crate::Error::SamplingMaskTokenCountMismatch { + request_id: raw.request_id, + token_count: raw.new_token_ids.len(), + row_count: mask.rows.len(), + }))); + } let cached_token_count = raw .prefill_stats .as_ref() @@ -291,6 +307,7 @@ impl Stream for GenerateOutputStream { cached_token_count, kv_transfer_params: raw.kv_transfer_params, ec_transfer_params: raw.ec_transfer_params, + sampling_mask, }; Poll::Ready(Some(Ok(output))) @@ -341,6 +358,7 @@ impl> + Send> T { while let Some(output) = stream.next().await.transpose()? { cached_token_count = cached_token_count.max(output.cached_token_count); + let sampling_mask = output.sampling_mask; if let Some(info) = output.prompt_info { if prompt_token_ids.is_none() { prompt_token_ids = Some(info.prompt_token_ids.to_vec()); @@ -359,6 +377,9 @@ impl> + Send> T { existing.logprobs = Some(step_logprobs); } } + if let Some(mut mask) = sampling_mask { + existing.sampling_mask.get_or_insert_default().rows.append(&mut mask.rows); + } } else { collected = Some(CollectedGenerateOutput { request_id: output.request_id, @@ -374,6 +395,7 @@ impl> + Send> T { }, kv_transfer_params: None, ec_transfer_params: None, + sampling_mask, }); } @@ -387,6 +409,15 @@ impl> + Send> T { }; collected.kv_transfer_params = output.kv_transfer_params; collected.ec_transfer_params = output.ec_transfer_params; + if let Some(mask) = collected.sampling_mask.as_ref() + && mask.rows.len() != collected.token_ids.len() + { + return Err(crate::Error::SamplingMaskTokenCountMismatch { + request_id: collected.request_id, + token_count: collected.token_ids.len(), + row_count: mask.rows.len(), + }); + } return Ok(collected); } } diff --git a/rust/src/llm/tests/generate.rs b/rust/src/llm/tests/generate.rs index 2d09aff889..09b5199767 100644 --- a/rust/src/llm/tests/generate.rs +++ b/rust/src/llm/tests/generate.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: Copyright contributors to the vLLM project use std::collections::BTreeSet; -use std::sync::Once; +use std::sync::{Arc, Once}; use std::time::Duration; use futures::StreamExt as _; @@ -18,6 +18,7 @@ use vllm_engine_core_client::protocol::output::{ }; use vllm_engine_core_client::protocol::request::EngineCoreRequest; use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams; +use vllm_engine_core_client::protocol::sampling_mask::{MaybeWireSamplingMask, SamplingMask}; use vllm_engine_core_client::protocol::stats::PrefillStats; use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task}; use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId}; @@ -326,6 +327,11 @@ async fn collect_output_aggregates_raw_tokens_logprobs_and_terminal_metadata() { num_local_cached_tokens: 1, ..Default::default() }), + new_sampling_mask: Some(MaybeWireSamplingMask::Direct( + SamplingMask { + rows: vec![vec![1, 33, 99]], + }, + )), ..request_output_with_logprobs( &request.request_id, vec![33], @@ -334,15 +340,22 @@ async fn collect_output_aggregates_raw_tokens_logprobs_and_terminal_metadata() { Some(prompt_logprobs()), ) }, - request_output_with_logprobs_and_kv( - &request.request_id, - vec![44], - Some(EngineCoreFinishReason::Stop), - Some(logprobs_for_position(44, -0.3, 1, 88, -0.4)), - None, - Some(serde_json::json!({"connector": "x"})), - None, - ), + EngineCoreOutput { + new_sampling_mask: Some(MaybeWireSamplingMask::Direct( + SamplingMask { + rows: vec![vec![2, 44, 88]], + }, + )), + ..request_output_with_logprobs_and_kv( + &request.request_id, + vec![44], + Some(EngineCoreFinishReason::Stop), + Some(logprobs_for_position(44, -0.3, 1, 88, -0.4)), + None, + Some(serde_json::json!({"connector": "x"})), + None, + ) + }, ], ..Default::default() } @@ -375,6 +388,42 @@ async fn collect_output_aggregates_raw_tokens_logprobs_and_terminal_metadata() { collected.kv_transfer_params, Some(serde_json::json!({"connector": "x"})) ); + assert_eq!( + collected.sampling_mask, + Some(SamplingMask { + rows: vec![vec![1, 33, 99], vec![2, 44, 88]], + }) + ); +} + +#[tokio::test] +async fn collect_output_rejects_partial_sampling_mask() { + let output = vllm_llm::GenerateOutput { + request_id: "req-partial-mask".to_string(), + prompt_info: Some(GeneratePromptInfo { + prompt_token_ids: Arc::from([11_u32, 22]), + prompt_logprobs: None, + }), + token_ids: vec![33, 44], + logprobs: None, + finish_reason: Some(FinishReason::Length), + cached_token_count: 0, + kv_transfer_params: None, + ec_transfer_params: None, + sampling_mask: Some(SamplingMask { + rows: vec![vec![1, 33, 99]], + }), + }; + + let error = futures::stream::iter([Ok(output)]).collect_output().await.unwrap_err(); + assert!(matches!( + error, + Error::SamplingMaskTokenCountMismatch { + token_count: 2, + row_count: 1, + .. + } + )); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/rust/src/mock-engine/src/engine.rs b/rust/src/mock-engine/src/engine.rs index 38f9b21be1..4844aba120 100644 --- a/rust/src/mock-engine/src/engine.rs +++ b/rust/src/mock-engine/src/engine.rs @@ -95,12 +95,10 @@ fn utility_response( "get_supported_tasks" => utility_envelope(vec!["generate"]), "is_sleeping" => utility_envelope(false), "reset_prefix_cache" => utility_envelope(true), - "reset_mm_cache" - | "reset_encoder_cache" - | "profile" - | "sleep" - | "wake_up" - | "execute_dummy_batch" => utility_envelope(()), + "wake_up" => utility_envelope(true), + "reset_mm_cache" | "reset_encoder_cache" | "profile" | "sleep" | "execute_dummy_batch" => { + utility_envelope(()) + } _ => utility_envelope(Value::Nil), }?; diff --git a/rust/src/mock-engine/src/tests.rs b/rust/src/mock-engine/src/tests.rs index bee710fd0c..c1cfbb49cb 100644 --- a/rust/src/mock-engine/src/tests.rs +++ b/rust/src/mock-engine/src/tests.rs @@ -195,6 +195,7 @@ async fn utility_requests_return_minimal_success_responses() { assert!(client.reset_prefix_cache(false, false).await.expect("reset prefix cache")); client.reset_mm_cache().await.expect("reset mm cache"); client.reset_encoder_cache().await.expect("reset encoder cache"); + assert!(client.wake_up(Some(vec!["kv_cache".to_string()])).await.expect("wake up")); shutdown_mock(client, shutdown, task).await; } diff --git a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v32.rs b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v32.rs index 632eda600f..9bbe39367b 100644 --- a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v32.rs +++ b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v32.rs @@ -193,6 +193,34 @@ mod tests { ); } + #[test] + fn deepseek_v32_parse_complete_keeps_parameter_without_string_attr() { + // The model sometimes omits `string="..."`; the parameter must still + // be kept and converted through the schema, like `string="false"`. + let mut parser = DeepSeekV32ToolParser::new(&test_tools()); + let output = parser + .parse_complete( + "<|DSML|function_calls>\n\ + <|DSML|invoke name=\"convert\">\n\ + <|DSML|parameter name=\"whole\">5.0\n\ + <|DSML|parameter name=\"flag\" >true\n\ + <|DSML|parameter name=\"payload\" string=\"true\">{\"nested\":true}\n\ + \n\ + ", + ) + .unwrap(); + + assert_eq!(output.calls().len(), 1); + assert_eq!( + serde_json::from_str::(&output.calls()[0].arguments).unwrap(), + json!({ + "whole": 5.0, + "flag": true, + "payload": "{\"nested\":true}", + }) + ); + } + #[test] fn deepseek_v32_parse_complete_preserves_raw_closing_tag_text_in_parameter_value() { let mut parser = DeepSeekV32ToolParser::new(&test_tools()); diff --git a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v4.rs b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v4.rs index b65df235ca..032cfbe5e2 100644 --- a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v4.rs +++ b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v4.rs @@ -147,6 +147,29 @@ mod tests { ); } + #[test] + fn deepseek_v4_streaming_keeps_parameter_without_string_attr() { + let mut parser = DeepSeekV4ToolParser::new(&test_tools()); + let output = collect_stream( + &mut parser, + &[ + "<|DSML|tool_calls>\n<|DSML|invoke name=\"get_weather\">\n", + "<|DSML|parameter name=\"loc", + "ation\">Par", + "is\n", + "<|DSML|parameter name=\"date\" string=\"true\">tomorrow\n", + "\n", + ], + ); + + assert!(output.normal_text().is_empty()); + assert_eq!(output.calls().len(), 1); + assert_eq!( + serde_json::from_str::(&output.calls()[0].arguments).unwrap(), + json!({ "location": "Paris", "date": "tomorrow" }) + ); + } + #[test] fn tool_framing_preserves_body_whitespace_across_chunk_boundaries() { assert_tool_framing_preserves_body_whitespace::( diff --git a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41.rs b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41.rs index 44658b1d44..be68b6d2e5 100644 --- a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41.rs +++ b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41.rs @@ -4,8 +4,6 @@ use super::{DeepSeekDsmlToolParser, DsmlTokens}; use crate::tool::{Result, StructuralTagBuilder, Tool, ToolParser, ToolParserOutput}; -mod structural_tag; - /// Tool parser for DeepSeek V4.1's spaced DSML tags. /// /// Arguments are emitted only after a full `invoke` block is parsed. @@ -27,7 +25,7 @@ impl ToolParser for DeepSeekV41ToolParser { } fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> { - Some(&structural_tag::DeepSeekV41StructuralTagBuilder) + Some(xgrammar_structural_tag::Model::DeepSeekV41.builder()) } fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> { diff --git a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/structural_tag.rs b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/structural_tag.rs deleted file mode 100644 index d5fb6118fb..0000000000 --- a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/structural_tag.rs +++ /dev/null @@ -1,108 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -//! Visible-text grammar for DeepSeek V4.1 tool calls. -//! -//! Adapted from xgrammar-structural-tag 0.2.0's DeepSeek V4 builder, with -//! V4.1's spaced DSML tags and schema-unconstrained parameter values. - -use serde_json::Value; -use xgrammar_structural_tag::Result; -use xgrammar_structural_tag::builders::{ - StructuralTagBuilder, StructuralTagContext, StructuralTagOptions, -}; -use xgrammar_structural_tag::format::{ - Format, JsonSchemaFormat, StructuralTag, TagFormat, TriggeredTagsFormat, -}; -use xgrammar_structural_tag::tool::{BuilderToolChoice, FunctionToolParam}; - -use super::DsmlTokens; - -const TOKENS: DsmlTokens = DsmlTokens::V41; - -pub(super) struct DeepSeekV41StructuralTagBuilder; - -impl StructuralTagBuilder for DeepSeekV41StructuralTagBuilder { - fn build(&self, ctx: StructuralTagContext<'_>) -> Result { - let tags = ctx - .function_tools - .iter() - .map(|tool| call_tag(tool, ctx.options)) - .collect::>(); - let excludes = if ctx.options.exclude_special_tokens { - &["", ""][..] - } else { - &[] - }; - let format = match ctx.tool_choice { - BuilderToolChoice::Auto if tags.is_empty() => Format::any_text_excluding(excludes), - BuilderToolChoice::Auto => Format::TriggeredTags( - TriggeredTagsFormat::new( - &[TOKENS.tool_calls_start], - vec![TagFormat::new( - format!("{}\n", TOKENS.tool_calls_start), - Format::tags_with_separator(tags, "", true, false), - TOKENS.tool_calls_end, - )], - ) - .with_excludes(excludes), - ), - BuilderToolChoice::Forced | BuilderToolChoice::Required => Format::sequence(vec![ - Format::const_string(format!("\n\n{}\n", TOKENS.tool_calls_start)), - Format::tags_with_separator( - tags, - "", - true, - ctx.tool_choice == BuilderToolChoice::Forced, - ), - Format::const_string(TOKENS.tool_calls_end), - ]), - }; - // Reasoning boundaries remain owned by the engine reasoner; serving - // calls this visible-text builder with reasoning=false. - Ok(StructuralTag::new(format)) - } -} - -fn call_tag(tool: &FunctionToolParam, options: StructuralTagOptions) -> TagFormat { - // TODO: enforce tool.parameters and strict schema semantics once V4.1 - // argument conversion is available. This builder intentionally ignores - // parameters and strict: it constrains tool names and DSML/value syntax - // only. Parameter names, presence, uniqueness and value schemas remain - // unconstrained, including for strict=true. Serving still uses strict to - // decide whether auto tool choice activates a grammar. - TagFormat::new( - format!("{} name=\"{}\">\n", TOKENS.invoke_start, tool.function.name), - Format::star(parameter(options)), - format!("{}\n", TOKENS.invoke_end), - ) -} - -fn parameter(options: StructuralTagOptions) -> Format { - Format::tag( - format!("{} name=\"", TOKENS.parameter_start), - Format::sequence(vec![ - Format::regex(r#"[^"]+"#), - Format::const_string("\" string=\""), - Format::or(vec![ - Format::sequence(vec![ - Format::const_string("true\">"), - Format::any_text_excluding(&[ - TOKENS.parameter_end, - TOKENS.invoke_end, - TOKENS.tool_calls_end, - ]), - ]), - Format::sequence(vec![ - Format::const_string("false\">"), - Format::JsonSchema( - JsonSchemaFormat::new(Value::Bool(true)) - .with_any_order(options.any_order) - .with_max_whitespace_cnt(options.max_whitespace_cnt), - ), - ]), - ]), - ]), - format!("{}\n", TOKENS.parameter_end), - ) -} diff --git a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/tests.rs b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/tests.rs index d40f82d533..208d88ea33 100644 --- a/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/tests.rs +++ b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/tests.rs @@ -71,6 +71,28 @@ fn spaced_dsml_streaming_preserves_shared_schema_coercion() { } } +#[test] +fn spaced_dsml_keeps_parameter_without_string_attr() { + let wire = concat!( + "<|DSML| calls>\n", + "<|DSML| invoke name=\"lookup\">\n", + "<|DSML| parameter name=\"query\">value\n", + "<|DSML| parameter name=\"limit\">2\n", + "\n", + "", + ); + let mut parser = DeepSeekV41ToolParser::create(&tools()).unwrap(); + + let output = collect_stream(parser.as_mut(), &[wire]); + + assert!(output.normal_text().is_empty()); + assert_eq!(output.calls().len(), 1); + assert_eq!( + serde_json::from_str::(&output.calls()[0].arguments).unwrap(), + json!({ "query": "value", "limit": 2 }) + ); +} + #[test] fn v4_tags_remain_text_in_v41_dialect() { let mut parser = DeepSeekV41ToolParser::create(&tools()).unwrap(); @@ -80,7 +102,7 @@ fn v4_tags_remain_text_in_v41_dialect() { } #[test] -fn structural_tag_ignores_parameter_schemas_and_strict() { +fn structural_tag_respects_parameter_schemas_and_strict() { let parser = DeepSeekV41ToolParser::create(&tools()).unwrap(); let builder = parser.structural_tag_builder().unwrap(); let options = StructuralTagOptions::default().with_reasoning(false); @@ -106,7 +128,13 @@ fn structural_tag_ignores_parameter_schemas_and_strict() { ] { let mut function = FunctionDefinition::new("lookup").with_parameters(parameters); function.strict = strict; - assert_eq!(build(function), expected); + let tag = build(function); + if strict == Some(false) { + assert_eq!(tag, expected); + } else { + assert_ne!(tag, expected); + } + assert!(tag.to_json_string().unwrap().contains("deepseek_v4_1_xml")); } } } diff --git a/rust/src/parser/src/tool/deepseek_dsml/mod.rs b/rust/src/parser/src/tool/deepseek_dsml/mod.rs index 01a3fbe282..9c7dd20331 100644 --- a/rust/src/parser/src/tool/deepseek_dsml/mod.rs +++ b/rust/src/parser/src/tool/deepseek_dsml/mod.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: Copyright contributors to the vLLM project use winnow::ascii::{multispace0 as ws0, multispace1 as ws1}; -use winnow::combinator::{alt, delimited, eof, repeat, seq, terminated}; +use winnow::combinator::{alt, delimited, eof, opt, preceded, repeat, seq, terminated}; use winnow::prelude::*; use winnow::stream::Partial; use winnow::token::{literal, rest, take_until}; @@ -293,13 +293,16 @@ fn parse_invoke_params(invoke_body: &str, tokens: DsmlTokens) -> ModalResult ModalResult { seq! {DsmlParameter { _: literal(tokens.parameter_start), _: ws1, name: name_attr.map(|name: &str| name.to_string()), - _: ws1, - is_string: string_attr.map(|value| value == "true"), + is_string: opt(preceded(ws1, string_attr)).map(|value| value == Some("true")), _: ws0, _: ">", value: take_until(0.., tokens.parameter_end).map(str::to_string), @@ -315,7 +318,7 @@ fn name_attr<'i>(input: &mut &'i str) -> ModalResult<&'i str> { /// Parse a string attribute. fn string_attr<'i>(input: &mut &'i str) -> ModalResult<&'i str> { - delimited("string=\"", alt(("true", "false")), "\"").parse_next(input) + delimited("string=\"", take_until(0.., "\""), "\"").parse_next(input) } /// Parse a DSML name attribute. diff --git a/rust/src/parser/src/tool/hy/structural_tag.rs b/rust/src/parser/src/tool/hy/structural_tag.rs index cce98d5b09..e883acbafd 100644 --- a/rust/src/parser/src/tool/hy/structural_tag.rs +++ b/rust/src/parser/src/tool/hy/structural_tag.rs @@ -219,7 +219,7 @@ mod tests { ToolChoice::required(), ); - expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"\n"},{"type":"tags_with_separator","tags":[{"begin":"get_weather\n","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"city"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""]},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]},{"type":"star","content":{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"days"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""]},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]}}]},"end":""}],"separator":"\n","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":"\n"}]}}"#]].assert_eq(&tag.to_json_string().unwrap()); + expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"\n"},{"type":"tags_with_separator","tags":[{"type":"tag","begin":"get_weather\n","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"city"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""],"max_tokens":null,"max_chars":null},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]},{"type":"star","content":{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"days"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""],"max_tokens":null,"max_chars":null},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]}}]},"end":""}],"separator":"\n","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":"\n"}]}}"#]].assert_eq(&tag.to_json_string().unwrap()); } #[test] @@ -238,7 +238,7 @@ mod tests { ToolChoice::required(), ); - expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"tags_with_separator","tags":[{"begin":"get_weather","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"city"},{"type":"const_string","value":""},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","",""]},{"type":"const_string","value":""}]}]},"end":""}],"separator":"","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":""}]}}"#]].assert_eq(&tag.to_json_string().unwrap()); + expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"tags_with_separator","tags":[{"type":"tag","begin":"get_weather","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"city"},{"type":"const_string","value":""},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","",""],"max_tokens":null,"max_chars":null},{"type":"const_string","value":""}]}]},"end":""}],"separator":"","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":""}]}}"#]].assert_eq(&tag.to_json_string().unwrap()); } #[test] @@ -259,7 +259,7 @@ mod tests { ); let value = serde_json::to_value(tag).unwrap(); - expect![[r#"{"type":"sequence","elements":[{"type":"star","content":{"type":"or","elements":[{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"query"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""]},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]},{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"limit"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""]},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]}]}}]}"#]].assert_eq( + expect![[r#"{"type":"sequence","elements":[{"type":"star","content":{"type":"or","elements":[{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"query"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""],"max_tokens":null,"max_chars":null},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]},{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"limit"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""],"max_tokens":null,"max_chars":null},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]}]}}]}"#]].assert_eq( &serde_json::to_string( &value["format"]["elements"][1]["tags"][0]["content"], ) @@ -276,8 +276,8 @@ mod tests { let auto = build("", &tools, ToolChoice::auto()).to_json_string().unwrap(); let forced = build("", &tools, ToolChoice::function("lookup")).to_json_string().unwrap(); - expect![[r#"{"type":"structural_tag","format":{"type":"triggered_tags","triggers":[""],"tags":[{"begin":"\n","content":{"type":"tags_with_separator","tags":[{"begin":"search\n","content":{"type":"any_text","excludes":[]},"end":""},{"begin":"lookup\n","content":{"type":"any_text","excludes":[]},"end":""}],"separator":"\n","at_least_one":true,"stop_after_first":false},"end":"\n"}],"at_least_one":false,"stop_after_first":false,"excludes":[]}}"#]].assert_eq(&auto); - expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"\n"},{"type":"tag","begin":"lookup\n","content":{"type":"any_text","excludes":[]},"end":""},{"type":"const_string","value":"\n"}]}}"#]].assert_eq(&forced); + expect![[r#"{"type":"structural_tag","format":{"type":"triggered_tags","triggers":[""],"tags":[{"type":"tag","begin":"\n","content":{"type":"tags_with_separator","tags":[{"type":"tag","begin":"search\n","content":{"type":"any_text","excludes":[],"max_tokens":null,"max_chars":null},"end":""},{"type":"tag","begin":"lookup\n","content":{"type":"any_text","excludes":[],"max_tokens":null,"max_chars":null},"end":""}],"separator":"\n","at_least_one":true,"stop_after_first":false},"end":"\n"}],"at_least_one":false,"stop_after_first":false,"excludes":[]}}"#]].assert_eq(&auto); + expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"\n"},{"type":"tag","begin":"lookup\n","content":{"type":"any_text","excludes":[],"max_tokens":null,"max_chars":null},"end":""},{"type":"const_string","value":"\n"}]}}"#]].assert_eq(&forced); } #[test] @@ -314,7 +314,7 @@ mod tests { ); let value = serde_json::to_value(tag).unwrap(); - expect![[r#"{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"query"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""]},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]},{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"tenant"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""]},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]}]}"#]].assert_eq( + expect![[r#"{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"query"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""],"max_tokens":null,"max_chars":null},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]},{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"tenant"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""],"max_tokens":null,"max_chars":null},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]}]}"#]].assert_eq( &serde_json::to_string( &value["format"]["elements"][1]["tags"][0]["content"], ) diff --git a/rust/src/parser/src/unified/kimi_k3/structural_tag.rs b/rust/src/parser/src/unified/kimi_k3/structural_tag.rs index 0a86a274e1..fc30f455e9 100644 --- a/rust/src/parser/src/unified/kimi_k3/structural_tag.rs +++ b/rust/src/parser/src/unified/kimi_k3/structural_tag.rs @@ -6,14 +6,14 @@ use serde_json::{Map, Value}; use xgrammar_structural_tag::Result; use xgrammar_structural_tag::builders::{ - StructuralTagBuilder, StructuralTagContext, StructuralTagOptions, + ReasoningMode, StructuralTagBuilder, StructuralTagContext, StructuralTagOptions, }; use xgrammar_structural_tag::format::{Format, JsonSchemaFormat, StructuralTag, TagFormat}; use xgrammar_structural_tag::tool::{BuilderToolChoice, FunctionToolParam, function_parameters}; use super::{ ARG_CLOSE, CALL_CLOSE, CLOSE, END_OF_MSG, JSON_CLOSE, JSON_OPEN, MESSAGE_CLOSE, OPEN, - RESPONSE_CLOSE, RESPONSE_OPEN, SEP, THINK_CLOSE, TOOLS_CLOSE, TOOLS_OPEN, + RESPONSE_CLOSE, RESPONSE_OPEN, SEP, THINK_CLOSE, THINK_OPEN, TOOLS_CLOSE, TOOLS_OPEN, }; pub(super) static KIMI_K3_STRUCTURAL_TAG_BUILDER: KimiK3StructuralTagBuilder = @@ -50,14 +50,23 @@ impl StructuralTagBuilder for KimiK3StructuralTagBuilder { } } -fn response_prefix(reasoning: bool) -> Vec { +fn response_prefix(reasoning: ReasoningMode) -> Vec { let mut elements = Vec::new(); - if reasoning { - elements.push(Format::tag( - "", + if reasoning != ReasoningMode::Disabled { + let prefix = Format::tag( + if reasoning == ReasoningMode::Auto { + THINK_OPEN + } else { + "" + }, Format::any_text_excluding(&[THINK_CLOSE, END_OF_MSG]), THINK_CLOSE, - )); + ); + elements.push(if reasoning == ReasoningMode::Auto { + Format::optional(prefix) + } else { + prefix + }); elements.push(Format::const_string(RESPONSE_OPEN)); } else { elements.push(Format::optional(Format::const_string(RESPONSE_OPEN))); @@ -386,7 +395,7 @@ mod tests { ) .unwrap(); - expect![[r##"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"optional","content":{"type":"const_string","value":"<|open|>response<|sep|>"}},{"type":"tag","begin":"","content":{"type":"any_text","excludes":["<|open|>","<|close|>","<|end_of_msg|>"]},"end":"<|close|>response<|sep|>"},{"type":"tag","begin":"<|open|>tools<|sep|>","content":{"type":"tags_with_separator","tags":[{"begin":"<|open|>call tool=\"get_weather\" index=\"","content":{"type":"sequence","elements":[{"type":"regex","pattern":"[1-9][0-9]*"},{"type":"const_string","value":"\"<|sep|>"},{"type":"or","elements":[{"type":"plus","content":{"type":"or","elements":[{"type":"tag","begin":"<|open|>argument key=\"unit\" type=\"string\"<|sep|>","content":{"type":"or","elements":[{"type":"const_string","value":"celsius"},{"type":"const_string","value":"fahrenheit"}]},"end":"<|close|>argument<|sep|>"},{"type":"tag","begin":"<|open|>argument key=\"place\" type=\"object\"<|sep|>","content":{"type":"json_schema","json_schema":{"$ref":"#/$defs/place","type":"object","$defs":{"place":{"type":"object","properties":{"city":{"type":"string"}}}}},"style":"json","any_order":false,"max_whitespace_cnt":null},"end":"<|close|>argument<|sep|>"}]}},{"type":"tag","begin":"<|open|>json type=\"object\"<|sep|>","content":{"type":"json_schema","json_schema":{"$defs":{"place":{"type":"object","properties":{"city":{"type":"string"}}}},"type":"object","properties":{"unit":{"type":"string","enum":["celsius","fahrenheit"]},"place":{"$ref":"#/$defs/place","type":"object"}},"required":["place"]},"style":"json","any_order":false,"max_whitespace_cnt":null},"end":"<|close|>json<|sep|>"}]}]},"end":"<|close|>call<|sep|>"}],"separator":"","at_least_one":true,"stop_after_first":false},"end":"<|close|>tools<|sep|>"},{"type":"optional","content":{"type":"const_string","value":"<|close|>message<|sep|>"}}]}}"##]].assert_eq(&tag.to_json_string().unwrap()); + expect![[r##"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"optional","content":{"type":"const_string","value":"<|open|>response<|sep|>"}},{"type":"tag","begin":"","content":{"type":"any_text","excludes":["<|open|>","<|close|>","<|end_of_msg|>"],"max_tokens":null,"max_chars":null},"end":"<|close|>response<|sep|>"},{"type":"tag","begin":"<|open|>tools<|sep|>","content":{"type":"tags_with_separator","tags":[{"type":"tag","begin":"<|open|>call tool=\"get_weather\" index=\"","content":{"type":"sequence","elements":[{"type":"regex","pattern":"[1-9][0-9]*"},{"type":"const_string","value":"\"<|sep|>"},{"type":"or","elements":[{"type":"plus","content":{"type":"or","elements":[{"type":"tag","begin":"<|open|>argument key=\"unit\" type=\"string\"<|sep|>","content":{"type":"or","elements":[{"type":"const_string","value":"celsius"},{"type":"const_string","value":"fahrenheit"}]},"end":"<|close|>argument<|sep|>"},{"type":"tag","begin":"<|open|>argument key=\"place\" type=\"object\"<|sep|>","content":{"type":"json_schema","json_schema":{"$ref":"#/$defs/place","type":"object","$defs":{"place":{"type":"object","properties":{"city":{"type":"string"}}}}},"style":"json","any_order":false,"max_whitespace_cnt":null},"end":"<|close|>argument<|sep|>"}]}},{"type":"tag","begin":"<|open|>json type=\"object\"<|sep|>","content":{"type":"json_schema","json_schema":{"$defs":{"place":{"type":"object","properties":{"city":{"type":"string"}}}},"type":"object","properties":{"unit":{"type":"string","enum":["celsius","fahrenheit"]},"place":{"$ref":"#/$defs/place","type":"object"}},"required":["place"]},"style":"json","any_order":false,"max_whitespace_cnt":null},"end":"<|close|>json<|sep|>"}]}]},"end":"<|close|>call<|sep|>"}],"separator":"","at_least_one":true,"stop_after_first":false},"end":"<|close|>tools<|sep|>"},{"type":"optional","content":{"type":"const_string","value":"<|close|>message<|sep|>"}}]}}"##]].assert_eq(&tag.to_json_string().unwrap()); } #[test] diff --git a/rust/src/server/Cargo.toml b/rust/src/server/Cargo.toml index caccbdefc5..8990ba2f3b 100644 --- a/rust/src/server/Cargo.toml +++ b/rust/src/server/Cargo.toml @@ -19,6 +19,7 @@ itertools.workspace = true libc.workspace = true llm-multimodal.workspace = true openssl.workspace = true +prost.workspace = true prost-types.workspace = true rmp-serde.workspace = true rmpv.workspace = true diff --git a/rust/src/server/examples/external_engine_openai_qwen.rs b/rust/src/server/examples/external_engine_openai_qwen.rs index ad3872843a..3de227babd 100644 --- a/rust/src/server/examples/external_engine_openai_qwen.rs +++ b/rust/src/server/examples/external_engine_openai_qwen.rs @@ -16,6 +16,7 @@ use clap::Parser; use futures::StreamExt as _; use tokio_util::sync::CancellationToken; use tracing_subscriber::EnvFilter; +use vllm_chat::ToolStrictLevel; use vllm_engine_core_client::TransportMode; use vllm_server::{ ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, CorsConfig, @@ -70,6 +71,7 @@ async fn main() -> Result<()> { }, tool_call_parser: ParserSelection::Auto, reasoning_parser: ParserSelection::Auto, + tool_strict_level: ToolStrictLevel::Auto, renderer: RendererSelection::Auto, language_model_only: false, chat_template: None, diff --git a/rust/src/server/src/config.rs b/rust/src/server/src/config.rs index 4c337adbef..afcfaeccfb 100644 --- a/rust/src/server/src/config.rs +++ b/rust/src/server/src/config.rs @@ -14,6 +14,7 @@ use serde_json::Value; use vllm_chat::multimodal::MmLimitPerPrompt; use vllm_chat::{ ChatTemplateContentFormatOption, GenerationConfigMode, ParserSelection, RendererSelection, + ToolStrictLevel, }; use vllm_engine_core_client::{CoordinatorMode as EngineCoreCoordinatorMode, TransportMode}; use vllm_text::backend::hf::HfOverrides; @@ -229,6 +230,8 @@ pub struct Config { pub tool_call_parser: ParserSelection, /// Reasoning parser selection. pub reasoning_parser: ParserSelection, + /// Server-side floor for tool-call structural tags. + pub tool_strict_level: ToolStrictLevel, /// Chat renderer selection. pub renderer: RendererSelection, /// Disable frontend-side multimodal preprocessing and render the model as diff --git a/rust/src/server/src/grpc/convert.rs b/rust/src/server/src/grpc/convert.rs index 5f6ce5d42b..0f28e147ef 100644 --- a/rust/src/server/src/grpc/convert.rs +++ b/rust/src/server/src/grpc/convert.rs @@ -316,6 +316,11 @@ pub fn to_sequence_output( _ => (vec![], vec![], vec![]), }; + let sampling_mask = finished + .and_then(|finished| finished.sampling_mask.as_ref()) + .map(|mask| mask.rows.iter().map(|row| pb::TokenIds { ids: row.clone() }).collect()) + .unwrap_or_default(); + Ok(pb::SequenceOutput { index: 0, // TODO: multi-sequence (n > 1) not supported text: if opts.output_text { @@ -333,6 +338,7 @@ pub fn to_sequence_output( ranks: rank_values, candidate_tokens: candidates, finish_info, + sampling_mask, }) } @@ -524,6 +530,7 @@ impl ResponseOpts { mod tests { use prost::Message as _; use vllm_engine_core_client::protocol::output::StopReason; + use vllm_engine_core_client::protocol::sampling_mask::SamplingMask; use vllm_text::{ FinishReason, Finished, Prompt, SamplingHints, SamplingLimits, lower_sampling_params, }; @@ -720,6 +727,7 @@ mod tests { finish_reason: reason, kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, } } @@ -804,4 +812,40 @@ mod tests { assert_eq!(finish.finish_reason, PbFinishReason::Stop as i32); assert_eq!(finish.stop_reason, Some(PbStopReason::EosTokenId(30))); } + + #[test] + fn sequence_output_only_carries_sampling_mask_on_terminal_output() { + let mut fin = finished(FinishReason::Length); + fin.sampling_mask = Some(SamplingMask { + rows: vec![vec![1, 10], vec![2, 20]], + }); + + let terminal = + to_sequence_output("", &[10, 20], None, Some(&fin), &ResponseOpts::default()) + .expect("terminal output"); + let intermediate = to_sequence_output("", &[9], None, None, &ResponseOpts::default()) + .expect("intermediate output"); + + assert_eq!( + terminal.sampling_mask, + vec![ + pb::TokenIds { ids: vec![1, 10] }, + pb::TokenIds { ids: vec![2, 20] } + ] + ); + assert!(intermediate.sampling_mask.is_empty()); + } + + #[test] + fn sampling_mask_uses_additive_proto_tag_ten() { + let response = pb::SequenceOutput { + sampling_mask: vec![pb::TokenIds { ids: vec![7, 8] }], + ..Default::default() + }; + + assert_eq!( + response.encode_to_vec(), + vec![0x52, 0x04, 0x0a, 0x02, 0x07, 0x08] + ); + } } diff --git a/rust/src/server/src/grpc/inference.rs b/rust/src/server/src/grpc/inference.rs index 72943b6af0..d56415c747 100644 --- a/rust/src/server/src/grpc/inference.rs +++ b/rust/src/server/src/grpc/inference.rs @@ -257,6 +257,7 @@ impl pb::inference_server::Inference for InferenceServiceImpl { finish_reason: collected.finish_reason, kv_transfer_params: collected.kv_transfer_params, ec_transfer_params: collected.ec_transfer_params, + sampling_mask: collected.sampling_mask, }; let outputs = convert::to_sequence_output( diff --git a/rust/src/server/src/grpc/tests.rs b/rust/src/server/src/grpc/tests.rs index f4b35c9df4..18b6e6eb29 100644 --- a/rust/src/server/src/grpc/tests.rs +++ b/rust/src/server/src/grpc/tests.rs @@ -244,6 +244,7 @@ impl ChatBackend for FakeTextBackend { self.tokenizer(), options.tool_call_parser, options.reasoning_parser, + options.tool_strict_level, )?)) } } @@ -292,6 +293,7 @@ impl ChatBackend for FakeMultimodalBackend { self.tokenizer(), options.tool_call_parser, options.reasoning_parser, + options.tool_strict_level, )?)) } } diff --git a/rust/src/server/src/lib.rs b/rust/src/server/src/lib.rs index 49d3d57f32..91564db73c 100644 --- a/rust/src/server/src/lib.rs +++ b/rust/src/server/src/lib.rs @@ -139,7 +139,8 @@ async fn build_state(config: &Config) -> Result> { let chat = ChatLlm::new(text, chat_backend) .with_tool_call_parser(config.tool_call_parser.clone()) - .with_reasoning_parser(config.reasoning_parser.clone()); + .with_reasoning_parser(config.reasoning_parser.clone()) + .with_tool_strict_level(config.tool_strict_level); let state = Arc::new( AppState::new(served_model_names, chat) diff --git a/rust/src/server/src/middleware/metrics.rs b/rust/src/server/src/middleware/metrics.rs index 1e6a79a556..de07620cf7 100644 --- a/rust/src/server/src/middleware/metrics.rs +++ b/rust/src/server/src/middleware/metrics.rs @@ -26,6 +26,7 @@ const EXCLUDED_HANDLERS: &[&str] = &[ "/reset_encoder_cache", "/collective_rpc", "/sleep", + "/release_kv_cache_memory", "/wake_up", "/is_sleeping", ]; diff --git a/rust/src/server/src/render.rs b/rust/src/server/src/render.rs index 4fc55fe60b..50485e26a4 100644 --- a/rust/src/server/src/render.rs +++ b/rust/src/server/src/render.rs @@ -10,7 +10,7 @@ use tokio_util::sync::CancellationToken; use tracing::info; use vllm_chat::{ ChatRequestProcessor, ChatTemplateContentFormatOption, LoadModelBackendsOptions, - ParserSelection, RendererSelection, load_model_backends, + ParserSelection, RendererSelection, ToolStrictLevel, load_model_backends, }; use vllm_text::TextRequestProcessor; use vllm_text::backend::hf::HfOverrides; @@ -32,6 +32,7 @@ pub struct RenderConfig { pub port: u16, pub tool_call_parser: ParserSelection, pub reasoning_parser: ParserSelection, + pub tool_strict_level: ToolStrictLevel, pub renderer: RendererSelection, pub chat_template: Option, pub default_chat_template_kwargs: HashMap, @@ -88,10 +89,12 @@ async fn build_state(config: &RenderConfig) -> Result> { let max_model_len = config.max_model_len.unwrap_or(u32::MAX); let text = TextRequestProcessor::new(loaded.text_backend, max_model_len) .with_max_logprobs(config.max_logprobs); - let chat = ChatRequestProcessor::render_only(loaded.chat_backend).with_parser_selections( - config.tool_call_parser.clone(), - config.reasoning_parser.clone(), - ); + let chat = ChatRequestProcessor::render_only(loaded.chat_backend) + .with_parser_selections( + config.tool_call_parser.clone(), + config.reasoning_parser.clone(), + ) + .with_tool_strict_level(config.tool_strict_level); Ok(Arc::new(RenderState { model: config.model.clone(), served_model_names, @@ -154,6 +157,7 @@ mod tests { shutdown.cancel(); let error = serve_render( RenderConfig { + tool_strict_level: ToolStrictLevel::Auto, model: "test-model".to_string(), revision: None, hf_overrides: Default::default(), diff --git a/rust/src/server/src/routes.rs b/rust/src/server/src/routes.rs index 41e4987882..4611d0c6d6 100644 --- a/rust/src/server/src/routes.rs +++ b/rust/src/server/src/routes.rs @@ -6,6 +6,7 @@ mod cache; mod collective_rpc; mod health; mod inference; +mod kv_event_sources; mod load; mod lora; mod metrics; @@ -91,6 +92,7 @@ fn build_router_with_options( .route("/metrics", get(metrics::scrape)) .route("/load", get(load::load)) .route("/version", get(version::version)) + .route("/kv_event_sources", get(kv_event_sources::kv_event_sources)) // OpenAI-compatible endpoints .route("/v1/models", get(openai::list_models)) .route("/v1/completions", post(openai::completions)) @@ -142,6 +144,10 @@ fn build_router_with_options( .route("/weight_info", get(weight_transfer::weight_info)) .route("/abort_requests", post(abort_requests::abort_requests)) .route("/sleep", post(sleep::sleep)) + .route( + "/release_kv_cache_memory", + post(sleep::release_kv_cache_memory), + ) .route("/wake_up", post(sleep::wake_up)) .route("/is_sleeping", get(sleep::is_sleeping)) .route("/pause", post(pause::pause)) diff --git a/rust/src/server/src/routes/http_client_tests.rs b/rust/src/server/src/routes/http_client_tests.rs index a0e0872e64..762d974b9b 100644 --- a/rust/src/server/src/routes/http_client_tests.rs +++ b/rust/src/server/src/routes/http_client_tests.rs @@ -168,6 +168,7 @@ impl ChatBackend for FakeChatBackend { self.tokenizer(), options.tool_call_parser, options.reasoning_parser, + options.tool_strict_level, )?)) } } diff --git a/rust/src/server/src/routes/inference/generate.rs b/rust/src/server/src/routes/inference/generate.rs index 2ce42d8c33..f82059dffd 100644 --- a/rust/src/server/src/routes/inference/generate.rs +++ b/rust/src/server/src/routes/inference/generate.rs @@ -481,6 +481,7 @@ mod tests { cached_token_count: 0, kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, }), Ok(GenerateOutput { request_id: String::new(), @@ -494,6 +495,7 @@ mod tests { cached_token_count: 2, kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, }), ]); @@ -575,6 +577,7 @@ mod tests { cached_token_count: 0, kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, } } @@ -658,6 +661,7 @@ mod tests { kv_transfer_params: None, ec_transfer_params: None, prompt_token_ids: vec![10, 20], + sampling_mask: None, }; let response = collect_generate( @@ -679,36 +683,6 @@ mod tests { assert!(response.mm_placeholders.is_none()); } - #[test] - fn stream_chunk_omits_absent_fields() { - // Contract shared with the Python frontend: absent optional fields - // are omitted from stream chunks, not serialized as null. - let chunk = GenerateStreamResponse { - request_id: "raw-1".to_string(), - choices: vec![GenerateResponseStreamChoice { - index: 0, - logprobs: None, - finish_reason: None, - token_ids: vec![1], - }], - usage: None, - prompt_token_ids: None, - mm_placeholders: None, - }; - - let json = serde_json::to_value(&chunk).expect("serialize chunk"); - - for key in ["usage", "prompt_token_ids", "mm_placeholders"] { - assert!(json.get(key).is_none(), "{key} should be omitted"); - } - for key in ["logprobs", "finish_reason"] { - assert!( - json["choices"][0].get(key).is_none(), - "{key} should be omitted" - ); - } - } - #[test] fn extract_mm_placeholders_returns_none_without_features() { assert!(extract_mm_placeholders(None).is_none()); @@ -774,6 +748,7 @@ mod tests { kv_transfer_params: None, ec_transfer_params: None, prompt_token_ids: vec![10, 20], + sampling_mask: None, }; let response = collect_generate( @@ -806,6 +781,7 @@ mod tests { }, kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, prompt_token_ids, }; diff --git a/rust/src/server/src/routes/kv_event_sources.rs b/rust/src/server/src/routes/kv_event_sources.rs new file mode 100644 index 0000000000..c3d36493a9 --- /dev/null +++ b/rust/src/server/src/routes/kv_event_sources.rs @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +use std::collections::BTreeMap; +use std::sync::Arc; + +use axum::Json; +use axum::extract::State; +use vllm_engine_core_client::protocol::handshake::KvEventsConfig; + +use crate::state::AppState; + +/// KV-event publisher config of each data-parallel rank, keyed by rank. +pub async fn kv_event_sources( + State(state): State>, +) -> Json> { + Json( + state + .engine_core_client() + .ready_responses() + .into_iter() + .filter_map(|ready| Some((ready.data_parallel_rank, ready.kv_events_config.clone()?))) + .collect(), + ) +} diff --git a/rust/src/server/src/routes/openai/completions.rs b/rust/src/server/src/routes/openai/completions.rs index a122afbcd9..06874a43bd 100644 --- a/rust/src/server/src/routes/openai/completions.rs +++ b/rust/src/server/src/routes/openai/completions.rs @@ -694,6 +694,7 @@ mod tests { ))), kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, }), )), ]); @@ -806,6 +807,7 @@ mod tests { finish_reason: FinishReason::Length, kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, }), )), ]); @@ -858,6 +860,7 @@ mod tests { finish_reason: FinishReason::Length, kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, }), )), ]); @@ -913,6 +916,7 @@ mod tests { finish_reason: FinishReason::Length, kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, }), )), ]); @@ -985,6 +989,7 @@ mod tests { finish_reason: FinishReason::Length, kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, }), )), ]); @@ -1059,6 +1064,7 @@ mod tests { finish_reason: FinishReason::Length, kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, }), )), ]); diff --git a/rust/src/server/src/routes/sleep.rs b/rust/src/server/src/routes/sleep.rs index 4bf9981402..d64a0850b6 100644 --- a/rust/src/server/src/routes/sleep.rs +++ b/rust/src/server/src/routes/sleep.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use axum::Json; use axum::extract::rejection::QueryRejection; -use axum::extract::{Query, State}; +use axum::extract::{Query, RawQuery, State}; use axum::http::StatusCode; use serde::{Deserialize, Serialize}; use vllm_engine_core_client::protocol::utility::PauseMode; @@ -27,12 +27,6 @@ pub(crate) struct SleepParams { mode: PauseMode, } -#[derive(Debug, Default, Deserialize)] -pub(crate) struct WakeUpParams { - #[serde(default)] - tags: Option>, -} - const fn default_sleep_level() -> u32 { 1 } @@ -57,14 +51,35 @@ pub async fn sleep( Ok(StatusCode::OK) } -/// Wake the engine from sleep mode. -pub async fn wake_up( +/// Release KV cache memory while keeping model weights resident. +pub async fn release_kv_cache_memory( State(state): State>, - Query(params): Query, ) -> Result { state .engine_core_client() - .wake_up(params.tags) + .release_kv_cache_memory() + .await + .map_err(|error| utility_call_error("release_kv_cache_memory", error))?; + + Ok(StatusCode::OK) +} + +/// Wake the engine from sleep mode. +pub async fn wake_up( + State(state): State>, + RawQuery(query): RawQuery, +) -> Result { + let tags = query.and_then(|query| { + let tags = url::form_urlencoded::parse(query.as_bytes()) + .filter(|(key, _)| key == "tags") + .map(|(_, value)| value.into_owned()) + .collect::>(); + (!tags.is_empty()).then_some(tags) + }); + + state + .engine_core_client() + .wake_up(tags) .await .map_err(|error| utility_call_error("wake_up", error))?; diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 4731958e2f..89ce35ce68 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -517,6 +517,7 @@ impl ChatBackend for FakeChatBackend { self.tokenizer(), options.tool_call_parser, options.reasoning_parser, + options.tool_strict_level, )?)) } } @@ -2053,6 +2054,56 @@ async fn version_returns_engine_vllm_version() { ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn kv_event_sources_reports_each_ready_engine() { + let ready = vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse { + data_parallel_rank: 1, + kv_events_config: Some( + vllm_engine_core_client::protocol::handshake::KvEventsConfig { + enable_kv_cache_events: true, + publisher: "zmq".to_string(), + endpoint: "tcp://10.0.0.2:41233".to_string(), + replay_endpoint: None, + buffer_steps: 10_000, + hwm: 100_000, + max_queue_size: 100_000, + topic: "kv".to_string(), + }, + ), + ..default_ready_response() + }; + let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready).await; + let response = app + .call( + Request::builder() + .uri("/kv_event_sources") + .body(Body::empty()) + .expect("build request"), + ) + .await + .expect("call app"); + + assert_eq!(response.status(), StatusCode::OK); + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); + let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json"); + assert_eq!( + json, + json!({ + "1": { + "enable_kv_cache_events": true, + "publisher": "zmq", + "endpoint": "tcp://10.0.0.2:41233", + "replay_endpoint": null, + "buffer_steps": 10000, + "hwm": 100000, + "max_queue_size": 100000, + "topic": "kv", + } + }) + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn server_info_endpoint_is_dev_mode_only() { @@ -5805,7 +5856,46 @@ async fn sleep_route_uses_python_compatible_default_query_values() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] -async fn wake_up_route_without_tags_sends_none() { +async fn release_kv_cache_memory_route_sends_expected_utility_call() { + let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| { + boxed_test_future(async move { + let utility = recv_engine_message(dealer).await; + assert_eq!(utility[0].as_ref(), &[0x03]); + + let payload = decode_value(&utility[1]).expect("decode utility payload"); + let array = payload.as_array().expect("utility payload array"); + let call_id = array[1].as_u64().expect("call id"); + + assert_eq!(array[2], Value::from("release_kv_cache_memory")); + assert_eq!(array[3], Value::Array(Vec::new())); + + send_outputs(push, utility_outputs(call_id, utility_none_result())).await; + }) + }) + .await; + + let response = app + .clone() + .call( + Request::builder() + .method("POST") + .uri("/release_kv_cache_memory") + .body(Body::empty()) + .expect("build request"), + ) + .await + .expect("call app"); + + let status = response.status(); + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); + assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body)); + assert!(body.is_empty()); + engine_task.await.expect("mock engine task"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn wake_up_route_without_tags_accepts_bool_result() { let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| { boxed_test_future(async move { let utility = recv_engine_message(dealer).await; @@ -5818,7 +5908,7 @@ async fn wake_up_route_without_tags_sends_none() { assert_eq!(array[2], Value::from("wake_up")); assert_eq!(array[3], Value::Array(vec![Value::Nil])); - send_outputs(push, utility_outputs(call_id, utility_none_result())).await; + send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await; }) }) .await; @@ -5842,6 +5932,51 @@ async fn wake_up_route_without_tags_sends_none() { engine_task.await.expect("mock engine task"); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn wake_up_route_accepts_repeated_tags() { + let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| { + boxed_test_future(async move { + let utility = recv_engine_message(dealer).await; + assert_eq!(utility[0].as_ref(), &[0x03]); + + let payload = decode_value(&utility[1]).expect("decode utility payload"); + let array = payload.as_array().expect("utility payload array"); + let call_id = array[1].as_u64().expect("call id"); + + assert_eq!(array[2], Value::from("wake_up")); + assert_eq!( + array[3], + Value::Array(vec![Value::Array(vec![ + Value::from("weights"), + Value::from("kv_cache"), + ])]) + ); + + send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await; + }) + }) + .await; + + let response = app + .clone() + .call( + Request::builder() + .method("POST") + .uri("/wake_up?tags=weights&tags%5B%5D=ignored&tags=kv_cache") + .body(Body::empty()) + .expect("build request"), + ) + .await + .expect("call app"); + + let status = response.status(); + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); + assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body)); + assert!(body.is_empty()); + engine_task.await.expect("mock engine task"); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn is_sleeping_route_returns_json_payload() { @@ -6160,6 +6295,7 @@ async fn admin_routes_are_hidden_when_dev_mode_is_disabled() { for (method, uri) in [ ("GET", "/is_sleeping"), ("POST", "/sleep"), + ("POST", "/release_kv_cache_memory"), ("POST", "/wake_up"), ("GET", "/is_paused"), ("POST", "/pause"), diff --git a/rust/src/text/src/output/decoded.rs b/rust/src/text/src/output/decoded.rs index a438c8e267..d3dc422656 100644 --- a/rust/src/text/src/output/decoded.rs +++ b/rust/src/text/src/output/decoded.rs @@ -9,6 +9,7 @@ use serde::{Deserialize, Serialize}; use tracing::{Level, debug, trace}; use vllm_engine_core_client::AbortCause; use vllm_engine_core_client::protocol::output::StopReason; +use vllm_engine_core_client::protocol::sampling_mask::SamplingMask; use vllm_llm::{FinishReason, GenerateOutput, TokenUsage}; use vllm_tokenizer::{DecodedText, DynTokenizer, IncrementalDecoder}; @@ -50,6 +51,8 @@ pub struct Finished { /// Connector-specific encoder cache transfer parameters for disaggregated /// serving. pub ec_transfer_params: Option, + /// Sampling support sets aligned with all generated token positions. + pub sampling_mask: Option, } /// Sample metadata emitted by one engine output update. @@ -120,6 +123,7 @@ pub async fn decoded_text_event_stream( let mut sampled_token_ids = Vec::new(); let mut output_token_count: usize = 0; let mut sampled_logprobs: Option = None; + let mut sampling_mask: Option = None; while let Some(next) = raw_stream.next().await { let output = next?; @@ -213,6 +217,7 @@ pub async fn decoded_text_event_stream( let mut new_token_ids = output.token_ids; let mut new_logprobs = output.logprobs; + let mut new_sampling_mask = output.sampling_mask; // Trim tokens and logprobs if we matched stop string. if let Some(num_tokens) = truncate_tokens_to { @@ -220,9 +225,15 @@ pub async fn decoded_text_event_stream( if let Some(logprobs) = &mut new_logprobs { logprobs.positions.truncate(num_tokens); } + if let Some(mask) = &mut new_sampling_mask { + mask.rows.truncate(num_tokens); + } } output_token_count += new_token_ids.len(); + if let Some(mut mask) = new_sampling_mask { + sampling_mask.get_or_insert_default().rows.append(&mut mask.rows); + } let decoded_logprobs = new_logprobs .as_ref() @@ -246,6 +257,16 @@ pub async fn decoded_text_event_stream( } if let Some(reason) = finish_reason { + if let Some(mask) = sampling_mask.as_ref() + && mask.rows.len() != output_token_count + { + return Err(vllm_llm::Error::SamplingMaskTokenCountMismatch { + request_id, + token_count: output_token_count, + row_count: mask.rows.len(), + } + .into()); + } // Flush any remaining buffered text. let (last_chunk, full_decoded) = decoder.flush(truncate_output_to)?; let text_len = full_decoded.text.len(); @@ -293,6 +314,7 @@ pub async fn decoded_text_event_stream( finish_reason: reason, kv_transfer_params, ec_transfer_params, + sampling_mask, })), }) .await; @@ -354,6 +376,7 @@ mod tests { use futures::{Stream, StreamExt as _, stream}; use vllm_engine_core_client::AbortCause; + use vllm_engine_core_client::protocol::logprobs::{Logprobs, PositionLogprobs, TokenLogprob}; use vllm_llm::GenerateOutput; use vllm_tokenizer::{TokenAnchor, TokenAttribution, test_utils::TestTokenizer}; @@ -384,6 +407,73 @@ mod tests { s.bytes().map(u32::from).collect() } + #[tokio::test] + async fn sampling_masks_survive_text_decode_and_collection() { + let prompt: Arc<[u32]> = Arc::from([]); + let mut first = GenerateOutput::for_test(Some(prompt), vec![b'a' as u32], None); + first.sampling_mask = Some(SamplingMask { + rows: vec![vec![1, b'a' as u32]], + }); + let mut terminal = + GenerateOutput::for_test(None, vec![b'b' as u32], Some(FinishReason::Length)); + terminal.sampling_mask = Some(SamplingMask { + rows: vec![vec![2, b'b' as u32]], + }); + let tokenizer: DynTokenizer = Arc::new(TestTokenizer::new()); + + let collected = decoded_text_event_stream( + "test-mask".into(), + tokenizer, + stream::iter([Ok(first), Ok(terminal)]), + TextDecodeOptions::default(), + false, + ) + .collect_output() + .await + .unwrap(); + + assert_eq!( + collected.sampling_mask, + Some(SamplingMask { + rows: vec![vec![1, b'a' as u32], vec![2, b'b' as u32]], + }) + ); + } + + #[tokio::test] + async fn sampling_mask_mismatch_reuses_llm_error() { + let prompt: Arc<[u32]> = Arc::from([]); + let mut terminal = GenerateOutput::for_test( + Some(prompt), + vec![b'a' as u32, b'b' as u32], + Some(FinishReason::Length), + ); + terminal.sampling_mask = Some(SamplingMask { + rows: vec![vec![1, b'a' as u32]], + }); + let tokenizer: DynTokenizer = Arc::new(TestTokenizer::new()); + + let error = decoded_text_event_stream( + "test-mask-mismatch".into(), + tokenizer, + stream::iter([Ok(terminal)]), + TextDecodeOptions::default(), + false, + ) + .collect_output() + .await + .unwrap_err(); + + assert!(matches!( + error, + crate::Error::Llm(vllm_llm::Error::SamplingMaskTokenCountMismatch { + token_count: 2, + row_count: 1, + .. + }) + )); + } + fn opts(stop: &[&str], min_tokens: u32) -> TextDecodeOptions { TextDecodeOptions { stop_strings: Some(stop.iter().map(|s| s.to_string()).collect()), @@ -472,6 +562,7 @@ mod tests { finish_reason: FinishReason::Length, kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, }), ), ], @@ -503,6 +594,7 @@ mod tests { ))), kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, }), ), ], @@ -576,6 +668,60 @@ mod tests { assert!(output.finish_reason.is_stop()); } + #[tokio::test] + async fn stream_stop_string_mid_chunk_keeps_token_logprob_mask_alignment() { + let token_ids = ascii_tokens("abcxyz"); + let mut raw = GenerateOutput::for_test( + Some(Arc::from([])), + token_ids.clone(), + Some(FinishReason::Length), + ); + raw.logprobs = Some(Logprobs { + positions: token_ids + .iter() + .map(|&token_id| PositionLogprobs { + entries: vec![TokenLogprob { + token_id, + logprob: -0.25, + rank: 1, + }], + }) + .collect(), + }); + raw.sampling_mask = Some(SamplingMask { + rows: token_ids.iter().map(|&token_id| vec![token_id, token_id + 100]).collect(), + }); + let tokenizer: DynTokenizer = Arc::new(TestTokenizer::new()); + + let output = decoded_text_event_stream( + "test-mid-chunk-stop".into(), + tokenizer, + stream::iter([Ok(raw)]), + opts(&["cx"], 0), + false, + ) + .collect_output() + .await + .unwrap(); + + assert_eq!(output.text, "ab"); + assert!(output.finish_reason.is_stop()); + assert_eq!(output.token_ids, ascii_tokens("abcx")); + assert_eq!(output.logprobs.as_ref().unwrap().positions.len(), 4); + assert_eq!( + output.sampling_mask, + Some(SamplingMask { + rows: vec![ + vec![b'a' as u32, b'a' as u32 + 100], + vec![b'b' as u32, b'b' as u32 + 100], + vec![b'c' as u32, b'c' as u32 + 100], + vec![b'x' as u32, b'x' as u32 + 100], + ], + }) + ); + assert_eq!(output.usage.output_token_count, 4); + } + #[tokio::test] async fn stream_stop_string_at_end() { let output = run_to_completion(ascii_tokens("abcxyz"), opts(&["xyz"], 0)).await; diff --git a/rust/src/text/src/output/mod.rs b/rust/src/text/src/output/mod.rs index a415bb8900..1293ac6025 100644 --- a/rust/src/text/src/output/mod.rs +++ b/rust/src/text/src/output/mod.rs @@ -17,6 +17,7 @@ mod logprobs; use std::sync::Arc; use futures::{StreamExt as _, pin_mut}; +use vllm_engine_core_client::protocol::sampling_mask::SamplingMask; use crate::{Error, FinishReason, Result, TextOutputStream}; @@ -35,6 +36,8 @@ pub struct CollectedTextOutput { /// Connector-specific encoder cache transfer parameters for disaggregated /// serving. pub ec_transfer_params: Option, + /// Sampling support sets aligned with generated token positions. + pub sampling_mask: Option, } #[allow(clippy::manual_async_fn, reason = "specify `Send` bound")] @@ -90,6 +93,7 @@ impl T { usage: vllm_llm::TokenUsage::default(), kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, }) }; @@ -99,6 +103,7 @@ impl T { collected.usage = finished.usage; collected.kv_transfer_params = finished.kv_transfer_params; collected.ec_transfer_params = finished.ec_transfer_params; + collected.sampling_mask = finished.sampling_mask; return Ok(collected); } } @@ -173,6 +178,7 @@ mod tests { finish_reason: FinishReason::stop_eos(), kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, })), }), ]); @@ -295,6 +301,7 @@ mod tests { finish_reason: FinishReason::stop_eos(), kv_transfer_params: None, ec_transfer_params: None, + sampling_mask: None, })), }), ]); diff --git a/setup.py b/setup.py index b24bcc1a2c..ed9616195f 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,18 @@ from setuptools_rust.build import build_rust from setuptools_scm import get_version from torch.utils.cpp_extension import CUDA_HOME, ROCM_HOME +# Select vLLM release tags, excluding crate tags such as "proto-v0.3.0". +VLLM_GIT_DESCRIBE_COMMAND = [ + "git", + "describe", + "--dirty", + "--tags", + "--long", + "--abbrev=40", + "--match", + "v[0-9]*", +] + def load_module_from_path(module_name, path): spec = importlib.util.spec_from_file_location(module_name, path) @@ -1249,9 +1261,15 @@ def get_vllm_version() -> str: if env_version := os.getenv("VLLM_VERSION_OVERRIDE"): print(f"Overriding VLLM version with {env_version} from VLLM_VERSION_OVERRIDE") os.environ["SETUPTOOLS_SCM_PRETEND_VERSION"] = env_version - return get_version(write_to="vllm/_version.py") + return get_version( + write_to="vllm/_version.py", + git_describe_command=VLLM_GIT_DESCRIBE_COMMAND, + ) - version = get_version(write_to="vllm/_version.py") + version = get_version( + write_to="vllm/_version.py", + git_describe_command=VLLM_GIT_DESCRIBE_COMMAND, + ) sep = "+" if "+" not in version else "." # dev versions might contain + if not envs.VLLM_SKIP_VERSION_SUFFIX: diff --git a/tests/basic_correctness/test_basic_correctness.py b/tests/basic_correctness/test_basic_correctness.py index 1b3c92ee5b..688d355c31 100644 --- a/tests/basic_correctness/test_basic_correctness.py +++ b/tests/basic_correctness/test_basic_correctness.py @@ -11,8 +11,6 @@ from unittest.mock import Mock import pytest import torch -from packaging.version import Version -from transformers import __version__ as TRANSFORMERS_VERSION import vllm.envs as envs from vllm import LLM @@ -149,15 +147,6 @@ def test_models( if enable_prompt_embeds: with torch.no_grad(): prompt_embeds = hf_model.get_prompt_embeddings(example_prompts) - if model == "hmellor/tiny-random-Gemma2ForCausalLM" and ( - Version(TRANSFORMERS_VERSION) < Version("5.3.0.dev0") - ): - # For Gemma 1/2 models with Transformers 5.4.0+, the prompt embeddings - # are normalised in `get_prompt_embeddings`, like Gemma 3. - # For older versions, we need to manually normalise. - embed_scale = hf_model.config.hidden_size**0.5 - normalizer = torch.tensor(embed_scale, dtype=prompt_embeds[0].dtype) - prompt_embeds = [p_e * normalizer for p_e in prompt_embeds] with VllmRunner( model, diff --git a/tests/basic_correctness/test_mem.py b/tests/basic_correctness/test_mem.py index 6b4895c643..6a483d26aa 100644 --- a/tests/basic_correctness/test_mem.py +++ b/tests/basic_correctness/test_mem.py @@ -109,6 +109,100 @@ def test_basic_cumem(): assert torch.allclose(output, torch.ones_like(output) * 3) +@pytest.mark.parametrize("full_sleep", [0, 1, 2], ids=["kv-only", "sleep-1", "sleep-2"]) +@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") +def test_release_kv_cache_memory_preserves_generation(full_sleep, monkeypatch): + """Rejected release and KV-only/full sleep cycles preserve greedy tokens.""" + monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") + kv_cache_memory_bytes = 256 * 1024 * 1024 + llm = LLM( + "Qwen/Qwen3-0.6B", + enable_sleep_mode=True, + enforce_eager=True, + kv_cache_memory_bytes=kv_cache_memory_bytes, + max_model_len=1024, + max_num_seqs=4, + gpu_memory_utilization=0.05, + ) + prompt = "How are you?" + sampling_params = SamplingParams(temperature=0, max_tokens=10) + expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids + + def get_mapped_bytes_by_tag(worker): + allocator = get_mem_allocator_instance() + mapped_bytes = mapped_usage(allocator) + kv_cache_bytes = sum( + data.handle[1] + for data in allocator.pointer_to_data.values() + if data.tag == "kv_cache" and not data.is_asleep + ) + return mapped_bytes, kv_cache_bytes + + mapped_before, kv_cache_before = llm.collective_rpc(get_mapped_bytes_by_tag)[0] + # Utility RPC transports the engine error as a plain Exception. + with pytest.raises(Exception, match="requires a completed pause"): + llm.release_kv_cache_memory() + assert llm.collective_rpc(get_mapped_bytes_by_tag)[0][0] == mapped_before + assert not llm.llm_engine.is_sleeping() + assert llm.generate(prompt, sampling_params)[0].outputs[0].token_ids == expected + + llm.sleep(level=0) + llm.release_kv_cache_memory() + mapped_after, kv_cache_after = llm.collective_rpc(get_mapped_bytes_by_tag)[0] + assert kv_cache_before > 0 + assert kv_cache_after == 0 + assert mapped_before - mapped_after >= kv_cache_before + assert mapped_after > 0 + assert llm.llm_engine.is_sleeping() + + if full_sleep: + llm.sleep(level=full_sleep) + assert llm.collective_rpc(get_mapped_bytes_by_tag)[0][0] == 0 + llm.wake_up(tags=None if full_sleep else ["kv_cache"]) + if full_sleep == 2: + llm.collective_rpc("reload_weights") + assert llm.collective_rpc(get_mapped_bytes_by_tag)[0][0] == mapped_before + assert not llm.llm_engine.is_sleeping() + actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids + assert actual == expected + + +@pytest.mark.parametrize("first_level", [1, 2]) +@pytest.mark.parametrize("second_level", [1, 2]) +@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") +def test_sleep_with_only_weights_asleep(first_level, second_level, monkeypatch): + """Repeated sleep after KV-only wake preserves mappings and recoverability.""" + monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") + llm = LLM( + "Qwen/Qwen3-0.6B", + enable_sleep_mode=True, + enforce_eager=True, + kv_cache_memory_bytes=256 * 1024 * 1024, + max_model_len=1024, + max_num_seqs=4, + gpu_memory_utilization=0.05, + ) + prompt = "How are you?" + sampling_params = SamplingParams(temperature=0, max_tokens=10) + expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids + + def get_mapped_bytes(worker): + return mapped_usage(get_mem_allocator_instance()) + + llm.sleep(level=first_level) + llm.wake_up(tags=["kv_cache"]) + mapped_before = llm.collective_rpc(get_mapped_bytes)[0] + assert mapped_before > 0 + llm.sleep(level=second_level) + assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before + llm.wake_up(tags=["weights"]) + if first_level == 2: + llm.collective_rpc("reload_weights") + assert not llm.llm_engine.is_sleeping() + actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids + assert actual == expected + + @create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") def test_discard_tags(): """Test that discard(tags) selectively frees GPU memory for specific diff --git a/tests/compile/test_config.py b/tests/compile/test_config.py index 5c9132ccbf..7465ea442c 100644 --- a/tests/compile/test_config.py +++ b/tests/compile/test_config.py @@ -541,6 +541,25 @@ def _mock_config_for_cudagraph_sizes( return config +@pytest.mark.parametrize("max_num_seqs", [100, 101]) +def test_default_cudagraph_capture_sizes_cover_off_stride_max_num_seqs( + max_num_seqs: int, +) -> None: + compilation_config = CompilationConfig( + cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE + ) + config = _mock_config_for_cudagraph_sizes( + max_num_seqs=max_num_seqs, + num_speculative_tokens=0, + max_num_batched_tokens=32768, + compilation_config=compilation_config, + ) + + VllmConfig._set_cudagraph_sizes(config) + + assert max_num_seqs in compilation_config.cudagraph_capture_sizes + + @pytest.mark.parametrize( ("max_num_seqs", "num_speculative_tokens", "widest_is_captured"), [ diff --git a/tests/conftest.py b/tests/conftest.py index a52a38f357..0d70be62eb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -241,7 +241,7 @@ def dist_init(): rank=0, distributed_init_method=f"file://{temp_file}", local_rank=0, - backend="nccl", + backend="gloo" if current_platform.is_cpu() else "nccl", ) initialize_model_parallel(1, 1) yield diff --git a/tests/distributed/conftest.py b/tests/distributed/conftest.py index 85a9d73aa5..1837d3706e 100644 --- a/tests/distributed/conftest.py +++ b/tests/distributed/conftest.py @@ -91,8 +91,11 @@ class MockSubscriber: if isinstance(replay_endpoints, str): replay_endpoints = [replay_endpoints] + ipv6 = any("[" in ep for ep in pub_endpoints + (replay_endpoints or [])) + # Set up subscriber socket - connect to all endpoints self.sub = self.ctx.socket(zmq.SUB) + self.sub.setsockopt(zmq.IPV6, ipv6) self.sub.setsockopt(zmq.SUBSCRIBE, topic.encode("utf-8")) for endpoint in pub_endpoints: self.sub.connect(endpoint) @@ -103,6 +106,7 @@ class MockSubscriber: if replay_endpoints: for replay_endpoint in replay_endpoints: replay = self.ctx.socket(zmq.DEALER) + replay.setsockopt(zmq.IPV6, ipv6) replay.connect(replay_endpoint) self.replay_sockets.append(replay) diff --git a/tests/distributed/test_events.py b/tests/distributed/test_events.py index aae80d9cce..dee0b99bb0 100644 --- a/tests/distributed/test_events.py +++ b/tests/distributed/test_events.py @@ -332,3 +332,77 @@ def test_event_publisher_factory(random_port): publisher = EventPublisherFactory.create(config, DP_RANK) assert isinstance(publisher, ZmqEventPublisher) publisher.shutdown() + + +@pytest.mark.parametrize( + ("endpoint", "expected"), + [("tcp://*:5557", "tcp://*:5558"), ("tcp://*:0", "tcp://*:0")], +) +def test_offset_endpoint_port_keeps_port_zero(endpoint, expected): + """Port 0 is not offset: every rank asks the OS for its own port.""" + from vllm.distributed.kv_events import ZmqEventPublisher + + assert ZmqEventPublisher.offset_endpoint_port(endpoint, 1) == expected + + +@pytest.mark.parametrize("node_ip", ["127.0.0.1", "::1"]) +def test_ephemeral_publishers_report_bound_endpoints(monkeypatch, node_ip): + """Two ranks binding tcp://*:0 get distinct OS-assigned ports on this + node's IP and report endpoints a subscriber can use for events and replay.""" + from vllm.config.kv_events import KVEventsConfig + from vllm.distributed import kv_events + from vllm.utils.network_utils import split_zmq_path + + from .conftest import MockSubscriber + + monkeypatch.setattr(kv_events, "get_ip", lambda: node_ip) + config = KVEventsConfig( + enable_kv_cache_events=True, + publisher="zmq", + endpoint="tcp://*:0", + replay_endpoint="tcp://*:0", + topic="kv", + ) + pubs = [EventPublisherFactory.create(config, rank) for rank in range(2)] + try: + resolved = [pub.get_publisher_config() for pub in pubs] + endpoints = [e for c in resolved for e in (c.endpoint, c.replay_endpoint)] + hosts_ports = {split_zmq_path(e)[1:] for e in endpoints} + assert {h for h, _ in hosts_ports} == {node_ip} + assert len(hosts_ports) == 4 and "0" not in {p for _, p in hosts_ports} + assert config.endpoint == "tcp://*:0" + + sub = MockSubscriber(endpoints[::2], endpoints[1::2], topic="kv") + try: + # PUB/SUB drops messages sent before the subscription lands, so + # keep publishing until both ranks are heard. + ranks: set[int] = set() + deadline = time.time() + 10 + while ranks != {0, 1} and time.time() < deadline: + for pub in pubs: + pub.publish(create_test_events(1)) + while (result := sub.receive_one(timeout=100)) is not None: + ranks.add(result[1].data_parallel_rank) + assert ranks == {0, 1} + sub.request_replay(0, socket_idx=1) + assert sub.receive_replay(socket_idx=1) + finally: + sub.close() + finally: + for pub in pubs: + pub.shutdown() + + +@pytest.mark.parametrize("node_ip", ["0.0.0.0", "::"]) +def test_ephemeral_bind_needs_a_node_ip(monkeypatch, node_ip): + """tcp://*:0 is advertised as this node's IP; with no usable IP the + publisher refuses instead of reporting a wildcard.""" + from vllm.config.kv_events import KVEventsConfig + from vllm.distributed import kv_events + + monkeypatch.setattr(kv_events, "get_ip", lambda: node_ip) + config = KVEventsConfig( + enable_kv_cache_events=True, publisher="zmq", endpoint="tcp://*:0" + ) + with pytest.raises(ValueError, match="VLLM_HOST_IP"): + EventPublisherFactory.create(config, DP_RANK) diff --git a/tests/distributed/test_quick_all_reduce.py b/tests/distributed/test_quick_all_reduce.py index f9228fbd68..107d95fa61 100644 --- a/tests/distributed/test_quick_all_reduce.py +++ b/tests/distributed/test_quick_all_reduce.py @@ -3,12 +3,17 @@ import multiprocessing import random +import sys +import types +from types import SimpleNamespace import pytest import ray import torch import torch.distributed as dist +import vllm._aiter_ops as aiter_ops +import vllm.distributed as vllm_distributed import vllm.envs as envs from vllm import _custom_ops as ops from vllm.distributed.communication_op import tensor_model_parallel_all_reduce # noqa @@ -233,6 +238,173 @@ def test_quick_allreduce_passes_dynamic_quant_level( assert called_quant_level == QuickReduceRegime.FP.value +def test_rocm_aiter_fused_rmsnorm_uses_aiter_qr_rmsnorm_for_prefill( + monkeypatch: pytest.MonkeyPatch, +): + class FakeAiterAllReduce: + aiter_ca = SimpleNamespace(world_size=2, fully_connected=True) + + def use_1stage_fused_ar_rms(self, inp): + return False + + def custom_fused_ar_rms(self, *args, **kwargs): + raise AssertionError("expected fused QR+RMSNorm dispatch") + + class FakeQuickReduce: + disabled = False + + def __init__(self): + self.checked_input = None + + def should_quick_allreduce(self, inp): + self.checked_input = inp + return True + + def _get_qr_quant_level(self, inp): + return QuickReduceRegime.INT4.value + + qr_comm = FakeQuickReduce() + device_comm = SimpleNamespace(cpu_group=object(), qr_comm=qr_comm) + monkeypatch.setattr( + vllm_distributed, + "get_tp_group", + lambda: SimpleNamespace(device_communicator=device_comm), + ) + monkeypatch.setattr( + aiter_ops.rocm_aiter_ops, + "get_aiter_allreduce", + lambda: FakeAiterAllReduce(), + ) + monkeypatch.setattr(aiter_ops.dist, "get_world_size", lambda group: 2) + monkeypatch.setattr(aiter_ops.dist, "get_rank", lambda group: 0) + monkeypatch.setattr( + aiter_ops.dist, + "all_gather_object", + lambda handles, handle, group: handles.__setitem__(0, handle), + ) + + calls = [] + fake_aiter = types.SimpleNamespace( + init_custom_qr=lambda rank, world_size, qr_max_size: 123, + qr_get_handle=lambda ptr: b"handle", + qr_open_handles=lambda ptr, handles: None, + ) + + def fake_qr_all_reduce_rmsnorm( + ptr, + inp, + residual, + residual_out, + out, + weight, + eps, + hidden_dim, + quant_level, + cast_bf2half, + ): + calls.append( + { + "ptr": ptr, + "inp": inp, + "residual": residual, + "residual_out": residual_out, + "out": out, + "weight": weight, + "eps": eps, + "hidden_dim": hidden_dim, + "quant_level": quant_level, + "cast_bf2half": cast_bf2half, + } + ) + out.copy_(inp) + residual_out.copy_(residual) + + fake_aiter.qr_all_reduce_rmsnorm = fake_qr_all_reduce_rmsnorm + monkeypatch.setitem(sys.modules, "aiter", fake_aiter) + monkeypatch.setattr(envs, "VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB", None) + monkeypatch.setattr(envs, "VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16", True) + + # token_num > 80 forces vLLM's AITER fused AR+RMS wrapper onto its + # non-1stage prefill path. hidden_dim=4096 bf16 gives an 8 KiB row, so four + # independent RMSNorm rows fit in one 32 KiB QR tile. + inp = torch.randn(81, 4096, dtype=torch.bfloat16) + residual = torch.randn_like(inp) + weight = torch.randn(4096, dtype=torch.bfloat16) + + out, residual_out = aiter_ops._rocm_aiter_fused_allreduce_rmsnorm_impl( + inp, residual, weight, 1e-6 + ) + + assert qr_comm.checked_input is inp + assert len(calls) == 1 + assert calls[0]["ptr"] == 123 + assert calls[0]["inp"] is inp + assert calls[0]["residual"] is residual + assert calls[0]["out"] is out + assert calls[0]["residual_out"] is residual_out + assert calls[0]["weight"] is weight + assert calls[0]["hidden_dim"] == 4096 + assert calls[0]["quant_level"] == QuickReduceRegime.INT4.value + assert calls[0]["cast_bf2half"] is True + torch.testing.assert_close(out, inp) + torch.testing.assert_close(residual_out, residual) + + +def test_rocm_aiter_fused_rmsnorm_keeps_1stage_decode_path( + monkeypatch: pytest.MonkeyPatch, +): + calls = [] + + class FakeAiterCA: + world_size = 2 + fully_connected = True + + def custom_fused_ar_rms( + self, inp, residual, weight, epsilon, *, use_1stage, gemma_norm + ): + calls.append( + { + "inp": inp, + "residual": residual, + "weight": weight, + "eps": epsilon, + "use_1stage": use_1stage, + "gemma_norm": gemma_norm, + } + ) + return inp + 1, residual + 1 + + class FakeAiterAllReduce: + aiter_ca = FakeAiterCA() + + def use_1stage_fused_ar_rms(self, inp): + return True + + monkeypatch.setattr( + aiter_ops.rocm_aiter_ops, + "get_aiter_allreduce", + lambda: FakeAiterAllReduce(), + ) + monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False) + + inp = torch.randn(4, 4096, dtype=torch.bfloat16) + residual = torch.randn_like(inp) + weight = torch.randn(4096, dtype=torch.bfloat16) + + out, residual_out = aiter_ops._rocm_aiter_fused_allreduce_rmsnorm_impl( + inp, residual, weight, 1e-6 + ) + + assert len(calls) == 1 + assert calls[0]["inp"] is inp + assert calls[0]["residual"] is residual + assert calls[0]["weight"] is weight + assert calls[0]["use_1stage"] is True + assert calls[0]["gemma_norm"] is False + torch.testing.assert_close(out, inp + 1) + torch.testing.assert_close(residual_out, residual + 1) + + @ray.remote(num_gpus=1, max_calls=1) def graph_quickreduce( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/entrypoints/pooling/scoring/test_cross_encoder_online_vision.py b/tests/entrypoints/pooling/scoring/test_cross_encoder_online_vision.py index 9f2d7da801..bf3c771fdd 100644 --- a/tests/entrypoints/pooling/scoring/test_cross_encoder_online_vision.py +++ b/tests/entrypoints/pooling/scoring/test_cross_encoder_online_vision.py @@ -27,31 +27,10 @@ ROCM_ATTN_BACKENDS = [ ATTN_BACKENDS = ROCM_ATTN_BACKENDS if current_platform.is_rocm() else ["auto"] -# Per-backend tolerance with explicit entries; "default" is the fallback -BACKEND_TOL: dict[str, float] = { - "default": 0.05, # 5% tolerance for other backends (e.g. FLASH_ATTN) - # Relaxed tolerances for ROCm attn - # See: https://github.com/vllm-project/vllm/issues/35569 - "ROCM_ATTN": 0.09, # gfx950:~8.45%, gfx942:~3.70% - "ROCM_AITER_FA": 0.045, # gfx950:~2.00%, gfx942:~0.80% - "TRITON_ATTN": 0.045, # gfx950:~3.00%, gfx942:~2.20% - "FLEX_ATTENTION": 0.045, # gfx950:~3.25%, gfx942:~1.10% -} - -# Some ROCm attention backends show small absolute drift on the low -# text-vs-text probability even though larger scores remain well inside the -# relative tolerance. The absolute drift is uniform across score magnitudes -# (~0.005-0.010), so it only exceeds the relative tolerance for the small -# ~0.10 text-vs-text value. Keep the relative tolerances tight and add only a -# small absolute floor for the affected backends. -# TRITON_ATTN: gfx942/ROCm 7.2 drifts ~0.008 abs on text-vs-text (~7.9% rel). -BACKEND_ABS_TOL: dict[str, float] = { - "default": 0.0, - "auto": 0.007, - "ROCM_AITER_FA": 0.005, - "TRITON_ATTN": 0.009, - "FLEX_ATTENTION": 0.00875, -} +# The absolute floor matters only for the low ~0.10 text-vs-text score, where a +# few thousandths of drift would otherwise exceed the relative bound. +REL_TOL = 0.05 +ABS_TOL = 0.007 # ROCm: disable prefix caching and eliminate batch variance to reduce # test flakiness. @@ -62,17 +41,9 @@ ROCM_EXTRA_ARGS = ( ) -def get_tol(backend: str) -> float: - return BACKEND_TOL.get(backend, BACKEND_TOL["default"]) - - -def get_abs_tol(backend: str) -> float: - return BACKEND_ABS_TOL.get(backend, BACKEND_ABS_TOL["default"]) - - def assert_score(actual: float, expected: float, backend: str, label: str): - tol = get_tol(backend) - abs_tol = get_abs_tol(backend) + tol = REL_TOL + abs_tol = ABS_TOL diff = abs(actual - expected) rel_diff = diff / abs(expected) if expected != 0 else diff print( @@ -104,9 +75,16 @@ documents = [ }, ] -TEXT_VS_TEXT = 0.10040374100208282 -TEXT_VS_IMAGE = 0.7423753142356873 -TEXT_VS_TEXT_PLUS_IMAGE = 0.5298863053321838 +# One bf16 ULP in attention compounds to ~0.1% here, so ROCm backends each +# settle differently; these are their midpoint across backends and archs. +if current_platform.is_rocm(): + TEXT_VS_TEXT = 0.105243 + TEXT_VS_IMAGE = 0.742472 + TEXT_VS_TEXT_PLUS_IMAGE = 0.544568 +else: + TEXT_VS_TEXT = 0.10040374100208282 + TEXT_VS_IMAGE = 0.7423753142356873 + TEXT_VS_TEXT_PLUS_IMAGE = 0.5298863053321838 @pytest.fixture(scope="module", params=ATTN_BACKENDS) @@ -507,6 +485,4 @@ async def test_rerank_api_instruction_field_matches_chat_template_kwargs( kwargs_scores = [ r.relevance_score for r in sorted(kwargs_rerank.results, key=lambda x: x.index) ] - assert field_scores == pytest.approx( - kwargs_scores, rel=get_tol(backend), abs=get_abs_tol(backend) - ) + assert field_scores == pytest.approx(kwargs_scores, rel=REL_TOL, abs=ABS_TOL) diff --git a/tests/entrypoints/scale_out/derender/test_derender.py b/tests/entrypoints/scale_out/derender/test_derender.py index d7e0298ac3..9cc6124393 100644 --- a/tests/entrypoints/scale_out/derender/test_derender.py +++ b/tests/entrypoints/scale_out/derender/test_derender.py @@ -800,7 +800,12 @@ def _e2e_generate_response( @pytest.mark.asyncio async def test_e2e_plain_roundtrip(parser_client, parser_tokenizer): - """Plain text without reasoning markers roundtrips correctly.""" + """Plain text without reasoning markers roundtrips correctly. + + Markerless output has no ````, which deepseek_r1 classifies + wholly as reasoning, so the text lands there rather than in content. + What this pins is detokenization fidelity through the parser path. + """ messages = [{"role": "user", "content": "What is 2+2?"}] gen_req = await _e2e_render_chat(parser_client, PARSER_MODEL, messages) @@ -814,16 +819,22 @@ async def test_e2e_plain_roundtrip(parser_client, parser_tokenizer): "model": PARSER_MODEL, "generate_response": _e2e_generate_response(output_ids), "prompt_tokens": len(gen_req["token_ids"]), + "chat_request": {"model": PARSER_MODEL, "messages": messages}, }, ) assert resp.status_code == 200, resp.text - content = resp.json()["choices"][0]["message"]["content"] - assert content == expected + message = resp.json()["choices"][0]["message"] + assert message["reasoning"] == expected + assert message["content"] is None @pytest.mark.asyncio async def test_e2e_token_identity(parser_client, parser_tokenizer): - """encode(derender(token_ids)) == token_ids (RL invariant).""" + """encode(derender(token_ids)) == token_ids (RL invariant). + + Markerless output comes back as reasoning (see + ``test_e2e_plain_roundtrip``), so re-encode that. + """ messages = [{"role": "user", "content": "Hi"}] gen_req = await _e2e_render_chat(parser_client, PARSER_MODEL, messages) @@ -836,17 +847,22 @@ async def test_e2e_token_identity(parser_client, parser_tokenizer): "model": PARSER_MODEL, "generate_response": _e2e_generate_response(output_ids), "prompt_tokens": len(gen_req["token_ids"]), + "chat_request": {"model": PARSER_MODEL, "messages": messages}, }, ) assert resp.status_code == 200 - content = resp.json()["choices"][0]["message"]["content"] - re_encoded = _encode(parser_tokenizer, content) + reasoning = resp.json()["choices"][0]["message"]["reasoning"] + re_encoded = _encode(parser_tokenizer, reasoning) assert output_ids == re_encoded @pytest.mark.asyncio async def test_e2e_non_ascii_roundtrip(parser_client, parser_tokenizer): - """CJK + emoji roundtrip without U+FFFD.""" + """CJK + emoji roundtrip without U+FFFD. + + Markerless output comes back as reasoning (see + ``test_e2e_plain_roundtrip``). + """ messages = [{"role": "user", "content": "Reply in Chinese"}] gen_req = await _e2e_render_chat(parser_client, PARSER_MODEL, messages) @@ -859,11 +875,12 @@ async def test_e2e_non_ascii_roundtrip(parser_client, parser_tokenizer): "model": PARSER_MODEL, "generate_response": _e2e_generate_response(output_ids), "prompt_tokens": len(gen_req["token_ids"]), + "chat_request": {"model": PARSER_MODEL, "messages": messages}, }, ) assert resp.status_code == 200 - content = resp.json()["choices"][0]["message"]["content"] - assert "�" not in content + reasoning = resp.json()["choices"][0]["message"]["reasoning"] + assert "�" not in reasoning @pytest.mark.asyncio @@ -976,8 +993,10 @@ async def test_e2e_parsed_reasoning_and_tool_call(parser_client, parser_tokenize @pytest.mark.asyncio -async def test_e2e_no_chat_request_fallback(parser_client, parser_tokenizer): - """Without chat_request, derender falls back to plain detokenization.""" +async def test_e2e_no_chat_request_rejected(parser_client, parser_tokenizer): + """Without chat_request a parser configured model rejects with 400 + rather than silently falling back to plain detokenization. This is to + prevent the leak of raw reasoning/tool markup into content.""" messages = [{"role": "user", "content": "Hello"}] gen_req = await _e2e_render_chat(parser_client, PARSER_MODEL, messages) @@ -992,9 +1011,8 @@ async def test_e2e_no_chat_request_fallback(parser_client, parser_tokenizer): "prompt_tokens": len(gen_req["token_ids"]), }, ) - assert resp.status_code == 200 - content = resp.json()["choices"][0]["message"]["content"] - assert "Hi" in content + assert resp.status_code == 400 + assert "chat_request" in resp.json()["error"]["message"] # --------------------------------------------------------------------------- diff --git a/tests/entrypoints/scale_out/derender/test_derender_stream.py b/tests/entrypoints/scale_out/derender/test_derender_stream.py index df10b31d28..4032d9cbf3 100644 --- a/tests/entrypoints/scale_out/derender/test_derender_stream.py +++ b/tests/entrypoints/scale_out/derender/test_derender_stream.py @@ -8,6 +8,8 @@ Tests are split into two layers: 1. Unit tests (no server): covers ``_detokenize_delta`` correctness (chunked == one-shot) and ``derender_completion_stream`` / ``derender_chat_stream`` logic via a real tokenizer on a tiny model. + The parser path is covered both with a deterministic stub parser and + with the real ``HarmonyParser`` (skipped without ``openai_harmony``). 2. Integration tests (require a running render server): covers the full HTTP round-trip through the streaming endpoint. Marked with @@ -15,19 +17,142 @@ Tests are split into two layers: fixtures from the sibling ``test_derender.py``. """ +import json +from collections.abc import Callable + import pytest import pytest_asyncio +from vllm.entrypoints.generate.base.protocol import ( + DeltaFunctionCall, + DeltaMessage, + DeltaToolCall, +) from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( DerenderStreamState, + GenerateResponse, + GenerateResponseChoice, GenerateResponseStreamChoice, GenerateStreamResponse, ) from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo +from vllm.parser import Parser +from vllm.utils import random_uuid MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM" +def _merge_fake_deltas( + accumulated: DeltaMessage | None, new: DeltaMessage | None +) -> DeltaMessage | None: + """Fold one token's worth of `_FakeParser` output into a per call + accumulator, the same way a real parser merges multiple tokens' + effects into the single `DeltaMessage` it returns per `parse_delta` + call. + """ + if new is None: + return accumulated + if accumulated is None: + accumulated = DeltaMessage() + + if new.content: + accumulated.content = (accumulated.content or "") + new.content + if new.reasoning: + accumulated.reasoning = (accumulated.reasoning or "") + new.reasoning + + for tc in new.tool_calls: + starts_new_call = tc.id is not None or ( + tc.function is not None and tc.function.name is not None + ) + existing = ( + None + if starts_new_call + else next((t for t in accumulated.tool_calls if t.index == tc.index), None) + ) + if existing is None: + accumulated.tool_calls.append(tc) + continue + if existing.function is None: + existing.function = DeltaFunctionCall() + if tc.function is not None and tc.function.arguments: + existing.function.arguments = ( + existing.function.arguments or "" + ) + tc.function.arguments + + return accumulated + + +class _FakeParser(Parser): + """Deterministic `Parser` stub for unit testing the replay/merge/pin + mechanics of `OnlineDerenderer._derender_chat_stream_parsed` in + isolation from any real reasoning/tool parser's markup grammar. + + Processes every id in `delta_token_ids` in order and folds their + effects into one `DeltaMessage`, like a real parser handling a + multi-token delta in a single call. Ignores `delta_text`: + + - `TOOL_START` opens a new tool call at index 0 (id + name). + - `TOOL_ARG` appends one `"a"` to that tool call's arguments. + - `REASON` emits one `"r"` of reasoning (suppressed when the request + has `include_reasoning=False`, mirroring the real parsers). + - `CONTENT` emits one `"c"` of content. + - An empty `delta_token_ids` with `finished=True` (the finish only + flush call) emits a `"FLUSH"` content sentinel so tests can confirm + it happened. + - Anything else emits nothing. + """ + + TOOL_START = 9 + TOOL_ARG = 10 + REASON = 11 + CONTENT = 12 + + def parse_delta( + self, + delta_text, + delta_token_ids, + request, + prompt_token_ids=None, + *, + finished, + ): + if not delta_token_ids: + return DeltaMessage(content="FLUSH") if finished else None + + accumulated: DeltaMessage | None = None + for tok in delta_token_ids: + accumulated = _merge_fake_deltas( + accumulated, self._parse_one_token(tok, request) + ) + return accumulated + + def _parse_one_token(self, tok: int, request) -> DeltaMessage | None: + if tok == self.TOOL_START: + return DeltaMessage( + tool_calls=[ + DeltaToolCall( + id=f"call-{random_uuid()}", + type="function", + function=DeltaFunctionCall(name="get_weather", arguments=""), + index=0, + ) + ] + ) + if tok == self.TOOL_ARG: + return DeltaMessage( + tool_calls=[ + DeltaToolCall(index=0, function=DeltaFunctionCall(arguments="a")) + ] + ) + if tok == self.REASON: + if not request.include_reasoning: + return None + return DeltaMessage(reasoning="r") + if tok == self.CONTENT: + return DeltaMessage(content="c") + return None + + # --------------------------------------------------------------------------- # Helpers shared across tests # --------------------------------------------------------------------------- @@ -111,6 +236,40 @@ def derenderer(tokenizer): ) +@pytest.fixture(scope="module") +def parsed_derenderer(tokenizer): + """OnlineDerenderer with `_FakeParser` wired in as `self.parser`. + + The parser configured path runs `_derender_chat_stream_parsed` via + `make_async(..., executor=renderer._executor)`, so unlike + `derenderer` above (whose streaming paths never touch the executor), + `renderer._executor` must be a real `ThreadPoolExecutor` here. + `loop.run_in_executor` cannot submit work to a `MagicMock`. + """ + from concurrent.futures import ThreadPoolExecutor + from unittest.mock import MagicMock + + from vllm.renderers.online_derenderer import OnlineDerenderer + + renderer = MagicMock() + renderer.get_tokenizer.return_value = tokenizer + renderer._executor = ThreadPoolExecutor(max_workers=2) + + model_config = MagicMock() + model_config.hf_config.model_type = "llama" + model_config.model = MODEL_NAME + + dr = OnlineDerenderer( + model_config=model_config, + renderer=renderer, + request_logger=None, + chat_template=None, + chat_template_content_format="string", + ) + dr.parser = _FakeParser + return dr + + class TestDetokenizeDelta: """_detokenize_delta: chunked decode must equal one shot decode.""" @@ -378,49 +537,456 @@ class TestDerenderChatStream: assert streamed == one_shot @pytest.mark.asyncio - @pytest.mark.parametrize("with_chat_request", [True, False]) - async def test_parser_raises_not_implemented(self, tokenizer, with_chat_request): - """Stream chat with a parser active must fail closed (NotImplementedError). - - Checks that a parser configured model must 501 even when ``chat_request`` is - omitted, otherwise reasoning/tool markup would leak into ``delta.content` - via the plain detok fallback. - """ - from unittest.mock import MagicMock - - from vllm.renderers.online_derenderer import OnlineDerenderer - - renderer = MagicMock() - renderer.get_tokenizer.return_value = tokenizer - - model_config = MagicMock() - model_config.hf_config.model_type = "llama" - model_config.model = MODEL_NAME - - # Construct a derenderer WITH a tool/reasoning parser active. - # ParserManager.get_parser returns None when no parser is named, so - # we inject a mock parser class directly. - dr = OnlineDerenderer( - model_config=model_config, - renderer=renderer, - request_logger=None, - chat_template=None, - chat_template_content_format="string", - ) - dr.parser = MagicMock() # simulate a parser being active - - chat_request = MagicMock() if with_chat_request else None - token_ids = tokenizer.encode("hello")[:3] - - with pytest.raises(NotImplementedError): - await dr.derender_chat_stream( + async def test_parser_configured_missing_chat_request_raises( + self, parsed_derenderer + ): + """A parser configured model must never fall through to plain detok + even when `chat_request` is omitted. If allowed this would leak raw + reasoning/tool markup into `delta.content`. `ServingDerender` + pre-checks this too (400 before touching the tokenizer). This pins + the `OnlineDerenderer` level backstop.""" + with pytest.raises(ValueError, match="chat_request"): + await parsed_derenderer.derender_chat_stream( model=MODEL_NAME, - generate_chunk=_make_stream_chunk(token_ids), + generate_chunk=_make_stream_chunk([_FakeParser.CONTENT]), state=None, - chat_request=chat_request, + chat_request=None, ) +def _chat_request(**kwargs): + from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest + + kwargs.setdefault("messages", [{"role": "user", "content": "hi"}]) + kwargs.setdefault("model", MODEL_NAME) + return ChatCompletionRequest(**kwargs) + + +class TestDerenderChatStreamParsed: + """derender_chat_stream: parser branch (replay + parse_delta) exercised + against the deterministic `_FakeParser` so these pin OnlineDerenderer's + own replay/merge/pin/finish_reason logic independent of any real + parser's markup grammar (covered separately by the parser_server backed + integration tests below).""" + + @pytest.mark.asyncio + async def test_dispatches_and_emits_content(self, parsed_derenderer): + chunk, state = await parsed_derenderer.derender_chat_stream( + model=MODEL_NAME, + generate_chunk=_make_stream_chunk( + [_FakeParser.CONTENT], finish_reason="stop" + ), + chat_request=_chat_request(), + ) + assert chunk.choices[0].delta.content == "c" + assert chunk.choices[0].delta.role == "assistant" + assert chunk.choices[0].finish_reason == "stop" + assert state.output_token_ids == [_FakeParser.CONTENT] + + @pytest.mark.asyncio + async def test_role_sent_once(self, parsed_derenderer): + chat_request = _chat_request() + chunk1, state = await parsed_derenderer.derender_chat_stream( + model=MODEL_NAME, + generate_chunk=_make_stream_chunk([_FakeParser.CONTENT]), + chat_request=chat_request, + ) + chunk2, _ = await parsed_derenderer.derender_chat_stream( + model=MODEL_NAME, + generate_chunk=_make_stream_chunk( + [_FakeParser.CONTENT], finish_reason="stop" + ), + state=state, + chat_request=chat_request, + ) + assert chunk1.choices[0].delta.role == "assistant" + assert chunk2.choices[0].delta.role is None + + @pytest.mark.asyncio + async def test_finish_only_chunk_flushes(self, parsed_derenderer): + """A finish only chunk (no new tokens) still calls `parse_delta` + once with `finished=True` to flush buffered state.""" + chat_request = _chat_request() + _, state = await parsed_derenderer.derender_chat_stream( + model=MODEL_NAME, + generate_chunk=_make_stream_chunk([_FakeParser.CONTENT]), + chat_request=chat_request, + ) + chunk2, _ = await parsed_derenderer.derender_chat_stream( + model=MODEL_NAME, + generate_chunk=_make_stream_chunk([], finish_reason="stop"), + state=state, + chat_request=chat_request, + ) + assert chunk2.choices[0].delta.content == "FLUSH" + assert chunk2.choices[0].finish_reason == "stop" + + @pytest.mark.asyncio + async def test_chunking_invariance(self, parsed_derenderer): + """1 token per chunk vs a single whole chunk assemble identically. + + Every chunk (live or replayed) is fed to `parse_delta` at its own + producer granularity (see `_derender_chat_stream_parsed`'s + docstring). This proves that choice doesn't change the assembled + output for a well behaved incremental parser. Each call's effects + fold together the same way regardless of how many tokens land in + one call vs. many.""" + token_ids = [ + _FakeParser.REASON, + _FakeParser.REASON, + _FakeParser.TOOL_START, + _FakeParser.TOOL_ARG, + _FakeParser.TOOL_ARG, + _FakeParser.CONTENT, + ] + chat_request = _chat_request( + tools=[{"type": "function", "function": {"name": "get_weather"}}], + tool_choice="auto", + include_reasoning=True, + ) + + async def _assemble(chunks: list[list[int]]) -> dict: + state = None + content = "" + reasoning = "" + tool_args = "" + for i, tids in enumerate(chunks): + finish = "stop" if i == len(chunks) - 1 else None + chunk, state = await parsed_derenderer.derender_chat_stream( + model=MODEL_NAME, + generate_chunk=_make_stream_chunk(tids, finish_reason=finish), + state=state, + chat_request=chat_request, + ) + delta = chunk.choices[0].delta + content += delta.content or "" + reasoning += delta.reasoning or "" + for tc in delta.tool_calls: + if tc.function and tc.function.arguments: + tool_args += tc.function.arguments + return {"content": content, "reasoning": reasoning, "tool_args": tool_args} + + whole = await _assemble([token_ids]) + one_at_a_time = await _assemble([[t] for t in token_ids]) + + assert whole == one_at_a_time + assert whole == {"content": "c", "reasoning": "rr", "tool_args": "aa"} + + @pytest.mark.asyncio + async def test_replay_preserves_chunk_boundaries( + self, parsed_derenderer, monkeypatch + ): + """A multi-token chunk (e.g. a speculative decoding step) must be + replayed as one `parse_delta` call, not one per token. Parsers + with per-call state such as `history_tool_call_cnt` would + otherwise diverge from standard serving.""" + + class _CallCountingParser(_FakeParser): + """Emits how many `parse_delta` calls it has seen.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.num_calls = 0 + + def parse_delta( + self, + delta_text, + delta_token_ids, + request, + prompt_token_ids=None, + *, + finished, + ): + self.num_calls += 1 + return DeltaMessage(content=str(self.num_calls)) + + monkeypatch.setattr(parsed_derenderer, "parser", _CallCountingParser) + chat_request = _chat_request() + + _, state = await parsed_derenderer.derender_chat_stream( + model=MODEL_NAME, + generate_chunk=_make_stream_chunk([_FakeParser.CONTENT] * 3), + chat_request=chat_request, + ) + assert state.output_chunk_lens == [3] + + chunk, _ = await parsed_derenderer.derender_chat_stream( + model=MODEL_NAME, + generate_chunk=_make_stream_chunk( + [_FakeParser.CONTENT], finish_reason="stop" + ), + state=state, + chat_request=chat_request, + ) + assert chunk.choices[0].delta.content == "2" + + @pytest.mark.asyncio + async def test_tool_call_id_pinned_across_chunks(self, parsed_derenderer): + """Once an index's ID is recorded in `last_tool_call_ids`, a later + id bearing delta for that same index is pinned to the recorded + value rather than a freshly (re-)generated one.""" + chat_request = _chat_request( + tools=[{"type": "function", "function": {"name": "get_weather"}}], + tool_choice="auto", + ) + chunk1, state = await parsed_derenderer.derender_chat_stream( + model=MODEL_NAME, + generate_chunk=_make_stream_chunk([_FakeParser.TOOL_START]), + chat_request=chat_request, + ) + first_id = chunk1.choices[0].delta.tool_calls[0].id + assert first_id is not None + assert state.last_tool_call_ids == [first_id] + + chunk2, state = await parsed_derenderer.derender_chat_stream( + model=MODEL_NAME, + generate_chunk=_make_stream_chunk( + [_FakeParser.TOOL_START], finish_reason="stop" + ), + state=state, + chat_request=chat_request, + ) + assert chunk2.choices[0].delta.tool_calls[0].id == first_id + assert state.last_tool_call_ids == [first_id] + + @pytest.mark.asyncio + async def test_finish_reason_rewritten_to_tool_calls(self, parsed_derenderer): + chat_request = _chat_request( + tools=[{"type": "function", "function": {"name": "get_weather"}}], + tool_choice="auto", + ) + chunk, _ = await parsed_derenderer.derender_chat_stream( + model=MODEL_NAME, + generate_chunk=_make_stream_chunk( + [_FakeParser.TOOL_START, _FakeParser.TOOL_ARG], finish_reason="stop" + ), + chat_request=chat_request, + ) + assert chunk.choices[0].finish_reason == "tool_calls" + + @pytest.mark.asyncio + async def test_finish_reason_stop_for_named_tool_choice(self, parsed_derenderer): + # ChatCompletionRequest's `check_tool_usage` is a mode="before" + # validator, so it sees the raw value and only accepts "auto", + # "required" or a dict, never an already built + # ChatCompletionNamedToolChoiceParam. + chat_request = _chat_request( + tools=[{"type": "function", "function": {"name": "get_weather"}}], + tool_choice={"type": "function", "function": {"name": "get_weather"}}, + ) + chunk, _ = await parsed_derenderer.derender_chat_stream( + model=MODEL_NAME, + generate_chunk=_make_stream_chunk( + [_FakeParser.TOOL_START, _FakeParser.TOOL_ARG], finish_reason="stop" + ), + chat_request=chat_request, + ) + assert chunk.choices[0].finish_reason == "stop" + + @pytest.mark.asyncio + async def test_include_reasoning_false_suppresses_reasoning( + self, parsed_derenderer + ): + chat_request = _chat_request(include_reasoning=False) + chunk, _ = await parsed_derenderer.derender_chat_stream( + model=MODEL_NAME, + generate_chunk=_make_stream_chunk( + [_FakeParser.REASON, _FakeParser.CONTENT], finish_reason="stop" + ), + chat_request=chat_request, + ) + delta = chunk.choices[0].delta + assert delta.reasoning is None + assert delta.content == "c" + + +# --------------------------------------------------------------------------- +# Harmony / GPT-OSS replay — unit, no server +# --------------------------------------------------------------------------- + +HARMONY_MODEL = "openai/gpt-oss-20b" +HARMONY_REASONING = "The user wants 2 plus 3." +HARMONY_ANSWER = "The answer is 5." +HARMONY_PROMPT = "<|start|>user<|message|>Add 2 and 3.<|end|><|start|>assistant" +HARMONY_OUTPUT = ( + f"<|channel|>analysis<|message|>{HARMONY_REASONING}<|end|>" + f"<|start|>assistant<|channel|>final<|message|>{HARMONY_ANSWER}<|return|>" +) + + +@pytest.fixture(scope="module") +def harmony_encode(): + """Encoder for canned GPT-OSS harmony token sequences.""" + pytest.importorskip("openai_harmony") + + # Pre-caches the o200k_base BPE file that openai-harmony's Rust backend + # downloads on first use, same as the sibling suite's E2E harmony tests. + from tests.entrypoints.scale_out.derender.test_derender import ( + _ensure_harmony_vocab, + ) + from vllm.entrypoints.openai.parser.harmony_utils import get_encoding + + _ensure_harmony_vocab() + encoding = get_encoding() + + def _encode(harmony_str: str) -> list[int]: + return encoding.encode(harmony_str, allowed_special="all") + + return _encode + + +@pytest.fixture(scope="module") +def harmony_tokenizer(): + pytest.importorskip("openai_harmony") + from vllm.tokenizers import get_tokenizer + + return get_tokenizer(HARMONY_MODEL, trust_remote_code=True) + + +@pytest.fixture(scope="module") +def harmony_derenderer(harmony_tokenizer): + """OnlineDerenderer whose parser resolves to the real `HarmonyParser`. + + Same shape as `parsed_derenderer` (mocked renderer, real executor) but + with a real tokenizer and parser, so the replay path is exercised + against Harmony's actual channel grammar rather than a stub. + """ + from concurrent.futures import ThreadPoolExecutor + from unittest.mock import MagicMock + + from vllm.parser.harmony import HarmonyParser + from vllm.renderers.online_derenderer import OnlineDerenderer + + renderer = MagicMock() + renderer.get_tokenizer.return_value = harmony_tokenizer + renderer._executor = ThreadPoolExecutor(max_workers=2) + + model_config = MagicMock() + model_config.model = HARMONY_MODEL + model_config.hf_config.model_type = "gpt_oss" + model_config.hf_text_config.model_type = "gpt_oss" + model_config.hf_overrides = None + + dr = OnlineDerenderer( + model_config=model_config, + renderer=renderer, + request_logger=None, + chat_template=None, + chat_template_content_format="string", + enable_auto_tools=True, + tool_parser="openai", + reasoning_parser="openai_gptoss", + ) + assert dr.use_harmony + assert dr.parser is HarmonyParser + return dr + + +async def _stream_harmony_deltas( + derenderer, + chat_request, + output_ids: list[int], + prompt_ids: list[int], + chunk_size: int, +) -> list[DeltaMessage]: + """Drive `output_ids` through the parser path in fixed size chunks. + + Returns the per chunk `DeltaMessage`s in order so callers can assert on + intermediate emissions, not just the assembled result. + """ + deltas: list[DeltaMessage] = [] + state = None + for start in range(0, len(output_ids), chunk_size): + tids = output_ids[start : start + chunk_size] + is_last = start + chunk_size >= len(output_ids) + chunk, state = await derenderer.derender_chat_stream( + model=HARMONY_MODEL, + generate_chunk=_make_stream_chunk( + tids, finish_reason="stop" if is_last else None + ), + state=state, + chat_request=chat_request, + prompt_token_ids=prompt_ids, + ) + deltas.append(chunk.choices[0].delta) + return deltas + + +class TestDerenderChatStreamHarmony: + """derender_chat_stream: replay + `parse_delta` against real HarmonyParser. + + Skipped where `openai_harmony` is not installed. `prompt_token_ids` is + threaded through for call shape fidelity only, since HarmonyParser + derives its state from the output tokens alone. + """ + + @pytest.mark.asyncio + async def test_reasoning_never_leaks_as_content_midstream( + self, harmony_derenderer, harmony_encode + ): + """Regression guard for the RFC's original replay + diff design. + + `HarmonyParser.parse()` always flushes to EOS. Mid-stream that + raises `HarmonyError` and the recovery branch re-emits the in + flight message on the `final` channel, so partial analysis surfaces + as content. Replay + `parse_delta` must never do that. Driven one + token per chunk, the finest granularity a client can produce. + """ + output_ids = harmony_encode(HARMONY_OUTPUT) + prompt_ids = harmony_encode(HARMONY_PROMPT) + chat_request = _chat_request(model=HARMONY_MODEL, include_reasoning=True) + + deltas = await _stream_harmony_deltas( + harmony_derenderer, chat_request, output_ids, prompt_ids, chunk_size=1 + ) + + reasoning = "" + content = "" + for i, delta in enumerate(deltas): + reasoning += delta.reasoning or "" + content += delta.content or "" + # Any analysis text emitted as content breaks the prefix + # property, since the two channels share no prefix here. + assert HARMONY_ANSWER.startswith(content), ( + f"content after chunk {i} is not a prefix of the final " + f"channel text: {content!r}" + ) + assert HARMONY_REASONING.startswith(reasoning) + assert not delta.tool_calls + + assert reasoning == HARMONY_REASONING + assert content == HARMONY_ANSWER + + @pytest.mark.asyncio + async def test_stream_matches_batch(self, harmony_derenderer, harmony_encode): + """Streamed assembly equals one shot `/derender` over the same IDs.""" + output_ids = harmony_encode(HARMONY_OUTPUT) + prompt_ids = harmony_encode(HARMONY_PROMPT) + chat_request = _chat_request(model=HARMONY_MODEL, include_reasoning=True) + + deltas = await _stream_harmony_deltas( + harmony_derenderer, chat_request, output_ids, prompt_ids, chunk_size=3 + ) + reasoning = "".join(d.reasoning or "" for d in deltas) + content = "".join(d.content or "" for d in deltas) + + batch_choices = await harmony_derenderer.derender_chat( + GenerateResponse( + request_id="test-harmony-batch", + choices=[ + GenerateResponseChoice( + index=0, token_ids=output_ids, finish_reason="stop" + ) + ], + ), + chat_request, + ) + message = batch_choices[0].message + + assert reasoning == message.reasoning + assert content == message.content + + class TestDerenderStreamStateValidation: """DerenderStreamState rejects malformed caller supplied offsets/lengths.""" @@ -446,6 +1012,24 @@ class TestDerenderStreamStateValidation: state = DerenderStreamState(prev_tokens=["a"] * 1024) assert len(state.prev_tokens) == 1024 + def test_output_chunk_lens_mismatch_rejected(self): + from pydantic import ValidationError + + with pytest.raises(ValidationError): + DerenderStreamState(output_token_ids=[1, 2, 3], output_chunk_lens=[1, 1]) + + def test_output_chunk_lens_zero_entry_rejected(self): + from pydantic import ValidationError + + with pytest.raises(ValidationError): + DerenderStreamState(output_token_ids=[1, 2], output_chunk_lens=[0, 2]) + + def test_output_chunk_lens_matching_accepted(self): + state = DerenderStreamState( + output_token_ids=[1, 2, 3], output_chunk_lens=[1, 2] + ) + assert state.output_chunk_lens == [1, 2] + class TestServingDerenderStreamErrorHandling: """Malformed stream_state must surface as 400 and not an unhandled 500.""" @@ -458,8 +1042,10 @@ class TestServingDerenderStreamErrorHandling: models = MagicMock() models.is_base_model.return_value = True models.model_config = MagicMock() + models.model_config.max_model_len = 100_000 online_derenderer = MagicMock() + online_derenderer.parser = None online_derenderer.derender_completion_stream = AsyncMock( side_effect=side_effect ) @@ -504,6 +1090,179 @@ class TestServingDerenderStreamErrorHandling: assert result.error.code == 400 +class TestServingDerenderStreamValidation: + """If checks cannot run in `derender_chat_stream_response`: all must + reject with 400 before ever calling into `online_derenderer` (i.e. + before touching the tokenizer).""" + + def _make_serving(self, *, parser_configured: bool, max_model_len: int = 100_000): + from unittest.mock import AsyncMock, MagicMock + + from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionStreamResponse, + ) + from vllm.entrypoints.scale_out.derender.serving import ServingDerender + + models = MagicMock() + models.is_base_model.return_value = True + models.model_config = MagicMock() + models.model_config.max_model_len = max_model_len + + online_derenderer = MagicMock() + online_derenderer.parser = MagicMock() if parser_configured else None + online_derenderer.derender_chat_stream = AsyncMock( + return_value=( + ChatCompletionStreamResponse(id="t", model=MODEL_NAME, choices=[]), + DerenderStreamState(), + ) + ) + return ServingDerender(models=models, online_derenderer=online_derenderer) + + @pytest.mark.asyncio + async def test_missing_chat_request_with_parser_rejected(self): + from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( + DerenderChatStreamRequest, + ) + + serving = self._make_serving(parser_configured=True) + request = DerenderChatStreamRequest( + stream=True, + model=MODEL_NAME, + generate_chunk=_make_stream_chunk([1, 2]), + ) + result = await serving.derender_chat_stream_response(request) + assert isinstance(result, ErrorResponse) + assert result.error.code == 400 + assert "chat_request" in result.error.message + serving.online_derenderer.derender_chat_stream.assert_not_called() + + @pytest.mark.asyncio + async def test_missing_chat_request_without_parser_ok(self): + from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( + DerenderChatStreamRequest, + ) + + serving = self._make_serving(parser_configured=False) + request = DerenderChatStreamRequest( + stream=True, + model=MODEL_NAME, + generate_chunk=_make_stream_chunk([1, 2]), + ) + result = await serving.derender_chat_stream_response(request) + assert not isinstance(result, ErrorResponse) + + @pytest.mark.asyncio + async def test_missing_prompt_token_ids_with_parser_rejected(self): + """A parser configured model must reject a missing prompt_token_ids + the same way it rejects a missing chat_request. Without it, + parse_delta cannot tell whether the prompt left reasoning open and + would misclassify reasoning content as plain content.""" + from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( + DerenderChatStreamRequest, + ) + + serving = self._make_serving(parser_configured=True) + request = DerenderChatStreamRequest( + stream=True, + model=MODEL_NAME, + generate_chunk=_make_stream_chunk([1, 2]), + chat_request=_chat_request(), + ) + result = await serving.derender_chat_stream_response(request) + assert isinstance(result, ErrorResponse) + assert result.error.code == 400 + assert "prompt_token_ids" in result.error.message + serving.online_derenderer.derender_chat_stream.assert_not_called() + + @pytest.mark.asyncio + async def test_prompt_token_ids_present_with_parser_ok(self): + from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( + DerenderChatStreamRequest, + ) + + serving = self._make_serving(parser_configured=True) + request = DerenderChatStreamRequest( + stream=True, + model=MODEL_NAME, + generate_chunk=_make_stream_chunk([1, 2]), + chat_request=_chat_request(), + prompt_token_ids=[1, 2, 3], + ) + result = await serving.derender_chat_stream_response(request) + assert not isinstance(result, ErrorResponse) + + @pytest.mark.asyncio + async def test_oversized_output_token_ids_rejected(self): + from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( + DerenderChatStreamRequest, + ) + + serving = self._make_serving(parser_configured=False, max_model_len=4) + request = DerenderChatStreamRequest( + stream=True, + model=MODEL_NAME, + generate_chunk=_make_stream_chunk([1, 2, 3]), + stream_state=DerenderStreamState( + output_token_ids=[1, 2], output_chunk_lens=[2] + ), + ) + result = await serving.derender_chat_stream_response(request) + assert isinstance(result, ErrorResponse) + assert result.error.code == 400 + assert "max_model_len" in result.error.message + serving.online_derenderer.derender_chat_stream.assert_not_called() + + @pytest.mark.asyncio + async def test_oversized_prompt_token_ids_rejected(self): + """prompt_token_ids is caller supplied and otherwise unbounded. A + parser configured deployment rescans it in full on every chunk + (is_reasoning_end / adjust_initial_state_from_prompt), so it must be + bounded the same way output_token_ids is.""" + from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( + DerenderChatStreamRequest, + ) + + serving = self._make_serving(parser_configured=True, max_model_len=4) + request = DerenderChatStreamRequest( + stream=True, + model=MODEL_NAME, + generate_chunk=_make_stream_chunk([1]), + chat_request=_chat_request(), + prompt_token_ids=[1, 2, 3, 4, 5], + ) + result = await serving.derender_chat_stream_response(request) + assert isinstance(result, ErrorResponse) + assert result.error.code == 400 + assert "max_model_len" in result.error.message + serving.online_derenderer.derender_chat_stream.assert_not_called() + + @pytest.mark.asyncio + async def test_too_many_choices_rejected(self): + """Each streamed chunk contains at most one choice (a single + DerenderStreamState is threaded through every choice). The check + could never fire since derender_chat_stream itself rejects anything + above 1 first.""" + from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( + DerenderChatStreamRequest, + ) + + serving = self._make_serving(parser_configured=False) + two_choices = GenerateStreamResponse( + request_id="t", + choices=[ + GenerateResponseStreamChoice(index=i, token_ids=[1]) for i in range(2) + ], + ) + request = DerenderChatStreamRequest( + stream=True, model=MODEL_NAME, generate_chunk=two_choices + ) + result = await serving.derender_chat_stream_response(request) + assert isinstance(result, ErrorResponse) + assert result.error.code == 400 + assert "at most one choice" in result.error.message + serving.online_derenderer.derender_chat_stream.assert_not_called() + + # --------------------------------------------------------------------------- # Integration tests — require a live render server # --------------------------------------------------------------------------- @@ -757,3 +1516,416 @@ async def test_streaming_usage_chunk(client): assert d2["chunk"]["choices"] == [] assert d2["chunk"]["usage"]["prompt_tokens"] == 10 assert d2["chunk"]["usage"]["completion_tokens"] == len(token_ids) + + +# --------------------------------------------------------------------------- +# Integration tests for parser configured (reasoning + tool calls) require a +# live render server. Mirrors the parser_server / parser_tokenizer pattern +# from test_derender.py. +# --------------------------------------------------------------------------- + +PARSER_MODEL = "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B" + +_PARSER_TOOLS = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather for a city", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + }, + }, + } +] + + +@pytest.fixture(scope="module") +def parser_server(): + from tests.utils import RemoteLaunchRenderServer + + args = [ + "--enable-auto-tool-choice", + "--tool-call-parser", + "hermes", + "--reasoning-parser", + "deepseek_r1", + ] + with RemoteLaunchRenderServer(PARSER_MODEL, args) as remote_server: + yield remote_server + + +@pytest_asyncio.fixture +async def parser_client(parser_server): + import httpx + + async with httpx.AsyncClient( + base_url=parser_server.url_for(""), timeout=60.0 + ) as http_client: + yield http_client + + +@pytest.fixture(scope="module") +def parser_tokenizer(): + from vllm.tokenizers import get_tokenizer + + return get_tokenizer(PARSER_MODEL) + + +def _require_parser_markers(tokenizer, text: str, *markers: str) -> list[int]: + """Encode text and skip the test if any marker is lost in roundtrip.""" + ids = tokenizer.encode(text, add_special_tokens=False) + decoded = tokenizer.decode(ids, skip_special_tokens=False) + for m in markers: + if m not in decoded: + pytest.skip(f"Marker {m!r} lost in encode->decode roundtrip") + return ids + + +async def _render_parser_chat(client, messages: list[dict]) -> dict: + resp = await client.post( + "/v1/chat/completions/render", + json={"model": PARSER_MODEL, "messages": messages}, + ) + assert resp.status_code == 200, resp.text + return resp.json() + + +async def _stream_chat_derender( + client, + output_ids: list[int], + chunk_sizes: list[int], + chat_request: dict, + prompt_tokens: int, + prompt_token_ids: list[int], + on_chunk: Callable[[list[dict]], None] | None = None, +) -> dict: + """Feed `output_ids` through the streaming chat derender endpoint in + the given `chunk_sizes`, threading `stream_state` across calls and + return the assembled message. + + If `on_chunk` is given, it is called after every chunk with a + snapshot (deep copy) of the `tool_calls` accumulator so far, letting + callers assert properties of the intermediate deltas (e.g. monotonic + argument growth) rather than only the final assembled result. + """ + state = None + content = "" + reasoning = "" + tool_calls: list[dict] = [] + finish_reason = None + + pos = 0 + for i, size in enumerate(chunk_sizes): + tids = output_ids[pos : pos + size] + pos += size + is_last = i == len(chunk_sizes) - 1 + resp = await client.post( + "/v1/chat/completions/derender", + json={ + "stream": True, + "model": PARSER_MODEL, + "generate_chunk": { + "request_id": "stream-test", + "choices": [ + { + "index": 0, + "token_ids": tids, + "finish_reason": "stop" if is_last else None, + } + ], + }, + "stream_state": state, + "prompt_tokens": prompt_tokens, + "prompt_token_ids": prompt_token_ids, + "chat_request": chat_request, + }, + ) + assert resp.status_code == 200, resp.text + data = resp.json() + state = data["stream_state"] + delta = data["chunk"]["choices"][0]["delta"] + content += delta.get("content") or "" + reasoning += delta.get("reasoning") or "" + for tc in delta.get("tool_calls") or []: + idx = tc["index"] + while len(tool_calls) <= idx: + tool_calls.append({"id": None, "name": None, "arguments": ""}) + if tc.get("id"): + tool_calls[idx]["id"] = tc["id"] + fn = tc.get("function") or {} + if fn.get("name"): + tool_calls[idx]["name"] = fn["name"] + if fn.get("arguments"): + tool_calls[idx]["arguments"] += fn["arguments"] + if is_last: + finish_reason = data["chunk"]["choices"][0]["finish_reason"] + if on_chunk is not None: + on_chunk([dict(tc) for tc in tool_calls]) + + return { + "content": content or None, + "reasoning": reasoning or None, + "tool_calls": tool_calls, + "finish_reason": finish_reason, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "chunking", + [ + "whole", + "per_token", + pytest.param( + "triples", + marks=pytest.mark.xfail( + reason=" leaks into reasoning when it shares a " + "multi-token delta with reasoning text (vllm#55195)" + ), + ), + ], +) +async def test_stream_parsed_matches_batch_reasoning( + parser_client, parser_tokenizer, chunking +): + """Streamed == batch: assembled reasoning/content equal the non + streaming `/derender` result over the same token IDs, for every + chunking of the same output (chunking invariance).""" + messages = [{"role": "user", "content": "What is 2+3?"}] + gen_req = await _render_parser_chat(parser_client, messages) + + reasoning_text = "The user wants 2 plus 3. That is 5." + answer_text = "The answer is 5." + output_text = f"{reasoning_text}{answer_text}" + output_ids = _require_parser_markers(parser_tokenizer, output_text, "") + + chat_request = { + "model": PARSER_MODEL, + "messages": messages, + "include_reasoning": True, + } + + batch_resp = await parser_client.post( + "/v1/chat/completions/derender", + json={ + "model": PARSER_MODEL, + "generate_response": { + "request_id": "batch", + "choices": [ + {"index": 0, "token_ids": output_ids, "finish_reason": "stop"} + ], + }, + "prompt_tokens": len(gen_req["token_ids"]), + "chat_request": chat_request, + }, + ) + assert batch_resp.status_code == 200, batch_resp.text + batch_msg = batch_resp.json()["choices"][0]["message"] + + n = len(output_ids) + chunk_sizes = { + "whole": [n], + "per_token": [1] * n, + "triples": [3] * (n // 3) + [n % 3], + }[chunking] + chunk_sizes = [c for c in chunk_sizes if c > 0] + streamed = await _stream_chat_derender( + parser_client, + output_ids, + chunk_sizes, + chat_request, + len(gen_req["token_ids"]), + gen_req["token_ids"], + ) + assert streamed["content"] == batch_msg["content"] + assert streamed["reasoning"] == batch_msg.get("reasoning") + + +@pytest.mark.asyncio +async def test_stream_parsed_matches_batch_tool_call(parser_client, parser_tokenizer): + """Tool call name+id+arguments parity between streamed and batch, plus + the finish_reason -> "tool_calls" rewrite for auto tool choice.""" + messages = [{"role": "user", "content": "Weather in Paris?"}] + gen_req = await _render_parser_chat(parser_client, messages) + + output_text = ( + "Let me check." + '\n{"name": "get_weather", ' + '"arguments": {"city": "Paris"}}\n' + ) + output_ids = _require_parser_markers( + parser_tokenizer, output_text, "", "", "" + ) + chat_request = { + "model": PARSER_MODEL, + "messages": messages, + "tools": _PARSER_TOOLS, + "tool_choice": "auto", + } + + batch_resp = await parser_client.post( + "/v1/chat/completions/derender", + json={ + "model": PARSER_MODEL, + "generate_response": { + "request_id": "batch", + "choices": [ + {"index": 0, "token_ids": output_ids, "finish_reason": "stop"} + ], + }, + "prompt_tokens": len(gen_req["token_ids"]), + "chat_request": chat_request, + }, + ) + assert batch_resp.status_code == 200, batch_resp.text + batch_choice = batch_resp.json()["choices"][0] + if not batch_choice["message"]["tool_calls"]: + pytest.skip("Model did not emit a tool call") + + for chunk_sizes in ([len(output_ids)], [1] * len(output_ids)): + # Snapshot of tool_calls[0]["arguments"] after every chunk, to check + # it only ever grows by appending. It never retracts or duplicates + # already streamed text. That's the exact failure mode the + # replay + parse_delta design (over diff-based streaming) exists to + # avoid. Parity with the batch result alone wouldn't catch a + # transient mid-stream regression. + arg_snapshots: list[str] = [] + + def _record( + tool_calls: list[dict], _snapshots: list[str] = arg_snapshots + ) -> None: + if tool_calls and tool_calls[0]["arguments"]: + _snapshots.append(tool_calls[0]["arguments"]) + + streamed = await _stream_chat_derender( + parser_client, + output_ids, + chunk_sizes, + chat_request, + len(gen_req["token_ids"]), + gen_req["token_ids"], + on_chunk=_record, + ) + assert streamed["tool_calls"] + assert streamed["tool_calls"][0]["name"] == "get_weather" + assert streamed["tool_calls"][0]["id"] is not None + assert json.loads(streamed["tool_calls"][0]["arguments"]) == json.loads( + batch_choice["message"]["tool_calls"][0]["function"]["arguments"] + ) + assert streamed["finish_reason"] == "tool_calls" + + assert arg_snapshots, "expected at least one tool-call argument delta" + for prev, curr in zip(arg_snapshots, arg_snapshots[1:]): + assert curr.startswith(prev), ( + f"tool-call arguments regressed: {prev!r} -> {curr!r}" + ) + assert arg_snapshots[-1] == streamed["tool_calls"][0]["arguments"] + + +@pytest.mark.asyncio +async def test_stream_parsed_cjk_across_chunk_boundaries( + parser_client, parser_tokenizer +): + """CJK/emoji tokens split across chunk boundaries with a parser active + across the reasoning -> content transition (parser path regression + guard analogous to #46159 in the plain detok path).""" + messages = [{"role": "user", "content": "Reply in Chinese"}] + gen_req = await _render_parser_chat(parser_client, messages) + + reasoning_text = "思考中" + answer_text = "你好世界 😀" + output_text = f"{reasoning_text}{answer_text}" + output_ids = _require_parser_markers(parser_tokenizer, output_text, "") + + chat_request = { + "model": PARSER_MODEL, + "messages": messages, + "include_reasoning": True, + } + streamed = await _stream_chat_derender( + parser_client, + output_ids, + [1] * len(output_ids), + chat_request, + len(gen_req["token_ids"]), + gen_req["token_ids"], + ) + assert "�" not in (streamed["content"] or "") + assert "�" not in (streamed["reasoning"] or "") + assert answer_text in (streamed["content"] or "") + + +@pytest.mark.asyncio +async def test_stream_parsed_include_reasoning_false(parser_client, parser_tokenizer): + """include_reasoning=False emits no reasoning deltas on the streaming + parser path.""" + messages = [{"role": "user", "content": "What is 2+3?"}] + gen_req = await _render_parser_chat(parser_client, messages) + + output_text = "reasoning hereThe answer is 5." + output_ids = _require_parser_markers(parser_tokenizer, output_text, "") + + chat_request = { + "model": PARSER_MODEL, + "messages": messages, + "include_reasoning": False, + } + streamed = await _stream_chat_derender( + parser_client, + output_ids, + [1] * len(output_ids), + chat_request, + len(gen_req["token_ids"]), + gen_req["token_ids"], + ) + assert streamed["reasoning"] is None + + +@pytest.mark.asyncio +async def test_stream_parsed_missing_chat_request_rejected(parser_client): + """Parser configured + no chat_request on the streaming endpoint -> 400 + (the live-server counterpart to the mocked + `TestServingDerenderStreamValidation` checks above).""" + resp = await parser_client.post( + "/v1/chat/completions/derender", + json={ + "stream": True, + "model": PARSER_MODEL, + "generate_chunk": { + "request_id": "reject-test", + "choices": [{"index": 0, "token_ids": [1, 2], "finish_reason": None}], + }, + }, + ) + assert resp.status_code == 400 + assert "chat_request" in resp.json()["error"]["message"] + + +@pytest.mark.asyncio +async def test_stream_parsed_missing_prompt_token_ids_rejected(parser_client): + """Parser configured + chat_request given but no prompt_token_ids on the + streaming endpoint -> 400 (the live-server counterpart to the mocked + `test_missing_prompt_token_ids_with_parser_rejected` above). Without + prompt_token_ids, parse_delta cannot tell whether the prompt left + reasoning open and would silently misclassify reasoning as content.""" + messages = [{"role": "user", "content": "Hello"}] + resp = await parser_client.post( + "/v1/chat/completions/derender", + json={ + "stream": True, + "model": PARSER_MODEL, + "generate_chunk": { + "request_id": "reject-test", + "choices": [{"index": 0, "token_ids": [1, 2], "finish_reason": None}], + }, + "chat_request": { + "model": PARSER_MODEL, + "messages": messages, + "include_reasoning": True, + }, + }, + ) + assert resp.status_code == 400 + assert "prompt_token_ids" in resp.json()["error"]["message"] diff --git a/tests/entrypoints/scale_out/render/test_render.py b/tests/entrypoints/scale_out/render/test_render.py index e21343553d..e65a26a7d1 100644 --- a/tests/entrypoints/scale_out/render/test_render.py +++ b/tests/entrypoints/scale_out/render/test_render.py @@ -594,176 +594,6 @@ async def test_completion_render_multiple_prompts_token_offsets(client): assert 0 <= start <= end <= len(prompt) -@pytest.mark.asyncio -async def test_chat_completion_render_assistant_tokens_mask_default(client): - """Without return_assistant_tokens_mask, assistant_tokens_mask should be null.""" - response = await client.post( - "/v1/chat/completions/render", - json={ - "model": MODEL_NAME, - "messages": [ - {"role": "user", "content": "Hello"}, - {"role": "assistant", "content": "Hi!"}, - {"role": "user", "content": "How are you?"}, - ], - }, - ) - - assert response.status_code == 200 - data = response.json() - assert data.get("assistant_tokens_mask") is None - - -@pytest.mark.asyncio -async def test_chat_completion_render_assistant_tokens_mask_false(client): - """Explicitly setting return_assistant_tokens_mask=false gives null.""" - response = await client.post( - "/v1/chat/completions/render", - json={ - "model": MODEL_NAME, - "messages": [ - {"role": "user", "content": "Hello"}, - ], - "return_assistant_tokens_mask": False, - }, - ) - - assert response.status_code == 200 - data = response.json() - assert data.get("assistant_tokens_mask") is None - - -@pytest.mark.asyncio -async def test_chat_render_assistant_tokens_mask_null_without_gen_tags( - client, -): - """The tiny test model lacks ``{% generation %}`` tags, so the mask is null.""" - response = await client.post( - "/v1/chat/completions/render", - json={ - "model": MODEL_NAME, - "messages": [ - {"role": "user", "content": "Hello"}, - {"role": "assistant", "content": "Hi!"}, - ], - "return_assistant_tokens_mask": True, - }, - ) - - assert response.status_code == 200 - assert response.json().get("assistant_tokens_mask") is None - - -# A minimal chat template with {% generation %} tags so we can test that -# the mask correctly marks assistant tokens. -_TEMPLATE_WITH_GENERATION = ( - "{% for m in messages %}" - "{% if m['role'] == 'user' %}User: {{ m['content'] }}\n" - "{% elif m['role'] == 'assistant' %}" - "{% generation %}Assistant: {{ m['content'] }}\n{% endgeneration %}" - "{% endif %}" - "{% endfor %}" -) - - -@pytest.mark.asyncio -async def test_chat_completion_render_assistant_tokens_mask_with_generation_tags( - client, -): - """With a ``{% generation %}``-enabled template, the mask marks assistant - tokens and the masked tokens decode to the assistant content.""" - response = await client.post( - "/v1/chat/completions/render", - json={ - "model": MODEL_NAME, - "messages": [ - {"role": "user", "content": "Hello"}, - {"role": "assistant", "content": "Hi!"}, - {"role": "user", "content": "Bye"}, - ], - "chat_template": _TEMPLATE_WITH_GENERATION, - "return_assistant_tokens_mask": True, - }, - ) - - assert response.status_code == 200 - data = response.json() - - mask = data["assistant_tokens_mask"] - token_ids = data["token_ids"] - assert mask is not None - assert isinstance(mask, list) - assert len(mask) == len(token_ids) - assert all(v in (0, 1) for v in mask) - assert sum(mask) > 0, "mask should mark at least one assistant token" - - # Detokenize masked (assistant) and unmasked (non-assistant) tokens - # separately to verify the mask is correct, not just non-empty. - masked_ids = [t for t, m in zip(token_ids, mask, strict=True) if m] - unmasked_ids = [t for t, m in zip(token_ids, mask, strict=True) if not m] - - detok = await client.post( - "/detokenize", - json={"model": MODEL_NAME, "tokens": masked_ids}, - ) - assert detok.status_code == 200 - assert "Hi!" in detok.json()["prompt"] - - detok_rest = await client.post( - "/detokenize", - json={"model": MODEL_NAME, "tokens": unmasked_ids}, - ) - assert detok_rest.status_code == 200 - assert "Hi!" not in detok_rest.json()["prompt"] - assert "Bye" in detok_rest.json()["prompt"] - - -@pytest.mark.asyncio -async def test_chat_render_assistant_tokens_mask_follows_truncation(client): - """The assistant mask must be truncated with the prompt it describes. - - `assistant_tokens_mask` is positional: entry i labels token i. Truncating - `token_ids` from the left without truncating the mask leaves the two - describing different positions, and the mask ends up marking whichever - tokens happen to sit at the old offsets. - """ - messages = [ - # Deliberately lopsided: a long leading user turn and a short trailing - # one, so keeping the head of the mask is distinguishable from keeping - # its tail. - {"role": "user", "content": "Hello hello hello hello hello hello"}, - {"role": "assistant", "content": "Hi!"}, - {"role": "user", "content": "Bye"}, - ] - body = { - "model": MODEL_NAME, - "messages": messages, - "chat_template": _TEMPLATE_WITH_GENERATION, - "return_assistant_tokens_mask": True, - } - - full = await client.post("/v1/chat/completions/render", json=body) - assert full.status_code == 200 - full_token_ids = full.json()["token_ids"] - full_mask = full.json()["assistant_tokens_mask"] - assert sum(full_mask) > 0 - - keep = len(full_token_ids) - 4 - # Precondition: with this prompt the head and tail slices of the mask - # really do differ, so the assertion below can tell them apart. - assert full_mask[-keep:] != full_mask[:keep] - - truncated = await client.post( - "/v1/chat/completions/render", - json={**body, "truncate_prompt_tokens": keep, "truncation_side": "left"}, - ) - assert truncated.status_code == 200 - data = truncated.json() - - assert data["token_ids"] == full_token_ids[-keep:] - assert data["assistant_tokens_mask"] == full_mask[-keep:] - - @pytest.mark.asyncio async def test_messages_render_basic(client): """Test basic Anthropic Messages render endpoint.""" diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py b/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py index 0cb8a6d397..d9ae424cc8 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py @@ -569,40 +569,6 @@ async def test_stream_omits_prompt_metadata_by_default(): assert "mm_placeholders" not in chunk -@pytest.mark.asyncio -async def test_stream_chunks_omit_absent_fields(): - """Absent optional fields are omitted from stream chunks, like Rust.""" - engine = _mock_engine() - - async def mock_generate(*args, **kwargs): - yield _make_request_output("req-1", token_ids=[10]) - yield _make_request_output( - "req-1", token_ids=[20], finish_reason="stop", finished=True - ) - - engine.generate = MagicMock(side_effect=mock_generate) - serving = _build_serving_tokens(engine) - request = GenerateRequest( - token_ids=[1, 2, 3], - sampling_params=SamplingParams(max_tokens=2), - model=MODEL_NAME, - stream=True, - ) - - response = await serving.serve_tokens(request) - parsed = _parse_sse_chunks([chunk async for chunk in response]) - data_chunks = [c for c in parsed if isinstance(c, dict) and c.get("choices")] - - assert len(data_chunks) == 2 - for chunk in data_chunks: - assert "usage" not in chunk - assert "prompt_token_ids" not in chunk - assert "mm_placeholders" not in chunk - assert "logprobs" not in chunk["choices"][0] - assert "finish_reason" not in data_chunks[0]["choices"][0] - assert data_chunks[1]["choices"][0]["finish_reason"] == "stop" - - @pytest.mark.asyncio async def test_stream_text_only_prompt_omits_mm_placeholders_key(): engine = _mock_engine() diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py b/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py index 1c349e8d67..aa90618a90 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py @@ -422,6 +422,31 @@ async def test_stop_string_workflow(client, tokenizer, messages): assert generate_res == completions_res +@pytest.mark.asyncio +@pytest.mark.skipif( + envs.VLLM_USE_RUST_FRONTEND, + reason="--tokens-only is not supported by the Rust frontend", +) +@pytest.mark.parametrize("server", [["--tokens-only"]], indirect=True) +async def test_stop_strings_rejected_when_tokens_only(client): + """--tokens-only forces detokenize=False after request validation, so + stop strings must be rejected explicitly rather than silently ignored.""" + sampling_params = {"max_tokens": 5, "stop": ["never"]} + payload = { + "model": MODEL_NAME, + "token_ids": [1, 2, 3], + "sampling_params": sampling_params, + "stream": False, + } + resp = await client.post(GEN_ENDPOINT, json=payload) + assert resp.status_code == 400 + assert "stop strings" in resp.json()["error"]["message"] + + sampling_params["stop"] = None + resp = await client.post(GEN_ENDPOINT, json=payload) + resp.raise_for_status() + + @pytest.mark.asyncio @pytest.mark.parametrize( "server", diff --git a/tests/entrypoints/serve/dev/test_sleep.py b/tests/entrypoints/serve/dev/test_sleep.py index 5ebee50600..34e69c9ece 100644 --- a/tests/entrypoints/serve/dev/test_sleep.py +++ b/tests/entrypoints/serve/dev/test_sleep.py @@ -1,14 +1,37 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from unittest.mock import AsyncMock + +import pytest import requests +from fastapi import FastAPI +from fastapi.testclient import TestClient from prometheus_client.parser import text_string_to_metric_families from tests.utils import RemoteOpenAIServer +from vllm.entrypoints.serve.dev.sleep.api_router import attach_router MODEL_NAME = "meta-llama/Llama-3.2-1B" +@pytest.mark.cpu_test +@pytest.mark.parametrize("fails", [False, True]) +def test_release_kv_cache_memory_route(fails): + app = FastAPI() + app.state.engine_client = AsyncMock() + release = app.state.engine_client.release_kv_cache_memory + if fails: + release.side_effect = RuntimeError("requires a completed pause first") + attach_router(app) + + with TestClient(app, raise_server_exceptions=False) as client: + response = client.post("/release_kv_cache_memory") + + assert response.status_code == (500 if fails else 200) + release.assert_awaited_once_with() + + def test_sleep_mode(): # dtype, max-len etc set so that this can run in CI args = [ @@ -63,7 +86,7 @@ def test_sleep_mode(): ) assert response.status_code == 200 - # is sleeping should be false after waking up any part of the engine + # Partial wake keeps the engine sleeping. response = requests.get(remote_server.url_for("is_sleeping")) assert response.status_code == 200 assert response.json().get("is_sleeping") is True diff --git a/tests/evals/gsm8k/configs/GLM-5.2-NVFP4-TP2-PCP2-EP.yaml b/tests/evals/gsm8k/configs/GLM-5.2-NVFP4-TP2-PCP2-EP.yaml index bc5c11fa53..141bf68162 100644 --- a/tests/evals/gsm8k/configs/GLM-5.2-NVFP4-TP2-PCP2-EP.yaml +++ b/tests/evals/gsm8k/configs/GLM-5.2-NVFP4-TP2-PCP2-EP.yaml @@ -3,6 +3,8 @@ accuracy_threshold: 0.90 num_questions: 1319 num_fewshot: 5 max_concurrency: 100 +# 30B+ MoE with EP/PCP: download + kernel autotuning can exceed the 1200s default. +startup_max_wait_seconds: 2400 # Leave headroom for FlashInfer CUTLASS autotuning after KV-cache allocation. server_args: >- --compilation-config '{"cudagraph_mode":"NONE"}' diff --git a/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-BF16-moonep.yaml b/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-BF16-moonep.yaml new file mode 100644 index 0000000000..8704f29bcf --- /dev/null +++ b/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-BF16-moonep.yaml @@ -0,0 +1,5 @@ +model_name: "Qwen/Qwen3-30B-A3B" +accuracy_threshold: 0.88 +num_questions: 1319 +num_fewshot: 5 +server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --all2all-backend=moonep" diff --git a/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-Fp8-AutoFp8-deepgemm-deepep-ll.yaml b/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-Fp8-AutoFp8-deepgemm-deepep-ll.yaml index 5f10684d2e..f81eb1c188 100644 --- a/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-Fp8-AutoFp8-deepgemm-deepep-ll.yaml +++ b/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-Fp8-AutoFp8-deepgemm-deepep-ll.yaml @@ -2,6 +2,8 @@ model_name: "Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8" accuracy_threshold: 0.88 num_questions: 1319 num_fewshot: 5 +# 30B MoE + DeepGEMM/DeepEP JIT compile can exceed the 1200s default startup wait. +startup_max_wait_seconds: 2400 server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --all2all-backend deepep_low_latency --disable-uvicorn-access-log" env: VLLM_USE_DEEP_GEMM: "1" diff --git a/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-Fp8-CT-Block-deepgemm-deepep-ll.yaml b/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-Fp8-CT-Block-deepgemm-deepep-ll.yaml index 37e6039e96..3987065f6f 100644 --- a/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-Fp8-CT-Block-deepgemm-deepep-ll.yaml +++ b/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-Fp8-CT-Block-deepgemm-deepep-ll.yaml @@ -2,6 +2,8 @@ model_name: "RedHatAI/Qwen3-30B-A3B-FP8-block" accuracy_threshold: 0.85 num_questions: 1319 num_fewshot: 5 +# 30B MoE + DeepGEMM/DeepEP JIT compile can exceed the 1200s default startup wait. +startup_max_wait_seconds: 2400 server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --all2all-backend deepep_low_latency --disable-uvicorn-access-log" env: VLLM_USE_DEEP_GEMM: "1" diff --git a/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-NvFp4-CT-fi-cutedsl-deepep-ll.yaml b/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-NvFp4-CT-fi-cutedsl-deepep-ll.yaml index 6624cea1ef..004e3e15e3 100644 --- a/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-NvFp4-CT-fi-cutedsl-deepep-ll.yaml +++ b/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-NvFp4-CT-fi-cutedsl-deepep-ll.yaml @@ -2,4 +2,6 @@ model_name: "RedHatAI/Qwen3-30B-A3B-NVFP4" accuracy_threshold: 0.88 num_questions: 1319 num_fewshot: 5 +# 30B MoE + CuTeDSL/DeepEP JIT compile can exceed the 1200s default startup wait. +startup_max_wait_seconds: 2400 server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --all2all-backend deepep_low_latency --moe-backend=flashinfer_cutedsl" diff --git a/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-NvFp4-ModelOpt-fi-cutedsl-deepep-ll.yaml b/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-NvFp4-ModelOpt-fi-cutedsl-deepep-ll.yaml index f2d4588e3a..55892f2196 100644 --- a/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-NvFp4-ModelOpt-fi-cutedsl-deepep-ll.yaml +++ b/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-NvFp4-ModelOpt-fi-cutedsl-deepep-ll.yaml @@ -2,4 +2,6 @@ model_name: "nvidia/Qwen3-30B-A3B-NVFP4" accuracy_threshold: 0.88 num_questions: 1319 num_fewshot: 5 +# 30B MoE + CuTeDSL/DeepEP JIT compile can exceed the 1200s default startup wait. +startup_max_wait_seconds: 2400 server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --all2all-backend deepep_low_latency --moe-backend=flashinfer_cutedsl" diff --git a/tests/evals/gsm8k/configs/moe-refactor-dp-ep/config-b200.txt b/tests/evals/gsm8k/configs/moe-refactor-dp-ep/config-b200.txt index 7fb1f49685..6dcb7414a5 100644 --- a/tests/evals/gsm8k/configs/moe-refactor-dp-ep/config-b200.txt +++ b/tests/evals/gsm8k/configs/moe-refactor-dp-ep/config-b200.txt @@ -10,3 +10,4 @@ Qwen3-30B-A3B-Fp8-CT-Block-deepgemm-deepep-ht.yaml Qwen3-30B-A3B-Fp8-CT-Block-deepgemm-deepep-ll.yaml Qwen3-30B-A3B-Fp8-CT-Block-deepgemm.yaml Qwen3-30B-A3B-BF16-triton.yaml +Qwen3-30B-A3B-BF16-moonep.yaml diff --git a/tests/evals/gsm8k/test_gsm8k_correctness.py b/tests/evals/gsm8k/test_gsm8k_correctness.py index 477a69a16c..7411ca0f40 100644 --- a/tests/evals/gsm8k/test_gsm8k_correctness.py +++ b/tests/evals/gsm8k/test_gsm8k_correctness.py @@ -128,6 +128,12 @@ def test_gsm8k_correctness(config_filename): "Skipping Qwen3.5-35B-A3B-MXFP4-AITER-TP2 on non-GFX950 platforms. " "The quantization scheme is not supported on non-GFX950 platforms." ) + if "--all2all-backend=moonep" in eval_config.get("server_args", ""): + from vllm.utils.import_utils import has_moonep + + if not has_moonep(): + pytest.skip("Skipping MoonEP config: the moonep package is not installed.") + # Parse server arguments from config (use shlex to handle quoted strings) server_args_str = eval_config.get("server_args", "") server_args = shlex.split(server_args_str) if server_args_str else [] diff --git a/tests/kernels/attention/test_cache.py b/tests/kernels/attention/test_cache.py index 0d5066d2f1..41f0a1138b 100644 --- a/tests/kernels/attention/test_cache.py +++ b/tests/kernels/attention/test_cache.py @@ -960,14 +960,14 @@ def test_concat_and_cache_ds_mla( tile_end = (tile_idx + 1) * 128 tile_data[:] = kv_c_data[tile_start:tile_end] - # tile_scale = tile_data.amax().to(torch.float32) / 448. - # NOTE: Using torch's amax() gives different results, - # so this must be manually computed. + # Using torch's amax() gives different results, so this must be + # manually computed. tile_data_float = tile_data.to(torch.float32) manual_max = abs(tile_data_float[0]) for j in range(1, 128): manual_max = max(manual_max, abs(tile_data_float[j])) - tile_scale = manual_max / 448.0 + raw_scale = torch.clamp(manual_max / 448.0, min=1e-4) + tile_scale = torch.exp2(torch.ceil(torch.log2(raw_scale))) ref_cache_32bit[kv_lora_rank // 4 + tile_idx] = tile_scale @@ -1008,7 +1008,7 @@ def test_concat_and_cache_ds_mla( ref_rope = ref_cache_slice.view(dtype)[kv_lora_rank // 2 + 8 :] torch.testing.assert_close(kv_nope, ref_nope, atol=0.001, rtol=0.1) - torch.testing.assert_close(kv_scales, ref_scales, atol=0.001, rtol=0.1) + torch.testing.assert_close(kv_scales, ref_scales, atol=0, rtol=0) torch.testing.assert_close(kv_rope, ref_rope, atol=0.001, rtol=0.1) diff --git a/tests/kernels/attention/test_flashmla_sparse.py b/tests/kernels/attention/test_flashmla_sparse.py index fb93557a32..bfcd6ae10a 100644 --- a/tests/kernels/attention/test_flashmla_sparse.py +++ b/tests/kernels/attention/test_flashmla_sparse.py @@ -109,8 +109,9 @@ def test_sparse_flashmla_metadata_smoke(): assert num_splits is None -def test_sparse_flashmla_decode_smoke(): +def test_sparse_flashmla_decode_matches_cache_writer_scales(): import vllm.v1.attention.ops.flashmla as fm + from vllm import _custom_ops as ops ok, reason = fm.is_flashmla_sparse_supported() if not ok: @@ -130,7 +131,7 @@ def test_sparse_flashmla_decode_smoke(): # Metadata q_seq_per_hk = seqlen_q * num_heads_q // num_heads_k # q_heads_per_hk = num_heads_q // num_heads_k - cache_seqlens = torch.zeros(batch_size, dtype=torch.int32, device=device) + cache_seqlens = torch.ones(batch_size, dtype=torch.int32, device=device) tile_md, num_splits = fm.get_mla_metadata( cache_seqlens, q_seq_per_hk, @@ -146,14 +147,33 @@ def test_sparse_flashmla_decode_smoke(): dtype=torch.bfloat16, device=device, ) - k_cache = torch.zeros( - (1, page_block_size, num_heads_k, bytes_per_token), - dtype=torch.uint8, - device=device, + cache_rows = torch.zeros( + (1, page_block_size, bytes_per_token), dtype=torch.uint8, device=device ) - indices = torch.zeros( - (batch_size, seqlen_q, topk), dtype=torch.int32, device=device + # 336 / 448 = 0.75 distinguishes arbitrary FP32 from power-of-two scales. + kv_c = torch.full((1, 512), 336.0, dtype=torch.bfloat16, device=device) + kv_c[:, -128:] = 0 + ops.concat_and_cache_mla( + kv_c, + torch.zeros((1, 64), dtype=torch.bfloat16, device=device), + cache_rows, + torch.zeros(1, dtype=torch.int64, device=device), + "fp8_ds_mla", + torch.ones(1, dtype=torch.float32, device=device), ) + scales = cache_rows[0, 0].view(torch.float32)[128:132] + expected_scales = torch.tensor([1.0, 1.0, 1.0, 2**-13], device=device) + torch.testing.assert_close(scales, expected_scales, rtol=0, atol=0) + cached_nope = cache_rows[0, 0, :512].view(torch.float8_e4m3fn).float() + expected = (cached_nope * scales.repeat_interleave(128)).to(torch.bfloat16) + + k_cache = cache_rows.unsqueeze(2) + indices = torch.full( + (batch_size, seqlen_q, topk), -1, dtype=torch.int32, device=device + ) + indices[..., 0] = 0 + # SM90 sparse decode only supports a fixed top-k width. Padding with -1 + # exercises the same cache data on SM90 and SM100 without topk_length. block_table = torch.zeros((batch_size, 128), dtype=torch.int32, device=device) out, lse = fm.flash_mla_with_kvcache( @@ -170,6 +190,9 @@ def test_sparse_flashmla_decode_smoke(): assert out.shape[0] == batch_size assert out.shape[-1] == head_dim_v assert lse.shape[0] == batch_size + torch.testing.assert_close( + out, expected.view(1, 1, 1, 512).expand_as(out), rtol=0, atol=0 + ) @pytest.mark.parametrize("h_q", [64, 128]) diff --git a/tests/kernels/attention/test_kimi_k3_mla_key_concat_kv_cache.py b/tests/kernels/attention/test_kimi_k3_mla_key_concat_kv_cache.py index e1582b7621..af41722605 100644 --- a/tests/kernels/attention/test_kimi_k3_mla_key_concat_kv_cache.py +++ b/tests/kernels/attention/test_kimi_k3_mla_key_concat_kv_cache.py @@ -222,7 +222,8 @@ def test_decode_concat_ignores_negative_slots_for_cache(cache_format: str) -> No _assert_cache_matches_reference(mixed_cache, reference_cache, initial_cache) -def test_ds_mla_cache_insert_bit_compatible_with_reference() -> None: +@pytest.mark.parametrize("decode", [False, True]) +def test_ds_mla_cache_insert_bit_compatible_with_reference(decode: bool) -> None: """Fused ds_mla insertion must match the reference bit-for-bit.""" torch.manual_seed(0) num_tokens, num_blocks = 33, 16 @@ -242,5 +243,18 @@ def test_ds_mla_cache_insert_bit_compatible_with_reference() -> None: got = ref.clone() scale = torch.ones(1, device="cuda", dtype=torch.float32) ops.concat_and_cache_mla(kv_c, k_pe.squeeze(1), ref, slots, "fp8_ds_mla", scale) - fused_mla_key_concat_ds_mla_insert(q, k_nope, k_pe, kv_c, got, slots) + if decode: + ql_nope = torch.randn( + num_tokens, NUM_HEADS, KV_LORA_RANK, device="cuda", dtype=dt + ) + fused_mla_decode_q_concat_kv_cache_insert( + ql_nope, q[..., -ROPE_HEAD_DIM:], kv_c, k_pe, got, slots, ds_mla=True + ) + else: + fused_mla_key_concat_ds_mla_insert(q, k_nope, k_pe, kv_c, got, slots) assert torch.equal(ref, got) + rows = got.view(-1, DS_MLA_CACHE_ENTRY)[slots[slots >= 0]] + scales = rows[:, KV_LORA_RANK : KV_LORA_RANK + 16].view(torch.float32) + torch.testing.assert_close( + torch.log2(scales), torch.log2(scales).round(), rtol=0, atol=0 + ) diff --git a/tests/kernels/attention/test_rocm_triton_attn_dsv4.py b/tests/kernels/attention/test_rocm_triton_attn_dsv4.py index f0ac4d8385..256b50d49a 100644 --- a/tests/kernels/attention/test_rocm_triton_attn_dsv4.py +++ b/tests/kernels/attention/test_rocm_triton_attn_dsv4.py @@ -1430,3 +1430,189 @@ def test_get_cached_wo_a_bf16_fp8_blockscale_caches() -> None: # Second call returns the same cached object. assert _get_cached_wo_a_bf16(wo_a, n_local_groups, o_lora_rank, hidden_dim) is out + + +@requires_split_decode_arch +@torch.inference_mode() +def test_fp8_paged_mqa_logits_triton_matches_torch_ref() -> None: + """Block-flat Triton decode logits vs the eager torch reference. + + Compare only valid key positions: the kernel skips the -inf pad that + the torch ref writes past seq_len. + """ + from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod + from vllm.v1.worker.workspace import init_workspace_manager + + device = torch.device("cuda") + init_workspace_manager(device) + torch.manual_seed(0) + + fp8_dtype = current_platform.fp8_dtype() + batch_size = 2 + next_n = 1 + num_heads = 4 + head_size = 128 + block_size = 64 + max_model_len = 256 + seq_lens = [180, 64] + num_pages = [(seq_len + block_size - 1) // block_size for seq_len in seq_lens] + num_blocks = sum(num_pages) + + q = torch.randn( + batch_size, next_n, num_heads, head_size, device=device, dtype=torch.bfloat16 + ).to(fp8_dtype) + weights = torch.rand( + batch_size * next_n, num_heads, device=device, dtype=torch.float32 + ) + context_lens = torch.tensor(seq_lens, device=device, dtype=torch.int32) + block_tables = torch.full( + (batch_size, max(num_pages)), 0, device=device, dtype=torch.int32 + ) + page = 0 + for i, n_pages in enumerate(num_pages): + block_tables[i, :n_pages] = torch.arange( + page, page + n_pages, device=device, dtype=torch.int32 + ) + page += n_pages + + kv_cache = torch.empty( + num_blocks, block_size, 1, head_size + 4, device=device, dtype=torch.uint8 + ) + values = torch.randn( + num_blocks, block_size, head_size, device=device, dtype=torch.bfloat16 + ).to(fp8_dtype) + scales = torch.rand( + num_blocks, block_size, device=device, dtype=torch.float32 + ).clamp(min=1e-3) + kv_flat = kv_cache.view(num_blocks, -1) + scale_off = block_size * head_size + kv_flat[:, :scale_off] = values.reshape(num_blocks, -1).view(torch.uint8) + kv_flat[:, scale_off:] = scales.contiguous().view(torch.uint8).view(num_blocks, -1) + + got = mod.rocm_fp8_paged_mqa_logits_triton( + q, kv_cache, weights, context_lens, block_tables, max_model_len + ) + ref = mod.fp8_paged_mqa_logits_torch( + q, kv_cache, weights, context_lens, block_tables, max_model_len + ) + for i, seq_len in enumerate(seq_lens): + torch.testing.assert_close( + got[i, :seq_len], ref[i, :seq_len], atol=2e-3, rtol=2e-3 + ) + + +def test_indexer_k_is_c4a_block_flat_gate() -> None: + from vllm.v1.attention.ops.rocm_aiter_mla_sparse import ( + _indexer_k_is_c4a_block_flat, + ) + + assert not _indexer_k_is_c4a_block_flat(1) + assert not _indexer_k_is_c4a_block_flat(2) + assert _indexer_k_is_c4a_block_flat(4) + assert not _indexer_k_is_c4a_block_flat(128) + + +@requires_split_decode_arch +@torch.inference_mode() +def test_paged_mqa_logits_gate_v41_shuffle_vs_c4a_flat() -> None: + """Ratio 1/2 stay on AITER; ratio 4 takes Triton.""" + from vllm.v1.attention.ops.rocm_aiter_mla_sparse import ( + fp8_paged_mqa_logits_torch, + indexer_k_quant_and_cache_triton, + rocm_fp8_paged_mqa_logits, + rocm_fp8_paged_mqa_logits_triton, + ) + from vllm.v1.worker.workspace import init_workspace_manager + + device = torch.device("cuda") + init_workspace_manager(device) + torch.manual_seed(1) + + fp8_dtype = current_platform.fp8_dtype() + batch_size = 1 + next_n = 1 + num_heads = 32 + head_size = 128 + block_size = 64 + seq_len = 180 + max_model_len = 256 + num_pages = (seq_len + block_size - 1) // block_size + num_tokens = num_pages * block_size + + q = torch.randn( + batch_size, next_n, num_heads, head_size, device=device, dtype=torch.bfloat16 + ).to(fp8_dtype) + weights = torch.rand( + batch_size * next_n, num_heads, device=device, dtype=torch.float32 + ) + context_lens = torch.tensor([seq_len], device=device, dtype=torch.int32) + block_tables = torch.arange(num_pages, device=device, dtype=torch.int32).view(1, -1) + dummy_sched = torch.zeros(8, 2, device=device, dtype=torch.int32) + + k_bf16 = torch.randn(num_tokens, head_size, device=device, dtype=torch.bfloat16) + slot = torch.arange(num_tokens, device=device, dtype=torch.int32) + cache_shuffle = torch.zeros( + num_pages, block_size, head_size + 4, device=device, dtype=torch.uint8 + ) + indexer_k_quant_and_cache_triton(k_bf16, cache_shuffle, slot, 128, "ue8m0") + cache_shuffle = cache_shuffle.unsqueeze(2) + + aiter = rocm_fp8_paged_mqa_logits( + q, + cache_shuffle, + weights, + context_lens, + block_tables, + dummy_sched, + max_model_len, + compress_ratio=1, + ) + for ratio in (1, 2): + got = rocm_fp8_paged_mqa_logits( + q, + cache_shuffle, + weights, + context_lens, + block_tables, + dummy_sched, + max_model_len, + compress_ratio=ratio, + ) + torch.testing.assert_close(got[:, :seq_len], aiter[:, :seq_len], atol=0, rtol=0) + + cache_flat = torch.empty( + num_pages, block_size, 1, head_size + 4, device=device, dtype=torch.uint8 + ) + values = torch.randn( + num_pages, block_size, head_size, device=device, dtype=torch.bfloat16 + ).to(fp8_dtype) + scales = torch.rand( + num_pages, block_size, device=device, dtype=torch.float32 + ).clamp(min=1e-3) + kv_flat = cache_flat.view(num_pages, -1) + scale_off = block_size * head_size + kv_flat[:, :scale_off] = values.reshape(num_pages, -1).view(torch.uint8) + kv_flat[:, scale_off:] = scales.contiguous().view(torch.uint8).view(num_pages, -1) + + gated_c4a = rocm_fp8_paged_mqa_logits( + q, + cache_flat, + weights, + context_lens, + block_tables, + dummy_sched, + max_model_len, + compress_ratio=4, + ) + triton = rocm_fp8_paged_mqa_logits_triton( + q, cache_flat, weights, context_lens, block_tables, max_model_len + ) + ref = fp8_paged_mqa_logits_torch( + q, cache_flat, weights, context_lens, block_tables, max_model_len + ) + torch.testing.assert_close( + gated_c4a[:, :seq_len], triton[:, :seq_len], atol=0, rtol=0 + ) + torch.testing.assert_close( + gated_c4a[:, :seq_len], ref[:, :seq_len], atol=2e-3, rtol=2e-3 + ) diff --git a/tests/kernels/attention/test_triton_mla_causal_verify_flatten.py b/tests/kernels/attention/test_triton_mla_causal_verify_flatten.py new file mode 100644 index 0000000000..a27552c90e --- /dev/null +++ b/tests/kernels/attention/test_triton_mla_causal_verify_flatten.py @@ -0,0 +1,215 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for the multi-token decode flatten in ``TritonMLAImpl.forward_mqa``. + +``decode_attention_fwd`` has no causal flag: it launches one program per row of +``q`` and reads that row's KV extent from ``B_Seqlen``. So a ``query_len``-token +decode block is flattened to one row per query token, and intra-block causality +has to be expressed through the per-row sequence lengths -- row ``t`` gets +``seq_len - (query_len - 1) + t``, i.e. the committed prefix plus block tokens +``0..t``. A non-causal draft block instead gives every row the same full extent. + +The kernel is replaced by a spy, so these tests assert on the ``block_table`` +and ``seq_lens`` the backend actually submits and need no GPU or Triton runtime. +""" + +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +import torch + +from vllm.model_executor.layers.attention.mla_attention import QueryLenSupport +from vllm.v1.attention.backend import AttentionCGSupport +from vllm.v1.attention.backends.mla.triton_mla import ( + TritonMLAImpl, + TritonMLAMetadataBuilder, +) + +NUM_HEADS = 16 +KV_LORA_RANK = 512 +QK_ROPE_HEAD_DIM = 64 +HEAD_SIZE = KV_LORA_RANK + QK_ROPE_HEAD_DIM +PAGE_SIZE = 16 + +# Committed context per request, including a fresh request at zero context +# where the whole KV extent is the block itself. A decode block's seq_len +# already counts the block, so seq_len = context + query_len. +CONTEXT_LENS = [0, 1, 37, 512] + + +def _seq_lens(query_len: int) -> list[int]: + return [c + query_len for c in CONTEXT_LENS] + + +def _make_impl(dcp_world_size: int = 1) -> TritonMLAImpl: + """Only the attributes ``forward_mqa`` reads; __init__ wants a full config.""" + impl = object.__new__(TritonMLAImpl) + impl.kv_lora_rank = KV_LORA_RANK + impl.scale = HEAD_SIZE**-0.5 + impl._sm_count = 304 + impl.dcp_world_size = dcp_world_size + return impl + + +def _make_metadata(seq_lens: list[int], query_len: int, causal: bool): + num_decodes = len(seq_lens) + return SimpleNamespace( + num_decodes=num_decodes, + num_decode_tokens=num_decodes * query_len, + max_seq_len=max(seq_lens), + causal=causal, + decode=SimpleNamespace( + # arange page ids so each request's window is an identifiable slice. + block_table=torch.arange(num_decodes * 64, dtype=torch.int32).reshape( + num_decodes, 64 + ), + seq_lens=torch.tensor(seq_lens, dtype=torch.int32), + ), + ) + + +def _run_forward_mqa( + seq_lens: list[int], + query_len: int, + causal: bool, + q_rows=None, + dcp_world_size: int = 1, +): + """Drive forward_mqa with the decode kernel spied out. + + Returns the ``(block_table, seq_lens)`` handed to ``decode_attention_fwd``. + """ + captured: dict = {} + + def spy(q, kv_cache, kv_c_cache, o, lse, block_table, b_seq_len, *args, **kwargs): + captured["block_table"] = block_table.detach().clone() + captured["seq_lens"] = b_seq_len.detach().clone() + + metadata = _make_metadata(seq_lens, query_len, causal) + rows = metadata.num_decode_tokens if q_rows is None else q_rows + q = torch.zeros(rows, NUM_HEADS, HEAD_SIZE, dtype=torch.bfloat16) + kv_cache = torch.zeros( + len(seq_lens) * 64, PAGE_SIZE, HEAD_SIZE, dtype=torch.bfloat16 + ) + layer = SimpleNamespace(_k_scale=torch.ones(1), layer_name="test") + + with patch("vllm.v1.attention.backends.mla.triton_mla.decode_attention_fwd", spy): + _make_impl(dcp_world_size).forward_mqa(q, kv_cache, metadata, layer) + + assert captured, "forward_mqa did not reach the decode kernel" + return captured + + +def test_multi_token_decode_flags_are_declared_together(): + """Capture probes max_query_len > 1, which only clears the assert in + build_for_cudagraph_capture once query_len_support raised the threshold.""" + assert ( + TritonMLAMetadataBuilder._cudagraph_support == AttentionCGSupport.UNIFORM_BATCH + ) + assert TritonMLAMetadataBuilder.query_len_support == QueryLenSupport.UNIFORM + + +@pytest.mark.parametrize("query_len", [2, 3, 4, 5, 8]) +def test_causal_block_rows_see_prefix_plus_own_position(query_len): + """Causal verify: row t sees seq_len - (query_len - 1) + t KV entries. + + Regression guard: the causal branch used to be skipped entirely, so a verify + token could attend the draft siblings it is supposed to be checking. Fails + on unmodified upstream, where no flatten happens at all for causal blocks. + """ + seq_lens = _seq_lens(query_len) + captured = _run_forward_mqa(seq_lens, query_len, causal=True) + + got = captured["seq_lens"].tolist() + want = [s - (query_len - 1) + t for s in seq_lens for t in range(query_len)] + assert got == want, ( + "causal flatten is not causal: verify row r*query_len+t must get " + f"seq_len_r - {query_len - 1} + t entries.\n" + f" seq_lens {seq_lens}\n got {got}\n expected {want}" + ) + + # Per-request invariants the arithmetic must hold regardless of query_len. + for r, seq_len in enumerate(seq_lens): + rows = got[r * query_len : (r + 1) * query_len] + assert rows == sorted(rows) and len(set(rows)) == query_len, ( + f"request {r} rows must strictly increase, got {rows}" + ) + assert rows[-1] == seq_len, "last row must see the full sequence" + assert rows[0] == seq_len - (query_len - 1) + assert all(n > 0 for n in rows), f"non-positive KV extent in {rows}" + + +def test_cudagraph_padding_rows_present_no_kv_extent(): + """Causal flattening pins cudagraph padding rows (seq_len 0) to exactly + zero KV extent, keeping them in the kernel's skipped branch.""" + query_len = 4 + seq_lens = _seq_lens(query_len) + [0] + captured = _run_forward_mqa(seq_lens, query_len, causal=True) + + padding = captured["seq_lens"].tolist()[-query_len:] + assert padding == [0] * query_len, ( + f"padding rows must present exactly zero KV extent, got {padding}" + ) + + +@pytest.mark.parametrize("query_len", [2, 4]) +def test_non_causal_block_rows_all_see_the_full_prefix(query_len): + """The DSpark draft block is generated in one pass, so no row is masked.""" + seq_lens = _seq_lens(query_len) + captured = _run_forward_mqa(seq_lens, query_len, causal=False) + + want = [s for s in seq_lens for _ in range(query_len)] + assert captured["seq_lens"].tolist() == want + + +@pytest.mark.parametrize("causal", [True, False]) +def test_block_table_is_expanded_to_match_q_rows(causal): + """The kernel indexes block_table by program_id, so it needs one row per + query token -- a short block_table is read out of bounds.""" + query_len = 4 + captured = _run_forward_mqa(_seq_lens(query_len), query_len, causal=causal) + + num_rows = len(CONTEXT_LENS) * query_len + assert captured["block_table"].shape[0] == num_rows + assert captured["seq_lens"].shape[0] == num_rows + # repeat_interleave, not repeat: a request's rows must stay adjacent. + for r in range(len(CONTEXT_LENS)): + block = captured["block_table"][r * query_len : (r + 1) * query_len] + assert torch.equal(block, block[:1].expand_as(block)) + + +@pytest.mark.parametrize("causal", [True, False]) +def test_single_token_decode_is_untouched(causal): + """query_len == 1 must short-circuit: ordinary decode sees no expansion.""" + seq_lens = _seq_lens(1) + captured = _run_forward_mqa(seq_lens, query_len=1, causal=causal) + + assert captured["seq_lens"].tolist() == seq_lens + assert captured["block_table"].shape[0] == len(seq_lens) + + +def test_expansion_factor_follows_the_q_row_count(): + """The kernel indexes block_table and seq_lens by program_id, so the + expansion must follow q rows, not the token/request counts.""" + num_rows = len(CONTEXT_LENS) * 4 + captured = _run_forward_mqa(_seq_lens(1), query_len=1, causal=True, q_rows=num_rows) + + assert captured["block_table"].shape[0] == num_rows + assert captured["seq_lens"].shape[0] == num_rows + + +def test_non_uniform_block_is_rejected(): + """Q rows must divide evenly across the decode requests.""" + with pytest.raises(AssertionError, match="non-uniform decode block"): + _run_forward_mqa(_seq_lens(1), query_len=1, causal=True, q_rows=6) + + +def test_causal_multi_token_decode_is_rejected_under_dcp(): + """Per-row extents offset the global seq_len, but DCP passes the rank-local + slice, so the causal arithmetic would silently address the wrong KV.""" + with pytest.raises(AssertionError, match="not supported with DCP"): + _run_forward_mqa(_seq_lens(4), query_len=4, causal=True, dcp_world_size=2) + + # Non-causal rows all take the same extent, which stays correct when local. + _run_forward_mqa(_seq_lens(4), query_len=4, causal=False, dcp_world_size=2) diff --git a/tests/kernels/mamba/test_gdn_prefill_cutedsl.py b/tests/kernels/mamba/test_gdn_prefill_cutedsl.py index 861b686e32..d86cfde2c7 100644 --- a/tests/kernels/mamba/test_gdn_prefill_cutedsl.py +++ b/tests/kernels/mamba/test_gdn_prefill_cutedsl.py @@ -192,3 +192,86 @@ def test_gdn_chunk_cutedsl_correctness(num_seqs: int, state_dtype: torch.dtype): assert no_buffer_state_error.mean().item() < 6e-4 assert buffer_o_error.max().item() == 0 assert buffer_state_error.max().item() == 0 + + +def test_gdn_chunk_cutedsl_correlated_keys_stability(): + """Cover a KKT block where three BF16 Newton rounds diverge.""" + total_tokens = 64 + num_k_heads = 4 + num_v_heads = 8 + head_dim = 128 + rng = torch.Generator("cuda").manual_seed(3456) + + q = torch.zeros( + 1, + total_tokens, + num_k_heads, + head_dim, + device="cuda", + dtype=torch.bfloat16, + ) + k = torch.zeros_like(q) + q[..., 0] = 1 + k[..., 0] = 1 + v = torch.randn( + 1, + total_tokens, + num_v_heads, + head_dim, + device="cuda", + dtype=torch.bfloat16, + generator=rng, + ) + g = torch.zeros( + 1, + total_tokens, + num_v_heads, + device="cuda", + dtype=torch.float32, + ) + beta = torch.full_like(g, 0.9) + initial_state = torch.zeros( + 1, + num_v_heads, + head_dim, + head_dim, + device="cuda", + dtype=torch.float32, + ) + cu_seqlens = torch.tensor([0, total_tokens], device="cuda", dtype=torch.int32) + chunk_indices, chunk_offsets = prepare_metadata_cutedsl(cu_seqlens, total_tokens) + + ref_o, ref_state = chunk_gated_delta_rule( + q=q, + k=k, + v=v, + g=g, + beta=beta, + initial_state=initial_state, + output_final_state=True, + cu_seqlens=cu_seqlens, + use_qk_l2norm_in_kernel=False, + ) + actual_o, actual_state = chunk_gated_delta_rule_cutedsl( + q=q, + k=k, + v=v, + g=g, + beta=beta, + initial_state=initial_state, + cu_seqlens=cu_seqlens, + chunk_indices=chunk_indices, + chunk_offsets=chunk_offsets, + ) + torch.accelerator.synchronize() + + assert torch.isfinite(actual_o).all() + assert torch.isfinite(actual_state).all() + output_error = (actual_o.float() - ref_o.float()).abs() + state_error = ( + actual_state.float() - ref_state.to(actual_state.dtype).float() + ).abs() + assert output_error.max().item() < 1e-1 + assert output_error.mean().item() < 2e-3 + assert state_error.max().item() < 2e-1 + assert state_error.mean().item() < 6e-4 diff --git a/tests/kernels/mamba/test_gdn_rocm_layout_dispatch.py b/tests/kernels/mamba/test_gdn_rocm_layout_dispatch.py new file mode 100644 index 0000000000..ae34ffcb80 --- /dev/null +++ b/tests/kernels/mamba/test_gdn_rocm_layout_dispatch.py @@ -0,0 +1,124 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Layout dispatch tests for ``QwenGatedDeltaNetAttention._forward_core_rocm``. + +The AITER fused reshape+conv kernel learned Qwen3.5's flat ``[q|k|v|z]`` +packing in https://github.com/ROCm/aiter/pull/3251, so flat-layout models take +the decode fast path too instead of falling back to the generic path. + +These tests run host-side on CPU: ``_forward_core_rocm`` is bound to a stub +layer whose ``_forward_core_decode_aiter``/``_forward_core`` record which +branch ran, so no GPU or AITER install is needed. +""" + +from __future__ import annotations + +import types +from unittest.mock import patch + +import pytest +import torch + +from vllm.model_executor.layers.mamba.gdn import qwen_gdn_linear_attn +from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import ( + QwenGatedDeltaNetAttention, +) +from vllm.platforms import current_platform +from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata + +pytestmark = pytest.mark.skipif( + not current_platform.is_rocm(), reason="ROCm-specific tests" +) + +PREFIX = "model.layers.0.linear_attn" +H = 2 # num key heads +HV = 4 # num value heads +K = 8 # head_k_dim +V = 8 # head_v_dim + + +def _make_metadata( + *, + num_prefills: int = 0, + num_decodes: int = 2, + spec_sequence_masks: torch.Tensor | None = None, +): + """Only the fields the dispatch condition reads carry meaningful values.""" + return GDNAttentionMetadata( + num_prefills=num_prefills, + num_prefill_tokens=num_prefills, + num_decodes=num_decodes, + num_decode_tokens=num_decodes, + num_spec_decodes=0, + num_spec_decode_tokens=0, + num_actual_tokens=num_prefills + num_decodes, + spec_sequence_masks=spec_sequence_masks, + ) + + +def _make_layer(gqa_interleaved_layout: bool): + """Stub layer running the real ``_forward_core_rocm``, recording dispatch.""" + layer = types.SimpleNamespace() + layer.prefix = PREFIX + layer.gqa_interleaved_layout = gqa_interleaved_layout + layer.qkvz_layout = "interleaved" if gqa_interleaved_layout else "flat" + + layer.calls = [] + layer._forward_core_decode_aiter = lambda **kw: layer.calls.append("aiter") + layer._forward_core = lambda **kw: layer.calls.append("generic") + layer.prepare_gdn_attention_core_inputs = lambda qkvz, ba, n: ( + torch.zeros(n, H * K * 2 + HV * V), + torch.zeros(n, HV, V), + torch.zeros(n, HV), + torch.zeros(n, HV), + ) + layer._forward_core_rocm = types.MethodType( + QwenGatedDeltaNetAttention._forward_core_rocm, layer + ) + return layer + + +def _run(layer, meta) -> str: + num_tokens = max(meta.num_actual_tokens, 1) + ctx = types.SimpleNamespace(attn_metadata={PREFIX: meta}) + with patch.object(qwen_gdn_linear_attn, "get_forward_context", return_value=ctx): + layer._forward_core_rocm( + qkvz=torch.zeros(num_tokens, 2 * H * K + 2 * HV * V), + ba=torch.zeros(num_tokens, 2 * HV), + z_out=torch.zeros(num_tokens, HV, V), + core_attn_out=torch.zeros(num_tokens, HV, V), + ) + assert len(layer.calls) == 1 + return layer.calls[0] + + +@pytest.mark.parametrize("gqa_interleaved_layout", [True, False]) +def test_pure_decode_takes_the_fast_path(gqa_interleaved_layout: bool) -> None: + """Both packings reach the fast path; flat used to be guarded out.""" + layer = _make_layer(gqa_interleaved_layout) + assert _run(layer, _make_metadata()) == "aiter" + + +@pytest.mark.parametrize("gqa_interleaved_layout", [True, False]) +def test_layout_string_matches_the_packing(gqa_interleaved_layout: bool) -> None: + """The value handed to the kernel's ``qkvz_layout`` parameter.""" + layer = _make_layer(gqa_interleaved_layout) + expected = "interleaved" if gqa_interleaved_layout else "flat" + assert layer.qkvz_layout == expected + + +@pytest.mark.parametrize("gqa_interleaved_layout", [True, False]) +@pytest.mark.parametrize( + "meta_kwargs", + [ + {"num_prefills": 1}, + {"num_decodes": 0}, + {"spec_sequence_masks": torch.zeros(1, dtype=torch.bool)}, + ], + ids=["has_prefill", "no_decode", "spec_decode"], +) +def test_non_pure_decode_batches_use_the_generic_path( + gqa_interleaved_layout: bool, meta_kwargs: dict +) -> None: + layer = _make_layer(gqa_interleaved_layout) + assert _run(layer, _make_metadata(**meta_kwargs)) == "generic" diff --git a/tests/kernels/moe/test_cpu_fused_moe.py b/tests/kernels/moe/test_cpu_fused_moe.py index cf83ef6fc0..d4c5437221 100644 --- a/tests/kernels/moe/test_cpu_fused_moe.py +++ b/tests/kernels/moe/test_cpu_fused_moe.py @@ -306,17 +306,28 @@ def test_cpu_fused_moe( @pytest.mark.skipif( - current_platform.get_cpu_architecture() != CpuArchEnum.ARM, - reason="Requires Arm CPU", + current_platform.get_cpu_architecture() + not in (CpuArchEnum.ARM, CpuArchEnum.POWERPC), + reason="Requires Arm or POWER CPU", ) @pytest.mark.parametrize("batch_size", BATCH_SIZE) @pytest.mark.parametrize("expert_num", EXPERT_NUM) @pytest.mark.parametrize("hidden_size", HIDDEN_DIM) @pytest.mark.parametrize("intermediate_size", INTERMEDIATE_DIM) @pytest.mark.parametrize("use_bias", USE_BIAS) -@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) +@pytest.mark.parametrize( + "dtype", + ( + [torch.float32, torch.float16, torch.bfloat16] + if current_platform.get_cpu_architecture() == CpuArchEnum.ARM + else [torch.float32, torch.bfloat16] + ), +) @pytest.mark.parametrize("act", ACT) -@pytest.mark.parametrize("isa", ["neon"]) +@pytest.mark.parametrize( + "isa", + ["neon"] if current_platform.get_cpu_architecture() == CpuArchEnum.ARM else ["vsx"], +) def test_cpu_fused_moe_int8( batch_size: int, expert_num: int, @@ -386,7 +397,8 @@ def test_cpu_fused_moe_int8( isa, ) - torch.testing.assert_close(output, ref_output, atol=2e-2, rtol=2e-2) + atol = rtol = 1e-1 if isa == "vsx" else 2e-2 + torch.testing.assert_close(output, ref_output, atol=atol, rtol=rtol) # moe_intermediate_size not a multiple of 32, e.g. what tensor-parallel diff --git a/tests/kernels/moe/test_cpu_quant_fused_moe.py b/tests/kernels/moe/test_cpu_quant_fused_moe.py index 99cae4534b..e89fbb55a0 100644 --- a/tests/kernels/moe/test_cpu_quant_fused_moe.py +++ b/tests/kernels/moe/test_cpu_quant_fused_moe.py @@ -1114,5 +1114,203 @@ def test_int8_w8a8_cpu_fused_moe(M, N, K, E, topk, seed, is_vnni, inplace): ) +# =========================================================================== +# FP8 W8A8 MoE +# =========================================================================== + + +requires_cpu_fp8_w8a8 = pytest.mark.skipif( + not ops.cpu_has_amx_fp8(), + reason="requires native AMX-FP8 hardware", +) + +FP8_W8A8_MAX = torch.finfo(torch.float8_e4m3fn).max +FP8_W8A8_QUANT_GROUP = 128 # quantization group size for both K and N dimensions + + +def _make_fp8_w8a8_weight_w13( + E: int, two_n: int, K: int, group_K: int = FP8_W8A8_QUANT_GROUP +) -> tuple[torch.Tensor, torch.Tensor]: + """Create w13 [E, 2N, K] FP8 with group scales [E, 2N, G] where G = K//group_K.""" + G = K // group_K + w_list, s_list = [], [] + for _ in range(E): + w_f32 = torch.randn(two_n, K) + # Per-row, per-K-group quantization: scale [2N, G] + w_re = w_f32.view(two_n, G, group_K) + abs_max = w_re.abs().amax(dim=2, keepdim=True).clamp(min=1e-7) # [2N, G, 1] + scale = (abs_max / FP8_W8A8_MAX).squeeze(2) # [2N, G] + w_q = ( + (w_re / abs_max).clamp(-FP8_W8A8_MAX, FP8_W8A8_MAX).to(torch.float8_e4m3fn) + ) + w_list.append(w_q.view(two_n, K).contiguous()) + s_list.append(scale.float()) + return torch.stack(w_list), torch.stack(s_list) # [E, 2N, K], [E, 2N, G] + + +def _make_fp8_w8a8_weight_w2( + E: int, + K: int, + N: int, + group_K: int = FP8_W8A8_QUANT_GROUP, + group_N: int = FP8_W8A8_QUANT_GROUP, +) -> tuple[torch.Tensor, torch.Tensor]: + """Create w2 [E, K, N] FP8 with block scales [E, K//gK, N//gN].""" + nK = K // group_K + nN = N // group_N + w_list, s_list = [], [] + for _ in range(E): + w_f32 = torch.randn(K, N) + w_re = w_f32.view(nK, group_K, nN, group_N) + abs_max = w_re.abs().amax(dim=(1, 3), keepdim=True).clamp(min=1e-7) + scale = (abs_max / FP8_W8A8_MAX).squeeze(1).squeeze(2) # [nK, nN] + w_q = ( + (w_re / abs_max).clamp(-FP8_W8A8_MAX, FP8_W8A8_MAX).to(torch.float8_e4m3fn) + ) + w_list.append(w_q.view(K, N).contiguous()) + s_list.append(scale.float()) + return torch.stack(w_list), torch.stack(s_list) # [E, K, N], [E, K//gK, N//gN] + + +def _pack_fp8_w8a8_w13( + w13: torch.Tensor, # [E, 2N, K] FP8 + w13_scale: torch.Tensor, # [E, 2N, G] float32 +) -> tuple[torch.Tensor, torch.Tensor]: + """Pack w13 for CPU FP8 W8A8 MoE kernel using float8_linear_prepack_cpu.""" + E = w13.size(0) + packed_list, scale_list = [], [] + for i in range(E): + pw, ps = torch.ops._C.float8_linear_prepack_cpu( + w13[i].contiguous(), w13_scale[i].contiguous() + ) + packed_list.append(pw) + scale_list.append(ps) + return torch.stack(packed_list), torch.stack(scale_list) + + +def _ref_fp8_w8a8_moe( + hidden_states: torch.Tensor, # BF16 [M, K_hidden] (after FP8 round-trip dequant) + w13: torch.Tensor, # FP8 [E, 2N, K_hidden] + w13_scale: torch.Tensor, # float32 [E, 2N, G] (G = K_hidden//group_K) + w2: torch.Tensor, # FP8 [E, K_hidden, N] + w2_scale: torch.Tensor, # float32 [E, K_hidden//gK, N//gN] + topk_weights: torch.Tensor, # float32 [M, top_k] + topk_ids: torch.Tensor, # int64 [M, top_k] + N: int, + group_K: int = FP8_W8A8_QUANT_GROUP, + group_N: int = FP8_W8A8_QUANT_GROUP, +) -> torch.Tensor: + """Reference implementation: dequantize weights then compute MoE.""" + M, K_hidden = hidden_states.shape + top_k = topk_ids.shape[1] + output = torch.zeros(M, K_hidden, dtype=torch.bfloat16) + + for tok in range(M): + x = hidden_states[tok].float() # [K_hidden] + for k_idx in range(top_k): + expert_id = topk_ids[tok, k_idx].item() + w = topk_weights[tok, k_idx].item() + + # Stage 1: x @ w13.T — gate and up projections + w13_e = w13[expert_id].float() # [2N, K_hidden] + ws_e = w13_scale[expert_id].float() # [2N, G] + G = ws_e.shape[1] + w13_dq = torch.zeros_like(w13_e) + for g in range(G): + c0, c1 = g * group_K, (g + 1) * group_K + w13_dq[:, c0:c1] = w13_e[:, c0:c1] * ws_e[:, g : g + 1] + gate_up = (x.unsqueeze(0) @ w13_dq.T).squeeze(0) # [2N] + + gate = gate_up[:N] + up = gate_up[N:] + act = F.silu(gate) * up # [N] + + # Stage 2: act @ w2.T — down projection + w2_e = w2[expert_id].float() # [K_hidden, N] + ws2_e = w2_scale[expert_id].float() # [K_hidden//gK, N//gN] + nK2 = K_hidden // group_K + nN2 = N // group_N + w2_dq = torch.zeros_like(w2_e) + for gi in range(nK2): + for gj in range(nN2): + r0, r1 = gi * group_K, (gi + 1) * group_K + c0, c1 = gj * group_N, (gj + 1) * group_N + w2_dq[r0:r1, c0:c1] = w2_e[r0:r1, c0:c1] * ws2_e[gi, gj] + out_e = (act.unsqueeze(0) @ w2_dq.T).squeeze(0) # [K_hidden] + + output[tok] += (w * out_e).to(torch.bfloat16) + + return output + + +@requires_cpu_fp8_w8a8 +@pytest.mark.parametrize( + "E,N,K,M,top_k", + [ + # N and K must be multiples of FP8_W8A8_QUANT_GROUP=128 + (4, 128, 256, 1, 2), # single token (decode phase) + (4, 128, 256, 8, 2), + (8, 256, 512, 16, 2), + (4, 256, 512, 4, 1), + ], +) +def test_fp8_w8a8_cpu_fused_moe(E: int, N: int, K: int, M: int, top_k: int): + """Test fused_experts_cpu FP8_W8A8 shape/dtype and accuracy vs a + dequantized BF16 reference.""" + set_random_seed(42) + + w13, w13_scale = _make_fp8_w8a8_weight_w13(E, 2 * N, K) + w2, w2_scale = _make_fp8_w8a8_weight_w2(E, K, N) + + packed_w13, packed_w13_scale = _pack_fp8_w8a8_w13(w13, w13_scale) + packed_w2 = _prepack_experts(w2) + + hidden_states = torch.randn(M, K, dtype=torch.bfloat16) * 0.1 + x_fp8, x_scales = torch.ops._C.quantize_fp8e4m3_vec(hidden_states, True, None) + assert x_fp8.dtype == torch.float8_e4m3fn + assert x_scales.shape == (M,) + + gen = torch.Generator().manual_seed(0) + topk_weights = torch.softmax(torch.randn(M, top_k, generator=gen), dim=-1).float() + topk_ids = torch.zeros(M, top_k, dtype=torch.int32) + for i in range(M): + topk_ids[i] = torch.randperm(E, generator=torch.Generator().manual_seed(i))[ + :top_k + ].int() + + output = torch.empty(M, K, dtype=torch.bfloat16) + ops.fused_experts_cpu( + output, + x_fp8, + packed_w13, + packed_w2, + topk_weights, + topk_ids, + ops.CPUQuantMethod.FP8_W8A8, + packed_w13_scale, # w1_scale: [E, Nc, G, BLOCK_N] + w2_scale, # w2_scale: [E, K//128, N//128] + None, # w1_zero + None, # w2_zero + [FP8_W8A8_QUANT_GROUP, FP8_W8A8_QUANT_GROUP], # block_size + None, # w1_bias + None, # w2_bias + None, # alpha + None, # limit + True, # is_vnni (weights already packed) + x_scales, # a1_scale: per-token FP8 activation scales + ) + + assert output.shape == (M, K) + assert output.dtype == torch.bfloat16 + assert not torch.isnan(output).any() + assert not torch.isinf(output).any() + + x_dequant = (x_fp8.float() * x_scales.unsqueeze(1)).bfloat16() + ref = _ref_fp8_w8a8_moe( + x_dequant, w13, w13_scale, w2, w2_scale, topk_weights, topk_ids.long(), N + ) + torch.testing.assert_close(output.float(), ref.float(), atol=1e-2, rtol=1e-2) + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"])) diff --git a/tests/kernels/moe/test_deepgemm.py b/tests/kernels/moe/test_deepgemm.py index f6df64c51b..cee0faa592 100644 --- a/tests/kernels/moe/test_deepgemm.py +++ b/tests/kernels/moe/test_deepgemm.py @@ -26,6 +26,10 @@ from vllm.model_executor.layers.fused_moe.config import ( ) from vllm.model_executor.layers.fused_moe.deep_gemm_utils import ( deepgemm_moe_permute, + ep_gather, +) +from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import ( + _fp8_workspace_shape, ) from vllm.model_executor.layers.fused_moe.experts.triton_deep_gemm_moe import ( TritonOrDeepGemmExperts, @@ -43,6 +47,42 @@ from vllm.utils.deep_gemm import ( BLOCK_SIZE = [128, 128] +def test_ep_gather_uses_64_bit_row_offsets(): + hidden_size = 4096 + source_row = 1 << 18 + input_tensor = torch.empty( + (source_row + 1, hidden_size), + device="cuda", + dtype=torch.bfloat16, + ) + input_tensor[source_row].fill_(1) + output = torch.empty((1, hidden_size), device="cuda", dtype=torch.bfloat16) + + ep_gather( + input_tensor=input_tensor, + recv_topk_ids=torch.zeros((1, 1), device="cuda", dtype=torch.int64), + recv_topk_weight=torch.ones((1, 1), device="cuda", dtype=torch.float32), + input_index=torch.full((1, 1), source_row, device="cuda", dtype=torch.int32), + expert_map=None, + output_tensor=output, + ) + + torch.testing.assert_close(output, torch.ones_like(output), rtol=0, atol=0) + + +@pytest.mark.parametrize("workspace_dtype", [torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("num_columns", [2048, 6144, 6145]) +def test_fp8_workspace_shape(workspace_dtype, num_columns): + num_rows = 17 + shape = _fp8_workspace_shape(num_rows, num_columns, workspace_dtype) + + allocated_bytes = math.prod(shape) * workspace_dtype.itemsize + required_bytes = num_rows * num_columns * torch.float8_e4m3fn.itemsize + + assert allocated_bytes >= required_bytes + assert allocated_bytes - required_bytes < num_rows * workspace_dtype.itemsize + + @pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels") def test_deepgemm_moe_permute_initializes_padding_scales(workspace_init): hidden_states = torch.randn(2, 128, device="cuda", dtype=torch.bfloat16) diff --git a/tests/kernels/moe/test_flashinfer.py b/tests/kernels/moe/test_flashinfer.py index 2e022ede36..9189c5c641 100644 --- a/tests/kernels/moe/test_flashinfer.py +++ b/tests/kernels/moe/test_flashinfer.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import multiprocessing from dataclasses import dataclass +from unittest.mock import MagicMock import pytest import torch @@ -22,6 +24,9 @@ from vllm.model_executor.layers.fused_moe.config import ( from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import ( FlashInferExperts, ) +from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import ( + view_as_block_major_k, +) from vllm.model_executor.layers.fused_moe.experts.trtllm_fp8_moe import ( TrtLlmFp8ExpertsModular, TrtLlmFp8ExpertsMonolithic, @@ -433,6 +438,273 @@ def test_convert_moe_weights_to_flashinfer_trtllm_block_layout( assert w13_converted.shape[0] == num_experts assert w2_converted.shape[0] == num_experts + assert w13_converted.data_ptr() == w13.data_ptr() + assert w2_converted.data_ptr() == w2.data_ptr() + + +@pytest.mark.parametrize("is_gated_act_gemm", [True, False]) +def test_convert_moe_weights_to_flashinfer_trtllm_block_layout_values( + is_gated_act_gemm, +): + from flashinfer.fused_moe.core import ( + _maybe_get_cached_w3_w1_permute_indices, + get_w2_permute_indices_with_cache, + ) + + from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( + convert_moe_weights_to_flashinfer_trtllm_block_layout, + ) + + num_experts, intermediate, hidden = 2, 256, 256 + w13_multiplier = 2 if is_gated_act_gemm else 1 + w13 = torch.randn( + (num_experts, w13_multiplier * intermediate, hidden), + dtype=torch.bfloat16, + device="cuda", + ) + w2 = torch.randn( + (num_experts, hidden, intermediate), + dtype=torch.bfloat16, + device="cuda", + ) + + def _reference_block_layout( + weight: torch.Tensor, + is_w13: bool, + cache: dict[torch.Size, torch.Tensor], + ) -> torch.Tensor: + outputs = [] + for expert in weight: + expert_uint8 = expert.view(torch.uint8) + if is_w13: + indices = _maybe_get_cached_w3_w1_permute_indices( + cache, + expert_uint8, + 128, + is_gated_act_gemm=is_gated_act_gemm, + ) + if is_gated_act_gemm: + indices = (indices + expert_uint8.shape[0] // 2) % ( + expert_uint8.shape[0] + ) + else: + indices = get_w2_permute_indices_with_cache( + cache, + expert_uint8, + 128, + ) + rows, cols = expert_uint8.shape + blocks = expert_uint8.view(rows, cols // 128, 128).permute(1, 0, 2) + outputs.append(torch.index_select(blocks, 1, indices.to(weight.device))) + return torch.stack(outputs).view(torch.bfloat16) + + reference_cache: dict[torch.Size, torch.Tensor] = {} + expected_w13 = _reference_block_layout(w13, is_w13=True, cache=reference_cache) + expected_w2 = _reference_block_layout(w2, is_w13=False, cache=reference_cache) + w13_ptr = w13.data_ptr() + w2_ptr = w2.data_ptr() + + actual_w13, actual_w2 = convert_moe_weights_to_flashinfer_trtllm_block_layout( + {}, + w13, + w2, + is_gated_act_gemm=is_gated_act_gemm, + ) + + assert actual_w13.data_ptr() == w13_ptr + assert actual_w2.data_ptr() == w2_ptr + assert torch.equal(actual_w13, expected_w13) + assert torch.equal(actual_w2, expected_w2) + + +def _make_unquantized_flashinfer_test_layer( + monkeypatch, intermediate, is_gated, *, device="cuda" +): + from vllm.model_executor.layers.fused_moe.oracle.unquantized import ( + UnquantizedMoeBackend, + ) + from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import ( + UnquantizedFusedMoEMethod, + ) + + moe_config = make_dummy_moe_config( + num_experts=2, + hidden_dim=256, + intermediate_size=intermediate, + activation=MoEActivation.SILU if is_gated else MoEActivation.RELU2_NO_MUL, + ) + moe_config.intermediate_size_per_partition = (intermediate + 127) // 128 * 128 + method = object.__new__(UnquantizedFusedMoEMethod) + method.moe = moe_config + method.unquantized_backend = UnquantizedMoeBackend.FLASHINFER_TRTLLM + method.moe_kernel = None + mock_kernel = MagicMock() + monkeypatch.setattr( + method, + "_init_moe_kernel", + lambda _: setattr(method, "moe_kernel", mock_kernel), + ) + + layer = torch.nn.Module() + layer.moe_config = moe_config + with torch.device(device): + method.create_weights( + layer, + num_experts=2, + hidden_size=256, + intermediate_size_per_partition=moe_config.intermediate_size_per_partition, + params_dtype=torch.bfloat16, + ) + return method, layer + + +@pytest.mark.parametrize("intermediate", [192, 256]) +@pytest.mark.parametrize("is_gated", [True, False]) +def test_unquantized_flashinfer_trtllm_weights_can_be_reprocessed( + monkeypatch, intermediate, is_gated +): + """Each raw reload clears padding left dirty by prior in-place conversion.""" + from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( + convert_moe_weights_to_flashinfer_trtllm_block_layout, + ) + + method, layer = _make_unquantized_flashinfer_test_layer( + monkeypatch, intermediate, is_gated + ) + w13_shape, w2_shape = layer.w13_weight.shape, layer.w2_weight.shape + padded = w2_shape[-1] + w13_ptr = layer.w13_weight.data_ptr() + w2_ptr = layer.w2_weight.data_ptr() + load_numel = ( + layer.w13_weight.weight_loader_numel, + layer.w2_weight.weight_loader_numel, + ) + + for _ in range(2): + reloaded_w13 = torch.randn_like(layer.w13_weight) + reloaded_w2 = torch.randn_like(layer.w2_weight) + reloaded_w13[:, intermediate:padded].zero_() + if is_gated: + reloaded_w13[:, padded + intermediate :].zero_() + reloaded_w2[:, :, intermediate:].zero_() + expected_w13, expected_w2 = ( + convert_moe_weights_to_flashinfer_trtllm_block_layout( + {}, + reloaded_w13.clone(), + reloaded_w2.clone(), + is_gated_act_gemm=is_gated, + ) + ) + + # The loader writes only logical checkpoint slices, not padding. + layer.w13_weight.fill_(float("nan")) + layer.w2_weight.fill_(float("nan")) + layer.w13_weight[:, :intermediate].copy_(reloaded_w13[:, :intermediate]) + if is_gated: + layer.w13_weight[:, padded : padded + intermediate].copy_( + reloaded_w13[:, padded : padded + intermediate] + ) + layer.w2_weight[:, :, :intermediate].copy_(reloaded_w2[:, :, :intermediate]) + method.process_weights_after_loading(layer) + + assert layer.w13_weight.shape == w13_shape + assert layer.w2_weight.shape == w2_shape + assert layer.w13_weight.data_ptr() == w13_ptr + assert layer.w2_weight.data_ptr() == w2_ptr + assert ( + layer.w13_weight.weight_loader_numel, + layer.w2_weight.weight_loader_numel, + ) == load_numel + kernel_w13 = view_as_block_major_k(layer.w13_weight) + kernel_w2 = view_as_block_major_k(layer.w2_weight) + assert torch.equal(kernel_w13, expected_w13) + assert torch.equal(kernel_w2, expected_w2) + + +def _check_flashinfer_ipc_weights(entries, expected, mode, device_index): + from vllm.model_executor.model_loader.weight_cache.ipc_loader import IpcModelLoader + from vllm.model_executor.utils import weights_already_processed + + torch.accelerator.set_device_index(device_index) + with pytest.MonkeyPatch.context() as monkeypatch: + method, layer = _make_unquantized_flashinfer_test_layer( + monkeypatch, 192, is_gated=True, device="meta" + ) + loader = object.__new__(IpcModelLoader) + loader.mode = mode + loader._apply_entries(layer, entries, {}, device_index) + pointers = (layer.w13_weight.data_ptr(), layer.w2_weight.data_ptr()) + + # Like ipc_cache: a fresh method, only tensor metadata, no _setup_kernel. + with weights_already_processed(): + method.process_weights_after_loading(layer) + assert method.moe_kernel is not None + actual = ( + view_as_block_major_k(layer.w13_weight), + view_as_block_major_k(layer.w2_weight), + ) + for weight, reference, pointer in zip(actual, expected, pointers): + assert weight.data_ptr() == pointer + assert torch.equal(weight.cpu(), reference) + # Cross-process addresses need not match; writes prove sharing/isolation. + with torch.no_grad(): + for weight in actual: + weight.fill_(1.0) + torch.accelerator.synchronize() + + +@pytest.mark.parametrize("cache_ndim", [3, 4]) +@pytest.mark.parametrize("mode", ["copy", "zero_copy"]) +def test_unquantized_flashinfer_trtllm_cached_weights_need_no_method_state( + monkeypatch, cache_ndim, mode +): + """A spawned IPC consumer restores views without transient method state.""" + from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( + convert_moe_weights_to_flashinfer_trtllm_block_layout, + ) + from vllm.model_executor.model_loader.weight_cache.protocol import TensorEntry + + method, layer = _make_unquantized_flashinfer_test_layer( + monkeypatch, 192, is_gated=True, device="meta" + ) + packed = convert_moe_weights_to_flashinfer_trtllm_block_layout( + {}, + torch.randn_like(layer.w13_weight, device="cuda"), + torch.randn_like(layer.w2_weight, device="cuda"), + ) + expected = [weight.cpu() for weight in packed] + entries = {} + for name, weight in zip(("w13_weight", "w2_weight"), packed): + cached = weight.view_as(getattr(layer, name)) if cache_ndim == 3 else weight + entries[name] = TensorEntry.from_tensor(cached, kind="param") + consumer = multiprocessing.get_context("spawn").Process( + target=_check_flashinfer_ipc_weights, + args=( + entries, + expected, + mode, + torch.accelerator.current_device_index(), + ), + ) + consumer.start() + try: + # Keep producer allocations alive until the consumer releases its views. + consumer.join(timeout=180) + assert not consumer.is_alive(), "IPC consumer timed out" + assert consumer.exitcode == 0, "IPC consumer failed; see child traceback" + for source, reference in zip(packed, expected): + expected_source = ( + torch.ones_like(reference) if mode == "zero_copy" else reference + ) + assert torch.equal(source.cpu(), expected_source) + finally: + if consumer.is_alive(): + consumer.terminate() + consumer.join(timeout=10) + if consumer.is_alive(): + consumer.kill() + consumer.join(timeout=10) + consumer.close() @pytest.mark.parametrize( diff --git a/tests/kernels/moe/test_fp8_tp_loading.py b/tests/kernels/moe/test_fp8_tp_loading.py new file mode 100644 index 0000000000..680ccb812c --- /dev/null +++ b/tests/kernels/moe/test_fp8_tp_loading.py @@ -0,0 +1,311 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from types import SimpleNamespace + +import pytest +import torch + +import vllm.distributed as distributed +from tests.kernels.moe.utils import make_dummy_moe_config +from vllm.model_executor.layers.fused_moe import ( + unquantized_fused_moe_method as unquantized_module, +) +from vllm.model_executor.layers.fused_moe.config import RoutingMethodType +from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts +from vllm.model_executor.layers.quantization import fp8 as fp8_module +from vllm.model_executor.layers.quantization.fp8 import Fp8Config + + +def _make_fp8_tp_experts( + monkeypatch, + tp_size, + tp_rank, + backend="flashinfer_trtllm", + *, + mock_backend=True, + num_experts=2, + hidden_dim=256, + intermediate_size=640, + quant_config=None, + config_overrides=None, +): + # Exercise allocation and the public weight loader without selecting a GPU kernel. + if mock_backend: + monkeypatch.setattr( + fp8_module, "select_fp8_moe_backend", lambda **kwargs: (None, None) + ) + monkeypatch.setattr( + unquantized_module, + "select_unquantized_moe_backend", + lambda **kwargs: (None, None), + ) + monkeypatch.setattr( + fp8_module, "get_tensor_model_parallel_world_size", lambda: tp_size + ) + # The block-shape validator imports this function from vllm.distributed. + monkeypatch.setattr( + distributed, "get_tensor_model_parallel_world_size", lambda: tp_size + ) + config = make_dummy_moe_config( + num_experts=num_experts, + hidden_dim=hidden_dim, + intermediate_size=intermediate_size, + experts_per_token=min(num_experts, 10), + ) + config.routing_method = RoutingMethodType.RenormalizeNaive + config.moe_parallel_config.tp_size = tp_size + config.moe_parallel_config.tp_rank = tp_rank + config.intermediate_size_per_partition = intermediate_size // tp_size + config.intermediate_size_per_partition_unpadded = intermediate_size // tp_size + config.moe_backend = backend + for field, value in (config_overrides or {}).items(): + setattr(config, field, value) + return RoutedExperts( + "model.layers.0.mlp.experts", + torch.bfloat16, + config, + quant_config + or Fp8Config(is_checkpoint_fp8_serialized=True, weight_block_size=[128, 128]), + expert_map_manager=SimpleNamespace( + local_num_experts=num_experts, + placement_strategy="linear", + expert_map=None, + expert_mask=None, + routing_tables=None, + map_global_to_local=lambda index: index, + ), + ) + + +@pytest.mark.parametrize("tp_size", [2, 4, 8]) +@pytest.mark.parametrize("batched", [False, True]) +@pytest.mark.parametrize("intermediate_size", [640, 896]) +def test_fp8_block_aligned_tp_preserves_checkpoint( + monkeypatch, tp_size, batched, intermediate_size +): + """All ranks reconstruct the original dequantized projections, including + padding-only ranks. Reloading must clear stale weights and scales. + """ + generator = torch.Generator().manual_seed(42) + weights = { + "w1": torch.randn(2, intermediate_size, 256, generator=generator).to( + torch.float8_e4m3fn + ), + "w3": torch.randn(2, intermediate_size, 256, generator=generator).to( + torch.float8_e4m3fn + ), + "w2": torch.randn(2, 256, intermediate_size, generator=generator).to( + torch.float8_e4m3fn + ), + } + scales = { + name: torch.rand(2, w.shape[1] // 128, w.shape[2] // 128, generator=generator) + + 0.1 + for name, w in weights.items() + } + reconstructed: dict[str, list[torch.Tensor]] = {name: [] for name in weights} + num_blocks = intermediate_size // 128 + for rank in range(tp_size): + layer = _make_fp8_tp_experts( + monkeypatch, tp_size, rank, intermediate_size=intermediate_size + ) + assert layer.quant_method.weight_scale_refine is None + assert layer.quant_method.moe_block_shape == [128, 128] + width = layer.moe_config.intermediate_size_per_partition + assert width == ((num_blocks + tp_size - 1) // tp_size) * 128 + for name in weights: + prefix = "w2" if name == "w2" else "w13" + for suffix, checkpoint in ( + ("weight", weights[name]), + ("weight_scale_inv", scales[name]), + ): + param_name = f"{prefix}_{suffix}" + param = getattr(layer, param_name) + # Also reload after dirtying the destination, to catch padding leaks. + for _ in range(2): + if name != "w3": + param.data.fill_(7) + if batched: + assert param.weight_loader( + param, checkpoint, param_name, name, 0, return_success=True + ) + else: + for expert in range(2): + assert param.weight_loader( + param, + checkpoint[expert], + param_name, + name, + expert, + return_success=True, + ) + w = getattr(layer, f"{prefix}_weight").float() + s = getattr(layer, f"{prefix}_weight_scale_inv") + if name in ("w1", "w3"): + half = 0 if name == "w1" else 1 + w = w.chunk(2, dim=1)[half] + s = s.chunk(2, dim=1)[half] + dim = 2 if name == "w2" else 1 + valid = max(0, min(width, intermediate_size - rank * width)) + assert torch.count_nonzero(w.narrow(dim, valid, width - valid)) == 0 + assert torch.all(s.narrow(dim, valid // 128, (width - valid) // 128) == 1) + dequant = w * s.repeat_interleave(128, dim=1).repeat_interleave(128, dim=2) + reconstructed[name].append(dequant.narrow(dim, 0, valid)) + for name, parts in reconstructed.items(): + expected = weights[name].float() * scales[name].repeat_interleave( + 128, dim=1 + ).repeat_interleave(128, dim=2) + assert torch.equal(torch.cat(parts, dim=2 if name == "w2" else 1), expected) + + +@pytest.mark.parametrize("backend", ["auto", "triton"]) +@pytest.mark.parametrize("tp_size,block", [(2, 64), (4, 32)]) +def test_fp8_tp_default_keeps_refined_layout(monkeypatch, backend, tp_size, block): + layer = _make_fp8_tp_experts(monkeypatch, tp_size, 0, backend) + assert layer.moe_config.intermediate_size_per_partition == 640 // tp_size + assert layer.quant_method.moe_block_shape == [block, block] + + +@pytest.mark.parametrize( + "field,value", + [ + ("hidden_dim", 192), + ("intermediate_size", 672), + ("is_lora_enabled", True), + ("has_bias", True), + ], +) +def test_fp8_block_aligned_tp_rejects_unsupported_layout(monkeypatch, field, value): + with pytest.raises(ValueError, match="Block-aligned FP8 TP sharding requires"): + _make_fp8_tp_experts(monkeypatch, 4, 0, config_overrides={field: value}) + + +@pytest.mark.parametrize("tp_size", [1, 5]) +def test_fp8_aligned_tp_keeps_original_layout(monkeypatch, tp_size): + layer = _make_fp8_tp_experts(monkeypatch, tp_size, 0) + assert layer.moe_config.intermediate_size_per_partition == 640 // tp_size + assert not layer.moe_config.tp_shard_with_padding + assert layer.quant_method.weight_scale_refine is None + + +def test_fp8_skipped_layer_keeps_original_tp_layout(monkeypatch, default_vllm_config): + layer = _make_fp8_tp_experts( + monkeypatch, + 4, + 0, + quant_config=Fp8Config( + is_checkpoint_fp8_serialized=True, + weight_block_size=[128, 128], + ignored_layers=["model.layers.0.mlp.experts"], + ), + ) + assert isinstance(layer.quant_method, unquantized_module.UnquantizedFusedMoEMethod) + assert not layer.moe_config.tp_shard_with_padding + assert layer.moe_config.intermediate_size_per_partition == 160 + assert layer.w13_weight.shape == (2, 320, 256) + assert layer.w2_weight.shape == (2, 256, 160) + + +def test_fp8_block_aligned_tp_rejects_presharded_weights(monkeypatch): + layer = _make_fp8_tp_experts(monkeypatch, 4, 0) + with pytest.raises(ValueError, match="unsharded checkpoint"): + layer.w13_weight.weight_loader( + layer.w13_weight, + torch.zeros(160, 256, dtype=torch.float8_e4m3fn), + "w13_weight", + "w1", + 0, + ) + + +@pytest.mark.parametrize("tp_size", [2, 4, 8]) +@pytest.mark.parametrize("num_tokens", [1, 17]) +@torch.inference_mode() +def test_fp8_block_aligned_tp_flashinfer_matches_unsharded( + monkeypatch, + tp_size, + num_tokens, + dist_init, + workspace_init, +): + """Compare sharded FI with unsharded FI, reporting the cross-backend + difference separately because activation quantization can differ. + """ + from vllm.config import VllmConfig, set_current_vllm_config + from vllm.model_executor.layers.fused_moe.config import fp8_w8a8_moe_quant_config + from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts + from vllm.platforms import current_platform + + if not current_platform.is_device_capability_family(100): + pytest.skip("Requires datacenter Blackwell and FlashInfer TRTLLM") + torch.manual_seed(42) + e, h, n, topk = 16, 2560, 640, 10 + with torch.device("cuda"), set_current_vllm_config(VllmConfig()): + x = torch.randn(num_tokens, h, dtype=torch.bfloat16) / 10 + logits = torch.randn(num_tokens, e, dtype=torch.float32) + probabilities = logits.softmax(dim=-1) + topk_weights, topk_ids = probabilities.topk(topk, dim=-1) + topk_weights /= topk_weights.sum(dim=-1, keepdim=True) + weights = { + "w1": torch.randn(e, n, h).to(torch.float8_e4m3fn), + "w3": torch.randn(e, n, h).to(torch.float8_e4m3fn), + "w2": torch.randn(e, h, n).to(torch.float8_e4m3fn), + } + scales = { + name: (torch.rand(e, w.shape[1] // 128, w.shape[2] // 128) + 0.5) / 32 + for name, w in weights.items() + } + ref = fused_experts( + x, + torch.cat([weights["w1"], weights["w3"]], dim=1), + weights["w2"], + topk_weights, + topk_ids, + quant_config=fp8_w8a8_moe_quant_config( + w1_scale=torch.cat([scales["w1"], scales["w3"]], dim=1), + w2_scale=scales["w2"], + block_shape=[128, 128], + ), + ) + total = torch.zeros_like(ref, dtype=torch.float32) + fi_reference = None + for rank in range(-1, tp_size): + layer = _make_fp8_tp_experts( + monkeypatch, + 1 if rank == -1 else tp_size, + max(rank, 0), + mock_backend=False, + num_experts=e, + hidden_dim=h, + ) + for name in weights: + prefix = "w2" if name == "w2" else "w13" + for suffix, checkpoint in ( + ("weight", weights[name]), + ("weight_scale_inv", scales[name]), + ): + param_name = f"{prefix}_{suffix}" + param = getattr(layer, param_name) + param.weight_loader(param, checkpoint, param_name, name, 0) + method = layer.quant_method + method.process_weights_after_loading(layer) + assert method.is_monolithic + out = method.apply_monolithic(layer, x, logits) + assert torch.isfinite(out).all() + if rank == -1: + fi_reference = out.float().clone() + else: + total += out.float() + # Different FP8 GEMM reduction orders need not be bitwise identical. + assert fi_reference is not None + relative_l2 = (total - fi_reference).norm() / fi_reference.norm().clamp_min( + 1e-8 + ) + cross_backend_l2 = ( + fi_reference - ref.float() + ).norm() / ref.float().norm().clamp_min(1e-8) + print( + f"TP{tp_size} M={num_tokens}: shard_l2={relative_l2.item():.6f}, " + f"unsharded_fi_vs_triton_l2={cross_backend_l2.item():.6f}" + ) + assert relative_l2 < 0.01, relative_l2.item() diff --git a/tests/kernels/moe/test_gpt_oss_triton_kernels.py b/tests/kernels/moe/test_gpt_oss_triton_kernels.py index 86028a7feb..c1d83d8349 100644 --- a/tests/kernels/moe/test_gpt_oss_triton_kernels.py +++ b/tests/kernels/moe/test_gpt_oss_triton_kernels.py @@ -7,7 +7,7 @@ import torch import torch.nn.functional as F from vllm.platforms import current_platform -from vllm.utils.import_utils import has_triton_kernels +from vllm.utils.import_utils import get_triton_kernels_version, has_triton_kernels if not has_triton_kernels(): pytest.skip( @@ -15,9 +15,15 @@ if not has_triton_kernels(): allow_module_level=True, ) -import triton_kernels.matmul_ogs_details.opt_flags as opt_flags import triton_kernels.swiglu -from triton_kernels.matmul_ogs import FlexCtx, PrecisionConfig + +if get_triton_kernels_version() == "3.8": + # 3.8: matmul_ogs -> matmul, matmul_ogs_details -> matmul_details + import triton_kernels.matmul_details.opt_flags as opt_flags + from triton_kernels.matmul import FlexCtx, PrecisionConfig +else: + import triton_kernels.matmul_ogs_details.opt_flags as opt_flags + from triton_kernels.matmul_ogs import FlexCtx, PrecisionConfig from triton_kernels.numerics import InFlexData from triton_kernels.numerics_details.mxfp import downcast_to_mxfp, upcast_from_mxfp from triton_kernels.tensor import FP4, convert_layout, wrap_torch_tensor @@ -28,9 +34,10 @@ from vllm.model_executor.layers.fused_moe.config import mxfp4_w4a16_moe_quant_co from vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe import ( triton_kernel_moe_forward, ) +from vllm.model_executor.layers.quantization.utils.mxfp4_utils import mx_scale_kwargs from vllm.utils.math_utils import round_up -from .utils import shuffle_weight +from .utils import mxfp4_w_layouts, shuffle_weight def deshuffle(w: torch.Tensor): @@ -133,12 +140,12 @@ def init_compute_data(M, K, N, E, a_dtype: str, w_dtype: str, num_warps: int): x_tri = F.pad(x_tri, (0, x_pad, 0, 0), mode="constant", value=0) - w_layout, w_layout_opts = layout.make_default_matmul_mxfp4_w_layout(mx_axis=1) - w_scale_layout, w_scale_layout_opts = ( - layout.make_default_matmul_mxfp4_w_scale_layout( - mx_axis=1, num_warps=num_warps - ) - ) + ( + w_layout, + w_layout_opts, + w_scale_layout, + w_scale_layout_opts, + ) = mxfp4_w_layouts(mx_axis=1, num_warps=num_warps) w1_tri, w1_scale_tri = downcast_to_mxfp(w1_tri, torch.uint8, axis=1) w1 = upcast_from_mxfp(w1_tri, w1_scale_tri, torch.bfloat16, axis=1) @@ -165,10 +172,10 @@ def init_compute_data(M, K, N, E, a_dtype: str, w_dtype: str, num_warps: int): ) pc1 = PrecisionConfig( - weight_scale=w1_scale_tri, flex_ctx=FlexCtx(rhs_data=InFlexData()) + **mx_scale_kwargs(w1_scale_tri), flex_ctx=FlexCtx(rhs_data=InFlexData()) ) pc2 = PrecisionConfig( - weight_scale=w2_scale_tri, flex_ctx=FlexCtx(rhs_data=InFlexData()) + **mx_scale_kwargs(w2_scale_tri), flex_ctx=FlexCtx(rhs_data=InFlexData()) ) # tucuate so the rest can run properly @@ -281,8 +288,6 @@ class Case: @pytest.mark.parametrize("num_token", [2]) @pytest.mark.parametrize("tp", [1, 2, 4, 8]) def test_equiv(num_token, a_dtype, w_dtype, tp, workspace_init): - from triton_kernels.tensor_details import layout - if not hasattr(layout, "make_default_matmul_mxfp4_w_layout"): pytest.skip("make_default_matmul_mxfp4_w_layout not available") @@ -387,6 +392,11 @@ def test_routing_data_from_sparse_topk_parity(n_tokens, n_experts, topk): if gptoss_moe.triton_kernels_version == "3.5.1": pytest.skip("SparseMatrix path requires triton_kernels v3.6.0+") + if gptoss_moe.triton_kernels_version == "3.8": + pytest.skip( + "3.8 rebuilds routing from raw ids, so routing_data_from_sparse_topk " + "just forwards to make_routing_data and the comparison is vacuous" + ) from triton_kernels.topk import topk as topk_fn diff --git a/tests/kernels/moe/test_grouped_topk.py b/tests/kernels/moe/test_grouped_topk.py index c65cd381db..2004feae65 100644 --- a/tests/kernels/moe/test_grouped_topk.py +++ b/tests/kernels/moe/test_grouped_topk.py @@ -15,8 +15,10 @@ from vllm.config import ( get_cached_compilation_config, set_current_vllm_config, ) +from vllm.forward_context import set_forward_context from vllm.model_executor.layers.fused_moe.router.grouped_topk_router import ( GroupedTopk, + GroupedTopKRouter, fused_grouped_topk, ) from vllm.platforms import current_platform @@ -70,6 +72,36 @@ def _single_group_reference( return values, indices.to(torch.int32) +@pytest.mark.skipif( + not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform." +) +@pytest.mark.parametrize("skip_padding", [False, True]) +def test_kimi_grouped_topk_masks_padding_when_enabled(skip_padding: bool): + hidden_states = torch.randn((2, 16), device="cuda") + gating_output = torch.randn((2, 896), device="cuda") + is_padding = torch.tensor([False, True], device="cuda") + router = GroupedTopKRouter( + top_k=16, + global_num_experts=896, + num_expert_group=1, + topk_group=1, + skip_padding=skip_padding, + ) + + with set_forward_context(None, VllmConfig(), is_padding=is_padding): + topk_weights, topk_ids = router._compute_routing( + hidden_states, gating_output, torch.int32 + ) + + assert torch.all(topk_ids[0] >= 0) + if skip_padding: + assert torch.all(topk_ids[1] == -1) + assert torch.all(topk_weights[1] == 0) + else: + assert torch.all(topk_ids[1] >= 0) + assert torch.all(topk_weights[1] > 0) + + @pytest.mark.skipif( not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform." ) diff --git a/tests/kernels/moe/test_modular_oai_triton_moe.py b/tests/kernels/moe/test_modular_oai_triton_moe.py index 47be82dd2a..9665079e37 100644 --- a/tests/kernels/moe/test_modular_oai_triton_moe.py +++ b/tests/kernels/moe/test_modular_oai_triton_moe.py @@ -10,7 +10,7 @@ import torch.nn.functional as F from tests.utils import wait_for_gpu_memory_to_clear from vllm.model_executor.layers.fused_moe.activation import MoEActivation -from vllm.utils.import_utils import has_triton_kernels +from vllm.utils.import_utils import get_triton_kernels_version, has_triton_kernels if not has_triton_kernels(): pytest.skip( @@ -18,11 +18,14 @@ if not has_triton_kernels(): allow_module_level=True, ) -from triton_kernels.matmul_ogs import FlexCtx, PrecisionConfig +if get_triton_kernels_version() == "3.8": + # 3.8: matmul_ogs -> matmul + from triton_kernels.matmul import FlexCtx, PrecisionConfig +else: + from triton_kernels.matmul_ogs import FlexCtx, PrecisionConfig from triton_kernels.numerics import InFlexData from triton_kernels.numerics_details.mxfp import downcast_to_mxfp, upcast_from_mxfp from triton_kernels.tensor import FP4, convert_layout, wrap_torch_tensor -from triton_kernels.tensor_details import layout from triton_kernels.testing import assert_close from vllm.config import VllmConfig, set_current_vllm_config @@ -35,11 +38,12 @@ from vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe imp UnfusedOAITritonExperts, ) from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel +from vllm.model_executor.layers.quantization.utils.mxfp4_utils import mx_scale_kwargs from vllm.platforms import current_platform from vllm.utils.math_utils import round_up from vllm.utils.torch_utils import set_random_seed -from .utils import make_dummy_moe_config, shuffle_weight +from .utils import make_dummy_moe_config, mxfp4_w_layouts, shuffle_weight MNK = [ (1, 512, 384), @@ -130,10 +134,12 @@ def make_weights(dtype, k, n, e): w2 = w2[..., :n, :k] num_warps = 8 - w_layout, w_layout_opts = layout.make_default_matmul_mxfp4_w_layout(mx_axis=1) - w_scale_layout, w_scale_layout_opts = ( - layout.make_default_matmul_mxfp4_w_scale_layout(mx_axis=1, num_warps=num_warps) - ) + ( + w_layout, + w_layout_opts, + w_scale_layout, + w_scale_layout_opts, + ) = mxfp4_w_layouts(mx_axis=1, num_warps=num_warps) w1_tri = convert_layout(wrap_torch_tensor(w1_tri, FP4), w_layout, **w_layout_opts) w1_scale_tri = convert_layout( @@ -150,10 +156,10 @@ def make_weights(dtype, k, n, e): ) w1_precision_config = PrecisionConfig( - weight_scale=w1_scale_tri, flex_ctx=FlexCtx(rhs_data=InFlexData()) + **mx_scale_kwargs(w1_scale_tri), flex_ctx=FlexCtx(rhs_data=InFlexData()) ) w2_precision_config = PrecisionConfig( - weight_scale=w2_scale_tri, flex_ctx=FlexCtx(rhs_data=InFlexData()) + **mx_scale_kwargs(w2_scale_tri), flex_ctx=FlexCtx(rhs_data=InFlexData()) ) return ( diff --git a/tests/kernels/moe/test_moe_weight_loading_padded.py b/tests/kernels/moe/test_moe_weight_loading_padded.py index d3716a196b..ef31de06d1 100644 --- a/tests/kernels/moe/test_moe_weight_loading_padded.py +++ b/tests/kernels/moe/test_moe_weight_loading_padded.py @@ -12,7 +12,13 @@ correctly handles this mismatch. import pytest import torch +from vllm.model_executor.layers.fused_moe.oracle.unquantized import ( + UnquantizedMoeBackend, +) from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts +from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import ( + UnquantizedFusedMoEMethod, +) from .utils import make_dummy_moe_config @@ -249,6 +255,57 @@ class TestNarrowExpertDataForPadding: class TestWeightLoadingWithPaddedHiddenSize: """Integration-style tests that simulate padded weight loading.""" + @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") + @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float32]) + @pytest.mark.parametrize("tp_rank", [0, 1]) + def test_load_w2_chunks_noncontiguous_cpu_source_to_padded_cuda( + self, dtype, tp_rank + ): + hidden = 4096 + intermediate = 1024 + loaded_weight = ( + torch.arange(hidden * intermediate * 2) + .remainder(251) + .to(dtype) + .reshape(hidden, intermediate * 2) + ) + tp_source = loaded_weight[ + :, intermediate * tp_rank : intermediate * (tp_rank + 1) + ] + expert_data_full = torch.zeros( + hidden + 8, intermediate + 8, device="cuda", dtype=dtype + ) + destination = expert_data_full[:hidden, :intermediate] + + assert not tp_source.is_contiguous() + assert tp_source.nbytes > 1 << 20 + assert not destination.is_contiguous() + + experts = object.__new__(RoutedExperts) + torch.nn.Module.__init__(experts) + experts.moe_config = make_dummy_moe_config() + experts.moe_config.moe_parallel_config.tp_size = 2 + + torch.accelerator.synchronize() + allocated_before = torch.accelerator.memory_allocated() + torch.accelerator.reset_peak_memory_stats() + experts._load_w2( + expert_data=expert_data_full, + shard_dim=1, + loaded_weight=loaded_weight, + tp_rank=tp_rank, + ) + torch.accelerator.synchronize() + peak_extra = torch.accelerator.max_memory_allocated() - allocated_before + + # A single copy into the strided CUDA view allocates a full-shard + # temporary. Correct values alone would not catch that regression. + assert peak_extra < tp_source.nbytes + + torch.testing.assert_close(destination.cpu(), tp_source) + assert torch.count_nonzero(expert_data_full[hidden:, :]) == 0 + assert torch.count_nonzero(expert_data_full[:hidden, intermediate:]) == 0 + def test_load_w2_with_padding(self): """Simulate loading w2 weights when hidden_size is padded.""" padded_hidden = 3072 @@ -364,6 +421,50 @@ class TestWeightLoadingWithPaddedHiddenSize: ) +class TestUnquantizedTrtLlmPrePadding: + @staticmethod + def _make_method( + intermediate: int, + backend: UnquantizedMoeBackend = UnquantizedMoeBackend.FLASHINFER_TRTLLM, + ) -> UnquantizedFusedMoEMethod: + moe_config = make_dummy_moe_config( + num_experts=2, + hidden_dim=64, + intermediate_size=intermediate, + ) + method = object.__new__(UnquantizedFusedMoEMethod) + method.moe = moe_config + method.unquantized_backend = backend + method.moe_kernel = None + return method + + @pytest.mark.parametrize( + "backend,original,expected", + [ + (UnquantizedMoeBackend.FLASHINFER_TRTLLM, 1344, 1408), + (UnquantizedMoeBackend.FLASHINFER_TRTLLM, 1408, 1408), + (UnquantizedMoeBackend.TRITON, 1344, 1344), + ], + ) + def test_rounds_intermediate_before_weight_allocation( + self, + backend: UnquantizedMoeBackend, + original: int, + expected: int, + ): + method = self._make_method(original, backend) + + hidden, intermediate = method.maybe_roundup_sizes( + hidden_size=64, + intermediate_size_per_partition=original, + act_dtype=torch.bfloat16, + moe_parallel_config=method.moe.moe_parallel_config, + ) + + assert hidden == 64 + assert intermediate == expected + + class TestLoadWeightsExpertBias: """Some quantized exports (e.g. GPTQ, llm-compressor NVFP4) materialize all-zero per-expert `.bias` tensors for models whose experts have no bias diff --git a/tests/kernels/moe/test_moonep_bf16_poc.py b/tests/kernels/moe/test_moonep_bf16_poc.py new file mode 100644 index 0000000000..ba1e78fb7b --- /dev/null +++ b/tests/kernels/moe/test_moonep_bf16_poc.py @@ -0,0 +1,412 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Test MoonEP dispatch / prefetch / combine logic (BF16 PoC). + +Two paths, both compared against the pure-PyTorch reference MoE: +- MoonEPPrepareAndFinalize + a reference segment-loop expert runner over + MoonEP's expert-grouped ``[NvS, H]`` layout; +- the full modular kernel, MoonEPPrepareAndFinalize + MoonEPExperts through + FusedMoEKernel. +Requires NVSwitch multicast capable GPUs. +""" + +import dataclasses +import types + +import pytest +import torch +import torch.nn.functional as F +from torch.distributed import ProcessGroup + +from tests.kernels.moe.utils import make_test_weights +from tests.kernels.utils import torch_experts +from vllm.config import VllmConfig, set_current_vllm_config +from vllm.model_executor.layers.fused_moe.activation import MoEActivation +from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import ( + TopKWeightAndReduceNoOP, +) +from vllm.utils.import_utils import has_moonep +from vllm.utils.torch_utils import set_random_seed + +from ...utils import multi_gpu_test +from .parallel_utils import ProcessGroupInfo, parallel_launch + +if has_moonep(): + from vllm.model_executor.layers.fused_moe.prepare_finalize.moonep import ( + MoonEPExpertWeightLayout, + MoonEPPrepareAndFinalize, + make_moonep_weight_layout, + ) + +requires_moonep = pytest.mark.skipif( + not has_moonep(), + reason="Requires MoonEP", +) + + +class MulticastNotAvailableError(RuntimeError): + pass + + +@dataclasses.dataclass +class TestConfig: + topk: int + m: int + k: int + n: int + num_experts: int + router_skew: float + apply_router_weight_on_input: bool = False + + +@dataclasses.dataclass +class TestTensors: + rank_tokens: torch.Tensor + topk: torch.Tensor + topk_weights: torch.Tensor + config: TestConfig + + @staticmethod + def make(config: TestConfig) -> "TestTensors": + rank_tokens = ( + torch.randn((config.m, config.k), device="cuda", dtype=torch.bfloat16) / 10 + ) + # Skewed routing via a per-expert bias: scaling logits by a scalar + # preserves their ordering and would not change the selected topk + # ids, but a shared bias makes a few experts hot on every token so + # the planner has to fill redundant-expert prefetch slots. + expert_bias = config.router_skew * torch.randn( + 1, config.num_experts, device="cuda", dtype=torch.float32 + ) + logits = ( + torch.randn( + config.m, config.num_experts, device="cuda", dtype=torch.float32 + ) + + expert_bias + ) + topk_weights, topk = torch.topk(logits, config.topk, dim=-1) + topk_weights = torch.softmax(topk_weights, dim=-1) + return TestTensors( + rank_tokens=rank_tokens, + topk=topk.to(dtype=torch.int64), + topk_weights=topk_weights, + config=config, + ) + + +def reference_moonep_experts( + hidden_nvsh: torch.Tensor, + route_weights_nvs: torch.Tensor, + cu_seqlens: torch.Tensor, + weight_layout: "MoonEPExpertWeightLayout", +) -> torch.Tensor: + """Segment loop over MoonEP's expert-grouped layout. + + ``prefetch_weight`` has already materialized redundant experts' weights + in rows ``[E, E+B)``, so every segment reads its own row. Route weights + are applied here; MoonEP's combine does the K-sum. + """ + output = torch.empty_like(hidden_nvsh) + prev = 0 + for row, cur in enumerate(cu_seqlens.tolist()): + if cur == prev: + continue + x = hidden_nvsh[prev:cur] + gate = F.linear(x, weight_layout.full_gate_weight[row]) + up = F.linear(x, weight_layout.full_up_weight[row]) + y = F.linear(F.silu(gate) * up, weight_layout.full_down_weight[row]) + y = y * route_weights_nvs[prev:cur].to(dtype=y.dtype).unsqueeze(-1) + output[prev:cur].copy_(y) + prev = cur + if prev < hidden_nvsh.shape[0]: + output[prev:].zero_() + return output + + +def make_moonep_prepare_finalize( + pg: ProcessGroup, + pgi: ProcessGroupInfo, + hidden_size: int, + num_experts: int, + topk: int, + max_tokens_per_rank: int, + weight_layout: "MoonEPExpertWeightLayout", + pass_layout_to_pf: bool = True, +): + from moonep._C import nvl_multicast_supported + + from vllm.model_executor.layers.fused_moe.prepare_finalize.moonep import ( + MoonEPBufferPool, + ) + + if not nvl_multicast_supported(): + raise MulticastNotAvailableError("NVSwitch multicast not available") + + pool = MoonEPBufferPool( + dict( + H=hidden_size, + K=topk, + E=num_experts, + num_ep_ranks=pgi.world_size, + B=weight_layout.num_prefetch_slots, + group=pg, + explicitly_destroy=True, + ), + max_tokens_per_rank=max_tokens_per_rank, + ) + return pool, MoonEPPrepareAndFinalize( + buffer_pool=pool, + max_tokens_per_rank=max_tokens_per_rank, + num_dispatchers=pgi.world_size, + num_global_experts=num_experts, + weight_layout=weight_layout if pass_layout_to_pf else None, + ) + + +def moonep_moe_impl( + pg: ProcessGroup, + pgi: ProcessGroupInfo, + test_tensors: TestTensors, + w1: torch.Tensor, + w2: torch.Tensor, + num_prefetch_slots: int, +) -> torch.Tensor: + config = test_tensors.config + hidden_size = test_tensors.rank_tokens.shape[1] + max_tokens_per_rank = 128 * ((config.m + 127) // 128) + + weight_layout = make_moonep_weight_layout(w1, w2, num_prefetch_slots) + pool, pf = make_moonep_prepare_finalize( + pg, + pgi, + hidden_size, + config.num_experts, + config.topk, + max_tokens_per_rank, + weight_layout, + ) + try: + hidden_nvsh, _, _, _, route_weights_nvs = pf.prepare( + test_tensors.rank_tokens, + test_tensors.topk_weights, + test_tensors.topk, + num_experts=config.num_experts, + expert_map=None, + apply_router_weight_on_input=config.apply_router_weight_on_input, + quant_config=_no_quant_config(), + ) + assert route_weights_nvs is not None + if config.apply_router_weight_on_input: + # prepare() already applied the weights to the inputs; the + # runner must not apply them again. + route_weights_nvs = torch.ones_like(route_weights_nvs) + if config.router_skew >= 8 and config.m >= 100: + # Heavy skew must actually engage the redundant-expert planner. + assert (pf.plan.experts_to_copy >= 0).any(), ( + "no prefetch slot used despite heavy router skew" + ) + expert_out = reference_moonep_experts( + hidden_nvsh, route_weights_nvs, pf.cu_seqlens, weight_layout + ) + output = torch.empty_like(test_tensors.rank_tokens) + pf.finalize( + output, + expert_out, + test_tensors.topk_weights, + test_tensors.topk, + apply_router_weight_on_input=False, + weight_and_reduce_impl=TopKWeightAndReduceNoOP(), + ) + torch.accelerator.synchronize() + return output + finally: + pool.destroy() + + +def _no_quant_config(): + from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig + + return FusedMoEQuantConfig.make(quant_dtype=None) + + +def moonep_modular_kernel_impl( + pg: ProcessGroup, + pgi: ProcessGroupInfo, + test_tensors: TestTensors, + w1: torch.Tensor, + w2: torch.Tensor, + num_prefetch_slots: int, +) -> torch.Tensor: + """Full modular-kernel path: MoonEPPrepareAndFinalize + MoonEPExperts.""" + from tests.kernels.moe.utils import make_dummy_moe_config + from vllm.model_executor.layers.fused_moe.experts.moonep_experts import ( + MoonEPExperts, + ) + from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel + + config = test_tensors.config + hidden_size = test_tensors.rank_tokens.shape[1] + max_tokens_per_rank = 128 * ((config.m + 127) // 128) + + weight_layout = make_moonep_weight_layout(w1, w2, num_prefetch_slots) + # weight_layout deliberately not passed to the P/F: the engine path + # resolves it through post_init_setup + the experts' weight hook. + pool, pf = make_moonep_prepare_finalize( + pg, + pgi, + hidden_size, + config.num_experts, + config.topk, + max_tokens_per_rank, + weight_layout, + pass_layout_to_pf=False, + ) + try: + moe_config = make_dummy_moe_config( + num_experts=config.num_experts, + experts_per_token=config.topk, + hidden_dim=hidden_size, + intermediate_size=config.n, + max_num_tokens=max_tokens_per_rank, + ) + experts = MoonEPExperts(moe_config=moe_config, quant_config=_no_quant_config()) + # Stand-in for the layer carrying the layout, as + # convert_to_unquantized_kernel_format leaves it in the engine path. + fake_layer = types.SimpleNamespace(_moonep_weight_layout=weight_layout) + experts.process_weights_after_loading(fake_layer) + kernel = FusedMoEKernel(prepare_finalize=pf, fused_experts=experts) + out = kernel.apply( + hidden_states=test_tensors.rank_tokens, + w1=weight_layout.full_gate_weight, + w2=weight_layout.full_down_weight, + topk_weights=test_tensors.topk_weights, + topk_ids=test_tensors.topk, + activation=MoEActivation.SILU, + global_num_experts=config.num_experts, + expert_map=None, + apply_router_weight_on_input=config.apply_router_weight_on_input, + ) + torch.accelerator.synchronize() + return out + finally: + pool.destroy() + + +def _moonep_moe( + pgi: ProcessGroupInfo, + config: TestConfig, + w1: torch.Tensor, + w2: torch.Tensor, + num_prefetch_slots: int, + use_modular_kernel: bool, +): + from vllm.v1.worker.workspace import init_workspace_manager + + device_idx = torch.accelerator.current_device_index() + init_workspace_manager(torch.device("cuda", device_idx)) + w1 = w1.to(device=device_idx) + w2 = w2.to(device=device_idx) + + pg = torch.distributed.new_group(list(range(pgi.world_size))) + set_random_seed(7 + pgi.rank) + test_tensors = TestTensors.make(config) + + with set_current_vllm_config(VllmConfig()): + torch_combined = torch_experts( + test_tensors.rank_tokens, + w1, + w2, + test_tensors.topk_weights, + test_tensors.topk, + apply_router_weights_on_input=config.apply_router_weight_on_input, + ) + impl = moonep_modular_kernel_impl if use_modular_kernel else moonep_moe_impl + moonep_combined = impl(pg, pgi, test_tensors, w1, w2, num_prefetch_slots) + + torch.testing.assert_close( + torch_combined, + moonep_combined, + atol=6e-2, + rtol=6e-2, + ) + + +MNKs = [ + (1, 256, 512), + (37, 256, 512), + (100, 512, 1024), + (512, 768, 2048), +] + + +@pytest.mark.parametrize("m,n,k", MNKs) +@pytest.mark.parametrize("num_experts", [32]) +@pytest.mark.parametrize("topk", [4]) +@pytest.mark.parametrize("router_skew", [1.0, 8.0]) +@pytest.mark.parametrize("num_prefetch_slots", [4]) +@pytest.mark.parametrize("world_size", [2]) +@pytest.mark.parametrize("use_modular_kernel", [False, True]) +@multi_gpu_test(num_gpus=2) +@requires_moonep +def test_moonep_bf16_moe( + m: int, + n: int, + k: int, + num_experts: int, + topk: int, + router_skew: float, + num_prefetch_slots: int, + world_size: int, + use_modular_kernel: bool, +): + set_random_seed(7) + config = TestConfig( + topk=topk, + m=m, + k=k, + n=n, + num_experts=num_experts, + router_skew=router_skew, + ) + (_, w1, _, _), (_, w2, _, _) = make_test_weights(num_experts, n, k) + + try: + parallel_launch( + world_size, + _moonep_moe, + config, + w1, + w2, + num_prefetch_slots, + use_modular_kernel, + ) + except Exception as exc: + if "MulticastNotAvailableError" in str(exc): + pytest.skip("NVSwitch multicast not available") + raise + + +@pytest.mark.parametrize("use_modular_kernel", [False, True]) +@multi_gpu_test(num_gpus=2) +@requires_moonep +def test_moonep_input_weighted_moe(use_modular_kernel: bool): + """Llama-4-style routing applies route weights to the expert inputs.""" + set_random_seed(7) + config = TestConfig( + topk=1, + m=100, + k=1024, + n=512, + num_experts=32, + router_skew=1.0, + apply_router_weight_on_input=True, + ) + (_, w1, _, _), (_, w2, _, _) = make_test_weights( + config.num_experts, config.n, config.k + ) + try: + parallel_launch(2, _moonep_moe, config, w1, w2, 4, use_modular_kernel) + except Exception as exc: + if "MulticastNotAvailableError" in str(exc): + pytest.skip("NVSwitch multicast not available") + raise diff --git a/tests/kernels/moe/test_mxfp8_aiter_backend_selection.py b/tests/kernels/moe/test_mxfp8_aiter_backend_selection.py index 51aff4ea3b..61b26af62c 100644 --- a/tests/kernels/moe/test_mxfp8_aiter_backend_selection.py +++ b/tests/kernels/moe/test_mxfp8_aiter_backend_selection.py @@ -2,10 +2,11 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """MXFP8 MoE backend selection for the AITER FlyDSL kernel (gfx950). -GPU-free: mocks the platform (gfx950) and the ``flydsl`` package check, then +GPU-free: mocks the platform (gfx950) and the aiter fused-MoE enable flag, then exercises the oracle so the FlyDSL backend is auto-picked when usable (including under expert parallelism, since apply() forwards the expert_map as aiter's -expert_mask) and skipped (native fallback) when the device/package is missing. +expert_mask) and skipped (native fallback) when the device is unsupported or the +aiter runtime is disabled. """ import dataclasses @@ -48,6 +49,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( # noqa: kMxfp8Static, ) +from vllm._aiter_ops import rocm_aiter_ops # noqa: E402 # isort: skip + _AITER_MOD = "vllm.model_executor.layers.fused_moe.experts.aiter_mxfp8_moe" @@ -82,8 +85,8 @@ def _gfx950(): ) -def _flydsl_installed(present: bool): - return patch(f"{_AITER_MOD}.is_aiter_mxfp8_moe_available", return_value=present) +def _aiter_moe_enabled(present: bool): + return patch.object(rocm_aiter_ops, "is_fused_moe_enabled", return_value=present) def test_aiter_mxfp8_registered(): @@ -109,13 +112,18 @@ def test_ep_supported(ep_size): @pytest.mark.parametrize( "present,ep_size,supported,reason_substr", [ - (True, 1, True, None), # gfx950 + flydsl + TP -> selectable - (True, 2, True, None), # gfx950 + flydsl + EP -> selectable (expert_mask) - (False, 1, False, "flydsl package"), # package missing -> clear reason + (True, 1, True, None), # gfx950 + aiter MoE + TP -> selectable + (True, 2, True, None), # gfx950 + aiter MoE + EP -> selectable (expert_mask) + ( + False, + 1, + False, + "does not support current device", + ), # disabled -> not selected ], ) def test_is_supported_config(present, ep_size, supported, reason_substr): - with _gfx950(), _flydsl_installed(present): + with _gfx950(), _aiter_moe_enabled(present): ok, reason = AiterMxfp8Experts.is_supported_config( AiterMxfp8Experts, _config(ep_size), @@ -130,8 +138,8 @@ def test_is_supported_config(present, ep_size, supported, reason_substr): def test_explicit_moe_backend_aiter(): """--moe-backend aiter: returns FlyDSL when usable (TP or EP), else a clear - ValueError when the flydsl package is missing.""" - with _gfx950(), _flydsl_installed(True): + ValueError when the aiter MoE runtime is disabled.""" + with _gfx950(), _aiter_moe_enabled(True): assert ( _select_kernel_cls(Fp8MoeBackend.AITER_MXFP8, _config(1)) is AiterMxfp8Experts @@ -142,18 +150,17 @@ def test_explicit_moe_backend_aiter(): ) with ( _gfx950(), - _flydsl_installed(False), - pytest.raises(ValueError, match="flydsl package"), + _aiter_moe_enabled(False), + pytest.raises(ValueError, match="does not support current device"), ): _select_kernel_cls(Fp8MoeBackend.AITER_MXFP8, _config(1)) def test_gfx950_picks_aiter(): - """Auto-select on real ROCm hardware with flydsl usable -> FlyDSL wins.""" - # NOTE: Fp8MoeBackend.AITER_MXFP8 does not require VLLM_ROCM_USE_AITER=1 + """Auto-select on real ROCm hardware with aiter MoE enabled -> FlyDSL wins.""" with ( patch(f"{_AITER_MOD}.current_platform.supports_mx", return_value=True), - _flydsl_installed(True), + _aiter_moe_enabled(True), ): backend, experts_cls = select_mxfp8_moe_backend(_config()) assert backend is Fp8MoeBackend.AITER_MXFP8 diff --git a/tests/kernels/moe/test_routing.py b/tests/kernels/moe/test_routing.py index 174950e06c..4baad24cc3 100644 --- a/tests/kernels/moe/test_routing.py +++ b/tests/kernels/moe/test_routing.py @@ -87,6 +87,21 @@ def test_multiple_expert_groups_use_grouped_topk() -> None: ) assert isinstance(router, GroupedTopKRouter) + assert not router.skip_padding + + +def test_grouped_topk_padding_skip_must_be_enabled() -> None: + router = create_fused_moe_router( + top_k=4, + global_num_experts=128, + use_grouped_topk=True, + num_expert_group=8, + topk_group=4, + skip_padding=True, + ) + + assert isinstance(router, GroupedTopKRouter) + assert router.skip_padding def test_degenerate_grouped_config_with_bias_uses_topk_bias() -> None: diff --git a/tests/kernels/moe/test_trtllm_bf16_moe.py b/tests/kernels/moe/test_trtllm_bf16_moe.py index 9e50b4fc3b..5e569e153a 100644 --- a/tests/kernels/moe/test_trtllm_bf16_moe.py +++ b/tests/kernels/moe/test_trtllm_bf16_moe.py @@ -12,6 +12,7 @@ import pytest import torch import vllm.model_executor.layers.fused_moe.modular_kernel as mk +from tests.kernels.moe.utils import make_dummy_moe_config from tests.kernels.utils import torch_moe from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config from vllm.model_executor.layers.fused_moe.activation import MoEActivation @@ -27,6 +28,9 @@ from vllm.model_executor.layers.fused_moe.config import ( from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import ( TrtLlmBf16ExpertsModular, ) +from vllm.model_executor.layers.fused_moe.experts.trtllm_lora_moe import ( + TrtLlmBf16LoRAExperts, +) from vllm.model_executor.layers.fused_moe.oracle.unquantized import ( UnquantizedMoeBackend, convert_to_unquantized_kernel_format, @@ -75,6 +79,10 @@ def test_trtllm_bf16_moe_modular_no_graph( a = torch.randn((m, k), device="cuda", dtype=dtype) / 10 w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10 w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10 + # The FlashInfer conversion may rewrite unpadded input storage in place. + # Preserve the original layout for the independent torch reference. + reference_w1 = w1.clone() + reference_w2 = w2.clone() score = torch.randn((m, e), device="cuda", dtype=dtype) scores = torch.softmax(score, dim=-1, dtype=torch.float32) topk_weights, topk_ids = torch.topk(scores, topk) @@ -133,8 +141,8 @@ def test_trtllm_bf16_moe_modular_no_graph( torch_output = torch_moe( a, - w1, - w2, + reference_w1, + reference_w2, score, topk, activation=MoEActivation.SILU, @@ -146,3 +154,48 @@ def test_trtllm_bf16_moe_modular_no_graph( atol=1e-1, rtol=2e-1, ) + + +@pytest.mark.parametrize("has_lora_delta", [False, True]) +@torch.inference_mode() +def test_trtllm_bf16_lora_accepts_checkpoint_shaped_weights(has_lora_delta): + """LoRA dispatch accepts both 3D parameters and legacy 4D packed views.""" + config = make_dummy_moe_config( + num_experts=128, hidden_dim=256, intermediate_size=128 + ) + experts = TrtLlmBf16LoRAExperts(config, FUSED_MOE_UNQUANTIZED_CONFIG) + x = torch.randn(2, 256, device="cuda", dtype=torch.bfloat16) / 10 + w1, w2 = convert_to_unquantized_kernel_format( + UnquantizedMoeBackend.FLASHINFER_TRTLLM, + config, + torch.randn(128, 256, 256, device="cuda", dtype=torch.bfloat16) / 10, + torch.randn(128, 256, 128, device="cuda", dtype=torch.bfloat16) / 10, + ) + assert w1.ndim == w2.ndim == 3 + topk = ( + torch.tensor([[0], [1]], device="cuda", dtype=torch.int32), + torch.ones(2, 1, device="cuda", dtype=torch.float32), + ) + delta = torch.randn(2, 1, 256, device="cuda", dtype=torch.bfloat16) + + def invoke(w1, w2): + result = experts.invoke_routed_moe( + hidden_states=x, + w1=w1, + w2=w2, + topk_ids_and_weights=topk, + gemm1_lora_delta=delta if has_lora_delta else None, + global_num_experts=128, + a1q_scale=None, + output=torch.empty_like(x), + ) + if has_lora_delta: + # Only compare rows belonging to real tokens, excluding padding. + indices = result[2].flatten().long() + return result[0][indices], result[3][indices] + return result + + actual = invoke(w1, w2) + expected = invoke(w1.view(128, 4, 256, 64), w2.view(128, 2, 256, 64)) + for actual_tensor, expected_tensor in zip(actual, expected): + torch.testing.assert_close(actual_tensor, expected_tensor) diff --git a/tests/kernels/moe/test_unquantized_backend_selection.py b/tests/kernels/moe/test_unquantized_backend_selection.py index d4c037d978..9d0f4d6900 100644 --- a/tests/kernels/moe/test_unquantized_backend_selection.py +++ b/tests/kernels/moe/test_unquantized_backend_selection.py @@ -6,6 +6,7 @@ import pytest import torch from tests.kernels.moe.utils import make_dummy_moe_config +from vllm.model_executor.layers.fused_moe.activation import MoEActivation from vllm.model_executor.layers.fused_moe.config import RoutingMethodType from vllm.model_executor.layers.fused_moe.oracle.unquantized import ( UnquantizedMoeBackend, @@ -228,6 +229,65 @@ def test_select_rocm_aiter_backend(mock_aiter_enabled, mock_has_flashinfer): assert expert_cls is not None +@patch( + "vllm.utils.flashinfer.has_flashinfer", + return_value=False, +) +@patch( + "vllm.model_executor.layers.fused_moe.oracle.unquantized.rocm_aiter_ops." + "is_fused_moe_enabled", + return_value=True, +) +@patch( + "vllm.model_executor.layers.fused_moe.oracle.unquantized.rocm_aiter_ops." + "is_rdna_aiter_enabled", + return_value=False, +) +@pytest.mark.skipif( + not current_platform.is_rocm(), reason="ROCm-specific backend selection test" +) +def test_select_rocm_aiter_backend_non_gated_activation_falls_back( + mock_rdna_disabled, mock_aiter_enabled, mock_has_flashinfer, monkeypatch +): + """Test ROCm backend selection falls back (not raises) for non-gated MoE.""" + monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1") + monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "1") + + with patch( + "vllm.model_executor.layers.fused_moe.oracle.unquantized.current_platform" + ) as mock_platform: + mock_platform.is_cuda.return_value = False + mock_platform.is_rocm.return_value = True + mock_platform.is_cpu.return_value = False + mock_platform.is_xpu.return_value = False + mock_platform.is_tpu.return_value = False + mock_platform.is_out_of_tree.return_value = False + + moe_config = make_dummy_moe_config(activation=MoEActivation.SILU_NO_MUL) + assert moe_config.is_act_and_mul is False + + selected_backend, expert_cls = select_unquantized_moe_backend( + moe_config=moe_config, + ) + + assert selected_backend != UnquantizedMoeBackend.AITER + assert expert_cls is not None + + +@pytest.mark.skipif( + not current_platform.is_rocm(), reason="ROCm-specific backend selection test" +) +def test_explicit_aiter_backend_non_gated_activation_still_raises(): + """Explicit `--moe-backend aiter` still raises for non-gated MoE; + only the env-var opt-in path falls back.""" + moe_config = make_dummy_moe_config(activation=MoEActivation.SILU_NO_MUL) + moe_config.moe_backend = "aiter" # explicit pin, not "auto" + assert moe_config.is_act_and_mul is False + + with pytest.raises(ValueError): + select_unquantized_moe_backend(moe_config=moe_config) + + @patch( "vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsMonolithic.is_supported_config", return_value=(True, None), diff --git a/tests/kernels/moe/utils.py b/tests/kernels/moe/utils.py index e762472950..d92e4d6129 100644 --- a/tests/kernels/moe/utils.py +++ b/tests/kernels/moe/utils.py @@ -705,3 +705,27 @@ def check_accuracy(a, b, atol, rtol, percent): f"Mismatch percentage is {mismatch_percent:.4f} for rtol {rtol} " f"(threshold: {1 - percent:.4f})" ) + + +def mxfp4_w_layouts(mx_axis: int, num_warps: int = 8): + """Weight/scale layouts for mxfp4 MoE, as (layout, opts) pairs. + + triton_kernels 3.8 returns layout instances; earlier versions return a + (layout, opts) tuple. + """ + from triton_kernels.tensor_details import layout + + from vllm.utils.import_utils import get_triton_kernels_version + + if get_triton_kernels_version() == "3.8": + w = layout.make_default_matmul_mxfp4_w_layout(mx_axis=mx_axis) + s = layout.make_default_matmul_mxfp4_w_scale_layout( + mx_axis=mx_axis, num_warps=num_warps + ) + return w, {}, s, {} + + w, w_opts = layout.make_default_matmul_mxfp4_w_layout(mx_axis=mx_axis) + s, s_opts = layout.make_default_matmul_mxfp4_w_scale_layout( + mx_axis=mx_axis, num_warps=num_warps + ) + return w, w_opts, s, s_opts diff --git a/tests/kernels/quantization/test_block_fp8.py b/tests/kernels/quantization/test_block_fp8.py index d02527aba0..038b94bd2d 100644 --- a/tests/kernels/quantization/test_block_fp8.py +++ b/tests/kernels/quantization/test_block_fp8.py @@ -162,11 +162,16 @@ def test_w8a8_block_fp8_matmul(M, N, K, block_size, out_dtype, seed): @pytest.mark.skipif( not current_platform.is_cuda(), reason="CUTLASS only supported on CUDA platform." ) +@pytest.mark.parametrize( + # 65/66/67 cover all M%4 residue classes above the SM100 swapAB + # threshold (m <= 64); 1026 crosses multiple 128-row SF atoms. + "M", + [32, 65, 66, 67, 1026], +) @torch.inference_mode() -def test_w8a8_block_fp8_cutlass_matmul(): +def test_w8a8_block_fp8_cutlass_matmul(M): # Test simple case where weight.shape % 128 != 0, # like in DSV3 kv_a_proj_with_mqa - M = 32 N = 576 K = 7168 block_size = [128, 128] diff --git a/tests/kernels/quantization/test_cpu_fp8_scaled_mm.py b/tests/kernels/quantization/test_cpu_fp8_scaled_mm.py index e4dadb53bd..0b0d12a66c 100644 --- a/tests/kernels/quantization/test_cpu_fp8_scaled_mm.py +++ b/tests/kernels/quantization/test_cpu_fp8_scaled_mm.py @@ -1,6 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for CPU FP8 W8A16 block-scaled GEMM kernel (fp8_scaled_mm_cpu). +"""Tests for CPU FP8 scaled-mm GEMM kernels. + +Covers: + * fp8_scaled_mm_cpu (W8A16) + * fp8_scaled_mm_with_quant (W8A8) Run `pytest tests/kernels/quantization/test_cpu_fp8_scaled_mm.py -v`. """ @@ -23,19 +27,31 @@ from vllm.platforms import current_platform if not current_platform.is_cpu(): pytest.skip("skipping CPU-only tests", allow_module_level=True) -if not ops._supports_cpu_fp8_w8a16: - pytest.skip("fp8_scaled_mm_cpu op not available", allow_module_level=True) +requires_cpu_fp8_w8a16 = pytest.mark.skipif( + not ops._supports_cpu_fp8_w8a16, reason="fp8_scaled_mm_cpu op not available" +) +requires_cpu_fp8_w8a8 = pytest.mark.skipif( + not ops.cpu_has_amx_fp8(), + reason="requires native AMX-FP8 hardware", +) if not torch.cpu._is_amx_tile_supported(): pytest.skip("requires AMX tile support", allow_module_level=True) +FP8_MAX = torch.finfo(torch.float8_e4m3fn).max BLOCK_SIZE = [128, 128] +GROUP_SIZE = 128 def cdiv(a: int, b: int) -> int: return -(a // -b) +# --------------------------------------------------------------------------- +# W8A16 helpers +# --------------------------------------------------------------------------- + + def quantize_weight_block_fp8( weight: torch.Tensor, block_size: list[int], @@ -126,7 +142,7 @@ def ref_fp8_block_scaled_mm( # --------------------------------------------------------------------------- -# Test parameters +# W8A16 test: fp8_scaled_mm_cpu # --------------------------------------------------------------------------- M_SIZES = [1, 4, 16, 64, 128] # (N, K) — weight shape is [N, K], output has N columns. @@ -141,6 +157,7 @@ NK_SIZES = [ ] +@requires_cpu_fp8_w8a16 @pytest.mark.parametrize("M", M_SIZES) @pytest.mark.parametrize("N,K", NK_SIZES) @pytest.mark.parametrize("use_bias", [False, True]) @@ -200,6 +217,77 @@ def ref_fp8_per_tensor_scaled_mm( return out.to(out_dtype) +# --------------------------------------------------------------------------- +# W8A8 helpers +# --------------------------------------------------------------------------- + + +def quantize_weight_per_tensor( + weight: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + """Quantize weight [N, K] to FP8 with a single per-tensor scale, broadcast + to [N, 1] (the shape float8_linear_prepack_cpu expects).""" + N = weight.shape[0] + abs_max = weight.abs().max() + scale = (abs_max / FP8_MAX).clamp(min=1e-7) + q = (weight.float() / scale).clamp(-FP8_MAX, FP8_MAX).to(torch.float8_e4m3fn) + return q, scale.expand(N).contiguous().view(N, 1) + + +def quantize_weight_per_channel( + weight: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + """Quantize weight [N, K] to FP8 with per-channel (per-row) scales [N, 1].""" + abs_max = weight.abs().amax(dim=1, keepdim=True) + scale = (abs_max / FP8_MAX).clamp(min=1e-7) + q = (weight.float() / scale).clamp(-FP8_MAX, FP8_MAX).to(torch.float8_e4m3fn) + return q, scale.float() + + +def quantize_weight_per_group( + weight: torch.Tensor, + group_size: int = GROUP_SIZE, +) -> tuple[torch.Tensor, torch.Tensor]: + """Quantize weight [N, K] to FP8 with per-group (128-block) scales [N, G].""" + N, K = weight.shape + G = K // group_size + w_groups = weight.view(N, G, group_size) + abs_max = w_groups.abs().amax(dim=2, keepdim=True) + scale = (abs_max / FP8_MAX).clamp(min=1e-7) + q = (w_groups.float() / scale).clamp(-FP8_MAX, FP8_MAX).to(torch.float8_e4m3fn) + return q.view(N, K), scale.squeeze(2).float() + + +def quantize_act_per_token(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """Reference per-token (row) dynamic FP8 quantization of activations.""" + x_f = x.float() + scale = (x_f.abs().amax(dim=1, keepdim=True) / FP8_MAX).clamp(min=1e-7) + scaled = (x_f / scale).clamp(-FP8_MAX, FP8_MAX) + # Round via fp16 to match the AVX10.2 kernel path. + q = scaled.half().to(torch.float8_e4m3fn) + return q, scale + + +def reference_static_fp8_linear( + x: torch.Tensor, # BF16 [M, K] + weight_fp8: torch.Tensor, # FP8 [N, K] + weight_scale: torch.Tensor, # float32 [N, 1] + act_scale: torch.Tensor, # float32 scalar + bias: torch.Tensor | None, + out_dtype: torch.dtype, +) -> torch.Tensor: + """Static per-tensor: quantize/dequant x with act_scale, dequant weight, matmul.""" + x_scaled = (x.float() / act_scale.item()).clamp(-FP8_MAX, FP8_MAX) + # Round via fp16 to match the AVX10.2 kernel path. + x_q = x_scaled.half().to(torch.float8_e4m3fn) + x_dq = x_q.float() * act_scale.item() + w_dq = weight_fp8.float() * weight_scale.view(-1, 1) + out = torch.mm(x_dq, w_dq.t()) + if bias is not None: + out = out + bias.float() + return out.to(out_dtype) + + NK_SIZES_PER_TENSOR = [ (32, 64), (5120, 5120), @@ -274,3 +362,136 @@ def test_cpu_fp8_per_tensor_kernel_rejects_non_multiple_of_32_n(n: int): ) supported, _ = CPUFp8PerTensorScaledMMLinearKernel.can_implement(config) assert not supported + + +def reference_dynamic_fp8_linear( + x: torch.Tensor, # BF16 [M, K] + weight_fp8: torch.Tensor, # FP8 [N, K] + weight_scale: torch.Tensor, # float32 [N, G] + bias: torch.Tensor | None, + out_dtype: torch.dtype, +) -> torch.Tensor: + """Dynamic per-token act x per-group weight: quantize x per-token, + dequant weight per-group, matmul.""" + N, K = weight_fp8.shape + G = weight_scale.shape[1] + group_size = K // G + x_q, x_scale = quantize_act_per_token(x) + x_dq = x_q.float() * x_scale + w_dq = ( + weight_fp8.float().view(N, G, group_size) * weight_scale.view(N, G, 1) + ).view(N, K) + out = torch.mm(x_dq, w_dq.t()) + if bias is not None: + out = out + bias.float() + return out.to(out_dtype) + + +# --------------------------------------------------------------------------- +# W8A8 tests: fp8_scaled_mm_with_quant +# --------------------------------------------------------------------------- +W8A8_M_SIZES = [1, 4, 16, 64] +W8A8_NK_SIZES = [(128, 256), (256, 512), (64, 128)] + + +@requires_cpu_fp8_w8a8 +@pytest.mark.parametrize("static_activation", [True, False]) +@pytest.mark.parametrize("M", W8A8_M_SIZES) +@pytest.mark.parametrize("N,K", W8A8_NK_SIZES) +@pytest.mark.parametrize("use_bias", [False, True]) +def test_cpu_fp8_w8a8_per_tensor( + M: int, N: int, K: int, use_bias: bool, static_activation: bool +): + """CPUFP8W8A8ScaledMMLinearKernel with per-tensor weight scale.""" + torch.manual_seed(42) + out_dtype = torch.bfloat16 + + x = torch.randn(M, K, dtype=torch.bfloat16) + weight = torch.randn(N, K, dtype=torch.bfloat16) + bias = torch.randn(N, dtype=torch.float32) if use_bias else None + + w_fp8, w_scale = quantize_weight_per_tensor(weight) + packed_weight, packed_scale = torch.ops._C.float8_linear_prepack_cpu(w_fp8, w_scale) + + if static_activation: + act_scale = (x.float().abs().max() / FP8_MAX).clamp(min=1e-7).view(1) + kernel_out = ops.fp8_scaled_mm_with_quant( + x, act_scale, False, packed_weight, packed_scale, bias, out_dtype + ) + ref_out = reference_static_fp8_linear( + x, w_fp8, w_scale, act_scale, bias, out_dtype + ) + atol = 0.05 + else: + kernel_out = ops.fp8_scaled_mm_with_quant( + x, None, True, packed_weight, packed_scale, bias, out_dtype + ) + ref_out = reference_dynamic_fp8_linear(x, w_fp8, w_scale, bias, out_dtype) + atol = 0.05 + + assert kernel_out.dtype == out_dtype + torch.testing.assert_close(kernel_out, ref_out, rtol=0.05, atol=atol) + + +@requires_cpu_fp8_w8a8 +@pytest.mark.parametrize("static_activation", [True, False]) +@pytest.mark.parametrize("M", W8A8_M_SIZES) +@pytest.mark.parametrize("N,K", W8A8_NK_SIZES) +@pytest.mark.parametrize("use_bias", [False, True]) +def test_cpu_fp8_w8a8_per_channel( + M: int, N: int, K: int, use_bias: bool, static_activation: bool +): + """CPUFP8W8A8ScaledMMLinearKernel with per-channel weight scale.""" + torch.manual_seed(42) + out_dtype = torch.bfloat16 + + x = torch.randn(M, K, dtype=torch.bfloat16) + weight = torch.randn(N, K, dtype=torch.bfloat16) + bias = torch.randn(N, dtype=torch.float32) if use_bias else None + + w_fp8, w_scale = quantize_weight_per_channel(weight) + packed_weight, packed_scale = torch.ops._C.float8_linear_prepack_cpu(w_fp8, w_scale) + + if static_activation: + act_scale = (x.float().abs().max() / FP8_MAX).clamp(min=1e-7).view(1) + kernel_out = ops.fp8_scaled_mm_with_quant( + x, act_scale, False, packed_weight, packed_scale, bias, out_dtype + ) + ref_out = reference_static_fp8_linear( + x, w_fp8, w_scale, act_scale, bias, out_dtype + ) + atol = 0.05 + else: + kernel_out = ops.fp8_scaled_mm_with_quant( + x, None, True, packed_weight, packed_scale, bias, out_dtype + ) + ref_out = reference_dynamic_fp8_linear(x, w_fp8, w_scale, bias, out_dtype) + atol = 0.05 + + assert kernel_out.dtype == out_dtype + torch.testing.assert_close(kernel_out, ref_out, rtol=0.05, atol=atol) + + +@requires_cpu_fp8_w8a8 +@pytest.mark.parametrize("M", W8A8_M_SIZES) +@pytest.mark.parametrize("N,K", W8A8_NK_SIZES) +@pytest.mark.parametrize("use_bias", [False, True]) +def test_cpu_fp8_w8a8_per_group(M: int, N: int, K: int, use_bias: bool): + """CPUFp8W8A8BlockScaledMMKernel: per-group (128-block) weight scale + + dynamic per-token activation scale.""" + torch.manual_seed(42) + out_dtype = torch.bfloat16 + + x = torch.randn(M, K, dtype=torch.bfloat16) + weight = torch.randn(N, K, dtype=torch.bfloat16) + bias = torch.randn(N, dtype=torch.float32) if use_bias else None + + w_fp8, w_scale = quantize_weight_per_group(weight) + packed_weight, packed_scale = torch.ops._C.float8_linear_prepack_cpu(w_fp8, w_scale) + kernel_out = ops.fp8_scaled_mm_with_quant( + x, None, True, packed_weight, packed_scale, bias, out_dtype + ) + ref_out = reference_dynamic_fp8_linear(x, w_fp8, w_scale, bias, out_dtype) + + assert kernel_out.dtype == out_dtype + torch.testing.assert_close(kernel_out, ref_out, rtol=0.05, atol=0.05) diff --git a/tests/kernels/quantization/test_rocm_compressed_tensors_w4a16.py b/tests/kernels/quantization/test_rocm_compressed_tensors_w4a16.py index a9b35a4ea6..8e4563ab80 100644 --- a/tests/kernels/quantization/test_rocm_compressed_tensors_w4a16.py +++ b/tests/kernels/quantization/test_rocm_compressed_tensors_w4a16.py @@ -17,7 +17,6 @@ from vllm.platforms import current_platform @pytest.mark.parametrize( "model_path", [ - # Listed in tests/weight_loading/models.txt "nm-testing/tinyllama-oneshot-w4a16-group128-v2", ], ) diff --git a/tests/kernels/quantization/test_rocm_mxfp4.py b/tests/kernels/quantization/test_rocm_mxfp4.py index 0e14e2d258..8d1a4bb0a7 100644 --- a/tests/kernels/quantization/test_rocm_mxfp4.py +++ b/tests/kernels/quantization/test_rocm_mxfp4.py @@ -156,22 +156,26 @@ def test_fp4_env_defaults(): """ROCm FP4 env defaults should stay stable for the AITER gates.""" import vllm.envs as envs + assert envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM is False assert envs.VLLM_ROCM_USE_AITER_FP4BMM is True @pytest.mark.parametrize( ( "use_aiter", + "use_fp4_asm_gemm", "use_fp4bmm", ), [ - (True, True), - (True, False), - (False, True), + (True, True, True), + (True, True, False), + (True, False, True), + (False, True, True), ], ) def test_rocm_aiter_fp4_enablement_follows_env_and_arch( use_aiter, + use_fp4_asm_gemm, use_fp4bmm, monkeypatch, ): @@ -187,11 +191,15 @@ def test_rocm_aiter_fp4_enablement_follows_env_and_arch( _assert_aiter_supported() on_gfx950_value = on_gfx950() - expected_asm_gemm = use_aiter and on_gfx950_value + expected_asm_gemm = use_aiter and use_fp4_asm_gemm and on_gfx950_value expected_fp4bmm = use_aiter and use_fp4bmm and on_gfx950_value with monkeypatch.context() as mp: mp.setenv("VLLM_ROCM_USE_AITER", "1" if use_aiter else "0") + mp.setenv( + "VLLM_ROCM_USE_AITER_FP4_ASM_GEMM", + "1" if use_fp4_asm_gemm else "0", + ) mp.setenv("VLLM_ROCM_USE_AITER_FP4BMM", "1" if use_fp4bmm else "0") _reload_envs() rocm_aiter_ops.refresh_env_variables() diff --git a/tests/kernels/test_compressor_kv_cache.py b/tests/kernels/test_compressor_kv_cache.py index 95ff6b3c05..06897f04c0 100644 --- a/tests/kernels/test_compressor_kv_cache.py +++ b/tests/kernels/test_compressor_kv_cache.py @@ -473,15 +473,18 @@ _E2M1_MAGNITUDES = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0] def _decode_nvfp4_row(packed: torch.Tensor, scales: torch.Tensor) -> torch.Tensor: - """Unpack one 256-byte e2m1 row and apply its 32 e4m3 tile scales.""" + """Unpack e2m1 rows and apply their e4m3 tile scales.""" mags = torch.tensor(_E2M1_MAGNITUDES, device=packed.device) - codes = torch.empty(512, dtype=torch.uint8, device=packed.device) - codes[0::2] = packed & 0xF # even element in the low nibble - codes[1::2] = packed >> 4 + codes = torch.empty( + (*packed.shape[:-1], 512), dtype=torch.uint8, device=packed.device + ) + codes[..., 0::2] = packed & 0xF # even element in the low nibble + codes[..., 1::2] = packed >> 4 vals = mags[(codes & 7).long()] * torch.where(codes >= 8, -1.0, 1.0) return ( - vals.view(32, 16) * scales.view(torch.float8_e4m3fn).float().view(32, 1) - ).flatten() + vals.reshape(*packed.shape[:-1], 32, 16) + * scales.view(torch.float8_e4m3fn).float().reshape(*packed.shape[:-1], 32, 1) + ).flatten(-2) @pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only") @@ -543,7 +546,10 @@ def test_v41_rope_insert_nvfp4_record(compress_ratio: int): @pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only") -def test_v41_nvfp4_gather_matches_insert(): +@pytest.mark.parametrize("num_reqs", [1, 8, 32]) +@pytest.mark.parametrize("num_tokens", [70, 128, 129, 2051]) +@pytest.mark.parametrize("partial", [False, True]) +def test_v41_nvfp4_gather_matches_insert(num_reqs, num_tokens, partial): """The NVFP4 gather dequantizes exactly what the insert kernel wrote.""" from vllm.models.deepseek_v41.common.ops import dequantize_and_gather_k_cache from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import ( @@ -553,38 +559,73 @@ def test_v41_nvfp4_gather_matches_insert(): torch.manual_seed(17) device = "cuda" block_size = 64 - num_tokens = 70 - num_blocks = 4 + num_blocks = math.ceil(num_tokens / block_size) page_bytes = math.ceil(block_size * 288 / 512) * 512 positions = torch.arange(num_tokens, dtype=torch.int64, device=device) latent = torch.randn(num_tokens, 512, dtype=torch.bfloat16, device=device) - angles = torch.randn(128, 32, device=device) + angles = torch.randn(num_tokens, 32, device=device) cos_sin = torch.cat((angles.cos(), angles.sin()), dim=-1) backing = torch.zeros(num_blocks, page_bytes, dtype=torch.uint8, device=device) cache = backing.as_strided((num_blocks, block_size, 288), (page_bytes, 288, 1)) slots = torch.arange(num_tokens, dtype=torch.int64, device=device) rope_quant_insert(latent, positions, cos_sin, cache, slots, 1) - out = torch.zeros(1, num_tokens, 512, dtype=torch.bfloat16, device=device) - dequantize_and_gather_k_cache( - out, - cache, - seq_lens=torch.tensor([num_tokens], dtype=torch.int32, device=device), - gather_lens=None, - block_table=torch.arange(num_blocks, dtype=torch.int32, device=device).view( - 1, -1 - ), - block_size=block_size, - offset=0, + offset = 7 if partial else 0 + seq_lens = num_tokens - torch.arange(num_reqs, dtype=torch.int32, device=device) + gather_lens = seq_lens - 3 if partial else None + storage = torch.full( + (num_reqs, num_tokens + offset + 16, 512), + -123.0, + dtype=torch.bfloat16, + device=device, ) - for t in (0, 1, block_size, num_tokens - 1): - page, row = divmod(t, block_size) - expected = _decode_nvfp4_row( - backing[page, row * 256 : (row + 1) * 256], - backing[page, block_size * 256 + row * 32 :][:32], + out = storage[:, : num_tokens + offset] + block_table = torch.arange(num_blocks, dtype=torch.int32, device=device).repeat( + num_reqs, 1 + ) + + def gather(): + dequantize_and_gather_k_cache( + out, + cache, + seq_lens=seq_lens, + gather_lens=gather_lens, + block_table=block_table, + block_size=block_size, + offset=offset, ) - torch.testing.assert_close(out[0, t].float(), expected, rtol=0, atol=0) + + expected = _decode_nvfp4_row( + backing[:, : block_size * 256].reshape(-1, 256), + backing[:, block_size * 256 : block_size * 288].reshape(-1, 32), + ) + + def check(shorter_by): + assert (storage[:, num_tokens + offset :] == -123).all() + for req in range(num_reqs): + start = 3 if partial else 0 + length = num_tokens - req - start - shorter_by + assert (out[req, :offset] == -123).all() + assert (out[req, offset + length :] == -123).all() + torch.testing.assert_close( + out[req, offset : offset + length].float(), + expected[start : start + length], + rtol=0, + atol=0, + ) + + gather() + check(0) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + gather() + storage.fill_(-123) + seq_lens.sub_(1) + if gather_lens is not None: + gather_lens.sub_(1) + graph.replay() + check(1) @pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only") diff --git a/tests/kernels/test_fused_deepseek_v32_norm_rope.py b/tests/kernels/test_fused_deepseek_v32_norm_rope.py index 7f74cde05d..61f977ddbd 100644 --- a/tests/kernels/test_fused_deepseek_v32_norm_rope.py +++ b/tests/kernels/test_fused_deepseek_v32_norm_rope.py @@ -507,8 +507,8 @@ def test_fused_norm_rope_materializes_pcp_cache_inputs(has_indexer: bool): def test_fused_norm_rope_ds_mla(num_tokens: int): """fp8_ds_mla MLA cache layout (FlashMLA sparse, bf16-query path; SM90/SM100). - Per-token 656-byte entry: 512 fp8 NoPE (4 per-128 tiles, dynamic float32 - scale) | 4 float32 scales | 64 bf16 (unquantized) RoPE. + Per-token 656-byte entry: 512 fp8 NoPE (4 per-128 tiles, power-of-two scales + stored as float32) | 4 float32 scales | 64 bf16 (unquantized) RoPE. """ torch.manual_seed(5) dev = "cuda" @@ -557,7 +557,8 @@ def test_fused_norm_rope_ds_mla(num_tokens: int): kv_ref = rms_norm(kv_c, kvw) # [N, 512] fp32 kpe_ref = rope(k_pe.float(), pos, mla_cos_sin, interleave=True) # [N, 64] tiles = kv_ref.view(num_tokens, 4, 128) - ref_scale = torch.clamp(tiles.abs().amax(dim=-1) / FP8_MAX, min=1.1754944e-38) + ref_scale = torch.clamp(tiles.abs().amax(dim=-1) / FP8_MAX, min=1e-4) + ref_scale = torch.exp2(torch.ceil(torch.log2(ref_scale))) ref_nope = (tiles / ref_scale[..., None]).reshape(num_tokens, KV_LORA).to(FP8) cache = mla_cache[0, :num_tokens] # [N, 656] uint8 @@ -566,7 +567,7 @@ def test_fused_norm_rope_ds_mla(num_tokens: int): rope_off = KV_LORA // 2 + 8 rope_vals = cache.view(torch.bfloat16)[:, rope_off : rope_off + ROPE_DIM] - torch.testing.assert_close(scales, ref_scale, rtol=1e-2, atol=1e-6) + torch.testing.assert_close(scales, ref_scale, rtol=0, atol=0) assert_fp8(nope, ref_nope, "ds_mla NoPE fp8") assert_bf16(rope_vals, kpe_ref, "ds_mla RoPE bf16") # No indexer on this call: top-k buffer must be untouched. diff --git a/tests/kernels/test_mhc_kernels.py b/tests/kernels/test_mhc_kernels.py index fafb37d97b..08b1a24c85 100644 --- a/tests/kernels/test_mhc_kernels.py +++ b/tests/kernels/test_mhc_kernels.py @@ -132,6 +132,7 @@ def test_v41_dspark_head_collapses_with_last_ffn_mix(num_tokens, monkeypatch): monkeypatch.setattr(dspark, "mhc_post_tilelang", lambda *args: streams) draft = SimpleNamespace( + use_mega_moe=False, use_sequence_parallel=False, hc_mult=hc_mult, layers=[make_layer(mix) for mix in mixes], @@ -452,7 +453,7 @@ def test_deepseek_v41_decoder_mixes_match_torch( device=DEVICE, dtype=torch.bfloat16 ) decoder.attn = lambda positions, x, _: x * 0.5 - decoder.ffn = lambda x, input_ids: x * 0.25 + decoder.ffn = lambda x, input_ids, mega_gate_metadata=None: x * 0.25 with torch.device(DEVICE): decoder.hc_attn_fn = torch.randn(24, 20480) * 0.02 decoder.hc_ffn_fn = torch.randn(24, 20480) * 0.02 @@ -536,7 +537,7 @@ def test_deepseek_v41_capture_previous_aux(entry, monkeypatch, default_vllm_conf device=DEVICE, dtype=torch.bfloat16 ) decoder.attn = lambda positions, x, _: x * 0.5 - decoder.ffn = lambda x, input_ids: x * 0.25 + decoder.ffn = lambda x, input_ids, mega_gate_metadata=None: x * 0.25 with torch.device(DEVICE): decoder.hc_attn_fn = torch.randn(24, 20480) * 0.02 decoder.hc_ffn_fn = torch.randn(24, 20480) * 0.02 diff --git a/tests/kernels/test_mhc_tilelang_jit.py b/tests/kernels/test_mhc_tilelang_jit.py index 4f39fae020..1c430e7ae3 100644 --- a/tests/kernels/test_mhc_tilelang_jit.py +++ b/tests/kernels/test_mhc_tilelang_jit.py @@ -24,19 +24,26 @@ class _PassConfigKey: def _install_tilelang_stub( monkeypatch: pytest.MonkeyPatch, ) -> dict[str, int]: - calls = {"jit_decorate": 0, "compiled_call": 0} + calls = {"jit_decorate": 0, "compiled_call": 0, "compiled_compile": 0} tilelang: Any = ModuleType("tilelang") + class _JitImpl: + def __init__(self, func: Any) -> None: + self.func = func + + def __call__(self, *args: Any, **kw: Any) -> Any: + calls["compiled_call"] += 1 + return self.func.__name__ + + def compile(self, *args: Any, **kw: Any) -> Any: + calls["compiled_compile"] += 1 + return self.func.__name__ + def jit(**kwargs: Any) -> Any: def decorate(func: Any) -> Any: calls["jit_decorate"] += 1 - - def compiled(*args: Any, **kw: Any) -> Any: - calls["compiled_call"] += 1 - return func.__name__ - - return compiled + return _JitImpl(func) return decorate @@ -78,6 +85,21 @@ def test_tilelang_jit_decorator_is_lazy_only_on_rocm( assert calls["compiled_call"] == 1 +@pytest.mark.skipif(not current_platform.is_rocm(), reason="Test requires ROCm") +def test_tilelang_jit_proxies_compile_only_warmup( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls = _install_tilelang_stub(monkeypatch) + module_name = "vllm.model_executor.kernels.mhc.tilelang_kernels" + monkeypatch.delitem(sys.modules, module_name, raising=False) + module = importlib.import_module(module_name) + + assert module.mhc_post_tilelang.compile() == "mhc_post_tilelang" + assert calls["jit_decorate"] == 1 + assert calls["compiled_compile"] == 1 + assert calls["compiled_call"] == 0 + + @pytest.mark.skipif(not current_platform.is_rocm(), reason="Test requires ROCm") def test_deepseek_v4_import_and_jit_monitor_do_not_hijack_hip_symbols() -> None: if importlib.util.find_spec("tilelang") is None: diff --git a/tests/kernels/test_minimax_m3_amd_ops.py b/tests/kernels/test_minimax_m3_amd_ops.py index 997c619809..27bd74ce41 100644 --- a/tests/kernels/test_minimax_m3_amd_ops.py +++ b/tests/kernels/test_minimax_m3_amd_ops.py @@ -10,6 +10,7 @@ tests assert the two agree within tolerance: * Fused MXFP8 activation quant (Triton) -> _mxfp8_e4m3_quantize_torch * Native MXFP8 linear (dot_scaled) -> dequant-to-bf16 @ matmul * Native MXFP8 MoE (dot_scaled grouped GEMM) -> dequant-to-bf16 MoE math + * Strided K/V sparse-PA cache insert -> contiguous-copy insert The native MXFP8 GEMMs also guard the ``dot_scaled`` rhs-scale orientation: the scale is loaded ``[N, K//32]`` and passed WITHOUT transpose; a stray ``.T`` @@ -39,6 +40,7 @@ from vllm.model_executor.layers.quantization.utils.mxfp8_utils import ( # noqa: _mxfp8_e4m3_quantize_triton, dequant_mxfp8_to_bf16, ) +from vllm.models.minimax_m3.amd.model import _kv_insert_operand # noqa: E402 from vllm.models.minimax_m3.amd.ops import ( # noqa: E402 gemma_fused_add_rmsnorm, gemma_rmsnorm, @@ -608,3 +610,77 @@ def test_routed_experts_expert_map_delegates_to_kernel(): assert torch.equal(resolve(True), mask) # AITER kernel -> 0/1 mask assert torch.equal(resolve(False), canonical) # non-AITER -> canonical map assert torch.equal(resolve(False, has_moe_kernel=False), canonical) # non-modular + + +# --------------------------------------------------------------------------- # +# Strided K/V operand for the AITER sparse-PA cache insert +# --------------------------------------------------------------------------- # +def _fused_qkv_kv_slices(num_tokens, num_kv_heads, head_dim, num_q_heads=16): + """K/V as the sparse-PA insert sees them: column slices of a fused qkv.""" + kv = num_kv_heads * head_dim + row = num_q_heads * head_dim + 2 * kv + 2 * head_dim + qkv = torch.randn((num_tokens, row), dtype=torch.bfloat16, device=DEVICE) + k_start = num_q_heads * head_dim + k = qkv[:, k_start : k_start + kv].view(num_tokens, num_kv_heads, head_dim) + v = qkv[:, k_start + kv : k_start + 2 * kv].view(num_tokens, num_kv_heads, head_dim) + return k, v + + +def _asm_kv_cache(pages, num_kv_heads, head_dim): + """Page-16 K/V cache views matching ``_ensure_aiter_sparse_pa_kv_cache``.""" + x = 16 // torch.tensor([], dtype=torch.bfloat16).element_size() + key = torch.zeros( + (pages, num_kv_heads, head_dim // x, 16, x), + dtype=torch.bfloat16, + device=DEVICE, + ) + value = torch.zeros( + (pages, num_kv_heads, 16 // x, head_dim, x), + dtype=torch.bfloat16, + device=DEVICE, + ) + return key, value + + +@pytest.mark.parametrize("num_tokens", [1, 8, 129, 1024]) +@pytest.mark.parametrize("num_kv_heads", [1, 2]) +def test_kv_insert_operand_matches_contiguous(num_tokens, num_kv_heads): + """A strided K/V slice must insert bit-identically to a contiguous copy.""" + reshape_and_cache = pytest.importorskip("aiter").reshape_and_cache + head_dim = 128 + pages = num_tokens // 16 + 4 + slots = torch.randperm(pages * 16, device=DEVICE)[:num_tokens].to(torch.int32) + + torch.manual_seed(num_tokens) + k, v = _fused_qkv_kv_slices(num_tokens, num_kv_heads, head_dim) + assert num_tokens == 1 or not k.is_contiguous() + + caches = [] + for operand in (lambda t: t.contiguous(), _kv_insert_operand): + key_cache, value_cache = _asm_kv_cache(pages, num_kv_heads, head_dim) + reshape_and_cache( + operand(k), + operand(v), + key_cache, + value_cache, + slots, + kv_cache_dtype="auto", + asm_layout=True, + ) + caches.append((key_cache, value_cache)) + + (ref_k, ref_v), (got_k, got_v) = caches + assert torch.equal(ref_k, got_k) + assert torch.equal(ref_v, got_v) + + +def test_kv_insert_operand_copies_unsupported_layouts(): + """Only a row-strided slice passes through; anything else is copied.""" + k, _ = _fused_qkv_kv_slices(32, 2, 128) + assert _kv_insert_operand(k) is k + + inner_strided = k[:, :, ::2] + assert _kv_insert_operand(inner_strided).is_contiguous() + + flat = k.reshape(32, -1) + assert _kv_insert_operand(flat).is_contiguous() diff --git a/tests/lora/test_lora_checkpoints.py b/tests/lora/test_lora_checkpoints.py index 8db529223b..9dd0296a88 100644 --- a/tests/lora/test_lora_checkpoints.py +++ b/tests/lora/test_lora_checkpoints.py @@ -147,10 +147,10 @@ def test_gemma4_lora_weights_mapping(): def test_gemma4_moe_lora_weights_mapping(): mapper = Gemma4ForCausalLM.hf_to_vllm_mapper name = ( - "base_model.model.model.language_model.layers.9.moe.experts." + "base_model.model.model.language_model.layers.9.experts." "gate_up_proj.lora_B.weight" ) assert parse_fine_tuned_lora_name(name, mapper) == ( - "model.layers.9.moe.gate_up_proj", + "model.layers.9.experts.gate_up_proj", False, ) diff --git a/tests/lora/test_qwenvl.py b/tests/lora/test_qwenvl.py index 063de2b587..360b9aff05 100644 --- a/tests/lora/test_qwenvl.py +++ b/tests/lora/test_qwenvl.py @@ -3,8 +3,6 @@ from dataclasses import dataclass import pytest -from packaging.version import Version -from transformers import __version__ as TRANSFORMERS_VERSION import vllm from tests.conftest import VllmRunner @@ -29,20 +27,12 @@ class TestConfig: def __post_init__(self): if self.mm_processor_kwargs is None: - # There is a bug in transformers v4 where size is ignored by - # `Qwen2VLProcessor.__call__` - if Version(TRANSFORMERS_VERSION) < Version("5.2.0"): - self.mm_processor_kwargs = { - "min_pixels": 28 * 28, - "max_pixels": 1280 * 28 * 28, - } - else: - self.mm_processor_kwargs = { - "size": { - "shortest_edge": 28 * 28, - "longest_edge": 1280 * 28 * 28, - } + self.mm_processor_kwargs = { + "size": { + "shortest_edge": 28 * 28, + "longest_edge": 1280 * 28 * 28, } + } class Qwen2VLTester: diff --git a/tests/model_executor/layers/test_activation.py b/tests/model_executor/layers/test_activation.py new file mode 100644 index 0000000000..d0ef42c99a --- /dev/null +++ b/tests/model_executor/layers/test_activation.py @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from functools import partial + +import pytest +import torch +import torch.nn as nn + +from vllm.model_executor.layers.activation import GeluAndMul, SiluAndMul +from vllm.platforms import CpuArchEnum, current_platform + + +@pytest.mark.skipif( + not current_platform.is_cpu() + or current_platform.get_cpu_architecture() == CpuArchEnum.POWERPC, + reason="Native CPU activations require a non-PowerPC CPU platform", +) +@pytest.mark.parametrize( + "activation_factory,reference", + [ + (SiluAndMul, nn.SiLU()), + (GeluAndMul, nn.GELU()), + (partial(GeluAndMul, approximate="tanh"), nn.GELU(approximate="tanh")), + ], + ids=["silu", "gelu", "gelu_tanh"], +) +def test_cpu_act_and_mul_without_compiled_ops( + default_vllm_config, monkeypatch, activation_factory, reference +): + """CPU activations must construct and run without compiled wheel operators.""" + default_vllm_config.compilation_config.custom_ops = ["all"] + monkeypatch.setattr(torch.ops, "_C", object()) + x = torch.tensor([[-2.0, -1.0, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0]], device="cpu") + + layer = activation_factory() + + torch.testing.assert_close(layer(x), reference(x[:, :4]) * x[:, 4:]) diff --git a/tests/model_executor/layers/test_mla_short_prefill_indexer.py b/tests/model_executor/layers/test_mla_short_prefill_indexer.py index 41a06d0546..340d59a392 100644 --- a/tests/model_executor/layers/test_mla_short_prefill_indexer.py +++ b/tests/model_executor/layers/test_mla_short_prefill_indexer.py @@ -365,6 +365,26 @@ def test_select_candidate_blocks_tolerates_empty_rows(): assert (out[0] >= 0).all() and (out[2, :2] >= 0).all() +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +@pytest.mark.parametrize("score", [0.0, float("nan")]) +@pytest.mark.parametrize("start", [0, 3]) +def test_candidate_selection_keeps_valid_blocks_with_nan_scores(score, start): + """NaN warmup scores must not turn real blocks into repeated sparse padding.""" + logits = torch.full((3, 64), score, device="cuda") + starts = torch.full((3,), start, device="cuda", dtype=torch.int32) + lengths = torch.tensor([0, 17, 35], device="cuda", dtype=torch.int32) + out = torch.empty(3, 8, device="cuda", dtype=torch.int32) + + sparse_indexer._select_candidate_blocks(logits, starts, starts + lengths, 8, 8, out) + + for row, count in enumerate([0, 3, 5]): + valid = out[row][out[row] >= 0].sort().values + torch.testing.assert_close( + valid, torch.arange(count, device="cuda", dtype=torch.int32) + ) + assert (out[row] == -1).sum().item() == 8 - count + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") @pytest.mark.parametrize( "width,block_size,k,decode", @@ -413,7 +433,7 @@ def test_candidate_kernels_preserve_packed_bounds_and_padding( top = reduced.topk(min(k, nblocks), dim=-1) expected = torch.full((rows, k), -1, device="cuda", dtype=torch.int32) expected[:, : top.indices.shape[1]] = torch.where( - top.values > -torch.inf, top.indices, -1 + top.values != -torch.inf, top.indices, -1 ).int() actual = torch.empty(rows, k * 2, device="cuda", dtype=torch.int32)[:, ::2] select_candidate_blocks(logits, starts, ends, k, block_size, actual, repeat) @@ -608,7 +628,10 @@ def test_candidate_blocks_to_sparse_indices_math(): @pytest.mark.skipif(not torch.cuda.is_available(), reason="Triton kernel") @pytest.mark.parametrize("cbk,sbk", [(8, 8), (16, 8), (16, 16)]) -def test_candidate_blocks_to_sparse_indices_matches_reference(cbk: int, sbk: int): +@pytest.mark.parametrize("end_offset", [0, 1]) +def test_candidate_blocks_to_sparse_indices_matches_reference( + cbk: int, sbk: int, end_offset: int +): """The Triton expansion equals the PyTorch reference on production-shaped random inputs, including strided (column-sliced) candidate rows and strided output buffers.""" @@ -643,7 +666,7 @@ def test_candidate_blocks_to_sparse_indices_matches_reference(cbk: int, sbk: int ke, cbk, sbk, - out=(si_buf[:, : num_candidates * ratio], end_buf[::2]), + out=(si_buf[:, : num_candidates * ratio], end_buf[end_offset::2]), ) assert torch.equal(got_indices, ref_indices) assert torch.equal(got_end, ref_end) @@ -651,7 +674,7 @@ def test_candidate_blocks_to_sparse_indices_matches_reference(cbk: int, sbk: int cand_buf[:, num_candidates:], torch.full_like(cand_buf[:, num_candidates:], -7) ) assert not si_buf[:, num_candidates * ratio :].any() - assert not end_buf[1::2].any() + assert not end_buf[1 - end_offset :: 2].any() def _skip_unless_sm100_sparse_kernels(): diff --git a/tests/model_executor/model_loader/test_reload.py b/tests/model_executor/model_loader/test_reload.py index b4a544d2fc..fde8b86cda 100644 --- a/tests/model_executor/model_loader/test_reload.py +++ b/tests/model_executor/model_loader/test_reload.py @@ -5,6 +5,7 @@ import importlib.machinery import inspect import sys import types +from types import SimpleNamespace from unittest.mock import Mock from weakref import WeakKeyDictionary, ref @@ -467,6 +468,111 @@ def test_model_cleanup(dist_init, default_vllm_config): assert len(mock_info_dict) == 0 +@pytest.mark.parametrize("is_gated", [False, True]) +@pytest.mark.parametrize("has_bias", [False, True]) +@pytest.mark.parametrize("tp_rank", [0, 1]) +@pytest.mark.parametrize("padded", [False, True]) +def test_padded_moe_reload_releases_each_layer( + monkeypatch, is_gated, has_bias, tp_rank, padded +): + """Checkpoint-sized copies finish each layer without global finalization.""" + from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts + from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import ( + UnquantizedFusedMoEMethod, + ) + + hidden, intermediate, experts = 4, 3, 2 + stored_hidden, stored_intermediate = (8, 8) if padded else (hidden, intermediate) + config = SimpleNamespace( + hidden_dim_unpadded=hidden, + intermediate_size_per_partition_unpadded=intermediate, + is_act_and_mul=is_gated, + has_bias=has_bias, + tp_rank=tp_rank, + tp_shard_with_padding=False, + moe_parallel_config=SimpleNamespace(tp_size=2), + ) + model = torch.nn.ModuleList() + processed: list[torch.nn.Module] = [] + for _ in range(2): + method = object.__new__(UnquantizedFusedMoEMethod) + torch.nn.Module.__init__(method) + method.moe = config + # The regression concerns streaming reload, not kernel conversion. + monkeypatch.setattr(method, "process_weights_after_loading", processed.append) + layer = object.__new__(RoutedExperts) + torch.nn.Module.__init__(layer) + layer.moe_config = config + layer.quant_config = None + layer.quant_method = method + layer.expert_map_manager = SimpleNamespace(map_global_to_local=lambda i: i) + layer._loaded_expert_biases = set() + method.create_weights( + layer, + experts, + stored_hidden, + stored_intermediate, + torch.float32, + weight_loader=layer.weight_loader, + ) + model.append(layer) + + record_metadata_for_reloading(model) + original_params = [dict(layer.named_parameters()) for layer in model] + shards = ["w1", "w3", "w2"] if is_gated else ["w1", "w2"] + for cycle in range(2): + initialize_layerwise_reload(model) + for layer_index, layer in enumerate(model): + info = reload_layerwise.get_layerwise_info(layer) + inputs = [] + expected = { + name: torch.full_like(p, float("nan")) + for name, p in original_params[layer_index].items() + } + params = dict(layer.named_parameters()) + calls = [ + (e, s, b) + for e in range(experts) + for s in shards + for b in ([False, True] if has_bias else [False]) + ] + for call_index, (expert, shard, bias) in enumerate(calls): + name = ("w2" if shard == "w2" else "w13") + ( + "_bias" if bias else "_weight" + ) + shape = ( + ((hidden,) if bias else (hidden, 2 * intermediate)) + if shard == "w2" + else ((2 * intermediate,) if bias else (2 * intermediate, hidden)) + ) + weight = torch.arange( + torch.Size(shape).numel(), dtype=torch.float32 + ).reshape(shape) + weight = weight + 100 * (1 + call_index + cycle) + inputs.append(ref(weight)) + # Direct checkpoint loading is the reference for deferred reload. + layer.weight_loader(expected[name], weight, name, shard, expert) + param = params[name] + param.weight_loader(param, weight, name, shard, expert) + del weight + if call_index != len(calls) - 1: + assert info.can_load(), ( + "Layer processed before its final checkpoint shard" + ) + + assert not info.can_load(), ( + "Padding must not defer the layer until finalization" + ) + assert not info.loaded_weights + assert len(processed) == cycle * len(model) + layer_index + 1 + assert all(source() is None for source in inputs) + for name, original in original_params[layer_index].items(): + assert getattr(layer, name) is original + # Kernel-specific tests cover padding, which is not checkpoint data. + mask = torch.isfinite(expected[name]) + assert torch.equal(original[mask], expected[name][mask]) + + def test_get_numel_loaded(): param = torch.empty(10, device="meta") loaded_weight = torch.empty(10) diff --git a/tests/models/glm5next/test_sparse_indexer_topk_dispatch.py b/tests/models/glm5next/test_sparse_indexer_topk_dispatch.py new file mode 100644 index 0000000000..83dfdc9f6f --- /dev/null +++ b/tests/models/glm5next/test_sparse_indexer_topk_dispatch.py @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Focused unit tests for the GLM decode top-k dispatch.""" + +import pytest +import torch + +from vllm.models.glm5next.nvidia.sparse_indexer import _use_cooperative_topk +from vllm.platforms import current_platform + +pytestmark = pytest.mark.skipif( + not current_platform.is_cuda(), reason="CUDA-only dispatch" +) + + +@pytest.mark.parametrize("num_rows", [1, 64], ids=["1row", "64rows"]) +@torch.inference_mode() +def test_cooperative_topk_is_selected_for_supported_batch(num_rows: int): + logits = torch.empty(num_rows, 8192, dtype=torch.float32, device="cuda") + assert _use_cooperative_topk(logits, 512, num_rows) + + +@pytest.mark.parametrize( + ("num_rows", "select_k", "stride_padding"), + [ + (65, 512, 0), + (1, 256, 0), + (1, 512, 1), + ], + ids=["too_many_rows", "unsupported_k", "unaligned_stride"], +) +@torch.inference_mode() +def test_cooperative_topk_falls_back_on_unsupported_batch( + num_rows: int, select_k: int, stride_padding: int +): + logits = torch.empty( + num_rows, 8192 + stride_padding, dtype=torch.float32, device="cuda" + ) + if stride_padding: + logits = logits[:, :8192] + assert not _use_cooperative_topk(logits, select_k, num_rows) diff --git a/tests/models/language/generation/test_common.py b/tests/models/language/generation/test_common.py index 33ff4e7ccb..f70d5a0c68 100644 --- a/tests/models/language/generation/test_common.py +++ b/tests/models/language/generation/test_common.py @@ -3,8 +3,6 @@ import pytest import torch -from packaging.version import Version -from transformers import __version__ as TRANSFORMERS_VERSION from vllm.platforms import current_platform @@ -26,7 +24,7 @@ AITER_MODEL_LIST = [ "meta-llama/Llama-3.2-1B-Instruct", "openbmb/MiniCPM3-4B", "Qwen/Qwen2.5-0.5B-Instruct", - "TitanML/tiny-mixtral", + "axolotl-ai-co/tiny-mixtral-30m", "Qwen/Qwen3-8B", ] @@ -95,7 +93,7 @@ AITER_MODEL_LIST = [ pytest.param("stabilityai/stablelm-3b-4e1t"), # stablelm pytest.param("bigcode/starcoder2-3b"), # starcoder2 pytest.param( - "TitanML/tiny-mixtral", # mixtral + "axolotl-ai-co/tiny-mixtral-30m", # mixtral marks=[pytest.mark.core_model], ), pytest.param("swiss-ai/Apertus-8B-Instruct-2509"), # apertus @@ -126,12 +124,6 @@ def test_models( model_info.check_available_online(on_fail="skip") model_info.check_transformers_version(on_fail="skip") - if current_platform.is_rocm() and model == "TitanML/tiny-mixtral": - # Its single-token router selects LLMM1, whose low-precision - # accumulation can change the top-2 experts. Keep the optimized kernel - # enabled generally, but use the reference GEMM for this accuracy test. - monkeypatch.setenv("VLLM_ROCM_USE_SKINNY_GEMM", "0") - if use_rocm_aiter and (model in AITER_MODEL_LIST): monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1") elif use_rocm_aiter and model not in AITER_MODEL_LIST: @@ -168,16 +160,6 @@ def test_models( if prompt_embeds is not None: embed = hf_model.model.get_input_embeddings()(token_ids) - if "gemma" in model.lower() and ( - Version(TRANSFORMERS_VERSION) < Version("5.3.0.dev0") - ): - # For Gemma 1/2 models with Transformers 5.4.0+, the prompt - # embeddings are normalised in `get_prompt_embeddings`, - # like Gemma 3. For older versions, we need to manually normalise. - embed_scale = hf_model.config.hidden_size**0.5 - normalizer = torch.tensor(embed_scale, dtype=embed.dtype) - embed *= normalizer - # MiniCPM models apply scale_emb to embeddings internally. # vLLM expects pre-scaled embeddings when using inputs_embeds. if model in EMBED_SCALING_MODELS: diff --git a/tests/models/language/generation/test_gemma.py b/tests/models/language/generation/test_gemma.py index c2c8e88e42..5cffe6fb19 100644 --- a/tests/models/language/generation/test_gemma.py +++ b/tests/models/language/generation/test_gemma.py @@ -14,7 +14,10 @@ from vllm.model_executor.models.gemma3n import ( Gemma3nTextModel, _kv_sharing_weights_mapper, ) -from vllm.model_executor.models.gemma4 import Gemma4Model +from vllm.model_executor.models.gemma4 import ( + Gemma4ForCausalLM, + _gemma4_layer_weights_mapper, +) MODELS = ["google/gemma-2b", "google/gemma-2-2b", "google/gemma-3-4b-it"] @@ -57,36 +60,70 @@ def test_checkpoint_lm_head_can_override_tied_config(monkeypatch) -> None: @pytest.mark.cpu_test -def test_gemma4_kv_shared_layer_loads_plain_q_proj() -> None: - """KV-shared layers have q_proj instead of a packed qkv_proj; their - redundant K/V tensors in original checkpoints have no parameter and are - skipped rather than failing the load.""" - model = torch.nn.Module() - model.config = SimpleNamespace(num_experts=0) - model.start_layer, model.end_layer = 0, 2 - model.layers = torch.nn.ModuleList([torch.nn.Module(), torch.nn.Module()]) - for layer in model.layers: - layer.self_attn = torch.nn.Module() - model.layers[0].self_attn.qkv_proj = torch.nn.Linear(2, 6, bias=False) - model.layers[1].self_attn.q_proj = torch.nn.Linear(2, 2, bias=False) - shards: list[str] = [] - model.layers[0].self_attn.qkv_proj.weight.weight_loader = ( - lambda param, weight, shard_id: shards.append(shard_id) +def test_gemma4_attention_mapper() -> None: + """Layers with a qkv_proj pack q/k/v; `attention_k_eq_v` full-attention + layers also load K as the V shard, and leave any v_proj such a checkpoint + ships unmapped so it fails the load rather than silently overwriting V; + KV-shared layers keep q_proj and drop the K/V tensors original checkpoints + still ship for them.""" + config = SimpleNamespace( + num_hidden_layers=3, + num_kv_shared_layers=1, + attention_k_eq_v=True, + layer_types=["sliding_attention", "full_attention", "sliding_attention"], ) - - q_weight = torch.full((2, 2), 2.0) weights = [ - (f"layers.{i}.self_attn.{tensor}.weight", q_weight) - for i in (0, 1) - for tensor in ("q_proj", "k_proj", "v_proj") - ] + [("layers.1.self_attn.k_norm.weight", torch.ones(2))] - loaded = Gemma4Model.load_weights(cast(Gemma4Model, model), weights) + (f"model.layers.{i}.self_attn.{tensor}.weight", torch.full((2, 2), i + 1.0)) + for i in range(3) + for tensor in ("q_proj", "k_proj", "k_norm") + ] + [ + ("model.layers.0.mlp.up_proj.weight", torch.empty(0)), + ("model.layers.1.self_attn.v_proj.weight", torch.empty(0)), + ] - assert shards == ["q", "k", "v"] - assert "layers.1.self_attn.q_proj.weight" in loaded - assert not any(name.startswith("layers.1.self_attn.k") for name in loaded) - assert not any(name.startswith("layers.1.self_attn.v") for name in loaded) - assert torch.equal(model.layers[1].self_attn.q_proj.weight, q_weight) + mapper = _gemma4_layer_weights_mapper(config) + mapped = list(mapper.apply(weights)) + + assert [(name, getattr(w, "shard_id", None)) for name, w in mapped] == [ + ("model.layers.0.self_attn.qkv_proj.weight", "q"), + ("model.layers.0.self_attn.qkv_proj.weight", "k"), + ("model.layers.0.self_attn.k_norm.weight", None), + ("model.layers.1.self_attn.qkv_proj.weight", "q"), + ("model.layers.1.self_attn.qkv_proj.weight", "k"), + ("model.layers.1.self_attn.qkv_proj.weight", "v"), + ("model.layers.1.self_attn.k_norm.weight", None), + ("model.layers.2.self_attn.q_proj.weight", None), + ("model.layers.0.mlp.gate_up_proj.weight", 1), + ("model.layers.1.self_attn.v_proj.weight", None), + ] + k_weight, v_weight = weights[4][1], mapped[5][1] + assert torch.equal(v_weight, k_weight) and v_weight is not k_weight + + +@pytest.mark.cpu_test +def test_gemma4_expert_names_strip_language_model_prefix() -> None: + """The text-only path reuses the conditional wrapper's checkpoint naming, + so fused and per-expert tensors reach the experts under `model.*`.""" + prefix = "model.language_model.layers.0." + weights = [ + (prefix + name, torch.empty(0)) + for name in ( + "experts.gate_up_proj", + "experts.3.down_proj.weight_packed", + "router.per_expert_scale", + ) + ] + + mapped = [ + (name, getattr(w, "shard_id", None)) + for name, w in Gemma4ForCausalLM.hf_to_vllm_mapper.apply(weights) + ] + + assert mapped == [ + ("model.layers.0.experts.gate_up_proj", None), + ("model.layers.0.experts.3.down_proj.weight_packed", None), + ("model.layers.0.router.per_expert_scale", None), + ] @pytest.mark.cpu_test diff --git a/tests/models/language/pooling/conftest.py b/tests/models/language/pooling/conftest.py index 0571f1cd6d..2da4c65133 100644 --- a/tests/models/language/pooling/conftest.py +++ b/tests/models/language/pooling/conftest.py @@ -2,12 +2,40 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Pytest configuration for vLLM language generation tests.""" +import gc + +import pytest import torch +from tests.utils import wait_for_memory_to_settle +from vllm.logger import init_logger from vllm.platforms import current_platform +logger = init_logger(__name__) + def pytest_sessionstart(session): """Configure ROCm-specific settings before test session starts.""" if current_platform.is_rocm(): torch.set_float32_matmul_precision("high") + + +@pytest.fixture(autouse=True) +def release_gpu_memory_between_tests(): + """Reclaim cached GPU memory between tests in this shard. + + These tests start many vLLM/HF engines sequentially in one pytest + process; without explicit reclamation between tests, VRAM fragmentation + accumulates and later tests fail at engine startup ("Engine core + initialization failed"). + """ + yield + gc.collect() + if torch.accelerator.is_available(): + torch.accelerator.empty_cache() + try: + wait_for_memory_to_settle() + except ValueError as e: + # Longer-lived (class/module-scoped) engine fixtures may legitimately + # still hold VRAM; nothing further to reclaim in that case. + logger.info("Failed to clean GPU memory: %s", e) diff --git a/tests/models/language/pooling/test_colbert.py b/tests/models/language/pooling/test_colbert.py index 75323d54bf..8829558a06 100644 --- a/tests/models/language/pooling/test_colbert.py +++ b/tests/models/language/pooling/test_colbert.py @@ -128,9 +128,12 @@ def _load_projection_weight(model_name: str, hf_spec: dict, device: torch.device """Download and return the ColBERT linear projection weight.""" from safetensors.torch import load_file - from vllm.transformers_utils.repo_utils import hf_api + from vllm.transformers_utils.repo_utils import hf_api, with_retry - path = hf_api().hf_hub_download(model_name, filename=hf_spec["weights_file"]) + path = with_retry( + lambda: hf_api().hf_hub_download(model_name, filename=hf_spec["weights_file"]), + f"Error downloading {hf_spec['weights_file']} from {model_name}", + ) weights = load_file(path) return weights[hf_spec["weights_key"]].to(device) diff --git a/tests/models/language/pooling/test_nomic_max_model_len.py b/tests/models/language/pooling/test_nomic_max_model_len.py index 6ea29f6d06..4ecbe77bc6 100644 --- a/tests/models/language/pooling/test_nomic_max_model_len.py +++ b/tests/models/language/pooling/test_nomic_max_model_len.py @@ -2,6 +2,8 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # ruff: noqa: SIM117 +from functools import partial + import pytest from ...utils import EmbedModelInfo @@ -26,6 +28,22 @@ original_max_position_embeddings = 2048 max_model_len = int(original_max_position_embeddings * factor) +@pytest.fixture(scope="module", autouse=True) +def predownload_models(): + """Warm the HF cache so engine startup never hits transient hub errors.""" + from vllm.transformers_utils.repo_utils import hf_api, with_retry + + for model_info in MODELS: + with_retry( + partial( + hf_api().snapshot_download, + model_info.name, + revision=model_info.revision, + ), + f"Error downloading {model_info.name}", + ) + + @pytest.mark.parametrize("model_info", MODELS) def test_default(model_info, vllm_runner): with vllm_runner( diff --git a/tests/models/language/pooling/test_token_classification.py b/tests/models/language/pooling/test_token_classification.py index d8d813f404..c205a3c42a 100644 --- a/tests/models/language/pooling/test_token_classification.py +++ b/tests/models/language/pooling/test_token_classification.py @@ -115,6 +115,7 @@ def test_modernbert_models( @pytest.mark.parametrize("model", ["Davlan/xlm-roberta-base-ner-hrl"]) @pytest.mark.parametrize("dtype", ["float"]) +@pytest.mark.flaky(reruns=3) @torch.inference_mode def test_xlm_roberta_models( hf_runner, @@ -197,6 +198,7 @@ def test_openai_privacy_filter( @pytest.mark.parametrize("model", ["bd2lcco/Qwen3-0.6B-finetuned"]) @pytest.mark.parametrize("dtype", ["float"]) +@pytest.mark.flaky(reruns=3) @torch.inference_mode def test_auto_conversion( hf_runner, diff --git a/tests/models/language/pooling_mteb_test/conftest.py b/tests/models/language/pooling_mteb_test/conftest.py new file mode 100644 index 0000000000..5a31a5f415 --- /dev/null +++ b/tests/models/language/pooling_mteb_test/conftest.py @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Pytest configuration for vLLM pooling MTEB tests.""" + +import gc + +import pytest +import torch + +from tests.utils import wait_for_memory_to_settle +from vllm.logger import init_logger + +logger = init_logger(__name__) + + +@pytest.fixture(autouse=True) +def release_gpu_memory_between_tests(): + """Reclaim cached GPU memory between tests in this shard. + + These tests start many vLLM/HF engines sequentially in one pytest + process; without explicit reclamation between tests, VRAM fragmentation + accumulates and later tests fail at engine startup ("Engine core + initialization failed"). + """ + yield + gc.collect() + if torch.accelerator.is_available(): + torch.accelerator.empty_cache() + try: + wait_for_memory_to_settle() + except ValueError as e: + # Longer-lived (class/module-scoped) engine fixtures may legitimately + # still hold VRAM; nothing further to reclaim in that case. + logger.info("Failed to clean GPU memory: %s", e) diff --git a/tests/models/language/pooling_mteb_test/mteb_embed_utils.py b/tests/models/language/pooling_mteb_test/mteb_embed_utils.py index b9deecff8d..0423e94d21 100644 --- a/tests/models/language/pooling_mteb_test/mteb_embed_utils.py +++ b/tests/models/language/pooling_mteb_test/mteb_embed_utils.py @@ -161,10 +161,14 @@ def mteb_test_embed_models( model_info: EmbedModelInfo, vllm_extra_kwargs=None, hf_model_callback=None, - atol=MTEB_EMBED_TOL, + atol: float | None = None, prompt_prefix: str | None = None, vllm_model_callback=None, ): + if atol is None: + atol = ( + model_info.mteb_tol if model_info.mteb_tol is not None else MTEB_EMBED_TOL + ) vllm_extra_kwargs = get_vllm_extra_kwargs(model_info, vllm_extra_kwargs) # Test embed_dims, isnan and whether to use normalize @@ -268,4 +272,8 @@ def mteb_test_embed_models( # We are not concerned that the vllm mteb results are better # than SentenceTransformers, so we only perform one-sided testing. - assert st_main_score - vllm_main_score < atol + diff = st_main_score - vllm_main_score + assert diff < atol, ( + f"diff={diff:.6g} tol={atol} model={model_info.name} " + f"(st={st_main_score}, vllm={vllm_main_score})" + ) diff --git a/tests/models/language/pooling_mteb_test/mteb_score_utils.py b/tests/models/language/pooling_mteb_test/mteb_score_utils.py index 16081cbe23..2d83a2ebaa 100644 --- a/tests/models/language/pooling_mteb_test/mteb_score_utils.py +++ b/tests/models/language/pooling_mteb_test/mteb_score_utils.py @@ -230,8 +230,12 @@ def mteb_test_rerank_models( hf_runner=HFMtebCrossEncoder, vllm_extra_kwargs=None, vllm_mteb_encoder=VllmMtebCrossEncoder, - atol=MTEB_RERANK_TOL, + atol: float | None = None, ): + if atol is None: + atol = ( + model_info.mteb_tol if model_info.mteb_tol is not None else MTEB_RERANK_TOL + ) vllm_extra_kwargs = get_vllm_extra_kwargs(model_info, vllm_extra_kwargs) # Maybe load chat_template. @@ -308,4 +312,8 @@ def mteb_test_rerank_models( # We are not concerned that the vllm mteb results are better # than SentenceTransformers, so we only perform one-sided testing. - assert st_main_score - vllm_main_score < atol + diff = st_main_score - vllm_main_score + assert diff < atol, ( + f"diff={diff:.6g} tol={atol} model={model_info.name} " + f"(st={st_main_score}, vllm={vllm_main_score})" + ) diff --git a/tests/models/language/pooling_mteb_test/test_baai.py b/tests/models/language/pooling_mteb_test/test_baai.py index ec11960fda..a3a4ffaef1 100644 --- a/tests/models/language/pooling_mteb_test/test_baai.py +++ b/tests/models/language/pooling_mteb_test/test_baai.py @@ -17,6 +17,7 @@ MODELS = [ "BAAI/bge-base-en", architecture="BertModel", mteb_score=0.779336792, + mteb_tol=2e-3, seq_pooling_type="CLS", attn_type="encoder_only", is_prefix_caching_supported=False, @@ -54,6 +55,7 @@ MODELS = [ "BAAI/bge-m3", architecture="XLMRobertaModel", mteb_score=0.787343078, + mteb_tol=2e-3, seq_pooling_type="CLS", attn_type="encoder_only", is_prefix_caching_supported=False, @@ -82,6 +84,7 @@ RERANK_MODELS = [ "BAAI/bge-reranker-base", architecture="XLMRobertaForSequenceClassification", mteb_score=0.32398, + mteb_tol=1e-2, seq_pooling_type="CLS", attn_type="encoder_only", is_prefix_caching_supported=False, @@ -101,6 +104,7 @@ RERANK_MODELS = [ ] +@pytest.mark.flaky(reruns=2) @pytest.mark.parametrize("model_info", MODELS) def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None: mteb_test_embed_models(hf_runner, vllm_runner, model_info) @@ -113,6 +117,7 @@ def test_embed_models_correctness( correctness_test_embed_models(hf_runner, vllm_runner, model_info, example_prompts) +@pytest.mark.flaky(reruns=2) @pytest.mark.parametrize("model_info", RERANK_MODELS) def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None: mteb_test_rerank_models(vllm_runner, model_info) diff --git a/tests/models/language/pooling_mteb_test/test_bge_reranker_v2_gemma.py b/tests/models/language/pooling_mteb_test/test_bge_reranker_v2_gemma.py index 23bc95548b..b750b622b9 100644 --- a/tests/models/language/pooling_mteb_test/test_bge_reranker_v2_gemma.py +++ b/tests/models/language/pooling_mteb_test/test_bge_reranker_v2_gemma.py @@ -26,6 +26,7 @@ RERANK_MODELS = [ "method": "no_post_processing", }, mteb_score=0.33757, + mteb_tol=1e-2, seq_pooling_type="LAST", attn_type="decoder", is_prefix_caching_supported=True, @@ -127,6 +128,7 @@ class GemmaRerankerHfRunner(MtebCrossEncoderMixin, HfRunner): return torch.Tensor(scores) +@pytest.mark.flaky(reruns=2) @pytest.mark.parametrize("model_info", RERANK_MODELS) def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None: mteb_test_rerank_models( diff --git a/tests/models/language/pooling_mteb_test/test_cross_encoder.py b/tests/models/language/pooling_mteb_test/test_cross_encoder.py index 0d1067d5e2..8f5d86fa8d 100644 --- a/tests/models/language/pooling_mteb_test/test_cross_encoder.py +++ b/tests/models/language/pooling_mteb_test/test_cross_encoder.py @@ -17,6 +17,7 @@ RERANK_MODELS = [ is_prefix_caching_supported=False, is_chunked_prefill_supported=False, mteb_score=0.32898, + mteb_tol=1e-2, ), RerankModelInfo( "tomaarsen/Qwen3-Reranker-0.6B-seq-cls", @@ -27,10 +28,12 @@ RERANK_MODELS = [ is_chunked_prefill_supported=True, chat_template_name="qwen3_reranker.jinja", mteb_score=0.33459, + mteb_tol=1e-2, ), ] +@pytest.mark.flaky(reruns=2) @pytest.mark.parametrize("model_info", RERANK_MODELS) def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None: mteb_test_rerank_models(vllm_runner, model_info) diff --git a/tests/models/language/pooling_mteb_test/test_gte.py b/tests/models/language/pooling_mteb_test/test_gte.py index 0a54262e12..c6b9ebc274 100644 --- a/tests/models/language/pooling_mteb_test/test_gte.py +++ b/tests/models/language/pooling_mteb_test/test_gte.py @@ -18,6 +18,7 @@ MODELS = [ EmbedModelInfo( "thenlper/gte-large", mteb_score=0.76807651, + mteb_tol=2e-3, architecture="BertModel", seq_pooling_type="MEAN", attn_type="encoder_only", @@ -44,6 +45,7 @@ MODELS = [ "Alibaba-NLP/gte-multilingual-base", architecture="GteNewModel", mteb_score=0.775074696, + mteb_tol=2e-3, hf_overrides={"architectures": ["GteNewModel"]}, seq_pooling_type="CLS", attn_type="encoder_only", @@ -79,6 +81,7 @@ MODELS = [ EmbedModelInfo( "Alibaba-NLP/gte-modernbert-base", mteb_score=0.748193353, + mteb_tol=2e-3, architecture="ModernBertModel", seq_pooling_type="CLS", attn_type="encoder_only", @@ -90,6 +93,7 @@ MODELS = [ EmbedModelInfo( "Qwen/Qwen3-Embedding-0.6B", mteb_score=0.771163695, + mteb_tol=2e-3, architecture="Qwen3ForCausalLM", seq_pooling_type="LAST", attn_type="decoder", @@ -109,6 +113,7 @@ RERANK_MODELS = [ # classifier_pooling: mean "Alibaba-NLP/gte-reranker-modernbert-base", mteb_score=0.33386, + mteb_tol=1e-2, architecture="ModernBertForSequenceClassification", seq_pooling_type="CLS", attn_type="encoder_only", @@ -119,6 +124,7 @@ RERANK_MODELS = [ RerankModelInfo( "Alibaba-NLP/gte-multilingual-reranker-base", mteb_score=0.33062, + mteb_tol=1e-2, architecture="GteNewForSequenceClassification", hf_overrides={"architectures": ["GteNewForSequenceClassification"]}, seq_pooling_type="CLS", @@ -130,6 +136,7 @@ RERANK_MODELS = [ ] +@pytest.mark.flaky(reruns=2) @pytest.mark.parametrize("model_info", MODELS) def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None: mteb_test_embed_models(hf_runner, vllm_runner, model_info) @@ -142,6 +149,7 @@ def test_embed_models_correctness( correctness_test_embed_models(hf_runner, vllm_runner, model_info, example_prompts) +@pytest.mark.flaky(reruns=2) @pytest.mark.parametrize("model_info", RERANK_MODELS) def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None: vllm_extra_kwargs = {} diff --git a/tests/models/language/pooling_mteb_test/test_intfloat.py b/tests/models/language/pooling_mteb_test/test_intfloat.py index adadb60eee..805a3f91ed 100644 --- a/tests/models/language/pooling_mteb_test/test_intfloat.py +++ b/tests/models/language/pooling_mteb_test/test_intfloat.py @@ -13,6 +13,7 @@ MODELS = [ "intfloat/e5-small", architecture="BertModel", mteb_score=0.742285423, + mteb_tol=2e-3, seq_pooling_type="MEAN", attn_type="encoder_only", is_prefix_caching_supported=False, @@ -29,6 +30,7 @@ MODELS = [ "intfloat/multilingual-e5-base", architecture="XLMRobertaModel", mteb_score=0.779325955, + mteb_tol=2e-3, seq_pooling_type="MEAN", attn_type="encoder_only", is_prefix_caching_supported=False, @@ -48,6 +50,7 @@ MODELS = [ ] +@pytest.mark.flaky(reruns=2) @pytest.mark.parametrize("model_info", MODELS) def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None: mteb_test_embed_models(hf_runner, vllm_runner, model_info) diff --git a/tests/models/language/pooling_mteb_test/test_jina.py b/tests/models/language/pooling_mteb_test/test_jina.py index 6c9308745e..753e612510 100644 --- a/tests/models/language/pooling_mteb_test/test_jina.py +++ b/tests/models/language/pooling_mteb_test/test_jina.py @@ -23,6 +23,7 @@ EMBEDDING_MODELS = [ EmbedModelInfo( "jinaai/jina-embeddings-v3", mteb_score=0.824413164, + mteb_tol=2e-3, architecture="XLMRobertaModel", is_matryoshka=True, seq_pooling_type="MEAN", @@ -33,6 +34,7 @@ EMBEDDING_MODELS = [ EmbedModelInfo( "jinaai/jina-embeddings-v5-text-small", mteb_score=0.794535707854956, + mteb_tol=2e-3, architecture="JinaEmbeddingsV5Model", seq_pooling_type="LAST", attn_type="decoder", @@ -42,6 +44,7 @@ EMBEDDING_MODELS = [ EmbedModelInfo( "jinaai/jina-embeddings-v5-text-nano", architecture="JinaEmbeddingsV5Model", + mteb_tol=2e-3, dtype="bfloat16" if current_platform.is_rocm() else "auto", seq_pooling_type="LAST", attn_type="encoder_only", @@ -54,6 +57,7 @@ RERANK_MODELS = [ RerankModelInfo( "jinaai/jina-reranker-v2-base-multilingual", mteb_score=0.33643, + mteb_tol=1e-2, architecture="XLMRobertaForSequenceClassification", seq_pooling_type="CLS", attn_type="encoder_only", @@ -63,6 +67,7 @@ RERANK_MODELS = [ ] +@pytest.mark.flaky(reruns=2) @pytest.mark.parametrize("model_info", EMBEDDING_MODELS) def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None: task = "retrieval" if "v5" in model_info.name else "text-matching" @@ -98,6 +103,7 @@ def test_embed_models_correctness( ) +@pytest.mark.flaky(reruns=2) @pytest.mark.parametrize("model_info", RERANK_MODELS) def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None: mteb_test_rerank_models(vllm_runner, model_info) diff --git a/tests/models/language/pooling_mteb_test/test_modernbert_fp8.py b/tests/models/language/pooling_mteb_test/test_modernbert_fp8.py index 8e062f0d76..f97057f11f 100644 --- a/tests/models/language/pooling_mteb_test/test_modernbert_fp8.py +++ b/tests/models/language/pooling_mteb_test/test_modernbert_fp8.py @@ -56,6 +56,7 @@ def _assert_modernbert_online_fp8(model) -> None: ) +@pytest.mark.flaky(reruns=2) @pytest.mark.skipif( _fp8_scaled_mm_unsupported(), reason="No FP8 ScaledMM kernel is available on this GPU type.", diff --git a/tests/models/language/pooling_mteb_test/test_mxbai_rerank.py b/tests/models/language/pooling_mteb_test/test_mxbai_rerank.py index 74fe760e78..1ad1f68b9a 100644 --- a/tests/models/language/pooling_mteb_test/test_mxbai_rerank.py +++ b/tests/models/language/pooling_mteb_test/test_mxbai_rerank.py @@ -30,6 +30,7 @@ RERANK_MODELS = [ is_chunked_prefill_supported=True, chat_template_name="mxbai_rerank_v2.jinja", mteb_score=0.33651, + mteb_tol=1e-2, enable_test=True, ), RerankModelInfo( @@ -104,6 +105,7 @@ class MxbaiRerankerHfRunner(MtebCrossEncoderMixin, HfRunner): return torch.Tensor(scores) +@pytest.mark.flaky(reruns=2) @pytest.mark.parametrize("model_info", RERANK_MODELS) def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None: mteb_test_rerank_models(vllm_runner, model_info, hf_runner=MxbaiRerankerHfRunner) diff --git a/tests/models/language/pooling_mteb_test/test_nemotron.py b/tests/models/language/pooling_mteb_test/test_nemotron.py index 79fae28339..711224a1a2 100644 --- a/tests/models/language/pooling_mteb_test/test_nemotron.py +++ b/tests/models/language/pooling_mteb_test/test_nemotron.py @@ -19,6 +19,7 @@ EMBEDDING_MODELS = [ "nvidia/llama-nemotron-embed-1b-v2", architecture="LlamaBidirectionalModel", mteb_score=0.689164662128673, + mteb_tol=2e-3, seq_pooling_type="MEAN", attn_type="encoder_only", is_prefix_caching_supported=False, @@ -32,6 +33,7 @@ RERANK_MODELS = [ architecture="LlamaBidirectionalForSequenceClassification", chat_template_name="nemotron-rerank.jinja", mteb_score=0.33994, + mteb_tol=1e-2, seq_pooling_type="MEAN", attn_type="encoder_only", is_prefix_caching_supported=False, @@ -40,11 +42,13 @@ RERANK_MODELS = [ ] +@pytest.mark.flaky(reruns=2) @pytest.mark.parametrize("model_info", EMBEDDING_MODELS) def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None: mteb_test_embed_models(hf_runner, vllm_runner, model_info) +@pytest.mark.flaky(reruns=2) @pytest.mark.parametrize("model_info", RERANK_MODELS) def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None: mteb_test_rerank_models(vllm_runner, model_info) diff --git a/tests/models/language/pooling_mteb_test/test_nomic.py b/tests/models/language/pooling_mteb_test/test_nomic.py index d7947f73f9..38e34b78b0 100644 --- a/tests/models/language/pooling_mteb_test/test_nomic.py +++ b/tests/models/language/pooling_mteb_test/test_nomic.py @@ -17,6 +17,7 @@ MODELS = [ # HF version and remove revision set. revision="720244025c1a7e15661a174c63cce63c8218e52b", mteb_score=0.737568559, + mteb_tol=2e-3, enable_test=True, seq_pooling_type="MEAN", attn_type="encoder_only", @@ -35,6 +36,7 @@ MODELS = [ "nomic-ai/nomic-embed-text-v2-moe", architecture="NomicBertModel", mteb_score=0.715488912, + mteb_tol=2e-3, enable_test=True, seq_pooling_type="MEAN", attn_type="encoder_only", @@ -44,6 +46,7 @@ MODELS = [ ] +@pytest.mark.flaky(reruns=2) @pytest.mark.parametrize("model_info", MODELS) def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None: mteb_test_embed_models(hf_runner, vllm_runner, model_info) diff --git a/tests/models/language/pooling_mteb_test/test_qwen3_reranker.py b/tests/models/language/pooling_mteb_test/test_qwen3_reranker.py index 3c182cb046..10c90e733e 100644 --- a/tests/models/language/pooling_mteb_test/test_qwen3_reranker.py +++ b/tests/models/language/pooling_mteb_test/test_qwen3_reranker.py @@ -32,6 +32,7 @@ RERANK_MODELS = [ is_prefix_caching_supported=True, is_chunked_prefill_supported=True, mteb_score=0.33459, + mteb_tol=1e-2, enable_test=True, ), RerankModelInfo( @@ -108,11 +109,13 @@ class Qwen3RerankerHfRunner(MtebCrossEncoderMixin, HfRunner): return torch.Tensor(scores) +@pytest.mark.flaky(reruns=2) @pytest.mark.parametrize("model_info", RERANK_MODELS) def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None: mteb_test_rerank_models(vllm_runner, model_info, hf_runner=Qwen3RerankerHfRunner) +@pytest.mark.flaky(reruns=2) @pytest.mark.parametrize("model_info", RERANK_MODELS) @multi_gpu_test(num_gpus=2) def test_rerank_models_mteb_tp(vllm_runner, model_info: RerankModelInfo) -> None: diff --git a/tests/models/language/pooling_mteb_test/test_snowflake_arctic_embed.py b/tests/models/language/pooling_mteb_test/test_snowflake_arctic_embed.py index f3afbe84fa..b68b2778b6 100644 --- a/tests/models/language/pooling_mteb_test/test_snowflake_arctic_embed.py +++ b/tests/models/language/pooling_mteb_test/test_snowflake_arctic_embed.py @@ -14,6 +14,7 @@ MODELS = [ is_matryoshka=False, architecture="BertModel", mteb_score=0.714927797, + mteb_tol=2e-3, seq_pooling_type="CLS", attn_type="encoder_only", is_prefix_caching_supported=False, @@ -37,6 +38,7 @@ MODELS = [ is_matryoshka=False, architecture="NomicBertModel", mteb_score=0.681146831, + mteb_tol=2e-3, seq_pooling_type="CLS", attn_type="encoder_only", is_prefix_caching_supported=False, @@ -54,6 +56,7 @@ MODELS = [ is_matryoshka=True, architecture="BertModel", mteb_score=0.649088363, + mteb_tol=2e-3, seq_pooling_type="CLS", attn_type="encoder_only", is_prefix_caching_supported=False, @@ -65,6 +68,7 @@ MODELS = [ is_matryoshka=True, architecture="XLMRobertaModel", mteb_score=0.712258299, + mteb_tol=2e-3, seq_pooling_type="CLS", attn_type="encoder_only", is_prefix_caching_supported=False, @@ -76,6 +80,7 @@ MODELS = [ is_matryoshka=True, architecture="GteModel", mteb_score=0.706622444, + mteb_tol=2e-3, seq_pooling_type="CLS", attn_type="encoder_only", is_prefix_caching_supported=False, @@ -85,6 +90,7 @@ MODELS = [ ] +@pytest.mark.flaky(reruns=2) @pytest.mark.parametrize("model_info", MODELS) def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None: mteb_test_embed_models(hf_runner, vllm_runner, model_info) diff --git a/tests/models/language/pooling_mteb_test/test_st_projector.py b/tests/models/language/pooling_mteb_test/test_st_projector.py index 395846347f..7397db8535 100644 --- a/tests/models/language/pooling_mteb_test/test_st_projector.py +++ b/tests/models/language/pooling_mteb_test/test_st_projector.py @@ -14,6 +14,7 @@ ST_PROJECTOR_MODELS = [ "TencentBAC/Conan-embedding-v1", architecture="BertModel", mteb_score=0.688611955, + mteb_tol=2e-3, seq_pooling_type="MEAN", attn_type="encoder_only", is_prefix_caching_supported=False, @@ -24,6 +25,7 @@ ST_PROJECTOR_MODELS = [ "google/embeddinggemma-300m", architecture="Gemma3TextModel", mteb_score=0.7473819294684156, + mteb_tol=2e-3, seq_pooling_type="MEAN", attn_type="encoder_only", is_prefix_caching_supported=False, @@ -33,6 +35,7 @@ ST_PROJECTOR_MODELS = [ ] +@pytest.mark.flaky(reruns=2) @pytest.mark.parametrize("model_info", ST_PROJECTOR_MODELS) def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None: mteb_test_embed_models(hf_runner, vllm_runner, model_info) diff --git a/tests/models/language/pooling_mteb_test/test_voyage.py b/tests/models/language/pooling_mteb_test/test_voyage.py index 99ef1de9ad..628ea31ca0 100644 --- a/tests/models/language/pooling_mteb_test/test_voyage.py +++ b/tests/models/language/pooling_mteb_test/test_voyage.py @@ -22,6 +22,7 @@ MODELS = [ "num_labels": 2048, }, mteb_score=0.7054, + mteb_tol=2e-3, # === MTEB Results === # STS12: 0.6613 # STS13: 0.6906 @@ -34,6 +35,7 @@ MODELS = [ ] +@pytest.mark.flaky(reruns=2) @pytest.mark.parametrize("model_info", MODELS) def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None: # Encoder-only attention models need enforce_eager=True to avoid diff --git a/tests/models/multimodal/generation/test_common.py b/tests/models/multimodal/generation/test_common.py index c364bab382..4f68887d1d 100644 --- a/tests/models/multimodal/generation/test_common.py +++ b/tests/models/multimodal/generation/test_common.py @@ -317,8 +317,6 @@ VLM_TEST_SETTINGS = { img_idx_to_prompt=lambda idx: "", auto_cls=AutoModelForImageTextToText, vllm_output_post_proc=model_utils.blip2_vllm_to_hf_output, - # FIXME: https://github.com/huggingface/transformers/pull/38510 - marks=[pytest.mark.skip("Model is broken")], ), "cosmos3": VLMTestInfo( models=["nvidia/Cosmos3-Nano"], @@ -819,10 +817,6 @@ VLM_TEST_SETTINGS = { patch_hf_runner=model_utils.paddleocr_vl_patch_hf_runner, image_size_factors=[(0.25,)], marks=[ - pytest.mark.skipif( - Version(TRANSFORMERS_VERSION) == Version("4.57.3"), - reason="This model is broken in Transformers v4.57.3", - ), pytest.mark.skipif( Version(TRANSFORMERS_VERSION) >= Version("5.0.0"), reason="Model's custom code uses ROPE_INIT_FUNCTIONS" diff --git a/tests/models/multimodal/generation/test_vit_cudagraph.py b/tests/models/multimodal/generation/test_vit_cudagraph.py index f8136ae8f3..1ed84ebc52 100644 --- a/tests/models/multimodal/generation/test_vit_cudagraph.py +++ b/tests/models/multimodal/generation/test_vit_cudagraph.py @@ -499,3 +499,118 @@ def test_vit_cudagraph_video(model_id, vllm_runner, video_assets): # Ensure the output is a string assert isinstance(output_text, str) + + +def _check_eonly_encoder_outputs(worker, batches): + """Compare real E-only replay with eager, retaining outputs across replay.""" + import torch + + from vllm.v1.worker.mm_encoder_model_runner import MMEncoderModelRunner + + runner = worker.model_runner + assert isinstance(runner, MMEncoderModelRunner) + manager = runner.model_state.encoder_runner.cudagraph_manager + assert manager is not None and manager.is_captured() + retained: list[tuple[torch.Tensor, torch.Tensor]] = [] + max_error = 0.0 + stats = [] + with torch.inference_mode(): + for batch in batches: + kwargs = {key: value.to(runner.device) for key, value in batch.items()} + actual = manager.execute(kwargs) + expected = runner.model.embed_multimodal(**kwargs) + assert len(actual) == len(expected) + for output, reference in zip(actual, expected): + assert torch.isfinite(output).all() + # Two BF16 ulps at unit scale, fixed before model validation. + torch.testing.assert_close(output, reference, rtol=0.016, atol=0.016) + max_error = max(max_error, (output - reference).abs().max().item()) + for output, snapshot in retained: + torch.testing.assert_close(output, snapshot, rtol=0, atol=0) + retained.extend((output, output.clone()) for output in actual) + stats.append(manager.get_cumulative_stats()) + assert stats[0]["graph_hits"] > 0 + assert stats[1]["graph_hits"] > stats[0]["graph_hits"] + assert stats[2]["graph_misses"] > stats[1]["graph_misses"] + assert stats[3]["graph_hits"] > stats[2]["graph_hits"] + assert stats[4]["graph_hits"] > stats[3]["graph_hits"] + assert stats[5]["graph_hits"] > stats[4]["graph_hits"] + assert stats[6]["graph_hits"] > stats[5]["graph_hits"] + return {"max_abs_error": max_error, "stats": stats} + + +@pytest.mark.parametrize( + "model_id", + params_with_marks({key: MODEL_CONFIGS[key] for key in ("qwen3_vl", "qwen3_5")}), +) +@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA graphs") +def test_eonly_vit_cudagraph_outputs( + model_id, vllm_runner, image_assets, video_assets, monkeypatch +): + """Exercise startup capture, image/video replay, fallback and output ownership.""" + import numpy as np + from PIL import Image + from transformers import AutoProcessor + + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") + monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") + config = MODEL_CONFIGS[model_id] + token_budgets = [64, 256, 2048] + processor = AutoProcessor.from_pretrained(config.model) + first, second = [asset.pil_image for asset in image_assets] + batches = [ + [first.resize((224, 224))], + [second.resize((448, 224)), first.resize((224, 224))], + [second.resize((1792, 1792))], + [second.resize((224, 224))], + [first.resize((1280, 720))], + [first.resize((224, 224))], + ] + frames = sample_frames_from_video(video_assets[0].np_ndarrays, 2) + video = np.stack( + [np.asarray(Image.fromarray(frame).resize((224, 224))) for frame in frames] + ) + with vllm_runner( + config.model, + dtype=config.dtype, + mm_encoder_only=True, + # FA padding NaNs are tracked separately in #57136. + mm_encoder_attn_backend="FLASHINFER", + enable_prefix_caching=False, + max_model_len=4096, + max_num_seqs=2, + limit_mm_per_prompt={"image": 2, "video": 1}, + compilation_config={ + "cudagraph_mode": "NONE", + "cudagraph_mm_encoder": True, + "encoder_cudagraph_token_budgets": token_budgets, + "encoder_cudagraph_max_vision_items_per_batch": 2, + "encoder_cudagraph_max_frames_per_batch": 2, + }, + ) as model: + mm_config = model.llm.llm_engine.vllm_config.model_config.multimodal_config + processor_kwargs = mm_config.merge_mm_processor_kwargs({}) + inputs = [ + dict( + processor.image_processor( + images=images, return_tensors="pt", **processor_kwargs + ) + ) + for images in batches + ] + inputs.append( + dict( + processor.video_processor( + videos=[video], + do_sample_frames=False, + return_tensors="pt", + **processor_kwargs, + ) + ) + ) + results = model.llm.collective_rpc( + partial(_check_eonly_encoder_outputs, batches=inputs) + ) + for result in results: + assert result["stats"][0]["token_budgets"] == token_budgets + print(f"E-only {model_id}: {results}") diff --git a/tests/models/multimodal/generation/vlm_utils/model_utils.py b/tests/models/multimodal/generation/vlm_utils/model_utils.py index 77beb64601..ae3f376e1f 100644 --- a/tests/models/multimodal/generation/vlm_utils/model_utils.py +++ b/tests/models/multimodal/generation/vlm_utils/model_utils.py @@ -308,16 +308,6 @@ def gemma3_patch_hf_runner(hf_model: HfRunner) -> HfRunner: hf_model.processor = processor - orig_generate = hf_model.model.generate - - def _generate(self, *args, **kwargs): - # FIXME: https://github.com/huggingface/transformers/issues/38333 - kwargs["disable_compile"] = True - - return orig_generate(*args, **kwargs) - - hf_model.model.generate = types.MethodType(_generate, hf_model.model) - return hf_model diff --git a/tests/models/multimodal/pooling/conftest.py b/tests/models/multimodal/pooling/conftest.py new file mode 100644 index 0000000000..32f598b30a --- /dev/null +++ b/tests/models/multimodal/pooling/conftest.py @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Pytest configuration for vLLM multimodal pooling tests.""" + +import gc + +import pytest +import torch + +from tests.utils import wait_for_memory_to_settle +from vllm.logger import init_logger + +logger = init_logger(__name__) + + +@pytest.fixture(autouse=True) +def release_gpu_memory_between_tests(): + """Reclaim cached GPU memory between tests in this shard. + + These tests start many vLLM/HF engines sequentially in one pytest + process; without explicit reclamation between tests, VRAM fragmentation + accumulates and later tests fail at engine startup ("Engine core + initialization failed"). + """ + yield + gc.collect() + if torch.accelerator.is_available(): + torch.accelerator.empty_cache() + try: + wait_for_memory_to_settle() + except ValueError as e: + # Longer-lived (class/module-scoped) engine fixtures may legitimately + # still hold VRAM; nothing further to reclaim in that case. + logger.info("Failed to clean GPU memory: %s", e) diff --git a/tests/models/multimodal/pooling/test_jinavl_reranker.py b/tests/models/multimodal/pooling/test_jinavl_reranker.py index c71a2f08cb..05433e7269 100644 --- a/tests/models/multimodal/pooling/test_jinavl_reranker.py +++ b/tests/models/multimodal/pooling/test_jinavl_reranker.py @@ -2,8 +2,6 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import pytest -import transformers -from packaging import version from transformers import AutoModel from vllm.assets.base import VLLM_S3_BUCKET_URL @@ -246,10 +244,6 @@ def _run_test( @pytest.mark.parametrize("model", MODELS) @pytest.mark.parametrize("dtype", ["half"]) -@pytest.mark.skipif( - version.parse(transformers.__version__) == version.parse("4.57.5"), - reason="Skipped for transformers==4.57.5, https://github.com/huggingface/transformers/issues/43295", -) def test_model_text_image( hf_runner, vllm_runner, @@ -269,10 +263,6 @@ def test_model_text_image( @pytest.mark.parametrize("model", MODELS) @pytest.mark.parametrize("dtype", ["half"]) -@pytest.mark.skipif( - version.parse(transformers.__version__) == version.parse("4.57.5"), - reason="Skipped for transformers==4.57.5, https://github.com/huggingface/transformers/issues/43295", -) def test_model_text_text( hf_runner, vllm_runner, @@ -292,10 +282,6 @@ def test_model_text_text( @pytest.mark.parametrize("model", MODELS) @pytest.mark.parametrize("dtype", ["half"]) -@pytest.mark.skipif( - version.parse(transformers.__version__) == version.parse("4.57.5"), - reason="Skipped for transformers==4.57.5, https://github.com/huggingface/transformers/issues/43295", -) def test_model_image_text( hf_runner, vllm_runner, @@ -315,10 +301,6 @@ def test_model_image_text( @pytest.mark.parametrize("model", MODELS) @pytest.mark.parametrize("dtype", ["half"]) -@pytest.mark.skipif( - version.parse(transformers.__version__) == version.parse("4.57.5"), - reason="Skipped for transformers==4.57.5, https://github.com/huggingface/transformers/issues/43295", -) def test_model_image_image( hf_runner, vllm_runner, @@ -338,10 +320,6 @@ def test_model_image_image( @pytest.mark.parametrize("model", MODELS) @pytest.mark.parametrize("dtype", ["half"]) -@pytest.mark.skipif( - version.parse(transformers.__version__) == version.parse("4.57.5"), - reason="Skipped for transformers==4.57.5, https://github.com/huggingface/transformers/issues/43295", -) def test_model_text_mixed_documents( hf_runner, vllm_runner, diff --git a/tests/models/multimodal/processing/test_audio_in_video.py b/tests/models/multimodal/processing/test_audio_in_video.py index f696d7a561..acb102f60f 100644 --- a/tests/models/multimodal/processing/test_audio_in_video.py +++ b/tests/models/multimodal/processing/test_audio_in_video.py @@ -63,36 +63,32 @@ def test_audio_in_video_cache_correctness(model_id: str, num_videos: int) -> Non mm_processor_cache_gb=1, ) - # Baseline: no cache, always processes from scratch. - baseline_processor = MULTIMODAL_REGISTRY.create_processor( - ctx.model_config, cache=None - ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + # Sender cache: on a cache hit returns (None, prompt_updates) for each # item, setting mm_kwargs["video"] = [None] – the exact condition that # triggered the original bug. sender_cache = MultiModalProcessorSenderCache(ctx.model_config) - cached_processor = MULTIMODAL_REGISTRY.create_processor( - ctx.model_config, cache=sender_cache - ) - video_token_id = baseline_processor.info.get_hf_config().video_token_id + video_token_id = processor.info.get_hf_config().video_token_id mm_data = create_mm_data(num_videos) hf_processor_mm_kwargs = {"use_audio_in_video": True} - def run(processor): + def run(processor, cache): return processor( [video_token_id] * num_videos, - mm_items=baseline_processor.info.parse_mm_data(mm_data), + mm_items=processor.info.parse_mm_data(mm_data), hf_processor_mm_kwargs=hf_processor_mm_kwargs, + cache=cache, )["prompt_token_ids"] - baseline_ids = run(baseline_processor) + baseline_ids = run(processor, cache=None) # First call on the sender-cache processor: cache miss. # mm_kwargs["video"] items are real tensors; use_audio_in_video is # detected normally from the item data. - first_ids = run(cached_processor) + first_ids = run(processor, cache=sender_cache) assert first_ids == baseline_ids, ( "Cache-miss call produced different prompt_token_ids than baseline.\n" f" baseline : {baseline_ids}\n" @@ -103,7 +99,7 @@ def test_audio_in_video_cache_correctness(model_id: str, num_videos: int) -> Non # MultiModalProcessorSenderCache.get_and_update_item returns (None, …), # so mm_kwargs["video"] = [None]. Before the fix, use_audio_in_video was # not detected, yielding wrong token ids. - second_ids = run(cached_processor) + second_ids = run(processor, cache=sender_cache) assert second_ids == baseline_ids, ( "Cache-hit call produced different prompt_token_ids than baseline.\n" "This is the regression introduced when use_audio_in_video detection\n" @@ -123,7 +119,7 @@ def test_use_audio_in_video_without_audio_track(model_id: str) -> None: model_id, limit_mm_per_prompt={"audio": 1, "image": 0, "video": 1}, ) - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config, cache=None) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) video_token_id = processor.info.get_hf_config().video_token_id rng = np.random.RandomState(0) diff --git a/tests/models/multimodal/processing/test_audioflamingo3.py b/tests/models/multimodal/processing/test_audioflamingo3.py index f5920a3be4..f223145d98 100644 --- a/tests/models/multimodal/processing/test_audioflamingo3.py +++ b/tests/models/multimodal/processing/test_audioflamingo3.py @@ -25,6 +25,7 @@ import torch from transformers import PretrainedConfig from tests.models.registry import HF_EXAMPLE_MODELS +from vllm.config.multimodal import MultiModalDummyOptions class MockAudioFlamingo3Config(PretrainedConfig): @@ -127,7 +128,7 @@ def test_dummy_data_generation(mock_ctx): builder = AudioFlamingo3DummyInputsBuilder(info) mm_counts = {"audio": 2} - dummy_data = builder.get_dummy_mm_data(100, mm_counts, {}) + dummy_data = builder.get_dummy_mm_data(100, mm_counts, MultiModalDummyOptions()) assert "audio" in dummy_data assert len(dummy_data["audio"]) == 2 diff --git a/tests/models/multimodal/processing/test_common.py b/tests/models/multimodal/processing/test_common.py index 1fbb2149a6..db6f37dc3d 100644 --- a/tests/models/multimodal/processing/test_common.py +++ b/tests/models/multimodal/processing/test_common.py @@ -13,6 +13,7 @@ from vllm.config.multimodal import ( AudioDummyOptions, BaseDummyOptions, ImageDummyOptions, + MultiModalDummyOptions, VideoDummyOptions, ) from vllm.inputs import MultiModalDataDict, MultiModalInput @@ -71,6 +72,8 @@ def glmasr_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict: _IGNORE_MM_KEYS = { + # Dithering causes minor divergence + "cohere_asr": {"input_features"}, # In Ultravox, the audio_features can be different depending on padding # The slight difference should not be a problem though, since # attention_mask lets us ignore the difference. @@ -174,7 +177,7 @@ def get_token_prompt( inputs = dummy_inputs.get_dummy_processor_inputs( model_config.max_model_len, mm_counts, - mm_options={}, + mm_options=MultiModalDummyOptions(), # Assume all Mistral models define this extra argument mm_data=mm_data, # type: ignore[call-arg] ) @@ -182,7 +185,7 @@ def get_token_prompt( inputs = dummy_inputs.get_dummy_processor_inputs( model_config.max_model_len, mm_counts, - mm_options={}, + mm_options=MultiModalDummyOptions(), ) if not isinstance(inputs.prompt, list): @@ -278,13 +281,14 @@ def _test_processing_correctness( return BaseDummyOptions(count=count) # Assign normalized DummyOptions to the model config - model_config.get_multimodal_config().limit_per_prompt = { - modality: _to_dummy_options(modality, count) - for modality, count in limit_mm_per_prompt_ints.items() - } + model_config.get_multimodal_config().limit_per_prompt = MultiModalDummyOptions( + { + modality: _to_dummy_options(modality, count) + for modality, count in limit_mm_per_prompt_ints.items() + } + ) - baseline_processor = factories.build_processor(ctx, cache=None) - cached_processor = factories.build_processor(ctx, cache=cache) + processor = factories.build_processor(ctx) rng = np.random.RandomState(0) @@ -333,41 +337,42 @@ def _test_processing_correctness( _test_processing_correctness_one( model_config, mm_data, - baseline_processor, - cached_processor, + processor, batch_idx, hit_rate, num_batches, simplify_rate, + cache=cache, ) def _test_processing_correctness_one( model_config: ModelConfig, mm_data: MultiModalDataDict, - baseline_processor: BaseMultiModalProcessor, - cached_processor: BaseMultiModalProcessor, + processor: BaseMultiModalProcessor, batch_idx: int, hit_rate: float, num_batches: int, simplify_rate: float, + cache: MultiModalProcessorOnlyCache, ): model_type = model_config.hf_config.model_type - token_prompt = get_token_prompt(baseline_processor, mm_data) - mm_items = baseline_processor.info.parse_mm_data(mm_data) + token_prompt = get_token_prompt(processor, mm_data) + mm_items = processor.info.parse_mm_data(mm_data) ignore_mm_keys = _IGNORE_MM_KEYS.get(model_type, set[str]()) - baseline_tokenized_result = baseline_processor( + baseline_tokenized_result = processor( token_prompt, mm_items=mm_items, hf_processor_mm_kwargs={}, ) - cached_tokenized_result = cached_processor( + cached_tokenized_result = processor( token_prompt, mm_items=mm_items, hf_processor_mm_kwargs={}, + cache=cache, ) _assert_inputs_equal( diff --git a/tests/models/multimodal/processing/test_gemma4.py b/tests/models/multimodal/processing/test_gemma4.py index a28cc50be5..6430f40d8b 100644 --- a/tests/models/multimodal/processing/test_gemma4.py +++ b/tests/models/multimodal/processing/test_gemma4.py @@ -221,7 +221,7 @@ def test_video_cache_is_independent_of_image_kwargs( mm_processor_cache_gb=1, ) cache = MultiModalProcessorOnlyCache(ctx.model_config) - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config, cache=cache) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) hf_processor = processor.info.get_hf_processor() image = PILImage.new("RGB", (48, 48), color=(128, 128, 128)) frames = np.stack([np.asarray(image)] * 2) @@ -230,15 +230,16 @@ def test_video_cache_is_independent_of_image_kwargs( {"image": image, "video": [(frames, metadata)]} ) - def process(mm_kwargs): + def process(mm_kwargs, cache): return processor( hf_processor.image_token + hf_processor.video_token, mm_items, mm_uuid_items={"video": [video_uuid]}, hf_processor_mm_kwargs=mm_kwargs, + cache=cache, ) - baseline = process({}) + baseline = process({}, cache=None) if kwargs_on_init: ctx = build_model_context( model_id, @@ -247,12 +248,12 @@ def test_video_cache_is_independent_of_image_kwargs( mm_processor_cache_gb=1, ) cache = MultiModalProcessorOnlyCache(ctx.model_config) - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config, cache=cache) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) request_kwargs = {} if kwargs_on_init else kwargs - process(request_kwargs) - cached = process(request_kwargs) + process(request_kwargs, cache=cache) + cached = process(request_kwargs, cache=cache) cache.clear_cache() - fresh = process(request_kwargs) + fresh = process(request_kwargs, cache=cache) def pixels(result, modality, field): return result["mm_kwargs"][modality][0][field].data diff --git a/tests/models/multimodal/processing/test_idefics3.py b/tests/models/multimodal/processing/test_idefics3.py index 7365db59f2..feffbf6568 100644 --- a/tests/models/multimodal/processing/test_idefics3.py +++ b/tests/models/multimodal/processing/test_idefics3.py @@ -3,9 +3,7 @@ """Tests for Idefics3's multimodal preprocessing kwargs.""" import pytest -from packaging.version import Version from transformers import Idefics3Config -from transformers import __version__ as TRANSFORMERS_VERSION from vllm.multimodal import MULTIMODAL_REGISTRY @@ -13,10 +11,6 @@ from ....conftest import ImageTestAssets from ...utils import build_model_context -@pytest.mark.skipif( - Version(TRANSFORMERS_VERSION) < Version("5.2.0"), - reason="See https://github.com/huggingface/transformers/pull/43948", -) @pytest.mark.parametrize("model_id", ["HuggingFaceM4/Idefics3-8B-Llama3"]) @pytest.mark.parametrize( ("mm_processor_kwargs", "expected_toks_per_img"), diff --git a/tests/models/multimodal/processing/test_mllama4.py b/tests/models/multimodal/processing/test_mllama4.py index e5ba6ae241..6cf946eff2 100644 --- a/tests/models/multimodal/processing/test_mllama4.py +++ b/tests/models/multimodal/processing/test_mllama4.py @@ -23,11 +23,9 @@ def test_profiling(model_id: str, max_model_len: int): model_config_kwargs=model_config_kwargs, limit_mm_per_prompt=mm_counts, ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) - mm_inputs = MULTIMODAL_REGISTRY.get_dummy_mm_inputs( - ctx.model_config, - mm_counts=mm_counts, - ) + mm_inputs = processor.get_dummy_mm_inputs(mm_counts) hf_config = ctx.get_hf_config(Llama4Config) image_size = hf_config.vision_config.image_size diff --git a/tests/models/multimodal/processing/test_molmo2.py b/tests/models/multimodal/processing/test_molmo2.py index 5633d47a97..68d3c8163c 100644 --- a/tests/models/multimodal/processing/test_molmo2.py +++ b/tests/models/multimodal/processing/test_molmo2.py @@ -6,7 +6,7 @@ from types import SimpleNamespace import pytest import torch -from vllm.config.multimodal import VideoDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions, VideoDummyOptions from vllm.model_executor.models.molmo2 import ( Molmo2DummyInputsBuilder, build_flat_image_bool_length, @@ -83,7 +83,9 @@ def test_dummy_video_num_frames_override_honors_min_of_two( data = builder.get_dummy_mm_data( seq_len=128, mm_counts={"image": 0, "video": 1}, - mm_options={"video": VideoDummyOptions(num_frames=num_frames_override)}, + mm_options=MultiModalDummyOptions( + video=VideoDummyOptions(num_frames=num_frames_override) + ), ) video, _metadata = data["video"][0] diff --git a/tests/models/multimodal/processing/test_moss_audio.py b/tests/models/multimodal/processing/test_moss_audio.py index 87e7079a81..17863b791c 100644 --- a/tests/models/multimodal/processing/test_moss_audio.py +++ b/tests/models/multimodal/processing/test_moss_audio.py @@ -168,7 +168,6 @@ def _build_moss_audio_processor(cache=None): MossAudioMultiModalProcessor( info, MossAudioDummyInputsBuilder(info), - cache=cache, ), ctx, ) @@ -259,21 +258,24 @@ def test_moss_audio_multimodal_processor_handles_token_and_cache_paths(): ) cache = MultiModalProcessorOnlyCache(ctx.model_config) - cached_processor, _ = _build_moss_audio_processor(cache=cache) + cached_processor, _ = _build_moss_audio_processor() cached_text_miss = cached_processor( prompt, mm_items=mm_items, hf_processor_mm_kwargs={}, + cache=cache, ) cached_text_hit = cached_processor( prompt, mm_items=mm_items, hf_processor_mm_kwargs={}, + cache=cache, ) cached_token_hit = cached_processor( token_prompt, mm_items=mm_items, hf_processor_mm_kwargs={}, + cache=cache, ) expected_audio_tokens = MossAudioEncoder.compute_num_audio_tokens(raw_mel_len) diff --git a/tests/models/multimodal/processing/test_qwen2_vl.py b/tests/models/multimodal/processing/test_qwen2_vl.py index 4976b05c3e..963b5959ca 100644 --- a/tests/models/multimodal/processing/test_qwen2_vl.py +++ b/tests/models/multimodal/processing/test_qwen2_vl.py @@ -3,9 +3,7 @@ import pytest import torch -from packaging.version import Version from PIL import Image -from transformers import __version__ as TRANSFORMERS_VERSION from vllm.model_executor.models.vision import FusedInputNorm from vllm.multimodal import MULTIMODAL_REGISTRY @@ -28,7 +26,6 @@ def test_jina_vl_processing_order() -> None: processor = MULTIMODAL_REGISTRY.create_processor( ctx.model_config, tokenizer=ctx.tokenizer, - cache=cache, ) placeholder = "<|vision_start|><|image_pad|><|vision_end|>" @@ -39,6 +36,7 @@ def test_jina_vl_processing_order() -> None: return processor( placeholder * len(images), mm_items=processor.info.parse_mm_data({"image": images}), + cache=cache, ) query = process([query_image]) @@ -133,12 +131,6 @@ def test_processor_override( kwargs_on_init: bool, ): """Ensure Qwen2VLMultiModalProcessor handles min/max pixels properly.""" - if ( - Version(TRANSFORMERS_VERSION) < Version("5.2.0") - and "size" in mm_processor_kwargs - ): - pytest.skip("`size` ignored by `Qwen2VLProcessor.__call__`") - ctx = build_model_context( model_id, mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None, @@ -184,12 +176,6 @@ def test_get_image_size_with_most_features( model_id: str, mm_processor_kwargs: dict[str, object], ): - if ( - Version(TRANSFORMERS_VERSION) < Version("5.2.0") - and "size" in mm_processor_kwargs - ): - pytest.skip("`size` ignored by `Qwen2VLProcessor.__call__`") - ctx = build_model_context( model_id, mm_processor_kwargs=mm_processor_kwargs, diff --git a/tests/models/multimodal/processing/test_qwen3_vl.py b/tests/models/multimodal/processing/test_qwen3_vl.py index c58904a9e2..04f87049c9 100644 --- a/tests/models/multimodal/processing/test_qwen3_vl.py +++ b/tests/models/multimodal/processing/test_qwen3_vl.py @@ -12,6 +12,7 @@ import numpy as np import pytest from vllm.config import ModelConfig +from vllm.config.multimodal import MultiModalDummyOptions from vllm.multimodal import MULTIMODAL_REGISTRY from ...registry import HF_EXAMPLE_MODELS @@ -339,7 +340,9 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non limit_mm_per_prompt={"image": 0, "video": 1}, ) capped = MULTIMODAL_REGISTRY.create_processor(capped_ctx.model_config) - capped_dummy = capped.dummy_inputs.get_dummy_mm_data(1024, {"video": 1}, {}) + capped_dummy = capped.dummy_inputs.get_dummy_mm_data( + 1024, {"video": 1}, MultiModalDummyOptions() + ) capped_frames = capped_dummy["video"][0][0].shape[0] assert capped_frames == 16, ( f"Expected the dummy to spread the budget over 16 frames, got {capped_frames}" @@ -351,7 +354,9 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non limit_mm_per_prompt={"image": 0, "video": 1}, ) uncapped = MULTIMODAL_REGISTRY.create_processor(uncapped_ctx.model_config) - uncapped_dummy = uncapped.dummy_inputs.get_dummy_mm_data(1024, {"video": 1}, {}) + uncapped_dummy = uncapped.dummy_inputs.get_dummy_mm_data( + 1024, {"video": 1}, MultiModalDummyOptions() + ) uncapped_frames = uncapped_dummy["video"][0][0].shape[0] assert uncapped_frames == 2, ( f"Expected the uncapped dummy to keep 2 frames, got {uncapped_frames}" diff --git a/tests/models/multimodal/processing/test_smolvlm.py b/tests/models/multimodal/processing/test_smolvlm.py index 678b3fd39d..f27c4eb7b8 100644 --- a/tests/models/multimodal/processing/test_smolvlm.py +++ b/tests/models/multimodal/processing/test_smolvlm.py @@ -3,9 +3,7 @@ """Tests for smolvlm's multimodal preprocessing kwargs.""" import pytest -from packaging.version import Version from transformers import SmolVLMConfig -from transformers import __version__ as TRANSFORMERS_VERSION from vllm.multimodal import MULTIMODAL_REGISTRY @@ -13,10 +11,6 @@ from ....conftest import ImageTestAssets from ...utils import build_model_context -@pytest.mark.skipif( - Version(TRANSFORMERS_VERSION) < Version("5.2.0"), - reason="See https://github.com/huggingface/transformers/pull/43948", -) @pytest.mark.parametrize("model_id", ["HuggingFaceTB/SmolVLM2-2.2B-Instruct"]) @pytest.mark.parametrize( ("mm_processor_kwargs", "expected_toks_per_img"), diff --git a/tests/models/multimodal/processing/test_tensor_schema.py b/tests/models/multimodal/processing/test_tensor_schema.py index e1fb0cef74..403a01a9e2 100644 --- a/tests/models/multimodal/processing/test_tensor_schema.py +++ b/tests/models/multimodal/processing/test_tensor_schema.py @@ -18,6 +18,7 @@ from vllm.config.multimodal import ( AudioDummyOptions, BaseDummyOptions, ImageDummyOptions, + MultiModalDummyOptions, VideoDummyOptions, ) from vllm.distributed import ( @@ -98,7 +99,7 @@ def create_batched_mm_kwargs( processor_inputs = dummy_inputs.get_dummy_processor_inputs( seq_len=model_config.max_model_len, mm_counts=mm_counts, - mm_options={}, + mm_options=MultiModalDummyOptions(), ) mm_items = processor_inputs.mm_data_items resized_mm_data = { @@ -248,11 +249,13 @@ def test_model_tensor_schema(model_id: str): return AudioDummyOptions(count=count) return BaseDummyOptions(count=count) - model_config.get_multimodal_config().limit_per_prompt = { - modality: _to_dummy_options(modality, count) - for modality, count in limit_mm_per_prompt.items() - } - processor = factories.build_processor(ctx, cache=None) + model_config.get_multimodal_config().limit_per_prompt = MultiModalDummyOptions( + { + modality: _to_dummy_options(modality, count) + for modality, count in limit_mm_per_prompt.items() + } + ) + processor = factories.build_processor(ctx) with initialize_dummy_model(model_cls, model_config) as model: for modality, _, mm_kwargs in create_batched_mm_kwargs(model_config, processor): diff --git a/tests/models/multimodal/processing/test_transformers_image.py b/tests/models/multimodal/processing/test_transformers_image.py index f5de7d66f4..8f5a47be7b 100644 --- a/tests/models/multimodal/processing/test_transformers_image.py +++ b/tests/models/multimodal/processing/test_transformers_image.py @@ -245,6 +245,7 @@ def test_repeated_image_hits_the_processor_cache(): prompt="\nWhat is this?", mm_items=mm_processor.info.parse_mm_data({"image": image}), hf_processor_mm_kwargs={}, + cache=cache, ) first, second = process(), process() diff --git a/tests/models/multimodal/processing/transformers_backend.py b/tests/models/multimodal/processing/transformers_backend.py index dc49036e83..869afe9456 100644 --- a/tests/models/multimodal/processing/transformers_backend.py +++ b/tests/models/multimodal/processing/transformers_backend.py @@ -48,4 +48,4 @@ def create_cached_processor(model_id: str, processor_cls): ) info = MultiModalProcessingInfo(ctx) cache = MultiModalProcessorOnlyCache(model_config) - return processor_cls(info, MultiModalDummyInputsBuilder(info), cache=cache), cache + return processor_cls(info, MultiModalDummyInputsBuilder(info)), cache diff --git a/tests/models/multimodal/test_aria.py b/tests/models/multimodal/test_aria.py new file mode 100644 index 0000000000..fed7b20646 --- /dev/null +++ b/tests/models/multimodal/test_aria.py @@ -0,0 +1,100 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch +from transformers import AriaTextConfig +from transformers.models.aria.modeling_aria import AriaTextMoELayer as HFMoE + +from vllm.config import VllmConfig +from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501 + CompressedTensorsConfig, +) +from vllm.model_executor.models.aria import AriaTextModel, AriaTextMoELayer +from vllm.model_executor.models.utils import AutoWeightsLoader + + +@pytest.mark.cpu_test +@pytest.mark.usefixtures("dist_init") +@pytest.mark.parametrize( + "checkpoint_name,loaded_name,param_name", + [ + ( + "experts.fc1.weight", + "experts.w13_weight", + "experts.routed_experts.w13_weight", + ), + ( + "experts.fc2.weight", + "experts.w2_weight", + "experts.routed_experts.w2_weight", + ), + ( + "shared_experts.down_proj.weight", + "shared_experts.down_proj.weight", + "shared_experts.down_proj.weight", + ), + ], +) +def test_aria_expert_weights_load_with_checkpoint_layout( + checkpoint_name: str, loaded_name: str, param_name: str +): + """Real Aria checkpoint weights must reach the right parameter and layout.""" + config = AriaTextConfig( + hidden_size=32, + intermediate_size=64, + moe_num_experts=2, + moe_topk=1, + moe_num_shared_experts=1, + ) + checkpoint_weight = HFMoE(config).state_dict()[checkpoint_name] + checkpoint_weight.copy_( + torch.arange( + checkpoint_weight.numel(), dtype=checkpoint_weight.dtype + ).reshape_as(checkpoint_weight) + ) + expected = checkpoint_weight.clone() + if checkpoint_name.startswith("experts."): + expected = expected.transpose(-1, -2) + + layer = AriaTextMoELayer(config, quant_config=None, prefix="mlp") + loaded = AutoWeightsLoader(layer).load_weights( + [(checkpoint_name, checkpoint_weight)], mapper=AriaTextModel.hf_to_vllm_mapper + ) + + assert loaded == {loaded_name} + torch.testing.assert_close( + dict(layer.named_parameters())[param_name], expected, rtol=0, atol=0 + ) + + +@pytest.mark.cpu_test +def test_aria_quant_config_renames_expert_modules(): + quant_config = CompressedTensorsConfig( + target_scheme_map={}, + ignore=[ + "model.layers.0.mlp.experts.fc1", + "model.layers.0.mlp.experts.fc2", + ], + quant_format="pack-quantized", + ) + AriaTextModel.__new__( + AriaTextModel, vllm_config=VllmConfig(quant_config=quant_config) + ) + + assert quant_config.ignore == [ + "model.layers.0.mlp.experts.gate_up_proj", + "model.layers.0.mlp.experts.down_proj", + ] + + +@pytest.mark.cpu_test +@pytest.mark.parametrize( + "name,expected", [("fc1", "gate_up_proj"), ("fc2", "down_proj")] +) +def test_aria_expert_scale_names(name: str, expected: str): + suffixes = ["weight_scale", "input_scale"] + names = [f"model.layers.0.mlp.experts.{name}.{suffix}" for suffix in suffixes] + assert AriaTextModel.hf_to_vllm_mapper.apply_list(names) == [ + f"model.layers.0.mlp.experts.{expected}.{suffix}" for suffix in suffixes + ] diff --git a/tests/models/multimodal/test_cohere_asr.py b/tests/models/multimodal/test_cohere_asr.py index 85a9b59dd5..950542f900 100644 --- a/tests/models/multimodal/test_cohere_asr.py +++ b/tests/models/multimodal/test_cohere_asr.py @@ -4,9 +4,65 @@ from types import SimpleNamespace import pytest +import torch from vllm.config import SpeechToTextConfig -from vllm.model_executor.models.cohere_asr import CohereAsrForConditionalGeneration +from vllm.model_executor.models.cohere_asr import ( + CohereAsrForConditionalGeneration, + RelPositionMultiHeadAttention, +) + + +class _ReferenceRelPositionMultiHeadAttention(RelPositionMultiHeadAttention): + """Relative attention implementation before score accumulation is fused.""" + + def forward(self, query, key, value, mask, pos_emb=None): + q, k, v = self.forward_qkv(query, key, value) + q = q.transpose(1, 2) + + assert pos_emb is not None + n_batch_pos = pos_emb.size(0) + p = self.linear_pos(pos_emb).view(n_batch_pos, -1, self.h, self.d_k) + p = p.transpose(1, 2) + + q_with_bias_u = (q + self.pos_bias_u).transpose(1, 2) + q_with_bias_v = (q + self.pos_bias_v).transpose(1, 2) + + matrix_bd = torch.matmul(q_with_bias_v, p.transpose(-2, -1)) + matrix_bd = self.rel_shift(matrix_bd) + matrix_ac = torch.matmul(q_with_bias_u, k.transpose(-2, -1)) + matrix_bd = matrix_bd[:, :, :, : matrix_ac.size(-1)] + scores = (matrix_ac + matrix_bd) / self.s_d_k + return self.forward_attention(v, scores, mask) + + +@pytest.mark.parametrize(("query_len", "key_len"), [(2, 2), (3, 5)]) +def test_relative_attention_fuses_score_accumulation(monkeypatch, query_len, key_len): + torch.manual_seed(7) + batch, heads, model_dim = 2, 4, 32 + optimized = RelPositionMultiHeadAttention(heads, model_dim, None, None) + reference = _ReferenceRelPositionMultiHeadAttention(heads, model_dim, None, None) + reference.load_state_dict(optimized.state_dict()) + + query = torch.randn(batch, query_len, model_dim) + key = torch.randn(batch, key_len, model_dim) + value = torch.randn(batch, key_len, model_dim) + pos_emb = torch.randn(1, query_len + key_len - 1, model_dim) + expected = reference(query, key, value, None, pos_emb) + + original_baddbmm = torch.baddbmm + calls = 0 + + def tracked_baddbmm(*args, **kwargs): + nonlocal calls + calls += 1 + return original_baddbmm(*args, **kwargs) + + monkeypatch.setattr(torch, "baddbmm", tracked_baddbmm) + actual = optimized(query, key, value, None, pos_emb) + + torch.testing.assert_close(actual, expected) + assert calls == 1 @pytest.mark.parametrize( diff --git a/tests/models/quantization/test_mimo_v2_qkv_shard.py b/tests/models/quantization/test_mimo_v2_qkv_shard.py new file mode 100644 index 0000000000..91eb036bc7 --- /dev/null +++ b/tests/models/quantization/test_mimo_v2_qkv_shard.py @@ -0,0 +1,225 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit test for MiMo-V2 fused fp8 ``qkv_proj`` sharding. + +The checkpoint stores the fused QKV pre-sharded at ``num_key_value_heads`` +chunks (``ckpt_tp``), each holding that slice's ``[Q_c | K_c | V_c]`` rows with +its own block scale grid. ``ckpt_tp`` is not the layer's KV-head count: a +MiMo-V2.5 SWA layer has 8 KV heads but 4 chunks of 3712 rows (= 29 blocks), so +each chunk carries two KV heads. Reading the scales as one slice per KV head +expands 14 x 128 = 1792 rows for an 1856-row slice and fails. + +These tests build such checkpoints for the GA, SWA and Pro geometries and check +that every rank gets exactly the weights it should own, for TP sizes below, +equal to and above ``ckpt_tp`` (the last case replicating KV heads). +""" + +import pytest +import torch + +from vllm.model_executor.models.mimo_v2 import _shard_fp8_qkv_proj +from vllm.utils.math_utils import cdiv + +pytestmark = pytest.mark.cpu_test + +BLOCK = 128 +FP8_DTYPE = torch.float8_e4m3fn +FP8_MAX = torch.finfo(FP8_DTYPE).max +COLS = 128 # one column block is enough to exercise the row mapping +# fp8 e4m3 has 3 mantissa bits, so a single block-quantization round trip costs +# ~2-3% relative L2; a wrong row -> scale pairing costs >= 10%. +TOL = 0.06 + +# (num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp): MiMo-V2.5 GA, its +# SWA/MTP layers, and MiMo-V2.5-Pro. +GEOMETRIES = { + "ga": (64, 4, 192, 128, 4), + "swa": (64, 8, 192, 128, 4), + "pro": (128, 8, 192, 128, 8), +} + + +def _chunk_rows(num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp): + return ( + num_heads // ckpt_tp * head_dim + + num_kv_heads // ckpt_tp * head_dim + + num_kv_heads // ckpt_tp * v_head_dim + ) + + +def _quantize_chunks(truth, num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp): + """Block-quantize each checkpoint chunk on its own grid, like the export.""" + rows_per_chunk = _chunk_rows(num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp) + weights, scales = [], [] + for chunk in range(ckpt_tp): + rows = truth[chunk * rows_per_chunk : (chunk + 1) * rows_per_chunk] + quantized = torch.zeros_like(rows) + scale = torch.zeros( + cdiv(rows_per_chunk, BLOCK), rows.shape[1] // BLOCK, dtype=torch.float32 + ) + for row in range(0, rows_per_chunk, BLOCK): + for col in range(0, rows.shape[1], BLOCK): + block = rows[row : row + BLOCK, col : col + BLOCK] + amax = block.abs().max().clamp(min=1e-12) / FP8_MAX + scale[row // BLOCK, col // BLOCK] = amax + quantized[row : row + BLOCK, col : col + BLOCK] = block / amax + weights.append(quantized.to(FP8_DTYPE)) + scales.append(scale) + return torch.cat(weights, dim=0), torch.cat(scales, dim=0) + + +def _owned_rows( + truth, num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp, tp_rank, tp_size +): + """Rows the rank must own, in the forward's `[Q | K | V]` order.""" + rows_per_chunk = _chunk_rows(num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp) + q_per_chunk = num_heads // ckpt_tp * head_dim + k_per_chunk = num_kv_heads // ckpt_tp * head_dim + kv_heads: list[int] = [] + if tp_size <= num_kv_heads: + kv_heads = list( + range( + tp_rank * (num_kv_heads // tp_size), + (tp_rank + 1) * (num_kv_heads // tp_size), + ) + ) + else: + kv_heads = [tp_rank // (tp_size // num_kv_heads)] + index = [] + for head in range( + tp_rank * (num_heads // tp_size), (tp_rank + 1) * (num_heads // tp_size) + ): + chunk, offset = divmod(head, num_heads // ckpt_tp) + start = chunk * rows_per_chunk + offset * head_dim + index += list(range(start, start + head_dim)) + for head in kv_heads: + chunk, offset = divmod(head, num_kv_heads // ckpt_tp) + start = chunk * rows_per_chunk + q_per_chunk + offset * head_dim + index += list(range(start, start + head_dim)) + for head in kv_heads: + chunk, offset = divmod(head, num_kv_heads // ckpt_tp) + start = chunk * rows_per_chunk + q_per_chunk + k_per_chunk + offset * v_head_dim + index += list(range(start, start + v_head_dim)) + return truth[torch.tensor(index)] + + +def _dequantize(weight, scale): + """What the kernel computes: scale[row // block] applied per weight row.""" + per_row = scale.repeat_interleave(BLOCK, dim=1) + return weight.to(torch.float32) * per_row[torch.arange(weight.shape[0]) // BLOCK] + + +def _relative_l2(got, expected): + return ((got - expected).norm() / expected.norm()).item() + + +@pytest.mark.parametrize("geometry", list(GEOMETRIES)) +@pytest.mark.parametrize("tp_size", [1, 2, 4, 8]) +def test_fused_qkv_proj_sharding(geometry, tp_size): + num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp = GEOMETRIES[geometry] + if num_heads % tp_size or (tp_size > num_kv_heads and tp_size % num_kv_heads): + pytest.skip("TP size does not divide the head counts") + + rows_per_chunk = _chunk_rows(num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp) + truth = torch.randn(ckpt_tp * rows_per_chunk, COLS) + for chunk in range(ckpt_tp): # per-chunk magnitude spread, as in the export + truth[chunk * rows_per_chunk : (chunk + 1) * rows_per_chunk] *= 1.0 + chunk + weight, scale = _quantize_chunks( + truth, num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp + ) + + kv_per_rank = (num_kv_heads // tp_size) if tp_size <= num_kv_heads else 1 + rows_rank = (num_heads // tp_size) * head_dim + kv_per_rank * ( + head_dim + v_head_dim + ) + for tp_rank in range(tp_size): + w_rank, s_rank = _shard_fp8_qkv_proj( + weight, + scale, + num_heads=num_heads, + num_kv_heads=num_kv_heads, + head_dim=head_dim, + v_head_dim=v_head_dim, + tp_rank=tp_rank, + tp_size=tp_size, + ckpt_tp=ckpt_tp, + ) + assert w_rank.shape == (rows_rank, COLS) + assert s_rank.shape == (cdiv(rows_rank, BLOCK), scale.shape[1]) + expected = _owned_rows( + truth, + num_heads, + num_kv_heads, + head_dim, + v_head_dim, + ckpt_tp, + tp_rank, + tp_size, + ) + assert _relative_l2(_dequantize(w_rank, s_rank), expected) <= TOL + + +@pytest.mark.parametrize("geometry", ["ga", "swa"]) +@pytest.mark.parametrize("tp_size", [1, 2, 4]) +def test_exact_chunk_matches_checkpoint(geometry, tp_size): + """At tp_size <= ckpt_tp the chunk count is the only thing that matters.""" + num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp = GEOMETRIES[geometry] + rows_per_chunk = _chunk_rows(num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp) + truth = torch.randn(ckpt_tp * rows_per_chunk, COLS) + weight, scale = _quantize_chunks( + truth, num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp + ) + if tp_size == ckpt_tp: + w_rank, s_rank = _shard_fp8_qkv_proj( + weight, + scale, + num_heads=num_heads, + num_kv_heads=num_kv_heads, + head_dim=head_dim, + v_head_dim=v_head_dim, + tp_rank=0, + tp_size=tp_size, + ckpt_tp=ckpt_tp, + ) + # The checkpoint chunk *is* the rank's slice: no re-quantization. + # (torch.equal has no fp8 kernel, so compare the underlying bytes.) + expected_w = weight.chunk(tp_size, dim=0)[0] + assert torch.equal(w_rank.view(torch.uint8), expected_w.view(torch.uint8)) + assert torch.equal(s_rank, scale.chunk(tp_size, dim=0)[0]) + + +def test_wrong_chunk_count_is_detected(): + """Reading SWA scales as one slice per KV head must not look correct. + + This is the regression the fix addresses: 116 scale rows are 4 x 29 (29 = + ceil(3712 / 128)), not 8 x 15, so ``ckpt_tp = num_kv_heads`` pairs every + weight row with the wrong scale. + """ + num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp = GEOMETRIES["swa"] + rows_per_chunk = _chunk_rows(num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp) + truth = torch.randn(ckpt_tp * rows_per_chunk, COLS) + for chunk in range(ckpt_tp): + truth[chunk * rows_per_chunk : (chunk + 1) * rows_per_chunk] *= 1.0 + chunk + weight, scale = _quantize_chunks( + truth, num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp + ) + assert scale.shape[0] == ckpt_tp * cdiv(rows_per_chunk, BLOCK) + assert scale.shape[0] != num_kv_heads * cdiv(rows_per_chunk, BLOCK) + + kw = dict( + num_heads=num_heads, + num_kv_heads=num_kv_heads, + head_dim=head_dim, + v_head_dim=v_head_dim, + tp_rank=0, + tp_size=1, + ) + correct = _shard_fp8_qkv_proj(weight, scale, ckpt_tp=ckpt_tp, **kw) + wrong = _shard_fp8_qkv_proj(weight, scale, ckpt_tp=num_kv_heads, **kw) + assert correct[0].shape == wrong[0].shape + + expected = _owned_rows( + truth, num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp, 0, 1 + ) + assert _relative_l2(_dequantize(*correct), expected) <= TOL + assert _relative_l2(_dequantize(*wrong), expected) > 5 * TOL diff --git a/tests/models/quantization/test_mxfp8.py b/tests/models/quantization/test_mxfp8.py index c12a72a09c..0463468eae 100644 --- a/tests/models/quantization/test_mxfp8.py +++ b/tests/models/quantization/test_mxfp8.py @@ -111,11 +111,6 @@ def test_mxfp8_aiter_requires_swigluoai_activation( "_supports_current_device", staticmethod(lambda: True), ) - monkeypatch.setattr( - aiter_mxfp8_moe, - "is_aiter_mxfp8_moe_available", - lambda: True, - ) config = FusedMoEConfig( num_experts=8, @@ -164,11 +159,6 @@ def test_mxfp8_aiter_requires_swigluoai_params( "_supports_current_device", staticmethod(lambda: True), ) - monkeypatch.setattr( - aiter_mxfp8_moe, - "is_aiter_mxfp8_moe_available", - lambda: True, - ) config = FusedMoEConfig( num_experts=8, @@ -218,11 +208,6 @@ def test_mxfp8_aiter_accepts_swigluoai_params( "_supports_current_device", staticmethod(lambda: True), ) - monkeypatch.setattr( - aiter_mxfp8_moe, - "is_aiter_mxfp8_moe_available", - lambda: True, - ) config = FusedMoEConfig( num_experts=8, diff --git a/tests/models/registry.py b/tests/models/registry.py index 8adb9ed152..6a0fe8318e 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -447,7 +447,10 @@ _TEXT_GENERATION_EXAMPLE_MODELS = { ), "MixtralForCausalLM": _HfExamplesInfo( "mistralai/Mixtral-8x7B-Instruct-v0.1", - {"tiny": "TitanML/tiny-mixtral"}, + { + "tiny": "axolotl-ai-co/tiny-mixtral-30m", + "tiny-random": "TitanML/tiny-mixtral", + }, ), "NemotronForCausalLM": _HfExamplesInfo("nvidia/Minitron-8B-Base"), "NemotronHForCausalLM": _HfExamplesInfo( diff --git a/tests/models/test_deepseek_v41_replay_start.py b/tests/models/test_deepseek_v41_replay_start.py new file mode 100644 index 0000000000..7d4e45fe81 --- /dev/null +++ b/tests/models/test_deepseek_v41_replay_start.py @@ -0,0 +1,112 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""SWA bounded replay in DeepseekV41ModelState: the batch's replay starts reach +the sliding-window builders, and the replayed tokens' slots are padded in the +prefix-cacheable groups only.""" + +from dataclasses import replace +from types import SimpleNamespace +from unittest.mock import MagicMock + +import numpy as np +import pytest +import torch + +from vllm.config import CUDAGraphMode +from vllm.models.deepseek_v41.nvidia.model_state import DeepseekV41ModelState +from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadataBuilder +from vllm.v1.attention.backends.utils import PAD_SLOT_ID +from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers +from vllm.v1.worker.gpu.model_states.default import DefaultModelState + +pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") + +DEVICE = torch.device("cuda") +WINDOW = 4 +# Group 0 is prefix-cacheable (compressed KV), group 1 is the replayed window. +KV_CACHE_CONFIG = SimpleNamespace( + kv_cache_groups=[ + SimpleNamespace( + kv_cache_spec=SimpleNamespace(prefix_cacheable=True, prefix_replay_tokens=0) + ), + SimpleNamespace( + kv_cache_spec=SimpleNamespace( + prefix_cacheable=False, prefix_replay_tokens=WINDOW + ) + ), + ] +) + + +@pytest.fixture +def state(monkeypatch): + cfg = MagicMock() + cfg.model_config.enable_prompt_embeds = False + cfg.model_config.uses_mrope = False + cfg.model_config.is_multimodal_model = False + cfg.scheduler_config.max_num_seqs = 8 + cfg.scheduler_config.max_num_batched_tokens = 64 + seen: dict = {} + + def prepare_attn(self, input_batch, cg_mode, block_tables, slot_mappings, *a, **kw): + seen["slot_mappings"] = slot_mappings.clone() + seen["extra"] = kw["model_specific_attn_metadata"] + return {} + + monkeypatch.setattr(DefaultModelState, "prepare_attn", prepare_attn) + state = DeepseekV41ModelState( + cfg, SimpleNamespace(token_lookback_depth=0), None, DEVICE + ) + state.seen = seen # type: ignore[attr-defined] + return state + + +def _batch(query_lens, seq_lens, idx_mapping, is_prefilling): + num_reqs, num_tokens = len(query_lens), sum(query_lens) + batch = InputBatch.make_dummy(num_reqs, num_tokens, InputBuffers(8, 64, DEVICE)) + return replace( + batch, + query_start_loc=torch.tensor( + [0, *np.cumsum(query_lens)], dtype=torch.int32, device=DEVICE + ), + idx_mapping_np=np.array(idx_mapping, dtype=np.intp), + is_prefilling_np=np.array(is_prefilling, dtype=np.bool_), + positions=torch.cat( + [torch.arange(s - q, s) for q, s in zip(query_lens, seq_lens)] + ).to(DEVICE), + ) + + +def _prepare(state, batch): + slot_mappings = torch.arange(2 * batch.num_tokens, device=DEVICE).view(2, -1) + state.prepare_attn( + batch, CUDAGraphMode.NONE, (), slot_mappings, [], KV_CACHE_CONFIG + ) + swa_builder = MagicMock(spec=DeepseekSparseSWAMetadataBuilder) + extra = state.seen["extra"] + replay_start = extra.get_extra_attn_kwargs(swa_builder, batch.num_reqs)[ + "replay_start" + ] + assert extra.get_extra_attn_kwargs(MagicMock(), batch.num_reqs) == {} + return replay_start.tolist(), state.seen["slot_mappings"] + + +def test_replayed_tokens_write_only_the_window_group(state): + # Request state 3 resumes a hit at 16: positions 16..19 are replayed. + state.add_request(3, SimpleNamespace(replay_start=16)) + state.add_request(1, SimpleNamespace(replay_start=0)) + batch = _batch([6, 2], [22, 2], idx_mapping=[3, 1], is_prefilling=[True, True]) + replay_start, slots = _prepare(state, batch) + assert replay_start == [16, 0] + padded = [PAD_SLOT_ID] * WINDOW + [4, 5, 6, 7] + assert slots[0].tolist() == padded # compressed KV keeps its cached rows + assert slots[1].tolist() == list(range(8, 16)) # the window is rebuilt + + +def test_only_prefills_carry_a_replay_start(state): + state.add_request(0, SimpleNamespace(replay_start=16)) + # The same request, decoding: its rows sit above the hit. + batch = _batch([1], [40], idx_mapping=[0], is_prefilling=[False]) + replay_start, slots = _prepare(state, batch) + assert replay_start == [0] + assert slots.tolist() == [[0], [1]] diff --git a/tests/models/test_deepseek_v4_mega_moe.py b/tests/models/test_deepseek_v4_mega_moe.py index 67029b50a9..f87dd75fb2 100644 --- a/tests/models/test_deepseek_v4_mega_moe.py +++ b/tests/models/test_deepseek_v4_mega_moe.py @@ -15,6 +15,7 @@ from vllm.models.deepseek_v4.nvidia.model import ( DeepseekV4MegaMoEExperts, DeepseekV4MoE, make_deepseek_v4_expert_params_mapping, + prepare_mega_gate_routing_metadata, ) from vllm.models.deepseek_v4.nvidia.mtp import DeepSeekV4MTP from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_inputs @@ -22,6 +23,7 @@ from vllm.models.deepseek_v41.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID from vllm.models.deepseek_v41.nvidia.model import DeepseekV4MoE as DeepseekV41MoE from vllm.platforms import current_platform from vllm.transformers_utils.configs.deepseek_v41 import DeepseekV41Config +from vllm.utils.torch_utils import set_default_torch_dtype pytestmark = pytest.mark.skipif( not current_platform.is_cuda(), @@ -33,6 +35,7 @@ pytestmark = pytest.mark.skipif( def v41_moe_config(dist_init): return SimpleNamespace( model_config=SimpleNamespace( + dtype=torch.bfloat16, hf_config=DeepseekV41Config( text_config=dict( hidden_size=128, @@ -63,28 +66,52 @@ def v41_moe_config(dist_init): ) +@pytest.mark.parametrize("use_cudagraph", [False, True]) +@pytest.mark.parametrize("above_threshold", [False, True]) @pytest.mark.parametrize("vision", [False, True]) -@pytest.mark.parametrize("layer_id,num_experts,top_k", [(0, 8, 2), (2, 4, 3)]) +@pytest.mark.parametrize("layer_id,num_experts,top_k", [(0, 384, 6), (2, 128, 3)]) def test_deepseek_v41_moe_routes_without_hash_table( - v41_moe_config, monkeypatch, vision, layer_id, num_experts, top_k + v41_moe_config, + monkeypatch, + vision, + layer_id, + num_experts, + top_k, + above_threshold, + use_cudagraph, ): """Main and draft layers select experts by score and preserve image routing.""" + if not current_platform.is_device_capability_family(100): + pytest.skip("DeepGEMM Mega Gate requires SM100") + config = v41_moe_config.model_config.hf_config + config.hidden_size = 256 + config.n_routed_experts = 384 + config.num_experts_per_tok = 6 + config.dspark_n_routed_experts = 128 + num_tokens = (1 if num_experts == 128 else 16) + int(above_threshold) config.vision_n_layers = int(vision) - with torch.device("cuda"): + with ( + set_default_torch_dtype(v41_moe_config.model_config.dtype), + torch.device("cuda"), + ): moe = DeepseekV41MoE(v41_moe_config, prefix=f"model.layers.{layer_id}.ffn") - hidden_states = torch.randn(4, config.hidden_size) + hidden_states = torch.randn( + num_tokens, config.hidden_size, dtype=torch.bfloat16 + ) input_ids = ( - torch.tensor([42, IMAGE_SENTINEL_BASE_ID, IMAGE_SENTINEL_BASE_ID, 129257]) + torch.tensor( + [42, IMAGE_SENTINEL_BASE_ID, IMAGE_SENTINEL_BASE_ID, 129257] + ).repeat((num_tokens + 3) // 4)[:num_tokens] if vision else None ) assert moe.gate.tid2eid is None assert moe.gate.weight.shape == (num_experts, config.hidden_size) assert isinstance(moe.experts, DeepseekV4MegaMoEExperts) - assert moe.experts.w13_weight.shape == (num_experts, 256, 64) + assert moe.experts.w13_weight.shape == (num_experts, 256, 128) assert moe.experts.top_k == top_k - assert (config.n_routed_experts, config.num_experts_per_tok) == (8, 2) + assert (config.n_routed_experts, config.num_experts_per_tok) == (384, 6) with torch.no_grad(): moe.gate.weight.normal_(std=0.01) @@ -93,7 +120,7 @@ def test_deepseek_v41_moe_routes_without_hash_table( moe.gate.bias_vl.copy_(-moe.gate.e_score_correction_bias) scores = torch.nn.functional.softplus( - torch.nn.functional.linear(hidden_states, moe.gate.weight) + torch.mm(hidden_states, moe.gate.weight.t(), out_dtype=torch.float32) ).sqrt() bias = moe.gate.e_score_correction_bias if vision: @@ -105,15 +132,49 @@ def test_deepseek_v41_moe_routes_without_hash_table( dim=-1, keepdim=True ) + routed = {} + def check_routing(x, weights, ids, *, activation_clamp): - torch.testing.assert_close(ids, expected_ids) - torch.testing.assert_close(weights, expected_weights) + routed["ids"], routed["weights"] = ids, weights assert activation_clamp == config.swiglu_limit return x.clone() monkeypatch.setattr(moe.experts, "forward", check_routing) monkeypatch.setattr(moe.shared_experts, "forward", lambda x: 2 * x) - torch.testing.assert_close(moe(hidden_states, input_ids), 3 * hidden_states) + routing_input_ids = ( + input_ids + if input_ids is not None + else torch.zeros(num_tokens, dtype=torch.int64, device="cuda") + ) + metadata = prepare_mega_gate_routing_metadata( + routing_input_ids, + has_hash_routing=False, + image_sentinel_base_id=IMAGE_SENTINEL_BASE_ID if vision else None, + ) + if use_cudagraph: + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + for _ in range(3): + moe(hidden_states, input_ids, metadata) + stream.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=stream): + output = moe(hidden_states, input_ids, metadata) + graph.replay() + torch.accelerator.synchronize() + else: + output = moe(hidden_states, input_ids, metadata) + + ids, weights = routed["ids"], routed["weights"] + torch.testing.assert_close(ids, expected_ids) + torch.testing.assert_close( + weights, + expected_weights, + rtol=1e-3, + atol=1e-4, + ) + torch.testing.assert_close(output, 3 * hidden_states) @pytest.mark.parametrize( @@ -156,6 +217,67 @@ def test_deepseek_v4_moe_preserves_configured_hash_layers(v41_moe_config): moe(torch.zeros(1, 128)) +@pytest.mark.parametrize("num_tokens", [16, 17]) +def test_deepseek_v4_mega_gate_hash_routing_correctness( + v41_moe_config, monkeypatch, num_tokens +): + if not current_platform.is_device_capability_family(100): + pytest.skip("DeepGEMM Mega Gate requires SM100") + + config = v41_moe_config.model_config.hf_config + config.hidden_size = 256 + config.num_hash_layers = 1 + config.vocab_size = 32 + with ( + set_default_torch_dtype(v41_moe_config.model_config.dtype), + torch.device("cuda"), + ): + moe = DeepseekV4MoE( + v41_moe_config, + prefix="model.layers.0.ffn", + num_hash_layers=config.num_hash_layers, + ) + hidden_states = torch.randn( + num_tokens, config.hidden_size, dtype=torch.bfloat16 + ) + input_ids = torch.arange(num_tokens) + + token_ids = torch.arange(config.vocab_size, device="cuda") + fixed_ids = torch.stack( + ( + token_ids % config.n_routed_experts, + (token_ids + 3) % config.n_routed_experts, + ), + dim=1, + ) + with torch.no_grad(): + moe.gate.weight.normal_(std=0.01) + moe.gate.tid2eid.copy_(fixed_ids) + + expected_ids = fixed_ids[input_ids] + scores = torch.nn.functional.softplus( + torch.mm(hidden_states, moe.gate.weight.t(), out_dtype=torch.float32) + ).sqrt() + expected_weights = scores.gather(1, expected_ids) + expected_weights *= config.routed_scaling_factor / expected_weights.sum( + dim=-1, keepdim=True + ) + + def check_routing(x, weights, ids, *, activation_clamp): + torch.testing.assert_close(ids, expected_ids) + torch.testing.assert_close(weights, expected_weights, rtol=1e-3, atol=1e-4) + return x.clone() + + monkeypatch.setattr(moe.experts, "forward", check_routing) + monkeypatch.setattr(moe.shared_experts, "forward", torch.zeros_like) + metadata = prepare_mega_gate_routing_metadata( + input_ids, + has_hash_routing=True, + image_sentinel_base_id=None, + ) + torch.testing.assert_close(moe(hidden_states, input_ids, metadata), hidden_states) + + def test_deepseek_v4_mega_moe_expert_mapping(): mapping = make_deepseek_v4_expert_params_mapping(2) @@ -513,6 +635,7 @@ def test_deepseek_v4_mega_moe_does_not_double_add_fused_shared_expert( monkeypatch, fused ): class FakeGate(torch.nn.Module): + weight = torch.empty(2, 128) tid2eid = None e_score_correction_bias = None @@ -563,12 +686,14 @@ def test_deepseek_v4_mega_moe_does_not_double_add_fused_shared_expert( not torch.cuda.is_available(), reason="DeepSeek V4 MegaMoE fused input staging requires CUDA.", ) -def test_deepseek_v4_mega_moe_fused_input_staging_is_bitwise_exact(): +@pytest.mark.parametrize("num_tokens", [7, 63, 64, 65, 127, 128, 135, 434, 16384]) +@pytest.mark.parametrize("hidden_size", [256, 5120]) +def test_deepseek_v4_mega_moe_fused_input_staging_is_bitwise_exact( + num_tokens, hidden_size +): from vllm.third_party.deep_gemm.utils import per_token_cast_to_fp8 device = torch.device("cuda") - num_tokens = 7 - hidden_size = 256 top_k = 8 generator = torch.Generator(device=device) @@ -723,6 +848,7 @@ def test_deepseek_v4_mega_moe_stages_shared_scale_tma_layout(shared_block_m): assert torch.equal(fused_x.view(torch.uint8), ref_x.view(torch.uint8)) assert torch.equal(fused_x_sf, ref_x_sf) assert torch.equal(fused_shared_x_sf[populated], ref_shared_x_sf[populated]) + assert torch.all(fused_shared_x_sf[~populated] == -1) def test_deepseek_v4_pwal_hook_finalizes_mega_moe_and_mhc_broadcast(): @@ -760,11 +886,14 @@ def test_deepseek_v4_drafter_pwal_hooks_finalize_mega_moe(): not torch.cuda.is_available(), reason="DeepSeek V4 MegaMoE fused input staging requires CUDA.", ) -def test_deepseek_v4_mega_moe_fused_input_staging_masks_padding(): +@pytest.mark.parametrize("nonfinite_padding", [False, True]) +@pytest.mark.parametrize("num_tokens", [7, 65, 135]) +def test_deepseek_v4_mega_moe_fused_input_staging_masks_padding( + nonfinite_padding, num_tokens +): from vllm.third_party.deep_gemm.utils import per_token_cast_to_fp8 device = torch.device("cuda") - num_tokens = 7 hidden_size = 256 top_k = 8 @@ -792,10 +921,11 @@ def test_deepseek_v4_mega_moe_fused_input_staging_masks_padding(): dtype=torch.float32, generator=generator, ) - is_padding = torch.tensor( - [False, True, False, False, True, False, True], - device=device, - ) + is_padding = torch.zeros(num_tokens, device=device, dtype=torch.bool) + is_padding[1::3] = True + is_padding[-1] = True + if nonfinite_padding: + topk_weights[is_padding] = float("nan") ref_x, ref_x_sf = per_token_cast_to_fp8( hidden_states, diff --git a/tests/models/test_dspark_mla.py b/tests/models/test_dspark_mla.py index d30eb9e7c9..3e70450f2c 100644 --- a/tests/models/test_dspark_mla.py +++ b/tests/models/test_dspark_mla.py @@ -11,6 +11,7 @@ from vllm.compilation.wrapper import TorchCompileWithNoGuardsWrapper from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.models.qwen3_dspark import DSparkMarkovHead from vllm.model_executor.models.registry import ModelRegistry +from vllm.models.deepseek_v4.nvidia import dspark as dsv4_dspark from vllm.models.kimi_k3.nvidia import dspark_mla from vllm.models.kimi_k3.nvidia.dspark_mla import K3DSparkForCausalLM, K3DSparkModel @@ -262,3 +263,75 @@ def test_v41_dspark_loads_linear_scales( assert loaded == {runtime_name} assert shards == [() if shard_id is None else (shard_id,)] torch.testing.assert_close(param, checkpoint_scale) + + +def test_dsv4_context_wkv_weights_are_duplicated_by_draft_layer(): + weights = [ + ("mtp.0.attn.wkv.weight", torch.arange(4)), + ("mtp.1.attn.wq_a.weight", torch.arange(3)), + ("mtp.2.attn.wkv.scale", torch.tensor(0.5)), + ("mtp.3.attn.wkv.weight", torch.arange(2)), + ] + + duplicated = list(dsv4_dspark._duplicate_context_wkv_weights(weights, 3)) + + assert [name for name, _ in duplicated] == [ + "mtp.0.attn.wkv.weight", + "context_wkv_proj.weight", + "mtp.1.attn.wq_a.weight", + "mtp.2.attn.wkv.scale", + "context_wkv_proj.scale", + "mtp.3.attn.wkv.weight", + ] + assert duplicated[1][1].shard_id == 0 + assert duplicated[4][1].shard_id == 2 + assert duplicated[0][1].data_ptr() == duplicated[1][1].data_ptr() + assert duplicated[3][1].data_ptr() == duplicated[4][1].data_ptr() + + +def test_dsv4_context_kv_uses_one_stacked_wkv_projection(monkeypatch): + calls = [] + stacked_output = torch.arange(24, dtype=torch.float32).view(2, 12) + + class StackedProjection: + def __init__(self): + self.calls = 0 + + def __call__(self, main_x): + self.calls += 1 + assert main_x.shape == (2, 5) + return stacked_output + + projection = StackedProjection() + layers = [ + SimpleNamespace(attn=SimpleNamespace(kv_norm=lambda kv, offset=i: kv + offset)) + for i in range(3) + ] + model = SimpleNamespace( + config=SimpleNamespace(head_dim=4), + context_wkv_proj=projection, + layers=layers, + num_dspark_layers=3, + ) + slot_mappings = [torch.tensor([0, 1]), None, torch.tensor([4, 5])] + monkeypatch.setattr( + dsv4_dspark, + "_insert_context_kv", + lambda attn, kv, positions, slots: calls.append( + (attn, kv.clone(), positions, slots) + ), + ) + + dsv4_dspark.DSparkDeepseekV4Model.precompute_and_store_context_kv( + model, + torch.zeros(2, 5), + torch.tensor([7, 8]), + slot_mappings, + ) + + assert projection.calls == 1 + assert len(calls) == 2 + assert torch.equal(calls[0][1], stacked_output.view(2, 3, 4)[:, 0]) + assert torch.equal(calls[1][1], stacked_output.view(2, 3, 4)[:, 2] + 2) + assert calls[0][3] is slot_mappings[0] + assert calls[1][3] is slot_mappings[2] diff --git a/tests/models/test_initialization.py b/tests/models/test_initialization.py index e3a08980e9..073f616af8 100644 --- a/tests/models/test_initialization.py +++ b/tests/models/test_initialization.py @@ -1,12 +1,17 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import gc +from collections.abc import Generator from functools import partial from unittest.mock import patch import pytest +import torch from vllm import LLM +from vllm.logger import init_logger +from vllm.platforms import current_platform from vllm.utils.mem_constants import GiB_bytes from vllm.v1.attention.backends.utils import resolve_kv_cache_layout from vllm.v1.core.kv_cache_utils import ( @@ -15,7 +20,12 @@ from vllm.v1.core.kv_cache_utils import ( ) from vllm.v1.engine.core import EngineCore as V1EngineCore -from ..utils import create_new_process_for_each_test, requires_spawn_multiprocessing +from ..utils import ( + create_new_process_for_each_test, + requires_spawn_multiprocessing, + wait_for_gpu_memory_to_clear, + wait_for_memory_to_settle, +) from .registry import ( _TRANSFORMERS_BACKEND_MODELS, AUTO_EXAMPLE_MODELS, @@ -24,6 +34,8 @@ from .registry import ( ) from .utils import dummy_hf_overrides +logger = init_logger(__name__) + # This minimal list of model architectures is smaller than the total list of # supported models. The intention is that in the "typical" regression testing # scenario, we only test initializing these models. This subset was chosen @@ -53,6 +65,37 @@ OTHER_MODEL_ARCH_LIST = set(HF_EXAMPLE_MODELS.get_supported_archs()) - set( ) +@pytest.fixture(autouse=True) +def cleanup_gpu_memory_between_tests() -> Generator[None, None, None]: + """Reclaim GPU memory left over by each test's forked subprocess. + + These tests are known to leave behind GPU memory (see the comment at + ``gpu_memory_utilization=0.80`` below). Each parametrized case builds an + LLM in a forked subprocess whose VRAM is only reclaimed asynchronously + when the subprocess exits, so the leftovers can OOM a later test in this + module (observed with the heavier multimodal archs, e.g. + InternVLChatModel). Mirrors ``clean_gpu_memory_between_tests`` in the + top-level conftest, but unconditionally for this module. + """ + yield + gc.collect() + if not torch.cuda.is_available(): + return + torch.accelerator.empty_cache() + if current_platform.is_rocm() or current_platform.is_xpu(): + wait_for_memory_to_settle() + return + try: + wait_for_gpu_memory_to_clear( + devices=list(range(torch.accelerator.device_count())), + threshold_ratio=0.1, + ) + except Exception as e: + # NVML is not usable on MIG slices (raises NVMLError_NoPermission); + # cleanup is best-effort, so never fail a test over it. + logger.info("Failed to clean GPU memory: %s", e) + + @create_new_process_for_each_test() def can_initialize( model_arch: str, monkeypatch: pytest.MonkeyPatch, EXAMPLE_MODELS: HfExampleModels diff --git a/tests/models/test_utils.py b/tests/models/test_utils.py index 02903504c6..0dac2dbf2a 100644 --- a/tests/models/test_utils.py +++ b/tests/models/test_utils.py @@ -217,3 +217,25 @@ def test_get_rename_mapper_keeps_only_renames(): for name in ("drop_regex.w", "drop_substr.w", "drop_prefix.w", "w.drop_suffix"): assert mapper._map_name(name) is None assert renames._map_name(name) == name + + +def test_weights_mapper_stacks_one_weight_into_several_shards(): + """One checkpoint tensor can feed more than one shard of a stacked + parameter, and a dropped name stays dropped rather than reappearing as a + shard of the stacked one.""" + mapper = WeightsMapper( + orig_to_new_substr={"layers.1.k_proj.": None}, + orig_to_new_stacked={".k_proj.": (".qkv_proj.", ["k", "v"])}, + ) + weight = torch.ones(2) + weights = [(f"layers.{i}.k_proj.weight", weight) for i in (0, 1)] + + mapped = list(mapper.apply(weights)) + + assert [(name, w.shard_id) for name, w in mapped] == [ + ("layers.0.qkv_proj.weight", "k"), + ("layers.0.qkv_proj.weight", "v"), + ] + # Each shard needs its own tensor object, but they alias one allocation. + assert mapped[0][1] is not mapped[1][1] + assert mapped[0][1].data_ptr() == mapped[1][1].data_ptr() == weight.data_ptr() diff --git a/tests/models/transformers/fusers/test_mla.py b/tests/models/transformers/fusers/test_mla.py index 9f51ab5662..625fc4dc08 100644 --- a/tests/models/transformers/fusers/test_mla.py +++ b/tests/models/transformers/fusers/test_mla.py @@ -53,7 +53,6 @@ def _match(q_lora_rank: int | None) -> MLAFuser | None: def test_discovers_modules_without_q_lora(): fuser = _match(q_lora_rank=None) assert isinstance(fuser, MLAFuser) - assert not fuser.has_q_lora assert fuser.q_proj_name == "q_proj" assert fuser.kv_a_proj_name == "kv_a_proj_with_mqa" assert fuser.kv_a_layernorm_name == "kv_a_layernorm" @@ -68,7 +67,6 @@ def test_discovers_modules_without_q_lora(): def test_discovers_modules_with_q_lora(): fuser = _match(q_lora_rank=64) assert isinstance(fuser, MLAFuser) - assert fuser.has_q_lora assert fuser.q_a_proj_name == "q_a_proj" assert fuser.q_a_layernorm_name == "q_a_layernorm" assert fuser.q_b_proj_name == "q_b_proj" @@ -107,7 +105,7 @@ def test_update_forward_rewrites_real_attention(q_lora_rank): assert isinstance(fuser, MLAFuser) fuser.update_forward(_attention(q_lora_rank)) names = set(fuser.fused_forward.__code__.co_names) - if fuser.has_q_lora: + if fuser.q_a_proj_name is not None: assert _FUSED_QKV_A_PROJ in names assert not {"q_a_proj", "kv_a_proj_with_mqa"} & names @@ -158,7 +156,7 @@ def test_discovers_modules_under_arbitrary_names(): module = RenamedMLA() fuser = MLAFuser.match(trace(module), module) assert isinstance(fuser, MLAFuser) - assert not fuser.has_q_lora + assert fuser.q_a_proj_name is None assert fuser.q_proj_name == "alpha" assert fuser.kv_a_proj_name == "beta" assert fuser.kv_a_layernorm_name == "gamma" diff --git a/tests/models/utils.py b/tests/models/utils.py index b52855bcdd..e4cc5e673b 100644 --- a/tests/models/utils.py +++ b/tests/models/utils.py @@ -396,6 +396,8 @@ class ModelInfo: @dataclass class EmbedModelInfo(ModelInfo): mteb_score: float | None = None + # Per-model MTEB score tolerance; falls back to MTEB_EMBED_TOL. + mteb_tol: float | None = None is_matryoshka: bool = False matryoshka_dimensions: list[int] | None = None @@ -403,6 +405,8 @@ class EmbedModelInfo(ModelInfo): @dataclass class RerankModelInfo(ModelInfo): mteb_score: float | None = None + # Per-model MTEB score tolerance; falls back to MTEB_RERANK_TOL. + mteb_tol: float | None = None chat_template_name: str | None = None diff --git a/tests/multimodal/media/test_connector.py b/tests/multimodal/media/test_connector.py index b222d57d3c..5949c66b56 100644 --- a/tests/multimodal/media/test_connector.py +++ b/tests/multimodal/media/test_connector.py @@ -299,6 +299,7 @@ async def test_fetch_video_http(video_url: str, num_frames: int): assert metadata_sync == metadata_async +@pytest.mark.flaky(reruns=3, reruns_delay=5) @pytest.mark.asyncio @pytest.mark.parametrize("video_url", TEST_VIDEO_URLS) @pytest.mark.parametrize("max_duration", [1, 60, 1800]) @@ -320,8 +321,11 @@ async def test_fetch_video_http_with_dynamic_loader( } ) - video_sync, metadata_sync = connector.fetch_video(video_url) - video_async, metadata_async = await connector.fetch_video_async(video_url) + try: + video_sync, metadata_sync = connector.fetch_video(video_url) + video_async, metadata_async = await connector.fetch_video_async(video_url) + except (TimeoutError, asyncio.TimeoutError) as e: + pytest.skip(f"Timeout fetching video (CI network flakiness): {e}") assert np.array_equal(video_sync, video_async) assert metadata_sync == metadata_async @@ -379,6 +383,7 @@ def test_placeholder_range_extract_embeds_range(offset, is_embed, expected): assert pr.extract_embeds_range() == expected +@pytest.mark.flaky(reruns=3, reruns_delay=5) @pytest.mark.asyncio @pytest.mark.parametrize("video_url", TEST_VIDEO_URLS) @pytest.mark.parametrize("num_frames", [-1, 32, 1800]) @@ -394,8 +399,12 @@ async def test_allowed_media_domains(video_url: str, num_frames: int): ], ) - video_sync, metadata_sync = connector.fetch_video(video_url) - video_async, metadata_async = await connector.fetch_video_async(video_url) + try: + video_sync, metadata_sync = connector.fetch_video(video_url) + video_async, metadata_async = await connector.fetch_video_async(video_url) + except (TimeoutError, asyncio.TimeoutError) as e: + pytest.skip(f"Timeout fetching video (CI network flakiness): {e}") + assert np.array_equal(video_sync, video_async) assert metadata_sync == metadata_async diff --git a/tests/multimodal/test_cache.py b/tests/multimodal/test_cache.py index f77dacd341..02bd511970 100644 --- a/tests/multimodal/test_cache.py +++ b/tests/multimodal/test_cache.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import multiprocessing as mp from types import SimpleNamespace +from unittest.mock import Mock import numpy as np import pytest @@ -34,8 +35,13 @@ from vllm.multimodal.inputs import ( PlaceholderRange, ) from vllm.multimodal.processing import PromptInsertion +from vllm.renderers import renderer_from_config +from vllm.utils.async_utils import make_async from vllm.utils.mem_constants import GiB_bytes, MiB_bytes +from vllm.v1.engine.async_llm import AsyncLLM +from vllm.v1.engine.llm_engine import LLMEngine +from ..models.utils import build_model_context from ..utils import create_new_process_for_each_test pytestmark = pytest.mark.cpu_test @@ -755,6 +761,74 @@ def test_processor_cache_shared_across_loras(): assert feature_lora_b.data == item_data +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize( + "release_error", + [ + None, + "requires a completed pause first", + "requires all executor memory to be resident", + ], + ids=["released", "not-paused", "nonresident-memory"], +) +@pytest.mark.asyncio +async def test_release_kv_cache_resends_mm_payload(use_async, release_error): + """Release must not leave a sender hit pointing at a cleared receiver.""" + ctx = build_model_context( + "llava-hf/llava-v1.6-mistral-7b-hf", + mm_processor_kwargs=None, + limit_mm_per_prompt={"image": 1}, + mm_processor_cache_gb=1, + ) + model_config = ctx.model_config + + sender = MultiModalProcessorSenderCache(model_config) + receiver = MultiModalReceiverCache(model_config) + item = _dummy_item({"pixel_values": 16}) + mm_hash = "image_A" + payload, _ = sender.get_and_update_item((item, []), mm_hash) + assert receiver.get_and_update_item(payload, mm_hash) is item + assert sender.get_and_update_item(None, mm_hash)[0] is None + + renderer = renderer_from_config(VllmConfig(model_config=model_config)) + renderer._mm_processor_cache = sender + + def release(): + if release_error: + raise RuntimeError(release_error) + receiver.clear_cache() + + engine = SimpleNamespace( + renderer=renderer, + engine_core=SimpleNamespace( + release_kv_cache_memory=release, + release_kv_cache_memory_async=make_async( + release, + executor=renderer._executor, + ), + ), + logger_manager=Mock(), + ) + + async def call_release(): + if use_async: + await AsyncLLM.release_kv_cache_memory(engine) + else: + LLMEngine.release_kv_cache_memory(engine) + + if release_error: + with pytest.raises(RuntimeError, match=release_error): + await call_release() + engine.logger_manager.record_sleep_state.assert_not_called() + else: + await call_release() + engine.logger_manager.record_sleep_state.assert_called_once_with(1, 0) + + payload, _ = sender.get_and_update_item((item, []), mm_hash) + assert payload is item + assert receiver.get_and_update_item(payload, mm_hash) is item + + _SLEEP_VISION_PROMPT = ( "<|im_start|>system\nYou are a helpful assistant.<|im_end|>" "\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>" diff --git a/tests/multimodal/test_hasher.py b/tests/multimodal/test_hasher.py index 1b738eeeb8..971d97a2ca 100644 --- a/tests/multimodal/test_hasher.py +++ b/tests/multimodal/test_hasher.py @@ -268,3 +268,43 @@ def test_hash_media_io_noop_config_preserves_hash(): assert hasher.hash_kwargs("blake3", image=loaded) == hasher.hash_kwargs( "blake3", image=plain ) + + +# The digest input is a concatenation of byte chunks, so it has to be uniquely +# decodable. Each case below is a pair of distinct processor kwargs that used to +# serialize to the same bytes, which made two requests share an mm hash -- and +# therefore share both the processor cache entry and the prefix-cache block key. +IMAGE = b"\x89PNG\r\n\x1a\n" + + +def _hash(**mm_processor_kwargs: object) -> str: + return MultiModalHasher.hash_kwargs( + "blake3", model_id="m", image=IMAGE, **mm_processor_kwargs + ) + + +def test_hash_collision_kwargs_key_value_boundary(): + # Both used to flatten to b"ab" + b"c". + assert _hash(**{"ab": "c"}) != _hash(**{"a": "bc"}) + + +def test_hash_collision_nested_vs_flattened_key(): + nested = _hash(size={"shortest_edge": 224}) + flattened = _hash(**{"size.shortest_edge": 224}) + assert nested != flattened + + +def test_hash_collision_sequence_vs_mapping(): + assert _hash(fps=[2, 4]) != _hash(fps={"0": 2, "1": 4}) + + +@pytest.mark.parametrize("empty", ["", b"", []]) +def test_hash_collision_none_vs_empty(empty): + assert _hash(video_pruning_rate=None) != _hash(video_pruning_rate=empty) + + +def test_hash_collision_empty_container_vs_omitted(): + omitted = _hash() + assert _hash(size={}) != omitted + assert _hash(size=[]) != omitted + assert _hash(size={}) != _hash(size=[]) diff --git a/tests/multimodal/test_processing.py b/tests/multimodal/test_processing.py index 7d6a7fd7e1..0bd19a0ea0 100644 --- a/tests/multimodal/test_processing.py +++ b/tests/multimodal/test_processing.py @@ -7,7 +7,7 @@ from contextlib import nullcontext import numpy as np import pytest -from vllm.config import ModelConfig +from vllm.config import ModelConfig, SchedulerConfig from vllm.exceptions import VLLMValidationError from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.hasher import MultiModalHasher @@ -32,6 +32,7 @@ from vllm.multimodal.processing.processor import ( ) from vllm.utils.collection_utils import flatten_2d_lists +from ..models.utils import build_model_context from .utils import random_image pytestmark = pytest.mark.cpu_test @@ -1324,3 +1325,74 @@ def test_processor_inputs_hashes_ignore_unrelated_kwargs(): ) assert inputs.get_mm_hashes("test-model", "blake3") == {"image": ["image-uuid"]} + + +@pytest.mark.parametrize( + ("left", "right"), + [ + # Shifting the key/value boundary: both flatten to the dotted key + # "mm_processor_kwargs.abc" followed by no value bytes. + ({"ab": "c"}, {"a": "bc"}), + # A nested mapping and a caller-supplied dotted key flatten alike. + ({"size": {"shortest_edge": 224}}, {"size.shortest_edge": 224}), + # A sequence and a mapping keyed by stringified indices flatten alike. + ({"fps": [2, 4]}, {"fps": {"0": 2, "1": 4}}), + # None contributes the key alone, which a zero-byte value also does. + ({"video_pruning_rate": None}, {"video_pruning_rate": ""}), + # An empty container contributes nothing, as does omitting the key. + ({"size": {}}, {}), + ], +) +def test_processor_inputs_hashes_distinguish_kwargs_shapes(left, right): + """Distinct processor kwargs must not share a multi-modal hash. + + ``hf_processor_mm_kwargs`` is per-request input, so both the keys and the + values here are caller-controlled. The hash is the identity of the + processor cache entry and is mixed into the prefix-cache block key, so two + requests sharing one is a cross-request cache hit. + """ + image = random_image(np.random.RandomState(0), min_wh=8, max_wh=9) + mm_data_items = MultiModalDataParser().parse_mm_data({"image": [image]}) + + def hash_with(hf_processor_mm_kwargs): + return ProcessorInputs( + prompt=[], + mm_data_items=mm_data_items, + hf_processor_mm_kwargs=hf_processor_mm_kwargs, + ).get_mm_hashes("test-model", "blake3")["image"][0] + + assert hash_with(left) != hash_with(right) + + +@pytest.mark.parametrize( + ("chunked_prefill", "max_model_len", "expected_seq_len"), + [(None, 491520, 491520), (True, 491520, 8192), (True, 128, 128), (False, 128, 128)], +) +def test_dummy_inputs_scheduler_budget( + chunked_prefill, max_model_len, expected_seq_len +): + ctx = build_model_context( + "llava-hf/llava-v1.6-mistral-7b-hf", + mm_processor_kwargs=None, + limit_mm_per_prompt={"image": 1}, + ) + ctx.model_config.max_model_len = max_model_len + + processor = MULTIMODAL_REGISTRY.create_processor( + ctx.model_config, + tokenizer=ctx.tokenizer, + ) + processor.apply = lambda *args, **kwargs: {"prompt_token_ids": [7]} + + kwargs = {} + if chunked_prefill is not None: + kwargs["scheduler_config"] = SchedulerConfig( + max_model_len=max_model_len, + is_encoder_decoder=False, + max_num_batched_tokens=8192, + max_num_seqs=1, + enable_chunked_prefill=chunked_prefill, + ) + + result = processor.get_dummy_mm_inputs({"image": 1}, **kwargs) + assert len(result["prompt_token_ids"]) == expected_seq_len diff --git a/tests/multimodal/test_registry.py b/tests/multimodal/test_registry.py index 6ca8874014..d7947845af 100644 --- a/tests/multimodal/test_registry.py +++ b/tests/multimodal/test_registry.py @@ -5,11 +5,9 @@ Qwen2.5-VL visual component loading behavior. """ from types import SimpleNamespace -from unittest.mock import MagicMock import pytest -from vllm.config import SchedulerConfig from vllm.multimodal import MULTIMODAL_REGISTRY from ..models.utils import build_model_context @@ -17,34 +15,6 @@ from ..models.utils import build_model_context pytestmark = pytest.mark.cpu_test -@pytest.mark.parametrize( - ("chunked_prefill", "max_model_len", "expected_seq_len"), - [(None, 491520, 491520), (True, 491520, 8192), (True, 128, 128), (False, 128, 128)], -) -def test_dummy_inputs_scheduler_budget( - chunked_prefill, max_model_len, expected_seq_len -): - model_config = MagicMock() - model_config.max_model_len = max_model_len - processor = MagicMock() - processor.apply.return_value = {"prompt_token_ids": [7]} - kwargs = {} - if chunked_prefill is not None: - kwargs["scheduler_config"] = SchedulerConfig( - max_model_len=max_model_len, - is_encoder_decoder=False, - max_num_batched_tokens=8192, - max_num_seqs=1, - enable_chunked_prefill=chunked_prefill, - ) - result = MULTIMODAL_REGISTRY.get_dummy_mm_inputs( - model_config, {"image": 1}, processor=processor, **kwargs - ) - get_inputs = processor.dummy_inputs.get_dummy_processor_inputs - assert get_inputs.call_args.kwargs["seq_len"] == expected_seq_len - assert len(result["prompt_token_ids"]) == expected_seq_len - - @pytest.mark.parametrize( "model_id,limit_mm_per_prompt,expected", [ diff --git a/tests/parser/engine/test_deepseek_v4.py b/tests/parser/engine/test_deepseek_v4.py index 7cbf9791a7..2cf3a8bb85 100644 --- a/tests/parser/engine/test_deepseek_v4.py +++ b/tests/parser/engine/test_deepseek_v4.py @@ -119,6 +119,35 @@ class TestArgConverter: result = json.loads(_dsml_arg_converter("", partial=False)) assert result == {} + def _bare(self, name: str, value: str) -> str: + return f'<|DSML|parameter name="{name}">{value}{_PARAM_CLOSE}' + + def test_missing_string_attribute_keeps_literal(self): + """The model sometimes omits ``string``; the parameter must survive.""" + raw = self._bare("name", "alpha") + assert json.loads(_dsml_arg_converter(raw, partial=False)) == {"name": "alpha"} + + def test_missing_string_attribute_parses_json(self): + raw = self._bare("count", "7") + self._bare("opts", '{"k": [1]}') + assert json.loads(_dsml_arg_converter(raw, partial=False)) == { + "count": 7, + "opts": {"k": [1]}, + } + + def test_missing_string_attribute_alongside_annotated(self): + raw = self._bare("a", "x") + _param("b", "true", "42") + assert json.loads(_dsml_arg_converter(raw, partial=False)) == { + "a": "x", + "b": "42", + } + + def test_missing_string_attribute_partial_value(self): + raw = _param("a", "true", "x") + '<|DSML|parameter name="n">12' + assert json.loads(_dsml_arg_converter(raw, partial=True)) == { + "a": "x", + "n": 12, + } + def test_invalid_json_fallback(self): raw = self._raw(("data", "false", "[broken")) result = json.loads(_dsml_arg_converter(raw, partial=False)) @@ -232,6 +261,47 @@ class TestImplicitParameterClose: } +class TestMissingStringAttribute: + def test_non_streaming_keeps_parameter(self, mock_tokenizer, mock_request): + text = ( + f"{DSML_TOOL_START}" + f"{DSML_INVOKE_PREFIX}get_weather{DSML_INVOKE_NAME_END}\n" + f'<|DSML|parameter name="location">Paris{_PARAM_CLOSE}\n' + f"{_param('date', 'true', 'tomorrow')}\n" + f"{DSML_INVOKE_END}{DSML_TOOL_END}" + ) + + result = DeepSeekV4Parser(mock_tokenizer).extract_tool_calls(text, mock_request) + + assert result.tools_called is True + assert json.loads(result.tool_calls[0].function.arguments) == { + "location": "Paris", + "date": "tomorrow", + } + + def test_streaming_keeps_parameter(self, mock_tokenizer, mock_request): + chunks = [ + DSML_TOOL_START, + f"{DSML_INVOKE_PREFIX}get_weather{DSML_INVOKE_NAME_END}\n", + '<|DSML|parameter name="loc', + 'ation">Par', + "is", + _PARAM_CLOSE, + f"\n{_param('date', 'true', 'tomorrow')}\n", + DSML_INVOKE_END, + DSML_TOOL_END, + ] + + results = simulate_tool_streaming( + DeepSeekV4Parser(mock_tokenizer), mock_request, chunks + ) + + assert json.loads(collect_tool_arguments(results)) == { + "location": "Paris", + "date": "tomorrow", + } + + # ── Bare absorption and duplicate absorption ───────── diff --git a/tests/parser/engine/test_deepseek_v41.py b/tests/parser/engine/test_deepseek_v41.py index cc7d6e0aed..d99ca6b2a1 100644 --- a/tests/parser/engine/test_deepseek_v41.py +++ b/tests/parser/engine/test_deepseek_v41.py @@ -137,10 +137,12 @@ def test_python_argument_conversion_and_partial_values(): '<|DSML| parameter name="object" string="false">' '{"a": [true, null]}' '<|DSML| parameter name="bad" string="false">[broken' + '<|DSML| parameter name="bare">7' '<|DSML| parameter name="text" string="true"> a= 7.5) or ROCm — not just gfx950. +These exercise ``select_int8_moe_backend`` only (no MoE kernels are launched), +so they run on any platform where the Triton INT8 MoE kernel is available — +CUDA (SM >= 7.5), ROCm, or XPU — not just gfx950. """ import pytest @@ -27,17 +27,22 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( ) from vllm.platforms import current_platform -# The Triton int8_w8a8 fused-MoE kernel is available on CUDA (Turing+) and on -# ROCm CDNA GPUs. Gate on that rather than on a specific arch. +# The Triton int8_w8a8 fused-MoE kernel is available on CUDA (Turing+), on +# ROCm CDNA GPUs and on XPU. Gate on that rather than on a specific arch. INT8_MOE_SUPPORTED = ( - current_platform.is_cuda() and current_platform.has_device_capability((7, 5)) -) or current_platform.is_rocm() + (current_platform.is_cuda() and current_platform.has_device_capability((7, 5))) + or current_platform.is_rocm() + or current_platform.is_xpu() +) requires_int8_moe = pytest.mark.skipif( not INT8_MOE_SUPPORTED, - reason="Requires a GPU with Triton INT8 MoE support (CUDA SM>=7.5 or ROCm)", + reason="Requires Triton INT8 MoE support (CUDA SM>=7.5, ROCm, or XPU)", ) +# So FusedMoEConfig.device names the device the test actually runs on. +DEVICE = current_platform.device_type + def _make_int8_moe_config(moe_backend: str = "auto") -> FusedMoEConfig: from vllm.model_executor.layers.fused_moe.activation import MoEActivation @@ -52,7 +57,7 @@ def _make_int8_moe_config(moe_backend: str = "auto") -> FusedMoEConfig: moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(), activation=MoEActivation.SILU, in_dtype=torch.bfloat16, - device="cuda", + device=DEVICE, routing_method=RoutingMethodType.Renormalize, moe_backend=moe_backend, ) @@ -62,15 +67,20 @@ def _make_int8_moe_config(moe_backend: str = "auto") -> FusedMoEConfig: @pytest.mark.parametrize( "weight_key,activation_key", [ - # per-channel weight + dynamic per-token activation - (kInt8StaticChannelSym, kInt8DynamicTokenSym), - # per-tensor weight + dynamic per-tensor activation - (kInt8StaticTensorSym, kInt8DynamicTensorSym), + pytest.param( + kInt8StaticChannelSym, + kInt8DynamicTokenSym, + id="per_channel_weight-per_token_act", + ), + pytest.param( + kInt8StaticTensorSym, + kInt8DynamicTensorSym, + id="per_tensor_weight-per_tensor_act", + ), ], ) def test_int8_dynamic_schemes_dispatch_to_triton(weight_key, activation_key): - """Both dynamic-activation INT8 MoE schemes (per-channel + per-tensor - weights) select the Triton backend.""" + """Both dynamic-activation INT8 MoE schemes select the Triton backend.""" config = _make_int8_moe_config() backend, experts_cls = select_int8_moe_backend( config, weight_key=weight_key, activation_key=activation_key @@ -79,6 +89,28 @@ def test_int8_dynamic_schemes_dispatch_to_triton(weight_key, activation_key): assert experts_cls is not None +@pytest.mark.skipif(not current_platform.is_xpu(), reason="XPU-only behaviour") +def test_scaled_int8_quant_is_available_on_xpu(): + """The per-tensor scheme's activation quantization works on XPU. + + ``_int8_quantize`` sends per-tensor activations through + ``ops.scaled_int8_quant``, which has an XPU branch. This is what lets the + per-tensor scheme be enabled here alongside CUDA and ROCm; pin it so a + future change cannot silently remove that branch. + """ + from vllm import _custom_ops as ops + + x = torch.randn(4, 16, device=DEVICE, dtype=torch.bfloat16) + scale = torch.full((1,), 0.05, device=DEVICE, dtype=torch.float32) + + q, returned_scale, azp = ops.scaled_int8_quant(x, scale=scale) + + assert q.dtype == torch.int8 + assert q.shape == x.shape + assert azp is None + torch.testing.assert_close(returned_scale, scale) + + @requires_int8_moe def test_int8_explicit_moe_backend_triton(): """An explicit --moe-backend triton selects the Triton INT8 backend.""" diff --git a/tests/quantization/test_modelopt.py b/tests/quantization/test_modelopt.py index 60b207c184..ae8548c370 100644 --- a/tests/quantization/test_modelopt.py +++ b/tests/quantization/test_modelopt.py @@ -144,6 +144,36 @@ def test_modelopt_mxfp8_preserves_per_row_checkpoint_scales(dist_init, monkeypat assert torch.equal(linear.weight_scale, scales) +def test_modelopt_mxfp8_pre_processed_weights_follow_kernel(monkeypatch): + """The weight cache daemon can only serve MXFP8 layers whose kernel needs + no post-load state beyond the parameters it exports.""" + from vllm.config.quantization import QuantSpec + from vllm.model_executor.kernels.linear.mxfp8 import Mxfp8LinearKernel + + method = ModelOptLinearMethod.__new__(ModelOptLinearMethod) + method.spec = QuantSpec(weight=kMxfp8Static, activation=kMxfp8Dynamic) + method.kernel = None + assert not method.supports_pre_processed_weights + + method.kernel = Mock(spec=Mxfp8LinearKernel) + method.kernel.supports_pre_processed_weights = False + assert not method.supports_pre_processed_weights + method.kernel.supports_pre_processed_weights = True + assert method.supports_pre_processed_weights + + monkeypatch.setattr( + "vllm.model_executor.layers.quantization.modelopt.is_weights_pre_processed", + lambda: True, + ) + layer = torch.nn.Module() + method.process_weights_after_loading(layer) + method.kernel.process_weights_after_loading.assert_not_called() + + method.kernel.supports_pre_processed_weights = False + with pytest.raises(RuntimeError, match="pre-processed"): + method.process_weights_after_loading(layer) + + def test_modelopt_fp8_updates_weight_dims_after_transpose(): """Humming reads weight.input_dim/output_dim. Swapping the ModelWeightParameter for a plain Parameter drops them, so the per-tensor @@ -341,7 +371,7 @@ def test_modelopt_mixed_precision_composes_gemma4_mappers(): "quant_algo": "NVFP4", "group_size": 16, }, - "model.language_model.layers.1.moe.experts.gate_up_proj": { + "model.language_model.layers.1.experts.gate_up_proj": { "quant_algo": "NVFP4", "group_size": 16, }, @@ -353,10 +383,10 @@ def test_modelopt_mixed_precision_composes_gemma4_mappers(): ) config.apply_vllm_mapper(Gemma4ForCausalLM.hf_to_vllm_mapper.get_rename_mapper()) - expected_prefix = "language_model.model.layers.0.moe.experts" + expected_prefix = "language_model.model.layers.0.experts" assert set(config.quantized_layers) == { expected_prefix, - "language_model.model.layers.1.moe.gate_up_proj", + "language_model.model.layers.1.experts.gate_up_proj", } assert config._resolve_quant_algo(expected_prefix) == "NVFP4" diff --git a/tests/quantization/test_quark.py b/tests/quantization/test_quark.py index ced034cb1a..39481456fd 100644 --- a/tests/quantization/test_quark.py +++ b/tests/quantization/test_quark.py @@ -58,12 +58,17 @@ from vllm.model_executor.layers.quantization.quark.quark import ( # noqa: E501 from vllm.model_executor.layers.quantization.quark.quark_moe import ( # noqa: E501 QuarkMoEMethod, QuarkW4A8Fp8MoEMethod, + QuarkW4A16Int4MoEMethod, QuarkW8A8Fp8MoEMethod, QuarkW8A8Int8MoEMethod, ) -from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme +from vllm.model_executor.layers.quantization.quark.schemes import ( + QuarkScheme, + QuarkW4A16Int4, +) from vllm.model_executor.layers.quantization.quark.utils import ( QuarkQTensorHint, + canonicalize_quark_packed_int4, should_ignore_layer, ) from vllm.model_executor.layers.quantization.utils.mxfp4_utils import ( @@ -97,6 +102,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( kNvfp4Static, ) from vllm.model_executor.models.llama import LlamaForCausalLM +from vllm.model_executor.models.utils import WeightsMapper from vllm.platforms import current_platform from vllm.transformers_utils.repo_utils import hf_api @@ -1876,3 +1882,435 @@ def test_glm5next_attn_loader_accepts_deepseek_weight_scale_inv(): ) assert target.loaded_shard is None # o_proj is a direct (non-fused) proj assert torch.equal(target.loaded_weight, _dequant_fp8_block(weight, scale, 128)) + + +_REVERSE_AWQ_PACK_ORDER = [0, 4, 1, 5, 2, 6, 3, 7] + + +def _quark_int4_config( + *, + pack_method: str = "reorder", + symmetric: bool = True, + exclude: list[str] | None = None, +) -> dict: + return { + "quant_method": "quark", + "export": {"pack_method": pack_method, "kv_cache_group": []}, + "global_quant_config": { + "weight": { + "dtype": "int4", + "group_size": 128, + "symmetric": symmetric, + } + }, + "exclude": exclude or [], + } + + +def _sign_extend_int4_nibbles(t: torch.Tensor) -> torch.Tensor: + mask = (t & 0x8).bool() + t = t.clone() + t[mask] = t[mask] | 0xF0 + return t + + +def _dequantize_quark_signed_awq_torch( + qweight: torch.Tensor, + scales: torch.Tensor, + qzeros: torch.Tensor, + group_size: int, + *, + pack_reorder: bool = True, +) -> torch.Tensor: + bits = 4 + shifts = torch.arange(0, 32, bits, device=qweight.device) + iweights = ((qweight[:, :, None] >> shifts[None, None, :]) & 0xF).to(torch.int8) + iweights = iweights.view(qweight.shape[0], -1) + zeros = ((qzeros[:, :, None] >> shifts[None, None, :]) & 0xF).to(torch.int8) + zeros = zeros.view(qzeros.shape[0], -1) + + if pack_reorder: + order = torch.tensor(_REVERSE_AWQ_PACK_ORDER, device=qweight.device) + else: + order = torch.arange(8, device=qweight.device) + iweights = iweights.view(qweight.shape[0], -1, 8)[:, :, order].reshape( + qweight.shape[0], -1 + ) + zeros = zeros.view(qzeros.shape[0], -1, 8)[:, :, order].reshape(qzeros.shape[0], -1) + iweights = _sign_extend_int4_nibbles(iweights & 0xF) + zeros = _sign_extend_int4_nibbles(zeros & 0xF) + + scales = scales.repeat_interleave(group_size, dim=0) + zeros = zeros.repeat_interleave(group_size, dim=0) + return (iweights - zeros) * scales + + +def _dequantize_awq_unsigned_torch( + qweight: torch.Tensor, + scales: torch.Tensor, + qzeros: torch.Tensor, + group_size: int, + *, + pack_reorder: bool = True, +) -> torch.Tensor: + bits = 4 + shifts = torch.arange(0, 32, bits, device=qweight.device) + iweights = ((qweight[:, :, None] >> shifts[None, None, :]) & 0xF).to(torch.int8) + iweights = iweights.view(qweight.shape[0], -1) + zeros = ((qzeros[:, :, None] >> shifts[None, None, :]) & 0xF).to(torch.int8) + zeros = zeros.view(qzeros.shape[0], -1) + + if pack_reorder: + order = torch.tensor(_REVERSE_AWQ_PACK_ORDER, device=qweight.device) + else: + order = torch.arange(8, device=qweight.device) + iweights = iweights.view(qweight.shape[0], -1, 8)[:, :, order].reshape( + qweight.shape[0], -1 + ) + zeros = zeros.view(qzeros.shape[0], -1, 8)[:, :, order].reshape(qzeros.shape[0], -1) + + scales = scales.repeat_interleave(group_size, dim=0) + zeros = zeros.repeat_interleave(group_size, dim=0) + return (iweights - zeros) * scales + + +def _pack_int4_nibbles(nibbles: torch.Tensor, *, pack_reorder: bool) -> torch.Tensor: + pack_order = ( + torch.tensor(_REVERSE_AWQ_PACK_ORDER, device=nibbles.device) + if pack_reorder + else torch.arange(8, device=nibbles.device) + ) + shifts = pack_order * 4 + return ((nibbles.to(torch.int64) & 0xF) << shifts).sum(dim=-1).to(torch.int32) + + +class TestQuarkInt4Format: + """Tests for Quark INT4 export format compatibility.""" + + def test_quark_order_pack_method_uses_native_int4_scheme(self): + quant_config = QuarkConfig.from_config(_quark_int4_config(pack_method="order")) + weight_config = quant_config.quant_config["global_quant_config"] + weight_key, act_key, scheme_cls = quant_config._get_scheme_cls_from_config( + weight_config + ) + assert scheme_cls is QuarkW4A16Int4 + + scheme = quant_config.init_scheme( + scheme_cls, + weight_quant_key=weight_key, + activation_quant_key=act_key, + weight_config=weight_config["weight"], + ) + assert isinstance(scheme, QuarkW4A16Int4) + assert not scheme.pack_reorder + + def test_quark_int4_scheme_supports_asymmetric_weights(self): + quant_config = QuarkConfig.from_config(_quark_int4_config(symmetric=False)) + weight_config = quant_config.quant_config["global_quant_config"] + weight_key, act_key, scheme_cls = quant_config._get_scheme_cls_from_config( + weight_config + ) + assert scheme_cls is QuarkW4A16Int4 + + scheme = quant_config.init_scheme( + scheme_cls, + weight_quant_key=weight_key, + activation_quant_key=act_key, + weight_config=weight_config["weight"], + ) + assert isinstance(scheme, QuarkW4A16Int4) + assert not scheme.is_symmetric + + @pytest.mark.parametrize("missing_field", ["group_size", "symmetric"]) + def test_quark_int4_scheme_requires_weight_config_fields(self, missing_field): + from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kInt4Static, + ) + + weight_config = { + "dtype": "int4", + "group_size": 128, + "symmetric": True, + } + weight_config.pop(missing_field) + quant_config = QuarkConfig.from_config( + { + "quant_method": "quark", + "export": {"pack_method": "reorder", "kv_cache_group": []}, + "global_quant_config": {"weight": weight_config}, + "exclude": [], + } + ) + + with pytest.raises(ValueError, match=missing_field): + quant_config.init_scheme( + QuarkW4A16Int4, + weight_quant_key=kInt4Static, + activation_quant_key=None, + weight_config=weight_config, + ) + + def test_quark_int4_moe_uses_native_moe_method(self): + from unittest.mock import MagicMock, patch + + from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kInt4StaticAsym, + ) + + quant_config = QuarkConfig.from_config(_quark_int4_config(symmetric=False)) + moe_config = type("MoeConfig", (), {})() + moe_config.has_bias = False + + mock_backend = MagicMock() + mock_experts_cls = MagicMock() + with patch( + "vllm.model_executor.layers.quantization.quark.quark_moe" + ".select_wna16_moe_backend", + return_value=(mock_backend, mock_experts_cls), + ): + method = QuarkW4A16Int4MoEMethod( + kInt4StaticAsym, + None, + quant_config.quant_config["global_quant_config"]["weight"], + quant_config.pack_method, + moe_config, + ) + + assert method.group_size == 128 + assert method.pack_reorder + assert method.use_wna16_backend + + def test_triton_wna16_experts_supports_asymmetric_int4(self): + from vllm.model_executor.layers.fused_moe.experts.triton_moe import ( + TritonWNA16Experts, + ) + from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kInt4Static, + kInt4Static32, + kInt4Static32Asym, + kInt4StaticAsym, + ) + + for key in [kInt4Static, kInt4Static32, kInt4StaticAsym, kInt4Static32Asym]: + assert TritonWNA16Experts._supports_quant_scheme(key, None), ( + f"{key} should be supported by TritonWNA16Experts" + ) + + def test_quark_moe_loader_shards_w2_zero_point_across_tp_ranks(self): + """w2 zero-points are sharded along the intermediate dim per TP rank. + + Loading the same Quark checkpoint tensor at tensor_parallel_size=2 for + both ranks must jointly reconstruct the tensor_parallel_size=1 (full) + result, with no overlap or gap. This guards the custom loader's TP + split for the packed zero-point layout. + """ + + class _FakeMoEConfig: + has_bias = False + + def __init__(self, tp_size, tp_rank): + self.tp_size = tp_size + self.tp_rank = tp_rank + + class _FakeLayer: + def __init__(self, moe_config): + self.moe_config = moe_config + self.intermediate_size_per_partition = 64 + self.group_size_div_factor = 1 + + def load_w2_zero_point(raw_zp, tp_size, tp_rank): + from unittest.mock import MagicMock, patch + + from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kInt4StaticAsym, + ) + + quant_config = QuarkConfig.from_config(_quark_int4_config(symmetric=False)) + moe_config = _FakeMoEConfig(tp_size, tp_rank) + with patch( + "vllm.model_executor.layers.quantization.quark.quark_moe" + ".select_wna16_moe_backend", + return_value=(MagicMock(), MagicMock()), + ): + method = QuarkW4A16Int4MoEMethod( + kInt4StaticAsym, + None, + quant_config.quant_config["global_quant_config"]["weight"], + quant_config.pack_method, + moe_config, + ) + + layer = _FakeLayer(moe_config) + # The loader converts the packed zero-point, then writes the + # per-rank slice into param.data[expert_id]. A single expert dim is + # enough; the last dim shrinks by tp_size after sharding. + param = torch.nn.Parameter( + torch.zeros(1, 16, 8 // tp_size, dtype=torch.uint8), + requires_grad=False, + ) + loader = method.get_weight_loader(layer, weight_loader=None) + loader( + param, + raw_zp.clone(), + weight_name="w2_weight_zero_point", + shard_id="w2", + expert_id=0, + ) + return param.data[0] + + # Quark-packed zero-point for one expert: rows = hidden // pack_factor, + # cols carry the (packed) intermediate dim, split evenly across ranks. + raw_zp = torch.randint(0, 256, (8, 16), dtype=torch.uint8) + + full = load_w2_zero_point(raw_zp, tp_size=1, tp_rank=0) + shard0 = load_w2_zero_point(raw_zp, tp_size=2, tp_rank=0) + shard1 = load_w2_zero_point(raw_zp, tp_size=2, tp_rank=1) + + expected = full.view(full.size(0), 2, -1) + torch.testing.assert_close(shard0, expected[:, 0]) + torch.testing.assert_close(shard1, expected[:, 1]) + + def test_quark_shared_expert_gate_keeps_quantized_tensors(self): + quant_config = QuarkConfig.from_config(_quark_int4_config()) + mapper = quant_config.get_cache_scale_mapper() + + output_names = { + name + for name, _ in mapper.apply( + [ + ( + "model.language_model.layers.0.mlp.shared_expert_gate.weight", + torch.zeros(1), + ), + ( + "model.language_model.layers.0.mlp.shared_expert_gate" + ".weight_scale", + torch.zeros(1), + ), + ( + "model.language_model.layers.0.mlp.shared_expert_gate" + ".weight_zero_point", + torch.zeros(1), + ), + ] + ) + } + + assert ( + "model.language_model.layers.0.mlp.shared_expert_gate.weight" + in output_names + ) + assert ( + "model.language_model.layers.0.mlp.shared_expert_gate.weight_scale" + in output_names + ) + assert ( + "model.language_model.layers.0.mlp.shared_expert_gate.weight_zero_point" + in output_names + ) + + def test_quark_apply_mapper_updates_exclude_and_layer_quant_config(self): + quant_config = QuarkConfig.from_config( + { + **_quark_int4_config(), + "exclude": ["lm_head"], + "layer_quant_config": { + "model.language_model.layers.0.mlp.gate": { + "weight": {"dtype": "float16"}, + }, + }, + } + ) + quant_config.apply_vllm_mapper( + WeightsMapper( + orig_to_new_prefix={ + "lm_head": "language_model.lm_head", + "model.language_model.": "language_model.model.", + }, + ) + ) + + layer_quant_config = quant_config.quant_config["layer_quant_config"] + assert "language_model.model.layers.0.mlp.gate" in layer_quant_config + assert "model.language_model.layers.0.mlp.gate" not in layer_quant_config + assert quant_config.quant_config["exclude"] == [ + "language_model.lm_head", + ] + + def test_quark_apply_mapper_ignores_non_string_list_entries(self): + quant_config = QuarkConfig.from_config( + { + **_quark_int4_config(), + "algo_config": [ + {"name": "qronos", "inside_layer_modules": ["self_attn.q_proj"]} + ], + } + ) + quant_config.apply_vllm_mapper(WeightsMapper()) + + assert quant_config.quant_config["algo_config"] == [ + {"name": "qronos", "inside_layer_modules": ["self_attn.q_proj"]} + ] + + def test_quark_exclude_matches_via_mapper_prefix(self): + quant_config = QuarkConfig.from_config(_quark_int4_config(exclude=["lm_head"])) + quant_config.apply_vllm_mapper( + WeightsMapper(orig_to_new_prefix={"lm_head": "language_model.lm_head"}) + ) + + exclude_layers = quant_config.quant_config["exclude"] + assert should_ignore_layer("language_model.lm_head", ignore=exclude_layers) + assert not should_ignore_layer( + "language_model.model.layers.0.mlp.gate", ignore=exclude_layers + ) + + +@pytest.mark.parametrize("symmetric", [False, True]) +@pytest.mark.parametrize("pack_method", ["order", "reorder"]) +def test_quark_int4_canonicalizes_pack_for_kernel_layout(pack_method, symmetric): + """Quark pack order is normalized to the layout expected by awq_* ops. + + This reuses the existing AWQ dequant/gemm kernels for compute only; loading + still goes through the native Quark quantization path, not AutoAWQ. + """ + pack_reorder = pack_method == "reorder" + group_size = 2 + packed_values = torch.tensor( + [ + [0, 1, 7, 8, 9, 15, 2, 14], + [15, 8, 0, 3, 12, 7, 1, 9], + ], + dtype=torch.int32, + ) + packed_zeros = torch.zeros((1, 8), dtype=torch.int32) + qweight = _pack_int4_nibbles(packed_values, pack_reorder=pack_reorder).view(2, 1) + qzeros = _pack_int4_nibbles(packed_zeros, pack_reorder=pack_reorder).view(1, 1) + scales = torch.ones((1, 8), dtype=torch.float16) + + dequantize = ( + _dequantize_quark_signed_awq_torch + if symmetric + else _dequantize_awq_unsigned_torch + ) + expected = dequantize( + qweight, + scales, + qzeros, + group_size, + pack_reorder=pack_reorder, + ) + canonical_weight = canonicalize_quark_packed_int4( + qweight, + pack_reorder=pack_reorder, + is_symmetric=symmetric, + ) + canonical_zero = canonicalize_quark_packed_int4( + qzeros, + pack_reorder=pack_reorder, + is_symmetric=symmetric, + ) + actual = _dequantize_awq_unsigned_torch( + canonical_weight, scales, canonical_zero, group_size + ) + + assert torch.equal(actual, expected) diff --git a/tests/renderers/test_warmup.py b/tests/renderers/test_warmup.py index f6f9f7a4db..2627cdd985 100644 --- a/tests/renderers/test_warmup.py +++ b/tests/renderers/test_warmup.py @@ -5,6 +5,8 @@ These tests exercise: - Zero-limit modalities are filtered from mm_counts passed to get_dummy_processor_inputs (e.g. --limit-mm-per-prompt image=0 ...) + - The warmup runs through the processor-only cache and clears it afterwards, + so its dummy inputs never land in the sender cache used by the serving path - MM warmup is skipped entirely when mm_processor is None - The multimodal warmup is launched as a task on the single-worker _mm_executor to overlap engine-core init (future lifecycle, join by @@ -21,10 +23,13 @@ that acts as the renderer instance. """ from concurrent.futures import Future, ThreadPoolExecutor +from types import MethodType, SimpleNamespace from unittest.mock import MagicMock, patch import pytest +from vllm.entrypoints.chat_utils import ChatTemplateResolutionError +from vllm.multimodal.processing import BaseMultiModalProcessor from vllm.renderers.base import BaseRenderer from vllm.renderers.params import ChatParams @@ -34,9 +39,11 @@ def _make_renderer_mock(mm_limits: dict[str, int]) -> MagicMock: render_chat is mocked to raise ChatTemplateResolutionError so the chat warmup block is skipped cleanly, keeping the test focused on MM warmup. - """ - from vllm.entrypoints.chat_utils import ChatTemplateResolutionError + The processor is a mock at its I/O boundaries (``dummy_inputs`` and + ``apply``) but keeps the real ``get_dummy_mm_inputs``, since that is where + the warmup's seq_len capping and cache plumbing actually live. + """ renderer = MagicMock() # chat warmup: make render_chat raise so we skip past it cleanly @@ -45,13 +52,17 @@ def _make_renderer_mock(mm_limits: dict[str, int]) -> MagicMock: # MM processor with configurable limits mm_processor = MagicMock() mm_processor.info.allowed_mm_limits = mm_limits + mm_processor.info.ctx.model_config = renderer.model_config mm_processor.apply.return_value = {"prompt_token_ids": [1]} + mm_processor.get_dummy_mm_inputs = MethodType( + BaseMultiModalProcessor.get_dummy_mm_inputs, mm_processor + ) renderer.mm_processor = mm_processor - renderer._readonly_mm_processor = None + renderer._mm_processor_cache = MagicMock(name="sender_cache") + renderer._mm_processor_only_cache = MagicMock(name="processor_only_cache") renderer._warmup_mm_processor = BaseRenderer._warmup_mm_processor.__get__( renderer, BaseRenderer ) - renderer._clear_processor_cache = BaseRenderer._clear_processor_cache renderer.warmup_mm = BaseRenderer.warmup_mm.__get__(renderer, BaseRenderer) renderer.start_mm_warmup_in_background = ( BaseRenderer.start_mm_warmup_in_background.__get__(renderer, BaseRenderer) @@ -64,7 +75,6 @@ def _make_renderer_mock(mm_limits: dict[str, int]) -> MagicMock: renderer._mm_warmup_future = None # MM warmup has not run yet; warmup_mm must actually execute on the mock. renderer._mm_warmup_done = False - renderer.clear_mm_cache = MagicMock() renderer.model_config.max_model_len = 128 renderer.config.scheduler_config.max_num_batched_tokens = 8192 renderer.config.scheduler_config.enable_chunked_prefill = True @@ -80,8 +90,7 @@ class TestMmWarmupZeroLimitFiltering: """A modality with limit=0 must not appear in mm_counts.""" renderer = _make_renderer_mock({"image": 1, "video": 0}) - with patch("vllm.multimodal.processing.TimingContext", autospec=True): - BaseRenderer.warmup(renderer, ChatParams()) + BaseRenderer.warmup(renderer, ChatParams()) get_inputs = renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs get_inputs.assert_called_once() @@ -93,8 +102,7 @@ class TestMmWarmupZeroLimitFiltering: """When all limits are 0, mm_counts must be empty.""" renderer = _make_renderer_mock({"image": 0, "video": 0}) - with patch("vllm.multimodal.processing.TimingContext", autospec=True): - BaseRenderer.warmup(renderer, ChatParams()) + BaseRenderer.warmup(renderer, ChatParams()) get_inputs = renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs get_inputs.assert_called_once() @@ -105,8 +113,7 @@ class TestMmWarmupZeroLimitFiltering: """All modalities with limit > 0 must be present in mm_counts.""" renderer = _make_renderer_mock({"image": 2, "video": 1}) - with patch("vllm.multimodal.processing.TimingContext", autospec=True): - BaseRenderer.warmup(renderer, ChatParams()) + BaseRenderer.warmup(renderer, ChatParams()) get_inputs = renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs get_inputs.assert_called_once() @@ -125,8 +132,7 @@ class TestMmWarmupRunsNormally: renderer = _make_renderer_mock({"image": 1, "video": 1}) renderer.model_config.max_model_len = max_model_len - with patch("vllm.multimodal.processing.TimingContext", autospec=True): - BaseRenderer.warmup(renderer, ChatParams()) + BaseRenderer.warmup(renderer, ChatParams()) get_inputs = renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs assert get_inputs.call_args.kwargs["seq_len"] == expected_seq_len @@ -134,28 +140,46 @@ class TestMmWarmupRunsNormally: def test_processor_apply_called(self): renderer = _make_renderer_mock({"image": 1}) - with patch("vllm.multimodal.processing.TimingContext", autospec=True): - BaseRenderer.warmup(renderer, ChatParams()) + BaseRenderer.warmup(renderer, ChatParams()) renderer.mm_processor.apply.assert_called_once() - def test_mm_cache_cleared_after_warmup(self): - renderer = _make_renderer_mock({"image": 1}) - - with patch("vllm.multimodal.processing.TimingContext", autospec=True): - BaseRenderer.warmup(renderer, ChatParams()) - - renderer.clear_mm_cache.assert_called_once() - def test_render_chat_called_with_warmup_message(self): renderer = _make_renderer_mock({"image": 1}) - with patch("vllm.multimodal.processing.TimingContext", autospec=True): - BaseRenderer.warmup(renderer, ChatParams()) + BaseRenderer.warmup(renderer, ChatParams()) renderer.render_chat.assert_called_once() +class TestMmWarmupCacheRouting: + # The warmup runs through the processor-only cache and drops it afterwards, + # so the dummy inputs never reach the sender cache used by the serving path. + + def test_warmup_passes_processor_only_cache(self): + renderer = _make_renderer_mock({"image": 1}) + + with patch.object( + renderer.mm_processor, + "get_dummy_mm_inputs", + wraps=renderer.mm_processor.get_dummy_mm_inputs, + ) as spy: + BaseRenderer.warmup(renderer, ChatParams()) + + assert spy.call_args.kwargs["cache"] is renderer._mm_processor_only_cache + assert ( + spy.call_args.kwargs["scheduler_config"] is renderer.config.scheduler_config + ) + + def test_only_cache_cleared_and_sender_cache_untouched(self): + renderer = _make_renderer_mock({"image": 1}) + + BaseRenderer.warmup(renderer, ChatParams()) + + renderer._mm_processor_only_cache.clear_cache.assert_called_once() + renderer._mm_processor_cache.clear_cache.assert_not_called() + + class TestMmWarmupSkippedWhenNoProcessor: """MM warmup must be skipped when mm_processor is None (text-only model).""" @@ -168,51 +192,30 @@ class TestMmWarmupSkippedWhenNoProcessor: renderer.model_config.get_multimodal_config.assert_not_called() -class TestReadonlyMmWarmup: - """Readonly MM processor warmup must mirror the render path behavior.""" - - def test_readonly_processor_apply_called_and_cache_cleared(self): - renderer = _make_renderer_mock({"image": 1}) - readonly_mm_processor = MagicMock() - readonly_mm_processor.info.allowed_mm_limits = {"image": 1} - renderer._readonly_mm_processor = readonly_mm_processor - - with patch("vllm.multimodal.processing.TimingContext", autospec=True): - BaseRenderer.warmup(renderer, ChatParams()) - - readonly_mm_processor.apply.assert_called_once() - readonly_mm_processor.cache.clear_cache.assert_called_once() - - class TestWarmupFaultIsolation: - # A failure during a multimodal processor warmup is caught so it does not - # abort the remaining warmup steps; warmup itself must not raise. + # A failure during a warmup step is caught so it does not abort the other + # step; warmup itself must not raise. def test_chat_failure_does_not_abort_mm_warmup(self): renderer = _make_renderer_mock({"image": 1}) renderer.render_chat.side_effect = RuntimeError("chat boom") - with patch("vllm.multimodal.processing.TimingContext", autospec=True): - BaseRenderer.warmup(renderer, ChatParams()) # must not raise + BaseRenderer.warmup(renderer, ChatParams()) # must not raise renderer.mm_processor.apply.assert_called_once() - def test_mm_failure_does_not_abort_readonly_warmup(self): + def test_mm_failure_is_swallowed_and_cache_cleared(self): renderer = _make_renderer_mock({"image": 1}) - readonly_mm_processor = MagicMock() - readonly_mm_processor.info.allowed_mm_limits = {"image": 1} - renderer._readonly_mm_processor = readonly_mm_processor - # main processor warmup blows up before apply() + # the warmup blows up before apply() renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs.side_effect = ( RuntimeError("mm boom") ) - with patch("vllm.multimodal.processing.TimingContext", autospec=True): - BaseRenderer.warmup(renderer, ChatParams()) # must not raise + BaseRenderer.warmup(renderer, ChatParams()) # must not raise - readonly_mm_processor.apply.assert_called_once() - # cache is still cleared in the failed task's finally - renderer.clear_mm_cache.assert_called_once() + renderer.mm_processor.apply.assert_not_called() + # the cache is still cleared in the failed warmup's finally + renderer._mm_processor_only_cache.clear_cache.assert_called_once() class TestBackgroundMmWarmup: @@ -234,16 +237,15 @@ class TestBackgroundMmWarmup: def test_start_submits_future_to_mm_executor(self): renderer = self._make_bg_renderer({"image": 1}) try: - with patch("vllm.multimodal.processing.TimingContext", autospec=True): - # Spy on the real executor's submit to confirm the warmup is - # dispatched through _mm_executor (not a separate Thread). - with patch.object( - renderer._mm_executor, "submit", wraps=renderer._mm_executor.submit - ) as spy: - renderer.start_mm_warmup_in_background() - assert isinstance(renderer._mm_warmup_future, Future) - assert spy.called - renderer._join_mm_warmup() + # Spy on the real executor's submit to confirm the warmup is + # dispatched through _mm_executor (not a separate Thread). + with patch.object( + renderer._mm_executor, "submit", wraps=renderer._mm_executor.submit + ) as spy: + renderer.start_mm_warmup_in_background() + assert isinstance(renderer._mm_warmup_future, Future) + assert spy.called + renderer._join_mm_warmup() assert renderer._mm_warmup_future is None renderer.mm_processor.apply.assert_called_once() finally: @@ -252,7 +254,6 @@ class TestBackgroundMmWarmup: def test_start_noop_for_text_only_model(self): renderer = _make_renderer_mock({}) renderer.mm_processor = None - # _readonly_mm_processor is already None renderer.start_mm_warmup_in_background() @@ -261,12 +262,11 @@ class TestBackgroundMmWarmup: def test_start_is_run_at_most_once(self): renderer = self._make_bg_renderer({"image": 1}) try: - with patch("vllm.multimodal.processing.TimingContext", autospec=True): - renderer.start_mm_warmup_in_background() - first = renderer._mm_warmup_future - renderer.start_mm_warmup_in_background() # must not spawn a second - assert renderer._mm_warmup_future is first - renderer._join_mm_warmup() + renderer.start_mm_warmup_in_background() + first = renderer._mm_warmup_future + renderer.start_mm_warmup_in_background() # must not spawn a second + assert renderer._mm_warmup_future is first + renderer._join_mm_warmup() finally: renderer._mm_executor.shutdown(wait=True) @@ -275,9 +275,8 @@ class TestBackgroundMmWarmup: # run only the chat warmup — the MM warmup must not run twice. renderer = self._make_bg_renderer({"image": 1}) try: - with patch("vllm.multimodal.processing.TimingContext", autospec=True): - renderer.start_mm_warmup_in_background() - BaseRenderer.warmup(renderer, ChatParams()) + renderer.start_mm_warmup_in_background() + BaseRenderer.warmup(renderer, ChatParams()) # MM apply called exactly once (by the background warmup task). renderer.mm_processor.apply.assert_called_once() @@ -294,13 +293,12 @@ class TestBackgroundMmWarmup: # re-run the MM warmup — the _mm_warmup_done flag survives the join. renderer = self._make_bg_renderer({"image": 1}) try: - with patch("vllm.multimodal.processing.TimingContext", autospec=True): - renderer.start_mm_warmup_in_background() - # Simulate reset_mm_cache joining the background warmup first - # (clears _mm_warmup_future, sets _mm_warmup_done via the join). - renderer._join_mm_warmup() - assert renderer._mm_warmup_future is None # joined & cleared - BaseRenderer.warmup(renderer, ChatParams()) + renderer.start_mm_warmup_in_background() + # Simulate reset_mm_cache joining the background warmup first + # (clears _mm_warmup_future, sets _mm_warmup_done via the join). + renderer._join_mm_warmup() + assert renderer._mm_warmup_future is None # joined & cleared + BaseRenderer.warmup(renderer, ChatParams()) # MM apply called exactly once (by the background warmup task); # warmup() did not re-run warmup_mm despite the future being cleared. @@ -313,9 +311,8 @@ class TestBackgroundMmWarmup: # Direct repeated calls to warmup_mm run the MM warmup only once. renderer = _make_renderer_mock({"image": 1}) - with patch("vllm.multimodal.processing.TimingContext", autospec=True): - renderer.warmup_mm() - renderer.warmup_mm() # second call must be a no-op + renderer.warmup_mm() + renderer.warmup_mm() # second call must be a no-op renderer.mm_processor.apply.assert_called_once() @@ -324,10 +321,9 @@ class TestBackgroundMmWarmup: # so the mm_processor_cache is never touched concurrently. renderer = self._make_bg_renderer({"image": 1}) - with patch("vllm.multimodal.processing.TimingContext", autospec=True): - renderer.start_mm_warmup_in_background() - future = renderer._mm_warmup_future - renderer.shutdown() + renderer.start_mm_warmup_in_background() + future = renderer._mm_warmup_future + renderer.shutdown() assert renderer._mm_warmup_future is None # The background warmup was allowed to complete (apply ran) and the @@ -344,10 +340,6 @@ class TestEngineStartWarmupHook: # These tests pin the renderer plumbing contract on EngineCoreClient. def _mock_config(self): - from types import SimpleNamespace - - from vllm.v1.engine.core_client import EngineCoreClient # noqa: F401 - cfg = SimpleNamespace( parallel_config=SimpleNamespace( data_parallel_size=1, diff --git a/tests/test_config.py b/tests/test_config.py index 5c892da6c6..9da6eced94 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -46,6 +46,7 @@ from vllm.config.utils import get_field from vllm.config.vllm import OPTIMIZATION_LEVEL_TO_CONFIG, OptimizationLevel from vllm.platforms import current_platform from vllm.transformers_utils.config import ( + _patch_hf_transformers_nested_rope_validation, get_pooling_config, try_get_dense_modules, ) @@ -54,6 +55,42 @@ from vllm.v1.attention.backend import AttentionCGSupport DEVICE_TYPE = current_platform.device_type +def test_nested_rope_validation_patch_preserves_flat_rope_parameters(monkeypatch): + calls = [] + + def original_validate_rope(config, *args, **kwargs): + calls.append(config) + + from transformers import PretrainedConfig + + monkeypatch.setattr(PretrainedConfig, "validate_rope", original_validate_rope) + _patch_hf_transformers_nested_rope_validation() + + nested_rope_parameters = { + "full_attention": {"rope_type": "default"}, + "original_max_position_embeddings": 32768, + } + PretrainedConfig.validate_rope( + SimpleNamespace(rope_parameters=nested_rope_parameters) + ) + assert nested_rope_parameters == {"full_attention": {"rope_type": "default"}} + + flat_rope_parameters = { + "rope_type": "linear", + "factor": 8.0, + "rope_theta": 500000.0, + } + PretrainedConfig.validate_rope( + SimpleNamespace(rope_parameters=flat_rope_parameters) + ) + assert flat_rope_parameters == { + "rope_type": "linear", + "factor": 8.0, + "rope_theta": 500000.0, + } + assert len(calls) == 2 + + def _write_json(path: Path, value: object) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(value), encoding="utf-8") @@ -615,6 +652,15 @@ def test_v2_model_runner_supports_extract_hidden_states(): assert config._get_v2_model_runner_unsupported_features() == [] +def test_v2_model_runner_supports_custom_logits_processors(): + config = VllmConfig() + config.model_config = cast( + ModelConfig, SimpleNamespace(logits_processors=["a.b:C"]) + ) + + assert config._get_v2_model_runner_unsupported_features() == [] + + def test_dflash2_draft_forces_v2_model_runner(): """A DFlash2 draft must reach the V2 speculator, the only one that runs its candidate selector; on V1 it would draft as DFlash1 without raising.""" diff --git a/tests/test_envs.py b/tests/test_envs.py index 1ef40cde0c..dfd08a9ec2 100644 --- a/tests/test_envs.py +++ b/tests/test_envs.py @@ -18,6 +18,11 @@ from vllm.envs import ( from vllm.exceptions import VLLMValidationError +def test_object_storage_shm_default_name(): + """The generated name must fit macOS's shared-memory name limit.""" + assert len(envs._generate_shm_name()) <= 30 + + def test_getattr_without_cache(monkeypatch: pytest.MonkeyPatch): assert envs.VLLM_HOST_IP == "" assert envs.VLLM_PORT is None diff --git a/tests/test_sampling_params.py b/tests/test_sampling_params.py index cd23ea9938..3c909f4969 100644 --- a/tests/test_sampling_params.py +++ b/tests/test_sampling_params.py @@ -51,6 +51,14 @@ def test_non_diffusion_models_unaffected(): params.verify(MockModelConfig(), None, None, None) +def test_verify_leaves_logits_processors_to_admission(): + """verify() is runner-agnostic; LP validation lives in the admission + layer, so an unimportable FQCN must not fail verify().""" + SamplingParams().verify( + MockModelConfig(logits_processors=["no.such:Cls"]), None, None, None + ) + + @pytest.mark.parametrize("value", [-(2**63) - 1, 2**64]) def test_extra_args_rejects_nested_integer_overflow(value): """Reject extension values before they reach the engine transport.""" diff --git a/tests/test_triton_utils.py b/tests/test_triton_utils.py index 0ae4c0d6d6..7f6422466f 100644 --- a/tests/test_triton_utils.py +++ b/tests/test_triton_utils.py @@ -97,7 +97,8 @@ def test_triton_placeholder_language(): lang = TritonLanguagePlaceholder() assert isinstance(lang, types.ModuleType) assert lang.__name__ == "triton.language" - assert lang.constexpr is None + assert lang.constexpr(2**31 - 1) == 2**31 - 1 + assert lang.constexpr(1.5) == 1.5 assert lang.dtype is None assert lang.int64 is None assert lang.int32 is None @@ -143,3 +144,4 @@ def test_no_triton_fallback(): assert triton.__class__.__name__ == "TritonPlaceholder" assert triton.language.__class__.__name__ == "TritonLanguagePlaceholder" assert tl.__class__.__name__ == "TritonLanguagePlaceholder" + assert tl.constexpr(2**31 - 1) == 2**31 - 1 diff --git a/tests/tokenizers_/fixtures/deepseek_v4/test_input_5.json b/tests/tokenizers_/fixtures/deepseek_v4/test_input_5.json new file mode 100644 index 0000000000..2a2fa6a89e --- /dev/null +++ b/tests/tokenizers_/fixtures/deepseek_v4/test_input_5.json @@ -0,0 +1,39 @@ +{ + "thinking_mode": "thinking", + "reasoning_effort": "max", + "messages": [ + { + "role": "system", + "content": "You are a helpful vision assistant." + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "请按“第一张、第二张”的顺序回答:第一张图" + }, + { + "type": "image_url", + "image_url": { + "url": "examples/images/carrots.jpeg" + } + }, + { + "type": "text", + "text": "和第二张图" + }, + { + "type": "image_url", + "image_url": { + "url": "examples/images/corn.jpeg" + } + }, + { + "type": "text", + "text": "中分别是什么食材?它们通常食用的部位分别是什么?" + } + ] + } + ] +} diff --git a/tests/tokenizers_/fixtures/deepseek_v4/test_output_1.txt b/tests/tokenizers_/fixtures/deepseek_v4/test_output_1.txt index dbd823476c..7e3c9bd5a3 100644 --- a/tests/tokenizers_/fixtures/deepseek_v4/test_output_1.txt +++ b/tests/tokenizers_/fixtures/deepseek_v4/test_output_1.txt @@ -1,4 +1,4 @@ -<|begin▁of▁sentence|> +<|begin▁of▁sentence|>You are a helpful assistant. ## Tools @@ -26,7 +26,7 @@ Otherwise, output directly after with tool calls or final response. {"name": "search", "description": "Search the web for information", "parameters": {"type": "object", "properties": {"query": {"type": "string", "description": "Search query"}, "num_results": {"type": "integer", "description": "Number of results to return"}}, "required": ["query"]}} You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. -You are a helpful assistant.<|User|>What's the weather in Beijing?<|Assistant|>The user wants to know the weather in Beijing. I should use the get_weather tool. +<|User|>What's the weather in Beijing?<|Assistant|>The user wants to know the weather in Beijing. I should use the get_weather tool. <|DSML|tool_calls> <|DSML|invoke name="get_weather"> diff --git a/tests/tokenizers_/fixtures/deepseek_v4/test_output_5.txt b/tests/tokenizers_/fixtures/deepseek_v4/test_output_5.txt new file mode 100644 index 0000000000..1203dc8db1 --- /dev/null +++ b/tests/tokenizers_/fixtures/deepseek_v4/test_output_5.txt @@ -0,0 +1,9 @@ +<|begin▁of▁sentence|>You are a helpful vision assistant.<|User|>请按“第一张、第二张”的顺序回答:第一张图 + +<|deepseek_image|> + +和第二张图 + +<|deepseek_image|> + +中分别是什么食材?它们通常食用的部位分别是什么?<|Assistant|> \ No newline at end of file diff --git a/tests/tokenizers_/test_deepseek_v4.py b/tests/tokenizers_/test_deepseek_v4.py index a55aa57666..4403c2a877 100644 --- a/tests/tokenizers_/test_deepseek_v4.py +++ b/tests/tokenizers_/test_deepseek_v4.py @@ -54,13 +54,9 @@ def _load_reference_case(case_id: int): def _render_reference_case(case_id: int, **kwargs): messages, tools = _load_reference_case(case_id) - conversation, _, _ = parse_chat_messages( - messages, - _model_config(), - content_format="string", - ) + # Preserve content blocks for encoding without fetching fixture media. return _tokenizer().apply_chat_template( - conversation=conversation, + conversation=messages, messages=messages, tools=tools, tokenize=False, @@ -397,6 +393,7 @@ def test_deepseek_v4_maps_xhigh_to_high_reasoning_effort(): (2, {"thinking": True, "reasoning_effort": "low"}), (3, {"thinking": True, "reasoning_effort": "low"}), (4, {"thinking": False}), + (5, {"thinking": False}), ], ) def test_deepseek_v4_matches_reference_golden_fixtures(case_id, kwargs): @@ -447,7 +444,10 @@ def test_deepseek_v4_image_blocks_become_placeholders(): thinking=False, ) - assert "<|User|>first:<|deepseek_image|>second:<|deepseek_image|>" in prompt + assert ( + "<|User|>first:\n\n<|deepseek_image|>\n\n" + "second:\n\n<|deepseek_image|>" in prompt + ) def test_deepseek_v4_image_sentinel_ids_match_tokenizer(): @@ -485,3 +485,146 @@ def test_deepseek_v4_image_sentinel_ids_match_tokenizer(): ] ) assert image_sentinel_mask(ids).tolist() == [False, True, True, False, False] + + +def _request_tools(): + return [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather for a city", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } + ] + + +def _encode_reference(messages): + from vllm.tokenizers.deepseek_v4_encoding import encode_messages + + return encode_messages(messages, thinking_mode="thinking", reasoning_effort="low") + + +def test_deepseek_v4_attaches_request_tools_to_existing_system_message(): + messages = [ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "Weather in Paris?"}, + ] + + prompt = _tokenizer().apply_chat_template( + messages, + tools=_request_tools(), + tokenize=False, + thinking=True, + reasoning_effort="low", + ) + + expected = _encode_reference( + [{**messages[0], "tools": _request_tools()}, messages[1]] + ) + assert prompt == expected + assert prompt.startswith("<|begin▁of▁sentence|>You are helpful.\n\n## Tools") + assert prompt.count("## Tools") == 1 + + +def test_deepseek_v4_synthetic_system_only_when_no_system_message(): + """Without a system message, request tools still get a synthetic leading + system entry.""" + messages = [{"role": "user", "content": "Weather in Paris?"}] + + prompt = _tokenizer().apply_chat_template( + messages, + tools=_request_tools(), + tokenize=False, + thinking=True, + reasoning_effort="low", + ) + + expected = _encode_reference( + [{"role": "system", "tools": _request_tools()}, messages[0]] + ) + assert prompt == expected + assert prompt.startswith("<|begin▁of▁sentence|>\n\n## Tools") + + +def test_deepseek_v4_request_tools_override_system_message_tools(): + message_tools = [ + { + "type": "function", + "function": { + "name": "message_level_tool", + "description": "Should not be rendered", + "parameters": {"type": "object", "properties": {}}, + }, + } + ] + system_message = { + "role": "system", + "content": "You are helpful.", + "tools": message_tools, + } + user_message = {"role": "user", "content": "Weather in Paris?"} + + prompt = _tokenizer().apply_chat_template( + [system_message, user_message], + tools=_request_tools(), + tokenize=False, + thinking=True, + reasoning_effort="low", + ) + + expected = _encode_reference( + [{**system_message, "tools": _request_tools()}, user_message] + ) + assert prompt == expected + assert '"name": "get_weather"' in prompt + assert "message_level_tool" not in prompt + + +def test_deepseek_v4_attaches_request_tools_to_first_system_message_only(): + """Only the first system message, at any index, gets the request tools.""" + messages = [ + {"role": "user", "content": "Hi"}, + {"role": "system", "content": "First system."}, + {"role": "system", "content": "Second system."}, + {"role": "user", "content": "Weather in Paris?"}, + ] + + prompt = _tokenizer().apply_chat_template( + messages, + tools=_request_tools(), + tokenize=False, + thinking=True, + reasoning_effort="low", + ) + + expected = _encode_reference( + [ + messages[0], + {**messages[1], "tools": _request_tools()}, + messages[2], + messages[3], + ] + ) + assert prompt == expected + assert "First system.\n\n## Tools" in prompt + assert prompt.count("## Tools") == 1 + + +def test_deepseek_v4_request_tools_do_not_mutate_caller_messages(): + import copy + + messages = [ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "Weather in Paris?"}, + ] + snapshot = copy.deepcopy(messages) + + _tokenizer().apply_chat_template(messages, tools=_request_tools(), tokenize=False) + + assert messages == snapshot diff --git a/tests/tool_parsers/test_structural_tag_registry.py b/tests/tool_parsers/test_structural_tag_registry.py index 06c8fb6129..3221d1fb2c 100644 --- a/tests/tool_parsers/test_structural_tag_registry.py +++ b/tests/tool_parsers/test_structural_tag_registry.py @@ -1,11 +1,12 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import json from types import SimpleNamespace from unittest.mock import MagicMock import pytest -from xgrammar import Grammar, StructuralTag, builtin_structural_tag +from xgrammar import Grammar, StructuralTag from xgrammar.testing import _is_grammar_accept_string from vllm.entrypoints.openai.chat_completion.protocol import ( @@ -22,6 +23,7 @@ from vllm.tool_parsers.deepseekv31_tool_parser import DeepSeekV31ToolParser from vllm.tool_parsers.deepseekv32_engine_tool_parser import ( DeepSeekV32EngineToolParser, ) +from vllm.tool_parsers.deepseekv41_engine_tool_parser import DeepSeekV41EngineToolParser from vllm.tool_parsers.glm47_moe_tool_parser import Glm47MoeModelToolParser from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser from vllm.tool_parsers.kimi_k2_tool_parser import KimiK2ToolParser @@ -33,9 +35,11 @@ from vllm.tool_parsers.structural_tag_registry import ( SUPPORTED_STRUCTURAL_TAG_MODELS, VLLM_BUILTIN_STRUCTURAL_TAG_MODELS, XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS, + ToolChoice, get_function_parameters, get_model_structural_tag, ) +from vllm.tool_parsers.tool_strict_level import ToolStrictLevel @pytest.fixture @@ -80,51 +84,9 @@ def test_supported_structural_tag_models_include_vllm_builtins(): assert "hermes" in VLLM_BUILTIN_STRUCTURAL_TAG_MODELS -@pytest.mark.parametrize("choice", ["auto", "required", "get_weather"]) -def test_deepseek_v41_older_xgrammar_keeps_format_constraints( - choice, - sample_tools_strict, - monkeypatch, -): - monkeypatch.setattr( - builtin_structural_tag, "get_deepseek_v4_1_structural_tag", None, raising=False - ) - tool_choice = ( - ChatCompletionNamedToolChoiceParam( - function=ChatCompletionNamedFunction(name=choice) - ) - if choice == "get_weather" - else choice - ) - tag = get_model_structural_tag( - "deepseek_v41", sample_tools_strict, tool_choice, reasoning=False - ) - grammar = Grammar.from_structural_tag(tag) - begin = '\n\n<|DSML| calls>\n<|DSML| invoke name="get_weather">\n' - end = "\n" - parameter = ( - '<|DSML| parameter name="undeclared" string="false">' - '{"nested": [true, null, 1.5]}\n' - ) - # Required city is omitted; extra and repeated fields remain unconstrained. - assert _is_grammar_accept_string(grammar, begin + end) - assert _is_grammar_accept_string(grammar, begin + parameter * 2 + end) - assert not _is_grammar_accept_string( - grammar, - begin + parameter.replace('{"nested": [true, null, 1.5]}', "invalid") + end, - ) - assert not _is_grammar_accept_string( - grammar, (begin + end).replace('name="get_weather"', 'name="unknown"') - ) - assert not _is_grammar_accept_string( - grammar, begin + parameter.replace("| parameter", "|parameter") + end - ) - assert _is_grammar_accept_string(grammar, "Hello") == (choice == "auto") - - def test_deepseek_v41_named_choice_emits_one_call(sample_tools): tag = get_model_structural_tag( - "deepseek_v41", + "deepseek_v4_1", sample_tools, ChatCompletionNamedToolChoiceParam( function=ChatCompletionNamedFunction(name="get_weather") @@ -144,19 +106,14 @@ def test_deepseek_v41_named_choice_emits_one_call(sample_tools): grammar, "\n\n<|DSML| calls>\n" + call * 2 + "" ) assert ( - get_model_structural_tag("deepseek_v41", sample_tools, "auto", reasoning=False) + get_model_structural_tag("deepseek_v4_1", sample_tools, "auto", reasoning=False) is None ) -@pytest.mark.skipif( - not hasattr(builtin_structural_tag, "get_deepseek_v4_1_structural_tag"), - reason="Requires XGrammar's DeepSeek V4.1 builtin", -) @pytest.mark.parametrize("choice", ["auto", "required", "get_weather"]) -@pytest.mark.parametrize("reasoning", [False, True]) def test_deepseek_v41_constrains_parameters_after_reasoning( - sample_tools_strict, choice, reasoning + sample_tools_strict, choice ): """Strict calls must enforce the schema without requiring another think close.""" tool_choice = ( @@ -168,7 +125,7 @@ def test_deepseek_v41_constrains_parameters_after_reasoning( ) sample_tools_strict[0].function.parameters["additionalProperties"] = False tag = get_model_structural_tag( - "deepseek_v41", sample_tools_strict, tool_choice, reasoning=reasoning + "deepseek_v4_1", sample_tools_strict, tool_choice, reasoning=False ) grammar = Grammar.from_structural_tag(tag) begin = '\n\n<|DSML| calls>\n<|DSML| invoke name="get_weather">\n' @@ -192,14 +149,10 @@ def test_deepseek_v41_constrains_parameters_after_reasoning( assert _is_grammar_accept_string(grammar, "Hello") == (choice == "auto") -@pytest.mark.skipif( - not hasattr(builtin_structural_tag, "get_deepseek_v4_1_structural_tag"), - reason="Requires XGrammar's DeepSeek V4.1 builtin", -) def test_deepseek_v41_non_strict_parallel_calls_keep_typed_dsml(sample_tools): sample_tools[0].function.strict = False tag = get_model_structural_tag( - "deepseek_v41", sample_tools, "required", reasoning=False + "deepseek_v4_1", sample_tools, "required", reasoning=False ) grammar = Grammar.from_structural_tag(tag) call = ( @@ -231,11 +184,11 @@ def test_get_model_structural_tag_supports_all_xgrammar_builtins( def test_get_model_structural_tag_supports_vllm_hermes( - sample_tools: list[ChatCompletionToolsParam], + sample_tools_strict: list[ChatCompletionToolsParam], ): tag = get_model_structural_tag( model="hermes", - tools=sample_tools, + tools=sample_tools_strict, tool_choice="required", reasoning=False, ) @@ -273,6 +226,22 @@ def test_get_model_structural_tag_supports_vllm_hermes( assert content["json_schema"] == expected_schema +def test_hermes_required_without_strict_leaves_arguments_free( + sample_tools: list[ChatCompletionToolsParam], +): + tag = get_model_structural_tag( + model="hermes", + tools=sample_tools, + tool_choice="required", + reasoning=False, + ) + + assert isinstance(tag, StructuralTag) + tags = tag.model_dump()["format"]["tags"] + assert tags + assert all(tag_dump["content"]["json_schema"] is True for tag_dump in tags) + + def test_hermes_required_tool_calls_use_empty_separator(): tools = [ ChatCompletionToolsParam( @@ -361,10 +330,19 @@ def _k3_tools(*calls: str) -> str: return f"{_K3_TOOLS_OPEN}{''.join(calls)}{_K3_TOOLS_CLOSE}" -def _k3_grammar(tool_choice, tools=None): +def _as_strict(tools: list[ChatCompletionToolsParam]) -> list[ChatCompletionToolsParam]: + for tool in tools: + tool.function.strict = True + return tools + + +def _k3_grammar(tool_choice, tools=None, *, strict: bool = True): + """Build the K3 grammar; tools are strict by default so the typed + argument channel is exercised (an unset ``strict`` leaves arguments free).""" + tools = tools if tools is not None else _k3_tools_by_name() tag = get_model_structural_tag( model="kimi_k3", - tools=tools if tools is not None else _k3_tools_by_name(), + tools=_as_strict(tools) if strict else tools, tool_choice=tool_choice, reasoning=False, ) @@ -467,6 +445,18 @@ def test_kimi_k3_required_rejects_invalid(body: str): assert not _is_grammar_accept_string(_k3_grammar("required"), body) +def test_kimi_k3_required_without_strict_leaves_arguments_free(): + # An unset ``strict`` pins the call envelope only: a value the schema + # would reject is accepted, and the declared-required argument may be + # omitted. + grammar = _k3_grammar("required", strict=False) + body = _k3_response() + _k3_tools( + _k3_call("get_weather", _k3_arg("days", "number", "abc")) + ) + assert _is_grammar_accept_string(grammar, body) + assert not _is_grammar_accept_string(_k3_grammar("required"), body) + + def test_kimi_k3_schema_without_required_accepts_empty_call(): tools = [ ChatCompletionToolsParam( @@ -531,6 +521,7 @@ def test_get_model_structural_tag_supports_named_tool_choice( (DeepSeekV31ToolParser, "deepseek_v3_1"), (DeepSeekV32EngineToolParser, "deepseek_v3_2"), (DeepSeekV4EngineToolParser, "deepseek_v4"), + (DeepSeekV41EngineToolParser, "deepseek_v4_1"), (Glm47MoeModelToolParser, "glm_4_7"), (Hermes2ProToolParser, "hermes"), (KimiK2ToolParser, "kimi"), @@ -598,7 +589,11 @@ def test_get_structural_tag_disables_reasoning( assert captured == [False] +@pytest.mark.parametrize( + "parser_cls", [Qwen3EngineToolParser, DeepSeekV41EngineToolParser] +) def test_unified_parser_get_structural_tag_disables_reasoning( + parser_cls, monkeypatch: pytest.MonkeyPatch, sample_tools_strict: list[ChatCompletionToolsParam], ): @@ -614,7 +609,7 @@ def test_unified_parser_get_structural_tag_disables_reasoning( ) class TestParser(DelegatingParser): - tool_parser_cls = Qwen3EngineToolParser + tool_parser_cls = parser_cls request = ChatCompletionRequest( messages=[], @@ -829,3 +824,209 @@ def test_kimi_k3_forced_tool_choice_builds_single_mandatory_call(): response_only = _k3_response("no call here") assert _is_grammar_accept_string(grammar, ok) assert not _is_grammar_accept_string(grammar, response_only) + + +def _pins_argument_schema(tag: StructuralTag) -> bool: + return '"json_schema": {' in json.dumps(tag.model_dump(), ensure_ascii=False) + + +def _dumped(tag: StructuralTag) -> str: + return json.dumps(tag.model_dump(), ensure_ascii=False) + + +def test_tool_strict_level_auto_is_the_default( + sample_tools: list[ChatCompletionToolsParam], +): + """Auto + no strict tool gets no tag unless the operator raises the floor.""" + assert ( + get_model_structural_tag( + model="deepseek_v4", + tools=sample_tools, + tool_choice="auto", + reasoning=False, + ) + is None + ) + + +@pytest.mark.parametrize("model", sorted(SUPPORTED_STRUCTURAL_TAG_MODELS)) +def test_tool_strict_level_function_lifts_auto_gate( + model: str, + sample_tools: list[ChatCompletionToolsParam], +): + tag = get_model_structural_tag( + model=model, + tools=sample_tools, + tool_choice="auto", + reasoning=False, + strict_level=ToolStrictLevel.FUNCTION, + ) + + assert tag is not None + + +@pytest.mark.parametrize("model", sorted(XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS)) +def test_tool_strict_level_function_pins_envelope_only( + model: str, + sample_tools: list[ChatCompletionToolsParam], +): + """The request's own tools stay untouched; unset ``strict`` stays free.""" + tag = get_model_structural_tag( + model=model, + tools=sample_tools, + tool_choice="auto", + reasoning=False, + strict_level=ToolStrictLevel.FUNCTION, + ) + + assert tag is not None + assert not _pins_argument_schema(tag) + assert all(tool.function.strict is None for tool in sample_tools) + + +@pytest.mark.parametrize("model", sorted(XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS)) +def test_tool_strict_level_parameter_pins_argument_schemas( + model: str, + sample_tools: list[ChatCompletionToolsParam], +): + tag = get_model_structural_tag( + model=model, + tools=sample_tools, + tool_choice="auto", + reasoning=False, + strict_level=ToolStrictLevel.PARAMETER, + ) + + assert tag is not None + assert _pins_argument_schema(tag) + + +def test_tool_strict_level_function_keeps_client_strict_tools_strict( + sample_tools_strict: list[ChatCompletionToolsParam], +): + tag = get_model_structural_tag( + model="deepseek_v4", + tools=sample_tools_strict, + tool_choice="auto", + reasoning=False, + strict_level=ToolStrictLevel.FUNCTION, + ) + + assert tag is not None + assert _pins_argument_schema(tag) + + +@pytest.mark.parametrize("model", sorted(XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS)) +def test_absent_strict_is_not_pinned_next_to_a_strict_tool(model: str): + """A strict tool must not drag a neighbour with unset ``strict`` into + schema enforcement.""" + weather = ChatCompletionToolsParam( + type="function", + function={ + "name": "get_weather", + "strict": True, + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + ) + search = ChatCompletionToolsParam( + type="function", + function={ + "name": "search", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + }, + }, + ) + tag = get_model_structural_tag( + model=model, + tools=[weather, search], + tool_choice="auto", + reasoning=False, + ) + + assert tag is not None + dumped = _dumped(tag) + assert '"city"' in dumped + assert '"query"' not in dumped + + +@pytest.mark.parametrize( + "tool_choice", + [ + "required", + ChatCompletionNamedToolChoiceParam( + function=ChatCompletionNamedFunction(name="get_weather") + ), + ], +) +@pytest.mark.parametrize("model", sorted(XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS)) +def test_forced_tool_choice_pins_schema_only_for_strict_tools( + model: str, + tool_choice: ToolChoice, + sample_tools: list[ChatCompletionToolsParam], + sample_tools_strict: list[ChatCompletionToolsParam], +): + """Required / named always constrain the call; the argument schema + follows the per-tool ``strict`` (or the parameter level).""" + free = get_model_structural_tag( + model=model, + tools=sample_tools, + tool_choice=tool_choice, + reasoning=False, + strict_level=ToolStrictLevel.FUNCTION, + ) + pinned = get_model_structural_tag( + model=model, + tools=sample_tools_strict, + tool_choice=tool_choice, + reasoning=False, + ) + floored = get_model_structural_tag( + model=model, + tools=sample_tools, + tool_choice=tool_choice, + reasoning=False, + strict_level=ToolStrictLevel.PARAMETER, + ) + + assert free is not None and not _pins_argument_schema(free) + assert pinned is not None and _pins_argument_schema(pinned) + assert floored is not None and _pins_argument_schema(floored) + + +def test_tool_strict_level_parameter_overrides_client_strict_false(): + tools = [ + ChatCompletionToolsParam( + type="function", + function={ + "name": "get_weather", + "parameters": {"type": "object", "properties": {}}, + "strict": False, + }, + ) + ] + tag = get_model_structural_tag( + model="deepseek_v4", + tools=tools, + tool_choice="required", + reasoning=False, + strict_level=ToolStrictLevel.PARAMETER, + ) + + assert tag is not None + assert _pins_argument_schema(tag) + + +def test_tool_strict_level_from_name(): + assert ToolStrictLevel.from_name("AUTO") is ToolStrictLevel.AUTO + assert ToolStrictLevel.from_name("Parameter") is ToolStrictLevel.PARAMETER + with pytest.raises(ValueError, match="expected one of auto, function, parameter"): + ToolStrictLevel.from_name("strict") + with pytest.raises(ValueError, match="expected one of auto, function, parameter"): + ToolStrictLevel.from_name("off") diff --git a/tests/tools/test_docker_build_metadata_args.py b/tests/tools/test_docker_build_metadata_args.py index 8095ad34aa..127768afc7 100644 --- a/tests/tools/test_docker_build_metadata_args.py +++ b/tests/tools/test_docker_build_metadata_args.py @@ -155,10 +155,15 @@ def test_vllm_openai_image_embeds_metadata_contract() -> None: def test_rust_build_cache_excludes_git_metadata() -> None: + import torch + from vllm.platforms import current_platform dockerfile_names = ["Dockerfile", "Dockerfile.cpu"] - if not current_platform.is_rocm(): + # CPU jobs can reuse ROCm artifacts, which omit the XPU Dockerfile. + if not current_platform.is_rocm() and ( + not current_platform.is_cpu() or torch.version.hip is None + ): dockerfile_names.append("Dockerfile.xpu") for name in dockerfile_names: dockerfile = (REPO_ROOT / "docker" / name).read_text() diff --git a/tests/transformers_utils/test_config.py b/tests/transformers_utils/test_config.py index fb7a7145ff..229abfba8c 100644 --- a/tests/transformers_utils/test_config.py +++ b/tests/transformers_utils/test_config.py @@ -28,6 +28,7 @@ from vllm.transformers_utils.configs.glm5_next import ( Glm5NextTextConfig, Glm5NextVisionConfig, ) +from vllm.transformers_utils.configs.mistral import adapt_config_dict def test_patch_legacy_rope_type_preserves_nope_layers(): @@ -77,6 +78,57 @@ def test_patch_legacy_rope_type_normalizes_telechat3_yarn(): } +def test_mistral_yarn_apply_scale_false_disables_yarn_magnitude_scaling(): + """`yarn.apply_scale: false` must reach the DeepSeek-style attentions. + + Transformers spells it `attention_factor = 1.0`, which DeepseekV2Attention + and its siblings read to select `deepseek_llama_scaling` over + `deepseek_yarn`; without it Mistral-Large-3 runs with a spurious + yarn_get_mscale(factor)^2 attention scaling. + """ + params = { + "dim": 7168, + "n_layers": 61, + "head_dim": 192, + "hidden_dim": 16384, + "n_heads": 128, + "n_kv_heads": 128, + "norm_eps": 1e-5, + "vocab_size": 131072, + "rope_theta": 10000.0, + "max_position_embeddings": 294912, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "qk_nope_head_dim": 128, + "qk_rope_head_dim": 64, + "v_head_dim": 128, + "moe": { + "num_experts": 128, + "num_experts_per_tok": 4, + "num_shared_experts": 1, + "expert_hidden_dim": 4096, + "first_k_dense_replace": 3, + "route_every_n": 1, + "routed_scale": 1.0, + "num_expert_groups": 1, + "num_expert_groups_per_tok": 1, + }, + "llama_4_scaling": {"beta": 0.1, "original_max_position_embeddings": 8192}, + "yarn": { + "alpha": 1, + "apply_scale": False, + "beta": 32, + "factor": 36, + "original_max_position_embeddings": 8192, + }, + } + + config = adapt_config_dict(params, defaults={}) + + assert config.architectures == ["MistralLarge3ForCausalLM"] + assert config.rope_parameters["attention_factor"] == 1.0 + + def test_glm5_next_accepts_deepseek_sparse_attention_layers(): layer_types = ["linear_attention", "deepseek_sparse_attention"] diff --git a/tests/utils_/test_argparse_utils.py b/tests/utils_/test_argparse_utils.py index baf57d994f..89e0f9977d 100644 --- a/tests/utils_/test_argparse_utils.py +++ b/tests/utils_/test_argparse_utils.py @@ -524,3 +524,20 @@ def test_flat_product(): (3, 4, "a", 5, 6), (3, 4, "b", 5, 6), ] + + +def test_group_description_is_summary_only(): + """Group descriptions are config docstrings, too long for the terminal.""" + parser = FlexibleArgumentParser() + group = parser.add_argument_group( + title="MyConfig", + description="Summary line.\n\nDetails which only belong in the docs.", + ) + group.add_argument("--my-arg") + + assert "Summary line." in parser.format_help() + assert "only belong in the docs" not in parser.format_help() + + parser._search_keyword = "myconfig" + assert "Summary line." in parser.format_help() + assert "only belong in the docs" not in parser.format_help() diff --git a/tests/v1/attention/test_deepseek_v4_swa_visible.py b/tests/v1/attention/test_deepseek_v4_swa_visible.py index 1b5097de8e..7a74ce8c81 100644 --- a/tests/v1/attention/test_deepseek_v4_swa_visible.py +++ b/tests/v1/attention/test_deepseek_v4_swa_visible.py @@ -11,6 +11,8 @@ at ``max(pos - (window - 1) - max(left - (window - 1), 0), 0)`` and ends at ``pos + right`` (inclusive). """ +import inspect + import pytest import torch from typing_extensions import TypedDict @@ -57,10 +59,13 @@ def ref_left_right( return lefts, rights -def ref_swa_bounds(pos: int, window: int, left: int, right: int) -> tuple[int, int]: - """Reference [start, end) window bounds for one query token.""" +def ref_swa_bounds( + pos: int, window: int, left: int, right: int, replay_start: int = 0 +) -> tuple[int, int]: + """Reference [start, end) window bounds for one query token. Under SWA + bounded replay no window KV exists below ``replay_start``.""" left_add = max(left - (window - 1), 0) - start = max(pos - (window - 1) - left_add, 0) + start = max(pos - (window - 1) - left_add, 0, replay_start) return start, pos + right + 1 @@ -97,12 +102,14 @@ def ref_swa_slot_rows( window: int, max_image_tokens: int, width: int, + replay_starts: list[int] | None = None, ) -> tuple[list[list[int]], list[int]]: """Reference paged slot-id rows and lens for every token in the batch.""" block_table_cpu = block_table.cpu() lefts, rights = ref_left_right( seq_lens, query_lens, spans_per_req, max_image_tokens ) + replay_starts = replay_starts or [0] * len(seq_lens) rows: list[list[int]] = [] lens: list[int] = [] token = 0 @@ -110,7 +117,9 @@ def ref_swa_slot_rows( prefix_len = seq_len - query_len for i in range(query_len): pos = prefix_len + i - start, end = ref_swa_bounds(pos, window, lefts[token], rights[token]) + start, end = ref_swa_bounds( + pos, window, lefts[token], rights[token], replay_starts[req] + ) row = [] for p in range(start, end): blk = int(block_table_cpu[req, p // BLOCK_SIZE]) @@ -139,6 +148,7 @@ def run_swa_kernel( swa_indices = torch.zeros(num_tokens, 1, width, dtype=torch.int32, device=device) swa_lens = torch.zeros(num_tokens, dtype=torch.int32, device=device) is_valid = slot_mapping >= 0 + replay_start_t = torch.zeros(len(seq_lens), dtype=torch.int32, device=device) if with_image: lefts, rights = ref_left_right( @@ -164,6 +174,7 @@ def run_swa_kernel( block_table, block_table.stride(0), BLOCK_SIZE, + replay_start_t, token_offset=0, HAS_IMAGE=with_image, TRITON_BLOCK_SIZE=1024, @@ -289,18 +300,25 @@ def combine_case( query_lens: list[int], spans: list[list[tuple[int, int]]], with_image: bool, + replay_starts: list[int] | None = None, + combine_fn=combine_topk_swa_indices, ): """Run combine_topk_swa_indices and return (indices, lens, expected).""" device = torch.device("cuda") num_reqs = len(seq_lens) + replay_starts = replay_starts or [0] * num_reqs query_start_loc = torch.zeros(num_reqs + 1, dtype=torch.int32, device=device) query_start_loc[1:] = torch.tensor( query_lens, dtype=torch.int32, device=device ).cumsum(0) num_tokens = int(query_start_loc[-1]) seq_lens_t = torch.tensor(seq_lens, dtype=torch.int32, device=device) + # The builder's gather covers only the context above the replay start. gather_lens = torch.tensor( - [q + min(s - q, WINDOW - 1) for s, q in zip(seq_lens, query_lens)], + [ + q + min(max(s - q - r, 0), WINDOW - 1) + for s, q, r in zip(seq_lens, query_lens, replay_starts) + ], dtype=torch.int32, device=device, ) @@ -319,7 +337,14 @@ def combine_case( else: left_t = right_t = None - combined_indices, combined_lens = combine_topk_swa_indices( + # V4.1's combine has no image windows (causal SWA only). + has_image = "left_visible" in inspect.signature(combine_fn).parameters + image_kwargs = ( + dict(left_visible=left_t, right_visible=right_t, max_image_tokens=MAX_IMG) + if has_image + else {} + ) + combined_indices, combined_lens = combine_fn( topk_indices, query_start_loc, seq_lens_t, @@ -329,9 +354,7 @@ def combine_case( topk, M, N, - left_visible=left_t, - right_visible=right_t, - max_image_tokens=MAX_IMG, + **image_kwargs, ) # Reference rows. @@ -342,7 +365,7 @@ def combine_case( MAX_IMG, ) topk_cpu = topk_indices.cpu() - width = WINDOW + MAX_IMG + width = WINDOW + (MAX_IMG if has_image else 0) combined_topk = (topk + width + 127) // 128 * 128 rows = [] lens = [] @@ -354,6 +377,8 @@ def combine_case( pos = prefix_len + i topk_len = min((pos + 1) // compress_ratio, topk) start, end = ref_swa_bounds(pos, WINDOW, lefts[token], rights[token]) + # The window never reaches below the gathered buffer. + start = max(start, gather_start) swa_len = end - start row = [-1] * combined_topk for j in range(topk_len): @@ -388,6 +413,31 @@ def test_combine_topk_swa_with_image_spans(cfg): assert indices.cpu().tolist() == rows +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +@pytest.mark.parametrize("cfg", COMBINE_CASES) +def test_v41_combine_topk_swa_stops_at_replay_start(cfg): + """SWA bounded replay: the gathered buffer starts at replay_start, so the + window never indexes below it.""" + from vllm.models.deepseek_v41.common.ops.cache_utils import ( + combine_topk_swa_indices as combine_v41, + ) + + # Request 0 replays [16, 40): the windows of its first rows would + # otherwise reach below 16. + indices, lens, rows, exp_lens = combine_case( + cfg["compress_ratio"], + cfg["topk"], + seq_lens=[40, 12], + query_lens=[24, 12], + spans=[[], []], + with_image=False, + replay_starts=[16, 0], + combine_fn=combine_v41, + ) + assert lens.cpu().tolist() == exp_lens + assert indices.cpu().tolist() == rows + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") @pytest.mark.parametrize("cfg", COMBINE_CASES) def test_combine_topk_swa_without_image_unchanged(cfg): @@ -472,6 +522,7 @@ def make_builder( DeepseekSparseSWAMetadataBuilder ] = DeepseekSparseSWAMetadataBuilder, mm_prefix_clamp_sliding_window: bool = True, + bounded_replay: bool = False, ) -> DeepseekSparseSWAMetadataBuilder: overrides: dict = {"sliding_window": window} if vision: @@ -498,6 +549,7 @@ def make_builder( sliding_window=window, cache_dtype_str="auto", model_version="deepseek_v4", + bounded_replay=bounded_replay, ) return builder_cls( kv_cache_spec=spec, @@ -512,11 +564,17 @@ def build_metadata( seq_lens: list[int], query_lens: list[int], mm_req_doc_ranges: dict[int, list[tuple[int, int]]] | None, + replay_starts: list[int] | None = None, ): device = torch.device("cuda") query_start_loc, seq_lens_t, _, slot_mapping, block_table = make_batch( seq_lens, query_lens, device ) + replay_start = ( + None + if replay_starts is None + else torch.tensor(replay_starts, dtype=torch.int32, device=device) + ) return builder.build( 0, CommonAttentionMetadata( @@ -533,6 +591,7 @@ def build_metadata( causal=True, mm_req_doc_ranges=mm_req_doc_ranges, ), + replay_start=replay_start, ) @@ -653,3 +712,108 @@ def test_v41_image_prefill_uses_causal_swa(compress_ratio, query_len): assert absolute_keys.tolist() == list(range(lo, hi)) assert paged_lens[token] == hi - lo assert paged_indices[token, 0, : hi - lo].tolist() == list(range(lo, hi)) + + +# SWA bounded replay: every prefill index path clamps the window at replay_start. + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_builder_replay_start_bounds_prefill_window_and_gather(): + """Paged-direct prefill indices stop at replay_start and the FlashMLA + gather only covers the context the request may see.""" + seq_lens = [40, 12, 30] + query_lens = [24, 12, 10] + replay_starts = [12, 0, 0] + builder = make_builder(vision=False) + md = build_metadata(builder, seq_lens, query_lens, None, replay_starts) + assert md.replay_start is not None + assert md.replay_start.cpu().tolist() == replay_starts + + _, _, _, _, block_table = make_batch(seq_lens, query_lens, torch.device("cuda")) + rows, lens = ref_swa_slot_rows( + seq_lens, + query_lens, + [[], [], []], + block_table, + WINDOW, + MAX_IMG, + WINDOW, + replay_starts=replay_starts, + ) + assert md.prefill_swa_lens.cpu().tolist() == lens + assert md.prefill_swa_indices[:, 0].cpu().tolist() == rows + # Request 0: the first replayed row (pos 16) sees [12, 16]; its last row + # (pos 39) is a full window above the replay start and sees all of it. + assert md.prefill_swa_lens[0].item() == 16 - 12 + 1 + assert md.prefill_swa_lens[23].item() == WINDOW + # gather_len = query_len + min(prefix_len - replay_start, WINDOW - 1). + assert md.prefill_gather_lens.cpu().tolist() == [ + 24 + min(16 - 12, WINDOW - 1), + 12, + 10 + min(20, WINDOW - 1), + ] + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_builder_replaying_group_builds_without_replay_start(): + """Graph captures build without model-specific kwargs: a replaying group's + metadata then bounds nothing.""" + builder = make_builder(vision=False, bounded_replay=True) + md = build_metadata(builder, [40, 12], [24, 12], None) + assert not md.replay_start.any() + assert md.prefill_swa_lens[0].item() == WINDOW + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_v41_flashinfer_mixed_sparse_indices_respect_replay_start(): + from vllm.models.deepseek_v41.common.ops.cache_utils import ( + build_flashinfer_mixed_sparse_indices as build_v41, + ) + + device = torch.device("cuda") + # 1 decode token (req 0) + two prefill requests; req 1 resumes at 16. + seq_lens = [20, 40, 9] + query_lens = [1, 24, 9] + replay_starts = [0, 16, 0] + query_start_loc = torch.tensor([0, 1, 25, 34], dtype=torch.int32, device=device) + seq_lens_t = torch.tensor(seq_lens, dtype=torch.int32, device=device) + token_to_req = torch.tensor( + [0] + [1] * 24 + [2] * 9, dtype=torch.int32, device=device + ) + block_table = torch.arange(3, dtype=torch.int32, device=device).view(3, 1) + kwargs = dict( + decode_swa_indices=torch.zeros((1, WINDOW), dtype=torch.int32, device=device), + decode_compressed_indices=None, + decode_compressed_topk_lens=None, + prefill_topk_indices=torch.empty((33, 0), dtype=torch.int32, device=device), + query_start_loc=query_start_loc, + seq_lens=seq_lens_t, + token_to_req_indices=token_to_req, + swa_block_table=block_table, + swa_block_size=BLOCK_SIZE, + compressed_block_table=None, + compressed_block_size=BLOCK_SIZE, + window_size=WINDOW, + compress_ratio=1, + topk=0, + ) + plain, _ = build_v41( + replay_start=torch.zeros(3, dtype=torch.int32, device=device), **kwargs + ) + bounded, _ = build_v41( + replay_start=torch.tensor(replay_starts, dtype=torch.int32, device=device), + **kwargs, + ) + rows, _ = ref_swa_slot_rows( + seq_lens, + query_lens, + [[], [], []], + block_table, + WINDOW, + MAX_IMG, + WINDOW, + replay_starts=replay_starts, + ) + # Decode row is copied through; prefill rows follow the bounded window. + assert bounded[0].cpu().tolist() == plain[0].cpu().tolist() + assert bounded[1:].cpu().tolist() == rows[1:] diff --git a/tests/v1/attention/test_dspark_noncausal_sparse_mla.py b/tests/v1/attention/test_dspark_noncausal_sparse_mla.py index d5c7941314..3d4af17413 100644 --- a/tests/v1/attention/test_dspark_noncausal_sparse_mla.py +++ b/tests/v1/attention/test_dspark_noncausal_sparse_mla.py @@ -604,3 +604,154 @@ def test_dspark_noncausal_differs_from_causal( f"non-causal backend output matches the causal reference " f"(max abs diff={causal_err}); future-pointing indices are not attended to" ) + + +@pytest.mark.parametrize("context_len", [20, 128, 900]) +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float8_e4m3fn]) +@pytest.mark.parametrize("num_heads", [16, 64]) +def test_dsv41_flashinfer_dspark_window_matches_reference( + context_len, dtype, num_heads, monkeypatch +): + """Draft queries see the full block without treating padded slots as keys.""" + if not current_platform.is_device_capability_family(100): + pytest.skip("DSV4 TRTLLM sparse attention requires SM100") + from vllm.models.deepseek_v41.nvidia.flashinfer_sparse import ( + DeepseekSparseSWAFlashInferMetadataBuilder, + DeepseekV4FlashInferMLAAttention, + ) + from vllm.models.deepseek_v41.sparse_mla import ( + DeepseekV41SparseSWAMetadataBuilder, + ) + + torch.manual_seed(123) + device = "cuda" + query_lens = [5, 3] + context_lens = [context_len, context_len + 137] + num_real_tokens, num_tokens, head_dim = 8, 9, 512 + cache = torch.randn(32, 128, head_dim, device=device, dtype=torch.bfloat16).to( + dtype + ) + query = torch.randn( + num_tokens, num_heads, head_dim, device=device, dtype=torch.bfloat16 + ).to(dtype) + indices = torch.full((num_tokens, 256), -1, device=device, dtype=torch.int32) + visible_indices = [] + visible_lens = [] + for req, (context, query_len) in enumerate(zip(context_lens, query_lens)): + visible = ( + torch.arange(max(context - 128, 0), context + query_len, device=device) + + req * 16 * 128 + ) + visible_indices.extend([visible] * query_len) + visible_lens.extend([visible.numel()] * query_len) + for token, visible in enumerate(visible_indices): + indices[token, : visible.numel()] = visible.to(torch.int32) + query_start_loc = torch.tensor([0, 5, 8, 9], dtype=torch.int32) + metadata = SimpleNamespace( + num_decodes=3, + num_prefills=0, + num_decode_tokens=num_tokens, + num_prefill_tokens=0, + seq_lens=torch.tensor( + [context_lens[0] + 5, context_lens[1] + 3, 1], + device=device, + dtype=torch.int32, + ), + query_start_loc=query_start_loc.to(device), + query_start_loc_cpu=query_start_loc, + token_to_req_indices=torch.tensor( + [0] * 5 + [1] * 3 + [2], device=device, dtype=torch.int32 + ), + decode_swa_indices=indices, + decode_swa_width=256, + decode_swa_lens=torch.tensor( + visible_lens + [0], device=device, dtype=torch.int32 + ), + block_table=torch.arange(48, device=device, dtype=torch.int32).view(3, -1), + block_size=128, + replay_start=torch.zeros(3, device=device, dtype=torch.int32), + flashinfer_sparse_index_cache={}, + max_decode_query_len=5, + ) + + # Exercise FlashInfer preparation without constructing a model/config. + def init_parent(builder): + builder._max_tokens = num_tokens + builder.device = device + builder.window_size = 128 + + monkeypatch.setattr(DeepseekV41SparseSWAMetadataBuilder, "__init__", init_parent) + monkeypatch.setattr( + DeepseekV41SparseSWAMetadataBuilder, + "build", + lambda *args: metadata, + ) + builder = DeepseekSparseSWAFlashInferMetadataBuilder() + common_metadata = SimpleNamespace(causal=False) + builder.build(0, common_metadata) + prepared = ( + metadata.flashinfer_decode_topk_lens, + metadata.flashinfer_decode_seq_lens, + ) + attention = SimpleNamespace( + kv_cache_torch_dtype=dtype, + window_size=128, + compress_ratio=0, + topk_indices_buffer=torch.empty( + num_tokens, 0, device=device, dtype=torch.int32 + ), + scale=1 / math.sqrt(head_dim), + _flashinfer_fp8_bmm1_scale=1 / math.sqrt(head_dim), + _flashinfer_fp8_bmm2_scale=1.0, + attn_sink=None, + ) + attention._build_sparse_index_metadata = MethodType( + DeepseekV4FlashInferMLAAttention._build_sparse_index_metadata, attention + ) + output = torch.empty_like(query, dtype=torch.bfloat16) + + def forward(): + metadata.flashinfer_sparse_index_cache.clear() + DeepseekV4FlashInferMLAAttention._forward( + attention, query, None, cache, metadata, None, True, output + ) + + def check_output(): + references = [] + for token, visible in enumerate(visible_indices): + keys = cache.flatten(0, 1)[visible].float() + weights = torch.softmax( + query[token].float() @ keys.T / math.sqrt(head_dim), -1 + ) + references.append(weights @ keys) + atol = 0.01 if dtype == torch.bfloat16 else 0.05 + torch.testing.assert_close( + output[:num_real_tokens].float(), + torch.stack(references), + atol=atol, + rtol=0.05, + ) + + forward() + check_output() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + forward() + # Replay must consume the current inputs, including KV written since capture. + query.copy_(torch.randn_like(query, dtype=torch.bfloat16).to(dtype)) + cache.copy_(torch.randn_like(cache, dtype=torch.bfloat16).to(dtype)) + # A new step changes visibility as well as Q/KV, without recapturing. + metadata.seq_lens.sub_(1) + metadata.decode_swa_lens[:num_real_tokens].sub_(1) + for token, visible in enumerate(visible_indices): + indices[token, visible.numel() - 1] = -1 + visible_indices[token] = visible[:-1] + builder.build(0, common_metadata) + assert metadata.flashinfer_decode_topk_lens.data_ptr() == prepared[0].data_ptr() + assert metadata.flashinfer_decode_seq_lens.data_ptr() == prepared[1].data_ptr() + torch.testing.assert_close(prepared[0], metadata.decode_swa_lens.clamp_min(128)) + torch.testing.assert_close( + prepared[1], metadata.seq_lens[metadata.token_to_req_indices.long()] + ) + graph.replay() + check_output() diff --git a/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py b/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py index f3b4b7f5ba..83fceef018 100644 --- a/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py +++ b/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py @@ -14,6 +14,7 @@ from vllm.models.deepseek_v41.sparse_mla import ( from vllm.v1.attention.backend import CommonAttentionMetadata from vllm.v1.attention.backends.mla.compressor_utils import ( CompressedSlotMappingKernel, + get_compressed_slot_mapping, ) from vllm.v1.attention.backends.mla.indexer import ( BuildPrefillChunkMetadataKernel, @@ -235,6 +236,28 @@ def test_compressed_slot_mapping_warmup_includes_index_kpool(): assert {(key.compress_ratio, key.block_size) for key in keys} == {(32, 2)} +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_compressed_slot_mapping_inherits_padded_token_slots(): + """A token whose own slot is padded (SWA bounded replay) closes no + compressed state either.""" + device = torch.device("cuda") + query_start_loc = torch.tensor([0, 8], dtype=torch.int32, device=device) + seq_lens = torch.tensor([8], dtype=torch.int32, device=device) + block_table = torch.tensor([[3]], dtype=torch.int32, device=device) + slot_mapping = torch.arange(8, dtype=torch.int64, device=device) + slot_mapping[:4] = -1 + compressed = get_compressed_slot_mapping( + 8, + slot_mapping, + query_start_loc, + seq_lens, + block_table, + block_size=4, + compress_ratio=2, + ) + assert compressed.tolist() == [-1, -1, -1, -1, -1, 3 * 4 + 2, -1, 3 * 4 + 3] + + def test_index_conversion_warmup_uses_physical_block_stride(): config = SimpleNamespace( cache_config=SimpleNamespace(block_size=64), diff --git a/tests/v1/attention/test_kpool_tail_slot_mapping.py b/tests/v1/attention/test_kpool_tail_slot_mapping.py index 0966f9b20a..cb39e54037 100644 --- a/tests/v1/attention/test_kpool_tail_slot_mapping.py +++ b/tests/v1/attention/test_kpool_tail_slot_mapping.py @@ -32,6 +32,7 @@ from vllm.v1.attention.backends.mla.indexer import ( ) from vllm.v1.kv_cache_interface import KpoolTailSpec, compute_layout_strides from vllm.v1.kv_cache_layout import KVCacheLayout +from vllm.v1.worker.block_table import get_block_table_width KPOOL = 4 @@ -55,6 +56,29 @@ def test_tail_backend_layout_matches_kernel_pointer_arithmetic(): assert content_stride == 1 +def test_tail_spec_opts_out_of_generic_slot_mapping(): + """The tail row is one block wide (padded to the block-table alignment), so + the generic kernel's ``pos // kpool`` column index runs off the end of the + allocation for long prompts. The spec must opt out of it entirely.""" + spec = KpoolTailSpec( + block_size=KPOOL, + num_kv_heads=2, + head_size=128, + head_size_v=0, + dtype=torch.bfloat16, + sliding_window=KPOOL, + ) + max_len = 1 << 20 + width = get_block_table_width( + spec.max_num_blocks_per_req(None, max_len), + spec.block_size, + token_alignment=spec.block_table_token_alignment, + ) + + assert width * KPOOL < max_len + assert spec.uses_slot_mapping is False + + def make_tail_block_table(own_blocks, width=64): """Tail-group block table as BlockTables produces it: column 0 holds the request's single KpoolTailManager block, the remaining columns are never diff --git a/tests/v1/attention/test_sparse_mla_backends.py b/tests/v1/attention/test_sparse_mla_backends.py index 8ff52ecb38..7704600d09 100644 --- a/tests/v1/attention/test_sparse_mla_backends.py +++ b/tests/v1/attention/test_sparse_mla_backends.py @@ -421,7 +421,18 @@ def test_sparse_backend_decode_correctness( workspace_init, q_scale: float, k_scale: float, + monkeypatch, ): + if ( + batch_name == "large_q_pure_prefill" + and backend_cls == FlashMLASparseBackend + and kv_cache_dtype == "fp8_ds_mla" + and tensor_parallel_size == 4 + ): + monkeypatch.setattr( + "vllm.v1.attention.backends.mla.flashmla_sparse.split_prefill_chunks", + lambda rows, capacity: [(i, i + 1) for i in range(len(rows))], + ) if kv_cache_dtype not in backend_cls.supported_kv_cache_dtypes: pytest.skip(f"{backend_cls.get_name()} does not support {kv_cache_dtype}") @@ -774,6 +785,10 @@ def test_sparse_backend_decode_correctness( ) with torch.inference_mode(): + if backend_cls == FlashMLASparseBackend and kv_cache_dtype == "fp8_ds_mla": + from vllm.v1.worker.workspace import current_workspace_manager + + current_workspace_manager().lock() backend_output = mock_layer.forward_impl( query_vllm, kv_c_vllm, diff --git a/tests/v1/core/test_contiguous_kv_packing.py b/tests/v1/core/test_contiguous_kv_packing.py index 2fcf70c6ef..58d4e20bf7 100644 --- a/tests/v1/core/test_contiguous_kv_packing.py +++ b/tests/v1/core/test_contiguous_kv_packing.py @@ -651,3 +651,67 @@ class TestCompressorRingGroup: scheduler_block_size=128, ) assert len(manager.coordinator.single_type_managers) == len(groups) + + +class TestSWABoundedReplayGrouping: + def test_non_cacheable_swa_leaves_hits_to_the_paged_group(self): + """Under SWA bounded replay only the paged MLA group hashes (hash + block = its block) and the coordinator probes one spec group.""" + config = _shared_layout_config() + config.cache_config.enable_prefix_caching = True + specs: dict[str, KVCacheSpec] = { + "layers.2.attn": MLAAttentionSpec( + block_size=128, + num_kv_heads=1, + head_size=584, + dtype=torch.uint8, + tokens_per_state=2, + alignment=576, + ), + } + swa = SlidingWindowMLASpec( + block_size=32, + num_kv_heads=1, + head_size=584, + dtype=torch.uint8, + sliding_window=128, + alignment=576, + bounded_replay=True, + ) + for layer in range(4): + specs[f"layers.{layer}.attn.swa_cache"] = swa + + groups = get_kv_cache_groups(config, specs) + # The worker reads the replay window off the (packed) group specs to + # arm the window clamp; the wrapper must forward it like cacheability. + assert sorted( + (g.kv_cache_spec.prefix_cacheable, g.kv_cache_spec.prefix_replay_tokens) + for g in groups + ) == [(False, 128)] * 4 + [(True, 0)] + kv_cache_config = get_kv_cache_config_from_groups( + config, groups, available_memory=64 * _get_kv_cache_bytes_per_block(groups) + ) + manager = KVCacheManager( + generate_scheduler_kv_cache_config([kv_cache_config]), + max_model_len=8192, + enable_caching=True, + hash_block_size=128, + scheduler_block_size=128, + ) + assert len(manager.coordinator.attention_groups) == 1 + assert isinstance( + manager.coordinator.attention_groups[0].spec, MLAAttentionSpec + ) + + # Allocating and caching a request goes through every group's manager; + # the 32-token SWA group must skip hashing rather than assert on the + # 128-token hash block. + from tests.v1.core.utils import create_requests + + request = create_requests(num_requests=1, num_tokens=256, block_size=128)[0] + assert manager.allocate_slots(request, 256) is not None + request.num_computed_tokens = 256 + manager.cache_blocks(request, 256) + for single in manager.coordinator.single_type_managers: + cached = single.num_cached_block.get(request.request_id, 0) + assert cached == (2 if single.kv_cache_spec.prefix_cacheable else 0) diff --git a/tests/v1/core/test_kv_cache_utils.py b/tests/v1/core/test_kv_cache_utils.py index 3c82180474..07c2222145 100644 --- a/tests/v1/core/test_kv_cache_utils.py +++ b/tests/v1/core/test_kv_cache_utils.py @@ -1068,6 +1068,22 @@ def _stats(requests: int, queries: int, hits: int) -> PrefixCacheStats: return PrefixCacheStats(requests=requests, queries=queries, hits=hits) +def test_metrics_empty_distinguishes_no_queries_from_no_hits(): + """`hit_rate` alone cannot tell the two apart; `empty` can. + + Both an unobserved window and a genuine all-miss window report a hit + rate of 0.0, so anything surfacing that number to a human has to check + `empty` first - which is what the prefix-cache log line does. + """ + metrics = CachingMetrics(max_recent_requests=5) + assert metrics.empty + assert metrics.hit_rate == 0.0 + + metrics.observe(_stats(1, 20, 0)) + assert not metrics.empty + assert metrics.hit_rate == 0.0 + + def test_metrics(): """Test the prefix caching metrics.""" metrics = CachingMetrics(max_recent_requests=5) diff --git a/tests/v1/core/test_prefix_caching.py b/tests/v1/core/test_prefix_caching.py index 01bf73b6e7..d6848a6a7f 100644 --- a/tests/v1/core/test_prefix_caching.py +++ b/tests/v1/core/test_prefix_caching.py @@ -6,7 +6,7 @@ import copy from collections.abc import Callable from dataclasses import replace from math import lcm -from types import SimpleNamespace +from types import MethodType, SimpleNamespace from unittest.mock import MagicMock import pytest @@ -325,7 +325,7 @@ def test_hisparse_reports_when_context_is_fully_resident(): def test_hisparse_host_prefix_can_be_completed_by_indexer_offload(): - """Keep indexer-only imports host-backed in the resident block table.""" + """Restore the final host page when completing an indexer-only import.""" manager = make_hisparse_kv_cache_manager( 32, 16, @@ -360,11 +360,15 @@ def test_hisparse_host_prefix_can_be_completed_by_indexer_offload(): source, indexer, resident, hot = manager.get_blocks(resumed.request_id).blocks assert len(source) == len(indexer) == len(resident) == 4 assert len(hot) == 2 - # The local prefix is adopted from shadow pages (GPU-resident), while the - # externally imported page stays host-backed until its tail allocation. assert not any(block.is_null for block in resident[:2]) - assert resident[2].is_null + assert not resident[2].is_null assert not resident[3].is_null + coordinator = get_hisparse_coordinator(manager) + assert not coordinator.build_offload_command().page_transfers + coordinator.finish_host_import(resumed.request_id, failed=False) + transfers = coordinator.build_offload_command().page_transfers + assert len(transfers) == 1 and transfers[0].restore + assert transfers[0].resident_block_ids == (resident[2].block_id,) def test_hisparse_indexer_offload_is_capped_by_missing_host_prefix(): @@ -465,7 +469,8 @@ def test_connector_completes_partial_prefix_without_importing_missing_host_kv( ) allocated = manager.get_blocks(resumed.request_id).blocks assert len(allocated[0]) == host_blocks - assert allocated[2][1].is_null == (external > 0) + assert allocated[2][1].is_null == (host_blocks > 2) + assert not allocated[2][-1].is_null manager.free(resumed) # A surviving indexer offload must not make absent host KV look computed. @@ -490,6 +495,87 @@ def allocate_external_prefix( ) +@pytest.mark.parametrize("num_tokens", [1, 15, 16, 17, 31, 32, 33, 127]) +def test_nixl_hisparse_full_block_import_keeps_a_writable_tail(num_tokens): + """Import every real token and restore the page used by last-token replay.""" + from tests.v1.kv_connector.unit.utils import create_vllm_config + from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import ( + NixlPullConnectorScheduler, + ) + + manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True) + config = create_vllm_config(block_size=HISPARSE_BLOCK_SIZE) + connector = NixlPullConnectorScheduler(config, "test", manager.kv_cache_config) + request = make_request( + "import", list(range(num_tokens)), HISPARSE_BLOCK_SIZE, sha256 + ) + request.kv_transfer_params = {"do_remote_prefill": True} + count, is_async = connector.get_num_new_matched_tokens(request, 0) + assert count == num_tokens and is_async + assert allocate_external_prefix(manager, request, count) is not None + source, _, resident, _ = manager.get_blocks(request.request_id).blocks + assert len(source) == len(resident) == (num_tokens + 15) // 16 + assert all(block.is_null for block in resident[:-1]) + assert not resident[-1].is_null + coordinator = get_hisparse_coordinator(manager) + assert not coordinator.build_offload_command().page_transfers + + coordinator.finish_host_import(request.request_id, failed=False) + transfers = coordinator.build_offload_command().page_transfers + assert len(transfers) == 1 + restore = transfers[0] + assert restore.restore and not restore.after_forward + assert restore.host_block_id == source[-1].block_id + assert restore.resident_block_ids == (resident[-1].block_id,) + request.num_computed_tokens = count + scheduler = SimpleNamespace( + connector=connector, + kv_cache_manager=manager, + failed_recving_kv_req_ids=set(), + finished_recving_kv_req_ids={request.request_id}, + prefix_replay_tokens=0, + ) + scheduler._mark_prefix_replay = MethodType(Scheduler._mark_prefix_replay, scheduler) + Scheduler._update_waiting_for_remote_kv(scheduler, request) + assert request.num_tokens - request.num_computed_tokens == 1 + assert manager.allocate_slots(request, num_new_tokens=1) is not None + assert coordinator.build_row_mirrors([(request.request_id, num_tokens - 1, 1)]) + # A pending restore must not expose uninitialized GPU copies to prefix hits. + assert all(resident[-1] not in copies for copies in coordinator.copies.values()) + + coordinator.update_spills({restore.transfer_id: 2}, {restore.transfer_id: 1}) + assert resident[-1].ref_cnt == 2 + assert all(resident[-1] not in copies for copies in coordinator.copies.values()) + coordinator.update_spills({}, {restore.transfer_id: 1}) + assert resident[-1].ref_cnt == 1 + assert coordinator.request_states[request.request_id].valid_pages == set( + range(num_tokens // HISPARSE_BLOCK_SIZE) + ) + if num_tokens % HISPARSE_BLOCK_SIZE: + assert source[-1].block_hash is None + else: + assert coordinator.copies[source[-1].block_hash] == (resident[-1],) + + +def test_hisparse_aborted_tail_restore_retains_both_endpoints(): + """An abort cannot recycle storage still used by a queued tail restore.""" + manager = make_hisparse_kv_cache_manager(16, 16) + request = make_request("import", list(range(17)), HISPARSE_BLOCK_SIZE, sha256) + assert allocate_external_prefix(manager, request, 17) is not None + source, _, resident, _ = manager.get_blocks(request.request_id).blocks + host_tail, gpu_tail = source[-1], resident[-1] + coordinator = get_hisparse_coordinator(manager) + coordinator.finish_host_import(request.request_id, failed=False) + restore = coordinator.build_offload_command().page_transfers[0] + manager.free(request) + assert host_tail.ref_cnt == gpu_tail.ref_cnt == 1 + coordinator.update_spills({restore.transfer_id: 2}, {restore.transfer_id: 1}) + assert host_tail.ref_cnt == gpu_tail.ref_cnt == 1 + coordinator.update_spills({}, {restore.transfer_id: 1}) + assert host_tail.ref_cnt == gpu_tail.ref_cnt == 0 + assert not coordinator.has_pending_work() + + @pytest.mark.parametrize("enable_caching", [False, True]) def test_hisparse_keeps_resident_pages_until_hot_buffer_is_allocated(enable_caching): """Later admissions must not remove a scheduled request's only readable tier.""" @@ -669,7 +755,7 @@ def test_hisparse_host_cow_copy_is_drained_without_a_gpu_pool(): def test_hisparse_inflight_host_import_reserves_remaining_gpu_pages(): """An in-flight host import must reserve its unwritten resident pages.""" manager = make_hisparse_kv_cache_manager( - 11, + 12, 16, transfer_device_cache=True, ) @@ -1127,9 +1213,10 @@ def test_hisparse_external_import_uses_hard_gpu_footprint(): assert len(source) == num_prompt_blocks assert len(indexer) == num_prompt_blocks assert len(resident) == num_prompt_blocks - assert all(block.is_null for block in resident) + assert all(block.is_null for block in resident[:-1]) + assert not resident[-1].is_null assert len(hot) == 2 - assert manager.block_pool.get_num_free_blocks() == 1 + assert manager.block_pool.get_num_free_blocks() == 0 def test_hisparse_external_import_survives_capacity_retry(): @@ -1149,7 +1236,8 @@ def test_hisparse_external_import_survives_capacity_retry(): manager.free(first) assert allocate_external_prefix(manager, second, len(tokens)) is not None _, _, resident, hot = manager.get_blocks(second.request_id).blocks - assert all(block.is_null for block in resident) + assert all(block.is_null for block in resident[:-1]) + assert not resident[-1].is_null assert len(hot) == 2 @@ -5849,3 +5937,29 @@ def test_device_eviction_keeps_host_prefix(): assert host_block.block_hash is not None, ( "Device eviction invalidated unrelated host KV" ) + + +def test_get_unhashed_block_ids_all_groups(): + """Unhashed, non-null block ids are reported per KV cache group. + A group with no unhashed blocks reports an empty list, not a missing entry. + """ + + def hashed(block_id: int, group_id: int) -> KVCacheBlock: + block = KVCacheBlock(block_id=block_id) + block.set_block_hash(make_block_hash_with_group_id(BlockHash(b"h"), group_id)) + return block + + blocks = KVCacheBlocks( + ( + [ + KVCacheBlock(block_id=1), # unhashed + hashed(2, group_id=0), # cached + KVCacheBlock(block_id=3, is_null=True), # null padding + KVCacheBlock(block_id=4), # unhashed + ], + # No unhashed blocks in this group. + [hashed(5, group_id=1), KVCacheBlock(block_id=6, is_null=True)], + ) + ) + + assert blocks.get_unhashed_block_ids_all_groups() == [[1, 4], []] diff --git a/tests/v1/core/test_prefix_replay.py b/tests/v1/core/test_prefix_replay.py new file mode 100644 index 0000000000..b2e78e1f92 --- /dev/null +++ b/tests/v1/core/test_prefix_replay.py @@ -0,0 +1,279 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""SWA bounded replay: after a prefix hit the scheduler recomputes the tail of +the hit to rebuild the non-cacheable sliding-window group, keeps the hit's +blocks, and hands the worker the replay start.""" + +import pytest +import torch + +from tests.v1.kv_connector.unit.utils import MockKVConfig, create_model_runner_output +from vllm.v1.core.sched.scheduler import Scheduler +from vllm.v1.kv_cache_interface import ( + FullAttentionSpec, + KVCacheConfig, + KVCacheGroupSpec, + SlidingWindowMLASpec, +) +from vllm.v1.outputs import ModelRunnerOutput +from vllm.v1.request import RequestStatus +from vllm.v1.structured_output import StructuredOutputManager + +from .utils import create_requests, create_scheduler + +BLOCK_SIZE = 16 +WINDOW = 32 +NUM_PROMPT_TOKENS = 100 +# 100 tokens -> 6 full blocks cached -> 96-token hit. +HIT_TOKENS = NUM_PROMPT_TOKENS // BLOCK_SIZE * BLOCK_SIZE +FULL, SWA = 0, 1 + + +def _replay_scheduler( + *, + long_prefill_token_threshold: int = 0, + use_kv_connector: MockKVConfig | None = None, +) -> Scheduler: + """A hybrid layout: one prefix-cacheable full-attention group and one + replayed sliding-window group.""" + base = create_scheduler( + block_size=BLOCK_SIZE, + enable_prefix_caching=True, + long_prefill_token_threshold=long_prefill_token_threshold, + use_kv_connector=use_kv_connector, + ) + vllm_config = base.vllm_config + kv_cache_config = KVCacheConfig( + num_blocks=10000, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec( + ["full"], + FullAttentionSpec( + block_size=BLOCK_SIZE, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ), + ), + KVCacheGroupSpec( + ["swa"], + SlidingWindowMLASpec( + block_size=BLOCK_SIZE, + num_kv_heads=1, + head_size=64, + dtype=torch.bfloat16, + sliding_window=WINDOW, + bounded_replay=True, + ), + ), + ], + ) + scheduler = Scheduler( + vllm_config=vllm_config, + kv_cache_config=kv_cache_config, + block_size=BLOCK_SIZE, + log_stats=True, + structured_output_manager=StructuredOutputManager(vllm_config), + ) + scheduler.use_v2_model_runner = True + assert scheduler.prefix_replay_tokens == WINDOW + return scheduler + + +def _prefill(scheduler: Scheduler, request) -> None: + scheduler.add_request(request) + out = scheduler.schedule() + assert out.num_scheduled_tokens[request.request_id] == request.num_prompt_tokens + scheduler.update_from_output(out, _step_output(out, [request])) + + +def _step_output(out, requests) -> ModelRunnerOutput: + """Model output for every scheduled request: a token once its prefill is + done (non-zero, so outputs never extend the all-zero shared prompt in the + cache), nothing while it is still prefilling. schedule() has already + advanced num_computed_tokens past this step's chunk.""" + scheduled = [r for r in requests if r.request_id in out.num_scheduled_tokens] + return ModelRunnerOutput( + req_ids=[r.request_id for r in scheduled], + req_id_to_index={r.request_id: i for i, r in enumerate(scheduled)}, + sampled_token_ids=[ + [1000] if r.num_computed_tokens >= r.num_prompt_tokens else [] + for r in scheduled + ], + logprobs=None, + prompt_logprobs_dict={}, + pooler_output=None, + ) + + +def _new_req_data(out, request): + return next(r for r in out.scheduled_new_reqs if r.req_id == request.request_id) + + +def test_hit_replays_window_without_reallocating(): + scheduler = _replay_scheduler() + expected_replay = WINDOW + first, second = create_requests( + num_requests=2, + num_tokens=NUM_PROMPT_TOKENS, + same_prompt=True, + block_size=BLOCK_SIZE, + ) + _prefill(scheduler, first) + manager = scheduler.kv_cache_manager + + scheduler.add_request(second) + out = scheduler.schedule() + + replay_start = HIT_TOKENS - expected_replay + new_req = _new_req_data(out, second) + assert new_req.num_computed_tokens == replay_start + assert new_req.replay_start == replay_start + assert ( + out.num_scheduled_tokens[second.request_id] == NUM_PROMPT_TOKENS - replay_start + ) + + first_blocks = manager.get_blocks(first.request_id).blocks + second_blocks = manager.get_blocks(second.request_id).blocks + # The full-attention hit is shared (replayed tokens included); only the + # tail block is new. + num_hit_blocks = HIT_TOKENS // BLOCK_SIZE + assert second_blocks[FULL][:num_hit_blocks] == first_blocks[FULL][:num_hit_blocks] + assert all(b.ref_cnt == 2 for b in second_blocks[FULL][:num_hit_blocks]) + assert len(second_blocks[FULL]) == num_hit_blocks + 1 + # The window group never hits: blocks below the window are null, the + # window (replayed and new tokens) gets fresh blocks. + num_skipped_blocks = (HIT_TOKENS - WINDOW + 1) // BLOCK_SIZE + assert all(b.is_null for b in second_blocks[SWA][:num_skipped_blocks]) + num_window_blocks = len(second_blocks[SWA]) - num_skipped_blocks + assert num_window_blocks == -(-NUM_PROMPT_TOKENS // BLOCK_SIZE) - num_skipped_blocks + assert not any(b.is_null for b in second_blocks[SWA][num_skipped_blocks:]) + + scheduler.update_from_output(out, _step_output(out, [first, second])) + assert second.num_computed_tokens == NUM_PROMPT_TOKENS + + +def test_replay_only_chunks_make_progress(): + """A chunk cap smaller than the replay window schedules replay-only + chunks; they advance the request instead of stalling it.""" + scheduler = _replay_scheduler(long_prefill_token_threshold=BLOCK_SIZE) + first, second = create_requests( + num_requests=2, + num_tokens=NUM_PROMPT_TOKENS, + same_prompt=True, + block_size=BLOCK_SIZE, + ) + scheduler.add_request(first) + while first.num_computed_tokens < NUM_PROMPT_TOKENS: + out = scheduler.schedule() + scheduler.update_from_output(out, _step_output(out, [first])) + + scheduler.add_request(second) + + def full_blocks(): + return scheduler.kv_cache_manager.get_blocks(second.request_id).blocks[FULL] + + scheduled: list[int] = [] + while second.num_computed_tokens < NUM_PROMPT_TOKENS: + out = scheduler.schedule() + if not scheduled: + new_req = _new_req_data(out, second) + assert new_req.num_computed_tokens == HIT_TOKENS - WINDOW + assert new_req.replay_start == HIT_TOKENS - WINDOW + # A replay-only chunk adopts the hit and allocates nothing new. + assert len(full_blocks()) == HIT_TOKENS // BLOCK_SIZE + scheduled.append(out.num_scheduled_tokens[second.request_id]) + scheduler.update_from_output(out, _step_output(out, [first, second])) + # 32 replayed + 4 new tokens in 16-token chunks: two replay-only chunks. + assert scheduled == [BLOCK_SIZE, BLOCK_SIZE, NUM_PROMPT_TOKENS - HIT_TOKENS] + assert len(full_blocks()) == -(-NUM_PROMPT_TOKENS // BLOCK_SIZE) + assert second.status == RequestStatus.RUNNING + + +def test_async_remote_kv_hit_replays_after_load(): + """A KV-connector hit loaded asynchronously carries no window state + either; the replay is applied when the load completes.""" + matched = 64 + scheduler = _replay_scheduler( + use_kv_connector=MockKVConfig(matched_tokens=matched, is_async=True) + ) + request = create_requests( + num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE + )[0] + scheduler.add_request(request) + out = scheduler.schedule() + assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS + assert request.request_id not in out.num_scheduled_tokens + scheduler.update_from_output( + out, create_model_runner_output([], finished_recving={request.request_id}) + ) + + out = scheduler.schedule() + new_req = _new_req_data(out, request) + assert new_req.num_computed_tokens == matched - WINDOW + assert new_req.replay_start == matched - WINDOW + assert ( + out.num_scheduled_tokens[request.request_id] + == NUM_PROMPT_TOKENS - matched + WINDOW + ) + + +def test_remote_kv_hit_is_taken_in_whole_blocks(): + """A hit ending one token short of a block boundary would put the replay + window's first token in a block the sliding-window group retires, so a + connector hit is cut back to whole blocks.""" + matched = 4 * BLOCK_SIZE - 1 + scheduler = _replay_scheduler( + use_kv_connector=MockKVConfig(matched_tokens=matched, is_async=True) + ) + request = create_requests( + num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE + )[0] + scheduler.add_request(request) + out = scheduler.schedule() + scheduler.update_from_output( + out, create_model_runner_output([], finished_recving={request.request_id}) + ) + out = scheduler.schedule() + new_req = _new_req_data(out, request) + hit = 3 * BLOCK_SIZE + assert new_req.replay_start == hit - WINDOW + assert new_req.num_computed_tokens == hit - WINDOW + swa_manager = scheduler.kv_cache_manager.coordinator.single_type_managers[SWA] + swa_blocks = scheduler.kv_cache_manager.get_blocks(request.request_id).blocks[SWA] + assert swa_blocks[new_req.replay_start // BLOCK_SIZE] is not swa_manager._null_block + + +@pytest.mark.parametrize("via_connector", [False, True]) +def test_hit_no_longer_than_window_is_ignored(via_connector): + """Such a hit would be recomputed in full anyway. It is not adopted (an + async load of it is not even started), so a request has computed tokens + iff it replays.""" + if via_connector: + scheduler = _replay_scheduler( + use_kv_connector=MockKVConfig(matched_tokens=WINDOW, is_async=True) + ) + request = create_requests( + num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE + )[0] + else: + scheduler = _replay_scheduler() + first = create_requests( + num_requests=1, num_tokens=WINDOW, same_prompt=True, block_size=BLOCK_SIZE + )[0] + _prefill(scheduler, first) + request = create_requests( + num_requests=1, + num_tokens=NUM_PROMPT_TOKENS, + same_prompt=True, + block_size=BLOCK_SIZE, + req_ids=["long"], + )[0] + scheduler.add_request(request) + out = scheduler.schedule() + new_req = _new_req_data(out, request) + assert request.status == RequestStatus.RUNNING + assert new_req.num_computed_tokens == 0 + assert new_req.replay_start == 0 + assert out.num_scheduled_tokens[request.request_id] == NUM_PROMPT_TOKENS diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index ef818f27c7..4951014d1b 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -36,10 +36,13 @@ from vllm.utils.hashing import sha256 from vllm.v1.core.encoder_cache_manager import EncoderCacheManager from vllm.v1.core.kv_cache_coordinator import HybridKVCacheCoordinator from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash +from vllm.v1.core.sched.interface import PauseState from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput from vllm.v1.core.sched.scheduler import Scheduler from vllm.v1.core.single_type_kv_cache_manager import register_all_kvcache_specs from vllm.v1.engine import FinishReason +from vllm.v1.engine.core import EngineCore +from vllm.v1.executor.uniproc_executor import UniProcExecutor from vllm.v1.hisparse.coordinator import get_hisparse_coordinator from vllm.v1.kv_cache_interface import ( FullAttentionSpec, @@ -1487,6 +1490,79 @@ def test_scheduler_reset_prefix_cache(): assert scheduler.waiting[i] == request +def test_kv_cache_release_after_keep_pause_preserves_requests(): + """A completed keep pause permits release and recomputation of live requests.""" + scheduler = create_scheduler(max_num_seqs=1, enable_prefix_caching=True) + running, waiting = create_requests(num_requests=2) + for request in (running, waiting): + scheduler.add_request(request) + scheduler_output = scheduler.schedule() + scheduler.update_from_output( + scheduler_output, + ModelRunnerOutput( + req_ids=[running.request_id], + req_id_to_index={running.request_id: 0}, + sampled_token_ids=[[1]], + ), + ) + assert scheduler.kv_cache_manager.get_block_ids(running.request_id)[0] + + core = object.__new__(EngineCore) + core.scheduler = scheduler + core.model_executor = Mock(is_sleeping=False) + core.mm_receiver_cache = None + core.batch_queue = None + + assert core.pause_scheduler(mode="keep", clear_cache=False) is None + assert scheduler.running == [running] + assert list(scheduler.waiting) == [waiting] + assert not scheduler.has_requests() + + core.release_kv_cache_memory() + + core.model_executor.discard.assert_called_once_with(("kv_cache",)) + assert scheduler.requests == { + request.request_id: request for request in (running, waiting) + } + assert not scheduler.running + assert list(scheduler.waiting) == [running, waiting] + assert running.status == RequestStatus.PREEMPTED + assert running.num_computed_tokens == 0 + assert list(running.output_token_ids) == [1] + assert not scheduler.kv_cache_manager.get_block_ids(running.request_id)[0] + assert scheduler.schedule().total_num_scheduled_tokens == 0 + + assert core.wake_up(tags=["kv_cache"]) + resumed = scheduler.schedule() + assert resumed.num_scheduled_tokens == {running.request_id: running.num_tokens} + + +@pytest.mark.parametrize( + "sleeping_tags", + [{"weights", "kv_cache"}, {"weights"}, {"kv_cache"}], + ids=["full-sleep", "weights-only", "kv-only-or-repeated-release"], +) +def test_kv_cache_release_rejects_nonresident_memory(sleeping_tags): + """Reject release without resetting caches or discarding more memory.""" + core = object.__new__(EngineCore) + core.scheduler = create_scheduler() + core.scheduler.set_pause_state(PauseState.PAUSED_ALL) + core.batch_queue = None + core._reset_caches = Mock() + core.model_executor = object.__new__(UniProcExecutor) + core.model_executor.sleeping_tags = sleeping_tags.copy() + core.model_executor.collective_rpc = Mock() + + with pytest.raises( + RuntimeError, match="requires all executor memory to be resident" + ): + core.release_kv_cache_memory() + + core._reset_caches.assert_not_called() + core.model_executor.collective_rpc.assert_not_called() + assert core.model_executor.sleeping_tags == sleeping_tags + + def test_reset_connector_cache_no_connector_is_no_op_success(): """``reset_connector_cache`` must return True when no connector is configured. @@ -2302,6 +2378,25 @@ def test_has_sync_kv_loads( assert output.has_sync_kv_loads is expected_has_sync_loads +def test_kv_connector_honors_skip_reading_prefix_cache(): + """A request that must score every prompt row takes no external hit.""" + BLOCK_SIZE = 16 + scheduler = create_scheduler( + enable_prefix_caching=True, + use_kv_connector=mock_kv(matched_tokens=BLOCK_SIZE * 2, is_async=False), + block_size=BLOCK_SIZE, + ) + plain, scoring = create_requests(num_requests=2, num_tokens=BLOCK_SIZE * 4) + scoring.sampling_params.prompt_logprobs = 1 + scoring.skip_reading_prefix_cache = True + scheduler.add_request(plain) + scheduler.add_request(scoring) + + output = scheduler.schedule() + assert output.num_scheduled_tokens[plain.request_id] == BLOCK_SIZE * 2 + assert output.num_scheduled_tokens[scoring.request_id] == BLOCK_SIZE * 4 + + @pytest.mark.parametrize("is_async", [False, True]) def test_kv_connector_basic(is_async: bool): """Test whether Scheduler with KVConnector schedules tokens, allocates diff --git a/tests/v1/cudagraph/test_encoder_cudagraph.py b/tests/v1/cudagraph/test_encoder_cudagraph.py index 096f0e4075..1f145bdc89 100644 --- a/tests/v1/cudagraph/test_encoder_cudagraph.py +++ b/tests/v1/cudagraph/test_encoder_cudagraph.py @@ -13,7 +13,9 @@ Test organization: """ from collections.abc import Hashable +from types import SimpleNamespace from typing import Any +from unittest.mock import MagicMock, patch import pytest import torch @@ -575,6 +577,99 @@ class TestEncoderCudaGraphCaptureReplay: assert out.shape == (4, _HIDDEN) +# --------------------------------------------------------------------------- +# E-only capture and output lifecycle +# --------------------------------------------------------------------------- + + +@pytest.mark.skipif( + not current_platform.is_cuda_alike(), reason="Skip if not cuda or rocm" +) +@pytest.mark.usefixtures("dist_init", "workspace_init") +@pytest.mark.parametrize("profile_only", [False, True]) +@torch.inference_mode() +def test_eonly_capture_preserves_outputs_across_replay_and_fallback(profile_only): + """The E-only entry captures only the encoder and preserves cached outputs.""" + from vllm.distributed.ec_transfer.ec_connector.base import ( + ECConnectorBase, + ECConnectorMetadata, + ) + from vllm.v1.worker.gpu.ec_connector import ActiveECConnector + from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner + from vllm.v1.worker.mm_encoder_model_runner import MMEncoderModelRunner + from vllm.v1.worker.workspace import lock_workspace + + device = torch.device("cuda:0") + dtype = torch.float16 + model = SimpleMockViTModel().to(device).half() + manager = _make_manager_for_gpu(model, _BUDGETS, _MAX_BATCH, device, dtype) + encoder = object.__new__(EncoderRunner) + encoder.device = device + encoder.cudagraph_manager = manager + runner = object.__new__(MMEncoderModelRunner) + runner.model_state = SimpleNamespace(encoder_runner=encoder) + # No decoder manager is installed: capture must be encoder-only. + with patch( + "vllm.v1.worker.mm_encoder_model_runner.lock_workspace", wraps=lock_workspace + ) as lock: + runner.capture_model(profile_only=profile_only) + assert lock.call_count == int(not profile_only) + assert len(manager.budget_graphs["default"]) == len(_BUDGETS) + + cache: dict[str, torch.Tensor] = {} + pending_sends: dict[str, torch.Tensor] = {} + connector = MagicMock(spec=ECConnectorBase) + connector.is_producer = True + connector.is_consumer = False + connector.get_finished.return_value = (None, None) + connector.save_caches.side_effect = lambda *, encoder_cache, mm_hash: ( + pending_sends.update({mm_hash: encoder_cache[mm_hash]}) + ) + with patch( + "vllm.v1.worker.gpu.ec_connector.get_ec_transfer", return_value=connector + ): + ec = ActiveECConnector(SimpleNamespace(), cache) + scheduled = SimpleNamespace( + ec_connector_metadata=ECConnectorMetadata(), finished_req_ids=frozenset() + ) + saved_outputs: list[tuple[torch.Tensor, torch.Tensor]] = [] + # Mixed sizes exercise packing order; 64 is the boundary, 81 falls back. + for grids in ([[1, 8, 8], [1, 4, 4]], [[1, 16, 16]], [[1, 18, 18]], [[1, 4, 4]]): + inputs = _make_mm_kwargs(grids, device, dtype) + expected = model.encoder_eager_forward(inputs).split( + [t * (h // 2) * (w // 2) for t, h, w in grids] + ) + with ec.maybe_get_output(scheduled) as ec_output: + outputs = manager.execute(inputs) + assert outputs is not None + cache[str(len(cache))] = outputs[0] + assert ec_output.finished_sending is None + assert outputs is not None + for actual, eager in zip(outputs, expected): + torch.testing.assert_close(actual, eager) + for previous, snapshot in saved_outputs: + torch.testing.assert_close(previous, snapshot, rtol=0, atol=0) + saved_outputs.extend((output, output.clone()) for output in outputs) + assert manager.graph_hits == 4 + assert manager.graph_misses == 1 + assert pending_sends.keys() == cache.keys() + connector.get_finished.return_value = (set(cache), None) + with ec.maybe_get_output(scheduled) as ec_output: + pass + assert ec_output.finished_sending == set(cache) + + +def test_eonly_without_encoder_graph_skips_capture(): + from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner + from vllm.v1.worker.mm_encoder_model_runner import MMEncoderModelRunner + + encoder = object.__new__(EncoderRunner) + encoder.cudagraph_manager = None + runner = object.__new__(MMEncoderModelRunner) + runner.model_state = SimpleNamespace(encoder_runner=encoder) + assert runner.capture_model() == 0 + + # --------------------------------------------------------------------------- # SimpleMockViTVideoModel — extends SimpleMockViTModel with video support # --------------------------------------------------------------------------- diff --git a/tests/v1/e2e/spec_decode/acceptance_rates/dflash/test_dflash.py b/tests/v1/e2e/spec_decode/acceptance_rates/dflash/test_dflash.py index 2b1800f782..0dff043336 100644 --- a/tests/v1/e2e/spec_decode/acceptance_rates/dflash/test_dflash.py +++ b/tests/v1/e2e/spec_decode/acceptance_rates/dflash/test_dflash.py @@ -26,6 +26,8 @@ class DFlashCorrectnessConfig: use_chat_completions: bool = False enforce_eager: bool = False disable_flashinfer_sampler: bool = False + language_model_only: bool = False + chat_template_kwargs: dict[str, object] | None = None QWEN3_DFLASH = DFlashCorrectnessConfig( @@ -35,11 +37,23 @@ QWEN3_DFLASH = DFlashCorrectnessConfig( expected_acceptance_len=3.5, ) +QWEN3_8_DFLASH2_NVFP4 = DFlashCorrectnessConfig( + model="nvidia/Qwen3.8-27B-NVFP4", + draft_model="z-lab/Qwen3.8-27B-DFlash2", + expected_accuracy=0.90, + expected_acceptance_len=6.14 * 0.95, + num_speculative_tokens=7, + max_num_seqs=32, + use_chat_completions=True, + language_model_only=True, + chat_template_kwargs={"enable_thinking": False}, +) + LAGUNA_DFLASH_NVFP4 = DFlashCorrectnessConfig( model="poolside/Laguna-XS-2.1-NVFP4", draft_model="poolside/Laguna-XS-2.1-DFlash-NVFP4", expected_accuracy=0.7, # Standard GSM8K sanity floor. - expected_acceptance_len=3.55 * 0.9, + expected_acceptance_len=3.05 * 0.9, num_speculative_tokens=15, max_model_len=8192, max_num_seqs=32, @@ -104,6 +118,11 @@ def test_dflash_reference_acceptance_lengths( True, id="laguna-nvfp4-mrv2", ), + pytest.param( + QWEN3_8_DFLASH2_NVFP4, + True, + id="qwen3.8-dflash2-nvfp4-mrv2", + ), ], ) def test_dflash_correctness( @@ -134,12 +153,14 @@ def test_dflash_correctness( disable_log_stats=False, enable_chunked_prefill=None, compilation_config=CompilationConfig(), + language_model_only=config.language_model_only, ) as spec_runner: spec_llm = spec_runner.llm results = evaluate_gsm8k_offline( spec_llm, num_questions=config.num_questions, use_chat_completions=config.use_chat_completions, + chat_template_kwargs=config.chat_template_kwargs, ) accuracy = results["accuracy"] acceptance_len = compute_acceptance_len(spec_llm.get_metrics()) diff --git a/tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh b/tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh index 05af1bb6fd..c4e9b0d069 100755 --- a/tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh +++ b/tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh @@ -20,6 +20,8 @@ # SKIP_BASELINE set to 1 to reuse existing BASELINE_FILE # CONCURRENCY / REPEAT concurrent requests / rounds (defaults 3 / 2) # MAX_MODEL_LEN context length (default 16384) +# E_CUDAGRAPH_MM_ENCODER enable encoder CUDA graphs (default 0) +# E_MM_ENCODER_ATTN_BACKEND optional encoder attention backend override set -euo pipefail @@ -35,6 +37,15 @@ if [[ "$USE_MM_PROMPTS" == "1" ]]; then TEST_ARGS+=(--use_mm_prompts --mm_smoke_test) fi MAX_MODEL_LEN="${MAX_MODEL_LEN:-16384}" +E_ATTN_ARGS=() +if [[ -n "${E_MM_ENCODER_ATTN_BACKEND:-}" ]]; then + E_ATTN_ARGS+=(--mm-encoder-attn-backend "$E_MM_ENCODER_ATTN_BACKEND") +fi +case "${E_CUDAGRAPH_MM_ENCODER:-0}" in + 0) E_COMPILATION_CONFIG='{"cudagraph_mm_encoder":false}' ;; + 1) E_COMPILATION_CONFIG='{"cudagraph_mm_encoder":true}' ;; + *) echo "E_CUDAGRAPH_MM_ENCODER must be 0 or 1"; exit 1 ;; +esac GPU_SINGLE="${GPU_SINGLE:-0}" GPU_E="${GPU_E:-0}" @@ -203,7 +214,7 @@ run_epd_mooncake() { cleanup_instances echo "Starting ENCODER on GPU $GPU_E port $ENCODE_PORT" - CUDA_VISIBLE_DEVICES="$GPU_E" "${VLLM_SERVE[@]}" "$MODEL" \ + VLLM_SERVER_DEV_MODE=1 CUDA_VISIBLE_DEVICES="$GPU_E" "${VLLM_SERVE[@]}" "$MODEL" \ --port "$ENCODE_PORT" \ --gpu-memory-utilization 0.35 \ --mm-tensor-ipc torch_shm \ @@ -215,6 +226,9 @@ run_epd_mooncake() { --limit-mm-per-prompt '{"image":2,"video":0}' \ --allowed-local-media-path "${GIT_ROOT}/tests/v1/ec_connector/integration" \ --ec-transfer-config "$ENC_EC_JSON" \ + --compilation-config "$E_COMPILATION_CONFIG" \ + "${E_ATTN_ARGS[@]}" \ + --worker-extension-cls tests.v1.ec_connector.integration.test_epd_correctness.EncoderGraphTestWorkerExtension \ >"${LOG_PATH}/mooncake_epd_encoder.log" 2>&1 & local ENCODER_PID=$! PIDS+=("$ENCODER_PID") @@ -266,6 +280,26 @@ run_epd_mooncake() { --baseline_file "$BASELINE_FILE" \ "${TEST_ARGS[@]}" + curl -fsS "http://localhost:${ENCODE_PORT}/collective_rpc" \ + -H 'Content-Type: application/json' \ + -d '{"method":"encoder_graph_stats"}' \ + >"${LOG_PATH}/encoder_graph_stats.json" + "$PYTHON_BIN" - "${LOG_PATH}/encoder_graph_stats.json" "${E_CUDAGRAPH_MM_ENCODER:-0}" <<'PY' +import json +import sys + +with open(sys.argv[1]) as f: + results = json.load(f)["results"] +assert results, "No encoder workers returned graph statistics" +for stats in results: + if sys.argv[2] == "1": + assert stats["is_captured"], stats + assert stats["graph_hits"] > 0, stats + else: + assert stats["graph_hits"] == 0, stats +print("Encoder graph statistics:", results) +PY + cleanup_instances } diff --git a/tests/v1/ec_connector/integration/test_epd_correctness.py b/tests/v1/ec_connector/integration/test_epd_correctness.py index b8565a0acd..4e37637ea4 100644 --- a/tests/v1/ec_connector/integration/test_epd_correctness.py +++ b/tests/v1/ec_connector/integration/test_epd_correctness.py @@ -26,6 +26,7 @@ import json import os import time from concurrent.futures import ThreadPoolExecutor +from typing import TYPE_CHECKING import openai import requests @@ -33,6 +34,26 @@ import requests from vllm.assets.image import ImageAsset from vllm.multimodal.utils import encode_image_url +if TYPE_CHECKING: + from vllm.v1.worker.mm_encoder_model_runner import MMEncoderModelRunner + + +class EncoderGraphTestWorkerExtension: + """Expose encoder replay evidence through the existing test RPC endpoint.""" + + model_runner: "MMEncoderModelRunner" + + def encoder_graph_stats(self): + from vllm.v1.worker.mm_encoder_model_runner import MMEncoderModelRunner + + runner = self.model_runner + assert isinstance(runner, MMEncoderModelRunner) + manager = runner.model_state.encoder_runner.cudagraph_manager + if manager is None: + return {"is_captured": False, "graph_hits": 0, "graph_misses": 0} + return {"is_captured": manager.is_captured(), **manager.get_cumulative_stats()} + + MAX_OUTPUT_LEN = 256 # Sample prompts with multimodal content diff --git a/tests/v1/ec_connector/unit/cpu/test_ec_shared_region.py b/tests/v1/ec_connector/unit/cpu/test_ec_shared_region.py index 722c0d89fa..c168970147 100644 --- a/tests/v1/ec_connector/unit/cpu/test_ec_shared_region.py +++ b/tests/v1/ec_connector/unit/cpu/test_ec_shared_region.py @@ -13,6 +13,8 @@ from unittest.mock import MagicMock, patch import pytest import torch +from vllm.config import ParallelConfig +from vllm.distributed.ec_transfer.ec_connector.cpu import common from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import ( ECSharedRegion, _wait_for_file_size, @@ -95,6 +97,32 @@ def test_second_instance_opens_existing_file_and_shares_memory(): r1.cleanup() +def test_dense_dp_engines_get_distinct_regions(monkeypatch): + """Dense DP engines reset data_parallel_rank to 0 but share instance_id, + so the region must be keyed by data_parallel_index.""" + monkeypatch.setattr(common, "_get_encoder_cache_hidden_dim", lambda cfg: 8) + instance_id = str(uuid.uuid4()) + regions = [] + try: + for dp_index in range(2): + parallel_config = ParallelConfig( + data_parallel_size=2, data_parallel_rank=dp_index + ) + parallel_config.reconfigure_for_independent_dp_rank() + cfg = MagicMock() + cfg.instance_id = instance_id + cfg.parallel_config = parallel_config + cfg.model_config.dtype = torch.float16 + cfg.ec_transfer_config.ec_connector_extra_config = {"ec_cpu_bytes": 64} + regions.append(common.create_ec_shared_region(cfg)) + + assert all(r._is_creator for r in regions) + assert regions[0]._mmap_path != regions[1]._mmap_path + finally: + for r in regions: + r.cleanup() + + def test_only_creator_unlinks_file_on_cleanup(): """Critical contract: if the non-creator unlinks, the creator's mmap path becomes a dangling backing file and a third opener would create a new one diff --git a/tests/v1/ec_connector/unit/test_ec_example_connector.py b/tests/v1/ec_connector/unit/test_ec_example_connector.py index dac809d1a4..a6645fccf4 100644 --- a/tests/v1/ec_connector/unit/test_ec_example_connector.py +++ b/tests/v1/ec_connector/unit/test_ec_example_connector.py @@ -16,6 +16,7 @@ from vllm.distributed.ec_transfer.ec_connector.example_connector import ( ECExampleConnectorMetadata, MMMeta, ) +from vllm.distributed.ec_transfer.ec_connector.utils import collect_ec_item_metadata from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange from vllm.v1.core.sched.output import SchedulerOutput @@ -41,6 +42,20 @@ class MockRequest: return self._token_counts[input_id] +def test_metadata_preserves_repeated_item_positions(): + request = MockRequest("request", ["A", "B", "A"], [4, 4, 4]) + resolver = Mock() + resolver.fields_for.return_value = {"image_grid_thw"} + for feature in request.mm_features: + feature.data = Mock() + feature.data.get_data.return_value = {"image_grid_thw": torch.tensor([1, 2, 2])} + items = collect_ec_item_metadata(request.mm_features, resolver) + assert items == { + "A": {"metadata": {"image_grid_thw": [1, 2, 2]}, "item_indices": [0, 2]}, + "B": {"metadata": {"image_grid_thw": [1, 2, 2]}, "item_indices": [1]}, + } + + @pytest.fixture def temp_storage(tmp_path): """Fixture providing temporary storage path.""" diff --git a/tests/v1/ec_connector/unit/test_epd_proxy_retry.py b/tests/v1/ec_connector/unit/test_epd_proxy_retry.py index 1a8c877c72..0c48dc84d6 100644 --- a/tests/v1/ec_connector/unit/test_epd_proxy_retry.py +++ b/tests/v1/ec_connector/unit/test_epd_proxy_retry.py @@ -9,6 +9,8 @@ change to them is what these tests catch. import asyncio import importlib.util from pathlib import Path +from types import SimpleNamespace +from typing import Any import httpx import msgspec @@ -83,6 +85,128 @@ class _EncoderSession: return _EncoderResponse(self._replies.pop(0)) +@pytest.mark.parametrize("no_rewrite", [False, True]) +@pytest.mark.parametrize( + "batch_size, expected_sizes", [(0, [3, 3]), (1, [1] * 6), (2, [2, 2, 1, 1])] +) +def test_image_batches_preserve_item_identity( + proxy, monkeypatch, no_rewrite, batch_size, expected_sizes +): + """Rehashed and repeated images retain metadata and per-occurrence transfers.""" + from vllm.distributed.ec_transfer.ec_connector.mooncake.scheduler import ( + ECMooncakeScheduler, + ) + + seen = [] + + class Session: + async def post(self, url, data=None, headers=None): + body = msgspec.json.decode(data) + seen.append((url, body)) + params: dict[str, dict[str, Any]] = {} + for index, item in enumerate(body["messages"][0]["content"]): + image = item["image_url"]["url"] + entry = params.setdefault( + image + "-processed", + { + "metadata": {"image_grid_thw": [1, 2, ord(image)]}, + "item_indices": [], + "peer_port": 1234, + }, + ) + entry["item_indices"].append(index) + # Response ordering must not affect the image-to-metadata mapping. + return _EncoderResponse(dict(reversed(list(params.items())))) + + monkeypatch.setattr(proxy, "encode_session", Session()) + monkeypatch.setattr(proxy, "NO_REWRITE", no_rewrite) + monkeypatch.setattr(proxy, "ENCODER_MAX_BATCH_SIZE", batch_size) + monkeypatch.setattr(proxy, "encoder_rr_idx", 0) + images = ["A", "A", "C", "D", "A", "F"] + body = { + "messages": [ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": image}} + for image in images + ], + } + ], + "mm_processor_kwargs": {"max_pixels": 262144}, + } + meta, handles = asyncio.run( + proxy.fanout_encoder_primer( + body, ["http://e0", "http://e1"], "r", "tcp://consumer:1" + ) + ) + assert [len(b["messages"][0]["content"]) for _, b in seen] == expected_sizes + assert ( + len( + { + x["transfer_id"] + for _, b in seen + for x in b["ec_transfer_params"]["ec_items"] + } + ) + == 6 + ) + for url, batch in seen: + assert batch["mm_processor_kwargs"] == body["mm_processor_kwargs"] + for item in batch["messages"][0]["content"]: + assert item["image_url"]["url"] in ( + {"A", "C"} if url.startswith("http://e0/") else {"A", "D", "F"} + ) + assert [meta[i]["image_grid_thw"][-1] for i in range(6)] == list(map(ord, images)) + assert [meta[i]["ec_mm_hash"] for i in range(6)] == [ + x + "-processed" for x in images + ] + assert handles["A-processed"]["peer_port"] == 1234 + assert "ec_transfer_params" not in body + + transfers_by_encoder: list[list[dict[str, str]]] = [[], []] + for url, batch in seen: + rank = 0 if url.startswith("http://e0/") else 1 + transfers_by_encoder[rank].extend(batch["ec_transfer_params"]["ec_items"]) + expected_transfers = [ + transfers_by_encoder[i % 2][i // 2]["transfer_id"] for i in range(len(images)) + ] + params = ( + handles + if no_rewrite + else proxy.rewrite_for_decode(body, meta)["ec_transfer_params"] + ) + request = SimpleNamespace( + ec_transfer_params=params, + mm_features=[ + SimpleNamespace(identifier=image + "-processed") for image in images + ], + ) + assert [ + ECMooncakeScheduler._request_transfer_id(request, i) for i in range(len(images)) + ] == expected_transfers + + +def test_batch_rejects_ambiguous_metadata(proxy, monkeypatch): + monkeypatch.setattr( + proxy, + "encode_session", + _EncoderSession([{"unknown": {"metadata": {"image_grid_thw": [1, 2, 2]}}}]), + ) + body = { + "messages": [ + { + "content": [ + {"type": "image_url", "image_url": {"url": image}} + for image in ["A", "B"] + ] + } + ] + } + with pytest.raises(proxy.HTTPException, match="cannot be matched"): + asyncio.run(proxy.fanout_encoder_primer(body, ["http://e0"], "r")) + + def test_a_decode_retry_does_not_inherit_the_previous_handles(proxy, monkeypatch): """The retry loop re-enters `prepare_for_decode` with the same body. @@ -187,20 +311,21 @@ async def test_http_roundtrip_preserves_payload_and_response_bytes( body = await request.json() seen[stage].append(body) if stage == "encode": - item = body["messages"][0]["content"][0] - mm_hash = item["uuid"] - if body.get("mm_processor_kwargs"): - mm_hash += "-processed" - return web.json_response( - { - "ec_transfer_params": { - mm_hash: { - "metadata": {"image_grid_thw": [[1, 2, 2]]}, - "peer_port": len(seen[stage]), - } - } - } - ) + params: dict[str, dict[str, Any]] = {} + for index, item in enumerate(body["messages"][0]["content"]): + mm_hash = item["uuid"] + if body.get("mm_processor_kwargs"): + mm_hash += "-processed" + entry = params.setdefault( + mm_hash, + { + "metadata": {"image_grid_thw": [[1, 2, 2]]}, + "peer_port": len(seen[stage]), + "item_indices": [], + }, + ) + entry["item_indices"].append(index) + return web.json_response({"ec_transfer_params": params}) if stage == "prefill": assert body["max_tokens"] == 1 and body["stream"] is False return web.json_response( @@ -263,7 +388,7 @@ async def test_http_roundtrip_preserves_payload_and_response_bytes( finally: await proxy.on_shutdown() - assert len(seen["encode"]) == 4 + assert len(seen["encode"]) == 2 assert len(seen["decode"]) == 2 for requests in seen.values(): for forwarded in requests: @@ -296,7 +421,7 @@ async def test_http_roundtrip_preserves_payload_and_response_bytes( if request_options: mm_hash += "-processed" assert set(final["ec_transfer_params"]) - {"ec_items"} == {mm_hash} - assert final["ec_transfer_params"][mm_hash]["peer_port"] in (3, 4) + assert final["ec_transfer_params"][mm_hash]["peer_port"] == 2 if prefill: assert final["kv_transfer_params"] == {"remote_block_ids": [2]} @@ -324,3 +449,62 @@ def test_pooled_connections_are_retired_before_the_server_closes_them( assert proxy.decode_session.connector._keepalive_timeout == pooled_for finally: asyncio.run(proxy.on_shutdown()) + + +@pytest.mark.parametrize( + "no_rewrite, transfer", + [(False, None), (True, None), (False, "push"), (False, "handle")], +) +def test_video_audio_fallback(proxy, monkeypatch, no_rewrite, transfer): + """Keep raw video and neighboring images, but never discard transfer handles.""" + import copy + + from vllm.distributed.ec_transfer.ec_connector.utils import collect_ec_item_metadata + + video = {"type": "video_url", "video_url": {"url": "video-with-audio"}} + images = [{"type": "image_url", "image_url": {"url": key}} for key in ("A", "B")] + body: dict[str, Any] = { + "messages": [{"content": [images[0], video, images[1]]}], + "mm_processor_kwargs": {"use_audio_in_video": True}, + } + original = copy.deepcopy(body) + # Collector indices describe two processed features from one video item. + video_metadata = collect_ec_item_metadata( + [SimpleNamespace(identifier=key, data=None) for key in ("video", "audio")], None + ) + if transfer == "handle": + video_metadata["audio"]["transfer_id"] = "reservation" + image_metadata = { + key: {"metadata": {"image_grid_thw": [1, 2, size]}, "item_indices": [i]} + for i, (key, size) in enumerate((("A", 2), ("B", 4))) + } + # Images form the first group, the singleton video the second. + monkeypatch.setattr(proxy, "ENCODER_MAX_BATCH_SIZE", 0) + monkeypatch.setattr(proxy, "NO_REWRITE", no_rewrite) + replies = _EncoderSession([image_metadata, video_metadata]) + monkeypatch.setattr(proxy, "encode_session", replies) + consumer = "tcp://consumer:1" if transfer == "push" else None + preparation = proxy.prepare_for_decode(body, "r", ["http://e0"], "", consumer) + if transfer: + with pytest.raises(proxy.HTTPException, match="cannot be matched"): + asyncio.run(preparation) + return + + prepared, _, _ = asyncio.run(preparation) + assert body == original + assert prepared["mm_processor_kwargs"] == body["mm_processor_kwargs"] + final = prepared["messages"][0]["content"] + assert final[1] == video + handles = prepared["ec_transfer_params"] + assert not video_metadata.keys() & handles.keys() + if no_rewrite: + assert final == original["messages"][0]["content"] + assert "ec_items" not in handles + else: + for position, image, key in ((0, images[0], "A"), (2, images[1], "B")): + assert final[position] == { + "type": "image_embeds", + "image_embeds": image_metadata[key]["metadata"], + "uuid": proxy.content_uuid(image), + } + assert [item["mm_hash"] for item in handles["ec_items"]] == ["A", "B"] diff --git a/tests/v1/ec_connector/unit/test_scheduler_nixl_producer.py b/tests/v1/ec_connector/unit/test_scheduler_nixl_producer.py index 156d98ae79..d7ef19b5f4 100644 --- a/tests/v1/ec_connector/unit/test_scheduler_nixl_producer.py +++ b/tests/v1/ec_connector/unit/test_scheduler_nixl_producer.py @@ -67,6 +67,7 @@ def test_request_finished_producer_emits_params(monkeypatch): assert params == { "h1": { "metadata": {}, + "item_indices": [0], "peer_host": "1.2.3.4", "peer_port": 5601, "size_bytes": 2 * 32 * 2, @@ -91,6 +92,7 @@ def test_request_finished_announces_not_ready_entry(monkeypatch): assert params == { "h1": { "metadata": {}, + "item_indices": [0], "peer_host": "1.2.3.4", "peer_port": 5601, "size_bytes": 2 * 32 * 2, @@ -110,5 +112,5 @@ def test_request_finished_skips_unallocated_entry(monkeypatch): assert delay is False # The item's placeholder metadata is still reported even though there's # no cache entry to transfer (empty "metadata": no fields, no transfer). - assert params == {"h1": {"metadata": {}}} + assert params == {"h1": {"metadata": {}, "item_indices": [0]}} s.shutdown() diff --git a/tests/v1/engine/test_admission_control.py b/tests/v1/engine/test_admission_control.py index 219850f5d4..15be166c8c 100644 --- a/tests/v1/engine/test_admission_control.py +++ b/tests/v1/engine/test_admission_control.py @@ -15,6 +15,7 @@ These tests cover: import argparse import asyncio import multiprocessing +from collections.abc import Awaitable, Callable from http import HTTPStatus from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock @@ -35,6 +36,7 @@ from vllm.exceptions import ( from vllm.pooling_params import PoolingParams from vllm.sampling_params import SamplingParams from vllm.utils.argparse_utils import human_readable_int +from vllm.v1.engine import EngineCoreRequest from vllm.v1.engine.admission_control import SharedAdmissionStats from vllm.v1.engine.async_llm import AsyncLLM from vllm.v1.engine.output_processor import OutputProcessor @@ -78,6 +80,45 @@ def _make_output_processor(**request_states) -> OutputProcessor: return op +def _make_engine_request(request_id: str, n: int) -> EngineCoreRequest: + return EngineCoreRequest( + request_id=request_id, + prompt_token_ids=[1], + mm_features=None, + sampling_params=SamplingParams(n=n), + pooling_params=None, + arrival_time=0, + lora_request=None, + cache_salt=None, + data_parallel_rank=None, + ) + + +def _make_request_test_llm( + max_num_queued_reqs: int, + add_request_async: Callable[[EngineCoreRequest], Awaitable[None]], +) -> AsyncLLM: + llm = _make_async_llm(max_num_queued_reqs=max_num_queued_reqs) + llm.output_processor = OutputProcessor(None, log_stats=False) + llm.engine_core = SimpleNamespace( + resources=SimpleNamespace(engine_dead=False), + add_request_async=AsyncMock(side_effect=add_request_async), + abort_requests_async=AsyncMock(), + shutdown=MagicMock(), + ) + llm.vllm_config = SimpleNamespace( + cache_config=SimpleNamespace(kv_sharing_fast_prefill=False) + ) + llm.output_handler = None + llm.log_requests = False + llm._run_output_handler = MagicMock() + llm.input_processor = MagicMock() + llm.input_processor.assign_request_id.side_effect = lambda request: setattr( + request, "external_req_id", request.request_id + ) + return llm + + def _make_scheduler_config(**kwargs) -> SchedulerConfig: return SchedulerConfig( runner_type="generate", @@ -340,6 +381,55 @@ async def test_concurrent_single_request_admission_respects_limit(): assert len(request_states) == 1 +@pytest.mark.asyncio +@pytest.mark.parametrize("first_n", [1, 3]) +async def test_parallel_admission_is_all_or_nothing(first_n: int): + """An n>1 request reserves either all its capacity or none of it.""" + + async def add_request_async(_request): + await asyncio.sleep(0) + + llm = _make_request_test_llm(3, add_request_async) + second_n = 3 if first_n == 1 else 1 + first = _make_engine_request("first", first_n) + second = _make_engine_request("second", second_n) + results = await asyncio.gather( + llm.add_request("first", first, first.params), + llm.add_request("second", second, second.params), + return_exceptions=True, + ) + + assert not isinstance(results[0], Exception) + assert isinstance(results[1], QueueOverflowError) + assert llm.output_processor.get_num_unfinished_requests() == first_n + assert llm.engine_core.add_request_async.await_count == first_n + + +@pytest.mark.asyncio +async def test_parallel_admission_cancellation_cleans_up_all_children(): + """Cancellation during core submission must release every reserved slot.""" + first_submission_started = asyncio.Event() + + async def add_request_async(_request): + first_submission_started.set() + await asyncio.Event().wait() + + llm = _make_request_test_llm(3, add_request_async) + request = _make_engine_request("parallel", 3) + output = llm.generate(request, request.params, request.request_id) + generate_task = asyncio.create_task(anext(output)) + + await first_submission_started.wait() + generate_task.cancel() + with pytest.raises(asyncio.CancelledError): + await generate_task + + assert llm.engine_core.add_request_async.await_count == 1 + assert llm.output_processor.get_num_unfinished_requests() == 0 + aborted_ids = llm.engine_core.abort_requests_async.await_args.args[0] + assert set(aborted_ids) == {"0_parallel", "1_parallel", "2_parallel"} + + # --------------------------------------------------------------------------- # create_error_response integration # --------------------------------------------------------------------------- diff --git a/tests/v1/engine/test_engine_core.py b/tests/v1/engine/test_engine_core.py index ba1e9a4d20..e16e0550c2 100644 --- a/tests/v1/engine/test_engine_core.py +++ b/tests/v1/engine/test_engine_core.py @@ -23,6 +23,7 @@ from vllm.config import ( from vllm.engine.arg_utils import EngineArgs from vllm.platforms import current_platform from vllm.utils.torch_utils import set_default_torch_num_threads +from vllm.v1.core.sched.interface import PauseState from vllm.v1.engine import EngineCoreRequest from vllm.v1.engine.core import DPEngineCoreProc, EngineCore, EngineCoreProc from vllm.v1.executor.abstract import Executor @@ -667,6 +668,29 @@ def _pausable_engine_core_proc() -> EngineCoreProc: return core +@pytest.mark.parametrize( + "pause_state,has_requests,has_batches", + [ + pytest.param(PauseState.UNPAUSED, False, False, id="not-paused"), + pytest.param(PauseState.PAUSED_ALL, True, False, id="pending-requests"), + pytest.param(PauseState.PAUSED_ALL, False, True, id="pending-batches"), + ], +) +def test_kv_cache_release_rejects_unsafe_state(pause_state, has_requests, has_batches): + """Reject release before touching caches or memory if work can still use KV.""" + core = _pausable_engine_core_proc() + core.scheduler.pause_state = pause_state + core.scheduler.has_requests.return_value = has_requests + core.batch_queue = [object()] if has_batches else None + core._reset_caches = MagicMock() + + with pytest.raises(RuntimeError, match="requires a completed pause"): + core.release_kv_cache_memory() + + core._reset_caches.assert_not_called() + core.model_executor.discard.assert_not_called() + + @pytest.mark.parametrize("deferred", [False, True]) def test_pause_synchronizes_device_before_cache_reset(deferred: bool): """A resolved pause promises an idle device: the barrier must run before diff --git a/tests/v1/engine/test_engine_core_client.py b/tests/v1/engine/test_engine_core_client.py index a153b23b10..15dec72a41 100644 --- a/tests/v1/engine/test_engine_core_client.py +++ b/tests/v1/engine/test_engine_core_client.py @@ -214,6 +214,7 @@ def _make_dplb_client(num_engines: int = 3, client_count: int = 1) -> DPLBAsyncM client.core_engines = [bytes([i, 0]) for i in range(num_engines)] client.lb_engines = [[0, 0, 0.0] for _ in range(num_engines)] client.eng_start_index = 0 + client._kv_event_sources = {} return client @@ -1490,3 +1491,57 @@ def test_ready_response_queries_executor_with_weight_transfer(): executor.supports_draft_weight_updates.assert_called_once() assert response.supports_draft_weight_updates is True + + +def test_apply_ready_response_retains_kv_event_sources(): + """Each engine's resolved publisher config is kept by DP rank; engines + without KV events contribute nothing.""" + import msgspec + + from vllm.config.kv_events import KVEventsConfig + + client = object.__new__(MPClient) + client._effective_attention_block_sizes = set() + client._kv_event_sources = {} + client.vllm_config = SimpleNamespace( + cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0), + model_config=SimpleNamespace(max_model_len=8192), + ) + + def ready(rank: int, config: KVEventsConfig | None) -> bytes: + return msgspec.msgpack.encode( + EngineCoreReadyResponse( + max_model_len=8192, + num_gpu_blocks=100, + block_size=16, + dp_stats_address=None, + dtype="bfloat16", + vllm_version="test", + world_size=2, + data_parallel_size=2, + tensor_parallel_size=1, + pipeline_parallel_size=1, + decode_context_parallel_size=1, + data_parallel_rank=rank, + max_num_seqs=256, + max_num_batched_tokens=8192, + instance_id="test-instance", + supports_lora=False, + max_loras=0, + kv_events_config=config, + ) + ) + + rank1 = KVEventsConfig( + enable_kv_cache_events=True, publisher="zmq", endpoint="tcp://10.0.0.2:41233" + ) + client._apply_ready_response(ready(1, rank1)) + client._apply_ready_response(ready(0, None)) + assert client.get_kv_event_sources() == {1: rank1} + + # A rank that comes back (elastic re-add) reports its new port. + rank1_again = KVEventsConfig( + enable_kv_cache_events=True, publisher="zmq", endpoint="tcp://10.0.0.2:47001" + ) + client._apply_ready_response(ready(1, rank1_again)) + assert client.get_kv_event_sources() == {1: rank1_again} diff --git a/tests/v1/engine/test_input_processor_trace_replay.py b/tests/v1/engine/test_input_processor_trace_replay.py index 894293bc3d..24302f6de6 100644 --- a/tests/v1/engine/test_input_processor_trace_replay.py +++ b/tests/v1/engine/test_input_processor_trace_replay.py @@ -85,6 +85,7 @@ def _validate(enable_trace_replay: bool) -> None: speculative_config=None, structured_outputs_config=None, tokenizer=None, + validate_logits_processors_params=lambda params: None, ) params = SamplingParams(trace_decode_token_ids=[1, 2, 3]) with patch.object(SamplingParams, "verify"): diff --git a/tests/v1/engine/test_kv_cache_group_metadata.py b/tests/v1/engine/test_kv_cache_group_metadata.py new file mode 100644 index 0000000000..1b2dedec4d --- /dev/null +++ b/tests/v1/engine/test_kv_cache_group_metadata.py @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace + +import pytest +import torch + +from vllm.v1.core.kv_cache_manager import KVCacheManager +from vllm.v1.engine.core import EngineCore +from vllm.v1.kv_cache_interface import ( + KVCacheConfig, + KVCacheGroupSpec, + MambaSpec, + MLAAttentionSpec, +) + +pytestmark = pytest.mark.cpu_test + +BASE_BLOCK_SIZE = 1536 +DCP_WORLD_SIZE = 8 + + +def _make_engine_core_with_dcp_manager() -> EngineCore: + mla_spec = MLAAttentionSpec( + block_size=BASE_BLOCK_SIZE, + num_kv_heads=1, + head_size=576, + dtype=torch.bfloat16, + ) + mamba_spec = MambaSpec( + block_size=BASE_BLOCK_SIZE, + shapes=((1, 1),), + dtypes=(torch.float32,), + mamba_cache_mode="align", + ) + config = KVCacheConfig( + num_blocks=32, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec(["mla"], mla_spec), + KVCacheGroupSpec(["mamba"], mamba_spec), + ], + ) + manager = KVCacheManager( + config, + max_model_len=BASE_BLOCK_SIZE * DCP_WORLD_SIZE * 2, + scheduler_block_size=BASE_BLOCK_SIZE * DCP_WORLD_SIZE, + hash_block_size=BASE_BLOCK_SIZE, + dcp_world_size=DCP_WORLD_SIZE, + ) + scheduler = SimpleNamespace(kv_cache_config=config, kv_cache_manager=manager) + engine_core = object.__new__(EngineCore) + engine_core.scheduler = scheduler + return engine_core + + +def test_kv_cache_group_metadata_uses_effective_dcp_block_sizes() -> None: + engine_core = _make_engine_core_with_dcp_manager() + + metadata = engine_core.get_kv_cache_group_metadata() + + # DCP scales attention blocks but does not scale replicated Mamba state. + assert [item["block_size"] for item in metadata] == [12288, 1536] + assert [item["kind"] for item in metadata] == ["mla_attention", "mamba"] + + managers = engine_core.scheduler.kv_cache_manager.coordinator.single_type_managers + assert [item["block_size"] for item in metadata] == [ + manager.block_size for manager in managers + ] diff --git a/tests/v1/engine/test_output_processor.py b/tests/v1/engine/test_output_processor.py index 9df43088e6..69128f55b7 100644 --- a/tests/v1/engine/test_output_processor.py +++ b/tests/v1/engine/test_output_processor.py @@ -177,7 +177,8 @@ def test_incremental_detokenization( def test_remote_prefill_cached_tokens_override(do_remote_prefill: bool): """P/D disaggregation: num_cached_tokens should report the P worker's cache hits (passed via kv_transfer_params) instead of the local count, - which sees the KVs pulled from the remote prefill as a ~100% hit.""" + which sees the KVs pulled from the remote prefill as a ~100% hit. + """ output_processor = OutputProcessor(tokenizer=None, log_stats=False) prompt_tokens = [1, 2, 3, 4, 5, 6, 7, 8] diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_config.py b/tests/v1/kv_connector/unit/offloading_connector/test_config.py index acf55972df..fe0d258462 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_config.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_config.py @@ -19,6 +19,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import ( from vllm.platforms import current_platform from vllm.v1.core.kv_cache_utils import generate_scheduler_kv_cache_config from vllm.v1.kv_cache_interface import ( + CircularBufferSpec, FullAttentionSpec, HiddenStateCacheSpec, KVCacheConfig, @@ -340,6 +341,38 @@ def test_worker_kv_bytes_preserves_tensor_layout(packed: bool): assert offloading_config.cache.blocks_per_chunk == 2 +def test_offloading_skips_scratch_group(): + kv_cache_config = _make_mamba_hybrid_kv_cache_config() + kv_cache_config.kv_cache_groups.insert( + 1, + KVCacheGroupSpec( + ["scratch"], + CircularBufferSpec( + block_size=4, num_kv_heads=1, head_size=128, dtype=torch.float32 + ), + ), + ) + page_size = kv_cache_config.kv_cache_groups[0].kv_cache_spec.page_size_bytes + kv_cache_config.kv_cache_tensors = [ + KVCacheTensor( + size=page_size * kv_cache_config.num_blocks, + layers=["full_layer", "scratch", "mamba_layer"], + layer_stride=0, + block_stride=page_size, + ) + ] + config = _make_vllm_config() + + offloading_config = build_offloading_config(config, kv_cache_config) + scheduler_config = SchedulerOffloadConfig.from_spec( + MockOffloadingSpec(offloading_config), config, kv_cache_config + ) + + assert [group.group_id for group in offloading_config.groups] == [0, 2] + assert [group.group_idx for group in scheduler_config.kv_group_configs] == [0, 2] + assert offloading_config.worker_kv_bytes_per_block == page_size + + def test_hisparse_offloads_only_indexer_group(): source = KVCacheGroupSpec( ["source"], diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_events.py b/tests/v1/kv_connector/unit/offloading_connector/test_events.py index da0ccb02b1..c409fd125d 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_events.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_events.py @@ -682,3 +682,92 @@ def test_tiering_accepts_self_describing_kv_events(): assert spec.kv_events_config.enable_kv_cache_events assert spec.kv_events_config.self_describing_kv_events assert tracker.self_describing_enabled + + +def _build_tiering_spec(secondary_tiers, *, top_level_backpressure=None): + vllm_config = create_vllm_config( + block_size=4, + max_num_batched_tokens=16, + disable_hybrid_kv_cache_manager=False, + ) + extra_config = { + "spec_name": "TieringOffloadingSpec", + "cpu_bytes_to_use": 1 << 20, + "secondary_tiers": secondary_tiers, + } + if top_level_backpressure is not None: + extra_config["backpressure"] = top_level_backpressure + vllm_config.kv_transfer_config = KVTransferConfig( + kv_connector="OffloadingConnector", + kv_role="kv_both", + kv_connector_extra_config=extra_config, + ) + kv_cache_config = KVCacheConfig( + num_blocks=0, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec( + ["layer"], + FullAttentionSpec( + block_size=4, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ), + ) + ], + ) + return TieringOffloadingSpec(build_offloading_config(vllm_config, kv_cache_config)) + + +def test_partial_tier_backpressure_inherits_top_level_defaults(): + """A partial tier override still inherits missing fields. + + Field-by-field merge means an override that sets only ``high_water_s`` + keeps ``backpressure_cls`` (and any other field) from the top-level + default, so the resolved dict reaching the factory is complete. + """ + spec = _build_tiering_spec( + [{"type": "example", "backpressure": {"high_water_s": 0.1}}], + top_level_backpressure={ + "backpressure_cls": "EMABackpressureDetector", + "high_water_s": 0.5, + "low_water_s": 0.05, + }, + ) + + resolved = spec.secondary_tier_configs[0]["backpressure"] + # Tier override wins for the field it sets. + assert resolved["high_water_s"] == 0.1 + # Missing fields fall back to the top-level default. + assert resolved["backpressure_cls"] == "EMABackpressureDetector" + assert resolved["low_water_s"] == 0.05 + + +def test_tier_override_wins_over_top_level(): + """Precedence is tier override > top-level default, field-by-field.""" + spec = _build_tiering_spec( + [ + { + "type": "example", + "backpressure": {"high_water_s": 0.1, "low_water_s": 0.02}, + } + ], + top_level_backpressure={ + "backpressure_cls": "EMABackpressureDetector", + "high_water_s": 0.5, + "low_water_s": 0.05, + }, + ) + + resolved = spec.secondary_tier_configs[0]["backpressure"] + # Tier override wins for the fields it sets. + assert resolved["high_water_s"] == 0.1 + assert resolved["low_water_s"] == 0.02 + # Fields only in top-level still fill in. + assert resolved["backpressure_cls"] == "EMABackpressureDetector" + + +def test_tier_without_backpressure_stays_unconfigured(): + spec = _build_tiering_spec([{"type": "example"}]) + assert "backpressure" not in spec.secondary_tier_configs[0] diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_worker.py b/tests/v1/kv_connector/unit/offloading_connector/test_worker.py index 64f481d02e..32bd4f3eca 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_worker.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_worker.py @@ -13,6 +13,7 @@ from vllm.platforms import current_platform from vllm.utils.torch_utils import get_dtype_size from vllm.v1.attention.backends.registry import AttentionBackendEnum from vllm.v1.kv_cache_interface import ( + CircularBufferSpec, FullAttentionSpec, KVCacheConfig, KVCacheGroupSpec, @@ -520,6 +521,47 @@ def test_register_kv_caches(backend): assert actual.mapping is not None +def test_register_packed_kv_caches_skips_scratch_group(): + attn_spec = FullAttentionSpec( + block_size=BLOCK_SIZE, + num_kv_heads=NUM_KV_HEADS, + head_size=HEAD_SIZE, + dtype=DTYPE, + ) + page = attn_spec.page_size_bytes + layers = ["layer0", "layer1"] + kv_cache_config = KVCacheConfig( + num_blocks=NUM_BLOCKS, + kv_cache_tensors=[ + KVCacheTensor( + size=2 * page * NUM_BLOCKS, + layers=layers, + layer_stride=page, + block_stride=2 * page, + ) + ], + kv_cache_groups=[ + KVCacheGroupSpec(layers, attn_spec), + KVCacheGroupSpec( + ["scratch"], + CircularBufferSpec( + block_size=4, num_kv_heads=1, head_size=1, dtype=DTYPE + ), + ), + ], + ) + kv_caches = _allocate_kv_caches( + kv_cache_config, [], device=torch.device(f"{DEVICE_TYPE}:0") + ) + worker, spec = _make_worker(kv_cache_config) + worker.register_kv_caches(kv_caches) + + canonical = spec.get_worker.call_args[0][0] + assert len(canonical.tensors) == 1 + assert canonical.tensors[0].tensor.shape == (NUM_BLOCKS, 2 * page) + assert canonical.group_data_refs == [[CanonicalKVCacheRef(0, 2 * page)]] + + @pytest.mark.parametrize("backend", ATTN_BACKENDS) def test_register_kv_caches_uniform_type(backend): """Test register_kv_caches with UniformTypeKVCacheSpecs. diff --git a/tests/v1/kv_connector/unit/test_bidirectional_kv_transfer.py b/tests/v1/kv_connector/unit/test_bidirectional_kv_transfer.py index 78a6d930f9..3d539f97c8 100644 --- a/tests/v1/kv_connector/unit/test_bidirectional_kv_transfer.py +++ b/tests/v1/kv_connector/unit/test_bidirectional_kv_transfer.py @@ -969,6 +969,9 @@ def test_turn2_deadline_gate(dist_init, offset, expiry_delta, expect_declined): assert worker.xfer_stats.data["num_kv_expired_reqs"] == ( [1] if expect_declined else [] ) + # KV expiry must not pollute the transport failure counters. + assert worker.xfer_stats.data["num_failed_transfers"] == [] + assert worker.xfer_stats.data["num_failed_handshakes"] == [] _, done_recving = connector.get_finished(finished_req_ids=set()) assert "req" in done_recving diff --git a/tests/v1/kv_connector/unit/test_hidden_states_connector.py b/tests/v1/kv_connector/unit/test_hidden_states_connector.py index 8427833b12..735b042259 100644 --- a/tests/v1/kv_connector/unit/test_hidden_states_connector.py +++ b/tests/v1/kv_connector/unit/test_hidden_states_connector.py @@ -169,3 +169,62 @@ def test_hidden_state_group_isolated_from_packed_mixed_page_groups(): assert ( ExampleHiddenStatesConnector._find_cache_kv_group_id(cfg) == hidden_group_ids[0] ) + + +# ---- abort-path robustness -------------------------------------------------- + + +def _bare_connector() -> ExampleHiddenStatesConnector: + """Instance with scheduler-side state but bypassing ``__init__`` (no engine).""" + conn = ExampleHiddenStatesConnector.__new__(ExampleHiddenStatesConnector) + conn._request_filenames = {} + conn._pending_saves = {} + conn._lock_fds = {} + conn._cache_kv_group_id = 1 + conn._connector_metadata = None + conn._req_copy_events = {} + conn._accumulated_finished_req_ids = set() + return conn + + +def test_get_finished_count_is_one(): + # Only TP rank 0 writes, so KVOutputAggregator must expect a single + # finished_sending notification per request (not the TP world size). + assert _bare_connector().get_finished_count() == 1 + + +def test_request_finished_is_noop_for_never_scheduled_request(): + # A request aborted while still queued never reaches build_connector_meta, + # so no filename was recorded. request_finished must not raise KeyError. + conn = _bare_connector() + request = SimpleNamespace(request_id="cmpl-aborted", kv_transfer_params=None) + assert conn.request_finished(request, []) == (False, None) + + +def test_request_finished_all_groups_handles_missing_group(): + # Guard against indexing a nonexistent per-group block table. + conn = _bare_connector() + conn._cache_kv_group_id = 2 + request = SimpleNamespace(request_id="cmpl-aborted", kv_transfer_params=None) + assert conn.request_finished_all_groups(request, ([], [])) == (False, None) + + +def test_get_finished_does_not_report_untracked_request(): + # A never-scheduled aborted request has no copy event. get_finished must not + # report it as done_sending, or the scheduler asserts it is still tracked. + conn = _bare_connector() + assert conn.get_finished({"cmpl-aborted"}) == (None, None) + assert conn._accumulated_finished_req_ids == set() + + +def test_get_finished_reports_tracked_completed_request(): + conn = _bare_connector() + + class _DoneEvent: + def query(self) -> bool: + return True + + conn._req_copy_events["cmpl-done"] = _DoneEvent() + done_sending, done_recving = conn.get_finished({"cmpl-done"}) + assert done_sending == {"cmpl-done"} + assert done_recving is None diff --git a/tests/v1/kv_connector/unit/test_kv_load_failure_recovery.py b/tests/v1/kv_connector/unit/test_kv_load_failure_recovery.py index a54a023a13..a8bc2d0e12 100644 --- a/tests/v1/kv_connector/unit/test_kv_load_failure_recovery.py +++ b/tests/v1/kv_connector/unit/test_kv_load_failure_recovery.py @@ -35,6 +35,39 @@ def scheduler(): return create_scheduler(vllm_config) +@pytest.mark.parametrize("policy", ["fail", "recompute"]) +def test_failed_receive_completion_honors_failure_policy(policy): + """Failed receives still need completion to free blocks or resume computation.""" + scheduler = create_scheduler(create_vllm_config(kv_load_failure_policy=policy)) + request = create_request(num_tokens=3 * scheduler.block_size) + scheduler.add_request(request) + scheduler.connector = Mock() + scheduler.connector.get_num_new_matched_tokens.side_effect = [ + (2 * scheduler.block_size, True), + (0, False), + ] + scheduler.connector.request_finished.return_value = (False, None) + scheduler.connector.take_events.return_value = () + block_pool = scheduler.kv_cache_manager.block_pool + free_blocks_before = block_pool.get_num_free_blocks() + scheduler_output = scheduler.schedule() + assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS + + output = create_model_runner_output([], finished_recving={request.request_id}) + assert output.kv_connector_output is not None + output.kv_connector_output.failed_recving = {request.request_id} + scheduler.update_from_output(scheduler_output, output) + + if policy == "fail": + assert request.status == RequestStatus.FINISHED_ERROR + assert request.request_id not in scheduler.requests + assert block_pool.get_num_free_blocks() == free_blocks_before + else: + assert request.num_computed_tokens == 0 + next_output = scheduler.schedule() + assert request.request_id in next_output.num_scheduled_tokens + + @pytest.mark.parametrize( "num_prompt_blocks,num_external_computed_blocks,invalid_block_idxs", [ diff --git a/tests/v1/kv_connector/unit/test_multi_connector.py b/tests/v1/kv_connector/unit/test_multi_connector.py index f6d8ed97d8..1a85865560 100644 --- a/tests/v1/kv_connector/unit/test_multi_connector.py +++ b/tests/v1/kv_connector/unit/test_multi_connector.py @@ -567,6 +567,8 @@ class TestMultiConnectorStats: "num_descriptors": [10, 20], "num_failed_transfers": [], "num_failed_notifications": [], + "num_failed_handshakes": [], + "num_kv_expired_reqs": [], } } @@ -590,6 +592,8 @@ class TestMultiConnectorStats: "num_descriptors": [10], "num_failed_transfers": [], "num_failed_notifications": [], + "num_failed_handshakes": [], + "num_kv_expired_reqs": [], }, "MockConnector": {"mock_field": [1, 2, 3]}, } @@ -619,6 +623,8 @@ class TestMultiConnectorStats: "num_descriptors": [10], "num_failed_transfers": [], "num_failed_notifications": [], + "num_failed_handshakes": [], + "num_kv_expired_reqs": [], }, } @@ -638,6 +644,8 @@ class TestMultiConnectorStats: "num_descriptors": [10], "num_failed_transfers": [], "num_failed_notifications": [], + "num_failed_handshakes": [], + "num_kv_expired_reqs": [], } ) mock_stats = MockConnectorStats(data={"mock_field": [1, 2, 3]}) @@ -667,6 +675,8 @@ class TestMultiConnectorStats: "num_descriptors": [10], "num_failed_transfers": [], "num_failed_notifications": [], + "num_failed_handshakes": [], + "num_kv_expired_reqs": [], } ) @@ -698,6 +708,8 @@ class TestMultiConnectorStats: "num_descriptors": [10], "num_failed_transfers": [], "num_failed_notifications": [], + "num_failed_handshakes": [], + "num_kv_expired_reqs": [], }, "ExampleConnector": {"some_field": [1, 2, 3]}, } @@ -735,6 +747,8 @@ class TestMultiConnectorStats: "num_descriptors": [10], "num_failed_transfers": [], "num_failed_notifications": [], + "num_failed_handshakes": [], + "num_kv_expired_reqs": [], } ) } @@ -750,6 +764,8 @@ class TestMultiConnectorStats: "num_descriptors": [20], "num_failed_transfers": [], "num_failed_notifications": [], + "num_failed_handshakes": [], + "num_kv_expired_reqs": [], } ) } @@ -781,6 +797,8 @@ class TestMultiConnectorStats: "num_descriptors": [10], "num_failed_transfers": [], "num_failed_notifications": [], + "num_failed_handshakes": [], + "num_kv_expired_reqs": [], } ) } @@ -807,6 +825,8 @@ class TestMultiConnectorStats: "num_descriptors": [10, 20], "num_failed_transfers": [], "num_failed_notifications": [], + "num_failed_handshakes": [], + "num_kv_expired_reqs": [], } ) } @@ -833,6 +853,8 @@ class TestMultiConnectorStats: "num_descriptors": [10, 20], "num_failed_transfers": [], "num_failed_notifications": [], + "num_failed_handshakes": [], + "num_kv_expired_reqs": [], } ) } @@ -862,6 +884,8 @@ class TestMultiConnectorStats: "num_descriptors": [10, 20], "num_failed_transfers": [], "num_failed_notifications": [], + "num_failed_handshakes": [], + "num_kv_expired_reqs": [], } ) } diff --git a/tests/v1/kv_connector/unit/test_nixl_connector.py b/tests/v1/kv_connector/unit/test_nixl_connector.py index 2185ba62d2..39f8f88d5c 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector.py @@ -10,6 +10,7 @@ import textwrap import time import uuid from collections import defaultdict +from concurrent.futures import Future from types import SimpleNamespace from typing import Any, cast from unittest.mock import MagicMock, patch @@ -350,7 +351,8 @@ def test_prefill_exports_cached_tokens_in_kv_transfer_params(): """The P worker reports its own prefix-cache hits in the returned kv_transfer_params so the D worker can surface them in prompt_tokens_details instead of the ~100% local hit it measures - when pulling the KVs from the remote.""" + when pulling the KVs from the remote. + """ vllm_config = create_vllm_config() scheduler = create_scheduler(vllm_config) @@ -1681,6 +1683,83 @@ def test_kv_connector_stats_aggregation(): assert all(not isinstance(v, np.generic) for v in cli_stats.values()) +def test_kv_connector_stats_failure_grouping(): + """Transfer, handshake and notification failures are reported as one + transport-failure count, while KV expiry is reported separately: the + former are sporadic lower-transport-layer events, the latter an + autoscaler signal.""" + stats = NixlKVConnectorStats() + assert stats.is_empty() + + stats.record_failed_transfer() + stats.record_failed_handshake() + stats.record_failed_notification() + stats.record_kv_expired_req() + assert not stats.is_empty() + + # No successful transfers: latency stats are zero but the failure + # counts still surface. + reduced = stats.reduce() + assert reduced["Num successful transfers"] == 0 + assert reduced["Num failed transfers"] == 3 + assert reduced["Num KV expired reqs"] == 1 + + +def test_nixl_prom_metrics_group_handshake_with_transfer_failures(): + """vllm:nixl_num_failed_transfers counts handshake and notification + failures too, while vllm:nixl_num_kv_expired_reqs stays a separate + counter.""" + from prometheus_client import CollectorRegistry, Counter, Gauge, Histogram + + from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import ( + NixlPromMetrics, + ) + + registry = CollectorRegistry() + + class RegistryGauge(Gauge): + def __init__(self, *args, **kwargs): + super().__init__(*args, registry=registry, **kwargs) + + class RegistryCounter(Counter): + def __init__(self, *args, **kwargs): + super().__init__(*args, registry=registry, **kwargs) + + class RegistryHistogram(Histogram): + def __init__(self, *args, **kwargs): + super().__init__(*args, registry=registry, **kwargs) + + vllm_config = create_vllm_config() + metric_types = { + Gauge: RegistryGauge, + Counter: RegistryCounter, + Histogram: RegistryHistogram, + } + prom = NixlPromMetrics( + vllm_config, + metric_types, + labelnames=["engine"], + per_engine_labelvalues={0: ["engine-0"]}, + ) + + stats = NixlKVConnectorStats() + stats.record_failed_transfer() + stats.record_failed_handshake() + stats.record_failed_notification() + stats.record_kv_expired_req() + prom.observe(stats.data, engine_idx=0) + + def counter_value(name: str) -> float: + for metric in registry.collect(): + for sample in metric.samples: + if sample.name == name: + return sample.value + raise AssertionError(f"metric {name} not found in registry") + + assert counter_value("vllm:nixl_num_failed_transfers_total") == 3.0 + assert counter_value("vllm:nixl_num_kv_expired_reqs_total") == 1.0 + + def test_multi_kv_connector_stats_aggregation(): """Test MultiKVConnectorStats aggregation across TP ranks using KVOutputAggregator (used by MultiprocExecutor). @@ -2821,6 +2900,52 @@ def test_empty_recv_is_reported_only_when_awaited( assert (request_id in done_recving) is awaiting_kvs +@pytest.mark.parametrize("is_hma", [False, True]) +@pytest.mark.parametrize( + "local_block_ids,awaiting_kvs", + [((), False), (([],), False), (([],), True), (([1, 2, 3],), True)], +) +def test_handshake_failure_reports_only_awaited_recvs( + recv_worker, is_hma, local_block_ids, awaiting_kvs +): + """A failed cleanup handshake must not complete an already-aborted request.""" + worker = recv_worker + worker._is_hma_required = is_hma + worker._recving_metadata.clear() + worker._remote_agents = {} + worker._handshake_lock = contextlib.nullcontext() + worker._ready_requests = queue.Queue() + worker._reqs_to_process = set() + worker.pcp_rank = 0 + metadata = NixlConnectorMetadata() + metadata.add_new_req_to_recv( + request_id="request", + local_block_ids=local_block_ids, + kv_transfer_params={ + "remote_block_ids": ([4, 5, 6],), + "remote_engine_id": "prefill", + "remote_request_id": "prefill-request", + "remote_host": "localhost", + "remote_port": 1234, + }, + awaiting_kvs=awaiting_kvs, + ) + handshake = Future[None]() + handshake.set_exception(RuntimeError("handshake failed")) + with patch.object(worker, "_ensure_handshake", return_value=handshake): + worker.start_load_kv(metadata) + + results = worker.get_transfer_results() + expected = {"request"} if awaiting_kvs else set() + assert results.finished_recving == results.failed_recving == expected + assert worker.get_block_ids_with_load_errors() == ( + set(local_block_ids[0]) if awaiting_kvs and not is_hma else set() + ) + assert not worker._recving_metadata + assert not worker._recv_failures + assert not worker._pending_recv_notifs + + @patch( "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", FailingNixlWrapper, @@ -3028,6 +3153,11 @@ def test_handshake_failure_returns_finished(default_vllm_config, dist_init): assert request_id in done_recving assert connector.get_block_ids_with_load_errors() == {1, 2, 3} + # Handshake failures are recorded as transport failures, separately + # from KV expiry. + assert connector.connector_worker.xfer_stats.data["num_failed_handshakes"] + assert connector.connector_worker.xfer_stats.data["num_failed_transfers"] == [] + @patch( "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", diff --git a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py index f9cacabdd4..a4dfc168f3 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py @@ -1505,6 +1505,7 @@ def test_failed_load_rezeroes_unwritten_skipped_blocks(): scheduler.kv_cache_manager.cache_blocks = MagicMock() scheduler.failed_recving_kv_req_ids = {"req-1"} scheduler.finished_recving_kv_req_ids = {"req-1"} + scheduler.prefix_replay_tokens = 0 request = MagicMock() request.request_id = "req-1" diff --git a/tests/v1/kv_connector/unit/test_nixl_push_connector.py b/tests/v1/kv_connector/unit/test_nixl_push_connector.py index 7ce84bf685..e35ead0015 100644 --- a/tests/v1/kv_connector/unit/test_nixl_push_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_push_connector.py @@ -47,6 +47,9 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import ( NixlPushConnectorWorker, ) +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import ( + NixlKVConnectorStats, +) from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import TPMapping from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import ( get_base_request_id, @@ -360,6 +363,7 @@ class _StubWriterWorker(NixlPushConnectorWorker): w._recv_failures = set() w._recving_transfers = defaultdict(list) w._is_hma_required = False + w.xfer_stats = NixlKVConnectorStats() w._reqs_to_process = set() w._reqs_to_send = {} w.consumer_notification_counts_by_req = defaultdict(int) @@ -730,6 +734,10 @@ def test_do_start_push_kv_drops_request_on_handshake_failure(): assert xfer_calls == [] assert len(failures) == 1 assert failures[0]["failure_type"] == "push_handshake_failed" + # The handshake metric is counted once by _ensure_handshake's own + # callback (stubbed out here); the push side must not double-record it. + assert w.xfer_stats.data["num_failed_handshakes"] == [] + assert w.xfer_stats.data["num_failed_transfers"] == [] def test_writer_loop_drains_deferred_push_inbox(): diff --git a/tests/v1/kv_offload/cpu/test_canonical_layout.py b/tests/v1/kv_offload/cpu/test_canonical_layout.py index 4594e1eb09..749bd6fac5 100644 --- a/tests/v1/kv_offload/cpu/test_canonical_layout.py +++ b/tests/v1/kv_offload/cpu/test_canonical_layout.py @@ -241,6 +241,7 @@ def test_cross_topology_roundtrip(writer_tp: int, reader_tp: int): # The Triton load path dereferences CPU pointers on the GPU, which is # only legal on pinned memory; production pins via CPUOffloadingWorker pin_mmap_region(region) + assert region.is_pinned return region.create_next_canonical_view(_CANONICAL_PAGE) try: diff --git a/tests/v1/kv_offload/cpu/test_gpu_worker.py b/tests/v1/kv_offload/cpu/test_gpu_worker.py index 5306d05571..731d61a7e9 100644 --- a/tests/v1/kv_offload/cpu/test_gpu_worker.py +++ b/tests/v1/kv_offload/cpu/test_gpu_worker.py @@ -49,6 +49,22 @@ def test_rocm_cpu_to_gpu_uses_dma(monkeypatch: pytest.MonkeyPatch) -> None: ) +@pytest.mark.skipif(not gpu_worker.HAS_TRITON, reason="Requires Triton") +def test_unpinned_cpu_to_gpu_uses_dma(monkeypatch: pytest.MonkeyPatch) -> None: + """The Triton load path dereferences CPU pointers on the GPU, so pageable + host memory takes the DMA path even for pages where Triton would win.""" + monkeypatch.setattr(gpu_worker.current_platform, "is_xpu", lambda: False) + monkeypatch.setattr(gpu_worker.current_platform, "is_rocm", lambda: False) + + refs = [[CanonicalKVCacheRef(tensor_idx=0, page_size_bytes=512)]] + assert gpu_worker._select_swap_blocks_fn(refs, gpu_to_cpu=False) is not ( + ops.swap_blocks_batch + ) + assert gpu_worker._select_swap_blocks_fn( + refs, gpu_to_cpu=False, host_memory_is_pinned=False + ) is (ops.swap_blocks_batch) + + def test_worker_shutdown_releases_region_and_runs_both_handlers() -> None: """Both directions drain before the worker releases its shared region.""" worker = CPUOffloadingWorker.__new__(CPUOffloadingWorker) diff --git a/tests/v1/kv_offload/cpu/test_shared_offload_region.py b/tests/v1/kv_offload/cpu/test_shared_offload_region.py index 4f22d122db..f84e1bd729 100644 --- a/tests/v1/kv_offload/cpu/test_shared_offload_region.py +++ b/tests/v1/kv_offload/cpu/test_shared_offload_region.py @@ -15,6 +15,7 @@ from unittest.mock import MagicMock, call import pytest from vllm.utils.system_utils import get_mp_context +from vllm.v1.kv_offload.cpu import gpu_worker from vllm.v1.kv_offload.cpu import shared_offload_region as region_module from vllm.v1.kv_offload.cpu.shared_offload_region import ( SharedOffloadRegion, @@ -788,6 +789,87 @@ def test_cleanup_unregisters_every_pinned_chunk(iid, monkeypatch): ] +@pytest.fixture +def host_register(monkeypatch): + """Pretend to run on CUDA with host registrations capped at seven pages. + + Registration goes to a mock CudaRTLibrary and the region's cleanup to a + separate mock torch runtime. Both stay installed through the cleanup that + ``_region`` runs on exit. + """ + cudart = MagicMock() + cudart.cudaHostRegister.return_value = 0 + cudart.cudaHostUnregister.return_value = 0 + cudart.cudaGetLastError.return_value = 2 + torch_cudart = MagicMock() + torch_cudart.cudaHostUnregister.return_value = MagicMock(value=0) + monkeypatch.setattr(gpu_worker.current_platform, "is_cuda_alike", lambda: True) + monkeypatch.setattr(gpu_worker, "MAX_HOST_REGISTER_CHUNK_BYTES", 7 * PAGE_SIZE) + monkeypatch.setattr(gpu_worker, "CudaRTLibrary", lambda: cudart) + monkeypatch.setattr(region_module.torch.cuda, "cudart", lambda: torch_cudart) + return cudart, torch_cudart + + +def test_pin_mmap_region_registers_row_aligned_chunks(iid, host_register): + """Chunks end on row boundaries: 3-page rows under a 7-page cap register + as 6 + 6 + 3 pages, where a raw byte cap would give 7 + 7 + 1.""" + cudart, torch_cudart = host_register + with _region(iid, num_chunks=5, cpu_page_size=3 * PAGE_SIZE) as region: + gpu_worker.pin_mmap_region(region) + base = region._base.data_ptr() + assert cudart.mock_calls == [ + call.cudaHostRegister(base, 6 * PAGE_SIZE), + call.cudaHostRegister(base + 6 * PAGE_SIZE, 6 * PAGE_SIZE), + call.cudaHostRegister(base + 12 * PAGE_SIZE, 3 * PAGE_SIZE), + ] + assert region.is_pinned + + assert torch_cudart.cudaHostUnregister.call_args_list == [ + call(base + 12 * PAGE_SIZE), + call(base + 6 * PAGE_SIZE), + call(base), + ] + + +@pytest.mark.parametrize("fail_at", [0, 1, 2]) +def test_pin_mmap_region_failure_leaves_region_pageable(iid, host_register, fail_at): + """A failed chunk drains its pending error and unregisters the chunks + before it on the same handle, so the region is pinned whole or not at all.""" + cudart, torch_cudart = host_register + cudart.cudaHostRegister.side_effect = [0] * fail_at + [2] + with _region(iid, num_chunks=5, cpu_page_size=3 * PAGE_SIZE) as region: + gpu_worker.pin_mmap_region(region) + base = region._base.data_ptr() + assert not region.is_pinned + assert region.pinned_addresses == [] + + registered = [base + i * 6 * PAGE_SIZE for i in range(fail_at)] + failed_size = min(6, 15 - 6 * fail_at) * PAGE_SIZE + assert cudart.mock_calls == [ + *(call.cudaHostRegister(address, 6 * PAGE_SIZE) for address in registered), + call.cudaHostRegister(base + fail_at * 6 * PAGE_SIZE, failed_size), + call.cudaGetLastError(), + *(call.cudaHostUnregister(address) for address in reversed(registered)), + ] + torch_cudart.cudaHostUnregister.assert_not_called() + + +def test_pin_mmap_region_without_cuda_runtime_stays_pageable( + iid, host_register, monkeypatch +): + """A CUDA runtime that cannot be loaded leaves the region pageable instead + of failing worker startup.""" + + def missing_runtime(): + raise AssertionError("libcudart is not loaded in the current process") + + monkeypatch.setattr(gpu_worker, "CudaRTLibrary", missing_runtime) + with _region(iid) as region: + gpu_worker.pin_mmap_region(region) + assert not region.is_pinned + assert region.pinned_addresses == [] + + def test_cleanup_after_create_next_worker_view_releases_mmap(iid): """cleanup() must close the mmap even after create_next_worker_view was called. create_next_worker_view returns a view that shares storage with _base; both must be diff --git a/tests/v1/kv_offload/test_factory.py b/tests/v1/kv_offload/test_factory.py index 16539677c4..01ef756cae 100644 --- a/tests/v1/kv_offload/test_factory.py +++ b/tests/v1/kv_offload/test_factory.py @@ -275,11 +275,12 @@ def test_tiering_spec_create_worker_folds_device_index_for_sharded_layout(monkey @pytest.mark.parametrize("world_size", [2, 4, 8]) def test_cpu_spec_replicated_sizing_on_shared_region(monkeypatch, world_size: int): - # On shared-region (CUDA-alike) platforms the default spec now honors + # On CUDA the default spec honors # replicated layout: a single MLA copy (num_copies=1), matching tiering. import vllm.v1.kv_offload.cpu.spec as cpu_spec_module monkeypatch.setattr(cpu_spec_module.current_platform, "is_cuda_alike", lambda: True) + monkeypatch.setattr(cpu_spec_module.current_platform, "is_rocm", lambda: False) worker_kv_bytes_per_block = SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT spec = _create_spec( cpu_bytes_to_use=worker_kv_bytes_per_block * 8, @@ -295,19 +296,19 @@ def test_cpu_spec_replicated_sizing_on_shared_region(monkeypatch, world_size: in assert spec.num_chunks == 8 +@pytest.mark.parametrize("rocm", [False, True], ids=["non-cuda-alike", "rocm"]) @pytest.mark.parametrize("world_size", [2, 4, 8]) def test_cpu_spec_replicated_disabled_without_shared_region( - monkeypatch, world_size: int + monkeypatch, world_size: int, rocm: bool ): - # Data-loss guard: non-CUDA-alike platforms keep a per-rank private pinned + # Data-loss guard: platforms without shared regions keep a private pinned # tensor (no shared medium), so replicated layout MUST stay off. Otherwise # the rank-0 writer gate would ack rank>0 stores without writing, leaving # those private buffers empty and corrupting subsequent loads. import vllm.v1.kv_offload.cpu.spec as cpu_spec_module - monkeypatch.setattr( - cpu_spec_module.current_platform, "is_cuda_alike", lambda: False - ) + monkeypatch.setattr(cpu_spec_module.current_platform, "is_cuda_alike", lambda: rocm) + monkeypatch.setattr(cpu_spec_module.current_platform, "is_rocm", lambda: rocm) worker_kv_bytes_per_block = SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT spec = _create_spec( cpu_bytes_to_use=worker_kv_bytes_per_block * world_size * 2, @@ -324,17 +325,26 @@ def test_cpu_spec_replicated_disabled_without_shared_region( @pytest.mark.parametrize("config_replicated", [True, False]) -@pytest.mark.parametrize("cuda_alike", [True, False]) +@pytest.mark.parametrize( + ("cuda_alike", "rocm", "shared_region"), + [(True, False, True), (True, True, False), (False, False, False)], + ids=["cuda", "rocm", "non-cuda-alike"], +) def test_cpu_spec_replicated_layout_truth_matrix( - monkeypatch, cuda_alike: bool, config_replicated: bool + monkeypatch, + cuda_alike: bool, + rocm: bool, + shared_region: bool, + config_replicated: bool, ): # replicated_layout is enabled iff the config gate passes AND the deployment - # actually allocates on the shared region (CUDA-alike). + # actually allocates on the shared region. import vllm.v1.kv_offload.cpu.spec as cpu_spec_module monkeypatch.setattr( cpu_spec_module.current_platform, "is_cuda_alike", lambda: cuda_alike ) + monkeypatch.setattr(cpu_spec_module.current_platform, "is_rocm", lambda: rocm) worker_kv_bytes_per_block = SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT spec = _create_spec( cpu_bytes_to_use=worker_kv_bytes_per_block * 8, @@ -344,10 +354,10 @@ def test_cpu_spec_replicated_layout_truth_matrix( ) assert isinstance(spec, CPUOffloadingSpec) - assert spec.replicated_layout is (cuda_alike and config_replicated) + assert spec.replicated_layout is (shared_region and config_replicated) -def test_cpu_spec_create_worker_uses_mmap_on_cuda_alike(monkeypatch): +def test_cpu_spec_create_worker_uses_mmap_on_cuda(monkeypatch): import vllm.v1.kv_offload.cpu.spec as cpu_spec_module worker_kv_bytes_per_block = SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT @@ -371,6 +381,7 @@ def test_cpu_spec_create_worker_uses_mmap_on_cuda_alike(monkeypatch): return MagicMock() monkeypatch.setattr(cpu_spec_module.current_platform, "is_cuda_alike", lambda: True) + monkeypatch.setattr(cpu_spec_module.current_platform, "is_rocm", lambda: False) monkeypatch.setattr(cpu_spec_module, "SharedOffloadRegion", fake_region_ctor) monkeypatch.setattr(cpu_spec_module, "CPUOffloadingWorker", fake_worker_ctor) monkeypatch.setattr( @@ -386,7 +397,10 @@ def test_cpu_spec_create_worker_uses_mmap_on_cuda_alike(monkeypatch): assert worker_calls[0]["mmap_region"] is region -def test_cpu_spec_create_worker_uses_tensor_path_off_cuda_alike(monkeypatch): +@pytest.mark.parametrize("rocm", [False, True], ids=["non-cuda-alike", "rocm"]) +def test_cpu_spec_create_worker_uses_tensor_path_without_shared_region( + monkeypatch, rocm +): import vllm.v1.kv_offload.cpu.spec as cpu_spec_module spec = _create_spec(worker_kv_bytes_per_block=4096, world_size=4) @@ -403,15 +417,14 @@ def test_cpu_spec_create_worker_uses_tensor_path_off_cuda_alike(monkeypatch): worker_calls.append(kwargs) return MagicMock() - monkeypatch.setattr( - cpu_spec_module.current_platform, "is_cuda_alike", lambda: False - ) + monkeypatch.setattr(cpu_spec_module.current_platform, "is_cuda_alike", lambda: rocm) + monkeypatch.setattr(cpu_spec_module.current_platform, "is_rocm", lambda: rocm) monkeypatch.setattr(cpu_spec_module, "SharedOffloadRegion", fake_region_ctor) monkeypatch.setattr(cpu_spec_module, "CPUOffloadingWorker", fake_worker_ctor) spec.create_worker(MagicMock()) - # Non-CUDA-alike platforms keep the per-rank pinned-tensor path. + # ROCm and non-CUDA-alike platforms use the per-rank pinned-tensor path. assert region_calls == [] assert worker_calls[0]["mmap_region"] is None @@ -420,7 +433,7 @@ def test_cpu_spec_create_worker_skips_mmap_for_empty_cache(monkeypatch): import vllm.v1.kv_offload.cpu.spec as cpu_spec_module # worker_kv_bytes_per_block=0 yields num_chunks=0; a zero-byte region cannot - # be mmap'd, so even on CUDA-alike this must fall back to the tensor path. + # be mmap'd, so even on CUDA this must fall back to the tensor path. spec = _create_spec(worker_kv_bytes_per_block=0, world_size=4) assert isinstance(spec, CPUOffloadingSpec) assert spec.num_chunks == 0 @@ -429,6 +442,7 @@ def test_cpu_spec_create_worker_skips_mmap_for_empty_cache(monkeypatch): worker_calls: list[dict[str, Any]] = [] monkeypatch.setattr(cpu_spec_module.current_platform, "is_cuda_alike", lambda: True) + monkeypatch.setattr(cpu_spec_module.current_platform, "is_rocm", lambda: False) monkeypatch.setattr( cpu_spec_module, "SharedOffloadRegion", @@ -461,6 +475,7 @@ def test_cpu_spec_create_worker_rank_assignment( import vllm.v1.kv_offload.cpu.spec as cpu_spec_module monkeypatch.setattr(cpu_spec_module.current_platform, "is_cuda_alike", lambda: True) + monkeypatch.setattr(cpu_spec_module.current_platform, "is_rocm", lambda: False) worker_kv_bytes_per_block = SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT spec = _create_spec( cpu_bytes_to_use=worker_kv_bytes_per_block * 8, diff --git a/tests/v1/kv_offload/tiering/p2p/test_manager.py b/tests/v1/kv_offload/tiering/p2p/test_manager.py index aceec1cd8f..8e806582fd 100644 --- a/tests/v1/kv_offload/tiering/p2p/test_manager.py +++ b/tests/v1/kv_offload/tiering/p2p/test_manager.py @@ -180,6 +180,40 @@ class TestInitHashSeed: assert mgr._get_hash_seed() == "random-seed-abc" +class TestBackpressureRejected: + def _patch_transports(self, monkeypatch) -> None: + monkeypatch.setattr(manager_module, "NixlTransport", lambda *a, **k: object()) + monkeypatch.setattr(manager_module, "ZmqTransport", lambda *a, **k: object()) + monkeypatch.setattr( + manager_module.FileMapper, + "from_offloading_spec", + lambda **k: SimpleNamespace(get_run_config=lambda: {}), + ) + + def test_explicit_backpressure_detector_rejected(self, monkeypatch): + """An explicitly configured detector must fail fast for P2P. + + The generic store-latency detector cannot fail fast in PD mode and + rendezvous time makes store latency an unreliable pressure signal, + so P2P rejects it rather than degrading silently. + """ + self._patch_transports(monkeypatch) + with pytest.raises(ValueError, match="not supported for the P2P"): + P2PSecondaryTierManager( + offloading_spec=_init_offloading_spec(), + primary_kv_view=memoryview(bytearray(16)), + backpressure_detector=object(), + ) + + def test_no_detector_constructs_normally(self, monkeypatch): + self._patch_transports(monkeypatch) + mgr = P2PSecondaryTierManager( + offloading_spec=_init_offloading_spec(), + primary_kv_view=memoryview(bytearray(16)), + ) + assert mgr.bp_detector is None + + # --------------------------------------------------------------------------- # Tests for _peer_id_from_params # --------------------------------------------------------------------------- diff --git a/tests/v1/kv_offload/tiering/test_backpressure.py b/tests/v1/kv_offload/tiering/test_backpressure.py new file mode 100644 index 0000000000..b809bc4dc4 --- /dev/null +++ b/tests/v1/kv_offload/tiering/test_backpressure.py @@ -0,0 +1,706 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for back-pressure detection in TieringOffloadingManager. + +These tests use a DelayedSecondaryTierManager that holds completed jobs +until explicitly released, allowing precise control over when the manager +observes store completions and their apparent latency. +""" + +import time +from collections.abc import Iterable +from unittest.mock import MagicMock, patch + +import pytest +import torch + +from vllm.v1.kv_offload.base import ( + LookupResult, + OffloadKey, + OffloadPolicy, + ReqContext, + RequestOffloadingContext, + ScheduleEndContext, + make_offload_key, +) +from vllm.v1.kv_offload.tiering.backpressure import ( + DropAccountingPolicy, + DropStorePolicy, + EMABackpressureDetector, + ThrottledDropPolicy, +) +from vllm.v1.kv_offload.tiering.base import ( + JobResult, + SecondaryTierManager, + TieringOffloadingMetrics, + TransferJob, +) +from vllm.v1.kv_offload.tiering.manager import ( + CPUPrimaryTierOffloadingManager, + TieringOffloadingManager, +) + +_BP_EMA_ALPHA = EMABackpressureDetector.DEFAULT_ALPHA +_BP_WARMUP = EMABackpressureDetector.DEFAULT_WARMUP_COMPLETIONS +# Water marks in s/MiB, scaled for the test's tiny 16-byte blocks. +_BP_HIGH_WATER_S = 1000.0 +_BP_LOW_WATER_S = 500.0 + +_CTX = ReqContext(req_id="test") +_MOCK_OFFLOADING_SPEC = MagicMock() + + +def _mock_mmap_region(num_blocks: int, row_bytes: int = 16): + mock = MagicMock() + view = memoryview(torch.zeros((num_blocks, row_bytes), dtype=torch.int8).numpy()) + mock.create_kv_memoryview.return_value = view + return mock + + +def to_keys(int_ids: Iterable[int]) -> list[OffloadKey]: + return [make_offload_key(str(i).encode(), 0) for i in int_ids] + + +class DelayedSecondaryTierManager(SecondaryTierManager): + """Secondary tier that holds completed store jobs until released. + + Jobs are stored immediately but their completion is not reported + by get_finished_jobs() until release_jobs() is called. This lets + tests control the apparent latency of store operations by adjusting + submit_time on the JobMetadata before triggering the completion poll. + """ + + def __init__( + self, offloading_spec, primary_kv_view, tier_type, backpressure_detector=None + ): + super().__init__( + offloading_spec, + primary_kv_view, + tier_type, + backpressure_detector=backpressure_detector, + ) + self.blocks: dict[OffloadKey, bool] = {} + self._held_jobs: list[JobResult] = [] + self._released_jobs: list[JobResult] = [] + self._request_policy: OffloadPolicy | None = None + + def lookup(self, key, req_context): + return LookupResult.HIT if key in self.blocks else LookupResult.MISS + + def submit_store(self, job_metadata: TransferJob) -> None: + for key in job_metadata.keys: + self.blocks[key] = True + self._held_jobs.append(JobResult(job_id=job_metadata.job_id, success=True)) + + def submit_load(self, job_metadata: TransferJob) -> None: + self._released_jobs.append(JobResult(job_id=job_metadata.job_id, success=True)) + + def get_finished_jobs(self) -> Iterable[JobResult]: + result = self._released_jobs + self._released_jobs = [] + return result + + def release_jobs(self): + self._released_jobs.extend(self._held_jobs) + self._held_jobs.clear() + + def on_new_request(self, req_context): + if self._request_policy is not None: + return RequestOffloadingContext(policy=self._request_policy) + return RequestOffloadingContext() + + def drain_jobs(self): + self.release_jobs() + + def has_pending_work(self): + return bool(self._held_jobs) + + def get_num_blocks(self): + return len(self.blocks) + + +class TestDefaultConfig: + def test_fs_gets_local_watermarks(self): + cfg = EMABackpressureDetector.default_config("fs") + assert cfg["high_water_s"] == EMABackpressureDetector.LOCAL_HIGH_WATER_S + assert cfg["low_water_s"] == EMABackpressureDetector.LOCAL_LOW_WATER_S + + def test_obj_gets_network_watermarks(self): + cfg = EMABackpressureDetector.default_config("obj") + assert cfg["high_water_s"] == EMABackpressureDetector.NETWORK_HIGH_WATER_S + assert cfg["low_water_s"] == EMABackpressureDetector.NETWORK_LOW_WATER_S + + def test_p2p_gets_local_watermarks(self): + cfg = EMABackpressureDetector.default_config("p2p") + assert cfg["high_water_s"] == EMABackpressureDetector.LOCAL_HIGH_WATER_S + assert cfg["low_water_s"] == EMABackpressureDetector.LOCAL_LOW_WATER_S + + def test_fs_with_remote_locality_gets_network_watermarks(self): + cfg = EMABackpressureDetector.default_config("fs", locality="REMOTE") + assert cfg["high_water_s"] == EMABackpressureDetector.NETWORK_HIGH_WATER_S + assert cfg["low_water_s"] == EMABackpressureDetector.NETWORK_LOW_WATER_S + + def test_fs_with_local_locality_gets_local_watermarks(self): + cfg = EMABackpressureDetector.default_config("fs", locality="LOCAL") + assert cfg["high_water_s"] == EMABackpressureDetector.LOCAL_HIGH_WATER_S + assert cfg["low_water_s"] == EMABackpressureDetector.LOCAL_LOW_WATER_S + + +class TestIdleDecay: + """Tests for EMA idle decay — recovery from pressure without completions.""" + + def _make_detector(self, **kwargs): + defaults = dict( + high_water_s=_BP_HIGH_WATER_S, + low_water_s=_BP_LOW_WATER_S, + decay_half_life_s=2.0, + ) + defaults.update(kwargs) + return EMABackpressureDetector(**defaults) + + def test_ema_decays_while_under_pressure(self): + bp = self._make_detector() + bp._completions = _BP_WARMUP + bp._ema = _BP_HIGH_WATER_S * 2 + bp._under_pressure = True + bp._last_update = time.monotonic() + + with patch("time.monotonic", return_value=bp._last_update + 4.1): + # ~2.05 half-lives → EMA * 0.24, well below low water (500) + assert bp.is_under_pressure() is False + assert bp.store_latency_ema < _BP_LOW_WATER_S + + def test_ema_stays_pressured_if_not_enough_decay(self): + bp = self._make_detector() + bp._completions = _BP_WARMUP + bp._ema = _BP_HIGH_WATER_S * 4 + bp._under_pressure = True + bp._last_update = time.monotonic() + + with patch("time.monotonic", return_value=bp._last_update + 2.0): + # 2s = 1 half-life → EMA * 0.5 = HIGH * 2, still above LOW + assert bp.is_under_pressure() is True + assert bp.store_latency_ema == pytest.approx(_BP_HIGH_WATER_S * 2, rel=0.01) + + def test_no_decay_before_warmup(self): + bp = self._make_detector() + bp._ema = _BP_HIGH_WATER_S * 2 + bp._under_pressure = True + bp._last_update = time.monotonic() + + with patch("time.monotonic", return_value=bp._last_update + 10.0): + # Still in warmup (_completions=0 < _warmup_completions), + # so decay should not be applied in is_under_pressure. + assert bp.is_under_pressure() is True + assert bp.store_latency_ema == _BP_HIGH_WATER_S * 2 + + def test_no_decay_applied_during_new_sample(self): + """on_store_completed applies the new sample directly without + idle decay — decay only runs in is_under_pressure().""" + bp = self._make_detector() + bp._completions = _BP_WARMUP + bp._ema = _BP_HIGH_WATER_S * 2 + bp._under_pressure = True + t0 = time.monotonic() + bp._last_update = t0 + + block_bytes = 16 + with patch("time.monotonic", return_value=t0 + 4.0): + bp.on_store_completed(0.0, block_bytes) + # No decay in on_store_completed: EMA = alpha*0 + (1-alpha)*2000 + expected = (1 - _BP_EMA_ALPHA) * _BP_HIGH_WATER_S * 2 + assert bp.store_latency_ema == pytest.approx(expected, rel=0.01) + + def test_reset_clears_last_update(self): + bp = self._make_detector() + bp._last_update = time.monotonic() + bp.reset() + assert bp._last_update == 0.0 + + +class TestBackpressure: + @pytest.fixture + def setup(self): + mock_region = _mock_mmap_region(20) + self.primary = CPUPrimaryTierOffloadingManager( + num_chunks=20, mmap_region=mock_region + ) + mock_view = mock_region.create_kv_memoryview() + self.tier = DelayedSecondaryTierManager( + offloading_spec=_MOCK_OFFLOADING_SPEC, + primary_kv_view=mock_view, + tier_type="delayed", + backpressure_detector=EMABackpressureDetector( + high_water_s=_BP_HIGH_WATER_S, + low_water_s=_BP_LOW_WATER_S, + ), + ) + self.manager = TieringOffloadingManager( + primary_tier=self.primary, + secondary_tiers=[self.tier], + ) + + def _start_request(self, ctx=_CTX): + if ctx.req_id not in self.manager._req_state: + self.manager.on_new_request(ctx) + + def _store_blocks(self, keys, ctx=_CTX): + self._start_request(ctx) + result = self.manager.prepare_store(keys, ctx) + assert result is not None + self.manager.complete_store(keys, ctx, success=True) + + def _simulate_on_schedule_end(self): + ctx = ScheduleEndContext(new_req_ids=[], preempted_req_ids=()) + self.manager.on_schedule_end(ctx) + + def _backdate_held_jobs(self, age_s: float): + """Set submit_time on all in-flight store jobs so they appear + to have taken ``age_s`` seconds when completed.""" + self.manager._processed_jobs_this_step = False + now = time.monotonic() + for job_id, meta in self.manager._jobs.items(): + tj = meta.transfer_job + if not tj.is_promotion and tj.submit_time > 0: + tj.submit_time = now - age_s + + def test_ema_updates_on_store_completion(self, setup): + bp = self.tier.bp_detector + fast_latency = 0.01 + # EMA is in s/MiB; with 16-byte blocks, scale = MiB / block_bytes. + scale = EMABackpressureDetector._MIB / self.tier.block_size_bytes + + # During warmup, EMA stays at 0. + for i in range(_BP_WARMUP - 1): + keys = to_keys([300 + i]) + self._store_blocks(keys) + self._backdate_held_jobs(fast_latency) + self.tier.release_jobs() + self._simulate_on_schedule_end() + assert bp.store_latency_ema == 0.0 + + # Final warmup sample seeds EMA with the mean. + keys = to_keys([400]) + self._store_blocks(keys) + self._backdate_held_jobs(fast_latency) + self.tier.release_jobs() + self._simulate_on_schedule_end() + fast_s_per_mib = fast_latency * scale + assert bp.store_latency_ema == pytest.approx(fast_s_per_mib, rel=0.1) + + # After warmup, EMA updates normally. + new_latency = 0.05 + keys = to_keys([401]) + self._store_blocks(keys) + self._backdate_held_jobs(new_latency) + self.tier.release_jobs() + self._simulate_on_schedule_end() + new_s_per_mib = new_latency * scale + expected = _BP_EMA_ALPHA * new_s_per_mib + (1 - _BP_EMA_ALPHA) * fast_s_per_mib + assert bp.store_latency_ema == pytest.approx(expected, rel=0.1) + + def test_pressure_activates_above_high_water(self, setup): + bp = self.tier.bp_detector + + # Warm up the detector with fast completions first. + for i in range(_BP_WARMUP): + keys = to_keys([200 + i]) + self._store_blocks(keys) + self._backdate_held_jobs(0.001) + self.tier.release_jobs() + self._simulate_on_schedule_end() + + keys = to_keys([100]) + self._store_blocks(keys) + self._simulate_on_schedule_end() + + # 0.1s for a single 16-byte block → ~6553 s/MiB, well above high water. + self._backdate_held_jobs(0.1) + self.tier.release_jobs() + self._simulate_on_schedule_end() + + assert bp.store_latency_ema > _BP_HIGH_WATER_S + assert bp.is_under_pressure() is True + + def test_pressure_clears_below_low_water(self, setup): + bp = self.tier.bp_detector + bp.store_latency_ema = _BP_HIGH_WATER_S * 2 + bp._under_pressure = True + bp._completions = _BP_WARMUP + + # Drive several fast completions to bring EMA below low water. + block_id = 100 + for _ in range(50): + if bp.store_latency_ema < _BP_LOW_WATER_S: + break + keys = to_keys([block_id]) + block_id += 1 + self._store_blocks(keys) + # Pressure is on, so the cascade was skipped. Force-submit + # a store job manually to get a completion to feed the EMA. + job_meta = self.manager.create_store_job(keys, _CTX) + job_meta.submit_time = time.monotonic() - 0.001 + self.tier.submit_store(job_meta) + self.tier.release_jobs() + self._simulate_on_schedule_end() + else: + pytest.fail(f"EMA did not recover below low water: {bp.store_latency_ema}") + + assert bp.is_under_pressure() is False + + def test_hysteresis_prevents_oscillation(self, setup): + bp = self.tier.bp_detector + bp._completions = _BP_WARMUP + block_bytes = self.tier.block_size_bytes + + # Set EMA between low and high water marks (in s/MiB). + mid = (_BP_LOW_WATER_S + _BP_HIGH_WATER_S) / 2 + bp.store_latency_ema = mid + bp._last_update = time.monotonic() + + # Raw seconds that produce `mid` s/MiB for a given number of blocks. + def mid_latency_s(num_blocks: int) -> float: + return mid * (num_blocks * block_bytes / EMABackpressureDetector._MIB) + + # When not under pressure, mid-range EMA should not activate. + bp._under_pressure = False + keys = to_keys(range(2)) + self._store_blocks(keys) + self._backdate_held_jobs(mid_latency_s(len(keys))) + self.tier.release_jobs() + self._simulate_on_schedule_end() + assert bp.is_under_pressure() is False + + # When under pressure, mid-range EMA should not deactivate. + bp._under_pressure = True + bp._last_update = time.monotonic() + keys2 = to_keys(range(10, 12)) + # Force-submit since pressure is on (cascade would skip). + for k in keys2: + self.primary.prepare_store([k], _CTX) + self.primary.complete_store([k], _CTX) + job_meta = self.manager.create_store_job(keys2, _CTX) + job_meta.submit_time = time.monotonic() - mid_latency_s(len(keys2)) + self.tier.submit_store(job_meta) + self.tier.release_jobs() + self._simulate_on_schedule_end() + assert bp.is_under_pressure() is True + + def test_stores_skipped_under_pressure(self, setup): + bp = self.tier.bp_detector + bp._under_pressure = True + bp.store_latency_ema = _BP_HIGH_WATER_S * 4 + initial_blocks = self.tier.get_num_blocks() + + keys = to_keys(range(50, 53)) + self._store_blocks(keys) + + # Blocks should be in primary but NOT in secondary. + for k in keys: + assert self.primary.lookup(k, _CTX) is LookupResult.HIT + assert self.tier.get_num_blocks() == initial_blocks + + def test_stores_resume_after_pressure_clears(self, setup): + bp = self.tier.bp_detector + + # Start under pressure — stores skipped. + bp._under_pressure = True + bp.store_latency_ema = _BP_HIGH_WATER_S * 4 + keys1 = to_keys(range(60, 62)) + self._store_blocks(keys1) + assert all(k not in self.tier.blocks for k in keys1) + + # Clear pressure — next store should cascade. + bp._under_pressure = False + keys2 = to_keys(range(70, 72)) + self._store_blocks(keys2) + assert all(k in self.tier.blocks for k in keys2) + + def test_dropped_store_count_tracked(self, setup): + bp = self.tier.bp_detector + bp._under_pressure = True + bp.store_latency_ema = _BP_HIGH_WATER_S * 4 + policy = bp.policy + assert policy.pop_stores_dropped() == (0, 0) + + self._store_blocks(to_keys([80])) + assert policy._stores_dropped == 1 + assert policy._blocks_dropped == 1 + + self._store_blocks(to_keys([81, 82])) + assert policy._stores_dropped == 2 + assert policy._blocks_dropped == 3 + + def test_metrics_reported_via_get_stats(self, setup): + bp = self.tier.bp_detector + bp._under_pressure = True + bp.store_latency_ema = 2.5 + policy = bp.policy + policy._stores_dropped = 5 + policy._blocks_dropped = 12 + + stats = self.manager.get_stats() + assert stats is not None + reduced = stats.reduce() + + ema_key = TieringOffloadingMetrics.BACKPRESSURE_STORE_LATENCY_EMA + stores_key = TieringOffloadingMetrics.BACKPRESSURE_STORES_DROPPED + blocks_key = TieringOffloadingMetrics.BACKPRESSURE_BLOCKS_DROPPED + assert reduced[f"{ema_key}:('1:delayed',)"] == pytest.approx(2.5) + assert reduced[f"{stores_key}:('1:delayed',)"] == 5 + assert reduced[f"{blocks_key}:('1:delayed',)"] == 12 + + # Dropped counts reset after get_stats. + assert policy._stores_dropped == 0 + assert policy._blocks_dropped == 0 + + def test_reset_cache_clears_backpressure(self, setup): + bp = self.tier.bp_detector + bp.store_latency_ema = 5.0 + bp._under_pressure = True + + self.manager.reset_cache() + + assert bp.store_latency_ema == 0.0 + assert bp.is_under_pressure() is False + + def test_request_level_tier_respects_backpressure(self, setup): + """Request-level cascade also skips pressured tiers.""" + # Store blocks in primary first. + existing = to_keys(range(3)) + self._store_blocks(existing) + self.tier.release_jobs() + self._simulate_on_schedule_end() + self._simulate_on_schedule_end() + + # Make tier request-level for a new request. + self.tier._request_policy = OffloadPolicy.REQUEST_LEVEL + ctx = ReqContext(req_id="req_rl_bp") + self.manager.on_new_request(ctx) + + # Activate pressure. + bp = self.tier.bp_detector + bp._under_pressure = True + bp.store_latency_ema = _BP_HIGH_WATER_S * 4 + initial_blocks = self.tier.get_num_blocks() + + # prepare_store with existing + new blocks. + new = to_keys(range(10, 12)) + all_keys = existing + new + result = self.manager.prepare_store(all_keys, ctx) + assert result is not None + + # Existing blocks would normally cascade to request-level tier, + # but should be skipped under pressure. + assert self.tier.get_num_blocks() == initial_blocks + + def test_loads_continue_under_pressure(self, setup): + """Loads from a pressured tier still work.""" + blocks = to_keys(range(3)) + for b in blocks: + self.tier.blocks[b] = True + + bp = self.tier.bp_detector + bp._under_pressure = True + bp.store_latency_ema = _BP_HIGH_WATER_S * 4 + + # Lookups should still initiate promotion. + for b in blocks: + result = self.manager.lookup(b, _CTX) + assert result is LookupResult.HIT_PENDING + + +class TestThrottledDropPolicy: + """Tests for proportional throttling under pressure.""" + + def _make_detector(self, **kwargs): + defaults = dict( + high_water_s=_BP_HIGH_WATER_S, + low_water_s=_BP_LOW_WATER_S, + cooldown_s=0.0, + ) + defaults.update(kwargs) + return EMABackpressureDetector(**defaults) + + def test_no_drop_when_not_under_pressure(self): + bp = self._make_detector() + for _ in range(20): + assert bp.should_store(1) is True + + def test_no_drop_at_exactly_high_water(self): + bp = self._make_detector() + bp._under_pressure = True + bp.store_latency_ema = _BP_HIGH_WATER_S + for _ in range(20): + assert bp.should_store(1) is True + + def test_partial_drop_at_moderate_pressure(self): + bp = self._make_detector() + bp._under_pressure = True + bp.store_latency_ema = _BP_HIGH_WATER_S * 2 + results = [bp.should_store(1) for _ in range(20)] + assert any(results), "some stores should be allowed" + assert not all(results), "some stores should be dropped" + + def test_full_drop_at_severe_pressure(self): + bp = self._make_detector() + bp._under_pressure = True + bp.store_latency_ema = _BP_HIGH_WATER_S * 4 + for _ in range(20): + assert bp.should_store(1) is False + + def test_drop_rate_increases_with_severity(self): + """Higher EMA → higher drop rate.""" + drop_counts = [] + for multiplier in [1.5, 2.0, 2.5, 3.5]: + bp = self._make_detector() + bp._under_pressure = True + bp.store_latency_ema = _BP_HIGH_WATER_S * multiplier + drops = sum(1 for _ in range(100) if not bp.should_store(1)) + drop_counts.append(drops) + for i in range(len(drop_counts) - 1): + assert drop_counts[i] <= drop_counts[i + 1] + + def test_explicit_drop_store_policy_still_drops_all(self): + """DropStorePolicy (explicit) still drops everything under pressure.""" + bp = EMABackpressureDetector( + high_water_s=_BP_HIGH_WATER_S, + low_water_s=_BP_LOW_WATER_S, + policy=DropStorePolicy(), + cooldown_s=0.0, + ) + bp._under_pressure = True + bp.store_latency_ema = _BP_HIGH_WATER_S * 1.5 + for _ in range(20): + assert bp.should_store(1) is False + + +class TestSharedDropAccounting: + """Both policies share drop accounting via DropAccountingPolicy.""" + + @pytest.mark.parametrize("policy_cls", [DropStorePolicy, ThrottledDropPolicy]) + def test_shared_base(self, policy_cls): + assert issubclass(policy_cls, DropAccountingPolicy) + + @pytest.mark.parametrize("policy_cls", [DropStorePolicy, ThrottledDropPolicy]) + def test_pop_returns_and_resets(self, policy_cls): + policy = policy_cls() + policy.on_store_skipped(3) + policy.on_store_skipped(2) + assert policy.pop_stores_dropped() == (2, 5) + assert policy.pop_stores_dropped() == (0, 0) + + @pytest.mark.parametrize("policy_cls", [DropStorePolicy, ThrottledDropPolicy]) + def test_reset_clears_counters(self, policy_cls): + policy = policy_cls() + policy.on_store_skipped(4) + policy.reset() + assert policy.pop_stores_dropped() == (0, 0) + + def test_throttled_reset_also_clears_call_count(self): + policy = ThrottledDropPolicy() + policy._call_count = 7 + policy.reset() + assert policy._call_count == 0 + + +class TestHealthyBypass: + """Tests for the fast-path bypass when a tier is consistently healthy.""" + + def _make_detector(self, **kwargs): + defaults = dict( + high_water_s=_BP_HIGH_WATER_S, + low_water_s=_BP_LOW_WATER_S, + cooldown_s=0.0, + ) + defaults.update(kwargs) + return EMABackpressureDetector(**defaults) + + def test_not_healthy_initially(self): + bp = self._make_detector() + assert bp.is_healthy() is False + + def test_healthy_after_streak(self): + bp = self._make_detector() + bp._completions = _BP_WARMUP + bp._last_update = time.monotonic() + block_bytes = 16 + fast_s_per_mib = _BP_LOW_WATER_S * 0.5 + fast_elapsed = fast_s_per_mib * (block_bytes / EMABackpressureDetector._MIB) + for _ in range(EMABackpressureDetector._HEALTHY_THRESHOLD + _BP_WARMUP): + bp.on_store_completed(fast_elapsed, block_bytes) + assert bp.is_healthy() is True + + def test_healthy_resets_on_high_latency(self): + bp = self._make_detector() + bp._completions = _BP_WARMUP + bp._healthy_streak = EMABackpressureDetector._HEALTHY_THRESHOLD + bp._ema = _BP_LOW_WATER_S + bp._last_update = time.monotonic() + block_bytes = 16 + slow_s_per_mib = _BP_HIGH_WATER_S * 2 + slow_elapsed = slow_s_per_mib * (block_bytes / EMABackpressureDetector._MIB) + bp.on_store_completed(slow_elapsed, block_bytes) + assert bp.is_healthy() is False + + def test_healthy_bypasses_policy(self): + bp = self._make_detector() + bp._completions = _BP_WARMUP + bp._healthy_streak = EMABackpressureDetector._HEALTHY_THRESHOLD + assert bp.should_store(1) is True + + def test_reset_clears_healthy_streak(self): + bp = self._make_detector() + bp._healthy_streak = EMABackpressureDetector._HEALTHY_THRESHOLD + bp.reset() + assert bp._healthy_streak == 0 + assert bp.is_healthy() is False + + +class TestCooldown: + """Tests for the post-pressure cooldown period.""" + + def _make_detector(self, **kwargs): + defaults = dict( + high_water_s=_BP_HIGH_WATER_S, + low_water_s=_BP_LOW_WATER_S, + cooldown_s=1.0, + ) + defaults.update(kwargs) + return EMABackpressureDetector(**defaults) + + def test_cooldown_throttles_stores(self): + bp = self._make_detector() + bp._completions = _BP_WARMUP + bp._under_pressure = True + bp._ema = _BP_HIGH_WATER_S * 2 + bp._last_update = time.monotonic() + + with patch("time.monotonic") as mock_time: + t0 = bp._last_update + 10.0 + mock_time.return_value = t0 + assert bp.is_under_pressure() is False + assert bp._pressure_cleared_at > 0 + + mock_time.return_value = t0 + 0.1 + results = [bp.should_store(1) for _ in range(10)] + assert sum(results) == 5, "cooldown should allow ~50%" + + def test_no_cooldown_when_never_pressured(self): + bp = self._make_detector() + for _ in range(20): + assert bp.should_store(1) is True + + def test_no_cooldown_after_window_expires(self): + bp = self._make_detector() + bp._pressure_cleared_at = time.monotonic() - 2.0 + for _ in range(20): + assert bp.should_store(1) is True + + def test_reset_clears_cooldown(self): + bp = self._make_detector() + bp._pressure_cleared_at = time.monotonic() + bp._cooldown_count = 5 + bp.reset() + assert bp._pressure_cleared_at == 0.0 + assert bp._cooldown_count == 0 diff --git a/tests/v1/logits_processors/test_custom_offline.py b/tests/v1/logits_processors/test_custom_offline.py index 0d0b9531dc..f04f5cfa47 100644 --- a/tests/v1/logits_processors/test_custom_offline.py +++ b/tests/v1/logits_processors/test_custom_offline.py @@ -162,6 +162,9 @@ def test_custom_logitsprocs(monkeypatch, logitproc_source: CustomLogitprocSource """ # Test that logitproc info is passed to workers monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "1") + # These tests exercise the V1-interface logits processor; Model Runner V2 + # rejects V1-interface processors at load time by design. + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0") set_random_seed(40) # Choose LLM args based on logitproc source @@ -217,6 +220,9 @@ def test_custom_logitsprocs_req(monkeypatch): """ # Test that logitproc info is passed to workers monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "1") + # These tests exercise the V1-interface logits processor; Model Runner V2 + # rejects V1-interface processors at load time by design. + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0") set_random_seed(40) _run_test( {"logits_processors": [WrappedPerReqLogitsProcessor]}, logitproc_loaded=True @@ -261,6 +267,9 @@ def test_rejects_custom_logitsprocs( """ monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0") + # These tests exercise V1-interface rejection and V1 runner internals; + # Model Runner V2 rejects V1-interface processors at load time by design. + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0") set_random_seed(40) test_params: dict[str, dict[str, Any]] = { diff --git a/tests/v1/logits_processors/test_custom_online.py b/tests/v1/logits_processors/test_custom_online.py index 4a7ad8cc00..b117875625 100644 --- a/tests/v1/logits_processors/test_custom_online.py +++ b/tests/v1/logits_processors/test_custom_online.py @@ -79,6 +79,9 @@ def server(default_server_args, request, monkeypatch): """ # Test that logitproc info is passed to workers monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "1") + # These tests exercise the V1-interface logits processor; Model Runner V2 + # rejects V1-interface processors at load time by design. + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0") if request.param: # Launch server, append FQCN argument, inject dummy logitproc module diff --git a/tests/v1/sample/test_topk_topp_sampler.py b/tests/v1/sample/test_topk_topp_sampler.py index 0b570f49d9..8ff16e5599 100644 --- a/tests/v1/sample/test_topk_topp_sampler.py +++ b/tests/v1/sample/test_topk_topp_sampler.py @@ -237,6 +237,44 @@ def test_flashinfer_sampler(): ) +@pytest.mark.skipif( + not FLASHINFER_TOPK_TOPP_SUPPORTED, + reason="Requires CUDA and flashinfer", +) +def test_flashinfer_sampler_fallback_when_jit_cannot_target_gpu(monkeypatch): + """FlashInfer swallows arch-detection errors (e.g. SM 12.x with a CUDA + toolkit older than 12.9), leaving an empty target-arch set that makes + every JIT spec fail with a misleading "requires sm75 or higher" error at + first use, killing the engine during startup profiling (issue #42393). + The sampler gate must detect this and fall back to native sampling, and + surface the real reason when the user explicitly opted in. + """ + import flashinfer.jit.core + from flashinfer.compilation_context import CompilationContext + + from vllm.v1.sample.ops.topk_topp_sampler import flashinfer_sampler_supported + + def fake_check_cuda_arch(): + raise RuntimeError("FlashInfer requires GPUs with sm75 or higher") + + def fake_normalize_cuda_arch(major, minor): + raise RuntimeError("SM 12.x requires CUDA >= 12.9") + + monkeypatch.setattr(flashinfer.jit.core, "check_cuda_arch", fake_check_cuda_arch) + monkeypatch.setattr( + CompilationContext, + "_normalize_cuda_arch", + staticmethod(fake_normalize_cuda_arch), + ) + + monkeypatch.delenv("VLLM_USE_FLASHINFER_SAMPLER", raising=False) + assert flashinfer_sampler_supported() is False + + monkeypatch.setenv("VLLM_USE_FLASHINFER_SAMPLER", "1") + with pytest.raises(RuntimeError, match="SM 12.x requires CUDA >= 12.9"): + flashinfer_sampler_supported() + + # ============================================================================= # Triton kernel tests # ============================================================================= diff --git a/tests/v1/spec_decode/test_dflash2.py b/tests/v1/spec_decode/test_dflash2.py index e7ff2b6d5d..72379fa7d7 100644 --- a/tests/v1/spec_decode/test_dflash2.py +++ b/tests/v1/spec_decode/test_dflash2.py @@ -7,6 +7,7 @@ import pytest import torch from vllm.model_executor.models.qwen3_dflash2 import _grouped_conv, _score_edges +from vllm.platforms import current_platform from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator from vllm.v1.worker.gpu.spec_decode.dflash2.speculator import DFlash2Speculator @@ -35,6 +36,55 @@ def test_grouped_conv_matches_reference(block_size: int): torch.testing.assert_close(actual, expected.flatten(0, 1).flatten(-2)) +@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA") +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) +@pytest.mark.parametrize( + "batch,num_groups,group_size,block_size,taps", + [ + (7, 13, 11, 5, 1), + (7, 13, 11, 5, 3), + (7, 13, 11, 8, 2), + (16, 64, 16, 8, 2), + (16, 160, 16, 8, 2), + ], +) +def test_grouped_conv_triton_matches_reference( + dtype: torch.dtype, + batch: int, + num_groups: int, + group_size: int, + block_size: int, + taps: int, +): + torch.manual_seed(0) + rows = batch * block_size + hidden = torch.randn(rows, num_groups * group_size, device="cuda", dtype=dtype) + base = torch.randn(taps, num_groups * group_size, device="cuda", dtype=dtype) + projected = torch.randn(rows, 2, taps, num_groups, device="cuda", dtype=dtype) + delta = projected[:, 1] + + actual = _grouped_conv( + hidden, delta, base, block_size, num_groups, group_size, taps + ) + + hidden_blocks = hidden.float().view(batch, block_size, num_groups, group_size) + expected = torch.zeros_like(hidden_blocks) + base_blocks = base.float().view(taps, num_groups, group_size) + delta_blocks = delta.float().view(batch, block_size, taps, num_groups) + for position in range(block_size): + for tap in range(min(taps, position + 1)): + expected[:, position] += ( + base_blocks[tap] + delta_blocks[:, position, tap, :, None] + ) * hidden_blocks[:, position - tap] + + torch.testing.assert_close( + actual, + expected.flatten(0, 1).flatten(-2).to(dtype), + rtol=1e-2 if dtype is torch.bfloat16 else 1e-5, + atol=1e-2 if dtype is torch.bfloat16 else 1e-5, + ) + + def test_selector_edges_match_sequential_reference(): torch.manual_seed(1) batch, steps, top_k, rank = 2, 4, 3, 5 diff --git a/tests/v1/spec_decode/test_dflash_causality.py b/tests/v1/spec_decode/test_dflash_causality.py index 0f1de86cf9..dc13e55700 100644 --- a/tests/v1/spec_decode/test_dflash_causality.py +++ b/tests/v1/spec_decode/test_dflash_causality.py @@ -103,6 +103,18 @@ def test_dflash_fc_uses_aux_layer_count(): assert _get_dflash_fc_input_size(vllm_config) == 3 * 4096 +def test_dflash_target_layers_map_to_post_layer_boundaries(): + vllm_config = _vllm_config( + dflash_config={"target_layer_ids": [0, 16, 31]}, + ) + + assert get_eagle3_aux_layers_from_config(vllm_config.speculative_config) == ( + 1, + 17, + 32, + ) + + @pytest.mark.parametrize("config_name", ["dflash_config", "eagle_config"]) def test_eagle_aux_layers_preserves_legacy_layer_ids(config_name): layer_ids = [1, 17, 32] diff --git a/tests/v1/structured_output/test_utils.py b/tests/v1/structured_output/test_utils.py index f5f8020220..3e4a12f7e5 100644 --- a/tests/v1/structured_output/test_utils.py +++ b/tests/v1/structured_output/test_utils.py @@ -5,9 +5,12 @@ import pytest from xgrammar import Grammar from xgrammar.testing import _is_grammar_accept_string +from vllm.sampling_params import SamplingParams, StructuredOutputsParams from vllm.v1.structured_output.backend_xgrammar import ( has_xgrammar_unsupported_json_features, + validate_xgrammar_grammar, ) +from vllm.v1.structured_output.utils import choice_as_grammar pytestmark = pytest.mark.cpu_test @@ -232,3 +235,46 @@ def test_pattern_properties_constrains_keys_and_values(pattern_properties_schema assert grammar_accepts(pattern_properties_schema, '{"grade_1": 5}') assert not grammar_accepts(pattern_properties_schema, '{"other": 5}') assert not grammar_accepts(pattern_properties_schema, '{"grade_1": "five"}') + + +class TestIsGrammarAcceptString: + class TestPR48115Regressions: + @pytest.mark.parametrize("codepoint", [*range(0x20), 0x7F]) + def test_choice_as_grammar_preserves_control_characters(self, codepoint): + choice = f"a{chr(codepoint)}f" + grammar = Grammar.from_ebnf(choice_as_grammar([choice])) + + assert _is_grammar_accept_string(grammar, choice) + assert not _is_grammar_accept_string(grammar, "a") + assert not _is_grammar_accept_string(grammar, "af") + assert not _is_grammar_accept_string(grammar, f"a\\u{codepoint:04x}f") + + @pytest.mark.parametrize( + ("choice", "other"), + [ + (r"a\nf", "a\nf"), + ('a quote " and a backslash \\', 'a quote " and a backslash '), + ("café", "cafe"), + ("日本語", "日本"), + ("😀", "😁"), + ], + ) + def test_choice_as_grammar_preserves_literal_choices(self, choice, other): + grammar = Grammar.from_ebnf(choice_as_grammar(["yes", choice])) + + assert _is_grammar_accept_string(grammar, "yes") + assert _is_grammar_accept_string(grammar, choice) + assert not _is_grammar_accept_string(grammar, other) + assert not _is_grammar_accept_string(grammar, choice + "extra") + + def test_validate_xgrammar_preserves_multiline_choice(self): + structured_outputs = StructuredOutputsParams(choice=["yes", "no\nplease"]) + validate_xgrammar_grammar( + SamplingParams(structured_outputs=structured_outputs) + ) + + assert structured_outputs.choice is None + grammar = Grammar.from_ebnf(structured_outputs.grammar) + assert _is_grammar_accept_string(grammar, "yes") + assert _is_grammar_accept_string(grammar, "no\nplease") + assert not _is_grammar_accept_string(grammar, r"no\nplease") diff --git a/tests/v1/worker/test_attn_utils.py b/tests/v1/worker/test_attn_utils.py index 2dadc8c980..99481a1293 100644 --- a/tests/v1/worker/test_attn_utils.py +++ b/tests/v1/worker/test_attn_utils.py @@ -9,11 +9,13 @@ never addressed by the logical view. from types import SimpleNamespace +import numpy as np import pytest import torch import vllm.v1.hisparse.binding as attn_utils_module from tests.v1.attention.utils import dense_kv_cache_views +from vllm.config.compilation import CUDAGraphMode from vllm.v1.attention.backend import AttentionBackend, AttentionCGSupport, MultipleOf from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy from vllm.v1.hisparse.binding import allocate_hisparse_kv_caches @@ -30,9 +32,11 @@ from vllm.v1.kv_cache_interface import ( ) from vllm.v1.worker.gpu import attn_utils from vllm.v1.worker.gpu.attn_utils import ( + FastPrefillHelper, get_attn_cg_support, get_query_lens_mismatch_unsupported_backend, ) +from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor from vllm.v1.worker.utils import ( AttentionGroup, allocate_kv_cache, @@ -240,6 +244,63 @@ def test_get_kv_sharing_fast_prefill_eligible_layers(monkeypatch: pytest.MonkeyP assert attn_utils.get_kv_sharing_fast_prefill_eligible_layers(vllm_config) == set() +@pytest.mark.parametrize( + "num_tokens", [1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128] +) +@pytest.mark.parametrize("num_active_loras", [1, 2, 4]) +def test_fast_prefill_dispatch_preserves_active_lora_count( + num_tokens: int, num_active_loras: int +): + """Fast-prefill padding must match the main dispatch's LoRA variant.""" + + class FakeCudaGraphManager: + device = "cpu" + + def __init__(self): + self.dispatch_calls = [] + + def dispatch(self, **kwargs): + self.dispatch_calls.append(kwargs) + # A captured no-LoRA graph is padded to 8 tokens, while the + # active-LoRA path stays eager at the unpadded token count. + if kwargs["num_active_loras"] == 0: + num_tokens = 8 + mode = CUDAGraphMode.PIECEWISE + else: + num_tokens = kwargs["num_tokens"] + mode = CUDAGraphMode.NONE + return BatchExecutionDescriptor( + cg_mode=mode, + num_tokens=num_tokens, + num_reqs=kwargs["num_reqs"], + num_active_loras=kwargs["num_active_loras"], + num_ubatches=1, + ) + + manager = FakeCudaGraphManager() + helper = FastPrefillHelper(manager, max_num_tokens=max(32, num_tokens)) + metadata = helper.prepare( + torch.arange(num_tokens, dtype=torch.int32), + num_reqs=1, + cu_num_logits_np=np.array([0, num_tokens], dtype=np.int32), + has_prefill=True, + batch_desc=BatchExecutionDescriptor( + cg_mode=CUDAGraphMode.NONE, + num_tokens=num_tokens, + num_reqs=1, + num_active_loras=num_active_loras, + num_ubatches=1, + ), + num_active_loras=num_active_loras, + ) + + assert metadata is not None + assert metadata.num_logits_indices == num_tokens + assert metadata.logits_indices_padded.shape[0] == num_tokens + assert metadata.max_logits_per_req == num_tokens + assert manager.dispatch_calls[-1]["num_active_loras"] == num_active_loras + + class _FakeSharedHostRegion: def __init__(self) -> None: self.cleanup_calls = 0 diff --git a/tests/v1/worker/test_encoder_runner.py b/tests/v1/worker/test_encoder_runner.py index f8bd50c6d9..98aaa568f0 100644 --- a/tests/v1/worker/test_encoder_runner.py +++ b/tests/v1/worker/test_encoder_runner.py @@ -9,12 +9,15 @@ and tolerated (token-embedding fallback) when it is not, while a miss within the processed range still fails loudly. """ -from unittest.mock import MagicMock +from types import SimpleNamespace +from unittest.mock import MagicMock, patch import numpy as np import pytest import torch +from vllm.config.compilation import CUDAGraphMode +from vllm.model_executor.models.interfaces import SupportsEncoderCudaGraph from vllm.multimodal.inputs import ( MultiModalFeatureSpec, MultiModalFieldElem, @@ -31,6 +34,56 @@ pytestmark = pytest.mark.cpu_test HIDDEN = 4 +@pytest.mark.parametrize( + "enabled,enforce_eager,supported,expected", + [ + (True, False, True, True), + (False, False, True, False), + (True, True, True, False), + (True, False, False, False), + ], +) +def test_encoder_graph_configuration(enabled, enforce_eager, supported, expected): + """Encoder graph eligibility is independent of decoder graph mode.""" + config = MagicMock() + config.model_config.enforce_eager = enforce_eager + config.model_config.dtype = torch.float32 + config.model_config.get_inputs_embeds_size.return_value = HIDDEN + config.scheduler_config.max_num_batched_tokens = 8 + config.compilation_config.cudagraph_mode = CUDAGraphMode.NONE + config.compilation_config.cudagraph_mm_encoder = enabled + model = MagicMock(spec=SupportsEncoderCudaGraph) if supported else torch.nn.Module() + state = _model_state(EncoderCache()) + with patch("vllm.v1.worker.gpu.model_states.interface.EncoderCudaGraphManager"): + ModelState.__init__(state, config, model, state.encoder_cache, state.device) + assert state.encoder_runner.has_cudagraph() is expected + + +def test_encoder_graph_unsupported_modality_uses_eager_output(): + """An image graph must not intercept another modality's encoder output.""" + from vllm.v1.worker.encoder_cudagraph import EncoderCudaGraphManager + + runner = _make_runner([], []) + output = torch.ones(2, HIDDEN) + runner.model = MagicMock() + runner.model.embed_multimodal.return_value = [output] + manager = MagicMock(spec=EncoderCudaGraphManager) + manager.config = SimpleNamespace(modalities=["image"]) + manager.is_captured.return_value = True + manager.supports_modality.side_effect = lambda modality: ( + EncoderCudaGraphManager.supports_modality(manager, modality) + ) + runner.cudagraph_manager = manager + with patch( + "vllm.v1.worker.gpu.mm.encoder_runner.group_and_batch_mm_kwargs", + return_value=[("audio", 1, {"audio_values": torch.zeros(2)})], + ): + result = runner.execute_mm_encoder([]) + assert len(result) == 1 + assert result[0] is output + manager.execute.assert_not_called() + + def _model_state(cache: EncoderCache) -> MagicMock: """A mock ModelState backed by a real EncoderCache.""" state = MagicMock() diff --git a/tests/v1/worker/test_gpu_bad_words.py b/tests/v1/worker/test_gpu_bad_words.py index 150c4ed007..3cce549536 100644 --- a/tests/v1/worker/test_gpu_bad_words.py +++ b/tests/v1/worker/test_gpu_bad_words.py @@ -14,6 +14,10 @@ if not torch.cuda.is_available(): from vllm.sampling_params import SamplingParams from vllm.v1.worker.gpu.sample.bad_words import BadWordsState +from vllm.v1.worker.gpu.sample.logits_processor import ( + LogitsContext, + LogitsProcRequestState, +) from vllm.v1.worker.gpu.states import RequestState DEVICE = torch.device("cuda") @@ -47,7 +51,7 @@ def _make_state(bad_words_token_ids: list[list[int]]) -> tuple[BadWordsState, in req_states.apply_staged_writes() req_idx = req_states.req_id_to_index["req"] - state = BadWordsState(req_states) + state = BadWordsState(None, LogitsProcRequestState.from_request_state(req_states)) state.add_request(req_idx, SamplingParams(_bad_words_token_ids=bad_words_token_ids)) state.apply_staged_writes() return state, req_idx @@ -61,16 +65,55 @@ def _apply(bad_words_token_ids: list[list[int]]) -> torch.Tensor: expanded_idx_mapping = torch.tensor( [req_idx] * num_logits, dtype=torch.int32, device=DEVICE ) - state.apply_bad_words( + state.apply( logits, - expanded_idx_mapping, - idx_mapping_np, - torch.tensor(INPUT_IDS, dtype=torch.int32, device=DEVICE), - torch.tensor(LOCAL_POS, dtype=torch.int32, device=DEVICE), + LogitsContext( + expanded_idx_mapping=expanded_idx_mapping, + idx_mapping=torch.tensor([req_idx], dtype=torch.int32, device=DEVICE), + idx_mapping_np=idx_mapping_np, + expanded_local_pos=torch.tensor( + LOCAL_POS, dtype=torch.int32, device=DEVICE + ), + input_ids=torch.tensor(INPUT_IDS, dtype=torch.int32, device=DEVICE), + pos=torch.zeros(num_logits, dtype=torch.int32, device=DEVICE), + seq_lens_upper_bound_np=np.full(1, num_logits, dtype=np.int64), + ), ) return logits.cpu() +def test_v2_bad_words_slot_reuse_clears_state(): + """A slot recycled from a bad-words request must not keep masking. + + There is no removal hook by design; safety comes from add_request() + overwriting the slot's state unconditionally. + """ + state, req_idx = _make_state([[10, 11, 60]]) + state.add_request(req_idx, SamplingParams()) + state.apply_staged_writes() + assert int(state.num_bad_words.np[req_idx]) == 0 + + num_logits = len(INPUT_IDS) + idx_mapping = torch.tensor([req_idx], dtype=torch.int32, device=DEVICE) + out = state.apply( + torch.zeros((num_logits, VOCAB_SIZE), device=DEVICE), + LogitsContext( + expanded_idx_mapping=torch.tensor( + [req_idx] * num_logits, dtype=torch.int32, device=DEVICE + ), + idx_mapping=idx_mapping, + idx_mapping_np=np.array([req_idx], dtype=np.intp), + expanded_local_pos=torch.tensor( + LOCAL_POS, dtype=torch.int32, device=DEVICE + ), + input_ids=torch.tensor(INPUT_IDS, dtype=torch.int32, device=DEVICE), + pos=torch.zeros(num_logits, dtype=torch.int32, device=DEVICE), + seq_lens_upper_bound_np=np.full(1, num_logits, dtype=np.int64), + ), + ) + assert not torch.isinf(out).any() + + def test_v2_bad_words_prefix_inside_draft_tokens(): """A prefix matching entirely within the draft tokens must mask the bad word's last token at the draft position that completes the prefix.""" diff --git a/tests/v1/worker/test_gpu_batch_shard.py b/tests/v1/worker/test_gpu_batch_shard.py index 1309686d53..ffb0922098 100644 --- a/tests/v1/worker/test_gpu_batch_shard.py +++ b/tests/v1/worker/test_gpu_batch_shard.py @@ -183,6 +183,12 @@ def test_local_batch_partition(tp_size: int, seed: int): ).all() assert (_np(local.logits_indices) == _np(batch.logits_indices)[rows]).all() assert (_np(local.seq_lens) == _np(batch.seq_lens)[owned]).all() + # Indexed by batch position, like seq_lens: a consumer holding local row + # indices reads another request's length if this is not localised too. + assert ( + _np(local.seq_lens_cpu_upper_bound) + == _np(batch.seq_lens_cpu_upper_bound)[owned] + ).all() assert batch.num_draft_tokens_per_req is not None assert local.num_draft_tokens_per_req is not None assert ( diff --git a/tests/v1/worker/test_gpu_input_batch_v2.py b/tests/v1/worker/test_gpu_input_batch_v2.py index 8937f2dc18..13a92d2c0f 100644 --- a/tests/v1/worker/test_gpu_input_batch_v2.py +++ b/tests/v1/worker/test_gpu_input_batch_v2.py @@ -6,6 +6,9 @@ import numpy as np import pytest import torch +from vllm.config import VllmConfig +from vllm.forward_context import set_forward_context +from vllm.model_executor.layers.fused_moe.router.fused_topk_router import fused_topk from vllm.platforms import current_platform from vllm.v1.attention.backends.utils import get_dcp_local_seq_lens from vllm.v1.worker.gpu import cp_utils @@ -56,6 +59,28 @@ def test_make_dummy_distributes_remainder(num_reqs: int, num_tokens: int): ) +@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA top-k.") +@pytest.mark.parametrize("is_padding", [True, False]) +def test_make_dummy_padding_controls_moe_routing(monkeypatch, is_padding: bool): + """Dummy tokens marked as padding are dropped by the MoE router (top-k id + -1), which is wanted for idle DP ranks. Profile runs must not mark them, or + no token reaches the experts and MoE memory is never profiled.""" + monkeypatch.setenv("VLLM_MOE_SKIP_PADDING", "1") + num_tokens = 16 + buffers = InputBuffers( + max_num_reqs=4, max_num_tokens=num_tokens, device=torch.device(DEVICE) + ) + batch = InputBatch.make_dummy(4, num_tokens, buffers, is_padding=is_padding) + hidden_states = torch.randn(num_tokens, 4, device=DEVICE) + router_logits = torch.randn(num_tokens, 8, device=DEVICE) + + with set_forward_context(None, VllmConfig(), is_padding=batch.is_padding): + _, topk_ids, _ = fused_topk(hidden_states, router_logits, 2, False) + + assert bool((topk_ids == -1).all()) is is_padding + assert bool((topk_ids >= 0).all()) is not is_padding + + def test_maybe_prepare_dcp_local_seq_lens_uses_shared_buffer(monkeypatch): """The batch must view the caller-owned buffer, sliced to padded length. diff --git a/tests/v1/worker/test_gpu_kpool_tail_slot_mapping.py b/tests/v1/worker/test_gpu_kpool_tail_slot_mapping.py new file mode 100644 index 0000000000..8a7445f7d4 --- /dev/null +++ b/tests/v1/worker/test_gpu_kpool_tail_slot_mapping.py @@ -0,0 +1,98 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""GPU regression test for the kpool tail group's slot mapping. + +The kpool tail cache holds one circular block per request, so its block table +row is orders of magnitude shorter than the sequence it serves. +`_compute_slot_mappings_kernel` indexes block tables by +`position // kernel_block_size` with an effectively unmasked load, so +this leads to a read of neighboring requests' rows (and for the later requests, outside +the buffer entirely). +""" + +import pytest +import torch + +from vllm.platforms import current_platform +from vllm.v1.attention.backends.utils import PAD_SLOT_ID +from vllm.v1.kv_cache_interface import KpoolTailSpec +from vllm.v1.worker.block_table import get_block_table_width +from vllm.v1.worker.gpu.block_table import BlockTables + +pytestmark = pytest.mark.skipif( + not current_platform.is_cuda_alike(), + reason="requires CUDA or ROCm", +) + +KPOOL = 4 +MAX_MODEL_LEN = 1 << 20 +MLA_BLOCK_SIZE = 640 +MAX_NUM_REQS = 4 +POISON_BLOCK_ID = 1 << 20 + + +def make_tail_spec(): + return KpoolTailSpec( + block_size=KPOOL, + num_kv_heads=2, + head_size=128, + head_size_v=0, + dtype=torch.bfloat16, + sliding_window=KPOOL, + ) + + +def test_kpool_tail_group_never_position_indexes_its_block_table(): + device = torch.device("cuda") + spec = make_tail_spec() + tail_width = get_block_table_width( + spec.max_num_blocks_per_req(None, MAX_MODEL_LEN), + spec.block_size, + token_alignment=spec.block_table_token_alignment, + ) + mla_width = get_block_table_width(MAX_MODEL_LEN // MLA_BLOCK_SIZE, MLA_BLOCK_SIZE) + + # Positions chosen against the tail row: `in_row` still lands inside + # request 0's own row, `next_row` lands on request 1's poisoned row, and + # `past_table` is past the whole [MAX_NUM_REQS, tail_width] allocation. + in_row = KPOOL - 1 + next_row = tail_width * KPOOL + past_table = MAX_NUM_REQS * tail_width * KPOOL + KPOOL + assert past_table < MAX_MODEL_LEN + positions = [0, in_row, next_row, past_table] + + block_tables = BlockTables( + block_sizes=[MLA_BLOCK_SIZE, spec.block_size], + max_num_reqs=MAX_NUM_REQS, + max_num_batched_tokens=len(positions), + max_num_blocks_per_group=[mla_width, tail_width], + device=device, + kernel_block_sizes=[MLA_BLOCK_SIZE, spec.block_size], + slot_mapping_enabled=[True, spec.uses_slot_mapping], + ) + mla_blocks = list(range(1, mla_width + 1)) + block_tables.append_block_ids(0, (mla_blocks, [7]), overwrite=True) + block_tables.append_block_ids(1, ([2], [POISON_BLOCK_ID]), overwrite=True) + block_tables.apply_staged_writes() + + idx_mapping = torch.zeros(1, dtype=torch.int32, device=device) + query_start_loc = torch.tensor( + [0, len(positions)], dtype=torch.int32, device=device + ) + slot_mappings = block_tables.compute_slot_mappings( + idx_mapping, + query_start_loc, + torch.tensor(positions, dtype=torch.int64, device=device), + num_tokens_padded=len(positions), + ) + torch.accelerator.synchronize() + + tail_slots = slot_mappings[1].tolist() + assert tail_slots == [PAD_SLOT_ID] * len(positions) + assert POISON_BLOCK_ID * KPOOL not in tail_slots + + mla_slots = slot_mappings[0].tolist() + assert mla_slots == [ + mla_blocks[p // MLA_BLOCK_SIZE] * MLA_BLOCK_SIZE + p % MLA_BLOCK_SIZE + for p in positions + ] diff --git a/tests/v1/worker/test_gpu_logit_bias.py b/tests/v1/worker/test_gpu_logit_bias.py index 3bf62380bb..617a361204 100644 --- a/tests/v1/worker/test_gpu_logit_bias.py +++ b/tests/v1/worker/test_gpu_logit_bias.py @@ -14,6 +14,11 @@ if not torch.cuda.is_available(): from vllm.sampling_params import SamplingParams, StructuredOutputsParams from vllm.v1.worker.gpu.sample.logit_bias import LogitBiasState +from vllm.v1.worker.gpu.sample.logits_processor import ( + LogitsContext, + LogitsProcRequestState, +) +from vllm.v1.worker.gpu.states import RequestState DEVICE = torch.device("cuda") VOCAB_SIZE = 128 @@ -45,17 +50,35 @@ def _only_stop_token_left(num_rows: int) -> torch.Tensor: def _apply(logits: torch.Tensor, structured: list[bool]) -> torch.Tensor: """Run the fused bias kernel with one logits row per request.""" - state = LogitBiasState(max_num_reqs=4, device=DEVICE) + req_states = RequestState( + max_num_reqs=4, + max_model_len=64, + max_num_batched_tokens=16, + num_speculative_steps=1, + vocab_size=VOCAB_SIZE, + device=DEVICE, + ) + # add_request() reads prompt_len per slot; every slot here uses the same one. + req_states.prompt_len.np[:] = PROMPT_LEN + + state = LogitBiasState(None, LogitsProcRequestState.from_request_state(req_states)) for req_idx, is_structured in enumerate(structured): - state.add_request(req_idx, PROMPT_LEN, _params(is_structured)) + state.add_request(req_idx, _params(is_structured)) state.apply_staged_writes() num_reqs = len(structured) - state.apply_logit_bias( + idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=DEVICE) + state.apply( logits, - torch.arange(num_reqs, dtype=torch.int32, device=DEVICE), - np.arange(num_reqs, dtype=np.intp), - torch.full((num_reqs,), POS, dtype=torch.int32, device=DEVICE), + LogitsContext( + expanded_idx_mapping=idx_mapping, + idx_mapping=idx_mapping, + idx_mapping_np=np.arange(num_reqs, dtype=np.intp), + expanded_local_pos=torch.zeros(num_reqs, dtype=torch.int32, device=DEVICE), + input_ids=torch.zeros(num_reqs, dtype=torch.int32, device=DEVICE), + pos=torch.full((num_reqs,), POS, dtype=torch.int32, device=DEVICE), + seq_lens_upper_bound_np=np.full(num_reqs, POS + 1, dtype=np.int64), + ), ) return logits.cpu() diff --git a/tests/v1/worker/test_gpu_logits_processors.py b/tests/v1/worker/test_gpu_logits_processors.py new file mode 100644 index 0000000000..f0e5bd43a5 --- /dev/null +++ b/tests/v1/worker/test_gpu_logits_processors.py @@ -0,0 +1,114 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace +from typing import Any + +import numpy as np +import pytest +import torch + +pytest.importorskip("triton") +if not torch.cuda.is_available(): + pytest.skip( + "CUDA required for Model Runner V2 logits processor tests", + allow_module_level=True, + ) + +from vllm.sampling_params import SamplingParams +from vllm.v1.worker.gpu.sample.logits_processor import ( + LogitsContext, + LogitsProcessor, +) +from vllm.v1.worker.gpu.sample.sampler import Sampler +from vllm.v1.worker.gpu.states import RequestState + +DEVICE = torch.device("cuda") +VOCAB_SIZE = 128 +TARGET_TOKEN = 7 +BIAS = 5.0 +BOOST = 100.0 +TEMPERATURE = 2.0 + + +class MockReasoningConfig: + reasoning_start_token_ids = [90] + reasoning_end_token_ids = [91] + natural_reasoning_end_token_ids = [91] + + +class RecordingProcessor(LogitsProcessor): + """Adds BOOST to TARGET_TOKEN and records every lifecycle callback.""" + + def __init__(self, vllm_config: Any, req_state: Any): + self.events: list[str] = [] + self.seen_logits: torch.Tensor | None = None + + def add_request(self, req_idx: int, sampling_params: SamplingParams) -> bool: + self.events.append(f"add_request:{req_idx}") + return True + + def apply_staged_writes(self) -> None: + self.events.append("apply_staged_writes") + + def apply(self, logits: torch.Tensor, ctx: LogitsContext) -> torch.Tensor: + self.events.append("apply") + self.seen_logits = logits.clone() + logits[:, TARGET_TOKEN] += BOOST + return logits + + +def _make_sampler(processor: RecordingProcessor) -> Sampler: + req_states = RequestState( + max_num_reqs=4, + max_model_len=64, + max_num_batched_tokens=16, + num_speculative_steps=1, + vocab_size=VOCAB_SIZE, + device=DEVICE, + ) + return Sampler( + vllm_config=SimpleNamespace(reasoning_config=MockReasoningConfig()), + max_num_reqs=4, + vocab_size=VOCAB_SIZE, + device=DEVICE, + req_states=req_states, + custom_logits_processors=[processor], + ) + + +def test_custom_processor_lifecycle_and_pipeline_position(): + """A custom processor's hooks fire in order, it runs after the built-in + bias stage, and before temperature scaling.""" + proc = RecordingProcessor(None, None) + sampler = _make_sampler(proc) + sampler.add_request( + 0, + SamplingParams(logit_bias={TARGET_TOKEN: BIAS}, temperature=TEMPERATURE), + ) + sampler.apply_staged_writes() + + num_reqs = 1 + idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=DEVICE) + raw = torch.zeros(num_reqs, VOCAB_SIZE, device=DEVICE) + out = sampler.apply_sampling_params( + raw, + expanded_idx_mapping=idx_mapping, + idx_mapping=idx_mapping, + idx_mapping_np=np.arange(num_reqs, dtype=np.intp), + pos=torch.zeros(num_reqs, dtype=torch.int32, device=DEVICE), + input_ids=torch.zeros(num_reqs, dtype=torch.int32, device=DEVICE), + expanded_local_pos=torch.zeros(num_reqs, dtype=torch.int32, device=DEVICE), + seq_lens_upper_bound_np=np.ones(num_reqs, dtype=np.int64), + ) + + assert proc.events == [ + "add_request:0", + "apply_staged_writes", + "apply", + ] + # The processor sees the built-in bias applied but no temperature scaling. + assert proc.seen_logits is not None + assert proc.seen_logits[0, TARGET_TOKEN].item() == BIAS + # Its own modification is then temperature-scaled by the sampler. + assert out[0, TARGET_TOKEN].item() == (BIAS + BOOST) / TEMPERATURE diff --git a/tests/v1/worker/test_gpu_model_runner.py b/tests/v1/worker/test_gpu_model_runner.py index 80e8318fa1..32341ce944 100644 --- a/tests/v1/worker/test_gpu_model_runner.py +++ b/tests/v1/worker/test_gpu_model_runner.py @@ -174,6 +174,20 @@ def test_freeze_gc_disables_and_restores_automatic_gc( gc.disable() +def test_prepare_padding_mask_marks_sequence_parallel_padding(): + runner = GPUModelRunner.__new__(GPUModelRunner) + runner.is_padding = torch.empty(8, dtype=torch.bool) + + mask = runner._prepare_padding_mask(1, 8) + + assert mask.tolist() == [False, True, True, True, True, True, True, True] + assert mask.data_ptr() == runner.is_padding.data_ptr() + + mask = runner._prepare_padding_mask(0, 8) + + assert mask.all() + + @pytest.fixture def model_runner(): vllm_config = get_vllm_config() diff --git a/tests/v1/worker/test_gpu_model_runner_v2.py b/tests/v1/worker/test_gpu_model_runner_v2.py index 308c9fe9ea..83cd1e9d07 100644 --- a/tests/v1/worker/test_gpu_model_runner_v2.py +++ b/tests/v1/worker/test_gpu_model_runner_v2.py @@ -21,6 +21,20 @@ from vllm.v1.worker.gpu.block_table import BlockTables from vllm.v1.worker.gpu.model_runner import GPUModelRunner +def test_prepare_padding_mask_marks_sequence_parallel_padding(): + runner = GPUModelRunner.__new__(GPUModelRunner) + runner.input_buffers = SimpleNamespace(is_padding=torch.empty(8, dtype=torch.bool)) + + mask = runner._prepare_padding_mask(1, 8) + + assert mask.tolist() == [False, True, True, True, True, True, True, True] + assert mask.data_ptr() == runner.input_buffers.is_padding.data_ptr() + + mask = runner._prepare_padding_mask(0, 8) + + assert mask.all() + + def test_qsa_circular_group_uses_custom_slot_mapping(monkeypatch): runner = GPUModelRunner.__new__(GPUModelRunner) runner.max_model_len = 262144 diff --git a/tests/v1/worker/test_gpu_model_runner_v2_eplb.py b/tests/v1/worker/test_gpu_model_runner_v2_eplb.py index 1176857dff..b9cee16ac4 100644 --- a/tests/v1/worker/test_gpu_model_runner_v2_eplb.py +++ b/tests/v1/worker/test_gpu_model_runner_v2_eplb.py @@ -56,7 +56,16 @@ class FakeEplbState: def _make_runner(**overrides: Any) -> Any: runner: Any = mrv2.GPUModelRunner.__new__(mrv2.GPUModelRunner) runner.device = torch.device("cpu") - runner.model_config = SimpleNamespace(model="test-model") + runner.model_config = SimpleNamespace(model="test-model", logits_processors=None) + runner.req_states = SimpleNamespace( + device=runner.device, + max_num_reqs=8, + vocab_size=0, + all_token_ids=None, + prompt_len=None, + prefill_len=None, + total_len=None, + ) runner.load_config = SimpleNamespace(load_format="hf") runner.parallel_config = SimpleNamespace( enable_eplb=True, diff --git a/tests/v1/worker/test_gpu_rejection_sampler_chunking.py b/tests/v1/worker/test_gpu_rejection_sampler_chunking.py index 2e5adfabd1..efd2b59d75 100644 --- a/tests/v1/worker/test_gpu_rejection_sampler_chunking.py +++ b/tests/v1/worker/test_gpu_rejection_sampler_chunking.py @@ -47,6 +47,9 @@ def test_chunked_scores_match_full_batch(logprobs_mode: str): [np.arange(count, dtype=np.int32) for count in num_logits_per_req] ) ).to(device), + seq_lens_cpu_upper_bound=torch.from_numpy( + np.array([10, 20, 30, 40], dtype=np.int32) + ), ) rejection_sampler = object.__new__(RejectionSampler) rejection_sampler.sampler = SimpleNamespace(logprobs_mode=logprobs_mode) diff --git a/tests/v1/worker/test_gpu_sampler_flags.py b/tests/v1/worker/test_gpu_sampler_flags.py index 3fd1cb3b3c..3f05a2cfb5 100644 --- a/tests/v1/worker/test_gpu_sampler_flags.py +++ b/tests/v1/worker/test_gpu_sampler_flags.py @@ -1,6 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from collections.abc import Sequence +from types import SimpleNamespace + import numpy as np import pytest import torch @@ -10,6 +13,10 @@ if not torch.cuda.is_available(): pytest.skip("CUDA required for sampler flag tests", allow_module_level=True) from vllm.sampling_params import SamplingParams +from vllm.v1.worker.gpu.sample.logits_processor import ( + LogitsContext, + LogitsProcessor, +) from vllm.v1.worker.gpu.sample.sampler import Sampler from vllm.v1.worker.gpu.states import RequestState @@ -23,7 +30,7 @@ class MockReasoningConfig: natural_reasoning_end_token_ids = [91] -def _make_sampler() -> Sampler: +def _make_sampler(custom_logits_processors: Sequence[LogitsProcessor] = ()) -> Sampler: req_states = RequestState( max_num_reqs=4, max_model_len=64, @@ -33,11 +40,12 @@ def _make_sampler() -> Sampler: device=DEVICE, ) return Sampler( + vllm_config=SimpleNamespace(reasoning_config=MockReasoningConfig()), max_num_reqs=4, vocab_size=VOCAB_SIZE, device=DEVICE, req_states=req_states, - reasoning_config=MockReasoningConfig(), + custom_logits_processors=custom_logits_processors, ) @@ -65,23 +73,46 @@ def test_logits_processing_cache_matches_request_features( sampling_params: SamplingParams, expected: bool ): sampler = _make_sampler() - sampler.add_request(3, prompt_len=1, sampling_params=sampling_params) + sampler.add_request(3, sampling_params=sampling_params) assert sampler.needs_logits_processing[3] == expected def test_logits_processing_cache_is_overwritten_when_slot_is_reused(): sampler = _make_sampler() - sampler.add_request(3, 1, SamplingParams.for_sampler_warmup()) - sampler.add_request(3, 1, SamplingParams()) + sampler.add_request(3, SamplingParams.for_sampler_warmup()) + sampler.add_request(3, SamplingParams()) assert not sampler.needs_logits_processing[3] +class _GateProcessor(LogitsProcessor): + """Reports a fixed admission decision from add_request().""" + + def __init__(self, admitted: bool): + self.admitted = admitted + + def add_request(self, req_idx: int, sampling_params: SamplingParams) -> bool: + return self.admitted + + def apply(self, logits: torch.Tensor, ctx: LogitsContext) -> torch.Tensor: + raise AssertionError("the pipeline never runs in this test") + + +@pytest.mark.parametrize("admitted", [True, False]) +def test_custom_processor_add_request_gates_pipeline_flag(admitted: bool): + """A custom processor's add_request() return value is OR-ed into the + per-request needs_logits_processing flag.""" + sampler = _make_sampler(custom_logits_processors=[_GateProcessor(admitted)]) + sampler.add_request(3, SamplingParams()) + + assert sampler.needs_logits_processing[3] == admitted + + def test_logits_processing_cache_only_checks_active_requests(): sampler = _make_sampler() - sampler.add_request(0, 1, SamplingParams(temperature=0.0)) - sampler.add_request(2, 1, SamplingParams.for_sampler_warmup()) + sampler.add_request(0, SamplingParams(temperature=0.0)) + sampler.add_request(2, SamplingParams.for_sampler_warmup()) sampling_only = np.array([0], dtype=np.int32) with_processing = np.array([0, 2], dtype=np.int32) diff --git a/tests/v1/worker/test_gpu_thinking_budget.py b/tests/v1/worker/test_gpu_thinking_budget.py index 2bc697b084..2456027335 100644 --- a/tests/v1/worker/test_gpu_thinking_budget.py +++ b/tests/v1/worker/test_gpu_thinking_budget.py @@ -1,6 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from types import SimpleNamespace + +import numpy as np import pytest import torch @@ -12,6 +15,7 @@ if not torch.cuda.is_available(): ) from vllm.sampling_params import SamplingParams +from vllm.v1.worker.gpu.sample.logits_processor import LogitsContext from vllm.v1.worker.gpu.sample.sampler import Sampler from vllm.v1.worker.gpu.sample.thinking_budget import ThinkingBudgetState from vllm.v1.worker.gpu.states import RequestState @@ -75,11 +79,17 @@ def _apply( idx_mapping_np = idx_mapping.cpu().numpy() state.apply( logits, - expanded_idx_mapping, - idx_mapping, - idx_mapping_np, - torch.tensor(input_ids, dtype=torch.int32, device=DEVICE), - torch.tensor(local_pos, dtype=torch.int32, device=DEVICE), + LogitsContext( + expanded_idx_mapping=expanded_idx_mapping, + idx_mapping=idx_mapping, + idx_mapping_np=idx_mapping_np, + expanded_local_pos=torch.tensor( + local_pos, dtype=torch.int32, device=DEVICE + ), + input_ids=torch.tensor(input_ids, dtype=torch.int32, device=DEVICE), + pos=torch.zeros(len(input_ids), dtype=torch.int32, device=DEVICE), + seq_lens_upper_bound_np=np.full(1, len(input_ids), dtype=np.int64), + ), ) return logits.cpu() @@ -190,15 +200,14 @@ def test_v2_greedy_sampling_applies_thinking_budget(): """Greedy-only requests must not bypass thinking-budget processing.""" req_states = _make_req_states([1, START, 10, 11, 12], prompt_len=1) sampler = Sampler( + vllm_config=SimpleNamespace(reasoning_config=MockReasoningConfig()), max_num_reqs=4, vocab_size=VOCAB_SIZE, device=DEVICE, req_states=req_states, - reasoning_config=MockReasoningConfig(), ) sampler.add_request( req_idx=3, - prompt_len=1, sampling_params=SamplingParams( temperature=0.0, thinking_token_budget=3, @@ -219,6 +228,7 @@ def test_v2_greedy_sampling_applies_thinking_budget(): torch.tensor([4], dtype=torch.int32, device=DEVICE), input_ids, torch.tensor([0], dtype=torch.int32, device=DEVICE), + np.full(1, 4, dtype=np.int64), ) assert out[0, END].item() == pytest.approx(1.0e9) diff --git a/tests/v1/worker/test_gpu_ubatch_slicing.py b/tests/v1/worker/test_gpu_ubatch_slicing.py index e735953d8b..9d3a92e623 100644 --- a/tests/v1/worker/test_gpu_ubatch_slicing.py +++ b/tests/v1/worker/test_gpu_ubatch_slicing.py @@ -10,6 +10,7 @@ threaded execution of the microbatches. """ import threading +from contextlib import nullcontext from dataclasses import replace from types import SimpleNamespace from typing import Any, cast @@ -30,6 +31,7 @@ from vllm.config import ( from vllm.forward_context import create_forward_context from vllm.v1.attention.backends.utils import get_dcp_local_seq_lens from vllm.v1.kv_cache_interface import KVCacheConfig +from vllm.v1.worker import gpu_ubatch_wrapper as legacy_gpu_ubatch_wrapper from vllm.v1.worker.gpu import cp_utils as gpu_cp_utils from vllm.v1.worker.gpu import cudagraph_utils, dp_utils from vllm.v1.worker.gpu.cudagraph_utils import ( @@ -426,6 +428,94 @@ def test_slice_model_inputs_handles_mrope_positions(): assert sliced["intermediate_tensors"] is None +@pytest.mark.parametrize( + ("cudagraph_runtime_mode", "execution_method"), + [ + (CUDAGraphMode.FULL, "_capture_ubatches"), + (CUDAGraphMode.NONE, "_run_ubatches"), + ], +) +def test_legacy_ubatch_contexts_slice_padding_mask( + cudagraph_runtime_mode: CUDAGraphMode, execution_method: str +): + """Replacement contexts preserve each microbatch's padding rows.""" + ubatch_slices = [ + UBatchSlice(slice(0, 2), slice(0, 3)), + UBatchSlice(slice(2, 4), slice(3, 6)), + ] + is_padding = torch.tensor([False, True, False, True, False, True]) + forward_context = SimpleNamespace( + batch_descriptor=object(), + ubatch_slices=ubatch_slices, + cudagraph_runtime_mode=cudagraph_runtime_mode, + attn_metadata=[{}, {}], + slot_mapping={}, + dp_metadata=object(), + is_padding=is_padding, + ) + + wrapper = object.__new__(legacy_gpu_ubatch_wrapper.UBatchWrapper) + wrapper.vllm_config = SimpleNamespace( + parallel_config=SimpleNamespace(data_parallel_size=2) + ) + wrapper.comm_stream = object() + wrapper.ready_barrier = object() + wrapper.cudagraphs = {} + wrapper.sm_control = nullcontext() + wrapper.runnable = object() + + def make_contexts(*, forward_contexts, **kwargs): + return [ + SimpleNamespace(id=i, forward_context=context) + for i, context in enumerate(forward_contexts) + ] + + def create_context(attn_metadata, vllm_config, **kwargs): + return SimpleNamespace(attn_metadata=attn_metadata, **kwargs) + + model_inputs = { + "input_ids": torch.arange(6), + "positions": torch.arange(6), + "inputs_embeds": None, + "intermediate_tensors": None, + } + expected = object() + with ( + patch.object( + legacy_gpu_ubatch_wrapper, + "get_forward_context", + return_value=forward_context, + ), + patch.object( + legacy_gpu_ubatch_wrapper.torch.cuda, + "current_stream", + return_value=object(), + ), + patch.object( + legacy_gpu_ubatch_wrapper.DPMetadata, "make", return_value=object() + ), + patch.object( + legacy_gpu_ubatch_wrapper, + "create_forward_context", + side_effect=create_context, + ), + patch.object( + legacy_gpu_ubatch_wrapper, + "make_ubatch_contexts", + side_effect=make_contexts, + ), + patch.object(wrapper, execution_method, return_value=expected) as execute, + ): + result = wrapper(**model_inputs) + + assert result is expected + contexts = [ + metadata.context.forward_context for metadata in execute.call_args.args[0] + ] + torch.testing.assert_close(contexts[0].is_padding, is_padding[:3]) + torch.testing.assert_close(contexts[1].is_padding, is_padding[3:]) + + def _make_dbo_config() -> VllmConfig: return VllmConfig( model_config=ModelConfig(model="facebook/opt-125m", dtype="float16", seed=0), diff --git a/tests/v1/worker/test_gpu_worker_weight_transfer.py b/tests/v1/worker/test_gpu_worker_weight_transfer.py index e576ebd031..537653bda3 100644 --- a/tests/v1/worker/test_gpu_worker_weight_transfer.py +++ b/tests/v1/worker/test_gpu_worker_weight_transfer.py @@ -11,7 +11,7 @@ import pytest import torch import torch.nn as nn -from vllm.config import VllmConfig, get_current_vllm_config +from vllm.config import ParallelConfig, VllmConfig, get_current_vllm_config from vllm.lora.layers import BaseLayerWithLoRA from vllm.v1.worker.gpu_model_runner import _get_parameter_for_reload from vllm.v1.worker.gpu_worker import Worker @@ -136,12 +136,22 @@ def test_rank_local_update_selects_worker_payload(rank, expected): assert worker._weight_update_active is True -def test_rank_local_update_includes_data_parallel_rank(): +def test_rank_local_update_uses_data_parallel_index_after_reconfigure(): engine = _RecordingEngine() worker = _make_worker(engine) worker.rank = 0 - worker.vllm_config.parallel_config.data_parallel_size = 4 - worker.vllm_config.parallel_config.data_parallel_rank = 2 + parallel_config = ParallelConfig( + data_parallel_size=4, + data_parallel_rank=2, + ) + assert parallel_config.data_parallel_rank == 2 + assert parallel_config.data_parallel_index == 2 + + parallel_config.reconfigure_for_independent_dp_rank() + assert parallel_config.data_parallel_rank == 0 + assert parallel_config.data_parallel_index == 2 + + worker.vllm_config.parallel_config = parallel_config Worker.start_weight_update(worker) Worker.update_weights( diff --git a/tests/v1/worker/test_logits_processor_loader.py b/tests/v1/worker/test_logits_processor_loader.py new file mode 100644 index 0000000000..7e851e5237 --- /dev/null +++ b/tests/v1/worker/test_logits_processor_loader.py @@ -0,0 +1,228 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""CPU-only tests for the V2 custom logits processor loader. + +Load-time validation is pure Python (importlib + issubclass), so it runs +without CUDA. Pipeline behavior is covered by test_gpu_logits_processors.py. +""" + +import subprocess +import sys +from enum import Enum, auto +from types import SimpleNamespace +from typing import Any + +import pytest +import torch + +import vllm.v1.worker.gpu.sample.logits_processor.loader as loader +from vllm.exceptions import VLLMValidationError +from vllm.sampling_params import SamplingParams +from vllm.v1.sample.logits_processor import ( + LogitsProcessor as V1LogitsProcessor, +) +from vllm.v1.worker.gpu.sample.logits_processor import ( + LogitsContext, + LogitsProcessor, +) + + +class DummyV2Processor(LogitsProcessor): + """Records its constructor args so tests can assert how it was built.""" + + def __init__(self, vllm_config: Any, req_state: Any): + self.ctor_args = (vllm_config, req_state) + + def apply(self, logits: torch.Tensor, ctx: LogitsContext) -> torch.Tensor: + return logits + + +class AltV2Processor(DummyV2Processor): + """Second V2 processor, used to assert plugin/FQCN load ordering.""" + + +class DummyV1Processor(V1LogitsProcessor): + """A V1-interface processor. Never instantiated: the loader must reject + it at the issubclass check, before any constructor runs.""" + + +def not_a_processor() -> None: + """FQCN target that resolves but is not a class.""" + + +def _fake_entrypoint(cls: type) -> SimpleNamespace: + return SimpleNamespace( + name=cls.__name__.lower(), + value=f"fake:{cls.__name__}", + load=lambda: cls, + ) + + +@pytest.fixture(autouse=True) +def no_installed_plugins(monkeypatch: pytest.MonkeyPatch): + """Isolate tests from entry points registered in the environment.""" + monkeypatch.setattr(loader, "entry_points", lambda group: []) + + +class ProcSource(Enum): + CLASS = auto() + FQCN = auto() + ENTRYPOINT = auto() + + +@pytest.mark.parametrize("source", list(ProcSource)) +def test_loads_v2_processors(monkeypatch: pytest.MonkeyPatch, source: ProcSource): + """A V2 subclass loads via class object, FQCN string, or entrypoint.""" + fake_req_states = SimpleNamespace( + device="cpu", + max_num_reqs=4, + vocab_size=128, + all_token_ids=None, + prompt_len=None, + prefill_len=None, + total_len=None, + ) + custom: list[Any] + if source is ProcSource.CLASS: + custom = [DummyV2Processor] + elif source is ProcSource.FQCN: + custom = [f"{__name__}:DummyV2Processor"] + else: + monkeypatch.setattr( + loader, "entry_points", lambda group: [_fake_entrypoint(DummyV2Processor)] + ) + # Plugins load before user-specified processors. + custom = [f"{__name__}:AltV2Processor"] + + procs = loader.build_custom_logits_processors(None, fake_req_states, False, custom) + + if source is ProcSource.ENTRYPOINT: + assert [type(p) for p in procs] == [DummyV2Processor, AltV2Processor] + else: + assert [type(p) for p in procs] == [DummyV2Processor] + vllm_config, req_state = procs[0].ctor_args + assert vllm_config is None + assert isinstance(req_state, loader.LogitsProcRequestState) + assert req_state.max_num_reqs == 4 + + +@pytest.mark.parametrize( + ("custom", "plugin_cls", "exc_type", "msg_fragment"), + [ + pytest.param( + [DummyV1Processor], + None, + ValueError, + "not a subclass", + id="v1-class-object", + ), + pytest.param( + [f"{__name__}:DummyV1Processor"], + None, + ValueError, + "not a subclass", + id="v1-fqcn", + ), + pytest.param( + [], + DummyV1Processor, + ValueError, + "V1-interface plugins are not supported", + id="v1-entrypoint", + ), + pytest.param( + [f"{__name__}:not_a_processor"], + None, + ValueError, + "must be a type", + id="non-type-fqcn", + ), + pytest.param( + ["no.such.module:Nope"], + None, + RuntimeError, + "no.such.module:Nope", + id="missing-module", + ), + pytest.param( + ["no_colon_here"], + None, + ValueError, + "Expected format", + id="fqcn-missing-colon", + ), + pytest.param( + ["too:many:colons"], + None, + ValueError, + "Expected format", + id="fqcn-extra-colons", + ), + ], +) +def test_rejects_invalid( + monkeypatch: pytest.MonkeyPatch, + custom: list, + plugin_cls: type | None, + exc_type: type[Exception], + msg_fragment: str, +): + """V1 processors and malformed references fail at load time, not at + sampling time, with a message naming the offending reference.""" + if plugin_cls is not None: + monkeypatch.setattr( + loader, "entry_points", lambda group: [_fake_entrypoint(plugin_cls)] + ) + + with pytest.raises(exc_type, match=msg_fragment): + loader.build_custom_logits_processors(None, None, False, custom) + + +def test_pooling_model_rejects_custom_logitsprocs(): + """Pooling models reject custom processors instead of ignoring them.""" + with pytest.raises(ValueError, match="Pooling models do not support"): + loader.build_custom_logits_processors(None, None, True, [DummyV2Processor]) + assert loader.build_custom_logits_processors(None, None, True, []) == [] + + +class ValidatingProcessor(DummyV2Processor): + """Rejects target_token=-1 via the optional validate_params hook.""" + + @classmethod + def validate_params(cls, sampling_params: SamplingParams) -> None: + if (sampling_params.extra_args or {}).get("target_token") == -1: + raise ValueError("target_token must not be -1") + + +def test_validate_params_runs_at_admission(): + """The factory loads classes up front; the returned validator runs + validate_params per request and wraps rejections as VLLMValidationError.""" + loader._cached_load_v2_logitsprocs.cache_clear() + validate = loader.build_custom_logits_processors_params_validator( + [ValidatingProcessor] + ) + validate(SamplingParams(extra_args={"target_token": 1})) + with pytest.raises(VLLMValidationError, match="target_token"): + validate(SamplingParams(extra_args={"target_token": -1})) + + +def test_validator_factory_loads_classes_eagerly(): + """A bad reference fails when the validator is built (at startup), not on + the first request.""" + loader._cached_load_v2_logitsprocs.cache_clear() + with pytest.raises(RuntimeError, match="no.such.module:Nope"): + loader.build_custom_logits_processors_params_validator(["no.such.module:Nope"]) + + +def test_loader_import_stays_frontend_safe(): + """The frontend process imports the loader to validate params; importing + it must not pull in model-runner side modules.""" + code = ( + "import sys\n" + "import vllm.v1.worker.gpu.sample.logits_processor.loader\n" + "assert 'vllm.sampling_params' not in sys.modules\n" + "assert 'vllm.v1.sample.logits_processor' not in sys.modules\n" + "assert 'vllm.v1.worker.gpu.states' not in sys.modules\n" + "assert 'vllm.v1.worker.gpu.buffer_utils' not in sys.modules\n" + ) + subprocess.run([sys.executable, "-c", code], check=True) diff --git a/tests/v1/worker/test_utils.py b/tests/v1/worker/test_utils.py index 55cab287fd..9a345e926c 100644 --- a/tests/v1/worker/test_utils.py +++ b/tests/v1/worker/test_utils.py @@ -531,6 +531,71 @@ def test_hisparse_eager_mirror_records_transfer_without_page_copy(): worker._enqueue_transfers.assert_not_called() +@pytest.mark.parametrize("is_host_writer", [False, True]) +@pytest.mark.parametrize("device", ["cpu", "cuda"]) +def test_hisparse_tail_restore_preserves_imported_rows( + monkeypatch, request, is_host_writer, device +): + """Restore externally pinned host rows on every rank under sync checking.""" + if device == "cuda" and not torch.cuda.is_available(): + pytest.skip("CUDA is required for the device copy check") + worker = _make_hisparse_worker() + worker.is_host_writer = is_host_writer + worker.kernel_block_size = 4 + worker.host_caches = tuple( + torch.arange(24, dtype=torch.float32).reshape(12, 2) + layer * 100 + for layer in range(2) + ) + if device == "cuda": + # Match production's cudaHostRegister pool, which torch.is_pinned() + # does not recognize, rather than torch's caching pinned allocator. + host_pool, registered = hisparse_runtime_module.allocate_pinned_host_pool( + sum(cache.nbytes for cache in worker.host_caches) + ) + request.addfinalizer( + lambda: hisparse_runtime_module.release_pinned_state([], [registered]) + ) + host_caches = host_pool.view(torch.float32).view(2, 12, 2).unbind() + for destination, source in zip(host_caches, worker.host_caches): + destination.copy_(source) + worker.host_caches = host_caches + worker.resident_caches = tuple( + torch.full((4, 4, 2), -1.0, device=device) for _ in range(2) + ) + worker.cache_handles = [ + SimpleNamespace( + runtime=SimpleNamespace(resident_source_index=i, eager_host_mirror=True) + ) + for i in range(2) + ] + worker._record_transfer_completion = MagicMock() + stream = torch.cuda.current_stream() if device == "cuda" else MagicMock() + monkeypatch.setattr(hisparse_worker_module, "current_stream", lambda: stream) + transfer = SparseKVPageTransfer(7, 2, (1, 3), False, restore=True) + + restore_pages = worker._restore_pages + if device == "cuda": + import vllm.utils.gpu_sync_debug as gpu_sync_debug + + monkeypatch.setattr(gpu_sync_debug, "_SYNC_CHECK_MODE", "error") + monkeypatch.setattr(gpu_sync_debug, "_sync_check_enabled", False) + gpu_sync_debug.enable_gpu_sync_check() + restore_pages = gpu_sync_debug.with_gpu_sync_check(restore_pages) + restore_pages([transfer]) + + # Appending into unused capacity must preserve every imported prompt row. + for cache, host, block in zip( + worker.resident_caches, worker.host_caches, transfer.resident_block_ids + ): + assert torch.equal(cache[block].cpu(), host[8:12]) + cache[block, 2:] = 999 + assert torch.equal(cache[block, :2].cpu(), host[8:10]) + assert torch.all(cache[0] == -1) + worker._record_transfer_completion.assert_called_once_with( + [transfer], stream=stream + ) + + def test_hisparse_lazy_mirror_copies_transfer_pages(): worker = _make_hisparse_worker() worker.cache_handles = [ diff --git a/tests/weight_loading/models-amd.txt b/tests/weight_loading/models-amd.txt deleted file mode 100644 index a6a85736cb..0000000000 --- a/tests/weight_loading/models-amd.txt +++ /dev/null @@ -1,18 +0,0 @@ -# quantization, model, revision[, minimum capability[, safetensors load strategy]] -fp8, amd/Meta-Llama-3.1-8B-Instruct-FP8-KV, main -None, amd/Llama-3.2-1B-Instruct-FP8-KV, main -fp8, amd/Mixtral-8x7B-Instruct-v0.1-FP8-KV, main, 80, prefetch -gptq, robertgshaw2/zephyr-7b-beta-channelwise-gptq, main -gptq, TheBloke/Llama-2-7B-GPTQ, main -gptq, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, gptq-8bit--1g-actorder_True -compressed-tensors, nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change, main -compressed-tensors, nm-testing/tinyllama-oneshot-w8-channel-a8-tensor, main -compressed-tensors, nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2, main -compressed-tensors, nm-testing/tinyllama-oneshot-w8a8-channel-dynamic-token-v2, main -compressed-tensors, nm-testing/tinyllama-oneshot-w4a16-group128-v2, main -compressed-tensors, nm-testing/tinyllama-oneshot-w8a16-per-channel, main -compressed-tensors, nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test, main -compressed-tensors, nm-testing/Phi-3-mini-128k-instruct-FP8, main -compressed-tensors, neuralmagic/Phi-3-medium-128k-instruct-quantized.w4a16, main -awq, casperhansen/mixtral-instruct-awq, main -fp8, neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV, main diff --git a/tests/weight_loading/models-large-amd.txt b/tests/weight_loading/models-large-amd.txt deleted file mode 100644 index 41e8ee5200..0000000000 --- a/tests/weight_loading/models-large-amd.txt +++ /dev/null @@ -1,4 +0,0 @@ -# quantization, model, revision[, minimum capability[, safetensors load strategy]] -fp8, amd/Meta-Llama-3.1-70B-Instruct-FP8-KV, main, 80, prefetch -None, microsoft/phi-4, main, 80, prefetch -fp8, amd/Mixtral-8x22B-Instruct-v0.1-FP8-KV, main, 80, prefetch diff --git a/tests/weight_loading/models-large.txt b/tests/weight_loading/models-large.txt deleted file mode 100644 index 8b3913cc2d..0000000000 --- a/tests/weight_loading/models-large.txt +++ /dev/null @@ -1,6 +0,0 @@ -compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-quantized, main -compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-channel-quantized, main -compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W8A16-quantized, main -gptq, TheBloke/Mixtral-8x7B-v0.1-GPTQ, main -awq_marlin, casperhansen/deepseek-coder-v2-instruct-awq, main -compressed-tensors, RedHatAI/Llama-4-Scout-17B-16E-Instruct-quantized.w4a16, main diff --git a/tests/weight_loading/models.txt b/tests/weight_loading/models.txt deleted file mode 100644 index 87b9650870..0000000000 --- a/tests/weight_loading/models.txt +++ /dev/null @@ -1,18 +0,0 @@ -gptq, robertgshaw2/zephyr-7b-beta-channelwise-gptq, main -gptq, TheBloke/Llama-2-7B-GPTQ, main -gptq, LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit, main -gptq, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, gptq-8bit--1g-actorder_True -gptq, TechxGenus/gemma-2b-it-GPTQ, main -compressed-tensors, nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change, main -compressed-tensors, nm-testing/tinyllama-oneshot-w8-channel-a8-tensor, main -compressed-tensors, nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2, main -compressed-tensors, nm-testing/tinyllama-oneshot-w8a8-channel-dynamic-token-v2, main -compressed-tensors, nm-testing/tinyllama-oneshot-w4a16-group128-v2, main -compressed-tensors, nm-testing/tinyllama-oneshot-w8a16-per-channel, main -compressed-tensors, nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test, main -compressed-tensors, nm-testing/Phi-3-mini-128k-instruct-FP8, main -compressed-tensors, neuralmagic/Phi-3-medium-128k-instruct-quantized.w4a16, main -#compressed-tensors, mgoin/DeepSeek-Coder-V2-Lite-Instruct-FP8, main -awq, casperhansen/mixtral-instruct-awq, main -awq_marlin, casperhansen/mixtral-instruct-awq, main -fp8, neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV, main diff --git a/tests/weight_loading/run_model_weight_loading_test.sh b/tests/weight_loading/run_model_weight_loading_test.sh deleted file mode 100755 index 598f1620c7..0000000000 --- a/tests/weight_loading/run_model_weight_loading_test.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -SUCCESS=0 - -while getopts "c:" OPT; do - case ${OPT} in - c ) - CONFIG="$OPTARG" - ;; - \? ) - usage - exit 1 - ;; - esac -done - - -IFS=$'\n' read -d '' -r -a MODEL_CONFIGS < "$CONFIG" - -for MODEL_CONFIG in "${MODEL_CONFIGS[@]}" -do - if [[ $MODEL_CONFIG == \#* ]]; then - echo "=== SKIPPING MODEL: $MODEL_CONFIG ===" - continue - fi - - LOCAL_SUCCESS=0 - IFS=', ' read -r -a array <<< "$MODEL_CONFIG" - - echo "=== RUNNING MODEL: $MODEL_CONFIG ===" - - unset WEIGHT_LOADING_TEST_SAFETENSORS_LOAD_STRATEGY - export QUANTIZATION=${array[0]} - export MODEL_NAME=${array[1]} - export REVISION=${array[2]} - # If array length is larger than 3, then MIN_CAPABILITY is provided - if [ ${#array[@]} -gt 3 ]; then - export MIN_CAPABILITY=${array[3]} - fi - if [ ${#array[@]} -gt 4 ]; then - export WEIGHT_LOADING_TEST_SAFETENSORS_LOAD_STRATEGY=${array[4]} - fi - pytest -s weight_loading/test_weight_loading.py || LOCAL_SUCCESS=$? - - if [[ $LOCAL_SUCCESS == 0 ]]; then - echo "=== PASSED MODEL: ${MODEL_CONFIG} ===" - else - echo "=== FAILED MODEL: ${MODEL_CONFIG} ===" - fi - - SUCCESS=$((SUCCESS + LOCAL_SUCCESS)) - -done - -if [ "${SUCCESS}" -eq "0" ]; then - exit 0 -else - exit 1 -fi diff --git a/tests/weight_loading/test_weight_loading.py b/tests/weight_loading/test_weight_loading.py deleted file mode 100644 index c32ce71aca..0000000000 --- a/tests/weight_loading/test_weight_loading.py +++ /dev/null @@ -1,49 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import os - -import pytest -import torch - -from vllm.platforms import current_platform -from vllm.utils.mem_constants import GiB_bytes - -MAX_MODEL_LEN = 1024 -MODEL_NAME = os.environ.get( - "MODEL_NAME", "robertgshaw2/zephyr-7b-beta-channelwise-gptq" -) -REVISION = os.environ.get("REVISION", "main") -QUANTIZATION = os.environ.get("QUANTIZATION", "gptq_marlin") -MIN_CAPABILITY = os.environ.get("MIN_CAPABILITY", "80") -SAFETENSORS_LOAD_STRATEGY = os.environ.get( - "WEIGHT_LOADING_TEST_SAFETENSORS_LOAD_STRATEGY" -) - - -@pytest.mark.skipif( - MODEL_NAME == "casperhansen/deepseek-coder-v2-instruct-awq", reason="OOM in the CI" -) -@pytest.mark.skipif( - not current_platform.has_device_capability(int(MIN_CAPABILITY)), - reason="Current system does not have minimum capability.", -) -def test_weight_loading(vllm_runner): - """Test parameter weight loading with tp>1.""" - # MoE models need fp16. - NEEDS_FP16 = QUANTIZATION == "gptq" - with vllm_runner( - model_name=MODEL_NAME, - revision=REVISION, - dtype=torch.half if NEEDS_FP16 else "auto", - quantization=None if QUANTIZATION == "None" else QUANTIZATION, - max_model_len=MAX_MODEL_LEN, - tensor_parallel_size=2, - safetensors_load_strategy=SAFETENSORS_LOAD_STRATEGY, - # Generating 20 tokens needs a tiny KV cache, while sizing and - # allocating a full-size one dominates this test on large devices. - kv_cache_memory_bytes=2 * GiB_bytes, - ) as model: - output = model.generate_greedy("Hello world!", max_tokens=20) - print(output) - assert output diff --git a/tools/build_rust.py b/tools/build_rust.py index a000ec8169..320cf75e3d 100644 --- a/tools/build_rust.py +++ b/tools/build_rust.py @@ -16,13 +16,28 @@ from setuptools_scm import get_version ROOT_DIR = Path(__file__).resolve().parents[1] VLLM_RS_BUILD_VERSION = "VLLM_RS_BUILD_VERSION" +# Select vLLM release tags, excluding crate tags such as "proto-v0.3.0". +VLLM_GIT_DESCRIBE_COMMAND = [ + "git", + "describe", + "--dirty", + "--tags", + "--long", + "--abbrev=40", + "--match", + "v[0-9]*", +] + def prepare_build_environment() -> str | None: """Set the device-independent vLLM source version for Rust artifacts.""" version = os.getenv(VLLM_RS_BUILD_VERSION) or None if version is None: try: - version = get_version(root=ROOT_DIR) + version = get_version( + root=ROOT_DIR, + git_describe_command=VLLM_GIT_DESCRIBE_COMMAND, + ) except LookupError: return None diff --git a/tools/pre_commit/mypy.py b/tools/pre_commit/mypy.py index b7a278b9bd..71dd929d37 100755 --- a/tools/pre_commit/mypy.py +++ b/tools/pre_commit/mypy.py @@ -113,10 +113,7 @@ SEPARATE_GROUPS = [ EXCLUDE = [ r"vllm/model_executor/models/[kK]", - r"vllm/model_executor/models/[nN]", - r"vllm/model_executor/models/[oO]", r"vllm/model_executor/models/[qQ]", - r"vllm/model_executor/models/transformers", r"vllm/model_executor/models/[uU]", r"vllm/model_executor/models/[vV]", r"vllm/model_executor/models/[wW]", diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index 101e1dd10a..c514b7c0bf 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -4,9 +4,10 @@ import ctypes import functools import os from collections.abc import Callable -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import torch +import torch.distributed as dist from torch._ops import OpOverload import vllm.envs as envs @@ -34,6 +35,63 @@ except ImportError: # which is a host op, so we cache it once here. FP8_DTYPE = current_platform.fp8_dtype() _HIPB_MM_INITIALIZED_DEVICES: set[int] = set() +KB = 1024 +MB = 1024 * KB + + +def _get_or_create_aiter_qr_rmsnorm_comm( + device_comm: Any, +) -> tuple[int, int, bool] | None: + """Create the AITER QR communicator used by fused QR+RMSNorm. + + vLLM's normal QuickReduce communicator owns vLLM C++ QR buffers. The fused + QR+RMSNorm kernel added in AITER needs an AITER DeviceComms instance, so we + lazily create one on the same TP CPU group and keep it on the vLLM device + communicator for reuse. + """ + existing = getattr(device_comm, "_aiter_qr_rmsnorm_comm", None) + if existing is False: + return None + if existing is not None: + return existing + + try: + import aiter + except Exception: + device_comm._aiter_qr_rmsnorm_comm = False + return None + + if not hasattr(aiter, "qr_all_reduce_rmsnorm"): + device_comm._aiter_qr_rmsnorm_comm = False + return None + + group = getattr(device_comm, "cpu_group", None) + if group is None: + device_comm._aiter_qr_rmsnorm_comm = False + return None + + world_size = dist.get_world_size(group=group) + rank = dist.get_rank(group=group) + if world_size not in (2, 4, 8): + device_comm._aiter_qr_rmsnorm_comm = False + return None + + qr_max_size_mb = envs.VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB + qr_max_size = None if qr_max_size_mb is None else qr_max_size_mb * MB + + try: + ptr = aiter.init_custom_qr(rank, world_size, qr_max_size) + handle = aiter.qr_get_handle(ptr) + handles = [None] * world_size + dist.all_gather_object(handles, handle, group=group) + aiter.qr_open_handles(ptr, handles) + except Exception: + device_comm._aiter_qr_rmsnorm_comm = False + return None + + comm = (ptr, world_size, envs.VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16) + device_comm._aiter_qr_rmsnorm_comm = comm + return comm def _ensure_hipb_mm_extension_initialized() -> None: @@ -991,6 +1049,53 @@ def _rocm_aiter_fused_allreduce_rmsnorm_impl( ca = aiter_ar.aiter_ca use_1stage = aiter_ar.use_1stage_fused_ar_rms(input_) + if not use_1stage: + from vllm.distributed import get_tp_group + + device_comm = get_tp_group().device_communicator + qr_comm = getattr(device_comm, "qr_comm", None) + hidden_dim = input_.shape[-1] + row_size = hidden_dim * input_.element_size() + fused_qr_rmsnorm_ok = ( + qr_comm is not None + and not getattr(qr_comm, "disabled", True) + and hasattr(qr_comm, "should_quick_allreduce") + and qr_comm.should_quick_allreduce(input_) + and input_.shape == residual.shape + and input_.dtype in (torch.bfloat16, torch.float16) + and input_.dtype == residual.dtype + and input_.dtype == weight.dtype + and weight.dim() == 1 + and weight.numel() == hidden_dim + and input_.numel() % hidden_dim == 0 + and row_size > 0 + and row_size <= 32 * KB + and (32 * KB) % row_size == 0 + ) + if fused_qr_rmsnorm_ok: + assert qr_comm is not None + aiter_qr_comm = _get_or_create_aiter_qr_rmsnorm_comm(device_comm) + if aiter_qr_comm is not None: + import aiter + + ptr, _, cast_bf2half = aiter_qr_comm + quant_level = qr_comm._get_qr_quant_level(input_) + out = torch.empty_like(input_) + residual_out = torch.empty_like(residual) + aiter.qr_all_reduce_rmsnorm( + ptr, + input_, + residual, + residual_out, + out, + weight, + epsilon, + hidden_dim, + quant_level, + cast_bf2half, + ) + return out, residual_out + result = ca.custom_fused_ar_rms( input_, residual, @@ -1788,6 +1893,7 @@ class rocm_aiter_ops: VLLM_ROCM_USE_AITER_MHA: Controls MHA ops including flash_attn_varlen. VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION: Controls Triton unified attention. VLLM_ROCM_USE_AITER_FP8BMM: Controls FP8 batched matrix multiply. + VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: Controls FP4 assembly GEMM. VLLM_ROCM_USE_AITER_TRITON_ROPE: Controls Triton rotary embeddings. VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: Controls shared expert fusion. VLLM_ROCM_USE_AITER_MOE_SITUV2: Controls SiTUv2 FlyDSL MoE (a4w4). @@ -1855,6 +1961,8 @@ class rocm_aiter_ops: _FP8BMM_ENABLED = envs.VLLM_ROCM_USE_AITER_FP8BMM _FP4BMM_ENABLED = envs.VLLM_ROCM_USE_AITER_FP4BMM _LINEAR_HIPBMM_ENABLED = envs.VLLM_ROCM_USE_AITER_LINEAR_HIPBMM + # TODO: Consolidate under _LINEAR_ENABLED + _FP4_GEMM_DYNAMIC_QUANT_ASM = envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM # TODO: Consolidate under VLLM_ROCM_USE_AITER_ROPE _TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE _MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS @@ -1884,6 +1992,7 @@ class rocm_aiter_ops: cls._FP8BMM_ENABLED = envs.VLLM_ROCM_USE_AITER_FP8BMM cls._FP4BMM_ENABLED = envs.VLLM_ROCM_USE_AITER_FP4BMM cls._LINEAR_HIPBMM_ENABLED = envs.VLLM_ROCM_USE_AITER_LINEAR_HIPBMM + cls._FP4_GEMM_DYNAMIC_QUANT_ASM = envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM cls._TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE cls._MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS cls._MOE_SITUV2 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2 @@ -2125,7 +2234,7 @@ class rocm_aiter_ops: def is_asm_fp4_gemm_dynamic_quant_enabled(cls) -> bool: from vllm.platforms.rocm import on_gfx950 - return cls._AITER_ENABLED and on_gfx950() + return cls._AITER_ENABLED and cls._FP4_GEMM_DYNAMIC_QUANT_ASM and on_gfx950() @classmethod @if_aiter_supported diff --git a/vllm/_custom_ops.py b/vllm/_custom_ops.py index 96f47e2069..2a1654231f 100644 --- a/vllm/_custom_ops.py +++ b/vllm/_custom_ops.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from enum import IntEnum +from functools import cache from typing import TYPE_CHECKING, Literal import torch @@ -3384,6 +3385,7 @@ class CPUQuantMethod(IntEnum): FP8_W8A16 = 2 INT4_W4A8 = 3 MXFP4 = 4 + FP8_W8A8 = 5 if hasattr(torch.ops._C, "dynamic_4bit_int_moe"): @@ -3422,6 +3424,7 @@ def fused_experts_cpu( alpha: float | None = None, limit: float | None = None, is_vnni: bool = True, + a1_scale: torch.Tensor | None = None, ) -> None: torch.ops._C.fused_experts_cpu( out, @@ -3435,6 +3438,7 @@ def fused_experts_cpu( w2_scale, w1_zero, w2_zero, + a1_scale, block_size, w1_bias, w2_bias, @@ -3564,6 +3568,46 @@ def fp8_scaled_mm_cpu( ) +# FP8 W8A8 CPU kernels +@cache +def cpu_has_amx_fp8() -> bool: + """Whether this CPU has native AMX-FP8 MMA support.""" + if not hasattr(torch.ops._C, "cpu_has_amx_fp8"): + return False + return bool(torch.ops._C.cpu_has_amx_fp8()) + + +if hasattr(torch.ops._C, "fp8_scaled_mm_with_quant"): + + @register_fake("_C::fp8_scaled_mm_with_quant") + def fp8_scaled_mm_with_quant_fake( + act: torch.Tensor, + act_scales: torch.Tensor | None, + channelwise: bool, + weight: torch.Tensor, + weight_scales: torch.Tensor, + bias: torch.Tensor | None, + output_dtype: torch.dtype, + ) -> torch.Tensor: + M = act.reshape(-1, act.size(-1)).size(0) + N = weight.size(0) * weight.size(-1) + return torch.empty((M, N), dtype=output_dtype, device=act.device) + + +def fp8_scaled_mm_with_quant( + act: torch.Tensor, + act_scales: torch.Tensor | None, + channelwise: bool, + weight: torch.Tensor, + weight_scales: torch.Tensor, + bias: torch.Tensor | None, + output_dtype: torch.dtype, +) -> torch.Tensor: + return torch.ops._C.fp8_scaled_mm_with_quant( + act, act_scales, channelwise, weight, weight_scales, bias, output_dtype + ) + + def chunk_gated_delta_rule_cpu( query: torch.Tensor, key: torch.Tensor, diff --git a/vllm/compilation/decorators.py b/vllm/compilation/decorators.py index 242e7227e9..35c0e6dee5 100644 --- a/vllm/compilation/decorators.py +++ b/vllm/compilation/decorators.py @@ -7,7 +7,7 @@ import inspect import os import sys from collections.abc import Callable, Generator -from typing import TYPE_CHECKING, Any, TypeVar, overload +from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar, overload from unittest.mock import patch import torch @@ -32,6 +32,9 @@ from vllm.utils.torch_utils import is_torch_equal_or_newer from .monitor import monitor_profiling_run, monitor_torch_compile +DynamicArgDims: TypeAlias = dict[str, int | list[int] | dict[int, str]] +"""Argument name -> the dynamic dimension(s) of that argument.""" + # shape_id parameter was added to mark_unbacked in PyTorch 2.11.0 _SUPPORTS_SHAPE_ID = is_torch_equal_or_newer("2.11.0") @@ -89,7 +92,7 @@ def support_torch_compile( @overload def support_torch_compile( *, - dynamic_arg_dims: dict[str, int | list[int] | dict[int, str]] | None, + dynamic_arg_dims: DynamicArgDims | None, ) -> Callable[[type[_T]], type[_T]]: ... @@ -103,7 +106,7 @@ def support_torch_compile( @overload def support_torch_compile( *, - dynamic_arg_dims: dict[str, int | list[int] | dict[int, str]] | None = None, + dynamic_arg_dims: DynamicArgDims | None = None, mark_unbacked_dims: dict[str, int | list[int]] | None = None, enable_if: Callable[[VllmConfig], bool] | None = None, is_encoder: bool = False, @@ -117,7 +120,7 @@ def support_torch_compile(cls: type[_T]) -> type[_T]: ... def support_torch_compile( cls: type[_T] | None = None, *, - dynamic_arg_dims: dict[str, int | list[int] | dict[int, str]] | None = None, + dynamic_arg_dims: DynamicArgDims | None = None, mark_unbacked_dims: dict[str, int | list[int]] | None = None, enable_if: Callable[[VllmConfig], bool] | None = None, is_encoder: bool = False, @@ -328,7 +331,7 @@ def _try_load_aot_compiled_fn( def _support_torch_compile( cls: type[_T], - dynamic_arg_dims: dict[str, int | list[int] | dict[int, str]], + dynamic_arg_dims: DynamicArgDims, mark_unbacked_dims: dict[str, int | list[int]] | None = None, enable_if: Callable[[VllmConfig], bool] | None = None, is_encoder: bool = False, diff --git a/vllm/config/cache.py b/vllm/config/cache.py index 37ad5b2b34..c12433a367 100644 --- a/vllm/config/cache.py +++ b/vllm/config/cache.py @@ -227,6 +227,11 @@ class CacheConfig: necessary for implementing this optimization in some models (e.g. Gemma3n) """ + swa_bounded_replay: bool = True + """Keep the sliding-window KV of models that support it (DeepSeek-V4.1) + out of prefix caching and rebuild it after a prefix hit by recomputing the + hit's last window. Requires model runner V2.""" + kv_cache_memory_bytes: int | None = None """Size of KV Cache per GPU in bytes. By default, this is set to None and vllm can automatically infer the kv cache size based on @@ -281,8 +286,9 @@ class CacheConfig: "effective_attention_block_size", "kv_cache_size_tokens", "kv_cache_max_concurrency", - # WIP feature toggle not impacting compiled graph shape + # Feature toggles not impacting compiled graph shape "kv_sharing_fast_prefill", + "swa_bounded_replay", } from vllm.config.utils import get_hash_factors, hash_factors diff --git a/vllm/config/multimodal.py b/vllm/config/multimodal.py index 09e184b8ce..4eca42430c 100644 --- a/vllm/config/multimodal.py +++ b/vllm/config/multimodal.py @@ -3,11 +3,18 @@ from collections.abc import Mapping from pathlib import Path -from typing import Any, Literal, TypeAlias, TypedDict, final +from typing import Any, Literal, TypeAlias, TypeVar, final, overload import torch -from pydantic import ConfigDict, Field, field_validator, model_validator +from pydantic import ( + ConfigDict, + Field, + GetCoreSchemaHandler, + field_validator, + model_validator, +) from pydantic.dataclasses import dataclass +from pydantic_core import core_schema import vllm.envs as envs from vllm.config.ec_transfer import ECTransferConfig @@ -18,6 +25,8 @@ from vllm.v1.attention.backends.registry import AttentionBackendEnum logger = init_logger(__name__) +_T = TypeVar("_T") + @dataclass class BaseDummyOptions: @@ -51,17 +60,50 @@ class AudioDummyOptions(BaseDummyOptions): @final -class MultiModalDummyOptionsBuiltins(TypedDict, total=False): - """Type annotations for modality types predefined by vLLM.""" +class MultiModalDummyOptions(dict[str, BaseDummyOptions]): + """Dummy data options for each modality. - image: ImageDummyOptions - """Options for dummy images.""" + Lookups of the modalities predefined by vLLM return their own options + class, while any other modality returns + [`BaseDummyOptions`][vllm.config.multimodal.BaseDummyOptions]. + """ - video: VideoDummyOptions - """Options for dummy videos.""" + @overload # type: ignore[override] + def get(self, key: Literal["image"], /) -> ImageDummyOptions | None: ... - audio: AudioDummyOptions - """Options for dummy audios.""" + @overload + def get(self, key: Literal["image"], default: _T, /) -> ImageDummyOptions | _T: ... + + @overload + def get(self, key: Literal["video"], /) -> VideoDummyOptions | None: ... + + @overload + def get(self, key: Literal["video"], default: _T, /) -> VideoDummyOptions | _T: ... + + @overload + def get(self, key: Literal["audio"], /) -> AudioDummyOptions | None: ... + + @overload + def get(self, key: Literal["audio"], default: _T, /) -> AudioDummyOptions | _T: ... + + @overload + def get(self, key: str, /) -> BaseDummyOptions | None: ... + + @overload + def get(self, key: str, default: _T, /) -> BaseDummyOptions | _T: ... + + def get(self, key: str, default: object = None, /) -> object: + return super().get(key, default) + + @classmethod + def __get_pydantic_core_schema__( + cls, + source_type: Any, + handler: GetCoreSchemaHandler, + ) -> core_schema.CoreSchema: + return core_schema.no_info_after_validator_function( + cls, handler.generate_schema(dict[str, BaseDummyOptions]) + ) MMEncoderTPMode = Literal["weights", "data"] @@ -75,15 +117,6 @@ MMProcessorDevice: TypeAlias = str `"xpu"` on XPU). Validated against that set by the CLI.""" -MMDummyOptions: TypeAlias = dict[str, BaseDummyOptions] -""" -A dictionary containing an entry for each modality type of dummy data. - -The built-in modalities are defined by -[`MultiModalDummyOptionsBuiltins`][vllm.config.multimodal.MultiModalDummyOptionsBuiltins]. -""" - - @config class MultiModalConfig: """Controls the behavior of multimodal models.""" @@ -91,7 +124,9 @@ class MultiModalConfig: language_model_only: bool = False """If True, disables all multimodal inputs by setting all modality limits to 0. Equivalent to setting `--limit-mm-per-prompt` to 0 for every modality.""" - limit_per_prompt: MMDummyOptions = Field(default_factory=dict) + limit_per_prompt: MultiModalDummyOptions = Field( + default_factory=MultiModalDummyOptions + ) """The maximum number of input items and options allowed per prompt for each modality. @@ -257,8 +292,8 @@ class MultiModalConfig: def _validate_limit_per_prompt( cls, value: dict[str, int | dict[str, int]], - ) -> MMDummyOptions: - out: MMDummyOptions = {} + ) -> MultiModalDummyOptions: + out = MultiModalDummyOptions() for k, v in value.items(): # Handle legacy format where only count is specified diff --git a/vllm/config/parallel.py b/vllm/config/parallel.py index ba744e049f..666f2441ae 100644 --- a/vllm/config/parallel.py +++ b/vllm/config/parallel.py @@ -49,6 +49,7 @@ All2AllBackend = Literal[ "deepep_v2", "mori_high_throughput", "mori_low_latency", + "moonep", "nixl_ep", "allgather_reducescatter", "flashinfer_all2allv", # temporary alias for flashinfer_nvlink_two_sided @@ -206,6 +207,7 @@ class ParallelConfig: - "deepep_low_latency": Use deepep low-latency kernels - "mori_high_throughput": MoRI EP with InterNodeV1 for multi-node - "mori_low_latency": MoRI EP with InterNodeV1LL for multi-node + - "moonep": MoonEP balanced EP with dynamic redundant experts (NVLink) - "nixl_ep": Use nixl-ep kernels - "flashinfer_nvlink_one_sided": Use flashinfer high-throughput a2a kernels - "flashinfer_nvlink_two_sided": Use flashinfer two-sided kernels for mnnvl""" diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index e5710dced3..78e30cf9d0 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -1881,6 +1881,55 @@ class VllmConfig: # (e.g., XPU may lower max_num_batched_tokens when MLA is enabled) self._set_compile_ranges() + if self.parallel_config.all2all_backend == "moonep": + if ( + self.model_config is not None + and self.model_config.quantization is not None + ): + raise ValueError( + "The moonep all2all backend currently supports unquantized " + "BF16 models only; got " + f"quantization={self.model_config.quantization!r}. Use a " + "different --all2all-backend for quantized models." + ) + if ( + self.model_config is not None + and self.model_config.dtype != torch.bfloat16 + ): + raise ValueError( + "The moonep all2all backend currently supports BF16 models " + f"only; got dtype={self.model_config.dtype}. Use a " + "different --all2all-backend or --dtype bfloat16." + ) + if self.parallel_config.enable_eplb: + raise ValueError( + "The moonep all2all backend does not support EPLB yet: " + "EPLB rearranges expert parameters in a layout MoonEP's " + "replicated [E+B] weights do not follow. Disable " + "--enable-eplb or use a different --all2all-backend." + ) + if self.parallel_config.expert_placement_strategy != "linear": + raise ValueError( + "The moonep all2all backend requires linear expert " + "placement: its load-time all-gather assumes each rank " + "holds a contiguous chunk of the global expert range. Got " + "--expert-placement-strategy " + f"{self.parallel_config.expert_placement_strategy!r}." + ) + # Enforced here rather than in set_splitting_ops_for_v1 so it + # holds for every compilation mode, and keyed on use_all2all so + # PCP/SP-only topologies are covered too: MoonEP dispatch/combine + # are eager-only and must not be captured. + if ( + self.parallel_config.use_all2all + and self.compilation_config.cudagraph_mode != CUDAGraphMode.NONE + ): + logger.info( + "MoonEP: Disabling CUDA Graphs since the MoonEP " + "integration is currently eager-only." + ) + self.compilation_config.cudagraph_mode = CUDAGraphMode.NONE + # Do this after all the updates to compilation_config.mode effective_dp_size = ( self.parallel_config.data_parallel_size @@ -2223,9 +2272,9 @@ class VllmConfig: max_cudagraph_capture_size = ( self.compilation_config.max_cudagraph_capture_size ) - # Decode sizes to cover, in tokens. Populated only when a request - # is more than one token wide and only when the default is computed - # here, so an explicit capture range is left exactly as configured. + # Decode sizes to cover, in tokens. Populated only when the default + # is computed here, so an explicit capture range is left exactly as + # configured. uniform_decode_sizes: list[int] = [] if max_cudagraph_capture_size is None: from vllm.platforms import current_platform @@ -2316,6 +2365,8 @@ class VllmConfig: if n * query_len <= max_cudagraph_capture_size } ) + elif max_num_seqs <= max_cudagraph_capture_size: + uniform_decode_sizes = [max_num_seqs] max_num_tokens = self.scheduler_config.max_num_batched_tokens max_cudagraph_capture_size = min(max_num_tokens, max_cudagraph_capture_size) @@ -2829,7 +2880,6 @@ class VllmConfig: def _get_v2_model_runner_unsupported_features(self) -> list[str]: """Collect features not yet supported by the V2 model runner.""" unsupported: list[str] = [] - model_config = self.model_config speculative_config = self.speculative_config if self.compilation_config.mode == CompilationMode.STOCK_TORCH_COMPILE: @@ -2878,17 +2928,6 @@ class VllmConfig: if self.parallel_config.enable_elastic_ep: unsupported.append("elastic expert parallelism") - has_logitsproc_plugins = False - if model_config is not None: - from importlib.metadata import entry_points - - has_logitsproc_plugins = bool(entry_points(group="vllm.logits_processors")) - - if model_config is not None and ( - model_config.logits_processors or has_logitsproc_plugins - ): - unsupported.append("custom logits processors") - if self.cache_config.mamba_cache_mode == "all": unsupported.append("mamba cache mode 'all'") diff --git a/vllm/device_allocator/sleep_mode_backend.py b/vllm/device_allocator/sleep_mode_backend.py index 45a635d613..023af04275 100644 --- a/vllm/device_allocator/sleep_mode_backend.py +++ b/vllm/device_allocator/sleep_mode_backend.py @@ -73,6 +73,13 @@ class SleepModeBackend(ABC): (suspended) engine from a healthy-serving one (see RFC #34303).""" return self._state + def discard(self, tags: tuple[str, ...]) -> None: + """Discard selected tagged allocations without preserving contents.""" + raise RuntimeError( + f"Sleep-mode backend {type(self).__name__!r} does not support " + "selective discard." + ) + # -- Capability introspection (no instance required) -- @classmethod @@ -132,6 +139,13 @@ class CuMemBackend(SleepModeBackend): allocator.wake_up(tags) self._state = "RUNNING" + def discard(self, tags: tuple[str, ...]) -> None: + from vllm.device_allocator import get_mem_allocator_instance + + allocator = get_mem_allocator_instance() + allocator.discard(tags) + self._state = "SUSPENDED" + @classmethod def preserves_communicators(cls) -> bool: # Comm identity/topology survive memory suspension; no reinit needed. diff --git a/vllm/distributed/device_communicators/all2all.py b/vllm/distributed/device_communicators/all2all.py index 0c8ebf3792..9e53c12e24 100644 --- a/vllm/distributed/device_communicators/all2all.py +++ b/vllm/distributed/device_communicators/all2all.py @@ -21,7 +21,12 @@ from vllm.utils.flashinfer import ( has_flashinfer_nvlink_two_sided, ) from vllm.utils.func_utils import supports_kw -from vllm.utils.import_utils import has_deep_ep, has_deep_ep_v2, has_mori +from vllm.utils.import_utils import ( + check_moonep_system_support, + has_deep_ep, + has_deep_ep_v2, + has_mori, +) from .base_device_communicator import All2AllManagerBase, Cache @@ -1094,3 +1099,71 @@ class DeepEPV2All2AllManager(All2AllManagerBase): for _, handle in self.handle_cache._cache.items(): handle.destroy() self.handle_cache._cache.clear() + + +class MoonEPAll2AllManager(All2AllManagerBase): + """All2All communication based on MoonEP + (https://github.com/MoonshotAI/MoonEP). + + MoonEP keeps token loads perfectly balanced across EP ranks by planning + a small number of dynamically redundant experts online and prefetching + their weights before expert compute. Every rank receives exactly + S x K token slots regardless of router skew, so all communication and + compute shapes are static. + + Requires NVLink symmetric-memory / multicast capable topologies + (single node NVSwitch, e.g. H100/H200/B200/GB300 class). + """ + + def __init__(self, cpu_group, tcp_store_group=None, device_group=None): + check_moonep_system_support() + super().__init__(cpu_group, tcp_store_group) + self._device_group = device_group + self.handle_cache = Cache() + + def _make_buffer_kwargs( + self, + max_num_tokens_per_dp_rank: int, + token_hidden_size: int, + num_topk: int, + num_global_experts: int, + num_prefetch_slots: int, + token_padding: int, + num_sms: int, + ) -> dict: + return dict( + S=max_num_tokens_per_dp_rank, + H=token_hidden_size, + K=num_topk, + E=num_global_experts, + num_ep_ranks=self.world_size, + num_sms=num_sms, + token_padding=token_padding, + B=num_prefetch_slots, + group=self._device_group + if self._device_group is not None + else self.cpu_group, + explicitly_destroy=True, + ) + + def get_handle(self, kwargs): + from vllm.model_executor.layers.fused_moe.prepare_finalize.moonep import ( + MoonEPBufferPool, + ) + + buffer_kwargs = self._make_buffer_kwargs(**kwargs) + logger.debug("MoonEP all2all args %s", buffer_kwargs) + + def make_pool(**kw): + return MoonEPBufferPool(kw, max_tokens_per_rank=kw["S"]) + + handle: MoonEPBufferPool = self.handle_cache.get_or_create( + buffer_kwargs, make_pool + ) + return handle + + def destroy(self): + with self.handle_cache._lock: + for _, handle in self.handle_cache._cache.items(): + handle.destroy() + self.handle_cache._cache.clear() diff --git a/vllm/distributed/device_communicators/cuda_communicator.py b/vllm/distributed/device_communicators/cuda_communicator.py index fb4ccd9f1e..fb0c281b4a 100644 --- a/vllm/distributed/device_communicators/cuda_communicator.py +++ b/vllm/distributed/device_communicators/cuda_communicator.py @@ -214,6 +214,14 @@ class CudaCommunicator(DeviceCommunicatorBase): tcp_store_group, device_group=self.device_group, ) + elif self.all2all_backend == "moonep": + from .all2all import MoonEPAll2AllManager + + self.all2all_manager = MoonEPAll2AllManager( + self.cpu_group, + tcp_store_group, + device_group=self.device_group, + ) elif self.all2all_backend == "nixl_ep": from .all2all import NixlEPAll2AllManager diff --git a/vllm/distributed/device_communicators/cuda_wrapper.py b/vllm/distributed/device_communicators/cuda_wrapper.py index a5026163e9..d27d84c030 100644 --- a/vllm/distributed/device_communicators/cuda_wrapper.py +++ b/vllm/distributed/device_communicators/cuda_wrapper.py @@ -78,6 +78,16 @@ class CudaRTLibrary: cudaError_t, [ctypes.POINTER(ctypes.c_void_p), cudaIpcMemHandle_t, ctypes.c_uint], ), + # cudaError_t cudaHostRegister ( void* ptr, size_t size, unsigned int flags ) + Function( + "cudaHostRegister", + cudaError_t, + [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_uint], + ), + # cudaError_t cudaHostUnregister ( void* ptr ) + Function("cudaHostUnregister", cudaError_t, [ctypes.c_void_p]), + # cudaError_t cudaGetLastError ( void ) + Function("cudaGetLastError", cudaError_t, []), ] # https://rocm.docs.amd.com/projects/HIPIFY/en/latest/tables/CUDA_Runtime_API_functions_supported_by_HIP.html # noqa @@ -92,6 +102,9 @@ class CudaRTLibrary: "cudaMemcpy": "hipMemcpy", "cudaIpcGetMemHandle": "hipIpcGetMemHandle", "cudaIpcOpenMemHandle": "hipIpcOpenMemHandle", + "cudaHostRegister": "hipHostRegister", + "cudaHostUnregister": "hipHostUnregister", + "cudaGetLastError": "hipGetLastError", } # class attribute to store the mapping from the path to the library @@ -185,3 +198,16 @@ class CudaRTLibrary: ) ) return devPtr + + def cudaHostRegister(self, ptr: int, size: int, flags: int = 0) -> int: + """Return the raw error code instead of raising, since callers can + fall back to pageable memory. Drain a failure with cudaGetLastError.""" + return self.funcs["cudaHostRegister"](ctypes.c_void_p(ptr), size, flags) + + def cudaHostUnregister(self, ptr: int) -> int: + """Return the raw error code instead of raising.""" + return self.funcs["cudaHostUnregister"](ctypes.c_void_p(ptr)) + + def cudaGetLastError(self) -> int: + """Return and clear the error pending on this thread.""" + return self.funcs["cudaGetLastError"]() diff --git a/vllm/distributed/ec_transfer/ec_connector/cpu/common.py b/vllm/distributed/ec_transfer/ec_connector/cpu/common.py index e2c37b72a6..eff16e08b0 100644 --- a/vllm/distributed/ec_transfer/ec_connector/cpu/common.py +++ b/vllm/distributed/ec_transfer/ec_connector/cpu/common.py @@ -99,8 +99,8 @@ def create_ec_shared_region(vllm_config: "VllmConfig") -> ECSharedRegion: ec_config = vllm_config.ec_transfer_config assert ec_config is not None, "ec_transfer_config required to build region" - dp_rank = vllm_config.parallel_config.data_parallel_rank - engine_id = f"{vllm_config.instance_id}_dp{dp_rank}" + dp_index = vllm_config.parallel_config.data_parallel_index + engine_id = f"{vllm_config.instance_id}_dp{dp_index}" dtype = vllm_config.model_config.dtype hidden_dim = _get_encoder_cache_hidden_dim(vllm_config) diff --git a/vllm/distributed/ec_transfer/ec_connector/utils.py b/vllm/distributed/ec_transfer/ec_connector/utils.py index f54d6877c1..35d26bb44b 100644 --- a/vllm/distributed/ec_transfer/ec_connector/utils.py +++ b/vllm/distributed/ec_transfer/ec_connector/utils.py @@ -60,15 +60,16 @@ def collect_ec_item_metadata( Keyed by mm_hash, each entry carries a `metadata` dict with whatever placeholder fields `resolver` says this model needs published for its - modality, so a consumer can skip the image transform. `data` is None for - items served from the processor cache, in which case the metadata is + modality, so a consumer can skip the image transform. `item_indices` + identifies every occurrence in `mm_features`, including repeated hashes. + `data` is None for items served from the processor cache, so the metadata is unavailable here and the consumer has to fall back to processing the media itself. A connector that also has transfer coordinates to report (e.g. NIXL peer_host/peer_port/size_bytes) merges those in alongside `metadata`, not into it. """ items: dict[str, dict[str, Any]] = {} - for feature in mm_features: + for index, feature in enumerate(mm_features): metadata: dict[str, Any] = {} if feature.data is not None: wanted = resolver.fields_for(feature.modality) @@ -81,7 +82,9 @@ def collect_ec_item_metadata( # Some metadata (e.g. Qwen3-VL video timestamps) is # produced as a plain list rather than a tensor. metadata[key] = value - items[feature.identifier] = {"metadata": metadata} + entry = items.setdefault(feature.identifier, {"item_indices": []}) + entry["metadata"] = metadata + entry["item_indices"].append(index) return items diff --git a/vllm/distributed/kv_events.py b/vllm/distributed/kv_events.py index 3aa2baa171..9d7ec2f827 100644 --- a/vllm/distributed/kv_events.py +++ b/vllm/distributed/kv_events.py @@ -17,6 +17,12 @@ import zmq from vllm.config.kv_events import KVEventsConfig from vllm.logger import init_logger +from vllm.utils.network_utils import ( + get_ip, + get_tcp_uri, + is_valid_ipv6_address, + split_zmq_path, +) from vllm.v1.core.kv_cache_utils import ExternalBlockHash logger = init_logger(__name__) @@ -349,6 +355,8 @@ class ZmqEventPublisher(EventPublisher): replay_endpoint, self._dp_rank ) assert self._endpoint is not None + self._hwm = hwm + self._socket_setup() self._publisher_config = KVEventsConfig( enable_kv_cache_events=True, publisher="zmq", @@ -359,8 +367,6 @@ class ZmqEventPublisher(EventPublisher): max_queue_size=max_queue_size, topic=topic, ) - self._hwm = hwm - self._socket_setup() # Payload self._seq_gen = count() @@ -423,16 +429,10 @@ class ZmqEventPublisher(EventPublisher): if self._pub is None: self._pub = self._ctx.socket(zmq.PUB) self._pub.set_hwm(self._hwm) - # Heuristic: bind if wildcard / * present, else connect. - # bind stable, connect volatile convention - if self._endpoint is not None and ( - "*" in self._endpoint - or "::" in self._endpoint - or self._endpoint.startswith("ipc://") - or self._endpoint.startswith("inproc://") - ): - self._pub.bind(self._endpoint) - elif self._endpoint is not None: + assert self._endpoint is not None + if self._is_bind_endpoint(self._endpoint): + self._endpoint = self._bind(self._pub, self._endpoint) + else: self._pub.connect(self._endpoint) # Set up replay socket: use ROUTER @@ -441,7 +441,31 @@ class ZmqEventPublisher(EventPublisher): # 3) works in our non‑blocking poll loop alongside PUB if self._replay_endpoint is not None: self._replay = self._ctx.socket(zmq.ROUTER) - self._replay.bind(self._replay_endpoint) + self._replay_endpoint = self._bind(self._replay, self._replay_endpoint) + + @staticmethod + def _is_bind_endpoint(endpoint: str) -> bool: + """Bind wildcard and ipc/inproc endpoints; connect to the rest.""" + return ( + "*" in endpoint + or "::" in endpoint + or endpoint.startswith(("ipc://", "inproc://")) + ) + + @staticmethod + def _bind(sock: zmq.Socket, endpoint: str) -> str: + """Bind; tcp://*:0 binds this node's IP and returns the bound endpoint.""" + if endpoint != "tcp://*:0": + sock.bind(endpoint) + return endpoint + ip = get_ip() + if ip in ("0.0.0.0", "::"): + raise ValueError("tcp://*:0 needs a node IP; set VLLM_HOST_IP") + if is_valid_ipv6_address(ip): + sock.setsockopt(zmq.IPV6, 1) + sock.bind(get_tcp_uri(ip, 0)) + _, _, port = split_zmq_path(sock.getsockopt_string(zmq.LAST_ENDPOINT)) + return get_tcp_uri(ip, int(port)) def _publisher_thread(self) -> None: """Background thread that processes the event queue.""" @@ -514,7 +538,7 @@ class ZmqEventPublisher(EventPublisher): Returns: The endpoint with the port offset by data_parallel_rank - or suffix appended + or suffix appended; a tcp port of 0 is left for the OS. """ # Do nothing if input is None or data_parallel_rank is 0 @@ -529,6 +553,8 @@ class ZmqEventPublisher(EventPublisher): last_colon_idx = endpoint.rfind(":") base_addr = endpoint[:last_colon_idx] base_port = int(endpoint[last_colon_idx + 1 :]) + if base_port == 0: + return endpoint new_port = base_port + data_parallel_rank return f"{base_addr}:{new_port}" return endpoint diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py index e9a3e5d99a..47990d22a5 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py @@ -103,6 +103,18 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA): Must be used in conjunction with the `extract_hidden_states` spec decoding method. """ + def get_finished_count(self) -> int: + """Number of ``finished_sending`` notifications expected per request. + + Only TP rank 0 writes hidden states to disk, so each request yields a + single notification. The default (``None``) makes ``KVOutputAggregator`` + wait for every worker in the world size; since ranks 1..N-1 never submit + a copy, the request would never be reported as finished and its blocks + would be leaked forever (the scheduler then spins on + ``has_finished_requests()``). + """ + return 1 + @classmethod def _find_cache_kv_group_id(cls, kv_cache_config: "KVCacheConfig | None") -> int: """Index of the KV cache group holding the extracted hidden states. @@ -517,7 +529,13 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA): the hidden states from the KV cache. """ req_id = request.request_id - filename = self._request_filenames.pop(req_id) + filename = self._request_filenames.pop(req_id, None) + if filename is None: + # Request was aborted before it was ever scheduled, so + # build_connector_meta never recorded a filename for it. There are + # no hidden states to save and no blocks worth delaying. + self._pending_saves.pop(req_id, None) + return False, None kv_params = request.kv_transfer_params or {} if kv_params.get("include_output_tokens", False): # Exclude the final token — it was the model's output, never an @@ -566,15 +584,18 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA): done_sending: set[str] = set() for req_id in list(self._accumulated_finished_req_ids): event = self._req_copy_events.get(req_id) - if event is None or event.query(): - self._req_copy_events.pop(req_id, None) + if event is not None and not event.query(): + continue # DtoH copy still in flight + self._req_copy_events.pop(req_id, None) + self._accumulated_finished_req_ids.discard(req_id) + # Only report requests we actually tracked for sending. + if event is not None: done_sending.add(req_id) - self._accumulated_finished_req_ids.discard(req_id) - # Clean up any leftover lock fds (e.g. aborted requests - # that never went through _submit_async_write). - lock_fd = self._lock_fds.pop(req_id, None) - if lock_fd is not None: - os.close(lock_fd) + # Clean up any leftover lock fds (e.g. aborted requests that never + # went through _submit_async_write). + lock_fd = self._lock_fds.pop(req_id, None) + if lock_fd is not None: + os.close(lock_fd) return done_sending or None, None @@ -583,6 +604,10 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA): request: "Request", block_ids: tuple[list[int], ...], ) -> tuple[bool, dict[str, Any] | None]: + if not 0 <= self._cache_kv_group_id < len(block_ids): + # No per-group block table (e.g. a request that was never + # scheduled); treat as a no-op instead of indexing out of bounds. + return False, None return self.request_finished(request, block_ids[self._cache_kv_group_id]) @classmethod diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/worker.py index b7ffe9ec97..b223616401 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/worker.py @@ -22,6 +22,7 @@ from vllm.distributed.parallel_state import ( get_tensor_model_parallel_rank, get_tp_group, ) +from vllm.utils.gpu_sync_debug import gpu_sync_allowed from vllm.utils.torch_utils import current_stream from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy from vllm.v1.hisparse.layout import HISPARSE_HOT_SUFFIX @@ -414,11 +415,18 @@ class HiSparseConnectorWorker: transfers = ( metadata.command.page_transfers if metadata.command is not None else [] ) + self._restore_pages([transfer for transfer in transfers if transfer.restore]) self._post_forward_transfers = [ - transfer for transfer in transfers if transfer.after_forward + transfer + for transfer in transfers + if not transfer.restore and transfer.after_forward ] self._submit_transfers( - [transfer for transfer in transfers if not transfer.after_forward] + [ + transfer + for transfer in transfers + if not transfer.restore and not transfer.after_forward + ] ) self._pending_invalid_block_ids.extend(metadata.source_block_ids) if request_state_indices is not None: @@ -708,11 +716,17 @@ class HiSparseConnectorWorker: self._submitted_mirror_layers.update(pending_layers) def _record_transfer_completion( - self, transfers: list[SparseKVPageTransfer] + self, + transfers: list[SparseKVPageTransfer], + *, + stream: torch.Stream | None = None, ) -> None: - if not transfers or not self.is_host_writer: + if not transfers: return - stream = self.dma_stream + if stream is None: + if not self.is_host_writer: + return + stream = self.dma_stream assert stream is not None completion_event = torch.Event() completion_event.record(stream) @@ -720,6 +734,30 @@ class HiSparseConnectorWorker: self._pending_transfer_events.append((completion_event, transfer_ids)) self._enqueued_transfer_ids.extend(transfer_ids) + def _restore_pages(self, transfers: list[SparseKVPageTransfer]) -> None: + """Restore imported tails on every rank before its forward or graph replay.""" + if not transfers: + return + for layer_index, cache in enumerate(self.cache_handles): + source_index = cache.runtime.resident_source_index + source = self.host_caches[layer_index] + destination = self.resident_caches[layer_index] + for transfer in transfers: + host_start = transfer.host_block_id * self.kernel_block_size + resident_block = transfer.resident_block_ids[source_index] + # Each import pays one H2D page copy and enqueue per layer/rank + # instead of tail prefill on D. Restore the prompt rows before + # decode appends to this page; unused rows do not extend the + # logical sequence. + # cudaHostRegister pins this pool outside PyTorch's allocator, + # so the sync checker cannot recognize it via is_pinned(). + with gpu_sync_allowed(): + destination[resident_block].copy_( + source[host_start : host_start + self.kernel_block_size], + non_blocking=True, + ) + self._record_transfer_completion(transfers, stream=current_stream()) + def _submit_transfers(self, transfers: list[SparseKVPageTransfer]) -> None: if self.cache_handles[0].runtime.eager_host_mirror: self._record_transfer_completion(transfers) @@ -733,15 +771,12 @@ class HiSparseConnectorWorker: descriptor_count = len(transfers) * num_layers descriptors = self._acquire_dma_descriptors(descriptor_count) destination_rows = np.fromiter( - ( - transfer.destination_block_id * self.kernel_block_size - for transfer in transfers - ), + (transfer.host_block_id * self.kernel_block_size for transfer in transfers), dtype=np.int64, count=len(transfers), ) source_blocks_by_transfer = np.asarray( - [transfer.source_block_ids for transfer in transfers], dtype=np.int64 + [transfer.resident_block_ids for transfer in transfers], dtype=np.int64 ) if source_blocks_by_transfer.ndim != 2: raise RuntimeError( diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py index 7bcdd2531b..c2b20ffc59 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py @@ -1306,6 +1306,8 @@ class NixlBaseConnectorWorker: error=e, remote_engine_id=eid, ) + # Count once per handshake, regardless of waiting requests. + self.xfer_stats.record_failed_handshake() fut.add_done_callback(done_callback) return fut @@ -2811,13 +2813,17 @@ class NixlBaseConnectorWorker: done_sending.update(self._replicated_pcp_done_sending) self._replicated_pcp_done_sending.clear() - # Process receive failures reported by background threads. + # Process receive failures reported by background threads. Each + # producer already recorded a specific metric (handshake or + # notification), so only the failure bookkeeping runs here. while not self._failed_recv_reqs.empty(): try: req_id = self._failed_recv_reqs.get_nowait() except queue.Empty: break - self._handle_failed_transfer(req_id, None, self._recv_failures) + self._handle_failed_transfer( + req_id, None, self._recv_failures, record_failed_transfer=False + ) failed_recv_reqs: set[ReqId] = set() if self._recv_failures: @@ -3059,10 +3065,27 @@ class NixlBaseConnectorWorker: return True def _handle_failed_transfer( - self, req_id: str, handle: int | None, failed_req_ids: set[str] | None = None + self, + req_id: str, + handle: int | None, + failed_req_ids: set[str] | None = None, + record_failed_transfer: bool = True, ) -> bool: - """Record a failure and release its handle, returning False to retain it.""" - self.xfer_stats.record_failed_transfer() + """Record a failure and release its handle, returning False to retain it. + + Args: + req_id: The request ID. + handle: The transfer handle. + failed_req_ids: Requests observed as failed; each stays listed + until it has no outstanding handles. + record_failed_transfer: Whether to count the failure toward the + transport-failure metric. Callers that already recorded a more + specific metric (eg KV expiry, reported separately from + transport failures) pass False. + + """ + if record_failed_transfer: + self.xfer_stats.record_failed_transfer() if failed_req_ids is not None: failed_req_ids.add(req_id) return handle is None or self._try_release_xfer_handle(req_id, handle) @@ -3278,7 +3301,8 @@ class NixlBaseConnectorWorker: """Pair an uncached decode suffix with the same prefill regions.""" assert len(decode_block_ids) == len(prefill_block_ids) if not any(decode_block_ids): - return [], prefill_block_ids + empty_regions: list[list[int]] = [[] for _ in decode_block_ids] + return empty_regions, empty_regions.copy() if num_computed_blocks is not None: assert num_remote_blocks is not None diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py index fea3a00ce2..43fbbe0c3c 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py @@ -64,8 +64,14 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): len(meta.local_physical_block_ids), len(meta.remote.block_ids), ) - # always store metadata for failure recovery - self._recving_metadata[req_id] = meta + # Full local hits and aborted cleanup only notify P; no recv is awaited. + # On a full local hit: + # - Notification failure must not fail the request: its KV is local. + # - Receive completion must not be reported: num_external_tokens == 0, + # so the scheduler never entered WAITING_FOR_REMOTE_KVS to begin with. + # Aborted cleanup requests have already been removed from the scheduler. + if meta.awaiting_kvs or any(meta.local_block_ids): + self._recving_metadata[req_id] = meta if remote_engine_id not in self._remote_agents: # Initiate handshake with remote engine to exchange metadata. with self._handshake_lock: @@ -149,7 +155,11 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): engine_id, ) self.xfer_stats.record_kv_expired_req() - self._handle_failed_transfer(req_id, None, self._recv_failures) + # KV expiry is reported separately from transport failures, so only + # the state cleanup side of _handle_failed_transfer runs here. + self._handle_failed_transfer( + req_id, None, self._recv_failures, record_failed_transfer=False + ) return if any(len(group) > 0 for group in meta.local_block_ids): @@ -168,8 +178,14 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): local_block_ids = meta.local_physical_block_ids remote_region_groups = self.dst_region_group_ids[engine_id] local_region_groups = self.region_group_ids or remote_region_groups - groups_differ = local_region_groups != remote_region_groups - if groups_differ: + if not local_block_ids: + # Region expansion cannot index empty groups. Pass empty specs to + # _read_blocks so its existing cache-hit notification path runs. + read_specs = [ + ReadSpec(remote_rank=rank, local_block_ids=[], remote_block_ids=[]) + for rank in plan.all_source_ranks + ] + elif local_region_groups != remote_region_groups: if not self.use_mla or self._has_mamba: raise NotImplementedError( "Different NIXL cache-group layouts are only supported for " @@ -358,7 +374,20 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): remote_agents = self._remote_agents[meta.remote.engine_id] for rank_to_notify, agent in remote_agents.items(): if rank_to_notify != (0, read_specs[0].remote_rank): - self.nixl_wrapper.send_notif(agent, notif_msg=notif_id) + try: + self.nixl_wrapper.send_notif(agent, notif_msg=notif_id) + except Exception as e: + self._log_failure( + failure_type="notification_failed", + msg="Remote rank will not update request state. " + "This may indicate network issues.", + req_id=req_id, + error=e, + dst_engine_id=meta.remote.engine_id, + remote_rank=rank_to_notify[1], + remote_agent_name=agent, + ) + self.xfer_stats.record_failed_notification() def _read_blocks( self, diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py index 01c956a525..fbe85f46a3 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py @@ -365,6 +365,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): engine_id, req_id, ) + self.xfer_stats.record_failed_notification() self._failed_recv_reqs.put(req_id) return for rank, agent_name in agents.items(): @@ -377,6 +378,9 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): error=e, remote_rank=rank, ) + self.xfer_stats.record_failed_notification() + # Earlier registrations may still trigger WRITEs into D's blocks. + # Keep the receive pending until those writes are finished. logger.debug( "Sent PUSH_REG for %s to engine %s (%dB)", req_id, engine_id, len(notif_msg) ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py index 2cff3f843b..f6d0684b6b 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py @@ -39,6 +39,7 @@ class NixlKVConnectorStats(KVConnectorStats): "num_descriptors": [], "num_failed_transfers": [], "num_failed_notifications": [], + "num_failed_handshakes": [], "num_kv_expired_reqs": [], } @@ -57,6 +58,10 @@ class NixlKVConnectorStats(KVConnectorStats): """Record a failed NIXL notification (send_notif).""" self.data["num_failed_notifications"].append(1) + def record_failed_handshake(self): + """Record a failed NIXL handshake.""" + self.data["num_failed_handshakes"].append(1) + def record_kv_expired_req(self): """Record a request that had its KV blocks expire.""" self.data["num_kv_expired_reqs"].append(1) @@ -72,6 +77,7 @@ class NixlKVConnectorStats(KVConnectorStats): self.num_successful_transfers == 0 and len(self.data["num_failed_transfers"]) == 0 and len(self.data["num_failed_notifications"]) == 0 + and len(self.data["num_failed_handshakes"]) == 0 and len(self.data["num_kv_expired_reqs"]) == 0 ) @@ -84,10 +90,22 @@ class NixlKVConnectorStats(KVConnectorStats): return self def reduce(self) -> dict[str, int | float]: - # Compute compact representative stats suitable for CLI logging + # Compute compact representative stats suitable for CLI logging. + # Failure counts are reported on every interval: transfer, handshake + # and notification failures are grouped as sporadic + # lower-transport-layer events, while KV expiry is reported separately + # as it is an actionable autoscaler signal rather than a transport + # issue. + failure_counts = { + "Num failed transfers": len(self.data["num_failed_transfers"]) + + len(self.data["num_failed_handshakes"]) + + len(self.data["num_failed_notifications"]), + "Num KV expired reqs": len(self.data["num_kv_expired_reqs"]), + } if self.num_successful_transfers == 0: - # CLI logging only reports successful transfers stats. If all requests in - # the interval were unsuccessful, Prom will report failures stats instead. + # Timing / throughput stats only cover successful transfers. If + # all requests in the interval were unsuccessful, the failure + # counts above still surface what happened. return { "Num successful transfers": 0, "Avg xfer time (ms)": 0, @@ -97,6 +115,7 @@ class NixlKVConnectorStats(KVConnectorStats): "Avg MB per transfer": 0, "Throughput (MB/s)": 0, "Avg number of descriptors": 0, + **failure_counts, } xfer_time = np.asarray(self.data["transfer_duration"]) @@ -122,6 +141,7 @@ class NixlKVConnectorStats(KVConnectorStats): "Avg MB per transfer": round(avg_mb.item(), 3), "Throughput (MB/s)": round(throughput_mb_s.item(), 3), "Avg number of descriptors": round(descs.mean().item(), 1), + **failure_counts, } @property @@ -212,15 +232,20 @@ class NixlPromMetrics(KVConnectorPromMetrics): ) counter_nixl_num_failed_transfers = self._counter_cls( name="vllm:nixl_num_failed_transfers", - documentation="Number of failed NIXL KV Cache transfers.", + documentation="Number of failed NIXL KV Cache transfers, including" + " handshake and notification failures. NOTE: KV expiry is tracked" + " separately in vllm:nixl_num_kv_expired_reqs.", labelnames=labelnames, ) self.counter_nixl_num_failed_transfers = create_metric_per_engine( counter_nixl_num_failed_transfers, self.per_engine_labelvalues ) + counter_nixl_num_failed_notifications = self._counter_cls( name="vllm:nixl_num_failed_notifications", - documentation="Number of failed NIXL KV Cache notifications.", + documentation="Number of failed NIXL KV Cache notifications. " + "Retained for compatibility; these failures are also included in " + "vllm:nixl_num_failed_transfers.", labelnames=labelnames, ) self.counter_nixl_num_failed_notifications = create_metric_per_engine( @@ -254,13 +279,26 @@ class NixlPromMetrics(KVConnectorPromMetrics): ): for list_item in transfer_stats_data[list_item_key]: prom_obj[engine_idx].observe(list_item) - for counter_obj, counter_item_key in zip( + for counter_obj, counter_item_keys in zip( [ self.counter_nixl_num_failed_transfers, self.counter_nixl_num_failed_notifications, self.counter_nixl_num_kv_expired_reqs, ], - ["num_failed_transfers", "num_failed_notifications", "num_kv_expired_reqs"], + [ + # Transfer, handshake and notification failures are grouped: + # all are sporadic lower-transport-layer events. KV expiry is + # reported separately as it signals autoscaler behavior, not + # transport health. + ( + "num_failed_transfers", + "num_failed_handshakes", + "num_failed_notifications", + ), + ("num_failed_notifications",), + ("num_kv_expired_reqs",), + ], ): - for list_item in transfer_stats_data[counter_item_key]: - counter_obj[engine_idx].inc(list_item) + for counter_item_key in counter_item_keys: + for list_item in transfer_stats_data[counter_item_key]: + counter_obj[engine_idx].inc(list_item) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/config.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/config.py index ca503d4e67..e20c1189af 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/config.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/config.py @@ -35,7 +35,7 @@ if TYPE_CHECKING: def get_offloading_group_ids(kv_cache_config: "KVCacheConfig") -> tuple[int, ...]: if kv_cache_config.hisparse_host_num_blocks is None: - return tuple(range(len(kv_cache_config.kv_cache_groups))) + return kv_cache_config.prefix_cacheable_group_ids return tuple( group_id for group_id, group in enumerate(kv_cache_config.kv_cache_groups) @@ -135,12 +135,12 @@ def build_offloading_config( ) worker_kv_bytes_per_block = 0 - all_groups_selected = len(selected_groups) == len(kv_cache_config.kv_cache_groups) if ( - all_groups_selected + kv_cache_config.hisparse_host_num_blocks is None and kv_cache_config.num_blocks > 0 and kv_cache_config.kv_cache_tensors ): + # Scratch filtering must preserve the scheduler/worker allocation stride. # Every KVCacheTensor describes placement within the same backing allocation, # so its size is the total, not a per-tensor share. total_gpu_kv_bytes = kv_cache_config.kv_cache_tensors[0].size diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py index 5b14e50dda..2bb00d88f0 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py @@ -141,8 +141,9 @@ class OffloadingConnectorWorker: ), None, ) - if packed_layer_name is not None and len(selected_groups) == len( - kv_cache_config.kv_cache_groups + if ( + packed_layer_name is not None + and kv_cache_config.hisparse_host_num_blocks is None ): (tensor,) = tensors_per_block[packed_layer_name] num_blocks = tensor.shape[0] diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index d637391cd3..67d8a2f2a0 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -22,6 +22,7 @@ from typing import ( cast, get_args, get_origin, + is_typeddict, ) import huggingface_hub @@ -183,6 +184,18 @@ def is_type(type_hint: TypeHint, type: TypeHintT) -> TypeIs[TypeHintT]: return type_hint is type or get_origin(type_hint) is type +def is_dict_subclass(type_hint: TypeHint) -> bool: + """Check if the type hint is a subclass of `dict`. + + `TypedDict`s are excluded because they do not support class checks. + """ + return ( + isinstance(type_hint, type) + and not is_typeddict(type_hint) + and issubclass(type_hint, dict) + ) + + def contains_type(type_hints: set[TypeHint], type: TypeHintT) -> bool: """Check if the type hints contain a specific type.""" return any(is_type(type_hint, type) for type_hint in type_hints) @@ -305,6 +318,8 @@ def _compute_kwargs(cls: ConfigType) -> dict[str, dict[str, Any]]: for field in fields(cls): # Get the set of possible types for the field type_hints: set[TypeHint] = get_type_hints(field.type) + # Subclasses of dict (e.g. MultiModalDummyOptions) are CLI dicts + type_hints = {dict if is_dict_subclass(th) else th for th in type_hints} # If the field is a dataclass, we can use the model_validate_json generator = (th for th in type_hints if is_dataclass(th)) @@ -766,6 +781,7 @@ class EngineArgs: stream_interval: int = SchedulerConfig.stream_interval kv_sharing_fast_prefill: bool = CacheConfig.kv_sharing_fast_prefill + swa_bounded_replay: bool = CacheConfig.swa_bounded_replay optimization_level: OptimizationLevel = VllmConfig.optimization_level performance_mode: PerformanceMode = VllmConfig.performance_mode @@ -1295,6 +1311,9 @@ class EngineArgs: cache_group.add_argument( "--kv-sharing-fast-prefill", **cache_kwargs["kv_sharing_fast_prefill"] ) + cache_group.add_argument( + "--swa-bounded-replay", **cache_kwargs["swa_bounded_replay"] + ) cache_group.add_argument( "--mamba-cache-dtype", **cache_kwargs["mamba_cache_dtype"] ) @@ -2103,6 +2122,7 @@ class EngineArgs: prefix_cache_retention_interval=self.prefix_cache_retention_interval, kv_cache_dtype_skip_layers=self.kv_cache_dtype_skip_layers, kv_sharing_fast_prefill=self.kv_sharing_fast_prefill, + swa_bounded_replay=self.swa_bounded_replay, mamba_cache_dtype=self.mamba_cache_dtype, mamba_ssm_cache_dtype=self.mamba_ssm_cache_dtype, mamba_block_size=self.mamba_block_size, diff --git a/vllm/engine/protocol.py b/vllm/engine/protocol.py index c44fd44d0f..1e6011a230 100644 --- a/vllm/engine/protocol.py +++ b/vllm/engine/protocol.py @@ -7,6 +7,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any from vllm.config import ModelConfig, VllmConfig +from vllm.config.kv_events import KVEventsConfig from vllm.distributed.weight_transfer.base import ( WeightTransferInitRequest, WeightTransferUpdateRequest, @@ -62,6 +63,10 @@ class EngineClient(ABC): @abstractmethod def dead_error(self) -> BaseException: ... + def get_kv_event_sources(self) -> dict[int, KVEventsConfig]: + """KV-event publisher config of each engine, keyed by DP rank.""" + return {} + def check_admission( # noqa: B027 self, n: int = 1, request_id: str | None = None ) -> None: @@ -186,6 +191,11 @@ class EngineClient(ABC): """Sleep the engine.""" ... + @abstractmethod + async def release_kv_cache_memory(self) -> None: + """Discard KV cache physical GPU memory. Requires a completed pause.""" + ... + @abstractmethod async def wake_up(self, tags: list[str] | None = None) -> None: """Wake up the engine.""" diff --git a/vllm/entrypoints/anthropic/serving.py b/vllm/entrypoints/anthropic/serving.py index e90d70e289..6cc1271add 100644 --- a/vllm/entrypoints/anthropic/serving.py +++ b/vllm/entrypoints/anthropic/serving.py @@ -112,6 +112,7 @@ class AnthropicServingMessages(OpenAIServingChat): reasoning_parser: str = "", enable_auto_tools: bool = False, tool_parser: str | None = None, + tool_strict_level: str = "auto", enable_prompt_tokens_details: bool = False, enable_force_include_usage: bool = False, default_chat_template_kwargs: dict[str, Any] | None = None, @@ -127,6 +128,7 @@ class AnthropicServingMessages(OpenAIServingChat): return_tokens_as_token_ids=return_tokens_as_token_ids, reasoning_parser=reasoning_parser, enable_auto_tools=enable_auto_tools, + tool_strict_level=tool_strict_level, tool_parser=tool_parser, enable_prompt_tokens_details=enable_prompt_tokens_details, enable_force_include_usage=enable_force_include_usage, diff --git a/vllm/entrypoints/cohere/serving.py b/vllm/entrypoints/cohere/serving.py index 7885586452..63d00f0c57 100644 --- a/vllm/entrypoints/cohere/serving.py +++ b/vllm/entrypoints/cohere/serving.py @@ -249,6 +249,7 @@ class CohereServingChatV2(OpenAIServingChat): reasoning_parser: str = "", enable_auto_tools: bool = False, tool_parser: str | None = None, + tool_strict_level: str = "auto", enable_prompt_tokens_details: bool = False, enable_force_include_usage: bool = False, default_chat_template_kwargs: dict[str, Any] | None = None, @@ -265,6 +266,7 @@ class CohereServingChatV2(OpenAIServingChat): return_tokens_as_token_ids=return_tokens_as_token_ids, reasoning_parser=reasoning_parser, enable_auto_tools=enable_auto_tools, + tool_strict_level=tool_strict_level, tool_parser=tool_parser, enable_prompt_tokens_details=enable_prompt_tokens_details, enable_force_include_usage=enable_force_include_usage, diff --git a/vllm/entrypoints/generate/api_router.py b/vllm/entrypoints/generate/api_router.py index ac995a4f03..189c4e6779 100644 --- a/vllm/entrypoints/generate/api_router.py +++ b/vllm/entrypoints/generate/api_router.py @@ -112,6 +112,7 @@ async def init_generate_state( return_tokens_as_token_ids=args.return_tokens_as_token_ids, enable_auto_tools=args.enable_auto_tool_choice, tool_parser=args.tool_call_parser, + tool_strict_level=args.tool_strict_level, tool_server=state.tool_server, reasoning_parser=args.structured_outputs_config.reasoning_parser, enable_prompt_tokens_details=args.enable_prompt_tokens_details, @@ -137,6 +138,7 @@ async def init_generate_state( enable_auto_tools=args.enable_auto_tool_choice, exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, tool_parser=args.tool_call_parser, + tool_strict_level=args.tool_strict_level, reasoning_parser=args.structured_outputs_config.reasoning_parser, enable_prompt_tokens_details=args.enable_prompt_tokens_details, enable_force_include_usage=args.enable_force_include_usage, @@ -178,6 +180,7 @@ async def init_generate_state( return_tokens_as_token_ids=args.return_tokens_as_token_ids, enable_auto_tools=args.enable_auto_tool_choice, tool_parser=args.tool_call_parser, + tool_strict_level=args.tool_strict_level, reasoning_parser=args.structured_outputs_config.reasoning_parser, enable_prompt_tokens_details=args.enable_prompt_tokens_details, enable_force_include_usage=args.enable_force_include_usage, @@ -198,6 +201,7 @@ async def init_generate_state( return_tokens_as_token_ids=args.return_tokens_as_token_ids, enable_auto_tools=args.enable_auto_tool_choice, tool_parser=args.tool_call_parser, + tool_strict_level=args.tool_strict_level, reasoning_parser=args.structured_outputs_config.reasoning_parser, enable_prompt_tokens_details=args.enable_prompt_tokens_details, enable_force_include_usage=args.enable_force_include_usage, diff --git a/vllm/entrypoints/launchers/api_server/app_state.py b/vllm/entrypoints/launchers/api_server/app_state.py index 09bb0ec630..d33be65949 100644 --- a/vllm/entrypoints/launchers/api_server/app_state.py +++ b/vllm/entrypoints/launchers/api_server/app_state.py @@ -95,6 +95,7 @@ async def init_app_state( enable_auto_tools=args.enable_auto_tool_choice, exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, tool_parser=args.tool_call_parser, + tool_strict_level=args.tool_strict_level, reasoning_parser=args.structured_outputs_config.reasoning_parser, default_chat_template_kwargs=default_chat_template_kwargs, log_error_stack=args.log_error_stack, @@ -111,6 +112,7 @@ async def init_app_state( enable_auto_tools=args.enable_auto_tool_choice, exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, tool_parser=args.tool_call_parser, + tool_strict_level=args.tool_strict_level, reasoning_parser=args.structured_outputs_config.reasoning_parser, default_chat_template_kwargs=default_chat_template_kwargs, log_error_stack=args.log_error_stack, diff --git a/vllm/entrypoints/launchers/cli_args.py b/vllm/entrypoints/launchers/cli_args.py index 71ae94c106..488a7f2c43 100644 --- a/vllm/entrypoints/launchers/cli_args.py +++ b/vllm/entrypoints/launchers/cli_args.py @@ -20,6 +20,7 @@ from vllm.entrypoints.chat_utils import ( ) from vllm.entrypoints.openai.models.protocol import LoRAModulePath from vllm.tool_parsers import ToolParserManager +from vllm.tool_parsers.tool_strict_level import ToolStrictLevelName from vllm.utils.argparse_utils import FlexibleArgumentParser from .utils.constants import ( @@ -116,6 +117,13 @@ class BaseFrontendArgs: """Special the tool parser plugin write to parse the model-generated tool into OpenAI API format, the name register in this plugin can be used in `--tool-call-parser`.""" + tool_strict_level: ToolStrictLevelName = "auto" + """Server-side floor for structural-tag based tool calling, applied on top + of the per-tool `strict` field. `auto` follows the request's tool choice + and per-tool strictness; `function` constrains the + tool-call envelope (markup and function name) for every request with + tools; `parameter` additionally pins argument schemas, as if every tool + were `strict: true`.""" tool_server: str | None = None """Comma-separated list of host:port pairs (IPv4, IPv6, or hostname). Examples: 127.0.0.1:8000, [::1]:8000, localhost:1234. Or `demo` for diff --git a/vllm/entrypoints/launchers/render/app_state.py b/vllm/entrypoints/launchers/render/app_state.py index 33988443d1..1d242977be 100644 --- a/vllm/entrypoints/launchers/render/app_state.py +++ b/vllm/entrypoints/launchers/render/app_state.py @@ -60,6 +60,7 @@ async def init_render_app_state( enable_auto_tools=args.enable_auto_tool_choice, exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, tool_parser=args.tool_call_parser, + tool_strict_level=args.tool_strict_level, reasoning_parser=args.reasoning_parser, default_chat_template_kwargs=default_chat_template_kwargs, log_error_stack=args.log_error_stack, @@ -76,6 +77,7 @@ async def init_render_app_state( enable_auto_tools=args.enable_auto_tool_choice, exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, tool_parser=args.tool_call_parser, + tool_strict_level=args.tool_strict_level, reasoning_parser=args.reasoning_parser, default_chat_template_kwargs=default_chat_template_kwargs, log_error_stack=args.log_error_stack, diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py index d9a2a46701..999310aa61 100644 --- a/vllm/entrypoints/llm.py +++ b/vllm/entrypoints/llm.py @@ -830,6 +830,14 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin): """ self.llm_engine.sleep(level=level, mode=mode) + def release_kv_cache_memory(self) -> None: + """Release the GPU physical memory backing the KV cache. + + Requires a completed ``sleep(level=0)`` and resident executor memory. + Restore with ``wake_up(tags=["kv_cache"])``; kept requests are recomputed. + """ + self.llm_engine.release_kv_cache_memory() + def wake_up(self, tags: list[str] | None = None): """Wake up the engine from sleep mode. See the [sleep][vllm.LLM.sleep] method for more details. diff --git a/vllm/entrypoints/openai/chat_completion/protocol.py b/vllm/entrypoints/openai/chat_completion/protocol.py index badd5a0344..225e992441 100644 --- a/vllm/entrypoints/openai/chat_completion/protocol.py +++ b/vllm/entrypoints/openai/chat_completion/protocol.py @@ -453,18 +453,6 @@ class ChatCompletionRequest(OpenAIBaseModel): ), ) - return_assistant_tokens_mask: bool = Field( - default=False, - description=( - "If true, the /render response will include an " - "``assistant_tokens_mask`` field — a per-token list of 0/1 " - "values indicating which tokens were assistant-generated. " - "Requires the chat template to use ``{% generation %}`` " - "tags. When the template does not support it, " - "``assistant_tokens_mask`` will be ``null``." - ), - ) - cache_salt: str | None = Field( default=None, min_length=1, @@ -605,7 +593,6 @@ class ChatCompletionRequest(OpenAIBaseModel): extra_kwargs, ), media_io_kwargs=self.media_io_kwargs, - return_assistant_tokens_mask=bool(self.return_assistant_tokens_mask), # No-tools requests default to tool_choice="none" at the API # layer. Collapse that default before rendering, so K3 emits a # model-visible tool-choice instruction only for requests with a diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index 51c0af8e8f..3e2d8f7369 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -132,6 +132,7 @@ class OpenAIServingChat(GenerateBaseServing): enable_auto_tools: bool = False, exclude_tools_when_tool_choice_none: bool = False, tool_parser: str | None = None, + tool_strict_level: str = "auto", enable_prompt_tokens_details: bool = False, enable_force_include_usage: bool = False, enable_log_outputs: bool = False, @@ -161,6 +162,7 @@ class OpenAIServingChat(GenerateBaseServing): tool_parser_name=tool_parser, reasoning_parser_name=reasoning_parser, enable_auto_tools=enable_auto_tools, + tool_strict_level=tool_strict_level, model_name=self.model_config.model, is_harmony=self.model_config.hf_config.model_type == "gpt_oss", ) diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py index ee4f7d55ca..8108c5f775 100644 --- a/vllm/entrypoints/openai/responses/serving.py +++ b/vllm/entrypoints/openai/responses/serving.py @@ -109,6 +109,7 @@ class OpenAIServingResponses(GenerateBaseServing): reasoning_parser: str = "", enable_auto_tools: bool = False, tool_parser: str | None = None, + tool_strict_level: str = "auto", tool_server: ToolServer | None = None, enable_prompt_tokens_details: bool = False, enable_force_include_usage: bool = False, @@ -135,6 +136,7 @@ class OpenAIServingResponses(GenerateBaseServing): tool_parser_name=tool_parser, reasoning_parser_name=reasoning_parser, enable_auto_tools=enable_auto_tools, + tool_strict_level=tool_strict_level, model_name=self.model_config.model, is_harmony=self.model_config.hf_config.model_type == "gpt_oss", ) diff --git a/vllm/entrypoints/scale_out/derender/serving.py b/vllm/entrypoints/scale_out/derender/serving.py index 81e919d840..d3c5f3aac1 100644 --- a/vllm/entrypoints/scale_out/derender/serving.py +++ b/vllm/entrypoints/scale_out/derender/serving.py @@ -138,6 +138,13 @@ class ServingDerender(BaseServing): if bounds_error is not None: return bounds_error + if self.online_derenderer.parser is not None and request.chat_request is None: + return self.create_error_response( + "chat_request is required when a tool or reasoning parser is " + "configured because plain detokenization would leak raw parser " + "markup into content." + ) + try: choices = await self.online_derenderer.derender_chat( request.generate_response, request.chat_request @@ -248,14 +255,67 @@ class ServingDerender(BaseServing): Processes one ``GenerateStreamResponse`` chunk and returns the derendered chunk together with the updated client carried state. - - ``parser is None`` or no ``chat_request`` until reasoning/tool call - functionality added in future PR. """ error_check_ret = await self._check_model(request) if error_check_ret is not None: return error_check_ret + if self.online_derenderer.parser is not None and request.chat_request is None: + return self.create_error_response( + "chat_request is required when a tool or reasoning parser is " + "configured because plain detokenization would leak raw parser " + "markup into content." + ) + + if ( + self.online_derenderer.parser is not None + and request.prompt_token_ids is None + ): + return self.create_error_response( + "prompt_token_ids is required when a tool or reasoning " + "parser is configured. Without it parse_delta cannot tell " + "whether the prompt left reasoning open (e.g. a chat " + "template that pre-opens ) and would misclassify " + "reasoning content as plain content." + ) + + # Each streamed chunk contains at most one choice per SSE event. + # A single DerenderStreamState is threaded + # through every choice in the chunk, so >1 would corrupt detok/parser + # state across choices. See the matching check in + # OnlineDerenderer.derender_chat_stream which this fails ahead of to + # avoid touching the tokenizer at all on a malformed chunk. + num_choices = len(request.generate_chunk.choices) + if num_choices > 1: + return self.create_error_response( + f"derender_chat_stream expects at most one choice per chunk " + f"(got {num_choices})." + ) + + # prompt_token_ids is folded in here too. It is caller supplied and + # otherwise unbounded. A parser configured deployment rescans it + # in full on every chunk (is_reasoning_end / + # adjust_initial_state_from_prompt, see abstract_parser.parse_delta) + # since parser state cannot be carried across calls. + prompt_len = ( + len(request.prompt_token_ids) if request.prompt_token_ids is not None else 0 + ) + prior_len = ( + len(request.stream_state.output_token_ids) + if request.stream_state is not None + else 0 + ) + delta_len = sum( + len(c.token_ids) for c in request.generate_chunk.choices if c.token_ids + ) + max_model_len = self.model_config.max_model_len + if prompt_len + prior_len + delta_len > max_model_len: + return self.create_error_response( + f"prompt_token_ids length ({prompt_len}) plus " + f"output_token_ids length ({prior_len}) plus delta " + f"({delta_len}) exceeds max_model_len ({max_model_len})." + ) + model_name = request.model or self.models.model_name() try: chunk, updated_state = await self.online_derenderer.derender_chat_stream( @@ -264,15 +324,21 @@ class ServingDerender(BaseServing): state=request.stream_state, chat_request=request.chat_request, prompt_tokens=request.prompt_tokens, + prompt_token_ids=request.prompt_token_ids, ) - except NotImplementedError as exc: - return self.create_error_response(exc) - except ValueError as exc: - return self.create_error_response(str(exc)) - except (KeyError, IndexError) as exc: - return self.create_error_response( - f"invalid stream_state: detokenization failed ({exc!r})" + except Exception as exc: + # The two ValueErrors derender_chat_stream can raise directly + # (missing chat_request, >1 choice per chunk) are already + # pre-checked above, so anything reaching here comes from the + # parser replaying entirely client controlled state. Hermes + # swallows its own streaming exceptions but finalize_generation + # and engine based parsers don't, so an unexpected parser + # failure on malformed (but well typed) state must surface as + # a 400 here rather than an unhandled 500. + logger.debug( + "derender stream replay failed: %s: %s", type(exc).__name__, exc ) + return self.create_error_response("invalid stream_state: derender failed") logger.debug( "derender_chat_stream request_id=%s model=%s delta_tokens=%d", @@ -313,7 +379,7 @@ class ServingDerender(BaseServing): return self.create_error_response(str(exc)) except (KeyError, IndexError) as exc: return self.create_error_response( - f"invalid stream_state: detokenization failed ({exc!r})" + f"invalid stream_state: detokenization failed ({exc})" ) logger.debug( diff --git a/vllm/entrypoints/scale_out/render/serving.py b/vllm/entrypoints/scale_out/render/serving.py index b79138664f..7973ac3c3d 100644 --- a/vllm/entrypoints/scale_out/render/serving.py +++ b/vllm/entrypoints/scale_out/render/serving.py @@ -126,33 +126,11 @@ class ServingRender(BaseServing): ) params = request.to_sampling_params(max_tokens, self.default_sampling_params) - assistant_tokens_mask: list[int] | None = engine_input.get( # type: ignore[assignment] - "assistant_tokens_mask" - ) - if assistant_tokens_mask is not None and len(assistant_tokens_mask) != len( - token_ids - ): - logger.warning( - "assistant_tokens_mask length (%d) != token_ids length (%d); " - "this can happen with multimodal inputs where " - "placeholder expansion changes the token count. " - "The mask may be positionally misaligned.", - len(assistant_tokens_mask), - len(token_ids), - ) - if len(assistant_tokens_mask) < len(token_ids): - assistant_tokens_mask.extend( - [0] * (len(token_ids) - len(assistant_tokens_mask)) - ) - else: - assistant_tokens_mask = assistant_tokens_mask[: len(token_ids)] - request_id = f"chatcmpl-{random_uuid()}" return GenerateRequest( request_id=request_id, token_ids=token_ids, - assistant_tokens_mask=assistant_tokens_mask, features=self._extract_mm_features(engine_input), sampling_params=params, model=request.model, diff --git a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py index be8130e314..ae0fd62b6e 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from typing import Any, Literal, TypeAlias +from typing import Annotated, Any, Literal, TypeAlias from pydantic import ( BaseModel, @@ -150,14 +150,6 @@ class GenerateRequest(BaseModel): token_ids: list[int] = Field(min_length=1) """The token ids to generate text from.""" - assistant_tokens_mask: list[int] | None = None - """Per-token mask (1 = assistant-generated, 0 = not). - - Only populated when the render request sets ``return_assistant_tokens_mask=True`` - and the chat template supports ``{% generation %}``. - ``None`` when the mask was not requested or could not be computed. - """ - @field_validator("token_ids") @classmethod def validate_token_ids(cls, v: list[int]) -> list[int]: @@ -483,16 +475,32 @@ class DerenderStreamState(BaseModel): streaming derender endpoint. All fields are plain JSON serializable data. No opaque tokenizer or parser internals are stored here. + Two separate sets of fields support two different streaming modes: + + - For plain streaming (no parser configured including the completions path): + `prev_tokens`, `prefix_offset` and `read_offset` maintain a bounded incremental + decoding window. This requires O(window) transport and O(delta) computation + per chunk. + - For parser enabled chat streaming: `output_token_ids`, `output_chunk_lens`, + `tools_streamed` and `last_tool_call_ids` are used to replay `parse_delta()` + from scratch on every chunk because parser state cannot be serialized. This + incurs O(n) transport per chunk (O(n²) per generation) and O(n²) + `parse_delta()` calls per generation. Since many parsers re-scan the + entire accumulated text on each invocation, `parse_delta()` itself is + O(n) yielding a true worst case compute cost of O(n³) per generation. + No caching is performed. Work is bounded by `max_model_len`. + See `OnlineDerenderer._derender_chat_stream_parsed`. + The detokenization strategy carries the incremental decode offsets directly rather than re-sending the whole token history each chunk. - ``detokenize_incrementally`` only ever reads the trailing token window - ``prev_tokens[prefix_offset:]``, so we carry just that tail plus the two + `detokenize_incrementally` only ever reads the trailing token window + `prev_tokens[prefix_offset:]`, so we carry just that tail plus the two offsets. Each chunk resumes exactly where the last one stopped, including - any partially processed multi-byte character (tracked by ``read_offset``), + any partially processed multi-byte character (tracked by `read_offset`), then trims and rebases the window so it never grows with generation length. Performance: - - Compute per chunk is O(delta). One ``detokenize_incrementally`` call per + - Compute per chunk is O(delta). One `detokenize_incrementally` call per new token, independent of how many tokens preceded it. - Transport per chunk is O(window). The carried tail is bounded by the incremental detokenization offset, so cumulative bytes over the wire are @@ -500,18 +508,18 @@ class DerenderStreamState(BaseModel): """ prev_tokens: list[str] = Field(default_factory=list) - """Trailing decode window. Token strings from ``prefix_offset`` onward. + """Trailing decode window. Token strings from `prefix_offset` onward. Bounded, trimmed and rebased each chunk to the tail - ``detokenize_incrementally`` still reads, so it does not grow with the + `detokenize_incrementally` still reads, so it does not grow with the number of chunks. """ prefix_offset: int = Field(default=0, ge=0) - """Prefix offset into ``prev_tokens`` for incremental detokenization.""" + """Prefix offset into `prev_tokens` for incremental detokenization.""" read_offset: int = Field(default=0, ge=0) - """Read offset into ``prev_tokens`` for incremental detokenization.""" + """Read offset into `prev_tokens` for incremental detokenization.""" @field_validator("prev_tokens") @classmethod @@ -525,23 +533,53 @@ class DerenderStreamState(BaseModel): return v role_sent: bool = False - """True once the initial ``role: "assistant"`` delta has been emitted. + """True once the initial `role: "assistant"` delta has been emitted. Prevents re-emitting the role on subsequent chunks even when the detok window is transiently empty (e.g. usage only final chunk). """ - # TODO: Properties used in follow on PR for tool call parsing - last_content: str | None = None - """Last emitted cumulative assistant content text.""" + output_token_ids: list[int] = Field(default_factory=list) + """All output tokens seen so far. Parser path only. - last_reasoning: str | None = None - """Last emitted cumulative reasoning text.""" + Replay buffer: each chunk rebuilds a fresh parser and replays every + token in here through `parse_delta` (discarding the result) before + processing the current chunk's tokens since parser internal state + cannot be serialized into this stateless model. Unavoidably O(n) + bounded by ``max_model_len`` (enforced server side, not by a field + validator here since the bound is model dependent). + """ + + output_chunk_lens: list[Annotated[int, Field(gt=0)]] = Field(default_factory=list) + """Token count of each chunk in `output_token_ids`. Parser path only. + + Replay uses these to reproduce the original `parse_delta` call + boundaries. Must sum to `len(output_token_ids)`. + """ + + @model_validator(mode="after") + def _validate_output_chunk_lens(self) -> "DerenderStreamState": + total = sum(self.output_chunk_lens) + if total != len(self.output_token_ids): + raise ValueError( + f"output_chunk_lens must sum to len(output_token_ids) " + f"(got sum={total}, len(output_token_ids)=" + f"{len(self.output_token_ids)})" + ) + return self + + tools_streamed: bool = False + """True once a tool call delta has been emitted. Parser path only. + + Drives the `finish_reason` -> `"tool_calls"` rewrite on the final + chunk mirroring the generate streaming path. + """ last_tool_call_ids: list[str] = Field(default_factory=list) - """Stable tool-call IDs, assigned once when each call first appears. + """Stable tool call IDs, assigned once when each call first appears. - Prevents ID regeneration across re-parsing. + Indexed by tool call index. Parser path only. Prevents ID regeneration + when replay reprocesses a tool call that already has a pinned ID. """ @@ -570,6 +608,19 @@ class DerenderChatStreamRequest(BaseModel): prompt_tokens: int | None = None """Prompt token count for usage. Forwarded from the render step.""" + prompt_token_ids: list[int] | None = None + """Prompt token IDs. Required by the parser path's `parse_delta` to + settle its initial reasoning state (e.g. chat templates that pre-open + ````). `prompt_tokens` is a usage count and cannot serve this + purpose. Sourced from `GenerateRequest.token_ids` at the render step. + + Rejected with a 400 (by `ServingDerender`) when a tool or reasoning + parser is configured and this is omitted. Without it, `parse_delta` + cannot tell whether the prompt left reasoning open and would silently + misclassify reasoning content as plain content. Unused on the plain + detokenization path. + """ + chat_request: ChatCompletionRequest | None = None """The original (post adjust_request) ChatCompletionRequest from /render.""" diff --git a/vllm/entrypoints/scale_out/token_in_token_out/serving.py b/vllm/entrypoints/scale_out/token_in_token_out/serving.py index 838fe90c5c..7813c7facc 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/serving.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/serving.py @@ -134,6 +134,16 @@ class ServingTokens(GenerateBaseServing): f"sampling_params.n must be at most the server's max_num_seqs " f"({max_num_seqs}), got {sampling_params.n}." ) + if self.force_no_detokenize and sampling_params.stop: + # SamplingParams rejects stop with detokenize=False at request + # validation, but this server forces detokenize=False afterwards, + # so the combination must be rejected here or stop strings are + # silently never applied. + return self.create_error_response( + "stop strings are not supported on a --tokens-only server " + "because detokenization is disabled. Check stop strings on " + "the coordinator, or use stop_token_ids." + ) try: msgspec.msgpack.encode( ( @@ -486,9 +496,13 @@ class ServingTokens(GenerateBaseServing): total_tokens=(num_prompt_tokens + num_generated_tokens[i]), ) - # Omit absent fields, like the Rust frontend and the - # final usage chunk below. - yield f"data: {chunk.model_dump_json(exclude_none=True)}\n\n" + # Omit absent prompt metadata (Rust skips None fields). + exclude = { + name + for name in ("prompt_token_ids", "mm_placeholders") + if getattr(chunk, name) is None + } + yield f"data: {chunk.model_dump_json(exclude=exclude)}\n\n" total_completion_tokens = sum(num_generated_tokens) final_usage_info = UsageInfo( diff --git a/vllm/entrypoints/serve/dev/sleep/api_router.py b/vllm/entrypoints/serve/dev/sleep/api_router.py index fedff8da74..846a487706 100644 --- a/vllm/entrypoints/serve/dev/sleep/api_router.py +++ b/vllm/entrypoints/serve/dev/sleep/api_router.py @@ -27,6 +27,12 @@ async def sleep(raw_request: Request): return Response(status_code=200) +@router.post("/release_kv_cache_memory") +async def release_kv_cache_memory(raw_request: Request): + await engine_client(raw_request).release_kv_cache_memory() + return Response(status_code=200) + + @router.post("/wake_up") async def wake_up(raw_request: Request): tags = raw_request.query_params.getlist("tags") diff --git a/vllm/entrypoints/serve/instrumentator/basic.py b/vllm/entrypoints/serve/instrumentator/basic.py index be091a1f43..64201db3f3 100644 --- a/vllm/entrypoints/serve/instrumentator/basic.py +++ b/vllm/entrypoints/serve/instrumentator/basic.py @@ -54,3 +54,10 @@ async def get_server_load_metrics(request: Request): async def show_version(): ver = {"version": VLLM_VERSION} return JSONResponse(content=ver) + + +@router.get("/kv_event_sources") +async def kv_event_sources(raw_request: Request): + """KV-event publisher config per DP rank; empty on render-only servers.""" + client: EngineClient | None = raw_request.app.state.engine_client + return client.get_kv_event_sources() if client is not None else {} diff --git a/vllm/envs.py b/vllm/envs.py index c512df457a..26c00c6da6 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -143,6 +143,7 @@ if TYPE_CHECKING: VLLM_ROCM_USE_AITER_MLA: bool = True VLLM_ROCM_AITER_MLA_ASM_PADDING: Literal["auto", "gluon", "asm"] = "auto" VLLM_ROCM_USE_AITER_MHA: bool = True + VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: bool = False VLLM_ROCM_USE_AITER_TRITON_ROPE: bool = False VLLM_ROCM_USE_AITER_FP8BMM: bool = True VLLM_ROCM_USE_AITER_FP4BMM: bool = True @@ -534,6 +535,14 @@ def get_vllm_port() -> int | None: raise ValueError(f"VLLM_PORT '{port}' must be a valid integer") from err +def _generate_shm_name() -> str: + import pybase64 + + # Fit macOS's 30-character limit without truncating the UUID. + encoded_uuid = pybase64.urlsafe_b64encode(uuid.uuid4().bytes).decode("ascii") + return "vllm_mm_" + encoded_uuid.rstrip("=") + + def get_env_or_set_default( env_name: str, default_factory: Callable[[], str], @@ -1305,6 +1314,11 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_ROCM_USE_AITER_MHA": lambda: ( os.getenv("VLLM_ROCM_USE_AITER_MHA", "True").lower() in ("true", "1") ), + # Whether to use aiter fp4 gemm asm. + # By default is disabled. + "VLLM_ROCM_USE_AITER_FP4_ASM_GEMM": lambda: ( + os.getenv("VLLM_ROCM_USE_AITER_FP4_ASM_GEMM", "False").lower() in ("true", "1") + ), # Whether to use aiter rope. # By default is disabled. "VLLM_ROCM_USE_AITER_TRITON_ROPE": lambda: ( @@ -1937,7 +1951,7 @@ environment_variables: dict[str, Callable[[], Any]] = { # if not explicitly set. "VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME": get_env_or_set_default( "VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME", - lambda: f"VLLM_OBJECT_STORAGE_SHM_BUFFER_{uuid.uuid4().hex}", + _generate_shm_name, ), # The size in MB of the buffers (NVL and RDMA) used by DeepEP "VLLM_DEEPEP_BUFFER_SIZE_MB": lambda: int( diff --git a/vllm/inputs/engine.py b/vllm/inputs/engine.py index 429e8a829d..26b840f76f 100644 --- a/vllm/inputs/engine.py +++ b/vllm/inputs/engine.py @@ -43,11 +43,6 @@ class TokensInput(_InputOptions): """Char-level (start, end) offsets per token, propagated from the renderer's TokensPrompt when offsets were computed.""" - assistant_tokens_mask: NotRequired[list[int] | None] - """Per-token 0/1 mask marking assistant-generated tokens. - Populated when ``return_assistant_tokens_mask=True`` is set on the - render request and the chat template supports ``{% generation %}``.""" - def tokens_input( prompt_token_ids: list[int], @@ -155,11 +150,6 @@ class MultiModalInput(_InputOptions): `prompt_token_ids`. """ - assistant_tokens_mask: NotRequired[list[int] | None] - """Per-token 0/1 mask marking assistant-generated tokens. - Populated when ``return_assistant_tokens_mask=True`` is set on the - render request and the chat template supports ``{% generation %}``.""" - def mm_input( prompt_token_ids: list[int], diff --git a/vllm/model_executor/determinism/batch_invariant_configs.py b/vllm/model_executor/determinism/batch_invariant_configs.py index 39cda984f7..874447bb82 100644 --- a/vllm/model_executor/determinism/batch_invariant_configs.py +++ b/vllm/model_executor/determinism/batch_invariant_configs.py @@ -254,6 +254,233 @@ _BATCH_INVARIANT_MATMUL_TUNED_CONFIGS: dict[ ), ), }, + "sm120": { + (12288, 2048): _MatmulShapeConfig( # Qwen3-1.7B gate_up_proj + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 128, 8, 5)), + (4, _MatmulMConfig(16, 128, 8, 4)), + (8, _MatmulMConfig(16, 128, 8, 4)), + (16, _MatmulMConfig(16, 128, 4, 4)), + (32, _MatmulMConfig(32, 128, 8, 5)), + (64, _MatmulMConfig(64, 128, 4, 3)), + (256, _MatmulMConfig(64, 64, 4, 4)), + (512, _MatmulMConfig(64, 128, 4, 5)), + (1024, _MatmulMConfig(128, 128, 4, 3)), + (2048, _MatmulMConfig(128, 128, 4, 3)), + ), + ), + (2048, 6144): _MatmulShapeConfig( # Qwen3-1.7B down_proj + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 32, 8, 5)), + (4, _MatmulMConfig(16, 32, 8, 3)), + (8, _MatmulMConfig(16, 32, 8, 4)), + (16, _MatmulMConfig(16, 32, 4, 5)), + (32, _MatmulMConfig(32, 32, 4, 5)), + (64, _MatmulMConfig(32, 32, 4, 4)), + (256, _MatmulMConfig(64, 64, 4, 5)), + (512, _MatmulMConfig(128, 64, 4, 3)), + (1024, _MatmulMConfig(128, 128, 4, 3)), + (2048, _MatmulMConfig(128, 64, 8, 4)), + ), + ), + (4096, 2048): _MatmulShapeConfig( # Qwen3-1.7B qkv_proj + block_k=128, + m_buckets=( + (1, _MatmulMConfig(16, 64, 4, 4)), + (4, _MatmulMConfig(16, 64, 8, 5)), + (8, _MatmulMConfig(16, 64, 8, 5)), + (16, _MatmulMConfig(16, 64, 8, 3)), + (32, _MatmulMConfig(16, 64, 8, 5)), + (64, _MatmulMConfig(32, 64, 4, 2)), + (256, _MatmulMConfig(64, 128, 4, 3)), + (512, _MatmulMConfig(64, 64, 4, 3)), + (1024, _MatmulMConfig(64, 128, 4, 3)), + (2048, _MatmulMConfig(64, 128, 4, 3)), + ), + ), + (151936, 2048): _MatmulShapeConfig( # Qwen3-1.7B lm_head + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 64, 8, 5)), + (4, _MatmulMConfig(16, 128, 4, 4)), + (8, _MatmulMConfig(16, 128, 4, 5)), + (16, _MatmulMConfig(16, 128, 8, 3)), + (32, _MatmulMConfig(32, 128, 8, 3)), + (64, _MatmulMConfig(32, 128, 8, 5)), + (256, _MatmulMConfig(128, 128, 4, 3)), + (512, _MatmulMConfig(128, 128, 8, 3)), + (1024, _MatmulMConfig(128, 128, 4, 3)), + (2048, _MatmulMConfig(128, 128, 4, 3)), + ), + ), + (2048, 2048): _MatmulShapeConfig( # Qwen3-1.7B o_proj + block_k=32, + m_buckets=( + (1, _MatmulMConfig(16, 128, 4, 3)), + (4, _MatmulMConfig(16, 128, 4, 4)), + (8, _MatmulMConfig(16, 64, 4, 5)), + (16, _MatmulMConfig(32, 32, 8, 4)), + (32, _MatmulMConfig(16, 64, 4, 5)), + (64, _MatmulMConfig(32, 32, 4, 5)), + (256, _MatmulMConfig(128, 32, 4, 5)), + (512, _MatmulMConfig(128, 64, 4, 4)), + (1024, _MatmulMConfig(128, 128, 8, 5)), + (2048, _MatmulMConfig(128, 256, 8, 4)), + ), + ), + (19456, 2560): _MatmulShapeConfig( # Qwen3-4B gate_up_proj + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 128, 8, 4)), + (4, _MatmulMConfig(16, 128, 8, 3)), + (8, _MatmulMConfig(16, 128, 8, 5)), + (16, _MatmulMConfig(16, 128, 8, 5)), + (32, _MatmulMConfig(32, 128, 4, 4)), + (64, _MatmulMConfig(64, 128, 8, 4)), + (256, _MatmulMConfig(128, 128, 4, 3)), + (512, _MatmulMConfig(64, 128, 4, 5)), + (1024, _MatmulMConfig(128, 128, 4, 3)), + (2048, _MatmulMConfig(128, 128, 4, 3)), + ), + ), + (2560, 9728): _MatmulShapeConfig( # Qwen3-4B down_proj + block_k=128, + m_buckets=( + (1, _MatmulMConfig(16, 64, 4, 5)), + (4, _MatmulMConfig(16, 64, 4, 4)), + (8, _MatmulMConfig(16, 64, 4, 4)), + (16, _MatmulMConfig(16, 32, 4, 5)), + (32, _MatmulMConfig(32, 32, 4, 3)), + (64, _MatmulMConfig(32, 64, 8, 4)), + (256, _MatmulMConfig(64, 64, 4, 3)), + (512, _MatmulMConfig(64, 128, 4, 3)), + (1024, _MatmulMConfig(64, 128, 4, 3)), + (2048, _MatmulMConfig(64, 128, 4, 3)), + ), + ), + (6144, 2560): _MatmulShapeConfig( # Qwen3-4B qkv_proj + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 128, 8, 4)), + (4, _MatmulMConfig(16, 256, 4, 3)), + (8, _MatmulMConfig(16, 128, 4, 5)), + (16, _MatmulMConfig(16, 64, 8, 3)), + (32, _MatmulMConfig(16, 128, 8, 4)), + (64, _MatmulMConfig(64, 64, 4, 5)), + (256, _MatmulMConfig(128, 128, 8, 3)), + (512, _MatmulMConfig(64, 64, 4, 5)), + (1024, _MatmulMConfig(64, 128, 8, 4)), + (2048, _MatmulMConfig(128, 128, 4, 3)), + ), + ), + (151936, 2560): _MatmulShapeConfig( # Qwen3-4B lm_head + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 64, 8, 5)), + (4, _MatmulMConfig(16, 64, 8, 5)), + (8, _MatmulMConfig(16, 64, 4, 5)), + (16, _MatmulMConfig(16, 64, 4, 5)), + (32, _MatmulMConfig(32, 64, 4, 5)), + (64, _MatmulMConfig(64, 64, 4, 5)), + (256, _MatmulMConfig(128, 128, 4, 3)), + (512, _MatmulMConfig(128, 128, 4, 3)), + (1024, _MatmulMConfig(128, 128, 8, 3)), + (2048, _MatmulMConfig(128, 128, 4, 3)), + ), + ), + (2560, 4096): _MatmulShapeConfig( # Qwen3-4B o_proj + block_k=128, + m_buckets=( + (1, _MatmulMConfig(16, 64, 8, 3)), + (4, _MatmulMConfig(16, 64, 8, 4)), + (8, _MatmulMConfig(16, 64, 8, 5)), + (16, _MatmulMConfig(16, 64, 4, 4)), + (32, _MatmulMConfig(16, 32, 4, 4)), + (64, _MatmulMConfig(16, 64, 8, 3)), + (256, _MatmulMConfig(64, 64, 8, 3)), + (512, _MatmulMConfig(64, 128, 4, 3)), + (1024, _MatmulMConfig(64, 128, 4, 3)), + (2048, _MatmulMConfig(64, 128, 4, 3)), + ), + ), + (24576, 4096): _MatmulShapeConfig( # Qwen3-8B gate_up_proj + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 256, 8, 3)), + (4, _MatmulMConfig(16, 256, 8, 3)), + (8, _MatmulMConfig(16, 256, 8, 3)), + (16, _MatmulMConfig(16, 256, 8, 3)), + (32, _MatmulMConfig(32, 256, 8, 3)), + (64, _MatmulMConfig(64, 256, 4, 3)), + (256, _MatmulMConfig(64, 128, 4, 5)), + (512, _MatmulMConfig(128, 128, 4, 3)), + (1024, _MatmulMConfig(128, 128, 4, 3)), + (2048, _MatmulMConfig(128, 128, 4, 3)), + ), + ), + (4096, 12288): _MatmulShapeConfig( # Qwen3-8B down_proj + block_k=128, + m_buckets=( + (1, _MatmulMConfig(16, 64, 4, 5)), + (4, _MatmulMConfig(16, 64, 4, 5)), + (8, _MatmulMConfig(16, 64, 4, 4)), + (16, _MatmulMConfig(16, 32, 4, 4)), + (32, _MatmulMConfig(32, 32, 8, 5)), + (64, _MatmulMConfig(32, 64, 4, 5)), + (256, _MatmulMConfig(64, 128, 4, 3)), + (512, _MatmulMConfig(64, 64, 4, 3)), + (1024, _MatmulMConfig(64, 128, 4, 3)), + (2048, _MatmulMConfig(64, 128, 4, 3)), + ), + ), + (6144, 4096): _MatmulShapeConfig( # Qwen3-8B qkv_proj + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 64, 8, 4)), + (4, _MatmulMConfig(16, 128, 8, 4)), + (8, _MatmulMConfig(16, 128, 4, 4)), + (16, _MatmulMConfig(16, 64, 4, 5)), + (32, _MatmulMConfig(16, 128, 8, 5)), + (64, _MatmulMConfig(64, 64, 4, 4)), + (256, _MatmulMConfig(128, 128, 8, 3)), + (512, _MatmulMConfig(64, 64, 4, 5)), + (1024, _MatmulMConfig(128, 64, 4, 3)), + (2048, _MatmulMConfig(128, 128, 4, 3)), + ), + ), + (151936, 4096): _MatmulShapeConfig( # Qwen3-8B lm_head + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 64, 8, 5)), + (4, _MatmulMConfig(16, 64, 8, 5)), + (8, _MatmulMConfig(16, 64, 4, 5)), + (16, _MatmulMConfig(16, 64, 4, 5)), + (32, _MatmulMConfig(32, 64, 4, 5)), + (64, _MatmulMConfig(32, 128, 8, 5)), + (256, _MatmulMConfig(128, 128, 4, 3)), + (512, _MatmulMConfig(128, 128, 8, 3)), + (1024, _MatmulMConfig(128, 128, 4, 3)), + (2048, _MatmulMConfig(128, 128, 4, 3)), + ), + ), + (4096, 4096): _MatmulShapeConfig( # Qwen3-8B o_proj + block_k=128, + m_buckets=( + (1, _MatmulMConfig(16, 64, 8, 4)), + (4, _MatmulMConfig(16, 64, 4, 3)), + (8, _MatmulMConfig(16, 32, 4, 4)), + (16, _MatmulMConfig(16, 32, 4, 3)), + (32, _MatmulMConfig(16, 64, 4, 5)), + (64, _MatmulMConfig(64, 32, 8, 4)), + (256, _MatmulMConfig(64, 128, 4, 3)), + (512, _MatmulMConfig(64, 64, 4, 3)), + (1024, _MatmulMConfig(64, 128, 4, 3)), + (2048, _MatmulMConfig(64, 128, 4, 3)), + ), + ), + }, } _TUNED_MATMUL_CONFIGS_FOR_DEVICE: dict[tuple[int, int], _MatmulShapeConfig] | None = ( @@ -265,6 +492,8 @@ _TUNED_MATMUL_CONFIGS_RESOLVED = False def _get_tuned_matmul_arch_family(capability: DeviceCapability | None) -> str | None: if capability is None: return None + if capability.major == 12: + return "sm120" if capability.major == 10: return "blackwell" if capability.major == 9: diff --git a/vllm/model_executor/kernels/attention/dsa/candidate_blocks.py b/vllm/model_executor/kernels/attention/dsa/candidate_blocks.py index efc7c418ca..b9cdc923b1 100644 --- a/vllm/model_executor/kernels/attention/dsa/candidate_blocks.py +++ b/vllm/model_executor/kernels/attention/dsa/candidate_blocks.py @@ -66,9 +66,10 @@ def _store_candidates_kernel( cols = tl.program_id(1) * TILE + tl.arange(0, TILE) value = tl.load(values + row * k + cols, cols < k, other=-float("inf")) index = tl.load(indices + row * k + cols, cols < k, other=-1) + # NaN scores can occur during warmup; only -inf denotes padding. tl.store( output + row * out_stride_row + cols * out_stride_col, - tl.where(value > -float("inf"), index, -1), + tl.where(value != -float("inf"), index, -1), cols < OUT_K, ) diff --git a/vllm/model_executor/kernels/attention/dsa/sparse_mqa_logits.py b/vllm/model_executor/kernels/attention/dsa/sparse_mqa_logits.py index 0c2ea1aef7..345c711ae6 100644 --- a/vllm/model_executor/kernels/attention/dsa/sparse_mqa_logits.py +++ b/vllm/model_executor/kernels/attention/dsa/sparse_mqa_logits.py @@ -73,7 +73,7 @@ def check_deep_select_layout(num_sparse_cols: int, topk_tokens: int) -> None: _INT32_BIG = tl.constexpr(2**31 - 1) -@triton.jit +@triton.jit(do_not_specialize_on_alignment=["end_ptr"]) def _expand_candidates_kernel( cand_ptr, cand_stride, diff --git a/vllm/model_executor/kernels/linear/__init__.py b/vllm/model_executor/kernels/linear/__init__.py index 3c9c616d0c..fb7a170b82 100644 --- a/vllm/model_executor/kernels/linear/__init__.py +++ b/vllm/model_executor/kernels/linear/__init__.py @@ -186,6 +186,8 @@ from vllm.model_executor.kernels.linear.scaled_mm.b12x import ( from vllm.model_executor.kernels.linear.scaled_mm.cpu import ( CPUFp8BlockScaledMMKernel, CPUFp8PerTensorScaledMMLinearKernel, + CPUFp8W8A8BlockScaledMMKernel, + CPUFP8W8A8ScaledMMLinearKernel, CPUInt8ScaledMMLinearKernel, ) from vllm.model_executor.kernels.linear.scaled_mm.cutlass import ( @@ -436,6 +438,7 @@ _POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] = ChannelWiseTorchFP8ScaledMMLinearKernel, ], PlatformEnum.CPU: [ + CPUFP8W8A8ScaledMMLinearKernel, CPUFp8PerTensorScaledMMLinearKernel, PerTensorTorchFP8ScaledMMLinearKernel, ChannelWiseTorchFP8ScaledMMLinearKernel, @@ -468,6 +471,7 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[ TritonFp8BlockScaledMMKernel, ], PlatformEnum.CPU: [ + CPUFp8W8A8BlockScaledMMKernel, # W8A8 preferred; falls back to W8A16 below CPUFp8BlockScaledMMKernel, ], PlatformEnum.XPU: [ @@ -864,6 +868,7 @@ def choose_mp_linear_kernel( can_implement, failure_reason = kernel.can_implement(config) if can_implement: + logger.info_once("Using %s for mixed-precision linear", kernel.__name__) return kernel else: failure_reasons.append( diff --git a/vllm/model_executor/kernels/linear/mxfp8/Mxfp8LinearKernel.py b/vllm/model_executor/kernels/linear/mxfp8/Mxfp8LinearKernel.py index 3fe34029e5..fdcfeb264b 100644 --- a/vllm/model_executor/kernels/linear/mxfp8/Mxfp8LinearKernel.py +++ b/vllm/model_executor/kernels/linear/mxfp8/Mxfp8LinearKernel.py @@ -27,6 +27,10 @@ class Mxfp8LinearKernel(ABC): Marlin, emulation). """ + supports_pre_processed_weights: bool = False + """True if ``process_weights_after_loading`` only rewrites parameters, so + weights exported by the weight cache daemon can be used as-is.""" + def __init__(self, c: Mxfp8LinearLayerConfig) -> None: assert self.can_implement(c)[0] assert self.is_supported()[0] diff --git a/vllm/model_executor/kernels/linear/mxfp8/emulation.py b/vllm/model_executor/kernels/linear/mxfp8/emulation.py index 79b2fba388..da48655d37 100644 --- a/vllm/model_executor/kernels/linear/mxfp8/emulation.py +++ b/vllm/model_executor/kernels/linear/mxfp8/emulation.py @@ -16,6 +16,8 @@ from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig class EmulationMxfp8LinearKernel(Mxfp8LinearKernel): """Software emulation fallback for MXFP8 (dequant to BF16).""" + supports_pre_processed_weights = True + @classmethod def is_supported( cls, compute_capability: int | None = None diff --git a/vllm/model_executor/kernels/linear/mxfp8/flashinfer.py b/vllm/model_executor/kernels/linear/mxfp8/flashinfer.py index 02700a0ced..cc9b35d34b 100644 --- a/vllm/model_executor/kernels/linear/mxfp8/flashinfer.py +++ b/vllm/model_executor/kernels/linear/mxfp8/flashinfer.py @@ -27,6 +27,8 @@ from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig class FlashInferCutlassMxfp8LinearKernel(Mxfp8LinearKernel): """MXFP8 W8A8 GEMM via FlashInfer CUTLASS (SM100+).""" + supports_pre_processed_weights = True + @classmethod def is_supported( cls, compute_capability: int | None = None @@ -117,6 +119,8 @@ class FlashInferCutlassMxfp8LinearKernel(Mxfp8LinearKernel): class FlashInferCutedslMxfp8LinearKernel(Mxfp8LinearKernel): """MXFP8 W8A8 GEMM via FlashInfer CuTe-DSL (SM100/SM103).""" + supports_pre_processed_weights = True + @classmethod def is_supported( cls, compute_capability: int | None = None diff --git a/vllm/model_executor/kernels/linear/mxfp8/rocm_native.py b/vllm/model_executor/kernels/linear/mxfp8/rocm_native.py index 29739afae3..c7e6537f49 100644 --- a/vllm/model_executor/kernels/linear/mxfp8/rocm_native.py +++ b/vllm/model_executor/kernels/linear/mxfp8/rocm_native.py @@ -119,6 +119,18 @@ def _mxfp8_dot_scaled_linear( return out +# Triton 3.8 enables TRITON_HIP_USE_ASYNC_COPY by default on gfx950; its extra LDS +# buffer leaves no room for num_stages=3 at BLOCK_K=256. +# TODO(rasmith)(Rohan138): Remove the 3.8 check once +# https://github.com/vllm-project/vllm/pull/50605 merges. +_BK256_STAGES = 3 +if triton.__version__.startswith("3.8") and current_platform.is_rocm(): + from vllm.platforms.rocm import on_gfx950 + + if on_gfx950(): + _BK256_STAGES = 2 + + def _select_cfg(M, N, K): """(BLOCK_M, BLOCK_N, BLOCK_K, num_warps, num_stages) — graph-tuned on gfx950. @@ -166,7 +178,7 @@ def _select_cfg(M, N, K): # 3.6; on triton 3.7 its large BLOCK_M register/LDS footprint spills or hits # "out of resources", so use 128x128x256 -- within the known-good footprint.) if M >= 4096 and K >= 1024 and K % 256 == 0 and occ >= 256: - return 128, 128, 256, 8, 3 + return 128, 128, 256, 8, _BK256_STAGES return 128, 128, 128, 8, 3 # large-K (K >= 2048). BLOCK_K is K-divisibility-guarded (the K-loop is unmasked): # served large-K is 2048/6144 (%256==0), but fall back to 128 (always divides, since @@ -186,7 +198,7 @@ def _select_cfg(M, N, K): # Covers the qkv-class local N=1536 (TP=8 qkv / TP=4 shared_gate_up) and the deep-K # / very-large-M shapes. if K % 256 == 0 and (1280 < N <= 1536 or (occ >= 128 and (K >= 4096 or M >= 4096))): - return 128, 128, 256, 8, 3 + return 128, 128, 256, 8, _BK256_STAGES # small local-N (e.g. TP=8 shared_gate_up N=768): a 64-wide BLOCK_N doubles the # N-tile count -> better CU fill than 128x128 at this mid-large M (~1.4x there). if N <= 1024 and K % 256 == 0: @@ -200,6 +212,8 @@ _DOT_SCALED_K_ALIGN = 128 class RocmDotScaledMxfp8LinearKernel(Mxfp8LinearKernel): """Native CDNA4 (gfx950) MXFP8 linear via Triton ``tl.dot_scaled``.""" + supports_pre_processed_weights = True + @classmethod def is_supported( cls, compute_capability: int | None = None diff --git a/vllm/model_executor/kernels/linear/scaled_mm/__init__.py b/vllm/model_executor/kernels/linear/scaled_mm/__init__.py index f4a062b7f2..75db4bcaa9 100644 --- a/vllm/model_executor/kernels/linear/scaled_mm/__init__.py +++ b/vllm/model_executor/kernels/linear/scaled_mm/__init__.py @@ -10,6 +10,7 @@ from vllm.model_executor.kernels.linear.scaled_mm.BlockScaledMMLinearKernel impo from vllm.model_executor.kernels.linear.scaled_mm.cpu import ( CPUFp8BlockScaledMMKernel, CPUFp8PerTensorScaledMMLinearKernel, + CPUFP8W8A8ScaledMMLinearKernel, CPUInt8ScaledMMLinearKernel, ) from vllm.model_executor.kernels.linear.scaled_mm.cutlass import ( @@ -69,5 +70,6 @@ __all__ = [ "Fp8BlockScaledMMLinearKernel", "CPUFp8BlockScaledMMKernel", "CPUFp8PerTensorScaledMMLinearKernel", + "CPUFP8W8A8ScaledMMLinearKernel", "XPUFp8BlockScaledMMKernel", ] diff --git a/vllm/model_executor/kernels/linear/scaled_mm/cpu.py b/vllm/model_executor/kernels/linear/scaled_mm/cpu.py index e72dffcbbc..1d2ede1853 100644 --- a/vllm/model_executor/kernels/linear/scaled_mm/cpu.py +++ b/vllm/model_executor/kernels/linear/scaled_mm/cpu.py @@ -270,12 +270,17 @@ class CPUFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel): # Skip the base class process (FP8 padding / fnuz normalization) # which is GPU-oriented. Instead, VNNI-prepack weights for AMX. params = self._get_layer_params(layer) - packed_weight = torch.ops._C.convert_weight_packed(params.weight) - replace_parameter( - layer, - params.WEIGHT, - torch.nn.Parameter(packed_weight, requires_grad=False), - ) + + # MLA's kv_b_proj (see `_cpu_skip_gemm_dispatch`): MLA reads this + # weight to derive W_UK/W_UV, so keep the checkpoint [N, K] layout and + # let the kernel pack on the fly instead. + if not getattr(layer, "_cpu_skip_gemm_dispatch", False): + packed_weight = torch.ops._C.convert_weight_packed(params.weight) + replace_parameter( + layer, + params.WEIGHT, + torch.nn.Parameter(packed_weight, requires_grad=False), + ) # Re-wrap scale as a plain Parameter so the kernel can read it # without weight-loader metadata interfering. @@ -320,7 +325,7 @@ class CPUFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel): list(self.weight_group_shape), bias, x.dtype, - True, # is_vnni (weight already prepacked) + not getattr(layer, "_cpu_skip_gemm_dispatch", False), # is_vnni ) return out.reshape(x.shape[:-1] + (out.size(-1),)) if x.dim() > 2 else out @@ -430,3 +435,294 @@ class CPUFp8PerTensorScaledMMLinearKernel(FP8ScaledMMLinearKernel): raise NotImplementedError( "CPUFp8PerTensorScaledMMLinearKernel overrides apply_weights directly." ) + + +class CPUFp8W8A8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel): + """FP8 W8A8 block-quantized GEMM with dynamic per-token activation quant on CPU.""" + + # Activation quant is done inside fp8_scaled_mm_with_quant; + # the base class must not apply a second quant step. + apply_input_quant = False + + @classmethod + def is_supported( + cls, compute_capability: int | None = None + ) -> tuple[bool, str | None]: + if not current_platform.is_cpu(): + return False, "requires CPU platform." + if not torch.cpu._is_amx_tile_supported(): + return False, "requires AMX tile support (Sapphire Rapids or newer)." + if not ops.cpu_has_amx_fp8(): + return False, ( + "requires native AMX-FP8 hardware; falls back " + "to CPUFp8BlockScaledMMKernel (W8A16) otherwise." + ) + return True, None + + @classmethod + def can_implement( + cls, config: FP8ScaledMMLinearLayerConfig + ) -> tuple[bool, str | None]: + # Same block-shape requirements as the W8A16 sibling kernel. + weight_gs = config.weight_quant_key.scale.group_shape + if weight_gs.col <= 0 or weight_gs.col != 128: + return False, ( + "CPU FP8 W8A8 block kernel requires K-dimension block size of 128, " + f"got {weight_gs.col}." + ) + if weight_gs.row <= 0 or weight_gs.row % 32 != 0: + return False, ( + "CPU FP8 W8A8 block kernel requires N-dimension block size to be " + f"a positive multiple of 32, got {weight_gs.row}." + ) + if config.out_dtype not in (torch.bfloat16, torch.float32): + return False, "Only bfloat16/float32 output dtype supported." + n = config.weight_shape[0] + if n % 32 != 0: + # float8_linear_prepack_cpu tiles N in chunks of 32 and cannot + # handle a remainder tile smaller than that. + return False, f"requires weight output dim (N={n}) to be a multiple of 32." + return True, None + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + """Prepack block-quantized FP8 weight for W8A8 AMX BRGEMM. + + For block-quant FP8 models weight is stored as [N, K] (NOT transposed, + unlike non-block FP8 where vLLM stores [K, N]). + + float8_linear_prepack_cpu expects: + weight: [N, K] + scales: [N, G] where G = K / block_k + + weight_scale_inv from the checkpoint has shape [ceil(N/block_n), G], + so we expand rows from ceil(N/block_n) to N via repeat_interleave. + """ + params = self._get_layer_params(layer) + weight = params.weight # [N, K] + + # MLA's kv_b_proj (see `_cpu_skip_gemm_dispatch`): MLA reads this + # weight to derive W_UK/W_UV, so keep the checkpoint [N, K] layout and + # its block scales; apply_weights falls back to the W8A16 op. + if getattr(layer, "_cpu_skip_gemm_dispatch", False): + return + + weight_scale = ( + params.weight_scale_inv + if params.weight_scale_inv is not None + else params.weight_scale + ) # [ceil(N/block_n), ceil(K/block_k)] + assert weight_scale is not None, "FP8 W8A8 block kernel requires weight scale." + + N = weight.shape[0] + block_n = self.weight_group_shape.row # e.g. 128 + + # Expand scale rows: [ceil(N/block_n), G] → [N, G] + if weight_scale.size(0) != N: + weight_scale_expanded = weight_scale.repeat_interleave(block_n, dim=0)[ + :N, : + ].contiguous() + else: + weight_scale_expanded = weight_scale.contiguous() + + packed_weight, packed_scales = torch.ops._C.float8_linear_prepack_cpu( + weight, weight_scale_expanded.to(torch.float32) + ) + + scale_attr = ( + params.WEIGHT_SCALE_INV + if params.weight_scale_inv is not None + else params.WEIGHT_SCALE + ) + replace_parameter( + layer, + params.WEIGHT, + torch.nn.Parameter(packed_weight, requires_grad=False), + ) + replace_parameter( + layer, + scale_attr, + torch.nn.Parameter(packed_scales, requires_grad=False), + ) + + def apply_weights( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: torch.Tensor | None = None, + **kwargs, + ) -> torch.Tensor: + params = self._get_layer_params(layer) + weight_scale = ( + params.weight_scale_inv + if params.weight_scale_inv is not None + else params.weight_scale + ) + + x_2d = x.reshape(-1, x.shape[-1]) if x.dim() > 2 else x + if getattr(layer, "_cpu_skip_gemm_dispatch", False): + # Unpacked weight: fp8_scaled_mm_with_quant has no is_vnni option, + # so fall back to the W8A16 op (BF16 activation, block-scaled FP8 + # weight) for this layer. + out = torch.ops._C.fp8_scaled_mm_cpu( + x_2d, + params.weight, + weight_scale, + list(self.weight_group_shape), + bias, + x.dtype, + False, # is_vnni + ) + return out.reshape(x.shape[:-1] + (out.size(-1),)) if x.dim() > 2 else out + + # act_scales=None + channelwise=True → dynamic per-token activation quant + out = ops.fp8_scaled_mm_with_quant( + x_2d, + None, + True, + params.weight, + weight_scale, + bias, + x.dtype, + ) + return out.reshape(x.shape[:-1] + (out.size(-1),)) if x.dim() > 2 else out + + def apply_block_scaled_mm( + self, + A: torch.Tensor, + B: torch.Tensor, + As: torch.Tensor, + Bs: torch.Tensor, + ) -> torch.Tensor: + raise NotImplementedError( + "CPUFp8W8A8BlockScaledMMKernel overrides apply_weights directly." + ) + + +class CPUFP8W8A8ScaledMMLinearKernel(FP8ScaledMMLinearKernel): + """FP8 W8A8 GEMM with dynamic per-token activation quantization on CPU.""" + + @classmethod + def is_supported( + cls, compute_capability: int | None = None + ) -> tuple[bool, str | None]: + if not current_platform.is_cpu(): + return False, "requires CPU platform." + if not torch.cpu._is_amx_tile_supported(): + return False, "requires AMX tile support (Sapphire Rapids or newer)." + if not ops.cpu_has_amx_fp8(): + return False, ( + "requires native AMX-FP8 hardware (Diamond Rapids); falls back " + "to the Torch FP8 kernels otherwise." + ) + return True, None + + @classmethod + def can_implement( + cls, config: FP8ScaledMMLinearLayerConfig + ) -> tuple[bool, str | None]: + if config.out_dtype not in (torch.bfloat16, torch.float16, torch.float32): + return False, "Only bfloat16/float16/float32 output dtype supported." + # Reject block-quantized weights — those go to CPUFp8BlockScaledMMKernel + weight_gs = config.weight_quant_key.scale.group_shape + if weight_gs.col > 0 and weight_gs.row > 0: + return False, "Block-quantized weights not handled by FP8 W8A8 kernel." + n = config.weight_shape[0] + if n % 32 != 0: + # float8_linear_prepack_cpu tiles N in chunks of 32 and cannot + # handle a remainder tile smaller than that. + return False, f"requires weight output dim (N={n}) to be a multiple of 32." + return True, None + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + """Prepack weights with float8_linear_prepack_cpu (VNNI + block layout).""" + w_name, ws_name, xs_name, _ = self.layer_param_names + weight = getattr(layer, w_name) + weight_scale = getattr(layer, ws_name) + + # vLLM stores linear weight as [K, N], while the CPU prepack op expects [N, K]. + N = weight.shape[-1] + + # Ensure scale is 2D [N, G] + ws = weight_scale + if ws is None: + raise RuntimeError("FP8 W8A8 kernel requires weight scale.") + ws = ws.to(torch.float32) + + # Fused modules may carry per-shard scales (e.g. QKV/MLP shards). + # Expand those to per-output-channel scales before prepack. + if ( + len(getattr(layer, "logical_widths", [])) > 1 + and ws.dim() <= 1 + and ws.numel() == len(layer.logical_widths) + ): + ws = convert_to_channelwise(ws.reshape(-1), layer.logical_widths) + + if ws.dim() == 0: + ws = ws.view(1, 1).expand(N, 1) + elif ws.dim() == 1: + if ws.numel() == 1: + ws = ws.view(1, 1).expand(N, 1) + elif ws.numel() == N: + ws = ws.unsqueeze(1) # [N] -> [N, 1] + else: + raise RuntimeError( + "FP8 W8A8 kernel got unsupported 1D weight scale shape: " + f"{tuple(ws.shape)} for N={N}." + ) + + # Align scale rows to output channels N. + if ws.size(0) != N: + if ws.size(0) == 1: + ws = ws.expand(N, ws.size(1)).contiguous() + else: + raise RuntimeError( + "FP8 W8A8 kernel got mismatched weight scale rows: " + f"ws_rows={ws.size(0)}, expected N={N}." + ) + + w_2d = weight.t().contiguous() # [N, K] + packed_weight, packed_scales = torch.ops._C.float8_linear_prepack_cpu(w_2d, ws) + + replace_parameter( + layer, + w_name, + torch.nn.Parameter(packed_weight, requires_grad=False), + ) + replace_parameter( + layer, + ws_name, + torch.nn.Parameter(packed_scales, requires_grad=False), + ) + + def apply_weights( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: torch.Tensor | None = None, + **kwargs, + ) -> torch.Tensor: + w_name, ws_name, xs_name, _ = self.layer_param_names + weight = getattr(layer, w_name) + weight_scale = getattr(layer, ws_name) + # For static quantization, use pre-computed activation scale + act_scales = getattr(layer, xs_name, None) if xs_name else None + + x_2d = x.reshape(-1, x.shape[-1]) if x.dim() > 2 else x + # channelwise=True: per-token (per-row) quantization when act_scales is None + # channelwise=False: per-tensor quantization when act_scales is provided + channelwise = act_scales is None + out = ops.fp8_scaled_mm_with_quant( + x_2d, + act_scales, + channelwise, + weight, + weight_scale, + bias, + x.dtype, + ) + return out.reshape(x.shape[:-1] + (out.size(-1),)) if x.dim() > 2 else out + + def apply_scaled_mm(self, A, B, out_dtype, As, Bs, bias=None, **kwargs): + raise NotImplementedError( + "CPUFP8W8A8ScaledMMLinearKernel uses apply_weights directly." + ) diff --git a/vllm/model_executor/layers/activation.py b/vllm/model_executor/layers/activation.py index 7166d3f8fe..35a6fdf9b0 100644 --- a/vllm/model_executor/layers/activation.py +++ b/vllm/model_executor/layers/activation.py @@ -129,8 +129,11 @@ class SiluAndMul(CustomOp): super().__init__(compile_native=compile_native) if ( current_platform.is_cuda_alike() - or current_platform.is_cpu() or current_platform.is_xpu() + or ( + current_platform.is_cpu() + and current_platform.get_cpu_architecture() == CpuArchEnum.POWERPC + ) ): self.op = torch.ops._C.silu_and_mul @@ -424,8 +427,11 @@ class GeluAndMul(CustomOp): raise ValueError(f"Unknown approximate mode: {approximate}") if ( current_platform.is_cuda_alike() - or current_platform.is_cpu() or current_platform.is_xpu() + or ( + current_platform.is_cpu() + and current_platform.get_cpu_architecture() == CpuArchEnum.POWERPC + ) ): if approximate == "none": self.op = torch.ops._C.gelu_and_mul diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index 26547c7353..3104ac25c6 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -1055,7 +1055,7 @@ class MLAAttention(nn.Module, AttentionLayerBase): mqa_q_nope, self.impl._w_uk_packed, # type: ignore[attr-defined] True, - None, + self.impl._w_scale, # type: ignore[attr-defined] ) mqa_ql_nope = mqa_ql_nope.transpose(0, 1) else: @@ -1401,7 +1401,7 @@ class MLAAttention(nn.Module, AttentionLayerBase): x, self.impl._w_uv_packed, # type: ignore[attr-defined] True, - None, + self.impl._w_scale, # type: ignore[attr-defined] ) else: # Multiply + Transpose (N, B, L) x (N, L, V)->(N, B, V)->(B, N, V) diff --git a/vllm/model_executor/layers/fused_moe/all2all_utils.py b/vllm/model_executor/layers/fused_moe/all2all_utils.py index 775f595bfa..eb7edaef53 100644 --- a/vllm/model_executor/layers/fused_moe/all2all_utils.py +++ b/vllm/model_executor/layers/fused_moe/all2all_utils.py @@ -35,6 +35,7 @@ from vllm.platforms import current_platform from vllm.utils.import_utils import ( has_deep_ep, has_deep_ep_v2, + has_moonep, has_mori, has_nixl_ep, ) @@ -103,6 +104,13 @@ if current_platform.is_cuda_alike(): NIXL_EP_QUANT_BLOCK_SHAPE, NixlEPPrepareAndFinalize, ) + if has_moonep(): + from .prepare_finalize.moonep import ( + MOONEP_DEFAULT_NUM_PREFETCH_SLOTS, + MOONEP_DEFAULT_NUM_SMS, + MOONEP_DEFAULT_TOKEN_PADDING, + MoonEPPrepareAndFinalize, + ) def get_ep_all2all_manager( @@ -279,6 +287,28 @@ def maybe_make_prepare_finalize( sp_size=moe.moe_parallel_config.sp_size, ) + elif moe.use_moonep_kernels: + all_to_all_args = dict( + max_num_tokens_per_dp_rank=moe.max_num_tokens, + token_hidden_size=moe.hidden_dim, + num_topk=moe.experts_per_token, + num_global_experts=moe.num_experts, + num_prefetch_slots=MOONEP_DEFAULT_NUM_PREFETCH_SLOTS, + token_padding=MOONEP_DEFAULT_TOKEN_PADDING, + num_sms=MOONEP_DEFAULT_NUM_SMS, + ) + handle = all2all_manager.get_handle(all_to_all_args) + + # The [E+B] weight layout is picked up from the experts (their + # process_weights_after_loading hook) once the layer has loaded and + # converted its weights. + prepare_finalize = MoonEPPrepareAndFinalize( + buffer_pool=handle, + max_tokens_per_rank=moe.max_num_tokens, + num_dispatchers=all2all_manager.world_size, + num_global_experts=moe.num_experts, + ) + elif moe.use_mori_kernels: assert quant_config is not None diff --git a/vllm/model_executor/layers/fused_moe/config.py b/vllm/model_executor/layers/fused_moe/config.py index 9a2f8229b1..62ab2bf88a 100644 --- a/vllm/model_executor/layers/fused_moe/config.py +++ b/vllm/model_executor/layers/fused_moe/config.py @@ -1085,6 +1085,10 @@ class FusedMoEParallelConfig: def use_deepep_v2_kernels(self): return self.use_all2all_kernels and self.all2all_backend == "deepep_v2" + @property + def use_moonep_kernels(self): + return self.use_all2all_kernels and self.all2all_backend == "moonep" + @staticmethod def flatten_tp_across_dp_and_pcp( tp_size: int, dp_size: int, dp_rank: int, pcp_size: int, pcp_rank: int @@ -1309,6 +1313,8 @@ class FusedMoEConfig: # Set by __post_init__ intermediate_size_per_partition: int = -1 + # Use the allocated width as the checkpoint TP stride, including for scales. + tp_shard_with_padding: bool = False rocm_aiter_fmoe_enabled: bool = False aiter_fmoe_shared_expert_enabled: bool = False @@ -1471,6 +1477,10 @@ class FusedMoEConfig: def use_deepep_v2_kernels(self): return self.moe_parallel_config.use_deepep_v2_kernels + @property + def use_moonep_kernels(self): + return self.moe_parallel_config.use_moonep_kernels + @property def needs_round_robin_routing_tables(self): return self.moe_parallel_config.needs_round_robin_routing_tables diff --git a/vllm/model_executor/layers/fused_moe/deep_gemm_utils.py b/vllm/model_executor/layers/fused_moe/deep_gemm_utils.py index 4ea864eaac..33a2296955 100644 --- a/vllm/model_executor/layers/fused_moe/deep_gemm_utils.py +++ b/vllm/model_executor/layers/fused_moe/deep_gemm_utils.py @@ -393,12 +393,13 @@ def _fwd_kernel_ep_gather( source_token_index = tl.load( input_index + cur_token * input_index_stride0 + topk_index ) + source_token_index_i64 = source_token_index.to(tl.int64) acc_weight = tl.load( recv_topk_weight + cur_token * recv_topk_weight_stride0 + topk_index ) tmp = tl.load( input_tensor - + source_token_index * input_tensor_stride0 + + source_token_index_i64 * input_tensor_stride0 + cur_block * BLOCK_D + off_d ) diff --git a/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp8_moe.py b/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp8_moe.py index 93a4088fc4..c15a1a13d4 100644 --- a/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp8_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp8_moe.py @@ -11,6 +11,7 @@ import math import torch import vllm.model_executor.layers.fused_moe.modular_kernel as mk +from vllm._aiter_ops import rocm_aiter_ops from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe.activation import MoEActivation from vllm.model_executor.layers.fused_moe.experts.mxfp8_emulation_moe import ( @@ -24,41 +25,6 @@ _AITER_SWIGLU_ALPHA = 1.702 _AITER_SWIGLU_BETA = 1.0 -def is_aiter_mxfp8_moe_available() -> bool: - """True when the FlyDSL MXFP8 MoE can run here: gfx950, the ``flydsl`` - package is importable, AND the installed aiter carries the mxfp8 FlyDSL - 2-stage support from ROCm/aiter#3811. - - ``flydsl`` and ``aiter`` are separate packages, so ``is_flydsl_available()`` - (flydsl pkg + arch) is necessary but not sufficient: an older aiter without - #3811 still ships the flydsl pkg and the ``aiter.ops.flydsl`` module but a - broken/missing ``per_1x32 + fp8`` 2-stage path. Without this extra gate a - nightly lacking #3811 would wrongly select FlyDSL instead of falling back to - the native Triton dot_scaled path. #3811 added no probe-able public symbol, - so detect the ``minimax_m3_mxfp8`` tuned config it shipped. Every check fails - closed (returns False -> triton dot_scaled), which is always safe.""" - if not (current_platform.is_rocm() and current_platform.supports_mx()): - return False - try: - import os - - import aiter - from aiter.ops.flydsl.utils import is_flydsl_available - - if not is_flydsl_available(): - return False - return os.path.exists( - os.path.join( - os.path.dirname(aiter.__file__), - "configs", - "model_configs", - "minimax_m3_mxfp8_tuned_fmoe.csv", - ) - ) - except Exception: - return False - - class AiterMxfp8Experts(Mxfp8TritonExpertsBase): """MXFP8 MoE through AITER's FlyDSL two-stage grouped GEMM (gfx950).""" @@ -79,10 +45,7 @@ class AiterMxfp8Experts(Mxfp8TritonExpertsBase): @staticmethod def _supports_current_device() -> bool: - # Device capability only (gfx950 / MX-capable ROCm). The flydsl package - # check lives in is_supported_config so a missing package is reported - # distinctly from an unsupported device. - return current_platform.is_rocm() and current_platform.supports_mx() + return current_platform.supports_mx() and rocm_aiter_ops.is_fused_moe_enabled() @staticmethod def _supports_parallel_config(moe_parallel_config) -> bool: @@ -95,12 +58,6 @@ class AiterMxfp8Experts(Mxfp8TritonExpertsBase): is_supported, reason = super().is_supported_config( cls, moe_config, weight_key, activation_key, activation_format ) - # _supports_current_device() only gates on the device; surface a clear - # reason when the device is fine but the flydsl package is missing. - if is_supported and not is_aiter_mxfp8_moe_available(): - return False, ( - "kernel requires the aiter flydsl package, which is not installed" - ) if ( is_supported and moe_config.activation != MoEActivation.SWIGLUOAI_UNINTERLEAVE diff --git a/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py b/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py index 7b3193fb5f..3e7376fe90 100644 --- a/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py @@ -15,6 +15,7 @@ from vllm._custom_ops import ( convert_weight_packed_scale_zp, cpu_fused_moe, cpu_fused_moe_int8, + cpu_has_amx_fp8, cpu_prepack_moe_weight, cpu_prepack_moe_weight_int8, fused_experts_cpu, @@ -467,6 +468,7 @@ class CPUExpertsFp8(mk.FusedMoEExpertsModular): RoutingMethodType.Default, RoutingMethodType.Renormalize, RoutingMethodType.RenormalizeNaive, + RoutingMethodType.DeepSeekV3, RoutingMethodType.DeepseekV4, RoutingMethodType.DeepSeekV3, ] @@ -950,6 +952,29 @@ class CPUExpertsInt4(mk.FusedMoEExpertsModular): # =========================================================================== +def prepare_int8_moe_layer_for_cpu( + w13: torch.Tensor, + w2: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + """Prepack INT8 MoE weights for the current CPU architecture.""" + # SMMLA packing for AArch64 + if current_platform.get_cpu_architecture() == CpuArchEnum.ARM: + return ( + cpu_prepack_moe_weight_int8(w13, "neon"), + cpu_prepack_moe_weight_int8(w2, "neon"), + ) + # VSX packing for POWER + if current_platform.get_cpu_architecture() == CpuArchEnum.POWERPC: + return ( + cpu_prepack_moe_weight_int8(w13, "vsx"), + cpu_prepack_moe_weight_int8(w2, "vsx"), + ) + # VNNI packing for x86 + packed_w13 = torch.ops._C.convert_weight_packed(w13) + packed_w2 = torch.ops._C.convert_weight_packed(w2) + return packed_w13, packed_w2 + + class CPUExpertsInt8(mk.FusedMoEExpertsModular): """CPU INT8 W8A8 per-channel weight / dynamic per-token activation modular MoE experts.""" @@ -1283,6 +1308,160 @@ class ArmCPUExpertsInt8(mk.FusedMoEExpertsModular): ) +class PowerCPUExpertsInt8(mk.FusedMoEExpertsModular): + """POWER VSX INT8 MoE with per-token activation and channelwise weight quant.""" + + @property + def expects_unquantized_inputs(self) -> bool: + return True + + @staticmethod + def activation_format() -> mk.FusedMoEActivationFormat: + return mk.FusedMoEActivationFormat.Standard + + @staticmethod + def is_supported_config( + cls: type[mk.FusedMoEExperts], + moe_config: FusedMoEConfig, + weight_key: QuantKey | None, + activation_key: QuantKey | None, + activation_format: mk.FusedMoEActivationFormat, + ) -> tuple[bool, str | None]: + supported, reason = mk.FusedMoEExpertsModular.is_supported_config( + cls, + moe_config, + weight_key, + activation_key, + activation_format, + ) + if not supported: + return supported, reason + if moe_config.in_dtype not in (torch.float32, torch.bfloat16): + return False, "kernel requires float32 or bfloat16 activations" + if moe_config.hidden_dim % 16 != 0: + return False, "kernel requires hidden dim divisible by 16" + if moe_config.intermediate_size_per_partition % 16 != 0: + return False, "kernel requires intermediate dim divisible by 16" + return True, None + + @staticmethod + def _supports_current_device() -> bool: + return ( + current_platform.is_cpu() + and current_platform.get_cpu_architecture() == CpuArchEnum.POWERPC + and hasattr(torch.ops._C, "cpu_fused_moe_int8") + ) + + @staticmethod + def _supports_no_act_and_mul() -> bool: + return False + + @staticmethod + def _supports_activation(activation: MoEActivation) -> bool: + return activation in ( + MoEActivation.SILU, + MoEActivation.SWIGLUOAI, + MoEActivation.GELU, + MoEActivation.GELU_TANH, + ) + + @staticmethod + def _supports_parallel_config( + moe_parallel_config: FusedMoEParallelConfig, + ) -> bool: + return not moe_parallel_config.use_ep + + @staticmethod + def _supports_quant_scheme( + weight_key: QuantKey | None, + activation_key: QuantKey | None, + ) -> bool: + return (weight_key, activation_key) == ( + kInt8StaticChannelSym, + kInt8DynamicTokenSym, + ) + + @staticmethod + def _supports_routing_method( + routing_method: RoutingMethodType, + weight_key: QuantKey | None, + activation_key: QuantKey | None, + ) -> bool: + return routing_method in [ + RoutingMethodType.Default, + RoutingMethodType.Renormalize, + RoutingMethodType.RenormalizeNaive, + ] + + @staticmethod + def _supports_router_logits_dtype( + router_logits_dtype: torch.dtype | None, + routing_method: RoutingMethodType, + ) -> bool: + return True + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + w13 = cpu_prepack_moe_weight_int8(layer.w13_weight, "vsx") + w2 = cpu_prepack_moe_weight_int8(layer.w2_weight, "vsx") + replace_parameter(layer, "w13_weight", w13) + replace_parameter(layer, "w2_weight", w2) + + def workspace_shapes( + self, + M: int, + N: int, + K: int, + topk: int, + global_num_experts: int, + local_num_experts: int, + expert_tokens_meta: mk.ExpertTokensMetadata | None, + activation: MoEActivation, + ) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]: + # cpu_fused_moe_int8 manages its own scratch space. + return (0,), (0,), (M, K) + + def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce: + return TopKWeightAndReduceNoOP() + + def apply( + self, + output: torch.Tensor, + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + activation: MoEActivation, + global_num_experts: int, + expert_map: torch.Tensor | None, + a1q_scale: torch.Tensor | None, + a2_scale: torch.Tensor | None, + workspace13: torch.Tensor, + workspace2: torch.Tensor, + expert_tokens_meta: mk.ExpertTokensMetadata | None, + apply_router_weight_on_input: bool, + ) -> None: + # The modular prepare step applies router weights to the input when + # apply_router_weight_on_input is enabled. + assert self.w1_scale is not None + assert self.w2_scale is not None + cpu_fused_moe_int8( + output, + hidden_states, + w1, + w2, + self.w1_scale, + self.w2_scale, + self.w1_bias, + self.w2_bias, + topk_weights, + topk_ids, + activation.value, + "vsx", + skip_weighted=apply_router_weight_on_input, + ) + + class ZenCPUExpertsInt8(mk.FusedMoEExpertsModular): """AMD Zen INT8 MoE with per-token activation and channelwise weight quantization, dispatched through zentorch.""" @@ -1435,3 +1614,218 @@ class ZenCPUExpertsInt8(mk.FusedMoEExpertsModular): self._w1_scale_bf16, self._w2_scale_bf16, ) + + +# =========================================================================== +# FP8 W8A8 MoE +# =========================================================================== + + +def prepare_fp8_w8a8_moe_layer_for_cpu( + w13: torch.Tensor, + w2: torch.Tensor, + w13_scale: torch.Tensor, + w2_scale: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Prepack FP8 W8A8 MoE weights for CPU kernel.""" + E = w13.size(0) + + packed_w13_list = [] + packed_w13_scale_list = [] + + for i in range(E): + w1_exp = w13[i] # [2N, K] FP8 + ws_exp = w13_scale[i] # [2N, K/128] or [2N_blocks, K/128] + + # Align scale rows to weight rows if necessary (for block-128 checkpoints + # where scale may have fewer rows: [2N/128, K/128] → [2N, K/128]) + if ws_exp.size(0) < w1_exp.size(0): + repeat_factor = w1_exp.size(0) // ws_exp.size(0) + ws_exp = torch.repeat_interleave(ws_exp, repeat_factor, dim=0) + ws_exp = ws_exp[: w1_exp.size(0), :].contiguous() + + pw, ps = torch.ops._C.float8_linear_prepack_cpu(w1_exp, ws_exp) + packed_w13_list.append(pw) + packed_w13_scale_list.append(ps) + + packed_w13 = torch.stack(packed_w13_list) + packed_w13_scale = torch.stack(packed_w13_scale_list) + + # w2 uses the W8A16 path (BF16 activation × FP8 weight), VNNI packed + packed_w2 = torch.ops._C.convert_weight_packed(w2) + return packed_w13, packed_w13_scale, packed_w2, w2_scale + + +class CPUExpertsFp8W8A8(mk.FusedMoEExpertsModular): + """CPU FP8 W8A8 block-quantized modular MoE experts.""" + + def __init__( + self, + moe_config: FusedMoEConfig, + quant_config: FusedMoEQuantConfig, + ): + super().__init__(moe_config, quant_config) + + @property + def expects_unquantized_inputs(self) -> bool: + # Receives BF16 hidden states; quantization to FP8 is done inside apply() + return True + + @staticmethod + def activation_format() -> mk.FusedMoEActivationFormat: + return mk.FusedMoEActivationFormat.Standard + + @staticmethod + def _supports_current_device() -> bool: + # Requires native AMX-FP8 hardware. + return current_platform.is_cpu() and cpu_has_amx_fp8() + + @staticmethod + def _supports_no_act_and_mul() -> bool: + return False + + @staticmethod + def _supports_activation(activation: MoEActivation) -> bool: + return activation == MoEActivation.SILU + + @staticmethod + def _supports_parallel_config( + moe_parallel_config: FusedMoEParallelConfig, + ) -> bool: + return True + + @staticmethod + def _supports_quant_scheme( + weight_key: QuantKey | None, + activation_key: QuantKey | None, + ) -> bool: + # Block-quantized FP8. + return (weight_key, activation_key) == ( + kFp8Static128BlockSym, + kFp8Dynamic128Sym, + ) + + @staticmethod + def _supports_routing_method( + routing_method: RoutingMethodType, + weight_key: QuantKey | None, + activation_key: QuantKey | None, + ) -> bool: + return routing_method in [ + RoutingMethodType.Default, + RoutingMethodType.Renormalize, + RoutingMethodType.RenormalizeNaive, + RoutingMethodType.DeepSeekV3, + RoutingMethodType.DeepseekV4, + ] + + @staticmethod + def _supports_router_logits_dtype( + router_logits_dtype: torch.dtype | None, + routing_method: RoutingMethodType, + ) -> bool: + return True + + def moe_problem_size( + self, + a1: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_ids: torch.Tensor, + ) -> tuple[int, int, int, int, int]: + # float8_linear_prepack_cpu blocks w1 into [E, Nc, Kc, block_k, block_n], + # not the (E, N, K) layout the base implementation assumes -- N isn't + # recoverable from that shape, so read it from moe_config instead. + E = w1.shape[0] + K = a1.size(-1) + N = ( + self.moe_config.intermediate_size_per_partition + * self.moe_config.w13_num_shards + ) + M = a1.size(0) if a1.dim() == 2 else a1.size(1) + topk = topk_ids.size(1) + return E, M, N, K, topk + + def workspace_shapes( + self, + M: int, + N: int, + K: int, + topk: int, + global_num_experts: int, + local_num_experts: int, + expert_tokens_meta: mk.ExpertTokensMetadata | None, + activation: MoEActivation, + ) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]: + # fused_experts_cpu manages its own scratch space. + return (0,), (0,), (M, K) + + def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce: + return TopKWeightAndReduceNoOP() + + def apply( + self, + output: torch.Tensor, + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + activation: MoEActivation, + global_num_experts: int, + expert_map: torch.Tensor | None, + a1q_scale: torch.Tensor | None, + a2_scale: torch.Tensor | None, + workspace13: torch.Tensor, + workspace2: torch.Tensor, + expert_tokens_meta: mk.ExpertTokensMetadata | None, + apply_router_weight_on_input: bool, + ) -> None: + if apply_router_weight_on_input: + # fused_experts_cpu always applies topk_weights internally on + # combine; MoEPrepareAndFinalizeNoDPEPModular.prepare() would + # also pre-apply it to hidden_states, double-weighting the + # output. Not needed by any CPU FP8 model today. + raise NotImplementedError( + "CPUExpertsFp8W8A8 does not support apply_router_weight_on_input=True." + ) + + block_shape = ( + list(self.quant_config.block_shape) + if self.quant_config.block_shape + else ( + [ + self.quant_config._w1.shape.row, + self.quant_config._w1.shape.col, + ] + if self.quant_config._w1.shape is not None + else None + ) + ) + + # Quantize hidden_states (BF16) → FP8 with per-token scales + hidden_states_2d = hidden_states.reshape(-1, hidden_states.shape[-1]) + x_fp8, x_scales = torch.ops._C.quantize_fp8e4m3_vec( + hidden_states_2d, True, None + ) + + fused_experts_cpu( + output, + x_fp8, + w1, + w2, + topk_weights, + topk_ids, + CPUQuantMethod.FP8_W8A8, + self.w1_scale, # w1_scale + self.w2_scale, # w2_scale + None, # w1_zero + None, # w2_zero + block_shape, # block_size + None, # w1_bias + None, # w2_bias + None, # alpha + None, # limit + True, # is_vnni + x_scales, # a1_scale: per-token FP8 activation scales + ) diff --git a/vllm/model_executor/layers/fused_moe/experts/deep_gemm_moe.py b/vllm/model_executor/layers/fused_moe/experts/deep_gemm_moe.py index fe95b36858..699b029bad 100644 --- a/vllm/model_executor/layers/fused_moe/experts/deep_gemm_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/deep_gemm_moe.py @@ -50,6 +50,21 @@ from vllm.utils.import_utils import has_deep_gemm logger = init_logger(__name__) +def _fp8_workspace_shape( + num_rows: int, num_columns: int, workspace_dtype: torch.dtype +) -> tuple[int, int]: + """Size an FP8 byte buffer stored in a workspace of another dtype.""" + bytes_per_workspace_element = workspace_dtype.itemsize + fp8_columns_per_workspace_element = ( + bytes_per_workspace_element // torch.float8_e4m3fn.itemsize + ) + assert bytes_per_workspace_element % torch.float8_e4m3fn.itemsize == 0 + return ( + num_rows, + -(-num_columns // fp8_columns_per_workspace_element), + ) + + def _valid_deep_gemm_shape(M: int, N: int, K: int) -> bool: align = get_mk_alignment_for_contiguous_layout()[0] return align <= M and N % align == 0 and K % align == 0 @@ -216,7 +231,14 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular): assert M_sum % align_used == 0 activation_out_dim = self.adjust_N_for_activation(N, activation) - workspace1 = (M_sum, max(activation_out_dim, K)) + # workspace1 is allocated in the activation dtype by the workspace + # manager, but is only ever viewed and used as FP8 in apply(). Size it + # by bytes instead of reserving one BF16/FP16 element per FP8 element. + workspace1 = _fp8_workspace_shape( + M_sum, + max(activation_out_dim, K), + self.workspace_dtype(self.moe_config.in_dtype), + ) workspace2 = (M_sum, max(N, K)) output = (M, K) return (workspace1, workspace2, output) @@ -480,7 +502,14 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular): assert M_sum % align_used == 0 activation_out_dim = self.adjust_N_for_activation(N, activation) - workspace1 = (M_sum, max(activation_out_dim, K)) + # workspace1 holds the permuted and requantized FP8 activations. The + # workspace manager allocates it in the model activation dtype, so + # account for the dtype sizes rather than overallocating BF16/FP16. + workspace1 = _fp8_workspace_shape( + M_sum, + max(activation_out_dim, K), + self.workspace_dtype(self.moe_config.in_dtype), + ) workspace2 = (M_sum, max(N, K)) output = (M, K) return (workspace1, workspace2, output) diff --git a/vllm/model_executor/layers/fused_moe/experts/moonep_experts.py b/vllm/model_executor/layers/fused_moe/experts/moonep_experts.py new file mode 100644 index 0000000000..1af4b5f064 --- /dev/null +++ b/vllm/model_executor/layers/fused_moe/experts/moonep_experts.py @@ -0,0 +1,220 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""MoonEP experts: grouped GEMM over MoonEP's expert-grouped activations. + +MoonEP dispatch delivers tokens already contiguous per weight row +(``[NvS, H]`` in ``cu_seqlens[E+B]`` segment order), so the expert compute +is three grouped GEMMs over those segments with no permute/unpermute: + + gate = grouped_mm(x, w_gate[E+B]) # [NvS, I] + up = grouped_mm(x, w_up[E+B]) # [NvS, I] + act = silu(gate) * up * route_weight # route weights applied here + out = grouped_mm(act, w_down[E+B]) # [NvS, H] + +Rows ``[E, E+B)`` are the redundant-expert prefetch slots that +``MoonEPPrepareAndFinalize`` fills before ``apply`` runs. Empty segments +(including unused prefetch slots) are skipped by the grouped GEMM. + +Weight layout: MoonEP's ``prefetch_weight`` requires each projection to be +its own contiguous ``[E+B, ., .]`` tensor, so gate and up are separate +tensors rather than vLLM's interleaved ``[E, 2I, H]`` ``w13``. The modular +kernel passes ``w1``/``w2`` from the layer (``w1`` is the gate tensor); the +full layout is picked up from the layer in ``process_weights_after_loading`` +and shared with ``MoonEPPrepareAndFinalize`` via ``post_init_setup``. +""" + +import torch + +import vllm.model_executor.layers.fused_moe.modular_kernel as mk +from vllm.model_executor.layers.fused_moe.activation import MoEActivation +from vllm.model_executor.layers.fused_moe.config import ( + FusedMoEConfig, + FusedMoEParallelConfig, + FusedMoEQuantConfig, +) +from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import ( + TopKWeightAndReduceNoOP, +) +from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey +from vllm.platforms import current_platform + + +def moonep_grouped_gemm( + a: torch.Tensor, + w: torch.Tensor, + cu_seqlens: torch.Tensor, +) -> torch.Tensor: + """``out[seg_g] = a[seg_g] @ w[g].T`` for each ``cu_seqlens`` segment. + + Args: + a: ``[NvS, K]`` activations in segment order. + w: ``[G, N, K]`` weights (vLLM ``[E, N, K]`` convention). + cu_seqlens: ``[G]`` int32 cumulative segment end offsets. + + Returns: + ``[NvS, N]``; rows past ``cu_seqlens[-1]`` are zero-filled. + + """ + return torch._grouped_mm(a, w.transpose(1, 2), offs=cu_seqlens) + + +class MoonEPExperts(mk.FusedMoEExpertsModular): + """Grouped-GEMM experts over MoonEP's ``[NvS, H]`` layout (BF16).""" + + def __init__( + self, + moe_config: FusedMoEConfig, + quant_config: FusedMoEQuantConfig, + max_num_tokens: int | None = None, + num_dispatchers: int | None = None, + ): + super().__init__(moe_config, quant_config, max_num_tokens, num_dispatchers) + assert quant_config.quant_dtype is None, ( + "MoonEPExperts supports unquantized BF16 only" + ) + if ( + quant_config.gemm1_alpha is not None + or quant_config.gemm1_beta is not None + or quant_config.gemm1_clamp_limit is not None + ): + raise NotImplementedError( + "MoonEPExperts implements plain silu(gate) * up only; SwiGLU " + "variants with alpha/beta/clamp parameters are not supported" + ) + self._weight_layout = None + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + # The [E+B] gate/up/down layout built by + # convert_to_unquantized_kernel_format; MoonEPPrepareAndFinalize + # reads it from here (post_init_setup) for weight prefetch. + layout = getattr(layer, "_moonep_weight_layout", None) + assert layout is not None, ( + "MoonEPExperts requires the layer to carry _moonep_weight_layout " + "(set by convert_to_unquantized_kernel_format)" + ) + self._weight_layout = layout + + @property + def weight_layout(self): + return self._weight_layout + + @staticmethod + def activation_format() -> mk.FusedMoEActivationFormat: + return mk.FusedMoEActivationFormat.Standard + + @staticmethod + def _supports_current_device() -> bool: + return current_platform.is_cuda() + + @staticmethod + def _supports_no_act_and_mul() -> bool: + return False + + @staticmethod + def _supports_activation(activation: MoEActivation) -> bool: + return activation == MoEActivation.SILU + + @staticmethod + def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool: + # EPLB rearranges the named expert parameters as local_num_experts + # rows, which does not understand the replicated [E+B] layout. + return ( + moe_parallel_config.use_moonep_kernels + and not moe_parallel_config.enable_eplb + ) + + @staticmethod + def _supports_quant_scheme( + weight_key: QuantKey | None, + activation_key: QuantKey | None, + ) -> bool: + return weight_key is None and activation_key is None + + def supports_chunking(self) -> bool: + # NvS is static and segments must stay whole. + return False + + def supports_expert_map(self) -> bool: + # MoonEP addresses global expert rows directly. + return False + + def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce: + return TopKWeightAndReduceNoOP() + + def moe_problem_size( + self, + a1: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_ids: torch.Tensor, + ) -> tuple[int, int, int, int, int]: + # a1 is [NvS, H] in segment order, not token-major, so the base + # implementation's topk_ids/a1 row-count check does not apply. + assert a1.dim() == 2 and w1.dim() == 3 and w2.dim() == 3 + num_rows, intermediate, hidden = w1.shape # [E+B, I, H] + assert a1.size(1) == hidden + return num_rows, a1.size(0), intermediate, hidden, topk_ids.size(1) + + def workspace_shapes( + self, + M: int, + N: int, + K: int, + topk: int, + global_num_experts: int, + local_num_experts: int, + expert_tokens_meta: mk.ExpertTokensMetadata | None, + activation: MoEActivation, + ) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]: + # M is NvS. torch._grouped_mm allocates its own outputs, so no + # workspaces are needed. + return ((0,), (0,), (M, K)) + + def apply( + self, + output: torch.Tensor, + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + activation: MoEActivation, + global_num_experts: int, + expert_map: torch.Tensor | None, + a1q_scale: torch.Tensor | None, + a2_scale: torch.Tensor | None, + workspace13: torch.Tensor, + workspace2: torch.Tensor, + expert_tokens_meta: mk.ExpertTokensMetadata | None, + apply_router_weight_on_input: bool, + ) -> None: + # expert_map is ignored: MoonEP dispatches on global expert ids and + # the weights are addressed by global [E+B] row, on every rank. + assert activation == MoEActivation.SILU + assert self._weight_layout is not None, ( + "process_weights_after_loading() not called" + ) + assert expert_tokens_meta is not None + # MoonEPPrepareAndFinalize returns route weights in NvS order as the + # dispatched topk_weights, and per-row segment lengths as + # expert_tokens_meta. + route_weights_nvs = topk_weights + assert route_weights_nvs.dim() == 1 + cu_seqlens = torch.cumsum( + expert_tokens_meta.expert_num_tokens, dim=0, dtype=torch.int32 + ) + + assert cu_seqlens.numel() == w1.size(0) + + gate = moonep_grouped_gemm(hidden_states, w1, cu_seqlens) + up = moonep_grouped_gemm( + hidden_states, self._weight_layout.full_up_weight, cu_seqlens + ) + act = torch.nn.functional.silu(gate) + act.mul_(up) + if not apply_router_weight_on_input: + # Input-weighted routing (top-1, e.g. Llama 4) was already + # applied to the activations by prepare(), before dispatch. + act.mul_(route_weights_nvs.to(act.dtype).unsqueeze(-1)) + # Padding rows past the last segment come out zero-filled. + output.copy_(moonep_grouped_gemm(act, w2, cu_seqlens)) diff --git a/vllm/model_executor/layers/fused_moe/experts/triton_moe.py b/vllm/model_executor/layers/fused_moe/experts/triton_moe.py index 9f982644b5..700d113921 100644 --- a/vllm/model_executor/layers/fused_moe/experts/triton_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/triton_moe.py @@ -130,10 +130,18 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular): # INT8 requires at least 7.5 (Turing) on CUDA. ROCm CDNA GPUs # (e.g. MI2xx/MI3xx/gfx950) provide native INT8 matrix-core support and # the Triton int8_w8a8 fused MoE kernel handles them. + # XPU reaches the same Triton kernel through the same launcher, and both + # int8 activation-quant paths work there: per-token uses the Triton + # `per_token_quant_int8`, per-tensor resolves to the XPU branch of + # `scaled_int8_quant`, which is plain elementwise arithmetic. device_supports_int8 = ( - current_platform.is_cuda() - and current_platform.has_device_capability((7, 5)) - ) or current_platform.is_rocm() + ( + current_platform.is_cuda() + and current_platform.has_device_capability((7, 5)) + ) + or current_platform.is_rocm() + or current_platform.is_xpu() + ) supported: list[tuple[QuantKey | None, QuantKey | None]] = [(None, None)] if device_supports_int8: @@ -596,9 +604,9 @@ class TritonWNA16Experts(TritonExperts): ) -> bool: SUPPORTED_W = [ kInt4Static, + kInt4StaticAsym, kInt8Static, kInt4Static32, - kInt4StaticAsym, kInt4Static32Asym, # other group sizes? ] diff --git a/vllm/model_executor/layers/fused_moe/experts/trtllm_bf16_moe.py b/vllm/model_executor/layers/fused_moe/experts/trtllm_bf16_moe.py index 446acf46dd..54227d3cd1 100644 --- a/vllm/model_executor/layers/fused_moe/experts/trtllm_bf16_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/trtllm_bf16_moe.py @@ -31,6 +31,15 @@ from vllm.platforms import current_platform from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe +def view_as_block_major_k(weight: torch.Tensor) -> torch.Tensor: + """View packed storage, including legacy 4D IPC cache entries.""" + if weight.ndim == 4: + return weight + experts, rows, cols = weight.shape + block_k = 128 // weight.element_size() + return weight.view(experts, cols // block_k, rows, block_k) + + class TrtLlmBf16ExpertsBase: """BF16 unquantized TRTLLM-Gen MoE kernels. Shared base for modular and monolithic interfaces. @@ -200,8 +209,8 @@ class TrtLlmBf16ExpertsModular(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsModular) result = flashinfer.fused_moe.trtllm_bf16_routed_moe( topk_ids=(topk_ids, topk_weights), hidden_states=hidden_states, - gemm1_weights=w1, - gemm2_weights=w2, + gemm1_weights=view_as_block_major_k(w1), + gemm2_weights=view_as_block_major_k(w2), num_experts=global_num_experts, top_k=topk_ids.size(1), n_group=None, @@ -281,8 +290,8 @@ class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonol routing_logits=router_logits, routing_bias=e_score_correction_bias, hidden_states=hidden_states, - gemm1_weights=w1, - gemm2_weights=w2, + gemm1_weights=view_as_block_major_k(w1), + gemm2_weights=view_as_block_major_k(w2), num_experts=global_num_experts, top_k=self.topk, n_group=num_expert_group, diff --git a/vllm/model_executor/layers/fused_moe/experts/trtllm_lora_moe.py b/vllm/model_executor/layers/fused_moe/experts/trtllm_lora_moe.py index 4dc5076cf6..1c5698932a 100644 --- a/vllm/model_executor/layers/fused_moe/experts/trtllm_lora_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/trtllm_lora_moe.py @@ -34,6 +34,9 @@ from vllm.model_executor.layers.fused_moe.experts.lora_context import MoELoRACon from vllm.model_executor.layers.fused_moe.experts.lora_experts_mixin import ( LoRAExpertsMixin, ) +from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import ( + view_as_block_major_k, +) from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import ( TopKWeightAndReduceNoOP, ) @@ -527,8 +530,8 @@ class TrtLlmBf16LoRAExperts(_TrtLlmLoRAExpertsBase): ret = flashinfer.fused_moe.trtllm_bf16_routed_moe( topk_ids=topk_ids_and_weights, hidden_states=hidden_states, - gemm1_weights=w1, - gemm2_weights=w2, + gemm1_weights=view_as_block_major_k(w1), + gemm2_weights=view_as_block_major_k(w2), gemm1_lora_delta=gemm1_lora_delta, num_experts=global_num_experts, top_k=self.topk, diff --git a/vllm/model_executor/layers/fused_moe/layer.py b/vllm/model_executor/layers/fused_moe/layer.py index 2ee3c5f374..49557bb3f9 100644 --- a/vllm/model_executor/layers/fused_moe/layer.py +++ b/vllm/model_executor/layers/fused_moe/layer.py @@ -137,6 +137,7 @@ def FusedMoEFactory( runner_args: dict[str, Any] | None = None, routed_experts_cls: type[RoutedExperts] | None = None, routed_experts_args: dict[str, Any] | None = None, + skip_padding: bool = False, ) -> MoERunner: """Factory function for creating MoE execution pipeline. @@ -213,6 +214,7 @@ def FusedMoEFactory( runner_args: Additional arguments for runner constructor routed_experts_cls: Custom RoutedExperts class (None = use default) routed_experts_args: Additional arguments for routed_experts constructor + skip_padding: Whether grouped routing should invalidate padding rows. Returns: MoERunner: Configured MoE execution pipeline ready for forward passes @@ -308,6 +310,7 @@ def FusedMoEFactory( else 1.0, e_score_correction_bias=e_score_correction_bias, num_fused_shared_experts=num_fused_shared_experts, + skip_padding=(skip_padding and moe_parallel_config.use_deepep_v2_kernels), # Fused shared-expert slot weight. With apply_routed_scale_to_output # the runner scales the combined output by routed_scaling_factor, so # the shared slot weight must be 1/routed_scaling_factor for its net diff --git a/vllm/model_executor/layers/fused_moe/oracle/fp8.py b/vllm/model_executor/layers/fused_moe/oracle/fp8.py index 7c57e2e5c2..abf0896204 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/fp8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/fp8.py @@ -31,11 +31,14 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import ( prepare_fp8_moe_layer_for_marlin, ) from vllm.model_executor.layers.quantization.utils.quant_utils import ( + GroupShape, QuantKey, + create_fp8_quant_key, kFp8Dynamic128Sym, kFp8Static128BlockSym, ) from vllm.platforms import current_platform +from vllm.utils.math_utils import round_up logger = init_logger(__name__) @@ -54,7 +57,9 @@ class Fp8MoeBackend(Enum): VLLM_CUTLASS = "VLLM_CUTLASS" BATCHED_VLLM_CUTLASS = "BATCHED_VLLM_CUTLASS" XPU = "XPU" + # CPU FP8 W8A16 (BF16 activations); CPU_W8A8 needs native AMX-FP8. CPU = "CPU" + CPU_W8A8 = "CPU_W8A8" HPC = "HPC" # Dequantize-to-BF16 emulation for MXFP8 on devices without a native # MXFP8 MoE kernel (e.g. ROCm). Weights pass through unchanged here. @@ -89,6 +94,7 @@ def _get_priority_backends( Fp8MoeBackend.BATCHED_VLLM_CUTLASS, Fp8MoeBackend.BATCHED_TRITON, Fp8MoeBackend.XPU, + Fp8MoeBackend.CPU_W8A8, Fp8MoeBackend.CPU, Fp8MoeBackend.HPC, ] @@ -126,8 +132,10 @@ def _get_priority_backends( _move_to_front(_AVAILABLE_BACKENDS, Fp8MoeBackend.XPU) if current_platform.is_cpu(): - # CPU platform uses FP8 W8A16 fused MoE kernel. + # W8A8 first: it falls through to the W8A16 backend whenever the + # hardware (AMX-FP8) or the config isn't supported. _move_to_front(_AVAILABLE_BACKENDS, Fp8MoeBackend.CPU) + _move_to_front(_AVAILABLE_BACKENDS, Fp8MoeBackend.CPU_W8A8) return _AVAILABLE_BACKENDS @@ -235,6 +243,13 @@ def backend_to_kernel_cls( return [CPUExpertsFp8] + elif backend == Fp8MoeBackend.CPU_W8A8: + from vllm.model_executor.layers.fused_moe.experts.cpu_moe import ( + CPUExpertsFp8W8A8, + ) + + return [CPUExpertsFp8W8A8] + elif backend == Fp8MoeBackend.HPC: from vllm.model_executor.layers.fused_moe.hpc_moe import ( HPCExperts, @@ -299,6 +314,80 @@ def refine_fp8_moe_block_shape( return [refine, refine] +def pad_tp_shard_to_weight_blocks( + config: FusedMoEConfig, + weight_block_size: list[int], +) -> bool: + """Pad the TP shard to whole checkpoint blocks, keeping scales rank-local.""" + block_n, block_k = weight_block_size + if ( + block_n != block_k + or config.tp_size == 1 + or config.intermediate_size_per_partition % block_n == 0 + ): + return False + if ( + config.intermediate_size % block_n != 0 + or config.hidden_dim % block_n != 0 + or config.ep_size != 1 + or config.is_lora_enabled + or config.has_bias + ): + raise ValueError( + f"Block-aligned FP8 TP sharding requires {block_n}-aligned " + "global expert dimensions, pure TP, and no LoRA or " + "expert bias." + ) + config.tp_shard_with_padding = True + config.intermediate_size_per_partition = round_up( + config.intermediate_size_per_partition, block_n + ) + return True + + +def resolve_fp8_moe_weight_block_shape( + config: FusedMoEConfig, + weight_block_size: list[int], + activation_key: QuantKey, + is_checkpoint_fp8_serialized: bool, +) -> tuple[list[int], tuple[int, int] | None]: + """Return the TP-adapted block shape and refine factor: + refine if kernels allow, else pad to the TP shard.""" + refined_shape = refine_fp8_moe_block_shape(config, weight_block_size) + if is_checkpoint_fp8_serialized and config.moe_backend != "auto": + kernel_classes = backend_to_kernel_cls(map_fp8_backend(config.moe_backend)) + can_refine = refined_shape is not None and any( + k_cls._supports_quant_scheme( + create_fp8_quant_key( + static=True, group_shape=GroupShape(*refined_shape) + ), + activation_key, + ) + for k_cls in kernel_classes + ) + if not can_refine and pad_tp_shard_to_weight_blocks(config, weight_block_size): + logger.info_once( + "FP8 %s TP loading uses complete checkpoint blocks: " + "local allocation %d, without weight requantization.", + config.moe_backend, + config.intermediate_size_per_partition, + ) + return weight_block_size, None + if refined_shape is None: + return weight_block_size, None + logger.info_once( + "FP8 MoE block scales refined from %s to %s to fit " + "the TP-sharded intermediate size %d.", + str(weight_block_size), + str(refined_shape), + config.intermediate_size_per_partition, + ) + return refined_shape, ( + weight_block_size[0] // refined_shape[0], + weight_block_size[1] // refined_shape[1], + ) + + def select_fp8_moe_backend( config: FusedMoEConfig, weight_key: QuantKey | None, @@ -575,6 +664,14 @@ def convert_to_fp8_moe_kernel_format( ) w13, w2 = prepare_fp8_moe_layer_for_cpu(w13, w2) + elif fp8_backend == Fp8MoeBackend.CPU_W8A8: + from vllm.model_executor.layers.fused_moe.experts.cpu_moe import ( + prepare_fp8_w8a8_moe_layer_for_cpu, + ) + + w13, w13_scale, w2, w2_scale = prepare_fp8_w8a8_moe_layer_for_cpu( + w13, w2, w13_scale, w2_scale + ) else: if fp8_backend not in [ Fp8MoeBackend.TRITON, @@ -620,7 +717,15 @@ def make_fp8_moe_quant_config( In a future PR, we will have this function should be a method of the modular kernel itself. """ - # MARLIN and CPU are mixed precision W8A16 config. + if fp8_backend == Fp8MoeBackend.CPU_W8A8: + return fp8_w8a8_moe_quant_config( + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + block_shape=block_shape, + ) + + # MARLIN and CPU (W8A16) are mixed precision W8A16 configs. if fp8_backend == Fp8MoeBackend.MARLIN or fp8_backend == Fp8MoeBackend.CPU: return fp8_w8a16_moe_quant_config( w1_scale=w1_scale, diff --git a/vllm/model_executor/layers/fused_moe/oracle/int8.py b/vllm/model_executor/layers/fused_moe/oracle/int8.py index 3b8a3c4f87..1a23b01736 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/int8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/int8.py @@ -71,10 +71,16 @@ def backend_to_kernel_cls( from vllm.model_executor.layers.fused_moe.experts.cpu_moe import ( ArmCPUExpertsInt8, CPUExpertsInt8, + PowerCPUExpertsInt8, ZenCPUExpertsInt8, ) - return [ZenCPUExpertsInt8, ArmCPUExpertsInt8, CPUExpertsInt8] + return [ + ZenCPUExpertsInt8, + PowerCPUExpertsInt8, + ArmCPUExpertsInt8, + CPUExpertsInt8, + ] else: raise ValueError(f"Unknown Int8 MoE backend: {backend.value}") diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py index ce19ba39bf..7fed4124f5 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py @@ -31,7 +31,12 @@ from vllm.model_executor.layers.fused_moe.config import ( from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( swap_w13_to_w31, ) -from vllm.model_executor.layers.quantization.utils.mxfp4_utils import _swizzle_mxfp4 +from vllm.model_executor.layers.quantization.utils.mxfp4_utils import ( + _swizzle_mxfp4, +) +from vllm.model_executor.layers.quantization.utils.mxfp4_utils import ( + mx_scale_kwargs as _mx_scale_kwargs, +) from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import ( OCP_MX_BLOCK_SIZE, ) @@ -70,15 +75,6 @@ if triton_kernels_version is not None: ) -def _mx_scale_kwargs(scale): - """PrecisionConfig weight-scale kwargs: 3.8 uses b_mx_scale/b_microblock_size, - 3.5.1/3.6 use weight_scale. - """ - if triton_kernels_version == "3.8": - return {"b_mx_scale": scale, "b_microblock_size": 32} - return {"weight_scale": scale} - - def _pack_deepgemm_mxfp4_scales( w13_weight: torch.Tensor, w2_weight: torch.Tensor, @@ -769,18 +765,31 @@ def mxfp4_round_up_hidden_size_and_intermediate_size( intermediate_size = round_up(intermediate_size, 128) hidden_size = round_up(hidden_size, 128) elif current_platform.is_rocm(): - if backend == Mxfp4MoeBackend.AITER_MXFP4_BF16 and ( - activation == MoEActivation.SITU or activation == MoEActivation.SILU - ): - # K3's AITER A16W4 SiTU kernel handles K3's native intermediate size - # (moe_intermediate 3072; e.g. 384/partition at TP8). Align to 128 (a - # no-op for K3's shapes) rather than the generic ROCm 256 round-up, - # which would inflate weights and OOM. - intermediate_size = round_up(intermediate_size, 128) - hidden_size = round_up(hidden_size, 128) - else: - intermediate_size = round_up(intermediate_size, 256) - hidden_size = round_up(hidden_size, 256) + from vllm.platforms.rocm import get_cdna_version + + is_situ_or_silu = activation in ( + MoEActivation.SITU, + MoEActivation.SILU, + ) + + # K3's AITER A16W4 SiTU kernel handles K3's native intermediate size + # (moe_intermediate 3072; e.g. 384/partition at TP8). Align to 128 + # rather than the generic ROCm 256 round-up, which would inflate + # weights and OOM. + aiter_uses_128 = backend == Mxfp4MoeBackend.AITER_MXFP4_BF16 + + # matmul_ogs uses block_k=128 for MXFP4 on pre-CDNA4 GPUs. + # CDNA4's F16xMXFP4 configuration uses block_k=256. + triton_uses_128 = ( + backend == Mxfp4MoeBackend.TRITON_UNFUSED and get_cdna_version() != 4 + ) + + alignment = ( + 128 if is_situ_or_silu and (aiter_uses_128 or triton_uses_128) else 256 + ) + + intermediate_size = round_up(intermediate_size, alignment) + hidden_size = round_up(hidden_size, alignment) elif backend == Mxfp4MoeBackend.CPU: # CPU AMX kernel uses BLOCK_N=32, align to 32 intermediate_size = round_up(intermediate_size, 32) diff --git a/vllm/model_executor/layers/fused_moe/oracle/unquantized.py b/vllm/model_executor/layers/fused_moe/oracle/unquantized.py index 9101787059..b9f90012f4 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/unquantized.py +++ b/vllm/model_executor/layers/fused_moe/oracle/unquantized.py @@ -26,6 +26,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( swap_w13_to_w31, ) from vllm.platforms import current_platform +from vllm.utils.math_utils import round_up if TYPE_CHECKING: from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey @@ -39,6 +40,7 @@ class UnquantizedMoeBackend(Enum): AITER = "ROCm AITER" TRITON = "TRITON" BATCHED_TRITON = "BATCHED_TRITON" + MOONEP = "MOONEP" CPU = "CPU" XPU = "XPU" TPU = "TPU" @@ -137,6 +139,13 @@ def backend_to_kernel_cls( return [BatchedTritonExperts] + elif backend == UnquantizedMoeBackend.MOONEP: + from vllm.model_executor.layers.fused_moe.experts.moonep_experts import ( + MoonEPExperts, + ) + + return [MoonEPExperts] + elif backend == UnquantizedMoeBackend.XPU: from vllm.model_executor.layers.fused_moe.experts.xpu_moe import XPUExperts @@ -280,6 +289,13 @@ def select_unquantized_moe_backend( return backend, k_cls raise ValueError(_make_log_unsupported(backend, reason)) + # MoonEP owns the expert layout (expert-grouped [NvS, H] + cu_seqlens), + # so it is not interchangeable with the token-major experts backends. + if moe_config.moe_parallel_config.use_moonep_kernels: + return _return_or_raise( + UnquantizedMoeBackend.MOONEP, moe_config, activation_format + ) + runner_backend = moe_config.moe_backend # 'humming' is quantization-only; an unquantized layer (e.g. excluded via # modules_to_not_convert) falls through to auto instead of erroring. @@ -305,7 +321,33 @@ def select_unquantized_moe_backend( AVAILABLE_BACKENDS.remove(UnquantizedMoeBackend.AITER) else: backend = UnquantizedMoeBackend.AITER - return _return_or_raise(backend, moe_config, activation_format) + reason = None + for k_cls in backend_to_kernel_cls(backend): + supported, reason = k_cls.is_supported_config( + k_cls, moe_config, None, None, activation_format + ) + if supported: + logger.info_once(_make_log_backend(backend)) + return backend, k_cls + # AITER was explicitly requested but does not support this + # deployment configuration (e.g. a non-gated MoE activation, + # which AiterExperts._supports_no_act_and_mul() rejects + # unconditionally). Rather than hard-crashing at engine init, + # warn — explicitly stating that this is falling back, and + # which backends will be tried next — and fall back to the + # remaining backends in priority order below. + logger.warning_once( + "VLLM_ROCM_USE_AITER_MOE=1 was requested, but %s " + "Falling back to try the remaining available MoE backends: %s.", + _make_log_unsupported(backend, reason), + ", ".join( + b.value + for b in AVAILABLE_BACKENDS + if b != UnquantizedMoeBackend.AITER + ), + ) + if UnquantizedMoeBackend.AITER in AVAILABLE_BACKENDS: + AVAILABLE_BACKENDS.remove(UnquantizedMoeBackend.AITER) for backend in AVAILABLE_BACKENDS: for k_cls in backend_to_kernel_cls(backend): @@ -323,12 +365,58 @@ def select_unquantized_moe_backend( ) +def unquantized_round_up_hidden_size_and_intermediate_size( + backend: UnquantizedMoeBackend, + hidden_size: int, + intermediate_size: int, +) -> tuple[int, int]: + """Round up dimensions before allocation to satisfy the selected kernel.""" + if backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM: + intermediate_size = round_up(intermediate_size, 128) + return hidden_size, intermediate_size + + def convert_to_unquantized_kernel_format( unquantized_backend: UnquantizedMoeBackend, moe_config: FusedMoEConfig, w13_weight: torch.Tensor, w2_weight: torch.Tensor, + layer: torch.nn.Module | None = None, ) -> tuple[torch.Tensor, torch.Tensor]: + if unquantized_backend == UnquantizedMoeBackend.MOONEP: + if layer is None: + raise ValueError( + "MoonEP weight conversion requires the layer module to stash " + "the [E+B] weight layout on" + ) + # MoonEP addresses expert weights by global [E+B] row and needs one + # contiguous tensor per projection (gate / up / down). The layer's + # w13_weight becomes the gate tensor and w2_weight the down tensor; + # the up tensor and the full layout are stashed on the layer for + # the kernel wiring in _setup_kernel. + from vllm.model_executor.layers.fused_moe.prepare_finalize.moonep import ( + MOONEP_DEFAULT_NUM_PREFETCH_SLOTS, + gather_moonep_weight_layout, + ) + + if getattr(layer, "_moonep_weight_layout", None) is not None: + # On a reload the layer parameters were already replaced by the + # converted [E+B] gate/down tensors, so the source-format weights + # this conversion needs are gone (and the unregistered up + # projection cannot be recovered from them at all). Tracked in + # RFC #52095. + raise NotImplementedError( + "MoonEP does not support in-place weight reloads yet" + ) + layout = gather_moonep_weight_layout( + w13_weight, + w2_weight, + num_global_experts=moe_config.num_experts, + num_prefetch_slots=MOONEP_DEFAULT_NUM_PREFETCH_SLOTS, + ) + layer._moonep_weight_layout = layout + return layout.full_gate_weight, layout.full_down_weight + if unquantized_backend == UnquantizedMoeBackend.AITER: w13_weight, w2_weight = rocm_aiter_ops.shuffle_weights(w13_weight, w2_weight) w13_weight.is_shuffled = True @@ -351,13 +439,26 @@ def convert_to_unquantized_kernel_format( ) moe_config.intermediate_size_per_partition = padded_intermediate + # Reloads only overwrite checkpoint slices. An earlier in-place + # permutation can leave nonzero values in the raw padding slots. + unpadded = moe_config.intermediate_size_per_partition_unpadded + assert unpadded is not None + if padded_intermediate > unpadded: + w13_weight[:, unpadded:padded_intermediate].zero_() + if is_act_and_mul: + w13_weight[:, padded_intermediate + unpadded :].zero_() + w2_weight[:, :, unpadded:].zero_() + _cache_permute_indices: dict[torch.Size, torch.Tensor] = {} - w13_weight, w2_weight = convert_moe_weights_to_flashinfer_trtllm_block_layout( + convert_moe_weights_to_flashinfer_trtllm_block_layout( _cache_permute_indices, w13_weight, w2_weight, is_gated_act_gemm=is_act_and_mul, ) + # Keep checkpoint-shaped parameters for reload/IPC. The experts create + # BlockMajorK views at dispatch without changing the underlying storage. + return w13_weight, w2_weight if ( unquantized_backend == UnquantizedMoeBackend.TRITON @@ -471,9 +572,10 @@ class UnquantizedMoEKernelOracle(MoEKernelOracle[UnquantizedMoeBackend]): moe_config: FusedMoEConfig, w13_weight: torch.Tensor, w2_weight: torch.Tensor, + layer: torch.nn.Module | None = None, ) -> tuple[torch.Tensor, torch.Tensor]: return convert_to_unquantized_kernel_format( - backend, moe_config, w13_weight, w2_weight + backend, moe_config, w13_weight, w2_weight, layer=layer ) def make_kernel( diff --git a/vllm/model_executor/layers/fused_moe/prepare_finalize/moonep.py b/vllm/model_executor/layers/fused_moe/prepare_finalize/moonep.py new file mode 100644 index 0000000000..fdbed5ade8 --- /dev/null +++ b/vllm/model_executor/layers/fused_moe/prepare_finalize/moonep.py @@ -0,0 +1,429 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""MoonEP (https://github.com/MoonshotAI/MoonEP) prepare/finalize. + +BF16 correctness-first proof of concept on top of vLLM's modular kernel +interface. + +MoonEP differs from DeepEP-style backends in two ways that shape this +integration: + +- ``dispatch`` returns tokens already grouped by expert *row* (a fixed + ``[NvS, H]`` layout with ``NvS = S x K`` real slots plus padding) together + with a ``cu_seqlens[E+B]`` segment table and an opaque ``plan``. There is + no per-token topk id tensor after dispatch; the expert compute must be a + grouped GEMM over ``cu_seqlens`` segments. +- Rows ``[E, E+B)`` of the weight/segment space are dynamic redundant-expert + prefetch slots. ``plan.experts_to_copy`` names the source expert of each + slot and ``Buffer.prefetch_weight`` must run between dispatch and expert + compute. + +PoC limitations: +- BF16 / unquantized only, eager only. +- Expert weights are replicated in global-expert order on every rank + (memory-heavy). Production Kimi-K3 serving requires sharded + symmetric-memory expert ownership, where rows ``[0, E)`` physically alias + each home rank's parameter memory. +- Route weights are applied inside the expert compute and MoonEP's + ``combine`` performs the K-sum, so ``finalize`` requires + ``TopKWeightAndReduceNoOP``. +""" + +from typing import Any, NamedTuple + +import torch +import torch.nn.functional as F + +import vllm.model_executor.layers.fused_moe.modular_kernel as mk +from vllm.forward_context import get_forward_context +from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig +from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import ( + TopKWeightAndReduceNoOP, +) + +MOONEP_DEFAULT_NUM_PREFETCH_SLOTS = 4 +MOONEP_DEFAULT_TOKEN_PADDING = 128 +MOONEP_DEFAULT_NUM_SMS = 32 +# MoonEP's weight-prefetch kernel tiles hidden and intermediate dims by 128. +MOONEP_WEIGHT_TILE = 128 + + +class MoonEPExpertWeightLayout(NamedTuple): + """Contiguous BF16 expert weights in MoonEP ``[E+B, ...]`` layout. + + Rows ``[0, E)`` hold expert weights in global expert order; rows + ``[E, E+B)`` are mutable prefetch slots filled by + ``Buffer.prefetch_weight``. + + The prefetch slots never need re-zeroing between calls: the planner + marks unused slots with ``-1`` in ``plan.experts_to_copy`` (skipped by + ``prefetch_weight``) and gives them an empty ``cu_seqlens`` segment, so + stale slot contents are never read; used slots are fully overwritten. + """ + + full_gate_weight: torch.Tensor + full_up_weight: torch.Tensor + full_down_weight: torch.Tensor + num_prefetch_slots: int + + +def make_moonep_weight_layout( + w13_weight: torch.Tensor, + w2_weight: torch.Tensor, + num_prefetch_slots: int, +) -> MoonEPExpertWeightLayout: + """Build the replicated ``[E+B, ...]`` PoC weight layout. + + ``w13_weight`` must be ``[E, 2I, H]`` (gate rows first) and ``w2_weight`` + ``[E, H, I]``, both BF16 in global expert order. + """ + if num_prefetch_slots <= 0: + raise ValueError( + f"num_prefetch_slots must be positive, got {num_prefetch_slots}" + ) + if w13_weight.dtype != torch.bfloat16 or w2_weight.dtype != torch.bfloat16: + raise NotImplementedError("MoonEP PoC supports BF16 weights only.") + num_experts, two_i, hidden_size = w13_weight.shape + intermediate_size = two_i // 2 + if tuple(w2_weight.shape) != (num_experts, hidden_size, intermediate_size): + raise ValueError( + f"w2_weight shape {tuple(w2_weight.shape)} does not match " + f"w13_weight shape {tuple(w13_weight.shape)}" + ) + if hidden_size % MOONEP_WEIGHT_TILE or intermediate_size % MOONEP_WEIGHT_TILE: + raise ValueError( + "MoonEP weight prefetch requires hidden_size and intermediate_size " + f"to be multiples of {MOONEP_WEIGHT_TILE}; got H={hidden_size}, " + f"I={intermediate_size}" + ) + + full_gate_weight = torch.empty( + num_experts + num_prefetch_slots, + intermediate_size, + hidden_size, + dtype=torch.bfloat16, + device=w13_weight.device, + ) + full_up_weight = torch.empty_like(full_gate_weight) + full_down_weight = torch.empty( + num_experts + num_prefetch_slots, + hidden_size, + intermediate_size, + dtype=torch.bfloat16, + device=w2_weight.device, + ) + + full_gate_weight[:num_experts].copy_(w13_weight[:, :intermediate_size, :]) + full_up_weight[:num_experts].copy_(w13_weight[:, intermediate_size:, :]) + full_down_weight[:num_experts].copy_(w2_weight) + full_gate_weight[num_experts:].zero_() + full_up_weight[num_experts:].zero_() + full_down_weight[num_experts:].zero_() + + return MoonEPExpertWeightLayout( + full_gate_weight=full_gate_weight.contiguous(), + full_up_weight=full_up_weight.contiguous(), + full_down_weight=full_down_weight.contiguous(), + num_prefetch_slots=num_prefetch_slots, + ) + + +def gather_moonep_weight_layout( + w13_local: torch.Tensor, + w2_local: torch.Tensor, + num_global_experts: int, + num_prefetch_slots: int, +) -> MoonEPExpertWeightLayout: + """Build the replicated ``[E+B, ...]`` layout from this rank's local experts. + + PoC bridge: each EP rank loads only its own experts (linear placement), + so all-gather them once at load time into global expert order on every + rank. Production MoonEP instead maps rows ``[0, E)`` onto each home + rank's parameter memory via symmetric memory (RFC #52095 item 5). + """ + from vllm.distributed import get_ep_group + + ep_group = get_ep_group() + ep_size = ep_group.world_size + if ep_size == 1: + return make_moonep_weight_layout(w13_local, w2_local, num_prefetch_slots) + if w13_local.size(0) * ep_size != num_global_experts: + raise NotImplementedError( + "MoonEP PoC requires num_experts to be evenly divisible across EP " + f"ranks: {num_global_experts} experts, {ep_size} ranks" + ) + w13_global = ep_group.all_gather(w13_local.contiguous(), dim=0) + w2_global = ep_group.all_gather(w2_local.contiguous(), dim=0) + return make_moonep_weight_layout(w13_global, w2_global, num_prefetch_slots) + + +MOONEP_MIN_DISPATCH_TOKENS = 128 + + +class MoonEPBufferPool: + """Lazily creates ``moonep.Buffer`` instances at power-of-two capacities. + + ``Buffer`` fixes its token capacity ``S`` at construction, so a single + full-capacity buffer would pad every dispatch to + ``max_num_batched_tokens``. The pool instead serves the smallest + power-of-two capacity that fits the step's token count, so decode-sized + batches dispatch a few hundred slots rather than the full capacity. + + Selection must be identical on every EP rank (dispatch is collective and + ``Buffer`` construction is itself a collective): callers derive the token + count from ``dp_metadata`` (the max across DP ranks) so all ranks create + and pick the same buffer at the same step. + """ + + def __init__(self, buffer_kwargs: dict[str, Any], max_tokens_per_rank: int): + self._kwargs = {k: v for k, v in buffer_kwargs.items() if k != "S"} + self.max_tokens_per_rank = max_tokens_per_rank + self._buffers: dict[int, Any] = {} + + def capacity_for(self, num_tokens: int) -> int: + if num_tokens >= self.max_tokens_per_rank: + return self.max_tokens_per_rank + cap = 1 << max(num_tokens - 1, 0).bit_length() + return min(max(cap, MOONEP_MIN_DISPATCH_TOKENS), self.max_tokens_per_rank) + + def get(self, num_tokens: int) -> tuple[int, Any]: + """Return ``(capacity, buffer)`` for the given step token count.""" + from moonep import Buffer # type: ignore[import-not-found] + + capacity = self.capacity_for(num_tokens) + buffer = self._buffers.get(capacity) + if buffer is None: + buffer = Buffer(S=capacity, **self._kwargs) + self._buffers[capacity] = buffer + return capacity, buffer + + def destroy(self) -> None: + for buffer in self._buffers.values(): + buffer.destroy() + self._buffers.clear() + + +class MoonEPPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular): + """Prepare/Finalize using MoonEP balanced dispatch/combine. + + ``prepare`` pads the batch to the selected buffer's static token + capacity, dispatches, runs ``prefetch_weight`` for the planned redundant + experts, and stashes the ``plan`` for ``finalize`` (the same pattern + DeepEP-HT uses for its handle). Downstream expert compute must consume + the expert-grouped ``[NvS, H]`` layout via ``cu_seqlens``. + """ + + def __init__( + self, + buffer_pool: "MoonEPBufferPool", + max_tokens_per_rank: int, + num_dispatchers: int, + num_global_experts: int, + weight_layout: MoonEPExpertWeightLayout | None = None, + ): + super().__init__() + self.buffer_pool = buffer_pool + self.max_tokens_per_rank = max_tokens_per_rank + self.num_dispatchers_ = num_dispatchers + self.num_global_experts = num_global_experts + self.weight_layout = weight_layout + self._fused_experts: Any = None + + # dispatch state consumed by finalize (and the expert runner) + self._plan: Any = None + self._cu_seqlens: torch.Tensor | None = None + self._num_tokens: int = 0 + self._buffer: Any = None # the pool buffer used by the current step + + def post_init_setup(self, fused_experts: mk.FusedMoEExperts) -> None: + # The [E+B] weight layout is attached to the experts by their + # process_weights_after_loading hook (after this runs), so keep the + # reference and resolve the layout lazily in prepare(). + self._fused_experts = fused_experts + + def _resolve_weight_layout(self) -> MoonEPExpertWeightLayout: + if self.weight_layout is None and self._fused_experts is not None: + layout = getattr(self._fused_experts, "weight_layout", None) + if isinstance(layout, MoonEPExpertWeightLayout): + self.weight_layout = layout + # Redundant experts' weights must be in rows [E, E+B) before the + # expert compute reads them; skipping prefetch silently corrupts + # output. + assert self.weight_layout is not None, ( + "MoonEPPrepareAndFinalize: weight layout not available (the " + "experts' process_weights_after_loading has not run)" + ) + return self.weight_layout + + @property + def num_dispatched_slots(self) -> int: + """``NvS`` of the buffer selected by the current step's prepare().""" + assert self._buffer is not None, "prepare() has not been called" + return int(self._buffer._ctx["NvS"]) + + def num_dispatchers(self) -> int: + return self.num_dispatchers_ + + def output_is_reduced(self) -> bool: + # combine returns the fully weighted+reduced token-major output + return True + + @property + def activation_format(self) -> mk.FusedMoEActivationFormat: + return mk.FusedMoEActivationFormat.Standard + + def max_num_tokens_per_rank(self) -> int | None: + return self.max_tokens_per_rank + + def topk_indices_dtype(self) -> torch.dtype | None: + return torch.int32 + + def supports_async(self) -> bool: + return False + + @property + def cu_seqlens(self) -> torch.Tensor: + assert self._cu_seqlens is not None, "prepare() has not been called" + return self._cu_seqlens + + @property + def plan(self) -> Any: + assert self._plan is not None, "prepare() has not been called" + return self._plan + + def _pad_to_capacity( + self, + a1: torch.Tensor, + topk_ids: torch.Tensor, + topk_weights: torch.Tensor, + capacity: int, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, int]: + num_tokens = a1.shape[0] + if num_tokens > capacity: + raise ValueError( + f"MoonEP static buffer capacity exceeded: {num_tokens} tokens " + f"> S={capacity}." + ) + a1 = a1.contiguous() + topk_ids = topk_ids.to(dtype=torch.int32) + topk_weights = topk_weights.to(dtype=torch.float32) + # MoonEP's planner has no sentinel for invalid expert ids, so map + # negative ids (padded/invalid slots from the engine) to expert 0 + # with zero route weight — the same convention as the capacity + # padding rows below. Wasted compute, never wrong output. + invalid = topk_ids < 0 + topk_ids = torch.where(invalid, torch.zeros_like(topk_ids), topk_ids) + topk_weights = torch.where( + invalid, torch.zeros_like(topk_weights), topk_weights + ).contiguous() + topk_ids = topk_ids.contiguous() + if num_tokens == capacity: + return a1, topk_ids, topk_weights, num_tokens + pad = capacity - num_tokens + return ( + F.pad(a1, (0, 0, 0, pad)), + F.pad(topk_ids, (0, 0, 0, pad)), + F.pad(topk_weights, (0, 0, 0, pad)), + num_tokens, + ) + + def prepare( + self, + a1: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + num_experts: int, + expert_map: torch.Tensor | None, + apply_router_weight_on_input: bool, + quant_config: FusedMoEQuantConfig, + defer_input_quant: bool = False, + ) -> mk.PrepareResultType: + if a1.dtype != torch.bfloat16: + raise NotImplementedError("MoonEP PoC supports BF16 hidden states only.") + if quant_config.quant_dtype is not None: + raise NotImplementedError("MoonEP PoC does not support quantized dispatch.") + assert num_experts == self.num_global_experts + assert self._plan is None, ( + "MoonEPPrepareAndFinalize.prepare() called again before finalize()" + ) + if apply_router_weight_on_input: + assert topk_ids.size(1) == 1, ( + "apply_router_weight_on_input requires top-1 routing" + ) + a1 = a1 * topk_weights.to(a1.dtype) + + # Buffer choice must be rank-symmetric: use the max token count + # across DP ranks (every rank sees the same dp_metadata), falling + # back to the local count outside a forward context (tests). + num_tokens = a1.shape[0] + try: + dp_meta = get_forward_context().dp_metadata + except AssertionError: + dp_meta = None + if dp_meta is not None: + num_tokens = int(dp_meta.num_tokens_across_dp_cpu.max()) + capacity, self._buffer = self.buffer_pool.get(num_tokens) + + a1, topk_ids, topk_weights, self._num_tokens = self._pad_to_capacity( + a1, topk_ids, topk_weights, capacity + ) + tokens_per_expert = torch.bincount( + topk_ids.reshape(-1).to(dtype=torch.int64), + minlength=num_experts, + ).to(dtype=torch.int32) + + hidden_nvsh, route_weights_nvs, cu_seqlens, plan = self._buffer.dispatch( + a1, + topk_weights, + topk_ids, + tokens_per_expert, + ) + self._plan = plan + self._cu_seqlens = cu_seqlens + + weight_layout = self._resolve_weight_layout() + self._buffer.prefetch_weight( + plan=plan, + full_gate_weight=weight_layout.full_gate_weight, + full_up_weight=weight_layout.full_up_weight, + full_down_weight=weight_layout.full_down_weight, + ) + + # Segment sizes per [E+B] weight row. NOTE: this is per *row*, not + # per local expert, and NvS rows are expert-grouped rather than + # token-major — only MoonEP-aware expert implementations can consume + # this activation layout. + expert_num_tokens = torch.diff(cu_seqlens, prepend=cu_seqlens.new_zeros(1)) + expert_tokens_meta = mk.ExpertTokensMetadata( + expert_num_tokens=expert_num_tokens, + expert_num_tokens_cpu=None, + ) + + # MoonEP has no post-dispatch per-token topk ids/weights; route + # weights come back in NvS order. + return hidden_nvsh, None, expert_tokens_meta, None, route_weights_nvs + + def finalize( + self, + output: torch.Tensor, + fused_expert_output: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + apply_router_weight_on_input: bool, + weight_and_reduce_impl: mk.TopKWeightAndReduce, + ) -> None: + # Route weights are applied in the expert compute and MoonEP's + # combine performs the K-sum, so any weight/reduce work in finalize + # means MoonEP was paired with the wrong kind of experts. + assert isinstance(weight_and_reduce_impl, TopKWeightAndReduceNoOP), ( + "MoonEP requires TopKWeightAndReduceNoOP, got " + f"{type(weight_and_reduce_impl).__name__}" + ) + combined, _, _ = self._buffer.combine( + plan=self.plan, + hidden_nvsh=fused_expert_output, + route_weights_nvs=None, + ) + output.copy_(combined[: self._num_tokens]) + self._plan = None + self._cu_seqlens = None diff --git a/vllm/model_executor/layers/fused_moe/routed_experts.py b/vllm/model_executor/layers/fused_moe/routed_experts.py index 88841070f0..e3a287ddc2 100644 --- a/vllm/model_executor/layers/fused_moe/routed_experts.py +++ b/vllm/model_executor/layers/fused_moe/routed_experts.py @@ -27,6 +27,7 @@ from vllm.model_executor.layers.quantization.base_config import ( QuantizationConfig, resolve_quant_method, ) +from vllm.utils.math_utils import cdiv if TYPE_CHECKING: from vllm.model_executor.layers.fused_moe.runner.shared_experts import SharedExperts @@ -332,6 +333,7 @@ class RoutedExperts(PluggableLayer): shard_id: str, loaded_weight: torch.Tensor, tp_rank: int, + is_scale: bool = False, ): """Load grouped weight scales for group quantization or model weights @@ -341,14 +343,41 @@ class RoutedExperts(PluggableLayer): shard_id: either w1, w2, or w3 loaded_weight: checkpoint weight to load into the param tp_rank: tensor parallel rank + is_scale: whether padding should use unit scales instead of zero weights. """ + padded_tp = self.moe_config.tp_shard_with_padding + if padded_tp: + destination = expert_data + if shard_id in ("w1", "w3") and self.moe_config.is_act_and_mul: + half = destination.shape[shard_dim] // 2 + destination = destination.narrow( + shard_dim, 0 if shard_id == "w1" else half, half + ) + shard_size = destination.shape[shard_dim] + expected_size = ( + self.moe_config.intermediate_size + * shard_size + // self.moe_config.intermediate_size_per_partition + ) + if loaded_weight.shape[shard_dim] != expected_size: + raise ValueError( + "Block-aligned TP loading expects an unsharded checkpoint " + f"projection of size {expected_size}, got " + f"{loaded_weight.shape[shard_dim]}." + ) + start = min(tp_rank * shard_size, expected_size) + size = min(shard_size, expected_size - start) + loaded_weight = loaded_weight.narrow(shard_dim, start, size) + destination.fill_(1 if is_scale else 0) + if shard_id == "w2": self._load_w2( shard_dim=shard_dim, loaded_weight=loaded_weight, expert_data=expert_data, tp_rank=tp_rank, + load_full=padded_tp, ) elif shard_id in ("w1", "w3"): self._load_w13( @@ -357,6 +386,7 @@ class RoutedExperts(PluggableLayer): loaded_weight=loaded_weight, expert_data=expert_data, tp_rank=tp_rank, + load_full=padded_tp, ) def _load_per_channel_weight_scale( @@ -511,11 +541,12 @@ class RoutedExperts(PluggableLayer): shard_dim: int, loaded_weight: torch.Tensor, tp_rank: int, + load_full: bool = False, ): # Index the loaded weight for tp sharding. # down_proj: "RowParallel" so tp sharding on input_dim - # Only narrow if the loaded_weight is not a scalar (0-dim tensor). - if loaded_weight.ndim > 0: + # Padded TP weights have already been sliced by the grouped loader. + if not load_full and loaded_weight.ndim > 0: # Same padding fix as _load_w13: use unpadded per-rank size. tp_size = self.moe_config.moe_parallel_config.tp_size loaded_per_rank = loaded_weight.shape[shard_dim] // tp_size @@ -536,6 +567,23 @@ class RoutedExperts(PluggableLayer): hidden_dim=hidden_dim, shard_dim=shard_dim, ) + if ( + loaded_weight.device.type == "cpu" + and expert_data.device.type == "cuda" + and not expert_data.is_contiguous() + and expert_data.ndim == 2 + ): + # A strided CPU-to-CUDA copy can allocate a full-shard CUDA + # temporary. Make the TP slice contiguous on CPU and copy rows + # in chunks to limit the temporary used for the padded view. + loaded_weight = loaded_weight.contiguous() + num_chunks = max(cdiv(loaded_weight.nbytes, 1 << 20), 1) + for dst, src in zip( + expert_data.chunk(num_chunks, dim=0), + loaded_weight.chunk(num_chunks, dim=0), + ): + dst.copy_(src) + return expert_data.copy_(loaded_weight) def _load_single_value( @@ -800,6 +848,7 @@ class RoutedExperts(PluggableLayer): loaded_weight=loaded_weight, expert_data=expert_data, tp_rank=self.moe_config.tp_rank, + is_scale=True, ) elif quant_method == FusedMoeWeightScaleSupported.TENSOR.value: self._load_per_tensor_weight_scale( diff --git a/vllm/model_executor/layers/fused_moe/router/grouped_topk_router.py b/vllm/model_executor/layers/fused_moe/router/grouped_topk_router.py index 2df3489358..9b134d385a 100644 --- a/vllm/model_executor/layers/fused_moe/router/grouped_topk_router.py +++ b/vllm/model_executor/layers/fused_moe/router/grouped_topk_router.py @@ -8,6 +8,7 @@ from vllm import _custom_ops as ops from vllm import envs as envs from vllm._aiter_ops import rocm_aiter_ops from vllm.distributed.eplb.eplb_state import EplbLayerState +from vllm.forward_context import get_forward_context, is_forward_context_available from vllm.model_executor.custom_op import CustomOp from vllm.model_executor.layers.fused_moe.config import ( RoutingMethodType, @@ -258,6 +259,7 @@ class GroupedTopKRouter(BaseRouter): e_score_correction_bias: torch.Tensor | None = None, num_fused_shared_experts: int = 0, eplb_state: EplbLayerState | None = None, + skip_padding: bool = False, ): super().__init__( top_k=top_k, @@ -271,6 +273,7 @@ class GroupedTopKRouter(BaseRouter): self.routed_scaling_factor = routed_scaling_factor self.e_score_correction_bias = e_score_correction_bias self.num_fused_shared_experts = num_fused_shared_experts + self.skip_padding = skip_padding @property def routing_method_type(self) -> RoutingMethodType: @@ -346,4 +349,15 @@ class GroupedTopKRouter(BaseRouter): e_score_correction_bias=self.e_score_correction_bias, ) + if ( + self.skip_padding + and envs.VLLM_MOE_SKIP_PADDING + and is_forward_context_available() + ): + is_padding = get_forward_context().is_padding + if is_padding is not None: + is_padding = is_padding[: topk_ids.shape[0]].unsqueeze(1) + topk_weights = topk_weights.masked_fill(is_padding, 0) + topk_ids = topk_ids.masked_fill(is_padding, -1) + return topk_weights, topk_ids diff --git a/vllm/model_executor/layers/fused_moe/router/router_factory.py b/vllm/model_executor/layers/fused_moe/router/router_factory.py index 2037251824..f03fb106f7 100644 --- a/vllm/model_executor/layers/fused_moe/router/router_factory.py +++ b/vllm/model_executor/layers/fused_moe/router/router_factory.py @@ -65,6 +65,7 @@ def create_fused_moe_router( # Deepseek V4 vision routing bias parameters bias_vl: torch.Tensor | None = None, image_sentinel_lo: int = 0, + skip_padding: bool = False, ) -> FusedMoERouter: """Factory function to create the appropriate FusedMoERouter subclass based on the provided parameters. @@ -91,6 +92,7 @@ def create_fused_moe_router( topk_group: Top-k within each group (for grouped routing) scoring_func: Scoring function to use ("softmax" or "sigmoid") num_fused_shared_experts: Number of fused shared experts (for ROCm AITER) + skip_padding: Whether to invalidate routes for padding rows Grouped topk and fused topk bias arguments: routed_scaling_factor: Scaling factor for routed weights @@ -209,6 +211,7 @@ def create_fused_moe_router( routed_scaling_factor=routed_scaling_factor, e_score_correction_bias=e_score_correction_bias, num_fused_shared_experts=num_fused_shared_experts, + skip_padding=skip_padding, ) # Otherwise fall through to the non-grouped chain below. diff --git a/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py b/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py index 523d1df0d8..17481d4a91 100644 --- a/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py +++ b/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py @@ -11,6 +11,7 @@ from vllm.logger import init_logger from vllm.model_executor.custom_op import CustomOp from vllm.model_executor.layers.fused_moe.config import ( FusedMoEConfig, + FusedMoEParallelConfig, FusedMoEQuantConfig, biased_moe_quant_config, ) @@ -23,6 +24,7 @@ from vllm.model_executor.layers.fused_moe.oracle.unquantized import ( convert_to_unquantized_kernel_format, make_unquantized_moe_kernel, select_unquantized_moe_backend, + unquantized_round_up_hidden_size_and_intermediate_size, ) from vllm.model_executor.layers.fused_moe.runner.shared_experts import ( SharedExperts, @@ -47,13 +49,17 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): # --8<-- [end:unquantized_fused_moe] - supports_pre_processed_weights = True - def __init__(self, moe: FusedMoEConfig): super().__init__(moe) self.unquantized_backend, self.experts_cls = select_unquantized_moe_backend( moe_config=self.moe, ) + # MoonEP replaces the named parameters with its [E+B] gate/down + # split and keeps the up projection outside the parameter registry, + # so a pre-processed (e.g. ipc_cache) load cannot reconstruct it. + self.supports_pre_processed_weights = ( + self.unquantized_backend != UnquantizedMoeBackend.MOONEP + ) @property def supports_eplb(self) -> bool: @@ -68,10 +74,15 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): params_dtype: torch.dtype, **extra_weight_attrs, ): + unpadded_intermediate = self.moe.intermediate_size_per_partition_unpadded + assert unpadded_intermediate is not None if self.moe.is_act_and_mul: w13_up_dim = 2 * intermediate_size_per_partition else: w13_up_dim = intermediate_size_per_partition + unpadded_hidden = self.moe.hidden_dim_unpadded + assert unpadded_hidden is not None + unpadded_up = unpadded_intermediate * (2 if self.moe.is_act_and_mul else 1) # Fused gate_up_proj (column parallel) w13_weight = torch.nn.Parameter( torch.empty( @@ -84,6 +95,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): ) layer.register_parameter("w13_weight", w13_weight) set_weight_attrs(w13_weight, extra_weight_attrs) + w13_weight.weight_loader_numel = num_experts * unpadded_up * unpadded_hidden if self.moe.has_bias: w13_bias = torch.nn.Parameter( torch.zeros(num_experts, w13_up_dim, dtype=params_dtype), @@ -91,6 +103,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): ) layer.register_parameter("w13_bias", w13_bias) set_weight_attrs(w13_bias, extra_weight_attrs) + w13_bias.weight_loader_numel = num_experts * unpadded_up # down_proj (row parallel) w2_weight = torch.nn.Parameter( torch.empty( @@ -103,6 +116,9 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): ) layer.register_parameter("w2_weight", w2_weight) set_weight_attrs(w2_weight, extra_weight_attrs) + w2_weight.weight_loader_numel = ( + num_experts * unpadded_hidden * unpadded_intermediate + ) if self.moe.has_bias: w2_bias = torch.nn.Parameter( torch.zeros(num_experts, hidden_size, dtype=params_dtype), @@ -110,6 +126,24 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): ) layer.register_parameter("w2_bias", w2_bias) set_weight_attrs(w2_bias, extra_weight_attrs) + w2_bias.weight_loader_numel = num_experts * unpadded_hidden + + def maybe_roundup_sizes( + self, + hidden_size: int, + intermediate_size_per_partition: int, + act_dtype: torch.dtype, + moe_parallel_config: FusedMoEParallelConfig, + ) -> tuple[int, int]: + hidden_size, intermediate_size_per_partition = super().maybe_roundup_sizes( + hidden_size, + intermediate_size_per_partition, + act_dtype, + moe_parallel_config, + ) + return unquantized_round_up_hidden_size_and_intermediate_size( + self.unquantized_backend, hidden_size, intermediate_size_per_partition + ) def _maybe_pad_weight(self, weight: torch.Tensor) -> torch.Tensor: # Pad the weight tensor. This is an optimization on ROCm platform, which @@ -141,6 +175,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): moe_config=layer.moe_config, w13_weight=w13, w2_weight=w2, + layer=layer, ) # `moe_kernel` is initialized to None in FusedMoEMethodBase.__init__; # On the first call we replace the parameter normally. On subsequent @@ -161,14 +196,13 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): # does not need to be re-built. self._init_moe_kernel(layer) - if self.unquantized_backend == UnquantizedMoeBackend.CPU: - # The CPU experts need the layer itself for the setup that - # convert_to_unquantized_kernel_format cannot express, since - # it only sees the two weight tensors: padding and prepacking - # into the grouped-gemm layout (bias included), and capturing - # the router config that monolithic apply() cannot carry. - assert self.moe_kernel is not None - self.moe_kernel.fused_experts.process_weights_after_loading(layer) + # No-op by default. Experts that need the layer itself for setup + # convert_to_unquantized_kernel_format cannot express override + # it: CPU prepacks into its grouped-gemm layout and captures the + # router config; MoonEP picks up the [E+B] weight layout for + # prefetch and the up projection. + assert self.moe_kernel is not None + self.moe_kernel.fused_experts.process_weights_after_loading(layer) def _init_moe_kernel(self, layer: "RoutedExperts") -> None: """Build the MoE kernel from the layer's current (shuffled) weights.""" diff --git a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py index 807a5b9b1a..ccf1c60f0f 100644 --- a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py +++ b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py @@ -399,6 +399,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention): self.key_dim = self.head_k_dim * self.num_k_heads self.value_dim = self.head_v_dim * self.num_v_heads self.gqa_interleaved_layout = gqa_interleaved_layout + self.qkvz_layout = "interleaved" if gqa_interleaved_layout else "flat" if current_platform.is_xpu(): self._forward_method = self.forward_xpu elif current_platform.is_cpu(): @@ -1206,9 +1207,9 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention): """ROCm AITER fast path: conv1d + recurrent attention from packed qkvz/ba layout. - For decode-only (no spec, no prefill) interleaved-GQA layouts, - dispatches directly to ``_forward_core_decode_aiter``. Otherwise unpacks - the packed layout and falls through to ``_forward_core``. + For decode-only (no spec, no prefill) batches, dispatches directly to + ``_forward_core_decode_aiter``. Otherwise unpacks the packed layout and + falls through to ``_forward_core``. Args: qkvz: packed [q, k, v, z] projection (num_tokens, qkvz_dim) @@ -1231,12 +1232,8 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention): assert isinstance(attn_metadata, GDNAttentionMetadata) - # The AITER fused reshape/conv kernel expects Qwen3-Next's interleaved - # GQA layout. Qwen3.5 uses a non-interleaved q/k/v/z layout and must use - # the generic path below to split/rearrange inputs correctly. if ( - self.gqa_interleaved_layout - and attn_metadata.spec_sequence_masks is None + attn_metadata.spec_sequence_masks is None and attn_metadata.num_prefills == 0 and attn_metadata.num_decodes > 0 ): @@ -1625,6 +1622,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention): : attn_metadata.num_actual_tokens ], validate_data=True, + qkvz_layout=self.qkvz_layout, ) ) diff --git a/vllm/model_executor/layers/mamba/mamba_utils.py b/vllm/model_executor/layers/mamba/mamba_utils.py index db7e119560..8c89488d2d 100644 --- a/vllm/model_executor/layers/mamba/mamba_utils.py +++ b/vllm/model_executor/layers/mamba/mamba_utils.py @@ -222,12 +222,18 @@ class MambaStateShapeCalculator: @classmethod def append_replayssm_ring( cls, - base_shapes: tuple[tuple[int, ...], ...], + base_shapes: tuple[tuple[int, int], tuple[int, int, int]], n_groups: int, tp_world_size: int, logical_window: int, backend: MambaBackendEnum, - ) -> tuple[tuple[int, ...], ...]: + ) -> tuple[ + tuple[int, int], + tuple[int, int, int], + tuple[int, int, int], + tuple[int, int], + tuple[int, int, int], + ]: """Append the physical ReplaySSM ring shapes. ``base_shapes[1]`` is ``(nheads // tp, head_dim, state_size)``; diff --git a/vllm/model_executor/layers/mamba/ops/gdn_chunk_cutedsl/kernel_kkt_inv_uw.py b/vllm/model_executor/layers/mamba/ops/gdn_chunk_cutedsl/kernel_kkt_inv_uw.py index ba61ec68b5..4b4046a093 100644 --- a/vllm/model_executor/layers/mamba/ops/gdn_chunk_cutedsl/kernel_kkt_inv_uw.py +++ b/vllm/model_executor/layers/mamba/ops/gdn_chunk_cutedsl/kernel_kkt_inv_uw.py @@ -573,8 +573,8 @@ class Sm100ChunkUWKernel: for i in cutlass.range_constexpr(4): M[i] = _bf16x2_sub(_bf16x2_neg(eye[i]), M[i]) - # 3 rounds of Newton-Schulz - for _ in cutlass.range_constexpr(3): + # The fourth round keeps correlated-key KKT blocks stable in BF16. + for _ in cutlass.range_constexpr(4): # First MMA: -AiM = Ai @ (-M) cute.copy(stsm_atom, Ai_bf16, sA_ldsm[None, (warp_id_, warp_id_)]) cute.arch.sync_warp() diff --git a/vllm/model_executor/layers/quantization/auto_awq.py b/vllm/model_executor/layers/quantization/auto_awq.py index 2c123b8768..71b858e919 100644 --- a/vllm/model_executor/layers/quantization/auto_awq.py +++ b/vllm/model_executor/layers/quantization/auto_awq.py @@ -93,7 +93,7 @@ def _replace_or_register_parameter( def _convert_awq_to_standard_format( layer: torch.nn.Module, w_q_name: str, - w_zp_name: str, + w_zp_name: str | None, size_bits: int, ) -> None: """Convert AWQ weight and zero-point tensors to standard GPTQ-like format. @@ -101,6 +101,7 @@ def _convert_awq_to_standard_format( AWQ packs qweight along the output dim with a non-standard bit order. This converts to standard bit order and repacks qweight along the input dim, matching the format expected by the MPLinearKernel framework. + If w_zp_name is None (symmetric quantization), only the weight is converted. """ pack_factor = 32 // size_bits mask = (1 << size_bits) - 1 @@ -139,6 +140,9 @@ def _convert_awq_to_standard_format( ) setattr(layer, w_q_name, new_param) + if w_zp_name is None: + return + # --- Convert qzeros: fix AWQ bit ordering and repack # AWQ qzeros: (G, N // pack) packed along dim 1, AWQ bit order # Target: (N // pack, G) packed along dim 0, standard bit order diff --git a/vllm/model_executor/layers/quantization/fp8.py b/vllm/model_executor/layers/quantization/fp8.py index 5835af111a..c85a2c288e 100644 --- a/vllm/model_executor/layers/quantization/fp8.py +++ b/vllm/model_executor/layers/quantization/fp8.py @@ -8,7 +8,6 @@ import torch import vllm.envs as envs from vllm.config import get_current_vllm_config from vllm.distributed import get_tensor_model_parallel_world_size -from vllm.logger import init_logger from vllm.model_executor.kernels.linear import ( init_fp8_linear_kernel, ) @@ -24,14 +23,12 @@ from vllm.model_executor.layers.fused_moe import ( SharedExperts, UnquantizedFusedMoEMethod, ) -from vllm.model_executor.layers.fused_moe.config import ( - FusedMoEQuantConfig, -) +from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig from vllm.model_executor.layers.fused_moe.oracle.fp8 import ( convert_to_fp8_moe_kernel_format, make_fp8_moe_kernel, make_fp8_moe_quant_config, - refine_fp8_moe_block_shape, + resolve_fp8_moe_weight_block_shape, select_fp8_moe_backend, ) from vllm.model_executor.layers.linear import ( @@ -90,8 +87,6 @@ if TYPE_CHECKING: ACTIVATION_SCHEMES = ["static", "dynamic"] -logger = init_logger(__name__) - class Fp8Config(QuantizationConfig): """Config class for FP8.""" @@ -511,32 +506,16 @@ class Fp8MoEMethod(FusedMoEMethodBase): # Set weight key and activation key for kernel compatibility if self.block_quant: assert self.weight_block_size is not None - # TP shards the intermediate dim of the expert weights, so a - # per-shard size that is not a multiple of the checkpoint's block - # size makes the checkpoint's block scales impossible to shard - # exactly. When a finer block size (>= 32) divides both the - # checkpoint blocks and all involved dims, the weight scales are - # refined to that granularity at load time (a lossless upsampling, - # since the refined block divides the checkpoint block). The - # refined block shape is encoded in the weight key, so the oracle - # only selects kernels that support it (e.g. Triton, which takes - # the block shape as a runtime argument). - refined_shape = refine_fp8_moe_block_shape(self.moe, self.weight_block_size) - if refined_shape is not None: - block_n, block_k = self.weight_block_size - self.weight_scale_refine = ( - block_n // refined_shape[0], - block_k // refined_shape[1], + # Adapt the checkpoint block shape to TP sharding before + # building the weight key. + self.moe_block_shape, self.weight_scale_refine = ( + resolve_fp8_moe_weight_block_shape( + self.moe, + self.weight_block_size, + kFp8Dynamic128Sym, + self.quant_config.is_checkpoint_fp8_serialized, ) - self.moe_block_shape = refined_shape - logger.info_once( - "FP8 MoE block scales refined from %s to %s to fit " - "the TP-sharded intermediate size %d.", - str(self.weight_block_size), - str(refined_shape), - self.moe.intermediate_size_per_partition, - ) - assert self.moe_block_shape is not None + ) weight_key = create_fp8_quant_key( static=True, group_shape=GroupShape(*self.moe_block_shape) ) diff --git a/vllm/model_executor/layers/quantization/modelopt.py b/vllm/model_executor/layers/quantization/modelopt.py index c649061f33..364c52b4c8 100644 --- a/vllm/model_executor/layers/quantization/modelopt.py +++ b/vllm/model_executor/layers/quantization/modelopt.py @@ -2482,9 +2482,16 @@ class ModelOptLinearMethod(LinearMethodBase): @property def supports_pre_processed_weights(self) -> bool: # type: ignore[override] - # TODO(Isotr0py): support fp8/mxfp8 ModelOpt kernels transpose/repack. + # TODO(Isotr0py): support fp8 ModelOpt kernels transpose/repack. w = self.spec.weight - return isinstance(w, QuantKey) and w.dtype == FP4_DTYPE + return isinstance(w, QuantKey) and ( + w.dtype == FP4_DTYPE + or ( + w == kMxfp8Static + and self.kernel is not None + and self.kernel.supports_pre_processed_weights + ) + ) def create_weights( self, @@ -2524,7 +2531,13 @@ class ModelOptLinearMethod(LinearMethodBase): expose_input_quant_key(layer, self.kernel) def process_weights_after_loading(self, layer) -> None: + if self.spec.weight == kMxfp8Static and getattr(layer, "is_bmm", False): + self.kernel = init_mxfp8_linear_kernel(bmm_batch_size=layer.bmm_batch_size) if is_weights_pre_processed(): + if not self.supports_pre_processed_weights: + raise RuntimeError( + f"{type(self.kernel).__name__} cannot use pre-processed weights" + ) return self.fmt.pre_process(layer) self.wkey.process(layer, WEIGHT) @@ -2556,8 +2569,6 @@ class ModelOptLinearMethod(LinearMethodBase): persistent=False, ) layer._nvfp4_group_size_for_gather = self.ctx.group_size - if self.spec.weight == kMxfp8Static and getattr(layer, "is_bmm", False): - self.kernel = init_mxfp8_linear_kernel(bmm_batch_size=layer.bmm_batch_size) self.kernel.process_weights_after_loading(layer) def apply(self, layer, x, bias=None): diff --git a/vllm/model_executor/layers/quantization/online/fp8.py b/vllm/model_executor/layers/quantization/online/fp8.py index 2a8ef0b69f..ed018c81bf 100644 --- a/vllm/model_executor/layers/quantization/online/fp8.py +++ b/vllm/model_executor/layers/quantization/online/fp8.py @@ -737,11 +737,13 @@ class Fp8PtpcOnlineMoEMethod(_Fp8OnlineMoEBase): ) # Reject backends whose make_fp8_moe_quant_config branch silently # drops per_act_token_quant / per_out_ch_quant or collapses scales: - # MARLIN / CPU route through fp8_w8a16_moe_quant_config; FLASHINFER_* - # fold scales into a per-tensor alpha (oracle/fp8.py). + # MARLIN / CPU route through fp8_w8a16_moe_quant_config, CPU_W8A8 + # ignores both flags, and FLASHINFER_* fold scales into a per-tensor + # alpha (oracle/fp8.py). if self.fp8_backend in ( Fp8MoeBackend.MARLIN, Fp8MoeBackend.CPU, + Fp8MoeBackend.CPU_W8A8, Fp8MoeBackend.FLASHINFER_CUTLASS, Fp8MoeBackend.FLASHINFER_TRTLLM, ): diff --git a/vllm/model_executor/layers/quantization/quark/quark.py b/vllm/model_executor/layers/quantization/quark/quark.py index 88544fbe99..a3a3a36f00 100644 --- a/vllm/model_executor/layers/quantization/quark/quark.py +++ b/vllm/model_executor/layers/quantization/quark/quark.py @@ -32,6 +32,7 @@ from vllm.model_executor.layers.quantization.quark.schemes import ( QuarkOCP_MX, QuarkScheme, QuarkW4A8_MXFP4_FP8, + QuarkW4A16Int4, QuarkW8A8Fp8, QuarkW8A8Fp8PerBlock, QuarkW8A8Int8, @@ -39,6 +40,7 @@ from vllm.model_executor.layers.quantization.quark.schemes import ( from vllm.model_executor.layers.quantization.quark.utils import ( QuarkQTensorHint, deep_compare, + parse_w4a16_int4_weight_config, should_ignore_layer, ) from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import ( @@ -54,6 +56,10 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( kFp8Static128BlockSym, kFp8StaticChannelSym, kFp8StaticTensorSym, + kInt4Static, + kInt4Static32, + kInt4Static32Asym, + kInt4StaticAsym, kInt4W4A8StaticChannelSym, kInt8DynamicTensorAsym, kInt8DynamicTensorSym, @@ -172,8 +178,10 @@ class QuarkConfig(QuantizationConfig): quant_config_with_hf_to_vllm_mapper: dict[str, Any] = {} for k, v in self.quant_config.items(): - if isinstance(v, list): + if isinstance(v, list) and all(isinstance(item, str) for item in v): quant_config_with_hf_to_vllm_mapper[k] = hf_to_vllm_mapper.apply_list(v) + elif isinstance(v, list): + quant_config_with_hf_to_vllm_mapper[k] = v elif isinstance(v, dict): quant_config_with_hf_to_vllm_mapper[k] = hf_to_vllm_mapper.apply_dict(v) else: @@ -215,6 +223,9 @@ class QuarkConfig(QuantizationConfig): weight_quant_key=weight_quant_key, activation_quant_key=activation_quant_key, dynamic_mxfp4_quant=dynamic_mxfp4_quant, + weight_config=self._find_matched_config(prefix, type(layer)).get( + "weight" + ), ) layer.scheme = scheme return QuarkLinearMethod(self) @@ -227,6 +238,7 @@ class QuarkConfig(QuantizationConfig): method_cls=method_cls, weight_quant_key=weight_quant_key, activation_quant_key=activation_quant_key, + layer_name=prefix, ) return None @@ -572,6 +584,26 @@ class QuarkConfig(QuantizationConfig): ) return QuantKeyMatch(True, activation_quant_key, weight_quant_key) + def _is_w4a16_int4( + self, + weight_quant: dict[str, Any] | None, + input_quant: dict[str, Any] | None, + ) -> QuantKeyMatch: + if weight_quant is None or input_quant is not None: + return QuantKeyMatch(False, None, None) + + is_int4 = weight_quant.get("dtype") in ("int4", "uint4") + is_packed = self.pack_method in ("order", "reorder") + if not (is_int4 and is_packed): + return QuantKeyMatch(False, None, None) + + group_size, is_symmetric = parse_w4a16_int4_weight_config(weight_quant) + if is_symmetric: + weight_key = kInt4Static32 if group_size == 32 else kInt4Static + else: + weight_key = kInt4Static32Asym if group_size == 32 else kInt4StaticAsym + return QuantKeyMatch(True, None, weight_key) + def _is_w4a8_mxfp4_fp8( self, weight_quant: dict[str, Any] | None, @@ -890,6 +922,12 @@ class QuarkConfig(QuantizationConfig): match.activation_quant_key, QuarkW8A8Int8, ) + elif match := self._is_w4a16_int4(weight_config, input_config): + return ( + match.weight_quant_key, + match.activation_quant_key, + QuarkW4A16Int4, + ) elif match := self._is_w4a8_mxfp4_fp8(weight_config, input_config): return ( match.weight_quant_key, @@ -919,6 +957,7 @@ class QuarkConfig(QuantizationConfig): weight_quant_key: QuantKey | None, activation_quant_key: QuantKey | None, dynamic_mxfp4_quant: bool = False, + weight_config: dict[str, Any] | None = None, ) -> "QuarkScheme": """Construct a Quark scheme selected by get_scheme_cls.""" if scheme_cls not in ( @@ -928,9 +967,24 @@ class QuarkConfig(QuantizationConfig): QuarkOCP_MX, QuarkNVFP4, QuarkW4A8_MXFP4_FP8, + QuarkW4A16Int4, ): raise AssertionError(f"Unsupported Quark scheme class: {scheme_cls}") + scheme: QuarkScheme + if scheme_cls is QuarkW4A16Int4: + normalized_weight_config = self._unwrap_single_quant_config(weight_config) + assert normalized_weight_config is not None + assert weight_quant_key is not None + scheme = QuarkW4A16Int4( + weight_quant_key=weight_quant_key, + activation_quant_key=activation_quant_key, + pack_method=self.pack_method, + weight_config=normalized_weight_config, + ) + self._check_scheme_supported(scheme.get_min_capability()) + return scheme + kwargs: dict[str, Any] = {"activation_quant_key": activation_quant_key} # These classes handle different possible `weight_quant_key`. diff --git a/vllm/model_executor/layers/quantization/quark/quark_moe.py b/vllm/model_executor/layers/quantization/quark/quark_moe.py index d2472524a5..3a9f6d6366 100644 --- a/vllm/model_executor/layers/quantization/quark/quark_moe.py +++ b/vllm/model_executor/layers/quantization/quark/quark_moe.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import torch @@ -40,6 +40,12 @@ from vllm.model_executor.layers.fused_moe.oracle.int8 import ( make_int8_moe_quant_config, select_int8_moe_backend, ) +from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import ( + convert_to_wna16_moe_kernel_format, + make_wna16_moe_kernel, + make_wna16_moe_quant_config, + select_wna16_moe_backend, +) from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import ( TRITON_BACKENDS, Mxfp4MoeBackend, @@ -56,7 +62,10 @@ from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import ( make_nvfp4_moe_quant_config, select_nvfp4_moe_backend, ) -from vllm.model_executor.layers.quantization.quark.utils import QuarkQTensorHint +from vllm.model_executor.layers.quantization.quark.utils import ( + QuarkQTensorHint, + parse_w4a16_int4_weight_config, +) from vllm.model_executor.layers.quantization.utils.fp8_utils import ( validate_fp8_block_shape_moe, ) @@ -75,6 +84,10 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( kFp8Static128BlockSym, kFp8StaticChannelSym, kFp8StaticTensorSym, + kInt4Static, + kInt4Static32, + kInt4Static32Asym, + kInt4StaticAsym, kInt4W4A8StaticChannelSym, kInt8DynamicTensorAsym, kInt8DynamicTensorSym, @@ -102,6 +115,7 @@ logger = init_logger(__name__) __all__ = [ "QuarkMoEMethod", + "QuarkW4A16Int4MoEMethod", "QuarkW8A8Fp8MoEMethod", "QuarkOCP_MX_MoEMethod", "QuarkNvfp4MoEMethod", @@ -179,6 +193,12 @@ class QuarkMoEMethod(FusedMoEMethodBase): match.activation_quant_key, QuarkW8A8Fp8MoEMethod, ) + if match := quant_config._is_w4a16_int4(weight_config, input_config): + return ( + match.weight_quant_key, + match.activation_quant_key, + QuarkW4A16Int4MoEMethod, + ) if match := quant_config._is_w_ocp_mx_a_x( weight_config, input_config, allow_static_fp8=True ): @@ -202,6 +222,7 @@ class QuarkMoEMethod(FusedMoEMethodBase): method_cls: type["QuarkMoEMethod"], weight_quant_key: QuantKey | None, activation_quant_key: QuantKey | None, + layer_name: str | None = None, ) -> "QuarkMoEMethod": if weight_quant_key is None: raise RuntimeError("Unsupported FusedMoe scheme") @@ -226,10 +247,339 @@ class QuarkMoEMethod(FusedMoEMethodBase): return QuarkW8A8Int8MoEMethod( module.moe_config, weight_quant_key, activation_quant_key ) + elif method_cls is QuarkW4A16Int4MoEMethod: + assert layer_name is not None + layer_quant_config = quant_config._find_matched_config( + layer_name, type(module) + ) + weight_config = quant_config._unwrap_single_quant_config( + layer_quant_config.get("weight") + ) + if weight_config is None: + raise RuntimeError( + f"Missing weight quantization config for layer {layer_name}" + ) + return QuarkW4A16Int4MoEMethod( + weight_quant_key, + activation_quant_key, + weight_config, + quant_config.pack_method, + module.moe_config, + quant_config=quant_config, + ) else: raise RuntimeError("Unsupported FusedMoe scheme") +class QuarkW4A16Int4MoEMethod(QuarkMoEMethod): + """Quark packed INT4 weight-only MoE method.""" + + supported_activation_quant_keys: list[QuantKey | None] = [None] + supported_weight_quant_keys: list[QuantKey] = [ + kInt4Static, + kInt4Static32, + kInt4StaticAsym, + kInt4Static32Asym, + ] + + def __init__( + self, + weight_quant_key: QuantKey, + activation_quant_key: QuantKey | None, + weight_config: dict[str, Any], + pack_method: str, + moe: FusedMoEConfig, + quant_config: "QuarkConfig | None" = None, # type: ignore # noqa F821 + ): + super().__init__(moe, weight_quant_key, activation_quant_key) + + self.quant_config = quant_config + # group_size is read from the Quark config rather than weight_quant_key: + # the int4 QuantKeys only encode group_size 32 or -1, but Quark int4 + # checkpoints commonly use group_size 128, which the QuantKey cannot + # represent. Symmetry is unambiguous, so it comes from the key. + self.group_size = parse_w4a16_int4_weight_config(weight_config)[0] + self.is_symmetric = weight_quant_key.symmetric + self.bit8_pack_factor = 2 + self.pack_reorder = pack_method == "reorder" + + self.use_wna16_backend = True + self.wna16_backend, self.experts_cls = select_wna16_moe_backend( + config=self.moe, + weight_key=weight_quant_key, + quant_config=self.quant_config, + may_have_zp=not self.is_symmetric, + may_have_bias=False, + ) + + def create_weights( + self, + layer: RoutedExperts, + num_experts: int, + hidden_size: int, + intermediate_size_per_partition: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + group_size = self.group_size + group_size_div_factor = 1 + + while intermediate_size_per_partition % group_size or hidden_size % group_size: + group_size = group_size // 2 + group_size_div_factor *= 2 + assert group_size >= 32 + layer.group_size = group_size + layer.group_size_div_factor = group_size_div_factor + + strategy = FusedMoeWeightScaleSupported.GROUP.value + extra_weight_attrs.update({"quant_method": strategy, "is_transposed": False}) + + assert "weight_loader" in extra_weight_attrs + weight_loader = extra_weight_attrs["weight_loader"] + extra_weight_attrs["weight_loader"] = self.get_weight_loader( + layer, weight_loader + ) + + w13_weight = torch.nn.Parameter( + torch.empty( + num_experts, + 2 * intermediate_size_per_partition, + hidden_size // self.bit8_pack_factor, + dtype=torch.uint8, + ), + requires_grad=False, + ) + layer.register_parameter("w13_weight", w13_weight) + set_weight_attrs(w13_weight, extra_weight_attrs) + + w2_weight = torch.nn.Parameter( + torch.empty( + num_experts, + hidden_size, + intermediate_size_per_partition // self.bit8_pack_factor, + dtype=torch.uint8, + ), + requires_grad=False, + ) + layer.register_parameter("w2_weight", w2_weight) + set_weight_attrs(w2_weight, extra_weight_attrs) + + w13_weight_scale = torch.nn.Parameter( + torch.zeros( + num_experts, + 2 * intermediate_size_per_partition, + hidden_size // group_size, + dtype=params_dtype, + ), + requires_grad=False, + ) + layer.register_parameter("w13_weight_scale", w13_weight_scale) + set_weight_attrs(w13_weight_scale, extra_weight_attrs) + + w2_weight_scale = torch.nn.Parameter( + torch.zeros( + num_experts, + hidden_size, + intermediate_size_per_partition // group_size, + dtype=params_dtype, + ), + requires_grad=False, + ) + layer.register_parameter("w2_weight_scale", w2_weight_scale) + set_weight_attrs(w2_weight_scale, extra_weight_attrs) + + w13_weight_zero_point = torch.nn.Parameter( + torch.zeros( + num_experts, + 2 * intermediate_size_per_partition // self.bit8_pack_factor, + hidden_size // group_size, + dtype=torch.uint8, + ), + requires_grad=False, + ) + layer.register_parameter("w13_weight_zero_point", w13_weight_zero_point) + set_weight_attrs(w13_weight_zero_point, extra_weight_attrs) + + w2_weight_zero_point = torch.nn.Parameter( + torch.zeros( + num_experts, + hidden_size // self.bit8_pack_factor, + intermediate_size_per_partition // group_size, + dtype=torch.uint8, + ), + requires_grad=False, + ) + layer.register_parameter("w2_weight_zero_point", w2_weight_zero_point) + set_weight_attrs(w2_weight_zero_point, extra_weight_attrs) + + def process_weights_after_loading(self, layer: RoutedExperts) -> None: + w13_qzeros = layer.w13_weight_zero_point if not self.is_symmetric else None + w2_qzeros = layer.w2_weight_zero_point if not self.is_symmetric else None + + converted = convert_to_wna16_moe_kernel_format( + backend=self.wna16_backend, + layer=layer, + quant_config=self.quant_config, + input_dtype=None, + w13=layer.w13_weight, + w2=layer.w2_weight, + w13_scale=layer.w13_weight_scale, + w2_scale=layer.w2_weight_scale, + w13_qzeros=w13_qzeros, + w2_qzeros=w2_qzeros, + ) + + if converted is None: + # Backend rewrote the layer's params in place (e.g. Humming). + self._setup_kernel(layer) + return + + ( + w13_qweight, + w2_qweight, + w13_scales, + w2_scales, + _, # w13_qzeros + _, # w2_qzeros + _, # w13_input_global_scale + _, # w2_input_global_scale + _, # w13_bias + _, # w2_bias + ) = converted + + replace_parameter(layer, "w13_weight", w13_qweight) + replace_parameter(layer, "w2_weight", w2_qweight) + replace_parameter(layer, "w13_weight_scale", w13_scales) + replace_parameter(layer, "w2_weight_scale", w2_scales) + + self._setup_kernel(layer) + + def _setup_kernel(self, layer: RoutedExperts) -> None: + assert self.experts_cls is not None + self.moe_quant_config = self.get_fused_moe_quant_config(layer) + assert self.moe_quant_config is not None + self.moe_kernel = make_wna16_moe_kernel( + moe_quant_config=self.moe_quant_config, + moe_config=self.moe, + experts_cls=self.experts_cls, + backend=self.wna16_backend, + routing_tables=layer._expert_routing_tables(), + ) + + def get_fused_moe_quant_config( + self, layer: RoutedExperts + ) -> FusedMoEQuantConfig | None: + w1_zp = layer.w13_weight_zero_point if not self.is_symmetric else None + w2_zp = layer.w2_weight_zero_point if not self.is_symmetric else None + return make_wna16_moe_quant_config( + w1_scale=layer.w13_weight_scale, + w2_scale=layer.w2_weight_scale, + w1_zp=w1_zp, + w2_zp=w2_zp, + group_size=layer.group_size, + num_bits=4, + ) + + def apply( + self, + layer: RoutedExperts, + x: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + shared_experts: SharedExperts | None, + shared_experts_input: torch.Tensor | None, + ) -> torch.Tensor: + assert self.moe_kernel is not None + return self.moe_kernel.apply( + hidden_states=x, + w1=layer.w13_weight, + w2=layer.w2_weight, + topk_weights=topk_weights, + topk_ids=topk_ids, + activation=layer.activation, + global_num_experts=layer.global_num_experts, + apply_router_weight_on_input=layer.apply_router_weight_on_input, + expert_map=layer.expert_map, + shared_experts=shared_experts, + shared_experts_input=shared_experts_input, + ) + + def get_weight_loader(self, layer: RoutedExperts, weight_loader): + def convert_quark_tensor(tensor: torch.Tensor, tensor_type: str): + size0 = tensor.size(0) + tensor = tensor.view(torch.uint8) + + shifter = torch.tensor([0, 4], dtype=torch.uint8, device=tensor.device) + tensor = (tensor[:, :, None] >> shifter) & 0xF + + if self.pack_reorder: + reverse_awq_pack_order = [0, 4, 1, 5, 2, 6, 3, 7] + tensor = tensor.view(-1, 8)[:, reverse_awq_pack_order] + else: + tensor = tensor.view(-1, 8) + if self.is_symmetric: + tensor = tensor ^ 0x8 + tensor = tensor.view(size0, -1) + + tensor = tensor.T.contiguous() + if tensor_type == "weight": + tensor = tensor[:, 1::2] * 16 + tensor[:, ::2] + elif tensor_type == "zero_point": + tensor = tensor[1::2, :] * 16 + tensor[::2, :] + return tensor + + def quark_int4_weight_loader( + param: torch.nn.Parameter, + loaded_weight: torch.Tensor, + weight_name: str, + shard_id: str, + expert_id: int, + return_success: bool = False, + ): + loaded_weight = loaded_weight.to(param.data.device) + shard_size = layer.intermediate_size_per_partition + + if weight_name.endswith("_weight"): + loaded_weight = convert_quark_tensor(loaded_weight, "weight") + elif weight_name.endswith("_weight_zero_point"): + loaded_weight = convert_quark_tensor(loaded_weight, "zero_point") + elif weight_name.endswith("_weight_scale"): + loaded_weight = loaded_weight.T + + if layer.group_size_div_factor > 1 and ( + "weight_zero_point" in weight_name or "weight_scale" in weight_name + ): + loaded_weight = loaded_weight.repeat_interleave( + layer.group_size_div_factor, 1 + ) + + if "w13_weight_zero_point" in weight_name: + tensor = loaded_weight.view( + layer.moe_config.tp_size, -1, loaded_weight.size(1) + )[layer.moe_config.tp_rank] + if shard_id == "w1": + param.data[expert_id, : shard_size // 2] = tensor + else: + param.data[expert_id, shard_size // 2 :] = tensor + return True if return_success else None + elif "w2_weight_zero_point" in weight_name: + param.data[expert_id] = loaded_weight.view( + loaded_weight.size(0), layer.moe_config.tp_size, -1 + )[:, layer.moe_config.tp_rank] + return True if return_success else None + + return weight_loader( + param, + loaded_weight, + weight_name, + shard_id, + expert_id, + return_success=return_success, + ) + + return quark_int4_weight_loader + + class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod): supported_activation_quant_keys = [ kFp8StaticTensorSym, diff --git a/vllm/model_executor/layers/quantization/quark/schemes/__init__.py b/vllm/model_executor/layers/quantization/quark/schemes/__init__.py index 4db1ba3845..a5857a776f 100644 --- a/vllm/model_executor/layers/quantization/quark/schemes/__init__.py +++ b/vllm/model_executor/layers/quantization/quark/schemes/__init__.py @@ -5,6 +5,7 @@ from .quark_nvfp4 import QuarkNVFP4 from .quark_ocp_mx import QuarkOCP_MX from .quark_scheme import QuarkScheme from .quark_w4a8_mxfp4_fp8 import QuarkW4A8_MXFP4_FP8 +from .quark_w4a16_int4 import QuarkW4A16Int4 from .quark_w8a8_fp8 import QuarkW8A8Fp8, QuarkW8A8Fp8PerBlock from .quark_w8a8_int8 import QuarkW8A8Int8 @@ -14,6 +15,7 @@ __all__ = [ "QuarkW8A8Fp8PerBlock", "QuarkW8A8Int8", "QuarkOCP_MX", + "QuarkW4A16Int4", "QuarkW4A8_MXFP4_FP8", "QuarkNVFP4", ] diff --git a/vllm/model_executor/layers/quantization/quark/schemes/quark_w4a16_int4.py b/vllm/model_executor/layers/quantization/quark/schemes/quark_w4a16_int4.py new file mode 100644 index 0000000000..cded0f80a9 --- /dev/null +++ b/vllm/model_executor/layers/quantization/quark/schemes/quark_w4a16_int4.py @@ -0,0 +1,212 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import math +from collections.abc import Callable +from typing import Any + +import torch + +from vllm.model_executor.kernels.linear import ( + MPLinearKernel, + MPLinearLayerConfig, + choose_mp_linear_kernel, +) +from vllm.model_executor.layers.quantization.auto_awq import ( + _convert_awq_to_standard_format, +) +from vllm.model_executor.layers.quantization.quark.schemes.quark_scheme import ( + QuarkScheme, +) +from vllm.model_executor.layers.quantization.quark.utils import ( + canonicalize_quark_packed_int4, + parse_w4a16_int4_weight_config, +) +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + QuantKey, + kInt4Static, + kInt4Static32, + kInt4Static32Asym, + kInt4StaticAsym, +) +from vllm.model_executor.parameter import ( + GroupQuantScaleParameter, + PackedvLLMParameter, +) +from vllm.scalar_type import scalar_types + + +class QuarkW4A16Int4(QuarkScheme): + """Quark packed INT4 weight-only linear scheme via MPLinearKernel.""" + + supported_activation_quant_keys: list[QuantKey | None] = [None] + supported_weight_quant_keys: list[QuantKey] = [ + kInt4Static, + kInt4Static32, + kInt4StaticAsym, + kInt4Static32Asym, + ] + + def __init__( + self, + weight_quant_key: QuantKey, + activation_quant_key: QuantKey | None, + pack_method: str, + weight_config: dict[str, Any], + ): + super().__init__(weight_quant_key, activation_quant_key) + + # group_size is read from the Quark config rather than weight_quant_key: + # the int4 QuantKeys only encode group_size 32 or -1, but Quark int4 + # checkpoints commonly use group_size 128, which the QuantKey cannot + # represent. Symmetry is unambiguous, so it comes from the key. + self.group_size = parse_w4a16_int4_weight_config(weight_config)[0] + self.pack_factor = 8 + self.pack_reorder = pack_method == "reorder" + self.is_symmetric = weight_quant_key.symmetric + self.quant_type = ( + scalar_types.uint4b8 if self.is_symmetric else scalar_types.uint4 + ) + self.kernel: MPLinearKernel | None = None + + @classmethod + def get_min_capability(cls) -> int: + return 70 + + def create_weights( + self, + layer: torch.nn.Module, + output_partition_sizes: list[int], + input_size_per_partition: int, + params_dtype: torch.dtype, + weight_loader: Callable, + **kwargs, + ): + input_size = kwargs["input_size"] + output_size = kwargs["output_size"] + group_size = ( + self.group_size if self.group_size != -1 else input_size_per_partition + ) + if input_size_per_partition % group_size != 0: + raise ValueError( + "The input size is not aligned with the quantized weight shape. " + "This can be caused by too large tensor parallel size. " + f"input_size_per_partition={input_size_per_partition}, " + f"group_size={group_size}." + ) + + output_size_per_partition = sum(output_partition_sizes) + packed_output_size_per_partition = math.ceil( + output_size_per_partition / self.pack_factor + ) + layer.output_size_per_partition = output_size_per_partition + layer.packed_output_size_per_partition = packed_output_size_per_partition + + mp_linear_kernel_config = MPLinearLayerConfig( + full_weight_shape=(input_size, output_size), + partition_weight_shape=( + input_size_per_partition, + output_size_per_partition, + ), + weight_type=self.quant_type, + act_type=params_dtype, + group_size=group_size, + zero_points=not self.is_symmetric, + ) + kernel_type = choose_mp_linear_kernel(mp_linear_kernel_config) + + def weight_scale_loader( + param: torch.nn.Parameter, + loaded_weight: torch.Tensor, + *args, + **loader_kwargs, + ) -> None: + if loaded_weight.shape[1] < param.data.shape[1]: + padded_weight = loaded_weight.new_zeros(param.data.shape) + padded_weight[:, : loaded_weight.shape[1]] = loaded_weight + loaded_weight = padded_weight + weight_loader(param, loaded_weight, *args, **loader_kwargs) + + weight = PackedvLLMParameter( + data=torch.empty( + input_size_per_partition, + packed_output_size_per_partition, + dtype=torch.int32, + ), + input_dim=0, + output_dim=1, + packed_dim=1, + packed_factor=self.pack_factor, + weight_loader=weight_loader, + ) + num_groups = input_size_per_partition // group_size + weight_zero_point = PackedvLLMParameter( + data=torch.zeros( + num_groups, + packed_output_size_per_partition, + dtype=torch.int32, + ), + input_dim=0, + output_dim=1, + packed_dim=1, + packed_factor=self.pack_factor, + weight_loader=weight_loader, + ) + weight_scale = GroupQuantScaleParameter( + data=torch.empty( + num_groups, + packed_output_size_per_partition * self.pack_factor, + dtype=params_dtype, + ), + input_dim=0, + output_dim=1, + weight_loader=weight_scale_loader, + ) + + layer.register_parameter("weight", weight) + layer.register_parameter("weight_zero_point", weight_zero_point) + layer.register_parameter("weight_scale", weight_scale) + + self.kernel = kernel_type( + mp_linear_kernel_config, + w_q_param_name="weight", + w_s_param_name="weight_scale", + w_zp_param_name="weight_zero_point" if not self.is_symmetric else None, + ) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + layer.weight.data = canonicalize_quark_packed_int4( + layer.weight.data, + pack_reorder=self.pack_reorder, + is_symmetric=self.is_symmetric, + pack_factor=self.pack_factor, + ) + if not self.is_symmetric: + layer.weight_zero_point.data = canonicalize_quark_packed_int4( + layer.weight_zero_point.data, + pack_reorder=self.pack_reorder, + is_symmetric=self.is_symmetric, + pack_factor=self.pack_factor, + ) + output_size = layer.output_size_per_partition + packed_output_size = layer.packed_output_size_per_partition * self.pack_factor + if output_size < packed_output_size: + layer.weight_scale.data[:, output_size:].zero_() + + _convert_awq_to_standard_format( + layer, + "weight", + "weight_zero_point" if not self.is_symmetric else None, + self.quant_type.size_bits, + ) + assert self.kernel is not None + self.kernel.process_weights_after_loading(layer) + + def apply_weights( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: torch.Tensor | None = None, + ): + assert self.kernel is not None + return self.kernel.apply_weights(layer, x, bias) diff --git a/vllm/model_executor/layers/quantization/quark/utils.py b/vllm/model_executor/layers/quantization/quark/utils.py index 48c978263e..ac455e38b8 100644 --- a/vllm/model_executor/layers/quantization/quark/utils.py +++ b/vllm/model_executor/layers/quantization/quark/utils.py @@ -66,6 +66,63 @@ def should_ignore_layer( return all(shards_ignored) +def parse_w4a16_int4_weight_config( + weight_config: Mapping[str, Any], +) -> tuple[int, bool]: + """Parse required W4A16 INT4/UINT4 weight fields from Quark config.""" + if "group_size" not in weight_config: + raise ValueError( + "Quark W4A16 INT4/UINT4 configs must specify weight.group_size" + ) + if "symmetric" not in weight_config: + raise ValueError("Quark W4A16 INT4/UINT4 configs must specify weight.symmetric") + + group_size = weight_config["group_size"] + is_symmetric = weight_config["symmetric"] + if not isinstance(group_size, int) or group_size <= 0: + raise ValueError( + f"Quark W4A16 weight.group_size must be a positive int, got {group_size!r}" + ) + if not isinstance(is_symmetric, bool): + raise ValueError( + f"Quark W4A16 weight.symmetric must be a bool, got {is_symmetric!r}" + ) + return group_size, is_symmetric + + +def canonicalize_quark_packed_int4( + packed_weight: torch.Tensor, + *, + pack_reorder: bool, + is_symmetric: bool, + pack_factor: int = 8, +) -> torch.Tensor: + """Convert Quark export nibble layout to AWQ checkpoint layout.""" + from vllm.model_executor.layers.quantization.auto_awq import ( + _REVERSE_AWQ_PACK_ORDER, + ) + + if pack_reorder: + source_order = torch.tensor( + _REVERSE_AWQ_PACK_ORDER, device=packed_weight.device, dtype=torch.int32 + ) + else: + source_order = torch.arange( + pack_factor, device=packed_weight.device, dtype=torch.int32 + ) + target_order = torch.tensor( + _REVERSE_AWQ_PACK_ORDER, device=packed_weight.device, dtype=torch.int32 + ) + source_shifts = source_order * 4 + target_shifts = target_order * 4 + + values = (packed_weight.to(torch.int32)[..., None] >> source_shifts) & 0xF + if is_symmetric: + values = values ^ 0x8 + packed = (values.to(torch.int64) << target_shifts.to(torch.int64)).sum(dim=-1) + return packed.to(torch.int32) + + # utility for tensor dims > 2 cases def quark_quantize_weight_to_mxfp4(w: torch.Tensor): assert w.dtype == torch.bfloat16, ( diff --git a/vllm/model_executor/layers/quantization/utils/flashinfer_utils.py b/vllm/model_executor/layers/quantization/utils/flashinfer_utils.py index 044285ac85..1bfd7e61f9 100644 --- a/vllm/model_executor/layers/quantization/utils/flashinfer_utils.py +++ b/vllm/model_executor/layers/quantization/utils/flashinfer_utils.py @@ -112,8 +112,9 @@ def convert_moe_weights_to_flashinfer_trtllm_block_layout( ) -> tuple[torch.Tensor, torch.Tensor]: """Convert expert weights to FlashInfer's block layout. - This reorders W13 and W2 into the expected epilogue-tiled block layout and - returns the shuffled weight tensors. + This reorders W13 and W2 in place into the expected epilogue-tiled block + layout and returns views of the shuffled weight tensors. Using one expert + as scratch space avoids allocating another full copy of both weights. """ if w13_weight.dtype != torch.bfloat16 or w2_weight.dtype != torch.bfloat16: raise ValueError( @@ -147,51 +148,50 @@ def convert_moe_weights_to_flashinfer_trtllm_block_layout( out=out, ) - w13_rows, w13_cols = w13_weight[0].view(torch.uint8).shape - w2_rows, w2_cols = w2_weight[0].view(torch.uint8).shape - w13_weights_shuffled_tensor = torch.empty( - (num_experts, w13_cols // block_k, w13_rows, block_k), - dtype=torch.uint8, - device=w13_weight.device, - ) - w2_weights_shuffled_tensor = torch.empty( - (num_experts, w2_cols // block_k, w2_rows, block_k), - dtype=torch.uint8, - device=w2_weight.device, - ) - - for i in range(num_experts): - w13_expert_uint8 = w13_weight[i].view(torch.uint8) - - permute_indices = _maybe_get_cached_w3_w1_permute_indices( - cache_permute_indices, - w13_expert_uint8, - epilogue_tile_m, - is_gated_act_gemm=is_gated_act_gemm, - ) - if is_gated_act_gemm: - rows = w13_expert_uint8.shape[0] - permute_indices = (permute_indices + rows // 2) % rows - _copy_permuted_expert_to_block_layout( - w13_weights_shuffled_tensor[i], - w13_expert_uint8, - permute_indices, + def _convert_weight_in_place( + weight: torch.Tensor, + is_w13: bool, + ) -> torch.Tensor: + rows, cols = weight[0].view(torch.uint8).shape + block_layout_shape = (num_experts, cols // block_k, rows, block_k) + expert_scratch = torch.empty( + block_layout_shape[1:], + dtype=torch.uint8, + device=weight.device, ) - permute_indices = get_w2_permute_indices_with_cache( - cache_permute_indices, - w2_weight[i].view(torch.uint8), - epilogue_tile_m, - ) - _copy_permuted_expert_to_block_layout( - w2_weights_shuffled_tensor[i], - w2_weight[i].view(torch.uint8), - permute_indices, - ) + for i in range(num_experts): + expert_uint8 = weight[i].view(torch.uint8) + if is_w13: + permute_indices = _maybe_get_cached_w3_w1_permute_indices( + cache_permute_indices, + expert_uint8, + epilogue_tile_m, + is_gated_act_gemm=is_gated_act_gemm, + ) + if is_gated_act_gemm: + permute_indices = ( + permute_indices + expert_uint8.shape[0] // 2 + ) % expert_uint8.shape[0] + else: + permute_indices = get_w2_permute_indices_with_cache( + cache_permute_indices, + expert_uint8, + epilogue_tile_m, + ) + + _copy_permuted_expert_to_block_layout( + expert_scratch, + expert_uint8, + permute_indices, + ) + expert_uint8.view(-1).copy_(expert_scratch.view(-1)) + + return weight.view(torch.uint8).view(block_layout_shape).view(torch.bfloat16) return ( - w13_weights_shuffled_tensor.view(torch.bfloat16), - w2_weights_shuffled_tensor.view(torch.bfloat16), + _convert_weight_in_place(w13_weight, is_w13=True), + _convert_weight_in_place(w2_weight, is_w13=False), ) diff --git a/vllm/model_executor/layers/quantization/utils/mxfp4_utils.py b/vllm/model_executor/layers/quantization/utils/mxfp4_utils.py index a1d0501775..8cc626f386 100644 --- a/vllm/model_executor/layers/quantization/utils/mxfp4_utils.py +++ b/vllm/model_executor/layers/quantization/utils/mxfp4_utils.py @@ -36,6 +36,15 @@ def weight_mx_scale(precision_config): return precision_config.weight_scale +def mx_scale_kwargs(scale): + """PrecisionConfig weight-scale kwargs: 3.8 uses b_mx_scale/b_microblock_size, + 3.5.1/3.6 use weight_scale. + """ + if get_triton_kernels_version() == "3.8": + return {"b_mx_scale": scale, "b_microblock_size": 32} + return {"weight_scale": scale} + + def _swizzle_mxfp4(quant_tensor, scale, num_warps=8): """Weight swizzle for mxfp4 moe, used for OAI mxfp4 kernel.""" assert has_triton_kernels() diff --git a/vllm/model_executor/layers/sparse_attn_indexer.py b/vllm/model_executor/layers/sparse_attn_indexer.py index 7dde4db00f..ae45b77dce 100644 --- a/vllm/model_executor/layers/sparse_attn_indexer.py +++ b/vllm/model_executor/layers/sparse_attn_indexer.py @@ -642,6 +642,19 @@ def sparse_attn_indexer( assert decode_metadata is not None kv_cache = kv_cache_as_quant_view(kv_cache, head_dim, use_fp4_cache) decode_lens = decode_metadata.decode_lens + # requires_padding can be computed False for ragged warmup/mixed + # batches (seen on SM120 TP=2: 8 tokens over 6 seqs -> uniform + # reshape below would crash). The token-count/batch-size divisibility + # check is a necessary-but-not-sufficient proxy for "uniform batch" + # (e.g. decode_lens=[1, 2, 3] divides evenly by 3 but isn't uniform); + # it catches the specific case seen in practice without requiring a + # device sync to inspect decode_lens directly, which would break + # CUDA-graph-capture safety on this path. Shared by both the pack + # branch below and the matching unpack branch further down — they + # must agree on which path was taken. + needs_padded_path = decode_metadata.requires_padding or ( + num_decode_tokens % decode_lens.shape[0] != 0 + ) if num_decode_tokens == 0: padded_q_quant_decode_tokens = q_quant[:1].reshape(1, 1, *q_quant.shape[1:]) padded_q_scale = ( @@ -649,7 +662,7 @@ def sparse_attn_indexer( if q_scale is not None else None ) - elif decode_metadata.requires_padding: + elif needs_padded_path: # pad in edge case where we have short chunked prefill length < # decode_threshold since we unstrictly split # prefill and decode by decode_threshold @@ -773,7 +786,7 @@ def sparse_attn_indexer( cp_kv_cache_interleave_size, ) - if decode_metadata.requires_padding: + if needs_padded_path: # if padded, we need to unpack # the topk indices removing padded tokens topk_indices = unpack_seq_triton( diff --git a/vllm/model_executor/model_loader/reload/meta.py b/vllm/model_executor/model_loader/reload/meta.py index 4d08d49165..9874ebd67b 100644 --- a/vllm/model_executor/model_loader/reload/meta.py +++ b/vllm/model_executor/model_loader/reload/meta.py @@ -9,7 +9,7 @@ from torch.utils._python_dispatch import TorchDispatchMode from .sanitize import restore_layer_refs, sanitize_layer_refs from .types import LayerReloadingInfo, LayerTensors -from .utils import get_layer_params_buffers, get_layer_tensors +from .utils import get_layer_params_buffers, get_layer_tensors, get_tensor_load_numel __all__ = [ "to_meta_tensor", @@ -206,5 +206,5 @@ def get_numel_loaded( numel = counter.copied_numel param = args.arguments.get("param", None) if isinstance(param, torch.Tensor): - numel = min(numel, param.numel()) + numel = min(numel, get_tensor_load_numel(param)) return numel, return_value diff --git a/vllm/model_executor/model_loader/reload/utils.py b/vllm/model_executor/model_loader/reload/utils.py index b7dc361503..203aa82af3 100644 --- a/vllm/model_executor/model_loader/reload/utils.py +++ b/vllm/model_executor/model_loader/reload/utils.py @@ -11,6 +11,7 @@ __all__ = [ "get_layer_tensors", "get_layer_params_buffers", "get_layer_size", + "get_tensor_load_numel", "has_device_tensors", "get_info_size", ] @@ -30,6 +31,11 @@ def get_layer_params_buffers(layer: torch.nn.Module) -> LayerTensors: ) +def get_tensor_load_numel(tensor: torch.Tensor) -> int: + """Count checkpoint elements, excluding padding declared by the weight creator.""" + return getattr(tensor, "weight_loader_numel", tensor.numel()) + + def get_layer_size(layer: torch.nn.Module) -> int: """Calculate total number of elements across loadable tensors in a layer. @@ -39,7 +45,7 @@ def get_layer_size(layer: torch.nn.Module) -> int: from .meta import SKIP_LOAD_TENSORS return sum( - tensor.numel() + get_tensor_load_numel(tensor) for name, tensor in get_layer_tensors(layer).items() if name not in SKIP_LOAD_TENSORS ) diff --git a/vllm/model_executor/models/AXK1.py b/vllm/model_executor/models/AXK1.py index 97f3950428..9981f5904f 100644 --- a/vllm/model_executor/models/AXK1.py +++ b/vllm/model_executor/models/AXK1.py @@ -324,9 +324,9 @@ class AXK1Attention(nn.Module): assert config.rope_parameters is not None if config.rope_parameters["rope_type"] != "default": config.rope_parameters["rope_type"] = ( - "deepseek_yarn" - if config.rope_parameters.get("apply_yarn_scaling", True) - else "deepseek_llama_scaling" + "deepseek_llama_scaling" + if config.rope_parameters.get("attention_factor") == 1.0 + else "deepseek_yarn" ) self.rotary_emb = get_rope( @@ -496,9 +496,9 @@ class AXK1MLAAttention(nn.Module): assert config.rope_parameters is not None if config.rope_parameters["rope_type"] != "default": config.rope_parameters["rope_type"] = ( - "deepseek_yarn" - if config.rope_parameters.get("apply_yarn_scaling", True) - else "deepseek_llama_scaling" + "deepseek_llama_scaling" + if config.rope_parameters.get("attention_factor") == 1.0 + else "deepseek_yarn" ) self.rotary_emb = get_rope( diff --git a/vllm/model_executor/models/aimv2.py b/vllm/model_executor/models/aimv2.py index 3461b57137..4e31b99944 100644 --- a/vllm/model_executor/models/aimv2.py +++ b/vllm/model_executor/models/aimv2.py @@ -26,7 +26,10 @@ from vllm.transformers_utils.configs.ovis import AIMv2Config class AIMv2SwiGLUFFN(nn.Module): def __init__( - self, config: AIMv2Config, quant_config: QuantizationConfig, prefix: str + self, + config: AIMv2Config, + quant_config: QuantizationConfig | None, + prefix: str, ): super().__init__() hidden_features = config.intermediate_size @@ -91,7 +94,10 @@ class AIMv2ViTPreprocessor(nn.Module): class AIMv2Attention(nn.Module): def __init__( - self, config: AIMv2Config, quant_config: QuantizationConfig, prefix: str + self, + config: AIMv2Config, + quant_config: QuantizationConfig | None, + prefix: str, ): super().__init__() self.config = config @@ -144,7 +150,10 @@ class AIMv2Attention(nn.Module): class AIMv2Block(nn.Module): def __init__( - self, config: AIMv2Config, quant_config: QuantizationConfig, prefix: str + self, + config: AIMv2Config, + quant_config: QuantizationConfig | None, + prefix: str, ): super().__init__() self.attn = AIMv2Attention( @@ -166,7 +175,7 @@ class AIMv2Transformer(nn.Module): def __init__( self, config: AIMv2Config, - quant_config: QuantizationConfig, + quant_config: QuantizationConfig | None, *, require_post_norm: bool | None = None, prefix: str = "", @@ -206,7 +215,7 @@ class AIMv2Model(torch.nn.Module): def __init__( self, config: AIMv2Config, - quant_config: QuantizationConfig, + quant_config: QuantizationConfig | None, *, require_post_norm: bool | None = None, prefix: str = "", diff --git a/vllm/model_executor/models/aria.py b/vllm/model_executor/models/aria.py index 4240406f7b..da91c9d9bf 100644 --- a/vllm/model_executor/models/aria.py +++ b/vllm/model_executor/models/aria.py @@ -3,6 +3,7 @@ from collections.abc import Iterable, Mapping, Sequence from typing import Annotated, Literal +import regex as re import torch import torch.nn as nn from transformers import AriaConfig, AriaTextConfig, BatchFeature @@ -10,7 +11,7 @@ from transformers.models.aria.modeling_aria import AriaCrossAttention from transformers.models.aria.processing_aria import AriaProcessor from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.activation import get_act_fn from vllm.model_executor.layers.fused_moe import FusedMoEFactory @@ -222,6 +223,7 @@ class AriaTextMoELayer(nn.Module): intermediate_size=config.intermediate_size, quant_config=quant_config, prefix=f"{prefix}.experts", + is_fused_checkpoint_transposed=True, ) def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: @@ -267,14 +269,17 @@ class AriaTextModel(LlamaModel, SupportsQuant): "gate_up_proj": ["gate_proj", "up_proj"], } - # Aria packs all experts into single (transposed) fc1/fc2 tensors, which is - # exactly the pre-fused checkpoint layout FusedMoE self-loads once fc1/fc2 - # are renamed to the fused gate_up_proj/down_proj names. + # The fused expert loader expects names without the .weight suffix. hf_to_vllm_mapper = LlamaModel.hf_to_vllm_mapper | WeightsMapper( + orig_to_new_regex={ + re.compile(r"experts\.fc1\.weight$"): "experts.gate_up_proj", + re.compile(r"experts\.fc2\.weight$"): "experts.down_proj", + }, + # Quantization configs also use the expert module names. orig_to_new_substr={ "experts.fc1": "experts.gate_up_proj", "experts.fc2": "experts.down_proj", - } + }, ) def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): @@ -315,22 +320,18 @@ class AriaDummyInputsBuilder(BaseDummyInputsBuilder[AriaProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: vision_config = self.info.get_vision_config() max_image_size = vision_config.image_size - num_images = mm_counts.get("image", 0) - - image_overrides = mm_options.get("image") - assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions) return { "image": self._get_dummy_images( width=max_image_size, height=max_image_size, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/audioflamingo3.py b/vllm/model_executor/models/audioflamingo3.py index 36f2569277..4be996d278 100644 --- a/vllm/model_executor/models/audioflamingo3.py +++ b/vllm/model_executor/models/audioflamingo3.py @@ -30,7 +30,7 @@ from transformers.models.audioflamingo3 import ( from transformers.models.qwen2_audio import Qwen2AudioEncoder from vllm.config import VllmConfig -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import ModalityData, MultiModalDataDict from vllm.model_executor.layers.activation import get_act_fn from vllm.model_executor.models.module_mapping import MultiModelKeys @@ -216,21 +216,18 @@ class AudioFlamingo3DummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: hf_processor = self.info.get_hf_processor() feature_extractor = self.info.get_feature_extractor() sampling_rate = feature_extractor.sampling_rate audio_len = int(hf_processor.max_audio_len * sampling_rate) - num_audios = mm_counts.get("audio", 0) - audio_overrides = mm_options.get("audio") - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) return { "audio": self._get_dummy_audios( length=audio_len, - num_audios=num_audios, - overrides=audio_overrides, + num_audios=mm_counts.get("audio", 0), + overrides=mm_options.get("audio"), ) } diff --git a/vllm/model_executor/models/bagel.py b/vllm/model_executor/models/bagel.py index 9e0bd4ccbf..e9ebb0e5ed 100644 --- a/vllm/model_executor/models/bagel.py +++ b/vllm/model_executor/models/bagel.py @@ -14,7 +14,7 @@ import torch import torch.nn as nn from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.logger import init_logger from vllm.model_executor.layers.activation import get_act_fn @@ -248,23 +248,20 @@ class BagelDummyInputsBuilder(BaseDummyInputsBuilder[BagelProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) hf_config = self.info.get_hf_config() vit_config = hf_config.vit_config # Use the configured image size image_size = vit_config.image_size - image_overrides = mm_options.get("image") - assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions) return { "image": self._get_dummy_images( width=image_size, height=image_size, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), } diff --git a/vllm/model_executor/models/bailing_moe_v3_vl.py b/vllm/model_executor/models/bailing_moe_v3_vl.py index 7cd0568530..a236d0722b 100644 --- a/vllm/model_executor/models/bailing_moe_v3_vl.py +++ b/vllm/model_executor/models/bailing_moe_v3_vl.py @@ -13,7 +13,7 @@ import torch.nn as nn from transformers import AutoProcessor, BatchFeature from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -103,7 +103,7 @@ class BailingMoeV3VLDummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: target_width, target_height = self.info.get_image_size_with_most_features() return { diff --git a/vllm/model_executor/models/bee.py b/vllm/model_executor/models/bee.py index 4a57413b6e..cc6c0f09ab 100644 --- a/vllm/model_executor/models/bee.py +++ b/vllm/model_executor/models/bee.py @@ -8,7 +8,7 @@ import torch.nn as nn from transformers.activations import GELUActivation from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.multimodal import MULTIMODAL_REGISTRY @@ -90,21 +90,16 @@ class BeeDummyInputsBuilder(LlavaDummyInputsBuilder[BeeProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - target_width, target_height = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") - assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions) - return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), } diff --git a/vllm/model_executor/models/blip2.py b/vllm/model_executor/models/blip2.py index 48da4f63e8..cd019e41ec 100644 --- a/vllm/model_executor/models/blip2.py +++ b/vllm/model_executor/models/blip2.py @@ -15,7 +15,7 @@ from transformers import ( ) from vllm.config import CacheConfig, VllmConfig -from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.activation import get_act_fn from vllm.model_executor.layers.quantization import QuantizationConfig @@ -444,23 +444,19 @@ class Blip2DummyInputsBuilder(BaseDummyInputsBuilder[Blip2ProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: hf_config = self.info.get_hf_config() vision_config = hf_config.vision_config max_image_size = vision_config.image_size - num_images = mm_counts.get("image", 0) - - image_overrides = mm_options.get("image") - assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions) return { "image": self._get_dummy_images( width=max_image_size, height=max_image_size, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/clip.py b/vllm/model_executor/models/clip.py index 6fb311fb14..7c5ba496ea 100644 --- a/vllm/model_executor/models/clip.py +++ b/vllm/model_executor/models/clip.py @@ -15,7 +15,7 @@ from transformers import ( ) from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import divide, get_tensor_model_parallel_world_size from vllm.inputs import MultiModalDataDict, MultiModalInput from vllm.model_executor.layers.activation import get_act_fn @@ -205,20 +205,16 @@ class CLIPDummyInputsBuilder(BaseDummyInputsBuilder[CLIPProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - target_width, target_height = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") - return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/cohere2_moe.py b/vllm/model_executor/models/cohere2_moe.py index d657b78a5d..98e649dec6 100644 --- a/vllm/model_executor/models/cohere2_moe.py +++ b/vllm/model_executor/models/cohere2_moe.py @@ -35,7 +35,7 @@ from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors from .commandr import LayerNorm -from .interfaces import SupportsPP, SupportsQuant +from .interfaces import EagleModelMixin, SupportsEagle3, SupportsPP, SupportsQuant from .utils import ( AutoWeightsLoader, WeightsMapper, @@ -385,7 +385,7 @@ class Cohere2MoeDecoderLayer(nn.Module): @support_torch_compile -class Cohere2MoeModel(nn.Module): +class Cohere2MoeModel(nn.Module, EagleModelMixin): """Transformer decoder for Cohere2Moe.""" def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): @@ -439,7 +439,7 @@ class Cohere2MoeModel(nn.Module): positions: torch.Tensor, intermediate_tensors: IntermediateTensors | None = None, inputs_embeds: torch.Tensor | None = None, - ) -> torch.Tensor | IntermediateTensors: + ) -> torch.Tensor | IntermediateTensors | tuple[torch.Tensor, list[torch.Tensor]]: if get_pp_group().is_first_rank: if inputs_embeds is not None: hidden_states = inputs_embeds @@ -450,17 +450,28 @@ class Cohere2MoeModel(nn.Module): assert intermediate_tensors is not None hidden_states = intermediate_tensors["hidden_states"] residual = intermediate_tensors["residual"] - for layer in islice(self.layers, self.start_layer, self.end_layer): + + aux_hidden_states = [] + if self.start_layer in self.aux_hidden_state_layers: + aux_hidden_states.append(hidden_states) + for layer_idx, layer in enumerate( + islice(self.layers, self.start_layer, self.end_layer), + start=self.start_layer, + ): hidden_states, residual = layer(positions, hidden_states, residual) + if layer_idx + 1 in self.aux_hidden_state_layers: + aux_hidden_states.append(hidden_states) if not get_pp_group().is_last_rank: return IntermediateTensors( {"hidden_states": hidden_states, "residual": residual} ) hidden_states, _ = self.norm(hidden_states, residual) + if aux_hidden_states: + return hidden_states, aux_hidden_states return hidden_states -class Cohere2MoeForCausalLM(nn.Module, SupportsPP, SupportsQuant): +class Cohere2MoeForCausalLM(nn.Module, SupportsPP, SupportsQuant, SupportsEagle3): is_text_generation_model = True hf_to_vllm_mapper = WeightsMapper( @@ -520,7 +531,7 @@ class Cohere2MoeForCausalLM(nn.Module, SupportsPP, SupportsQuant): positions: torch.Tensor, intermediate_tensors: IntermediateTensors | None = None, inputs_embeds: torch.Tensor | None = None, - ) -> torch.Tensor | IntermediateTensors: + ) -> torch.Tensor | IntermediateTensors | tuple[torch.Tensor, list[torch.Tensor]]: return self.model(input_ids, positions, intermediate_tensors, inputs_embeds) def compute_logits( diff --git a/vllm/model_executor/models/cohere2_vision.py b/vllm/model_executor/models/cohere2_vision.py index a4748932d6..4fdaff4444 100644 --- a/vllm/model_executor/models/cohere2_vision.py +++ b/vllm/model_executor/models/cohere2_vision.py @@ -18,7 +18,7 @@ from transformers.models.cohere2_vision.processing_cohere2_vision import ( ) from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.activation import MulAndSilu from vllm.model_executor.layers.linear import ( @@ -202,19 +202,16 @@ class Cohere2VisionDummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) image_size = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") - return { "image": self._get_dummy_images( width=image_size.width, height=image_size.height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/cohere_asr.py b/vllm/model_executor/models/cohere_asr.py index 844e8c5e0e..c8a196ffdc 100644 --- a/vllm/model_executor/models/cohere_asr.py +++ b/vllm/model_executor/models/cohere_asr.py @@ -12,7 +12,7 @@ from transformers import PretrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.config.speech_to_text import SpeechToTextParams from vllm.distributed import get_tensor_model_parallel_world_size from vllm.inputs import MultiModalDataDict, PromptType, TokensPrompt @@ -1206,10 +1206,18 @@ class RelPositionMultiHeadAttention(CohereASRMultiHeadAttention): matrix_bd = torch.matmul(q_with_bias_v, p.transpose(-2, -1)) matrix_bd = self.rel_shift(matrix_bd) - # drops extra elements in the matrix_bd to match the matrix_ac's size - matrix_ac = torch.matmul(q_with_bias_u, k.transpose(-2, -1)) - matrix_bd = matrix_bd[:, :, :, : matrix_ac.size(-1)] - scores = (matrix_ac + matrix_bd) / self.s_d_k # (batch, head, time1, time2) + # drops extra elements in matrix_bd to match the key sequence length + matrix_bd = matrix_bd[:, :, :, : k.size(-2)] + batch, heads, query_len, key_len = matrix_bd.shape + scale = 1.0 / self.s_d_k + scores = torch.baddbmm( + matrix_bd.flatten(0, 1), + q_with_bias_u.flatten(0, 1), + k.transpose(-2, -1).flatten(0, 1), + beta=scale, + alpha=scale, + ).view(batch, heads, query_len, key_len) + return self.forward_attention(v, scores, mask) @@ -1936,7 +1944,7 @@ class CohereASRDummyInputsBuilder(BaseDummyInputsBuilder[CohereASRProcessingInfo self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, mm_processor_kwargs=None, ) -> MultiModalDataDict: feature_extractor = self.info.get_feature_extractor() diff --git a/vllm/model_executor/models/cohere_compass.py b/vllm/model_executor/models/cohere_compass.py index 1d0dd8c35b..9e8adabb80 100644 --- a/vllm/model_executor/models/cohere_compass.py +++ b/vllm/model_executor/models/cohere_compass.py @@ -51,7 +51,7 @@ from transformers.models.cohere_compass.image_processing_cohere_compass import ( from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import ( get_pp_group, get_tensor_model_parallel_world_size, @@ -1238,11 +1238,8 @@ class CohereCompassDummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - image_overrides = mm_options.get("image") - target_image_width, target_image_height = ( self.info.get_image_size_with_most_features() ) @@ -1251,8 +1248,8 @@ class CohereCompassDummyInputsBuilder( "image": self._get_dummy_images( width=target_image_width, height=target_image_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), } diff --git a/vllm/model_executor/models/colmodernvbert.py b/vllm/model_executor/models/colmodernvbert.py index 0c93e7ef52..a14476f774 100644 --- a/vllm/model_executor/models/colmodernvbert.py +++ b/vllm/model_executor/models/colmodernvbert.py @@ -15,7 +15,7 @@ from torch import nn from transformers import BatchFeature from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.pooler.tokwise import pooler_for_token_embed from vllm.multimodal import MULTIMODAL_REGISTRY @@ -135,18 +135,16 @@ class ColModernVBertDummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) target_width, target_height = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/deepseek_ocr.py b/vllm/model_executor/models/deepseek_ocr.py index 3d1921dc2b..62a46fc498 100644 --- a/vllm/model_executor/models/deepseek_ocr.py +++ b/vllm/model_executor/models/deepseek_ocr.py @@ -11,7 +11,7 @@ import torch.nn as nn from transformers import BatchFeature, CLIPVisionConfig from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.models.interfaces import ( MultiModalEmbeddings, @@ -268,17 +268,15 @@ class DeepseekOCRDummyInputsBuilder(BaseDummyInputsBuilder[DeepseekOCRProcessing self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - max_image_size = self.info.get_image_size_with_most_features() return { "image": self._get_dummy_images( width=max_image_size.width, height=max_image_size.height, - num_images=num_images, + num_images=mm_counts.get("image", 0), ) } diff --git a/vllm/model_executor/models/deepseek_ocr2.py b/vllm/model_executor/models/deepseek_ocr2.py index a64b71b64e..938598a68b 100644 --- a/vllm/model_executor/models/deepseek_ocr2.py +++ b/vllm/model_executor/models/deepseek_ocr2.py @@ -11,7 +11,7 @@ import torch.nn as nn from transformers import BatchFeature from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.models.interfaces import ( MultiModalEmbeddings, @@ -140,17 +140,15 @@ class DeepseekOCR2DummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - max_image_size = self.info.get_image_size_with_most_features() return { "image": self._get_dummy_images( width=max_image_size.width, height=max_image_size.height, - num_images=num_images, + num_images=mm_counts.get("image", 0), ) } diff --git a/vllm/model_executor/models/deepseek_v2.py b/vllm/model_executor/models/deepseek_v2.py index d7d4858044..ed702400be 100644 --- a/vllm/model_executor/models/deepseek_v2.py +++ b/vllm/model_executor/models/deepseek_v2.py @@ -540,9 +540,9 @@ class DeepseekV2Attention(nn.Module): ) if config.rope_parameters["rope_type"] != "default": config.rope_parameters["rope_type"] = ( - "deepseek_yarn" - if config.rope_parameters.get("apply_yarn_scaling", True) - else "deepseek_llama_scaling" + "deepseek_llama_scaling" + if config.rope_parameters.get("attention_factor") == 1.0 + else "deepseek_yarn" ) self.rotary_emb = get_rope( @@ -1107,9 +1107,9 @@ class DeepseekV2MLAAttention(nn.Module): if config.rope_parameters["rope_type"] != "default": config.rope_parameters["rope_type"] = ( - "deepseek_yarn" - if config.rope_parameters.get("apply_yarn_scaling", True) - else "deepseek_llama_scaling" + "deepseek_llama_scaling" + if config.rope_parameters.get("attention_factor") == 1.0 + else "deepseek_yarn" ) self.rotary_emb = get_rope( diff --git a/vllm/model_executor/models/deepseek_vl2.py b/vllm/model_executor/models/deepseek_vl2.py index 6f798a87e6..c720f62335 100644 --- a/vllm/model_executor/models/deepseek_vl2.py +++ b/vllm/model_executor/models/deepseek_vl2.py @@ -15,7 +15,7 @@ from einops import rearrange, repeat from transformers import BatchFeature from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import get_tensor_model_parallel_world_size from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.quantization import QuantizationConfig @@ -212,20 +212,16 @@ class DeepseekVL2DummyInputsBuilder(BaseDummyInputsBuilder[DeepseekVL2Processing self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - max_image_size = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") - return { "image": self._get_dummy_images( width=max_image_size.width, height=max_image_size.height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/diffusion_gemma.py b/vllm/model_executor/models/diffusion_gemma.py index bd1d14b6f6..0c1cc8c55e 100644 --- a/vllm/model_executor/models/diffusion_gemma.py +++ b/vllm/model_executor/models/diffusion_gemma.py @@ -46,7 +46,11 @@ from vllm.model_executor.models.gemma4_mm import ( ) from vllm.model_executor.models.module_mapping import MultiModelKeys from vllm.model_executor.models.transformers.utils import recursive_replace_linear -from vllm.model_executor.models.utils import WeightsMapper, maybe_prefix +from vllm.model_executor.models.utils import ( + AutoWeightsLoader, + WeightsMapper, + maybe_prefix, +) from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.platforms import current_platform from vllm.utils.torch_utils import async_tensor_h2d @@ -160,14 +164,12 @@ class DiffusionGemmaForConditionalGeneration( hf_to_vllm_mapper = WeightsMapper( orig_to_new_prefix={ + "model.decoder.self_conditioning.": "self_conditioning.", "model.decoder.": "model.", "model.encoder.language_model.": "model.", "model.encoder.vision_tower.": "vision_tower.", "model.encoder.embed_vision.": "embed_vision.", }, - orig_to_new_substr={ - ".experts.": ".moe.experts.", - }, ) packed_modules_mapping = { @@ -336,113 +338,12 @@ class DiffusionGemmaForConditionalGeneration( logits = _softcap_logits(logits, self.final_logit_softcapping) return logits - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): - """Load weights from checkpoint. - - Checkpoint layout (HF DiffusionGemma): - model.encoder.vision_tower.* → vision tower - model.encoder.embed_vision.* → vision embedder - model.encoder.language_model.layers.* → backbone - model.decoder.layers.* → backbone (tied) - model.decoder.embed_tokens.* → embeddings - model.decoder.self_conditioning.* → self-conditioning MLP - lm_head.* → LM head (tied) - - We load encoder weights into our single ``Gemma4Model`` backbone, - skip duplicate decoder backbone weights, handle vision tower and - self-conditioning separately. - """ - sc_params = dict( - (n, p) - for n, p in self.named_parameters() - if n.startswith("self_conditioning.") + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + # Some checkpoints carry a vestigial Gemma3n-style embedding table. + loader = AutoWeightsLoader( + self, ignore_unexpected_prefixes=["embed_vision.embedding."] ) - - # Collect vision tower + embedder parameters AND buffers for manual - # loading. The HF vision tower registers std_bias / std_scale as - # buffers (not parameters) when config.standardize is True, so we - # must include named_buffers() to avoid "not found in model" warnings. - vision_params: dict[str, torch.Tensor] = {} - for n, p in self.named_parameters(): - if n.startswith(("vision_tower.", "embed_vision.")): - vision_params[n] = p - for n, b in self.named_buffers(): - if n.startswith(("vision_tower.", "embed_vision.")): - vision_params[n] = b - - def _remap_weights(): - # Use full weight names (including suffixes like .weight_scale, - # .weight_packed) for dedup instead of just the base layer name. Critical - # for quantized checkpoints where each weight has multiple tensors; - # tracking only base names skips scales as duplicates. - seen_weights: set[str] = set() - for name, weight in weights: - # Self-conditioning lives under model.decoder.self_conditioning.* - # in the checkpoint but at self_conditioning.* in our model. - if "self_conditioning" in name: - sc_name = name.split("self_conditioning.", 1)[1] - sc_name = "self_conditioning." + sc_name - if sc_name in sc_params: - sc_params[sc_name].data.copy_(weight) - continue - - # Vision tower: model.encoder.vision_tower.* → vision_tower.* - # In HF, the vision tower is a sibling of language_model - # under the encoder module. - if name.startswith("model.encoder.vision_tower."): - vt_name = name[len("model.encoder.") :] - if vt_name in vision_params: - vision_params[vt_name].data.copy_(weight) - else: - logger.warning( - "Vision tower weight %s (mapped to %s) not found in model", - name, - vt_name, - ) - continue - - # Vision embedder: model.encoder.embed_vision.* → embed_vision.* - if name.startswith("model.encoder.embed_vision."): - ev_name = name[len("model.encoder.") :] - if ev_name in vision_params: - vision_params[ev_name].data.copy_(weight) - else: - logger.warning( - "Embed vision weight %s (mapped to %s) not found in model", - name, - ev_name, - ) - continue - - # Skip vestigial embed_vision.embedding weights. - if "embed_vision.embedding." in name: - continue - - # Encoder backbone → model.* - if name.startswith("model.encoder.language_model."): - name = name.replace("model.encoder.language_model.", "model.") - # Decoder backbone → model.* (skip exact duplicates) - elif name.startswith("model.decoder."): - name = name.replace("model.decoder.", "model.") - - # Skip only if we've seen the exact same weight name (including scales) - if name in seen_weights: - continue - seen_weights.add(name) - yield name, weight - - # Delegate to Gemma4ForCausalLM.load_weights for the backbone, - # which handles stacked params, MoE, k_eq_v, etc. - # Temporarily set self.config to text_config since Gemma4's - # load_weights expects it (e.g. tie_word_embeddings, layer_types). - from vllm.model_executor.models.gemma4 import Gemma4ForCausalLM - - saved_config = self.config - self.config = self.model.config - try: - Gemma4ForCausalLM.load_weights(self, _remap_weights()) - finally: - self.config = saved_config + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) @classmethod def get_placeholder_str(cls, modality: str, i: int) -> str | None: @@ -1066,6 +967,7 @@ class DiffusionSampler: tp_group_name: str = "", ): self.sampling_states = sampler.sampling_states + self.logprob_token_ids_state = sampler.logprob_token_ids_state self.req_states = sampler.req_states self.logits_mode = sampler.logprobs_mode in ("raw_logits", "processed_logits") # Self-conditioning soft embed = probs @ embed_weight * normalizer, @@ -1112,7 +1014,7 @@ class DiffusionSampler: # on the subsequent commit step when num_sampled=CANVAS_LEN. self._pending_logprobs: dict[int, LogprobsTensors] = {} - def add_request(self, req_idx: int, prompt_len: int, sampling_params: Any) -> None: + def add_request(self, req_idx: int, sampling_params: Any) -> None: if use_penalty(sampling_params): logger.warning_once( "DiffusionGemma does not support repetition/frequency/presence " @@ -1122,9 +1024,11 @@ class DiffusionSampler: # that was aborted between its converging denoise and commit steps. self._pending_logprobs.pop(req_idx, None) self.sampling_states.add_request(req_idx, sampling_params) + self.logprob_token_ids_state.add_request(req_idx, sampling_params) def apply_staged_writes(self) -> None: self.sampling_states.apply_staged_writes() + self.logprob_token_ids_state.apply_staged_writes() @property def penalties_state(self): @@ -1312,8 +1216,12 @@ class DiffusionSampler: is_committing = states.is_encoder_phase[decode_slots].clone() slots_np = input_batch.idx_mapping_np[:num_reqs] - is_decode_np = per_req_nlogits_np > 0 max_num_logprobs = self.sampling_states.max_num_logprobs(slots_np) + # Requests may ask for specific token ids' logprobs instead of, or as + # well as, a top-k. + max_token_ids = self.logprob_token_ids_state.max_num_token_ids(slots_np) + want_logprobs = max_num_logprobs >= 0 or max_token_ids > 0 + num_logprobs = max(max_num_logprobs, 0) # Sample over the [num_decode * CL, vocab] logits. The fp32 pipeline in # _compiled_sample_step keeps several live [group * CL, vocab] copies, so @@ -1371,7 +1279,7 @@ class DiffusionSampler: # Logprobs for denoise steps that just converged (is_encoder_phase # flipped False→True), stashed per tile so `scaled` is freed each tile. - if max_num_logprobs >= 0: + if want_logprobs: converged_mask = states.is_encoder_phase[tile_slots] just_converged = converged_mask & ~is_committing[tile] if just_converged.any(): @@ -1385,24 +1293,43 @@ class DiffusionSampler: # positions are never emitted. k_i = int(valid_canvas_len_np[start_req + li]) pos = li * CL + per_req_ids = max_token_ids > 0 self._pending_logprobs[slot.item()] = compute_topk_scores( flat_logits[pos : pos + k_i], - max_num_logprobs, + num_logprobs, argmax_tokens[local_idx][:k_i], + logprob_token_ids_state=( + self.logprob_token_ids_state if per_req_ids else None + ), + # every row of this stash belongs to one slot + expanded_idx_mapping=( + torch.full( + (k_i,), + slot.item(), + dtype=torch.int32, + device=flat_logits.device, + ) + if per_req_ids + else None + ), + max_per_req_token_ids=max_token_ids, logits_mode=self.logits_mode, ) # Commit steps: is_committing was True at entry. Reassemble previously # stashed logprobs and attach to SamplerOutput. logprobs_tensors = None - if max_num_logprobs >= 0 and is_committing.any() and self._pending_logprobs: + if want_logprobs and is_committing.any() and self._pending_logprobs: + committing_slots = set( + decode_slots_np[is_committing.cpu().numpy()].tolist() + ) parts_ids, parts_lp, parts_ranks = [], [], [] cu_gen: list[int] = [] flat_offset = 0 for i in range(num_reqs): cu_gen.append(flat_offset) slot = int(slots_np[i]) - if is_decode_np[i] and slot in self._pending_logprobs: + if slot in committing_slots and slot in self._pending_logprobs: lp = self._pending_logprobs.pop(slot) parts_ids.append(lp.logprob_token_ids) parts_lp.append(lp.logprobs) diff --git a/vllm/model_executor/models/dots_ocr.py b/vllm/model_executor/models/dots_ocr.py index 36b1cd530d..adcc560ba5 100644 --- a/vllm/model_executor/models/dots_ocr.py +++ b/vllm/model_executor/models/dots_ocr.py @@ -9,7 +9,7 @@ from torch.nn import LayerNorm from transformers.models.qwen2_vl import Qwen2VLProcessor from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import utils as dist_utils from vllm.distributed.parallel_state import ( get_tensor_model_parallel_rank, @@ -104,20 +104,16 @@ class DotsOCRDummyInputsBuilder(Qwen2VLDummyInputsBuilder): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - target_width, target_height = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") - return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), } diff --git a/vllm/model_executor/models/ernie45_vl.py b/vllm/model_executor/models/ernie45_vl.py index e1b847c294..a800458bbf 100644 --- a/vllm/model_executor/models/ernie45_vl.py +++ b/vllm/model_executor/models/ernie45_vl.py @@ -36,7 +36,7 @@ from einops import rearrange from transformers import BaseImageProcessor, BatchFeature from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions, VideoDummyOptions from vllm.distributed import parallel_state from vllm.distributed import utils as dist_utils from vllm.inputs import MultiModalDataDict @@ -1276,33 +1276,26 @@ class Ernie4_5_VLDummyInputsBuilder(BaseDummyInputsBuilder[Ernie4_5_VLProcessing self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - num_videos = mm_counts.get("video", 0) - target_width, target_height = self.info.get_image_size_with_most_features() target_num_frames = self.info.get_num_frames_with_most_features( seq_len, mm_counts ) - image_overrides = mm_options.get("image") - video_overrides = mm_options.get("video") - assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions) - return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), "video": self._get_dummy_videos( width=target_width, height=target_height, num_frames=target_num_frames, - num_videos=num_videos, - overrides=video_overrides, + num_videos=mm_counts.get("video", 0), + overrides=mm_options.get("video"), ), } diff --git a/vllm/model_executor/models/fireredasr2.py b/vllm/model_executor/models/fireredasr2.py index 82a4a6b506..be5f5efbff 100644 --- a/vllm/model_executor/models/fireredasr2.py +++ b/vllm/model_executor/models/fireredasr2.py @@ -12,7 +12,7 @@ from transformers import ( ) from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.config.speech_to_text import SpeechToTextParams from vllm.inputs import MultiModalDataDict, PromptType from vllm.logger import init_logger @@ -207,20 +207,18 @@ class FireRedASR2DummyInputsBuilder(BaseDummyInputsBuilder[FireRedASR2Processing self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: feature_extractor = self.info.get_feature_extractor() sampling_rate = feature_extractor.sampling_rate audio_len = feature_extractor.chunk_length * sampling_rate - num_audios = mm_counts.get("audio", 0) - - audio_overrides = mm_options.get("audio") - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) ret = { "audio": self._get_dummy_audios( - length=audio_len, num_audios=num_audios, overrides=audio_overrides + length=audio_len, + num_audios=mm_counts.get("audio", 0), + overrides=mm_options.get("audio"), ) } return ret diff --git a/vllm/model_executor/models/funasr.py b/vllm/model_executor/models/funasr.py index 7bf253ab40..f8db523785 100644 --- a/vllm/model_executor/models/funasr.py +++ b/vllm/model_executor/models/funasr.py @@ -14,7 +14,7 @@ from transformers import ( ) from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.config.speech_to_text import SpeechToTextParams from vllm.distributed import get_tensor_model_parallel_world_size from vllm.inputs import MultiModalDataDict, PromptType @@ -734,22 +734,18 @@ class FunASRDummyInputsBuilder(BaseDummyInputsBuilder[FunASRProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: feature_extractor = self.info.get_feature_extractor() sampling_rate = feature_extractor.sampling_rate audio_len = feature_extractor.chunk_length * sampling_rate - num_audios = mm_counts.get("audio", 0) - - audio_overrides = mm_options.get("audio") - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) return { "audio": self._get_dummy_audios( length=audio_len, - num_audios=num_audios, - overrides=audio_overrides, + num_audios=mm_counts.get("audio", 0), + overrides=mm_options.get("audio"), ), } diff --git a/vllm/model_executor/models/funaudiochat.py b/vllm/model_executor/models/funaudiochat.py index 7a093cc2e7..8b40ddf0cd 100644 --- a/vllm/model_executor/models/funaudiochat.py +++ b/vllm/model_executor/models/funaudiochat.py @@ -26,7 +26,7 @@ from transformers.feature_extraction_utils import BatchFeature from transformers.modeling_outputs import BaseModelOutput from vllm.config import VllmConfig -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear @@ -580,7 +580,7 @@ class FunAudioChatDummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: feature_extractor = self.info.get_feature_extractor() sampling_rate = int(feature_extractor.sampling_rate) @@ -597,15 +597,12 @@ class FunAudioChatDummyInputsBuilder( 1, (target_num_frames * sampling_rate + token_fps - 1) // token_fps, ) - num_audios = int(mm_counts.get("audio", 0)) - audio_overrides = mm_options.get("audio") - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) return { "audio": self._get_dummy_audios( length=audio_len, - num_audios=num_audios, - overrides=audio_overrides, + num_audios=int(mm_counts.get("audio", 0)), + overrides=mm_options.get("audio"), ) } diff --git a/vllm/model_executor/models/gemma3_mm.py b/vllm/model_executor/models/gemma3_mm.py index 5bccd9aec0..c6166bc2f4 100644 --- a/vllm/model_executor/models/gemma3_mm.py +++ b/vllm/model_executor/models/gemma3_mm.py @@ -11,7 +11,7 @@ from transformers.models.gemma3.image_processing_gemma3 import Gemma3ImageProces from transformers.models.gemma3.processing_gemma3 import Gemma3ProcessorKwargs from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.logger import init_logger from vllm.model_executor.layers.layernorm import GemmaRMSNorm @@ -244,20 +244,16 @@ class Gemma3DummyInputsBuilder(BaseDummyInputsBuilder[Gemma3ProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - target_width, target_height = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") - return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/gemma3n_mm.py b/vllm/model_executor/models/gemma3n_mm.py index d0f536afe8..be2e8edeb2 100644 --- a/vllm/model_executor/models/gemma3n_mm.py +++ b/vllm/model_executor/models/gemma3n_mm.py @@ -18,7 +18,7 @@ from transformers.models.gemma3n import ( from transformers.models.siglip import SiglipImageProcessorFast from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.config.speech_to_text import SpeechToTextParams from vllm.inputs import MultiModalDataDict, PromptType, TextPrompt from vllm.logger import init_logger @@ -232,10 +232,8 @@ class Gemma3nDummyInputsBuilder(BaseDummyInputsBuilder[Gemma3nProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - num_audios = mm_counts.get("audio", 0) processor = self.info.get_hf_processor() audio_feature_extractor: Gemma3nAudioFeatureExtractor = ( processor.feature_extractor @@ -245,21 +243,17 @@ class Gemma3nDummyInputsBuilder(BaseDummyInputsBuilder[Gemma3nProcessingInfo]): img_width = image_processor.size.get("width", 224) img_height = image_processor.size.get("height", 224) - image_overrides = mm_options.get("image") - audio_overrides = mm_options.get("audio") - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) - return { "image": self._get_dummy_images( width=img_width, height=img_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), "audio": self._get_dummy_audios( length=audio_len, - num_audios=num_audios, - overrides=audio_overrides, + num_audios=mm_counts.get("audio", 0), + overrides=mm_options.get("audio"), ), } diff --git a/vllm/model_executor/models/gemma4.py b/vllm/model_executor/models/gemma4.py index 04e22523c2..e3dfe65c2f 100644 --- a/vllm/model_executor/models/gemma4.py +++ b/vllm/model_executor/models/gemma4.py @@ -22,12 +22,12 @@ from collections.abc import Iterable from dataclasses import replace from itertools import islice -import regex as re import torch from torch import nn from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig +from vllm.config.utils import getattr_iter from vllm.distributed import ( get_pp_group, get_tensor_model_parallel_rank, @@ -40,7 +40,7 @@ from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( FusedMoEFactory, GateLinear, - fused_moe_make_expert_params_mapping, + MoERunner, ) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( @@ -57,10 +57,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - maybe_remap_kv_scale_name, -) from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors from vllm.transformers_utils.configs.gemma4 import gemma4_layer_config @@ -73,27 +69,63 @@ from .interfaces import ( SupportsEagle3, SupportsLoRA, SupportsPP, + SupportsQuant, ) from .utils import ( AutoWeightsLoader, + PPMissingLayer, + ShardIds, WeightsMapper, extract_layer_index, - is_pp_missing_parameter, make_layers, maybe_prefix, ) logger = init_logger(__name__) -_GEMMA4_EXPERT_PARENT_MAPPER = WeightsMapper( - orig_to_new_regex={ - re.compile(r"(? WeightsMapper: + """Stack each decoder layer's projections the way its modules are built. + + gate/up pack into `mlp.gate_up_proj`. Layers with a qkv_proj pack q/k/v + into it; `attention_k_eq_v` full-attention layers ship k_proj only, so K + is loaded into both the K and V shards and no v_proj is expected. KV-shared + layers only have q_proj; the K/V tensors original checkpoints still ship + for them are dropped. + """ + num_layers = config.num_hidden_layers + first_kv_shared = num_layers - getattr(config, "num_kv_shared_layers", 0) + k_eq_v_layers = ( + { + i + for i, layer_type in enumerate(config.layer_types[:first_kv_shared]) + if layer_type == "full_attention" + } + if getattr(config, "attention_k_eq_v", False) + else set() + ) + qkv_shards: tuple[tuple[str, ShardIds], ...] = (("q", "q"), ("k", "k"), ("v", "v")) + k_eq_v_shards: tuple[tuple[str, ShardIds], ...] = (("q", "q"), ("k", ["k", "v"])) + stacked: dict[str, tuple[str, ShardIds]] = { + ".mlp.gate_proj.": (".mlp.gate_up_proj.", 0), + ".mlp.up_proj.": (".mlp.gate_up_proj.", 1), } -) - - -def _remap_gemma4_expert_weight_name(name: str) -> str: - return re.sub(r"(? tuple[torch.Tensor, torch.Tensor]: + """Softmax over ALL experts → top-k → renormalize. + + MoERunner's built-in fused_topk scopes softmax differently, so Gemma4 + needs its own routing for numerical correctness. + + NOTE: self.per_expert_scale is read at call time (not captured into a + local) so that torch.func.functional_call parameter substitution + reaches the routing function correctly. + """ + if current_platform.is_cuda_alike() or current_platform.is_xpu(): + return gemma4_fused_routing_kernel_triton( + gating_output, topk, self.per_expert_scale + ) + + return gemma4_routing_function_torch(gating_output, topk, self.per_expert_scale) + def forward(self, x: torch.Tensor) -> torch.Tensor: """Returns raw router logits [T, E].""" x = self.norm(x) @@ -305,77 +363,6 @@ class Gemma4Router(nn.Module): return router_logits -class Gemma4MoE(nn.Module): - """Mixture of Experts for Gemma4 using vLLM's MoERunner. - - Wraps MoERunner with custom routing. The router projection is - external (Gemma4Router) — this class only handles expert dispatch. - - Gemma4 routing: softmax over ALL experts → top-k → renormalize. - per_expert_scale is folded into routing weights for mathematical - correctness with MoERunner's fused kernel. - """ - - def __init__( - self, - config, - quant_config: QuantizationConfig | None = None, - prefix: str = "", - ) -> None: - super().__init__() - self.hidden_size = config.hidden_size - self.num_experts = config.num_experts - - # Per-expert output scale folded into routing weights so that - # MoERunner's fused kernel computes: Σ_e (expert_e * w_e * scale_e) - self.per_expert_scale = nn.Parameter(torch.ones(config.num_experts)) - - # Gemma4 routing: softmax over ALL experts → top-k → renormalize. - # MoERunner's built-in fused_topk scopes softmax differently, so - # a custom routing function is needed for numerical correctness. - # NOTE: self.per_expert_scale is read at call time (not captured into - # a local) so that torch.func.functional_call parameter substitution - # reaches the routing function correctly. - def routing_function( - hidden_states: torch.Tensor, - gating_output: torch.Tensor, - topk: int, - renormalize: bool, - ) -> tuple[torch.Tensor, torch.Tensor]: - if current_platform.is_cuda_alike() or current_platform.is_xpu(): - return gemma4_fused_routing_kernel_triton( - gating_output, topk, self.per_expert_scale - ) - - return gemma4_routing_function_torch( - gating_output, topk, self.per_expert_scale - ) - - # MoERunner experts with custom Gemma4 routing - intermediate_size = getattr( - config, - "moe_intermediate_size", - getattr(config, "expert_intermediate_size", None), - ) - if intermediate_size is None: - raise ValueError("Gemma4 MoE requires an expert intermediate size") - - self.experts = FusedMoEFactory( - num_experts=config.num_experts, - top_k=config.top_k_experts, - hidden_size=config.hidden_size, - intermediate_size=intermediate_size, - renormalize=True, - quant_config=quant_config, - prefix=f"{prefix}.experts", - custom_routing_function=routing_function, - activation="gelu_tanh", - ) - - def forward(self, x: torch.Tensor, router_logits: torch.Tensor) -> torch.Tensor: - return self.experts(x, router_logits) - - class Gemma4Attention(nn.Module): def __init__( self, @@ -433,7 +420,7 @@ class Gemma4Attention(nn.Module): else: # QKVParallelLinear handles GQA correctly for all layer types. # k_eq_v layers load K weights into both K and V slots via - # _weight_iterator remapping — no structural difference needed. + # hf_to_vllm_mapper — no structural difference needed. self.qkv_proj = QKVParallelLinear( hidden_size, self.head_dim, @@ -645,7 +632,7 @@ class Gemma4DecoderLayer(nn.Module): config, "use_second_mlp_block", False ) self.router: Gemma4Router | None - self.moe: Gemma4MoE | None + self.experts: MoERunner | None self.post_feedforward_layernorm_1: RMSNorm | None self.post_feedforward_layernorm_2: RMSNorm | None self.pre_feedforward_layernorm_2: RMSNorm | None @@ -655,10 +642,20 @@ class Gemma4DecoderLayer(nn.Module): quant_config=quant_config, prefix=f"{prefix}.router", ) - self.moe = Gemma4MoE( - config, + names = ("moe_intermediate_size", "expert_intermediate_size") + intermediate_size = getattr_iter(config, names) + if intermediate_size is None: + raise ValueError("Gemma4 MoE requires an expert intermediate size") + self.experts = FusedMoEFactory( + num_experts=config.num_experts, + top_k=config.top_k_experts, + hidden_size=config.hidden_size, + intermediate_size=intermediate_size, + renormalize=True, quant_config=quant_config, - prefix=f"{prefix}.moe", + prefix=f"{prefix}.experts", + custom_routing_function=self.router.routing_function, + activation="gelu_tanh", ) self.post_feedforward_layernorm_1 = RMSNorm( config.hidden_size, eps=config.rms_norm_eps @@ -671,7 +668,7 @@ class Gemma4DecoderLayer(nn.Module): ) else: self.router = None - self.moe = None + self.experts = None self.post_feedforward_layernorm_1 = None self.post_feedforward_layernorm_2 = None self.pre_feedforward_layernorm_2 = None @@ -747,13 +744,13 @@ class Gemma4DecoderLayer(nn.Module): assert self.post_feedforward_layernorm_1 is not None assert self.pre_feedforward_layernorm_2 is not None assert self.router is not None - assert self.moe is not None + assert self.experts is not None assert self.post_feedforward_layernorm_2 is not None hidden_states_1 = self.post_feedforward_layernorm_1(hidden_states) hidden_states_2 = self.pre_feedforward_layernorm_2(residual) router_logits = self.router(residual) - hidden_states_2 = self.moe(hidden_states_2, router_logits) + hidden_states_2 = self.experts(hidden_states_2, router_logits) hidden_states_2 = self.post_feedforward_layernorm_2(hidden_states_2) # Combine MLP and MoE outputs @@ -983,7 +980,7 @@ class Gemma4CrossDecoderLayers(nn.Module): @support_torch_compile( enable_if=lambda vllm_config: not vllm_config.cache_config.kv_sharing_fast_prefill ) -class Gemma4Model(nn.Module, EagleModelMixin): +class Gemma4Model(nn.Module, EagleModelMixin, SupportsQuant): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() config = _get_text_config(vllm_config.model_config.hf_config) @@ -991,6 +988,7 @@ class Gemma4Model(nn.Module, EagleModelMixin): quant_config = vllm_config.quant_config self.config = config self.quant_config = quant_config + self.hf_to_vllm_mapper = _gemma4_layer_weights_mapper(config) # PLE config values (default to 0 if not present — disables PLE) self.hidden_size_per_layer_input = getattr( @@ -1391,152 +1389,21 @@ class Gemma4Model(nn.Module, EagleModelMixin): return hidden_states def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - - # MoE expert weight mapping: checkpoint can have either: - # 1. 3D packed tensors (exploded in _weight_iterator to per-expert 2D) - # 2. Already per-expert 2D weights (if quantized) - # Map to MoERunner parameters: - # moe.experts.{id}.gate_proj → MoERunner w1 (shard of w13) - # moe.experts.{id}.up_proj → MoERunner w3 (shard of w13) - # moe.experts.{id}.down_proj → MoERunner w2 - num_experts = getattr(self.config, "num_experts", None) or 0 - # Strategy A: dot-separated suffix - # (standard AWQ/GPTQ e.g. .qweight, .scales, .weight) - dot_suffix_expert_params_mapping = fused_moe_make_expert_params_mapping( - self, - ckpt_gate_proj_name="gate_proj", - ckpt_down_proj_name="down_proj", - ckpt_up_proj_name="up_proj", - num_experts=num_experts, - ) - # Strategy B: underscore-separated suffix - # (CompressedTensors-format AWQ/W4A16 _packed, _scale) - underscore_suffix_expert_params_mapping = [ - ( - f"{param_name}weight_", - f"{weight_name.rstrip('.')}_", - expert_id, - shard_id, - ) - for ( - param_name, - weight_name, - expert_id, - shard_id, - ) in dot_suffix_expert_params_mapping - ] - expert_params_mapping = ( - dot_suffix_expert_params_mapping + underscore_suffix_expert_params_mapping - ) - params_dict = dict(self.named_parameters()) - # Include buffers (e.g. layer_scalar) so they can be loaded too - params_dict.update(dict(self.named_buffers())) - loaded_params: set[str] = set() - for name, loaded_weight in weights: - if name.endswith((".k_scale", ".v_scale", ".q_scale", ".prob_scale")): - remapped_name = maybe_remap_kv_scale_name(name, params_dict) - if remapped_name is not None and remapped_name in params_dict: - param = params_dict[remapped_name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - loaded_params.add(remapped_name) - continue - - for param_name, shard_name, shard_id in stacked_params_mapping: - if shard_name not in name: - continue - stacked_name = name.replace(shard_name, param_name) - # KV-shared layers have a plain q_proj instead of a packed - # qkv_proj: fall through to the direct load, which also - # skips the redundant K/V tensors original checkpoints ship. - if stacked_name not in params_dict: - continue - if is_pp_missing_parameter(stacked_name, self): - continue - param = params_dict[stacked_name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - loaded_params.add(stacked_name) - break - else: - for ( - param_name, - weight_name, - expert_id, - shard_id, - ) in expert_params_mapping: - # Match both: - # - Bare weights: "experts.0.down_proj" (from 3D explosion) - # - With suffix: "experts.0.down_proj.weight_scale" (2D quantized) - # weight_name has trailing dot, so check with and without it - weight_name_base = weight_name.rstrip(".") - if weight_name in name: - # Has suffix (e.g., .weight_scale) - moe_name = name.replace(weight_name, param_name) - elif name.endswith(weight_name_base): - # Bare weight (no suffix) - moe_name = name.replace( - weight_name_base, param_name.rstrip("_") + "_weight" - ) - else: - continue - if moe_name not in params_dict: - continue - if is_pp_missing_parameter(moe_name, self): - continue - param = params_dict[moe_name] - # Expert weights are already in the correct - # orientation for MoERunner after _weight_iterator: - # gate/up: [I, H] → w1/w3 expects [I, H] - # down: [H, I] → w2 expects [H, I] - # Scales and other quantization params may be 1D or scalar. - weight_loader = param.weight_loader - weight_loader( - param, - loaded_weight, - moe_name, # Pass mapped name (handles both weights and scales) - shard_id=shard_id, - expert_id=expert_id, - ) - loaded_params.add(moe_name) - break - else: - if name.endswith(".bias") and name not in params_dict: - continue - remapped_name = maybe_remap_kv_scale_name(name, params_dict) - if remapped_name is None: - continue - name = remapped_name - if is_pp_missing_parameter(name, self): - continue - # Skip if name doesn't exist in params_dict (e.g., individual - # expert weights that should have been handled above) - if name not in params_dict: - continue - param = params_dict[name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - loaded_params.add(name) - - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) class Gemma4ForCausalLM( nn.Module, SupportsLoRA, SupportsPP, MixtureOfExperts, SupportsEagle3 ): - hf_to_vllm_mapper = _GEMMA4_EXPERT_PARENT_MAPPER | WeightsMapper( + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_substr={ + # Multimodal weights are handled by the multimodal wrapper. + "audio_tower.": None, + "vision_tower.": None, + "embed_audio.": None, + "embed_vision.": None, + }, orig_to_new_prefix={ # Gemma4ForConditionalGeneration already loads the text stack # from `model.language_model.*`. We reuse that same checkpoint @@ -1544,13 +1411,6 @@ class Gemma4ForCausalLM( # so LoRA keys from the conditional wrapper map onto `model.*`. "model.language_model.": "model.", }, - orig_to_new_substr={ - # Gemma4ForConditionalGeneration names MoE adapter targets under - # `...moe.experts.*`, while the text-only model exposes them - # under `...moe.*`. - ".moe.experts.gate_up_proj": ".moe.gate_up_proj", - ".moe.experts.down_proj": ".moe.down_proj", - }, ) # KV-shared layers only have q_proj, so qkv_proj packing applies to # the other layers. @@ -1596,26 +1456,18 @@ class Gemma4ForCausalLM( ) # --- MixtureOfExperts protocol --- - self.moe_layers: list[nn.Module] = [] - example_moe: Gemma4MoE | None = None - - for layer in self.model.layers: - if hasattr(layer, "moe") and isinstance(layer.moe, Gemma4MoE): - example_moe = layer.moe - self.moe_layers.append(layer.moe.experts) - + self.moe_layers: list[nn.Module] = [ + layer.experts + for layer in self.model.layers + if not isinstance(layer, PPMissingLayer) and layer.experts is not None + ] self.num_moe_layers = len(self.moe_layers) - if example_moe is not None: - self.num_logical_experts = example_moe.num_experts - self.num_physical_experts = example_moe.num_experts - self.num_local_physical_experts = example_moe.num_experts - self.num_routed_experts = example_moe.num_experts - else: - self.num_logical_experts = 0 - self.num_physical_experts = 0 - self.num_local_physical_experts = 0 - self.num_routed_experts = 0 + num_experts = config.num_experts if self.num_moe_layers else 0 + self.num_logical_experts = num_experts + self.num_physical_experts = num_experts + self.num_local_physical_experts = num_experts + self.num_routed_experts = num_experts self.num_expert_groups = 1 self.num_shared_experts = 0 @@ -1644,106 +1496,5 @@ class Gemma4ForCausalLM( return self.logits_processor(self.lm_head, hidden_states) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - # Checkpoint weight names use "language_model." prefix (from the - # Gemma4ForConditionalGeneration wrapper). Strip it to map to our - # model tree which is just "model.*". - def _weight_iterator(): - use_k_eq_v = getattr(self.config, "attention_k_eq_v", False) - # Build set of k_eq_v layer indices (full_attention layers - # when attention_k_eq_v is enabled). These layers have k_proj - # but no v_proj in checkpoint — we duplicate k_proj as v_proj. - k_eq_v_layer_indices: set[int] = set() - if use_k_eq_v: - for idx, lt in enumerate(self.config.layer_types): - if lt == "full_attention": - k_eq_v_layer_indices.add(idx) - - for name, weight in weights: - # Remap "language_model." → "" to match our model tree. - # Checkpoint: model.language_model.layers.X.* - # Our model: model.layers.X.* - name = name.replace("language_model.", "") - - # Remap new HF checkpoint naming to internal vLLM - # naming: HF moved per_expert_scale to router and - # renamed moe → experts in the MoE block. - name = name.replace( - ".router.per_expert_scale", - ".moe.per_expert_scale", - ) - if ".experts.gate_up_proj" in name: - name = name.replace( - ".experts.gate_up_proj", - ".moe.gate_up_proj", - ) - elif ".experts.down_proj" in name: - name = name.replace( - ".experts.down_proj", - ".moe.down_proj", - ) - - # Remap individual 2D expert weights: - # .experts.{id}.{proj} → .moe.experts.{id}.{proj} - # (This handles per-expert 2D quantized weights) - name = _remap_gemma4_expert_weight_name(name) - - # MoE expert weights: checkpoint stores as 3D packed - # tensors. Explode into per-expert 2D weights for - # MoERunner weight_loader. - # - # Checkpoint format: - # moe.gate_up_proj: [E, 2*I, H] (fused gate + up) - # moe.down_proj: [E, H, I] - # - # MoERunner expects per-expert: - # w1 (gate): [I, H] — first half of gate_up - # w3 (up): [I, H] — second half of gate_up - # w2 (down): [H, I] — as-is from checkpoint - # - # No transpose needed: checkpoint orientation already - # matches MoERunner's expected layout. - if "moe.gate_up_proj" in name and weight.dim() == 3: - num_experts = weight.size(0) - intermediate_size = weight.size(1) // 2 - for expert_id in range(num_experts): - gate_weight = weight[expert_id, :intermediate_size, :] - up_weight = weight[expert_id, intermediate_size:, :] - base = name.replace("moe.", f"moe.experts.{expert_id}.") - yield base.replace("gate_up_proj", "gate_proj"), gate_weight - yield base.replace("gate_up_proj", "up_proj"), up_weight - continue - - if "moe.down_proj" in name and weight.dim() == 3: - num_experts = weight.size(0) - for expert_id in range(num_experts): - expert_name = name.replace("moe.", f"moe.experts.{expert_id}.") - yield expert_name, weight[expert_id] - continue - - # k_eq_v layers: checkpoint has k_proj but no v_proj. - # QKVParallelLinear expects both, so duplicate k_proj - # as v_proj so V gets identical weights to K. - # ONLY for full_attention layers — sliding layers have - # their own real v_proj weights. - if "self_attn.k_proj" in name and k_eq_v_layer_indices: - m = re.search(r"layers\.(\d+)\.", name) - if m and int(m.group(1)) in k_eq_v_layer_indices: - yield name, weight - yield name.replace("k_proj", "v_proj"), weight.clone() - continue - - yield name, weight - - # Drop multimodal weights, which are handled by the multimodal wrapper. - # `_weight_iterator` already applies this model's renames by hand, so - # `hf_to_vllm_mapper` is deliberately not passed here. - mapper = WeightsMapper( - orig_to_new_substr={ - "audio_tower.": None, - "vision_tower.": None, - "embed_audio.": None, - "embed_vision.": None, - } - ) loader = AutoWeightsLoader(self) - return loader.load_weights(_weight_iterator(), mapper=mapper) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) diff --git a/vllm/model_executor/models/gemma4_mm.py b/vllm/model_executor/models/gemma4_mm.py index 6ad98cb650..796dfbb6e8 100644 --- a/vllm/model_executor/models/gemma4_mm.py +++ b/vllm/model_executor/models/gemma4_mm.py @@ -36,20 +36,13 @@ from transformers.models.gemma4.configuration_gemma4 import ( from vllm.config import VllmConfig from vllm.config.model import get_served_model_name -from vllm.config.multimodal import ( - AudioDummyOptions, - BaseDummyOptions, - VideoDummyOptions, -) +from vllm.config.multimodal import MultiModalDummyOptions, VideoDummyOptions from vllm.inputs import MultiModalDataDict from vllm.logger import init_logger from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.models.gemma3n_mm import batch_audio_features -from vllm.model_executor.models.gemma4 import ( - _GEMMA4_EXPERT_PARENT_MAPPER, - Gemma4ForCausalLM, -) +from vllm.model_executor.models.gemma4 import Gemma4ForCausalLM from vllm.model_executor.models.module_mapping import MultiModelKeys from vllm.model_executor.models.transformers.utils import recursive_replace_linear from vllm.multimodal import MULTIMODAL_REGISTRY @@ -276,10 +269,7 @@ class Gemma4ProcessingInfo(BaseProcessingInfo): num_frames = _VIDEO_MAX_FRAMES mm_config = self.ctx.model_config.get_multimodal_config() video_opts = mm_config.limit_per_prompt.get("video") - if ( - isinstance(video_opts, VideoDummyOptions) - and video_opts.num_frames is not None - ): + if video_opts is not None and video_opts.num_frames is not None: num_frames = min(num_frames, video_opts.num_frames) tokens["video"] = num_frames * (_VIDEO_MAX_SOFT_TOKENS + 2 + 6) return tokens @@ -505,9 +495,8 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions] | None = None, + mm_options: MultiModalDummyOptions | None = None, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) num_audios = mm_counts.get("audio", 0) num_videos = mm_counts.get("video", 0) processor = self.info.get_hf_processor() @@ -520,18 +509,12 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]): img_width = size.get("width", 224) img_height = size.get("height", 224) - image_overrides = mm_options.get("image") if mm_options else None - audio_overrides = mm_options.get("audio") if mm_options else None - video_overrides = mm_options.get("video") if mm_options else None - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) - assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions) - data: dict[str, Any] = { "image": self._get_dummy_images( width=img_width, height=img_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image") if mm_options else None, ), } @@ -540,7 +523,7 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]): data["audio"] = self._get_dummy_audios( length=audio_len, num_audios=num_audios, - overrides=audio_overrides, + overrides=mm_options.get("audio") if mm_options else None, ) if num_videos > 0: @@ -549,7 +532,7 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]): height=img_height, num_frames=_VIDEO_MAX_FRAMES, num_videos=num_videos, - overrides=video_overrides, + overrides=mm_options.get("video") if mm_options else None, ) return data @@ -1060,7 +1043,7 @@ class Gemma4ForConditionalGeneration( } # Maps checkpoint prefixes to vLLM module paths. - hf_to_vllm_mapper = _GEMMA4_EXPERT_PARENT_MAPPER | WeightsMapper( + hf_to_vllm_mapper = WeightsMapper( orig_to_new_prefix={ # vision tower "model.vision_tower": "vision_tower", diff --git a/vllm/model_executor/models/gemma4_unified.py b/vllm/model_executor/models/gemma4_unified.py index 7c9aaa8499..1a26246515 100644 --- a/vllm/model_executor/models/gemma4_unified.py +++ b/vllm/model_executor/models/gemma4_unified.py @@ -28,7 +28,6 @@ from transformers.models.gemma4_unified.processing_gemma4_unified import ( ) from vllm.config import VllmConfig -from vllm.config.multimodal import VideoDummyOptions from vllm.model_executor.layers.linear import ColumnParallelLinear from vllm.model_executor.models.gemma3n_mm import batch_audio_features from vllm.model_executor.models.gemma4 import Gemma4ForCausalLM @@ -177,10 +176,7 @@ class Gemma4UnifiedProcessingInfo(Gemma4ProcessingInfo): num_frames = _VIDEO_MAX_FRAMES mm_config = self.ctx.model_config.get_multimodal_config() video_opts = mm_config.limit_per_prompt.get("video") - if ( - isinstance(video_opts, VideoDummyOptions) - and video_opts.num_frames is not None - ): + if video_opts is not None and video_opts.num_frames is not None: num_frames = min(num_frames, video_opts.num_frames) tokens["video"] = num_frames * (_VIDEO_MAX_SOFT_TOKENS + 2 + 6) return tokens diff --git a/vllm/model_executor/models/glm4_1v.py b/vllm/model_executor/models/glm4_1v.py index 8f3db8a3d1..95cdcc5a20 100644 --- a/vllm/model_executor/models/glm4_1v.py +++ b/vllm/model_executor/models/glm4_1v.py @@ -36,9 +36,7 @@ import numpy as np import torch import torch.nn as nn import torch.nn.functional as F -import transformers from einops import rearrange -from packaging.version import Version from transformers import BatchFeature, Glm4vProcessor from transformers.image_processing_base import ImageProcessingMixin from transformers.models.glm4v.configuration_glm4v import ( @@ -54,7 +52,7 @@ from transformers.video_processing_utils import BaseVideoProcessor from transformers.video_utils import VideoMetadata from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions, VideoDummyOptions from vllm.distributed import get_tensor_model_parallel_world_size, parallel_state from vllm.distributed import utils as dist_utils from vllm.inputs import MultiModalDataDict @@ -132,8 +130,6 @@ logger = init_logger(__name__) # For profile run _MAX_FRAMES_PER_VIDEO = 600 -TRANSFORMERS_WITH_GA = Version(transformers.__version__) >= Version("5.10.0.dev0") - def _to_video_metadata(metadata: Mapping[str, Any]) -> VideoMetadata: return VideoMetadata( @@ -1389,17 +1385,6 @@ class Glm4vProcessingInfo(BaseProcessingInfo): timestamps_list = full_second_idxs[::2] return list(timestamps_list) - def _get_video_frame_embed_token_id(self, hf_processor: object) -> int: - attr = ( - "image_token_id" - if isinstance(hf_processor, Glm4vProcessor) or TRANSFORMERS_WITH_GA - else "video_token_id" - ) - token_id = getattr(hf_processor, attr, None) - if not isinstance(token_id, int): - raise ValueError(f"Processor has no valid {attr}") - return token_id - def _construct_video_placeholder( self, video_array: np.ndarray, @@ -1437,7 +1422,7 @@ class Glm4vProcessingInfo(BaseProcessingInfo): num_tokens_per_frame = int(H * W) // merge_length placeholder = [] placeholder.append(bov_token_id) - frame_embed_token_id = self._get_video_frame_embed_token_id(hf_processor) + frame_embed_token_id = hf_processor.image_token_id for frame_idx in frames_idx_token: placeholder.append(boi_token_id) placeholder.extend([frame_embed_token_id] * num_tokens_per_frame) @@ -1470,33 +1455,26 @@ class Glm4vDummyInputsBuilder(BaseDummyInputsBuilder[Glm4vProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - num_videos = mm_counts.get("video", 0) - target_width, target_height = self.info.get_image_size_with_most_features() target_num_frames = self.info.get_num_frames_with_most_features( seq_len, mm_counts ) - image_overrides = mm_options.get("image") - video_overrides = mm_options.get("video") - assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions) - return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), "video": self._get_dummy_videos( width=target_width, height=target_height, num_frames=target_num_frames, - num_videos=num_videos, - overrides=video_overrides, + num_videos=mm_counts.get("video", 0), + overrides=mm_options.get("video"), ), } @@ -1601,10 +1579,7 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]): processor = self.info.get_hf_processor(**hf_kwargs) - use_direct_path = ( - not isinstance(processor, Glm4vProcessor) and TRANSFORMERS_WITH_GA - ) - if use_direct_path: + if not isinstance(processor, Glm4vProcessor): prepared_data, prepared_kwargs = self._get_direct_path_inputs( hf_data, hf_kwargs ) @@ -1627,8 +1602,6 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]): del hf_data["videos"] video_grid_thw_lst = [] pixel_values_videos_lst = [] - frame_embed_token_id = self.info._get_video_frame_embed_token_id(processor) - swap_video_frame_tokens = frame_embed_token_id == processor.image_token_id for item in videos: video_array, metadata = item @@ -1651,10 +1624,10 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]): video_mm_kwargs, ) input_ids = video_outputs.pop("input_ids") - if swap_video_frame_tokens: - input_ids[input_ids == processor.image_token_id] = ( - processor.video_token_id - ) + + input_ids[input_ids == processor.image_token_id] = ( + processor.video_token_id + ) video_placeholder = processor.tokenizer.batch_decode(input_ids)[0] prompt_text = prompt_text.replace( "<|begin_of_video|><|video|><|end_of_video|>", @@ -1670,17 +1643,15 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]): ) else: video_outputs = dict() - swap_video_frame_tokens = False processed_data = self.info.ctx.call_hf_processor( self.info.get_hf_processor(**hf_kwargs), dict(text=prompt_text, **hf_data), hf_kwargs, ) - if swap_video_frame_tokens: - input_ids = processed_data["input_ids"] - input_ids[input_ids == processor.video_token_id] = processor.image_token_id - processed_data["input_ids"] = input_ids + input_ids = processed_data["input_ids"] + input_ids[input_ids == processor.video_token_id] = processor.image_token_id + processed_data["input_ids"] = input_ids processed_data.update(video_outputs) return self._finalize_hf_mm_data( @@ -1728,7 +1699,7 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]): ) return PromptUpdateDetails.select_token_id( placeholder, - embed_token_id=self.info._get_video_frame_embed_token_id(hf_processor), + embed_token_id=hf_processor.image_token_id, ) return [ diff --git a/vllm/model_executor/models/glm4v.py b/vllm/model_executor/models/glm4v.py index 4fc6446ad3..7b42d5734f 100644 --- a/vllm/model_executor/models/glm4v.py +++ b/vllm/model_executor/models/glm4v.py @@ -16,7 +16,7 @@ from torch.nn import LayerNorm from transformers import BatchFeature from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import get_tensor_model_parallel_world_size from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn @@ -445,22 +445,19 @@ class GLM4VDummyInputsBuilder(BaseDummyInputsBuilder[GLM4VProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: hf_config = self.info.get_hf_config() vision_config = hf_config.vision_config target_width = target_height = vision_config["image_size"] - num_images = mm_counts.get("image", 0) - - image_overrides = mm_options.get("image") return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/glmasr.py b/vllm/model_executor/models/glmasr.py index 003b02c28e..cf12dedbe5 100644 --- a/vllm/model_executor/models/glmasr.py +++ b/vllm/model_executor/models/glmasr.py @@ -11,7 +11,7 @@ from transformers.models.glmasr import GlmAsrConfig, GlmAsrProcessor from transformers.models.whisper import WhisperFeatureExtractor from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.config.speech_to_text import SpeechToTextParams from vllm.distributed.parallel_state import get_tensor_model_parallel_world_size from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TokensPrompt @@ -694,13 +694,10 @@ class GlmAsrDummyInputsBuilder(BaseDummyInputsBuilder[GlmAsrProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: feature_extractor = self.info.get_feature_extractor() sampling_rate = feature_extractor.sampling_rate - num_audios = mm_counts.get("audio", 0) - audio_overrides = mm_options.get("audio") - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) max_audio_len = getattr( self.info.get_hf_processor(), "max_audio_len", DEFAULT_MAX_AUDIO_LEN_S @@ -710,8 +707,8 @@ class GlmAsrDummyInputsBuilder(BaseDummyInputsBuilder[GlmAsrProcessingInfo]): return { "audio": self._get_dummy_audios( length=audio_len, - num_audios=num_audios, - overrides=audio_overrides, + num_audios=mm_counts.get("audio", 0), + overrides=mm_options.get("audio"), ) } diff --git a/vllm/model_executor/models/granite_speech.py b/vllm/model_executor/models/granite_speech.py index 1c1e98be04..53cbe7df43 100644 --- a/vllm/model_executor/models/granite_speech.py +++ b/vllm/model_executor/models/granite_speech.py @@ -34,7 +34,7 @@ from torch import nn from transformers import BatchFeature, PretrainedConfig from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.config.speech_to_text import SpeechToTextParams from vllm.inputs import MultiModalDataDict, PromptType, TokensPrompt from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear @@ -208,17 +208,13 @@ class GraniteSpeechDummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_audios = mm_counts.get("audio", 0) - audio_overrides = mm_options.get("audio") - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) - return { "audio": self._get_dummy_audios( length=self.info.get_max_audio_len(), - num_audios=num_audios, - overrides=audio_overrides, + num_audios=mm_counts.get("audio", 0), + overrides=mm_options.get("audio"), ) } diff --git a/vllm/model_executor/models/hyperclovax_vision_v2.py b/vllm/model_executor/models/hyperclovax_vision_v2.py index 07106c9471..e269e2a059 100644 --- a/vllm/model_executor/models/hyperclovax_vision_v2.py +++ b/vllm/model_executor/models/hyperclovax_vision_v2.py @@ -18,7 +18,7 @@ import torch.nn as nn from transformers import BatchFeature from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.forward_context import set_forward_context from vllm.inputs import MultiModalDataDict from vllm.multimodal import MULTIMODAL_REGISTRY @@ -193,7 +193,7 @@ class HCXVisionV2DummyInputsBuilder(BaseDummyInputsBuilder[HCXVisionV2Processing self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions] | None = None, + mm_options: MultiModalDummyOptions | None = None, mm_processor_kwargs: Mapping[str, object] | None = None, ) -> ProcessorInputs: """Build dummy processor inputs for memory profiling.""" @@ -225,31 +225,25 @@ class HCXVisionV2DummyInputsBuilder(BaseDummyInputsBuilder[HCXVisionV2Processing self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions] | None = None, + mm_options: MultiModalDummyOptions | None = None, mm_processor_kwargs: Mapping[str, object] | None = None, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - num_videos = mm_counts.get("video", 0) - target_width, target_height = self.info.get_image_size_with_most_features() target_num_frames = 16 # Default for video - image_overrides = mm_options.get("image") if mm_options else None - video_overrides = mm_options.get("video") if mm_options else None - result: MultiModalDataDict = { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, # type: ignore + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image") if mm_options else None, ), "video": self._get_dummy_videos( width=target_width, height=target_height, num_frames=target_num_frames, - num_videos=num_videos, - overrides=video_overrides, # type: ignore + num_videos=mm_counts.get("video", 0), + overrides=mm_options.get("video") if mm_options else None, ), } diff --git a/vllm/model_executor/models/idefics3.py b/vllm/model_executor/models/idefics3.py index dd03e17d1e..80c71cfe79 100644 --- a/vllm/model_executor/models/idefics3.py +++ b/vllm/model_executor/models/idefics3.py @@ -29,7 +29,7 @@ from transformers import ( ) from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor @@ -221,21 +221,18 @@ class Idefics3DummyInputsBuilder(BaseDummyInputsBuilder[Idefics3ProcessingInfo]) self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) hf_processor = self.info.get_hf_processor() image_processor: Idefics3ImageProcessor = hf_processor.image_processor longest_edge = image_processor.max_image_size["longest_edge"] - image_overrides = mm_options.get("image") - return { "image": self._get_dummy_images( width=longest_edge, height=longest_edge, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/interfaces.py b/vllm/model_executor/models/interfaces.py index 1cefd63dd6..21a09cb7ba 100644 --- a/vllm/model_executor/models/interfaces.py +++ b/vllm/model_executor/models/interfaces.py @@ -91,6 +91,13 @@ MambaStateShapes: TypeAlias = ( tuple[int, int, int], tuple[int, int, int], ] + | tuple[ + tuple[int, int], + tuple[int, int, int], + tuple[int, int, int], + tuple[int, int], + tuple[int, int, int], + ] ) diff --git a/vllm/model_executor/models/interns1.py b/vllm/model_executor/models/interns1.py index 4a4264660d..8d818e9a78 100644 --- a/vllm/model_executor/models/interns1.py +++ b/vllm/model_executor/models/interns1.py @@ -22,7 +22,7 @@ from transformers.models.internvl.video_processing_internvl import ( ) from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.models.interns1_vit import InternS1VisionModel @@ -299,35 +299,29 @@ class InternS1DummyInputsBuilder(BaseDummyInputsBuilder[InternS1ProcessingInfo]) self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: target_width, target_height = self.info.get_image_size_with_most_features() target_num_frames = self.info.get_num_frames_with_most_features( seq_len, mm_counts ) - num_images = mm_counts.get("image", 0) - num_videos = mm_counts.get("video", 0) config = self.info.get_hf_config() image_size_h, image_size_w = config.vision_config.image_size - image_overrides = mm_options.get("image") - video_overrides = mm_options.get("video") - assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions) - return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), "video": self._get_dummy_videos( width=image_size_w, height=image_size_h, num_frames=target_num_frames, - num_videos=num_videos, - overrides=video_overrides, + num_videos=mm_counts.get("video", 0), + overrides=mm_options.get("video"), ), } diff --git a/vllm/model_executor/models/internvl.py b/vllm/model_executor/models/internvl.py index 873c594c54..3640f8d6a3 100644 --- a/vllm/model_executor/models/internvl.py +++ b/vllm/model_executor/models/internvl.py @@ -17,7 +17,7 @@ import torch.nn as nn from transformers import BatchFeature, PretrainedConfig from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig @@ -203,19 +203,16 @@ class BaseInternVLDummyInputsBuilder(BaseDummyInputsBuilder[_I]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: target_width, target_height = self.info.get_image_size_with_most_features() - num_images = mm_counts.get("image", 0) - - image_overrides = mm_options.get("image") return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } @@ -425,7 +422,7 @@ class InternVLDummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: dummy_image = super().get_dummy_mm_data(seq_len, mm_counts, mm_options) if self.info.ctx_video_token: @@ -436,9 +433,6 @@ class InternVLDummyInputsBuilder( ) num_videos = mm_counts.get("video", 0) video_overrides = mm_options.get("video") - assert video_overrides is None or isinstance( - video_overrides, VideoDummyOptions - ) dummy_video = { "video": self._get_dummy_videos( width=image_size, diff --git a/vllm/model_executor/models/isaac.py b/vllm/model_executor/models/isaac.py index f7c1ee2dd1..f8f6dee32c 100644 --- a/vllm/model_executor/models/isaac.py +++ b/vllm/model_executor/models/isaac.py @@ -13,7 +13,7 @@ from einops import rearrange from transformers.image_processing_utils import BatchFeature from vllm.config import ModelConfig, VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import parallel_state from vllm.distributed import utils as dist_utils from vllm.inputs import MultiModalDataDict @@ -386,19 +386,16 @@ class IsaacDummyInputsBuilder(BaseDummyInputsBuilder[IsaacProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - target_width, target_height = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), } diff --git a/vllm/model_executor/models/kanana_v.py b/vllm/model_executor/models/kanana_v.py index e3c27e479b..fd076d66e7 100644 --- a/vllm/model_executor/models/kanana_v.py +++ b/vllm/model_executor/models/kanana_v.py @@ -18,7 +18,7 @@ from transformers.modeling_outputs import BaseModelOutput from transformers.models.qwen2_vl.configuration_qwen2_vl import Qwen2VLVisionConfig from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.logger import init_logger from vllm.multimodal import MULTIMODAL_REGISTRY @@ -442,7 +442,7 @@ class KananaVDummyInputsBuilder(BaseDummyInputsBuilder[KananaVProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: num_images = mm_counts.get("image", 0) return { diff --git a/vllm/model_executor/models/keye.py b/vllm/model_executor/models/keye.py index 1322920900..bfd2af714b 100644 --- a/vllm/model_executor/models/keye.py +++ b/vllm/model_executor/models/keye.py @@ -17,7 +17,7 @@ from transformers.modeling_outputs import BaseModelOutput, BaseModelOutputWithPo from transformers.utils import torch_int from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import get_tensor_model_parallel_world_size from vllm.inputs import ModalityData, MultiModalDataDict from vllm.logger import init_logger @@ -1124,30 +1124,24 @@ class KeyeBaseDummyInputsBuilder(BaseDummyInputsBuilder[_I]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - num_videos = mm_counts.get("video", 0) - target_width, target_height = self.info.get_image_size_with_most_features() target_num_frames = self.info.get_num_frames_with_most_features(seq_len) - image_overrides = mm_options.get("image") - video_overrides = mm_options.get("video") - mm_data = { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), "video": self._get_dummy_videos( width=target_width, height=target_height, num_frames=target_num_frames, - num_videos=num_videos, - overrides=video_overrides, + num_videos=mm_counts.get("video", 0), + overrides=mm_options.get("video"), ), } diff --git a/vllm/model_executor/models/kimi_audio.py b/vllm/model_executor/models/kimi_audio.py index 219c7f0e7b..3541fcf371 100644 --- a/vllm/model_executor/models/kimi_audio.py +++ b/vllm/model_executor/models/kimi_audio.py @@ -13,7 +13,7 @@ from transformers import BatchFeature from transformers import WhisperConfig as HFWhisperConfig from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.config.speech_to_text import SpeechToTextParams from vllm.inputs import PromptType, TokensPrompt from vllm.model_executor.model_loader import DefaultModelLoader @@ -180,7 +180,7 @@ class KimiAudioDummyInputsBuilder(BaseDummyInputsBuilder[KimiAudioProcessingInfo self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> ProcessorInputs: dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options) dummy_mm_items = self.info.parse_mm_data(dummy_mm_data) diff --git a/vllm/model_executor/models/kimi_k25.py b/vllm/model_executor/models/kimi_k25.py index 9af46e441a..81b5ec1646 100644 --- a/vllm/model_executor/models/kimi_k25.py +++ b/vllm/model_executor/models/kimi_k25.py @@ -14,7 +14,7 @@ from torch import nn from transformers import BatchFeature from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.logger import init_logger from vllm.model_executor.layers.quantization import QuantizationConfig @@ -216,7 +216,7 @@ class KimiK25DummyInputsBuilder(BaseDummyInputsBuilder[KimiK25ProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: # TODO: Support mm_options for vision_chunk to allow user configuration dummy_items = self.get_dummy_mm_items() diff --git a/vllm/model_executor/models/kimi_vl.py b/vllm/model_executor/models/kimi_vl.py index 344dfdaaf0..13779042b6 100644 --- a/vllm/model_executor/models/kimi_vl.py +++ b/vllm/model_executor/models/kimi_vl.py @@ -53,7 +53,7 @@ from transformers import BatchFeature from transformers.activations import GELUActivation from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.models.interfaces import ( @@ -219,18 +219,14 @@ class KimiVLDummyInputsBuilder(BaseDummyInputsBuilder[KimiVLProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - - image_overrides = mm_options.get("image") - return { "image": self._get_dummy_images( width=MaxImageTokenMeta.width, height=MaxImageTokenMeta.height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/lfm2_vl.py b/vllm/model_executor/models/lfm2_vl.py index 1cd51b93a2..644da00c81 100644 --- a/vllm/model_executor/models/lfm2_vl.py +++ b/vllm/model_executor/models/lfm2_vl.py @@ -21,7 +21,7 @@ from transformers.models.lfm2_vl.image_processing_lfm2_vl_fast import ( from typing_extensions import Buffer from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.forward_context import set_forward_context from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.linear import ReplicatedLinear @@ -387,20 +387,16 @@ class Lfm2VLDummyInputsBuilder(BaseDummyInputsBuilder[Lfm2VLProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - target_width, target_height = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") - return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), } diff --git a/vllm/model_executor/models/llava.py b/vllm/model_executor/models/llava.py index c3a437eb36..22cd34fc85 100644 --- a/vllm/model_executor/models/llava.py +++ b/vllm/model_executor/models/llava.py @@ -19,13 +19,12 @@ from transformers.models.llava import LlavaProcessor from transformers.models.pixtral import PixtralProcessor from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.activation import get_act_fn from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.multimodal import MULTIMODAL_REGISTRY -from vllm.multimodal.cache import BaseMultiModalProcessorCache from vllm.multimodal.inputs import ( MultiModalFieldConfig, MultiModalKwargsItem, @@ -228,20 +227,16 @@ class LlavaDummyInputsBuilder(BaseDummyInputsBuilder[_I]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - target_width, target_height = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") - return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } @@ -411,21 +406,17 @@ def _build_llava_or_pixtral_hf_info( def _build_llava_or_pixtral_hf_processor( info: _I, dummy_inputs: BaseDummyInputsBuilder[_I], - *, - cache: BaseMultiModalProcessorCache | None = None, ) -> BaseMultiModalProcessor: if isinstance(info, PixtralHFProcessingInfo): return PixtralHFMultiModalProcessor( info, dummy_inputs, # type: ignore - cache=cache, ) if isinstance(info, LlavaProcessingInfo): return LlavaMultiModalProcessor( info, dummy_inputs, # type: ignore - cache=cache, ) raise NotImplementedError(type(info)) diff --git a/vllm/model_executor/models/llava_next_video.py b/vllm/model_executor/models/llava_next_video.py index 235d1adbac..a33480ef08 100644 --- a/vllm/model_executor/models/llava_next_video.py +++ b/vllm/model_executor/models/llava_next_video.py @@ -10,7 +10,7 @@ import torch.nn as nn from transformers import BatchFeature, LlavaNextVideoConfig, LlavaNextVideoProcessor from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.activation import get_act_fn from vllm.model_executor.models.clip import CLIPVisionModel @@ -171,25 +171,20 @@ class LlavaNextVideoDummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_videos = mm_counts.get("video", 0) - target_width, target_height = self.info.get_image_size_with_most_features() target_num_frames = self.info.get_num_frames_with_most_features( seq_len, mm_counts ) - video_overrides = mm_options.get("video") - assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions) - return { "video": self._get_dummy_videos( width=target_width, height=target_height, num_frames=target_num_frames, - num_videos=num_videos, - overrides=video_overrides, + num_videos=mm_counts.get("video", 0), + overrides=mm_options.get("video"), ) } diff --git a/vllm/model_executor/models/llava_onevision.py b/vllm/model_executor/models/llava_onevision.py index 66503d7741..6fc96c8e67 100644 --- a/vllm/model_executor/models/llava_onevision.py +++ b/vllm/model_executor/models/llava_onevision.py @@ -14,7 +14,7 @@ from transformers.models.llava_onevision.modeling_llava_onevision import ( ) from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.activation import get_act_fn from vllm.multimodal import MULTIMODAL_REGISTRY @@ -278,33 +278,26 @@ class LlavaOnevisionDummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - num_videos = mm_counts.get("video", 0) - target_width, target_height = self.info.get_image_size_with_most_features() target_num_frames = self.info.get_num_frames_with_most_features( seq_len, mm_counts ) - image_overrides = mm_options.get("image") - video_overrides = mm_options.get("video") - assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions) - return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), "video": self._get_dummy_videos( width=target_width, height=target_height, num_frames=target_num_frames, - num_videos=num_videos, - overrides=video_overrides, + num_videos=mm_counts.get("video", 0), + overrides=mm_options.get("video"), ), } diff --git a/vllm/model_executor/models/llava_onevision2.py b/vllm/model_executor/models/llava_onevision2.py index 6988982c23..d8d514be23 100644 --- a/vllm/model_executor/models/llava_onevision2.py +++ b/vllm/model_executor/models/llava_onevision2.py @@ -54,7 +54,7 @@ from vllm.compilation.decorators import ( support_torch_compile, ) from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import parallel_state from vllm.distributed import utils as dist_utils from vllm.inputs import ModalityData, MultiModalDataDict @@ -1471,7 +1471,7 @@ class LlavaOnevision2DummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions] | None = None, + mm_options: MultiModalDummyOptions | None = None, ) -> MultiModalDataDict: n_img = mm_counts.get("image", 0) n_vid = mm_counts.get("video", 0) diff --git a/vllm/model_executor/models/midashenglm.py b/vllm/model_executor/models/midashenglm.py index 7de6ab0e7e..6dae25ffb2 100644 --- a/vllm/model_executor/models/midashenglm.py +++ b/vllm/model_executor/models/midashenglm.py @@ -36,7 +36,7 @@ from torch.nn.functional import scaled_dot_product_attention from transformers import BatchFeature from vllm.config import VllmConfig -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import get_tensor_model_parallel_world_size from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.activation import get_act_fn @@ -564,18 +564,13 @@ class MiDashengLMDummyInputsBuilder(BaseDummyInputsBuilder[MiDashengLMProcessing self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_audios = mm_counts.get("audio", 0) - - audio_overrides = mm_options.get("audio") - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) - return { "audio": self._get_dummy_audios( length=self.info.get_max_audio_len(), - num_audios=num_audios, - overrides=audio_overrides, + num_audios=mm_counts.get("audio", 0), + overrides=mm_options.get("audio"), ) } diff --git a/vllm/model_executor/models/mimo_v2.py b/vllm/model_executor/models/mimo_v2.py index ec67d72cf2..be665e7e06 100644 --- a/vllm/model_executor/models/mimo_v2.py +++ b/vllm/model_executor/models/mimo_v2.py @@ -50,6 +50,7 @@ from vllm.model_executor.model_loader.weight_utils import ( ) from vllm.model_executor.models.utils import sequence_parallel_chunk from vllm.sequence import IntermediateTensors +from vllm.utils.math_utils import cdiv from vllm.v1.attention.backend import AttentionType from vllm.v1.attention.backends.registry import AttentionBackendEnum @@ -460,6 +461,28 @@ class MiMoV2FlashDecoderLayer(nn.Module): return self.config.hybrid_layer_pattern[self.layer_id] == 1 +def _requantize_fp8( + grouped: torch.Tensor, rows_rank: int, block: int, dtype: torch.dtype +) -> tuple[torch.Tensor, torch.Tensor]: + """Block-quantize a rank's ``[Q | K | V]`` rows back to fp8. + + A rank's rows need not end on a block boundary (a single 1856-row slice is + 14.5 blocks) while ``scaled_quantize`` requires both dims to be multiples of + the block size, so pad the tail with zeros: zero rows cannot raise a block's + amax, so every scale -- and the number of scale rows -- is unchanged. The + padding is dropped again here. + """ + padded = cdiv(rows_rank, block) * block + if padded != rows_rank: + grouped = torch.cat( + [grouped, grouped.new_zeros(padded - rows_rank, grouped.shape[1])], dim=0 + ) + w_rank, s_rank = scaled_quantize( + grouped, GroupShape(block, block), dtype, compute_dtype=torch.float32 + ) + return w_rank[:rows_rank], s_rank + + def _shard_fp8_qkv_proj( w_full: torch.Tensor, s_full: torch.Tensor, @@ -469,75 +492,136 @@ def _shard_fp8_qkv_proj( v_head_dim: int, tp_rank: int, tp_size: int, + ckpt_tp: int, block: int = 128, ) -> tuple[torch.Tensor, torch.Tensor]: """Shard the fp8 qkv_proj weights for ``tp_rank``. - The checkpoint stores the fused QKV as ``num_kv_heads`` contiguous groups - (one per KV head; ``n`` below), each ordered ``[Q | K | V]``: + The checkpoint stores the fused QKV pre-sharded for ``ckpt_tp`` ranks (the + model config's ``num_key_value_heads``), each chunk holding that slice's + Q, K and V rows: - [Q_1 | K_1 | V_1 | Q_2 | K_2 | V_2 | ... | Q_n | K_n | V_n] + [Q_0 | K_0 | V_0 | Q_1 | K_1 | V_1 | ... | Q_n | K_n | V_n] (n = ckpt_tp) - Per group, Q has ``(num_heads / num_kv_heads) * head_dim`` rows, K has - ``head_dim`` rows, and V has ``v_head_dim`` rows. + Per chunk, Q has ``(num_heads / ckpt_tp) * head_dim`` rows, K has + ``(num_kv_heads / ckpt_tp) * head_dim`` rows, and V has + ``(num_kv_heads / ckpt_tp) * v_head_dim`` rows, and the fp8 block scales + are tiled per chunk too (``ceil(rows_per_chunk / block)`` rows each). - Each TP rank owns ``g = num_kv_heads / tp_size`` of these groups, and the - forward expects them de-interleaved into a single Q, K, and V block: + ``ckpt_tp`` is not the layer's KV-head count: a MiMo-V2.5 SWA layer has 8 + KV heads over 4 chunks, so each chunk carries two KV heads (3712 rows = 29 + blocks) while a GA layer has 4 KV heads over 4 chunks (3392 rows each). + + The forward expects each rank's slice de-interleaved: [Q_1 | Q_2 | ... | Q_g | K_1 | K_2 | ... | K_g | V_1 | V_2 | ... | V_g] - When ``g == 1`` the rank's slice is already ``[Q | K | V]``, so a plain - chunk suffices. When ``g > 1`` we cannot reach the de-interleaved layout by - re-permuting the fp8 block scales: each scale covers a 128-row block, and - since K is 192 rows (1.5 blocks) a block straddles the K/V boundary, so no - whole-block permutation produces it. Instead we dequantize this rank's - groups to float (dropping the block constraint), reorder the rows into the - layout above (Q, K, and V then each span a whole number of blocks), and - re-quantize to fp8. + When ``tp_size == ckpt_tp`` the checkpoint chunk *is* that layout, so a + plain chunk of both weight and scale suffices. Otherwise each rank's Q, K + and V rows are gathered from the chunks that hold them, dequantized with + the chunk's own scales, reordered, and re-quantized to fp8. """ - assert tp_size <= num_kv_heads and num_kv_heads % tp_size == 0, ( - "TP size must evenly split the number of KV heads." + assert num_heads % tp_size == 0, ( + f"num_heads={num_heads} must be divisible by tp_size={tp_size}." ) - - kv_heads_per_rank = num_kv_heads // tp_size - if kv_heads_per_rank == 1: - # One KV head per rank. The weights and scale can be trivially sharded - # without re-quantization. - w = w_full.chunk(tp_size, dim=0)[tp_rank] - s = s_full.chunk(tp_size, dim=0)[tp_rank] - return w, s - - q_rows_per_group = (num_heads // num_kv_heads) * head_dim - k_rows_per_group = head_dim - v_rows_per_group = v_head_dim - rows_per_group = q_rows_per_group + k_rows_per_group + v_rows_per_group - scale_rows_per_group = s_full.shape[0] // num_kv_heads - qs, ks, vs = [], [], [] - for g_idx in range(tp_rank * kv_heads_per_rank, (tp_rank + 1) * kv_heads_per_rank): - row_start = g_idx * rows_per_group - scale_row_start = g_idx * scale_rows_per_group - # Dequantize this group's weights. - w_g = w_full[row_start : row_start + rows_per_group].to(torch.float32) - s_g = s_full[scale_row_start : scale_row_start + scale_rows_per_group].to( - torch.float32 + if ckpt_tp <= 0 or num_heads % ckpt_tp or num_kv_heads % ckpt_tp: + raise ValueError( + f"fused qkv_proj is pre-sharded at {ckpt_tp} chunks, which do not " + f"divide num_heads={num_heads} / num_kv_heads={num_kv_heads}." ) - s_g_expanded = s_g.repeat_interleave(block, dim=0).repeat_interleave( - block, dim=1 - )[:rows_per_group] - w_g_dequant = w_g * s_g_expanded - # Track the dequantized q, k, and v weights separately. - qs.append(w_g_dequant[:q_rows_per_group]) - ks.append(w_g_dequant[q_rows_per_group : q_rows_per_group + k_rows_per_group]) - vs.append(w_g_dequant[q_rows_per_group + k_rows_per_group :]) - - # Combine the q, k, and v weights into the following layout: - # [Q_1, Q_2, .., Q_g, K_1, K_2, ..., K_g, V_1, V_2, ..., V_g] - grouped = torch.cat([torch.cat(qs), torch.cat(ks), torch.cat(vs)], dim=0) - # Quantize back to fp8. - return scaled_quantize( - grouped, GroupShape(block, block), w_full.dtype, compute_dtype=torch.float32 + # When there are fewer KV heads than ranks, vLLM replicates them + # (`num_kv_head_replicas`) and rank r owns KV head r // replicas, which + # keeps every Q head grouped with the KV head it attends to. + if tp_size <= num_kv_heads: + assert num_kv_heads % tp_size == 0, ( + f"num_kv_heads={num_kv_heads} must be divisible by tp_size={tp_size}." + ) + kv_head_ids = list( + range( + tp_rank * (num_kv_heads // tp_size), + (tp_rank + 1) * (num_kv_heads // tp_size), + ) + ) + else: + assert tp_size % num_kv_heads == 0, ( + f"tp_size={tp_size} must be divisible by num_kv_heads={num_kv_heads}." + ) + kv_head_ids = [tp_rank // (tp_size // num_kv_heads)] + q_head_ids = list( + range(tp_rank * (num_heads // tp_size), (tp_rank + 1) * (num_heads // tp_size)) ) + rows_per_chunk = w_full.shape[0] // ckpt_tp + q_per_chunk = (num_heads // ckpt_tp) * head_dim + k_per_chunk = (num_kv_heads // ckpt_tp) * head_dim + v_per_chunk = (num_kv_heads // ckpt_tp) * v_head_dim + if q_per_chunk + k_per_chunk + v_per_chunk != rows_per_chunk: + raise ValueError( + f"fused qkv_proj has {w_full.shape[0]} rows, not {ckpt_tp} chunks of " + f"{q_per_chunk} Q + {k_per_chunk} K + {v_per_chunk} V rows." + ) + + # The scales are tiled per chunk; they collapse to one continuous grid when + # a chunk is a whole number of blocks (the SWA chunks are: 3712 = 29 * 128). + chunk_scale_rows = cdiv(rows_per_chunk, block) + rows = torch.arange(w_full.shape[0]) + per_chunk_scales = s_full.shape[0] == ckpt_tp * chunk_scale_rows + if per_chunk_scales: + scale_index = (rows // rows_per_chunk) * chunk_scale_rows + ( + rows % rows_per_chunk + ) // block + elif s_full.shape[0] == cdiv(w_full.shape[0], block): + scale_index = rows // block + else: + raise ValueError( + f"fused qkv_proj scale has {s_full.shape[0]} rows, expected either " + f"{ckpt_tp * chunk_scale_rows} ({ckpt_tp} chunks of " + f"{chunk_scale_rows} rows) or {cdiv(w_full.shape[0], block)} " + f"(one continuous grid)" + ) + + if tp_size == ckpt_tp and per_chunk_scales: + # One checkpoint chunk per rank: already [Q | K | V] for that rank, and + # its scale rows line up with ceil(rows_per_chunk / block). + return ( + w_full.chunk(ckpt_tp, dim=0)[tp_rank], + s_full.chunk(ckpt_tp, dim=0)[tp_rank], + ) + + # Gather this rank's Q, K and V rows from the chunks that hold them. + q_heads_per_chunk = num_heads // ckpt_tp + kv_heads_per_chunk = num_kv_heads // ckpt_tp + head_rows = torch.arange(head_dim) + v_head_rows = torch.arange(v_head_dim) + row_index: list[torch.Tensor] = [] + for head in q_head_ids: + chunk = head // q_heads_per_chunk + row_index.append( + chunk * rows_per_chunk + (head % q_heads_per_chunk) * head_dim + head_rows + ) + for head in kv_head_ids: + chunk = head // kv_heads_per_chunk + row_index.append( + chunk * rows_per_chunk + + q_per_chunk + + (head % kv_heads_per_chunk) * head_dim + + head_rows + ) + for head in kv_head_ids: + chunk = head // kv_heads_per_chunk + row_index.append( + chunk * rows_per_chunk + + q_per_chunk + + k_per_chunk + + (head % kv_heads_per_chunk) * v_head_dim + + v_head_rows + ) + index = torch.cat(row_index) + grouped = w_full[index].to(torch.float32) * s_full[ + scale_index[index] + ].repeat_interleave(block, dim=1) + return _requantize_fp8(grouped, index.numel(), block, w_full.dtype) + @support_torch_compile class MiMoV2Model(nn.Module, EagleModelMixin): @@ -819,6 +903,8 @@ class MiMoV2Model(nn.Module, EagleModelMixin): v_head_dim=attn.v_head_dim, tp_rank=tp_rank, tp_size=tp_size, + # The fused qkv_proj is pre-sharded for this many ranks. + ckpt_tp=self.config.num_key_value_heads, ) sharded = {"weight": w_rank, "weight_scale_inv": s_rank} for kind, tensor in sharded.items(): diff --git a/vllm/model_executor/models/mimo_v2_mtp.py b/vllm/model_executor/models/mimo_v2_mtp.py index b08ba229f7..6fd8309350 100644 --- a/vllm/model_executor/models/mimo_v2_mtp.py +++ b/vllm/model_executor/models/mimo_v2_mtp.py @@ -45,7 +45,7 @@ from .interfaces import ( SupportsMultiModal, _require_is_multimodal, ) -from .mimo_v2 import MiMoV2Attention, MiMoV2MLP +from .mimo_v2 import MiMoV2Attention, MiMoV2MLP, _shard_fp8_qkv_proj from .utils import _merge_multimodal_embeddings, maybe_prefix # MiMo-V2 checkpoints contain multiple MTP layers, but vLLM currently supports @@ -266,6 +266,7 @@ class MiMoV2MTP(nn.Module): params_dict = dict(self.named_parameters()) loaded_params: set[str] = set() + pending_qkv_proj: dict[str, dict[str, torch.Tensor]] = {} for name, loaded_weight in weights: if "rotary_emb.inv_freq" in name: @@ -279,14 +280,54 @@ class MiMoV2MTP(nn.Module): ): continue - # Support fused qkv_proj checkpoint (Pro format). - # The checkpoint is stored pre-sharded for TP=8 as - # [Q_rank0, K_rank0, V_rank0, Q_rank1, ...], so splitting along - # dim 0 with chunk(tp_size) gives each rank its Q+K+V slice for - # both the FP8 weight and the block weight_scale_inv. This matches - # how the main model loads the same layout. + # Fused qkv_proj: the fp8 weight and its block scales arrive as + # separate tensors and are pre-sharded at `config.num_key_value_heads` + # chunks of [Q_c | K_c | V_c] (see _shard_fp8_qkv_proj for the + # layout). Shard them together: a plain chunk(tp_size) only happens + # to be right when tp_size is that chunk count *and* the scales are + # tiled per chunk (they are 4 x 29 = 116 rows for MiMo-V2.5, not + # tp_size slices of an even split). if "qkv_proj" in name: - if name in params_dict: + is_fp8_weight = ( + name.endswith("qkv_proj.weight") + and loaded_weight.dtype == torch.float8_e4m3fn + ) + is_fp8_scale = name.endswith("qkv_proj.weight_scale_inv") + if is_fp8_weight or is_fp8_scale: + prefix, kind = name.rsplit(".", 1) + if ( + f"{prefix}.weight" not in params_dict + or f"{prefix}.weight_scale_inv" not in params_dict + ): + continue + entry = pending_qkv_proj.setdefault(prefix, {}) + entry[kind] = loaded_weight + if "weight" not in entry or "weight_scale_inv" not in entry: + # Waiting for the other half of the fused projection. + continue + del pending_qkv_proj[prefix] + + # Geometry comes from the module the weights belong to. + attn = self.get_submodule(prefix.rsplit(".", 1)[0]) + w_rank, s_rank = _shard_fp8_qkv_proj( + entry["weight"], + entry["weight_scale_inv"], + num_heads=attn.total_num_heads, + num_kv_heads=attn.total_num_kv_heads, + head_dim=attn.head_dim, + v_head_dim=attn.v_head_dim, + tp_rank=tp_rank, + tp_size=tp_size, + ckpt_tp=self.config.num_key_value_heads, + ) + for k, tensor in (("weight", w_rank), ("weight_scale_inv", s_rank)): + param = params_dict[f"{prefix}.{k}"] + if tensor.shape[0] > param.shape[0]: + tensor = tensor[: param.shape[0]] + default_weight_loader(param, tensor) + loaded_params.add(f"{prefix}.{k}") + elif name in params_dict: + # Non-fp8 fused checkpoint: unchanged behaviour. param = params_dict[name] loaded_weight = loaded_weight.chunk(tp_size, dim=0)[tp_rank] default_weight_loader(param, loaded_weight) diff --git a/vllm/model_executor/models/mimo_v2_omni.py b/vllm/model_executor/models/mimo_v2_omni.py index d74edd6d4a..91a3177b7f 100644 --- a/vllm/model_executor/models/mimo_v2_omni.py +++ b/vllm/model_executor/models/mimo_v2_omni.py @@ -15,11 +15,7 @@ from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize from typing_extensions import TypedDict from vllm.config import VllmConfig -from vllm.config.multimodal import ( - BaseDummyOptions, - ImageDummyOptions, - VideoDummyOptions, -) +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import parallel_state from vllm.distributed import utils as dist_utils from vllm.inputs import MultiModalDataDict @@ -1184,33 +1180,26 @@ class MiMoV2OmniDummyInputsBuilder(BaseDummyInputsBuilder[MiMoV2OmniProcessingIn self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - num_videos = mm_counts.get("video", 0) - target_width, target_height = self.info.get_image_size_with_most_features() target_num_frames = self.info.get_num_frames_with_most_features( seq_len, mm_counts ) - image_overrides = mm_options.get("image") - video_overrides = mm_options.get("video") - assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions) - assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions) return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), "video": self._get_dummy_videos( width=target_width, height=target_height, num_frames=target_num_frames, - num_videos=num_videos, - overrides=video_overrides, + num_videos=mm_counts.get("video", 0), + overrides=mm_options.get("video"), ), } diff --git a/vllm/model_executor/models/minicpmo.py b/vllm/model_executor/models/minicpmo.py index 2ec8567921..cc8a1066f2 100644 --- a/vllm/model_executor/models/minicpmo.py +++ b/vllm/model_executor/models/minicpmo.py @@ -40,7 +40,7 @@ from transformers.models.whisper.modeling_whisper import ( ) from vllm.config import VllmConfig -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import ModalityData, MultiModalDataDict from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems from vllm.multimodal.inputs import ( @@ -402,22 +402,18 @@ class MiniCPMODummyInputsBuilder(MiniCPMVDummyInputsBuilder[MiniCPMOProcessingIn self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_audios = mm_counts.get("audio", 0) audio_len = ( self.info.get_max_audio_chunks_with_most_features() * self.info.get_default_audio_sampling_rate() ) - audio_overrides = mm_options.get("audio") - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) - audio_mm_data = { "audio": self._get_dummy_audios( length=audio_len, - num_audios=num_audios, - overrides=audio_overrides, + num_audios=mm_counts.get("audio", 0), + overrides=mm_options.get("audio"), ) } diff --git a/vllm/model_executor/models/minicpmv.py b/vllm/model_executor/models/minicpmv.py index 14720c5c86..8027dc1027 100644 --- a/vllm/model_executor/models/minicpmv.py +++ b/vllm/model_executor/models/minicpmv.py @@ -45,9 +45,8 @@ from typing_extensions import TypedDict, TypeVar from vllm.config import VllmConfig from vllm.config.multimodal import ( - BaseDummyOptions, ImageDummyOptions, - VideoDummyOptions, + MultiModalDummyOptions, ) from vllm.inputs import ModalityData, MultiModalDataDict from vllm.model_executor.layers.quantization import QuantizationConfig @@ -848,9 +847,8 @@ class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) num_videos = mm_counts.get("video", 0) image_width, image_height = self.info.get_image_size_with_most_features() @@ -859,14 +857,12 @@ class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]): seq_len, mm_counts ) - image_overrides = mm_options.get("image") video_overrides = mm_options.get("video") - assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions) # Convert video overrides to image overrides for per-frame image generation, # and apply num_frames override to num_video_frames. video_frame_overrides: ImageDummyOptions | None = None - if isinstance(video_overrides, VideoDummyOptions): + if video_overrides is not None: if video_overrides.num_frames: num_video_frames = min(num_video_frames, video_overrides.num_frames) if video_overrides.width or video_overrides.height: @@ -880,8 +876,8 @@ class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]): "image": self._get_dummy_images( width=image_width, height=image_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), "video": [ self._get_dummy_images( diff --git a/vllm/model_executor/models/mistral3.py b/vllm/model_executor/models/mistral3.py index f9d7dafe70..41352dc427 100644 --- a/vllm/model_executor/models/mistral3.py +++ b/vllm/model_executor/models/mistral3.py @@ -11,7 +11,7 @@ from transformers import BatchFeature, Mistral3Config, PixtralVisionConfig from transformers.models.pixtral import PixtralProcessor from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.activation import get_act_fn from vllm.model_executor.layers.layernorm import RMSNorm @@ -261,20 +261,16 @@ class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - target_width, target_height = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") - return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/mllama4.py b/vllm/model_executor/models/mllama4.py index 797c06d180..89d4edbe86 100644 --- a/vllm/model_executor/models/mllama4.py +++ b/vllm/model_executor/models/mllama4.py @@ -36,7 +36,7 @@ from vllm.compilation.decorators import ( support_torch_compile, ) from vllm.config import VllmConfig, set_current_vllm_config -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import get_tensor_model_parallel_world_size from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.attention import MMEncoderAttention @@ -709,20 +709,16 @@ class Mllama4DummyInputsBuilder(BaseDummyInputsBuilder[Mllama4ProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - (target_width, target_height) = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") - return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/molmo.py b/vllm/model_executor/models/molmo.py index 9aeef3715c..17ad7f0be1 100644 --- a/vllm/model_executor/models/molmo.py +++ b/vllm/model_executor/models/molmo.py @@ -21,7 +21,7 @@ from transformers import ( from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import ( get_pp_group, get_tensor_model_parallel_rank, @@ -1104,19 +1104,16 @@ class MolmoDummyInputsBuilder(BaseDummyInputsBuilder[MolmoProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: target_width, target_height = self.info.get_image_size_with_most_features() - num_images = mm_counts.get("image", 0) - - image_overrides = mm_options.get("image") return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } diff --git a/vllm/model_executor/models/molmo2.py b/vllm/model_executor/models/molmo2.py index 4c0ce0868e..881be146e7 100644 --- a/vllm/model_executor/models/molmo2.py +++ b/vllm/model_executor/models/molmo2.py @@ -26,7 +26,7 @@ from typing_extensions import TypedDict from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig -from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions, VideoDummyOptions from vllm.distributed import ( get_pp_group, get_tensor_model_parallel_rank, @@ -1834,7 +1834,7 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: num_images = mm_counts.get("image", 0) num_videos = mm_counts.get("video", 0) @@ -1845,13 +1845,11 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]): if num_images > 0: target_width, target_height = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") - dummy_images = self._get_dummy_images( width=target_width, height=target_height, num_images=num_images, - overrides=image_overrides, + overrides=mm_options.get("image"), ) if num_videos > 0: @@ -1864,7 +1862,6 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]): video_overrides = mm_options.get("video") if video_overrides: - assert isinstance(video_overrides, VideoDummyOptions) num_frames_override = video_overrides.num_frames if num_frames_override: if num_frames_override > target_num_frames: diff --git a/vllm/model_executor/models/moondream3.py b/vllm/model_executor/models/moondream3.py index 45b9420184..1e750fc9d0 100644 --- a/vllm/model_executor/models/moondream3.py +++ b/vllm/model_executor/models/moondream3.py @@ -13,7 +13,7 @@ import torch.nn.functional as F from transformers import BatchFeature from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import ( get_pp_group, get_tensor_model_parallel_rank, @@ -943,15 +943,14 @@ class Moondream3DummyInputsBuilder(BaseDummyInputsBuilder[Moondream3ProcessingIn self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions] | None = None, + mm_options: MultiModalDummyOptions | None = None, mm_processor_kwargs: Mapping[str, object] | None = None, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) return { "image": self._get_dummy_images( width=378, height=378, - num_images=num_images, + num_images=mm_counts.get("image", 0), ) } diff --git a/vllm/model_executor/models/moss_audio.py b/vllm/model_executor/models/moss_audio.py index 6092812585..02551d5663 100644 --- a/vllm/model_executor/models/moss_audio.py +++ b/vllm/model_executor/models/moss_audio.py @@ -19,7 +19,7 @@ from typing_extensions import TypedDict from vllm.compilation.decorators import support_torch_compile from vllm.config import VllmConfig -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import ( get_pp_group, get_tensor_model_parallel_world_size, @@ -1341,16 +1341,13 @@ class MossAudioDummyInputsBuilder(BaseDummyInputsBuilder[MossAudioProcessingInfo self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_audios = mm_counts.get("audio", 0) - audio_overrides = mm_options.get("audio") - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) return { "audio": self._get_dummy_audios( length=16000, - num_audios=num_audios, - overrides=audio_overrides, + num_audios=mm_counts.get("audio", 0), + overrides=mm_options.get("audio"), ) } diff --git a/vllm/model_executor/models/moss_transcribe_diarize.py b/vllm/model_executor/models/moss_transcribe_diarize.py index 32ac69c9af..208d5d4bc1 100644 --- a/vllm/model_executor/models/moss_transcribe_diarize.py +++ b/vllm/model_executor/models/moss_transcribe_diarize.py @@ -20,7 +20,7 @@ from torch import nn from transformers import BatchFeature from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.config.speech_to_text import SpeechToTextParams from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TextPrompt from vllm.model_executor.models.interfaces import ( @@ -423,20 +423,18 @@ class MossTranscribeDiarizeDummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: num_audios = mm_counts.get("audio", 0) if num_audios == 0: return {} feature_extractor = self.info.get_feature_extractor() - audio_overrides = mm_options.get("audio") - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) return { "audio": self._get_dummy_audios( length=_get_max_audio_samples(feature_extractor), num_audios=num_audios, - overrides=audio_overrides, + overrides=mm_options.get("audio"), ) } @@ -444,7 +442,7 @@ class MossTranscribeDiarizeDummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> ProcessorInputs: dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options) dummy_mm_items = self.info.parse_mm_data(dummy_mm_data) diff --git a/vllm/model_executor/models/muse_glimmer.py b/vllm/model_executor/models/muse_glimmer.py index ed8064542b..a7938bb56b 100644 --- a/vllm/model_executor/models/muse_glimmer.py +++ b/vllm/model_executor/models/muse_glimmer.py @@ -45,11 +45,7 @@ from transformers import BatchFeature from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig -from vllm.config.multimodal import ( - BaseDummyOptions, - ImageDummyOptions, - VideoDummyOptions, -) +from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import ( divide, get_pp_group, @@ -234,7 +230,7 @@ class MuseGlimmerDummyInputsBuilder(BaseDummyInputsBuilder[MuseGlimmerProcessing self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: processor = self.info.get_hf_processor() video_processor = processor.video_processor @@ -245,16 +241,12 @@ class MuseGlimmerDummyInputsBuilder(BaseDummyInputsBuilder[MuseGlimmerProcessing * int(video_processor.downsample_factor) * video_grid ) - image_overrides = mm_options.get("image") - video_overrides = mm_options.get("video") - assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions) - assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions) return { "image": self._get_dummy_images( width=image_width, height=image_height, num_images=mm_counts.get("image", 0), - overrides=image_overrides, + overrides=mm_options.get("image"), ), "video": self._get_dummy_videos( width=video_size, @@ -263,7 +255,7 @@ class MuseGlimmerDummyInputsBuilder(BaseDummyInputsBuilder[MuseGlimmerProcessing seq_len, mm_counts ), num_videos=mm_counts.get("video", 0), - overrides=video_overrides, + overrides=mm_options.get("video"), ), } diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py index 6e500c0c2c..de61880b5f 100644 --- a/vllm/model_executor/models/nano_nemotron_vl.py +++ b/vllm/model_executor/models/nano_nemotron_vl.py @@ -12,7 +12,7 @@ import warnings from collections.abc import Iterable, Mapping, Sequence from functools import cached_property from io import BytesIO -from typing import Annotated, Literal, TypeAlias +from typing import Annotated, Literal, TypeAlias, TypedDict, cast import torch import torch.nn as nn @@ -20,7 +20,10 @@ from transformers import BatchFeature, PretrainedConfig from vllm import envs from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions +from vllm.config.multimodal import ( + MultiModalDummyOptions, + VideoDummyOptions, +) from vllm.inputs import MultiModalDataDict, MultiModalInput from vllm.logger import init_logger from vllm.lora.layers.base import BaseLayerWithLoRA @@ -114,7 +117,7 @@ class NanoNemotronVLAudioFeatureInputs(TensorSchema): type: Literal["audio_features"] = "audio_features" input_audio_features: Annotated[torch.Tensor, TensorShape("c", "t", "f")] feature_attention_mask: Annotated[torch.Tensor, TensorShape("c", "t")] - audio_num_clips: list[int] + audio_num_clips: list[int] | torch.Tensor class NanoNemotronVLImagePixelInputs(TensorSchema): @@ -195,6 +198,12 @@ NanoNemotronVLVideoInputs: TypeAlias = ( ) +class NanoNemotronVLMultiModalInputs(TypedDict, total=False): + images: NanoNemotronVLImageInputs | None + videos: NanoNemotronVLVideoInputs | None + audios: NanoNemotronVLAudioFeatureInputs | None + + class NanoNemotronVLProcessingInfo(BaseProcessingInfo): def get_hf_processor(self, **kwargs: object) -> NanoNemotronVLProcessor: # Not accepted by NanoNemotronVLProcessor.__init__ @@ -465,8 +474,11 @@ class NanoNemotronVLMultiModalProcessor( if isinstance(images, ImageEmbeddingItems): feature_size = images.get_feature_size(item_idx) elif self.info.is_dynamic_tiler: - feature_size = out_mm_data["num_tokens_per_image"][item_idx] + num_tokens_per_image = out_mm_data["num_tokens_per_image"] + assert isinstance(num_tokens_per_image, list) + feature_size = num_tokens_per_image[item_idx] else: + assert isinstance(images, ImageProcessorItems) image_size = images.get_image_size(item_idx) max_num_tiles = hf_processor.max_num_tiles feature_size = hf_processor.get_num_image_tokens( @@ -530,10 +542,12 @@ class NanoNemotronVLMultiModalProcessor( assert isinstance(num_patches, int) T = hf_processor.video_temporal_patch_size + num_tubelets: int | None if T > 1 and num_patches is not None: num_tubelets = math.ceil(num_patches / T) else: num_tubelets = num_patches + assert num_tubelets is not None video_pruning_rate = self.info.ctx.get_mm_config().video_pruning_rate if video_pruning_rate is not None and video_pruning_rate > 0.0: @@ -635,6 +649,7 @@ class NanoNemotronVLMultiModalProcessor( audio_items: list[AudioItem] = [] has_audio: list[bool] = [] for idx, metadata in enumerate(metadata_list): + assert metadata is not None video_bytes = metadata.get("original_video_bytes") if video_bytes is None or len(video_bytes) == 0: raise ValueError( @@ -661,21 +676,25 @@ class NanoNemotronVLMultiModalProcessor( # Create a new VideoProcessorItems with metadata that does not contain # the large video bytes, to avoid modifying the input `mm_items`. - new_metadata_list = [ - {k: v for k, v in meta.items() if k != "original_video_bytes"} - for meta in metadata_list - ] + new_metadata_list = metadata_list.copy() + for idx, metadata in enumerate(metadata_list): + assert metadata is not None + new_metadata_list[idx] = { + key: value + for key, value in metadata.items() + if key != "original_video_bytes" + } new_videos = VideoProcessorItems(data=videos.data, metadata=new_metadata_list) - audio_parsed = {} + audio_parsed = MultiModalDataItems() if audio_items: audio_parsed = self.data_parser.parse_mm_data({"audio": audio_items}) # Create a new MultiModalDataItems with the new video and audio items. new_mm_items_dict = {**mm_items, **audio_parsed, "video": new_videos} - mm_items = MultiModalDataItems(new_mm_items_dict) + new_mm_items = MultiModalDataItems(new_mm_items_dict) - return mm_items, audio_items, has_audio + return new_mm_items, audio_items, has_audio def apply( self, @@ -800,7 +819,7 @@ class NanoNemotronVLDummyInputsBuilder( ) videos = [v.copy() for v in videos] - video_items = [] + video_items: list[VideoItem] = [] for video in videos: video_num_frames = video.shape[0] video_metadata = { @@ -819,22 +838,19 @@ class NanoNemotronVLDummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) (target_width, target_height), _ = ( self.info.get_dummy_image_size_and_max_tokens(mm_counts) ) processor = self.info.get_hf_processor() - image_overrides = mm_options.get("image") - dummy_image = { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } @@ -946,6 +962,7 @@ class NemotronH_Nano_VL_V2( model_config = vllm_config.model_config config = model_config.hf_config multimodal_config = model_config.multimodal_config + assert multimodal_config is not None image_size = config.force_image_size patch_size = config.patch_size self.patch_size = patch_size @@ -1138,17 +1155,26 @@ class NemotronH_Nano_VL_V2( return None if self.dynamic_resolution: + assert isinstance(pixel_values_flat, (list, torch.Tensor)) pixel_values_flat = DynamicResolutionImageTiler.stack( pixel_values_flat, self.patch_size ) + imgs_sizes = kwargs.pop("imgs_sizes") + num_tokens_per_image = kwargs.pop("num_tokens_per_image") + assert isinstance(imgs_sizes, list) + assert isinstance(num_tokens_per_image, list) return NanoNemotronVLImagePixelInputsDynamic( - pixel_values_flat=pixel_values_flat, **kwargs + pixel_values_flat=pixel_values_flat, + imgs_sizes=imgs_sizes, + num_tokens_per_image=num_tokens_per_image, ) else: + assert isinstance(pixel_values_flat, torch.Tensor) + image_num_patches = kwargs.pop("image_num_patches") + assert isinstance(image_num_patches, torch.Tensor) return NanoNemotronVLImagePixelInputs( pixel_values_flat=pixel_values_flat, - num_patches=kwargs.pop("image_num_patches"), - **kwargs, + num_patches=image_num_patches, ) def _process_image_input_dynamic( @@ -1166,7 +1192,8 @@ class NemotronH_Nano_VL_V2( return image_embeds.split(num_tokens_per_image) def _process_image_input( - self, image_input: NanoNemotronVLImagePixelInputs + self, + image_input: NanoNemotronVLImagePixelInputs | NanoNemotronVLVideoPixelInputs, ) -> tuple[torch.Tensor, ...]: image_embeds = self.extract_feature(image_input["pixel_values_flat"]) num_patches = image_input["num_patches"] @@ -1362,7 +1389,8 @@ class NemotronH_Nano_VL_V2( # of them than max_num_batched_tokens. # Embed them with the base weights - # the LoRA delta is undefined for tokens with no mapping entry. - embed_tokens = self.get_language_model().model.embed_tokens + language_model = cast(NemotronHForCausalLM, self.get_language_model()) + embed_tokens = language_model.model.embed_tokens if isinstance(embed_tokens, BaseLayerWithLoRA): embed_tokens = embed_tokens.base_layer text_embeddings = embed_tokens(repl_token_ids) @@ -1388,26 +1416,29 @@ class NemotronH_Nano_VL_V2( return None if video_embeds is not None: + assert isinstance(video_embeds, (torch.Tensor, list)) return NanoNemotronVLVideoEmbeddingInputs( type="video_embeds", data=video_embeds, ) if pixel_values_flat_video is not None: - if torch.is_tensor(frames_indices): + if isinstance(frames_indices, torch.Tensor): frames_indices = frames_indices.flatten() else: + assert isinstance(frames_indices, (list, tuple)) frames_indices = torch.cat([f.flatten() for f in frames_indices], dim=0) - if torch.is_tensor(frame_duration_ms): + if isinstance(frame_duration_ms, torch.Tensor): frame_duration_ms = frame_duration_ms.flatten() else: + assert isinstance(frame_duration_ms, (list, tuple)) frame_duration_ms = torch.cat( [f.flatten() for f in frame_duration_ms], dim=0 ) if ( - torch.is_tensor(pixel_values_flat_video) + isinstance(pixel_values_flat_video, torch.Tensor) and pixel_values_flat_video.ndim == 5 ): # batched._reduce_data stacked same-shape videos into @@ -1415,9 +1446,11 @@ class NemotronH_Nano_VL_V2( # same-H,W cat path below handles it uniformly. pixel_values_flat_video = list(pixel_values_flat_video) - if not torch.is_tensor(pixel_values_flat_video): + if not isinstance(pixel_values_flat_video, torch.Tensor): + assert isinstance(pixel_values_flat_video, (list, tuple)) pixel_values_flat_video = torch.cat(pixel_values_flat_video, dim=0) + assert isinstance(video_num_patches, torch.Tensor) expected_h = pixel_values_flat_video.shape[-2] expected_w = pixel_values_flat_video.shape[-1] num_frames = video_num_patches[0].item() @@ -1434,8 +1467,10 @@ class NemotronH_Nano_VL_V2( raise AssertionError("This line should be unreachable.") - def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: - modalities = {} + def _parse_and_validate_multimodal_inputs( + self, **kwargs: object + ) -> NanoNemotronVLMultiModalInputs: + modalities: NanoNemotronVLMultiModalInputs = {} # Preserve the order of modalities if there are multiple of them # from the order of kwargs. for input_key in kwargs: @@ -1458,8 +1493,17 @@ class NemotronH_Nano_VL_V2( ) and "audios" not in modalities ): + input_audio_features = kwargs.get("input_audio_features") + feature_attention_mask = kwargs.get("feature_attention_mask") + audio_num_clips = kwargs.get("audio_num_clips") + assert isinstance(input_audio_features, torch.Tensor) + assert isinstance(feature_attention_mask, torch.Tensor) + assert isinstance(audio_num_clips, (list, torch.Tensor)) modalities["audios"] = NanoNemotronVLAudioFeatureInputs( - **kwargs, validate=False + input_audio_features=input_audio_features, + feature_attention_mask=feature_attention_mask, + audio_num_clips=audio_num_clips, + validate=False, ) return modalities @@ -1479,23 +1523,26 @@ class NemotronH_Nano_VL_V2( for modality in modalities: if modality == "images": image_input = modalities["images"] - if image_input["type"] == "image_embeds": + assert image_input is not None + if isinstance(image_input, NanoNemotronVLImageEmbeddingInputs): image_embeddings = image_input["data"] - elif self.dynamic_resolution: - assert image_input["type"] == "pixel_values_dynamic" + elif isinstance(image_input, NanoNemotronVLImagePixelInputsDynamic): image_embeddings = self._process_image_input_dynamic(image_input) else: image_embeddings = self._process_image_input(image_input) multimodal_embeddings += tuple(image_embeddings) if modality == "videos": video_input = modalities["videos"] + assert video_input is not None if video_input["type"] == "video_embeds": video_embeddings = video_input["data"] else: + assert isinstance(video_input, NanoNemotronVLVideoPixelInputs) video_embeddings = self._process_video_input(video_input) multimodal_embeddings += tuple(video_embeddings) if modality == "audios": audio_input = modalities["audios"] + assert audio_input is not None audio_embeddings = self._process_audio_input(audio_input) multimodal_embeddings += tuple(audio_embeddings) @@ -1538,6 +1585,7 @@ class NemotronH_Nano_VL_V2( def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): mm_config = self.model_config.multimodal_config + assert mm_config is not None load_multimodal_weights = not all( mm_config.get_limit_per_prompt(modality) == 0 for modality in ("image", "video", "audio") diff --git a/vllm/model_executor/models/nemotron_h.py b/vllm/model_executor/models/nemotron_h.py index e71b7d8e6c..e5451d16ba 100644 --- a/vllm/model_executor/models/nemotron_h.py +++ b/vllm/model_executor/models/nemotron_h.py @@ -68,6 +68,7 @@ from vllm.model_executor.models.interfaces import ( EagleModelMixin, HasInnerState, IsHybrid, + MambaStateShapes, MixtureOfExperts, SupportsEagle, SupportsEagle3, @@ -138,6 +139,7 @@ class NemotronHMoE(nn.Module): prefix: str = "", ): super().__init__() + assert parallel_config is not None self.tp_size = get_tensor_model_parallel_world_size() self.routed_scaling_factor = config.routed_scaling_factor @@ -188,6 +190,8 @@ class NemotronHMoE(nn.Module): prefix=f"{prefix}.shared_experts", ) + self.fc1_latent_proj: ReplicatedLinear | None + self.fc2_latent_proj: ReplicatedLinear | None if self.use_latent_moe: self.fc1_latent_proj = ReplicatedLinear( input_size=config.hidden_size, @@ -722,7 +726,7 @@ class NemotronHForCausalLM( SupportsReplaySSM, ): # Relevant only if self.has_moe is True - is_non_gated_moe: bool = True + is_non_gated_moe = True hf_to_vllm_mapper = WeightsMapper( orig_to_new_prefix={"backbone": "model", "mtp": None}, @@ -773,7 +777,7 @@ class NemotronHForCausalLM( def get_mamba_state_shape_from_config( cls, vllm_config: "VllmConfig", - ) -> tuple[tuple[int, ...], ...]: + ) -> MambaStateShapes: """Calculate shapes for Mamba's convolutional and state caches. Args: @@ -858,6 +862,7 @@ class NemotronHForCausalLM( self.moe_layers.append(layer.mixer.experts) self.num_moe_layers = len(self.moe_layers) + assert example_moe is not None self.num_logical_experts = example_moe.n_logical_experts self.num_physical_experts = example_moe.n_physical_experts self.num_local_physical_experts = example_moe.n_local_physical_experts # noqa: E501 diff --git a/vllm/model_executor/models/nemotron_h_mtp.py b/vllm/model_executor/models/nemotron_h_mtp.py index 96f1b6685e..6b9de63009 100644 --- a/vllm/model_executor/models/nemotron_h_mtp.py +++ b/vllm/model_executor/models/nemotron_h_mtp.py @@ -92,10 +92,12 @@ class NemotronHMTPAttentionDecoderLayer(NemotronHAttentionDecoderLayer): def forward( self, - inputs_embeds: torch.Tensor, positions: torch.Tensor, hidden_states: torch.Tensor, residual: torch.Tensor | None = None, + *, + inputs_embeds: torch.Tensor | None = None, + **kwargs: object, ) -> tuple[torch.Tensor, torch.Tensor | None]: # Start projections (Fusion) if self.has_start_projections: @@ -179,10 +181,11 @@ class NemotronHMTPMoEDecoderLayer(NemotronHMoEDecoderLayer): def forward( self, - inputs_embeds: torch.Tensor, - positions: torch.Tensor, hidden_states: torch.Tensor, residual: torch.Tensor | None = None, + *, + inputs_embeds: torch.Tensor | None = None, + **kwargs: object, ) -> tuple[torch.Tensor, torch.Tensor | None]: # Start projections (Fusion) if self.has_start_projections: @@ -305,13 +308,14 @@ class NemotronHMultiTokenPredictor(nn.Module): def forward( self, - input_ids: torch.Tensor, + input_ids: torch.Tensor | None, positions: torch.Tensor, hidden_states: torch.Tensor, intermediate_tensors: IntermediateTensors | None = None, inputs_embeds: torch.Tensor | None = None, ) -> torch.Tensor | IntermediateTensors: if inputs_embeds is None: + assert input_ids is not None inputs_embeds = self.get_input_embeddings(input_ids) residual = None @@ -386,6 +390,7 @@ class NemotronHMTP(nn.Module, SupportsPP, SupportsQuant): @staticmethod def _find_quant_config(*args, **kwargs) -> QuantizationConfig | None: vllm_config = kwargs.get("vllm_config") + assert isinstance(vllm_config, VllmConfig) return get_draft_quant_config(vllm_config) def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: @@ -393,14 +398,15 @@ class NemotronHMTP(nn.Module, SupportsPP, SupportsQuant): def forward( self, - input_ids: torch.Tensor, + input_ids: torch.Tensor | None, positions: torch.Tensor, - hidden_states: torch.Tensor, + hidden_states: torch.Tensor | None = None, intermediate_tensors: IntermediateTensors | None = None, inputs_embeds: torch.Tensor | None = None, **kwargs: object, ) -> torch.Tensor: """Forward - applies attention-based MTP.""" + assert hidden_states is not None hidden_states = self.model( input_ids, positions, @@ -473,9 +479,10 @@ class NemotronHMTP(nn.Module, SupportsPP, SupportsQuant): name = name.replace("backbone.", "model.") if "scale" in name or "zero_point" in name: - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: + remapped_name = maybe_remap_kv_scale_name(name, params_dict) + if remapped_name is None: continue + name = remapped_name # Handle stacked parameters (qkv_proj) for attention layers is_stacked = False @@ -547,6 +554,7 @@ class NemotronHMTP(nn.Module, SupportsPP, SupportsQuant): param = params_dict[name] weight_loader = getattr(param, "weight_loader", default_weight_loader) + assert weight_loader is not None weight_loader(param, loaded_weight) loaded_params.add(name) diff --git a/vllm/model_executor/models/nemotron_parse.py b/vllm/model_executor/models/nemotron_parse.py index 01b5578531..5c4375b600 100644 --- a/vllm/model_executor/models/nemotron_parse.py +++ b/vllm/model_executor/models/nemotron_parse.py @@ -22,7 +22,7 @@ from transformers import ( from vllm.config import CacheConfig, VllmConfig from vllm.config.lora import LoRAConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.logger import init_logger from vllm.model_executor.layers.activation import get_act_fn @@ -388,7 +388,7 @@ class NemotronParseDummyInputsBuilder( self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: num_images = mm_counts.get("image", 0) diff --git a/vllm/model_executor/models/nemotron_vl.py b/vllm/model_executor/models/nemotron_vl.py index bf05848858..7b2777a710 100644 --- a/vllm/model_executor/models/nemotron_vl.py +++ b/vllm/model_executor/models/nemotron_vl.py @@ -138,7 +138,7 @@ class LlamaNemotronVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, Suppor prefix=maybe_prefix(prefix, "language_model"), ) - self.img_context_token_id = None + self.img_context_token_id: int | None = None self.visual_token_mask = None self.make_empty_intermediate_tensors = ( @@ -146,7 +146,7 @@ class LlamaNemotronVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, Suppor ) def _patch_quant_config( - self, config: PretrainedConfig, quant_config: QuantizationConfig + self, config: PretrainedConfig, quant_config: QuantizationConfig | None ): # the awq models from OpenGVLab missing `modules_to_not_convert` # patch the quant_config to add `modules_to_not_convert` back @@ -518,12 +518,21 @@ class LlamaNemotronVLForEmbedding(LlamaNemotronVLChatModel, VllmModelForPooling) use_head=False, ) - def _init_mlp1(self, config: PretrainedConfig) -> nn.Module: + def _init_mlp1( + self, + config: PretrainedConfig, + vit_hidden_size: int | None = None, + vision_projection_hidden_size: int | None = None, + ) -> nn.Module: """Override to use different MLP structure for embedding model.""" + if vit_hidden_size is None: + vit_hidden_size = config.vision_config.hidden_size + if vision_projection_hidden_size is None: + vision_projection_hidden_size = config.get_text_config().hidden_size return super()._init_mlp1( config, - vit_hidden_size=config.vision_config.hidden_size, - vision_projection_hidden_size=config.get_text_config().hidden_size, + vit_hidden_size=vit_hidden_size, + vision_projection_hidden_size=vision_projection_hidden_size, ) def _call_vision_model(self, pixel_values: torch.Tensor) -> torch.Tensor: diff --git a/vllm/model_executor/models/nvlm_d.py b/vllm/model_executor/models/nvlm_d.py index d937fade1d..5efcdb6f53 100644 --- a/vllm/model_executor/models/nvlm_d.py +++ b/vllm/model_executor/models/nvlm_d.py @@ -13,7 +13,7 @@ import torch import torch.nn as nn from transformers import PretrainedConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.multimodal import MULTIMODAL_REGISTRY @@ -83,19 +83,16 @@ class NVLMDummyInputsBuilder(BaseInternVLDummyInputsBuilder[NVLMProcessingInfo]) self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: target_width, target_height = self.info.get_image_size_with_most_features() - num_images = mm_counts.get("image", 0) - - image_overrides = mm_options.get("image") return { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } @@ -128,6 +125,7 @@ class NVLMMultiModalProcessor(BaseInternVLMultiModalProcessor[NVLMProcessingInfo if isinstance(images, ImageEmbeddingItems): feature_size = images.get_feature_size(item_idx) else: + assert isinstance(images, ImageProcessorItems) image_size = images.get_image_size(item_idx) feature_size = self.info.get_num_image_tokens( image_width=image_size.width, diff --git a/vllm/model_executor/models/olmo_hybrid.py b/vllm/model_executor/models/olmo_hybrid.py index 9a24df5ec9..e32087ea21 100644 --- a/vllm/model_executor/models/olmo_hybrid.py +++ b/vllm/model_executor/models/olmo_hybrid.py @@ -143,6 +143,7 @@ class OlmoHybridAttention(nn.Module): rope_parameters["rope_theta"] is not None ) + self.rotary_emb: nn.Module | None if self._use_rope: self.rotary_emb = get_rope( self.head_dim, @@ -183,6 +184,7 @@ class OlmoHybridAttention(nn.Module): q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) q, k = self._apply_qk_norm(q, k) if self._use_rope: + assert self.rotary_emb is not None q, k = self.rotary_emb(positions, q, k) attn_output = self.attn(q, k, v) output, _ = self.o_proj(attn_output) diff --git a/vllm/model_executor/models/opencua.py b/vllm/model_executor/models/opencua.py index 6a875c813c..6b12d1d3fb 100644 --- a/vllm/model_executor/models/opencua.py +++ b/vllm/model_executor/models/opencua.py @@ -7,6 +7,7 @@ """Inference-only OpenCUA-7B model compatible with HuggingFace weights.""" +import typing from collections.abc import Mapping, Sequence from typing import Any @@ -27,6 +28,7 @@ from vllm.multimodal.inputs import ( ) from vllm.multimodal.parse import MultiModalDataItems from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, PromptReplacement, PromptUpdate, @@ -178,7 +180,13 @@ class OpenCUAMultiModalProcessor(BaseMultiModalProcessor[OpenCUAProcessingInfo]) ] -class OpenCUADummyInputsBuilder(Qwen2VLDummyInputsBuilder): +if typing.TYPE_CHECKING: + _OpenCUADummyInputsBuilderBase = BaseDummyInputsBuilder[OpenCUAProcessingInfo] +else: + _OpenCUADummyInputsBuilderBase = Qwen2VLDummyInputsBuilder + + +class OpenCUADummyInputsBuilder(_OpenCUADummyInputsBuilderBase): def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str: num_images = mm_counts.get("image", 0) @@ -220,7 +228,7 @@ class OpenCUAForConditionalGeneration(Qwen2_5_VLForConditionalGeneration): nn.Module.__init__(self) config = vllm_config.model_config.hf_config quant_config = vllm_config.quant_config - multimodal_config = vllm_config.model_config.multimodal_config + multimodal_config = vllm_config.model_config.get_multimodal_config() self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data" self.config = config diff --git a/vllm/model_executor/models/openpangu.py b/vllm/model_executor/models/openpangu.py index b8937f2e09..4aa603b458 100644 --- a/vllm/model_executor/models/openpangu.py +++ b/vllm/model_executor/models/openpangu.py @@ -175,6 +175,7 @@ class OpenPanguMoE(nn.Module): self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts self.n_local_physical_experts = self.n_physical_experts // self.ep_size + self.shared_experts: OpenPanguMLP | None if config.n_shared_experts is not None: intermediate_size = config.moe_intermediate_size * config.n_shared_experts self.shared_experts = OpenPanguMLP( @@ -283,7 +284,7 @@ class OpenPanguMLAAttention(nn.Module): ) self.q_a_layernorm = RMSNorm(self.q_lora_rank, eps=config.rms_norm_eps) self.q_b_proj = ColumnParallelLinear( - q_lora_rank, + self.q_lora_rank, self.num_heads * self.qk_head_dim, bias=False, quant_config=quant_config, @@ -568,6 +569,9 @@ class OpenPanguSinkAttention(nn.Module): self.qk_nope_dim = getattr(config, "qk_nope_dim", None) self.qk_rope_dim = getattr(config, "qk_rope_dim", None) self.v_channels = getattr(config, "v_channels", None) + assert self.qk_nope_dim is not None + assert self.qk_rope_dim is not None + assert self.v_channels is not None self.head_dim = self.qk_rope_dim + self.qk_nope_dim self.q_size = self.num_heads * self.head_dim self.k_size = self.num_kv_heads * self.head_dim diff --git a/vllm/model_executor/models/openpangu_mtp.py b/vllm/model_executor/models/openpangu_mtp.py index d25efd8c03..c91453342a 100644 --- a/vllm/model_executor/models/openpangu_mtp.py +++ b/vllm/model_executor/models/openpangu_mtp.py @@ -52,7 +52,9 @@ class OpenPanguMultiTokenPredictorLayer(DeepSeekMultiTokenPredictorLayer): def __init__(self, vllm_config: VllmConfig, prefix: str) -> None: nn.Module.__init__(self) - config = vllm_config.speculative_config.draft_model_config.hf_config + speculative_config = vllm_config.speculative_config + assert speculative_config is not None + config = speculative_config.draft_model_config.hf_config self.config = config quant_config = vllm_config.quant_config @@ -200,10 +202,15 @@ class OpenPanguMTP(nn.Module): break else: for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping - if weight_name not in name: + ( + expert_param_name, + expert_weight_name, + expert_id, + expert_shard_id, + ) = mapping + if expert_weight_name not in name: continue - name = name.replace(weight_name, param_name) + name = name.replace(expert_weight_name, expert_param_name) param = params_dict[name] weight_loader = param.weight_loader @@ -211,7 +218,7 @@ class OpenPanguMTP(nn.Module): param, loaded_weight, name, - shard_id=shard_id, + shard_id=expert_shard_id, expert_id=expert_id, ) break diff --git a/vllm/model_executor/models/openpangu_vl.py b/vllm/model_executor/models/openpangu_vl.py index 8b85d0371b..741ec2155f 100644 --- a/vllm/model_executor/models/openpangu_vl.py +++ b/vllm/model_executor/models/openpangu_vl.py @@ -22,7 +22,7 @@ from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from functools import lru_cache, partial -from typing import Annotated, Literal, Optional +from typing import Annotated, Literal, Optional, TypeAlias import torch import torch.nn as nn @@ -627,7 +627,7 @@ class OpenPanguVisionTransformer(nn.Module): pad[-(dim + 1) * 2 + 1] = pad_len return F.pad(w, pad, mode="constant", value=0) - stacked_params_mapping = [ + stacked_params_mapping: list[tuple[str, str, str | int]] = [ ("attn.qkv.", "attn.q.", "q"), ("attn.qkv.", "attn.k.", "k"), ("attn.qkv.", "attn.v.", "v"), @@ -677,15 +677,11 @@ class OpenPanguVLProcessingInfo(Qwen2_5_VLProcessingInfo): def get_hf_config(self): return self.ctx.model_config.hf_config - def get_hf_processor( - self, - *, - min_pixels: int | None = None, - max_pixels: int | None = None, - size: dict[str, int] | None = None, - fps: float | list[float] | None = None, - **kwargs: object, - ): + def get_hf_processor(self, **kwargs: object): + kwargs.pop("min_pixels", None) + kwargs.pop("max_pixels", None) + kwargs.pop("size", None) + fps = kwargs.pop("fps", None) if fps is not None: kwargs["fps"] = fps @@ -747,11 +743,19 @@ class OpenPanguVLVideoEmbeddingInputs(TensorSchema): ] +OpenPanguVLImageInputs: TypeAlias = ( + OpenPanguVLImagePixelInputs | OpenPanguVLImageEmbeddingInputs +) +OpenPanguVLVideoInputs: TypeAlias = ( + OpenPanguVLVideoPixelInputs | OpenPanguVLVideoEmbeddingInputs +) + + class OpenPanguVLMultiModalProcessor(Qwen2_5_VLMultiModalProcessor): def _get_prompt_updates( self, mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, any], + hf_processor_mm_kwargs: Mapping[str, object], out_mm_kwargs: MultiModalKwargsItems, ) -> Sequence[PromptUpdate]: hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) @@ -878,10 +882,12 @@ class OpenPanguVLForConditionalGeneration( self.do_rescale = image_processor.do_rescale self.rescale_factor = image_processor.rescale_factor self.do_normalize = image_processor.do_normalize - self.image_mean = tuple(image_processor.image_mean) - self.image_std = tuple(image_processor.image_std) + self.image_mean: tuple[float, ...] = tuple(image_processor.image_mean) + self.image_std: tuple[float, ...] = tuple(image_processor.image_std) - def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig): + def _maybe_ignore_quant_config( + self, quant_config: QuantizationConfig | None + ) -> QuantizationConfig | None: if isinstance(quant_config, AutoGPTQConfig): return None return quant_config @@ -904,7 +910,9 @@ class OpenPanguVLForConditionalGeneration( else: return torch.concat(mm_input) - def _parse_and_validate_image_input(self, **kwargs: object): + def _parse_and_validate_image_input( + self, **kwargs: object + ) -> OpenPanguVLImageInputs | None: pixel_values = kwargs.pop("pixel_values", None) image_embeds = kwargs.pop("image_embeds", None) image_grid_thw = kwargs.pop("image_grid_thw", None) @@ -932,26 +940,27 @@ class OpenPanguVLForConditionalGeneration( image_grid_thw=image_grid_thw, ) - if image_embeds is not None: - image_embeds = self._validate_and_reshape_mm_tensor( - image_embeds, "image embeds" - ) - image_grid_thw = self._validate_and_reshape_mm_tensor( - image_grid_thw, "image grid_thw" - ) + assert image_embeds is not None + image_embeds = self._validate_and_reshape_mm_tensor( + image_embeds, "image embeds" + ) + image_grid_thw = self._validate_and_reshape_mm_tensor( + image_grid_thw, "image grid_thw" + ) - if not isinstance(image_embeds, torch.Tensor): - raise ValueError( - "Incorrect type of image embeddings. " - f"Got type: {type(image_embeds)}" - ) - return OpenPanguVLImageEmbeddingInputs( - type="image_embeds", - image_embeds=image_embeds, - image_grid_thw=image_grid_thw, + if not isinstance(image_embeds, torch.Tensor): + raise ValueError( + f"Incorrect type of image embeddings. Got type: {type(image_embeds)}" ) + return OpenPanguVLImageEmbeddingInputs( + type="image_embeds", + image_embeds=image_embeds, + image_grid_thw=image_grid_thw, + ) - def _parse_and_validate_video_input(self, **kwargs: object): + def _parse_and_validate_video_input( + self, **kwargs: object + ) -> OpenPanguVLVideoInputs | None: pixel_values_videos = kwargs.pop("pixel_values_videos", None) video_embeds = kwargs.pop("video_embeds", None) video_grid_thw = kwargs.pop("video_grid_thw", None) @@ -973,27 +982,30 @@ class OpenPanguVLForConditionalGeneration( video_grid_thw=video_grid_thw, ) - if video_embeds is not None: - video_embeds = self._validate_and_reshape_mm_tensor( - video_embeds, "video embeds" - ) - video_grid_thw = self._validate_and_reshape_mm_tensor( - video_grid_thw, "video grid_thw" - ) + assert video_embeds is not None + video_embeds = self._validate_and_reshape_mm_tensor( + video_embeds, "video embeds" + ) + video_grid_thw = self._validate_and_reshape_mm_tensor( + video_grid_thw, "video grid_thw" + ) - if not isinstance(video_embeds, torch.Tensor): - raise ValueError( - "Incorrect type of video embeddings. " - f"Got type: {type(video_embeds)}" - ) - return OpenPanguVLVideoEmbeddingInputs( - type="video_embeds", - video_embeds=video_embeds, - video_grid_thw=video_grid_thw, + if not isinstance(video_embeds, torch.Tensor): + raise ValueError( + f"Incorrect type of video embeddings. Got type: {type(video_embeds)}" ) + return OpenPanguVLVideoEmbeddingInputs( + type="video_embeds", + video_embeds=video_embeds, + video_grid_thw=video_grid_thw, + ) - def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: - mm_input_by_modality = {} + def _parse_and_validate_multimodal_inputs( + self, **kwargs: object + ) -> dict[str, OpenPanguVLImageInputs | OpenPanguVLVideoInputs | None]: + mm_input_by_modality: dict[ + str, OpenPanguVLImageInputs | OpenPanguVLVideoInputs | None + ] = {} for input_key in kwargs: if ( input_key in ("pixel_values", "image_embeds") @@ -1019,16 +1031,19 @@ class OpenPanguVLForConditionalGeneration( multimodal_embeddings: tuple[torch.Tensor, ...] = () for modality in mm_input_by_modality: - multimodal_input = mm_input_by_modality[modality] if modality == "image": - vision_embeddings = self._process_image_input(multimodal_input) + image_input = mm_input_by_modality["image"] + assert isinstance(image_input, OpenPanguVLImageInputs) + vision_embeddings = self._process_image_input(image_input) multimodal_embeddings = ( multimodal_embeddings if not vision_embeddings else (multimodal_embeddings + vision_embeddings) ) if modality == "video": - video_embeddings = self._process_video_input(multimodal_input) + video_input = mm_input_by_modality["video"] + assert isinstance(video_input, OpenPanguVLVideoInputs) + video_embeddings = self._process_video_input(video_input) multimodal_embeddings = ( multimodal_embeddings if not video_embeddings @@ -1041,20 +1056,19 @@ class OpenPanguVLForConditionalGeneration( input_ids: torch.Tensor, multimodal_embeddings: MultiModalEmbeddings | None = None, ) -> torch.Tensor: - if multimodal_embeddings is None: - return self.language_model.embed_input_ids(input_ids) + inputs_embeds = self.language_model.embed_input_ids(input_ids) + if multimodal_embeddings is not None: + inputs_embeds = self.embed_input_ids( # type: ignore[call-overload] + input_ids, + inputs_embeds, + multimodal_embeddings, + [self.config.image_token_id, self.config.video_token_id], + ) + return inputs_embeds - mm_token_ids = input_ids.new_tensor( - [self.config.image_token_id, self.config.video_token_id] - ) - is_multimodal = torch.isin(input_ids, mm_token_ids) - return self.embed_input_ids( - input_ids, - multimodal_embeddings, - is_multimodal=is_multimodal, - ) - - def _process_image_input(self, image_input) -> tuple[torch.Tensor, ...]: + def _process_image_input( + self, image_input: OpenPanguVLImageInputs + ) -> tuple[torch.Tensor, ...]: grid_thw = image_input["image_grid_thw"] if grid_thw.ndim != 2: raise ValueError(f"grid_thw.ndim must be 2, but it is {grid_thw.ndim}") @@ -1085,7 +1099,9 @@ class OpenPanguVLForConditionalGeneration( sizes = grid_thw.prod(-1) // merge_size // merge_size return image_embeds.split(sizes.tolist()) - def _process_video_input(self, video_input) -> torch.Tensor: + def _process_video_input( + self, video_input: OpenPanguVLVideoInputs + ) -> tuple[torch.Tensor, ...]: grid_thw = video_input["video_grid_thw"] if grid_thw.ndim != 2: raise ValueError(f"grid_thw.ndim must be 2, but it is {grid_thw.ndim}") @@ -1158,8 +1174,13 @@ class OpenPanguVLForConditionalGeneration( for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset): offset = mm_feature.mm_position.offset modality = mm_feature.modality + feature_data = mm_feature.data + assert feature_data is not None if modality == "image": - t, h, w = mm_feature.data["image_grid_thw"].data.tolist() + grid_thw_item = feature_data["image_grid_thw"] + grid_thw = grid_thw_item.data + assert isinstance(grid_thw, torch.Tensor) + t, h, w = grid_thw.tolist() assert t == 1, f"Image must have 1 frame, got {t}" yield ( modality, @@ -1169,7 +1190,10 @@ class OpenPanguVLForConditionalGeneration( w // spatial_merge_size, ) elif modality == "video": - t, h, w = mm_feature.data["video_grid_thw"].data.tolist() + grid_thw_item = feature_data["video_grid_thw"] + grid_thw = grid_thw_item.data + assert isinstance(grid_thw, torch.Tensor) + t, h, w = grid_thw.tolist() yield ( modality, offset, @@ -1281,14 +1305,17 @@ def normalize(image, mean, std): @lru_cache(maxsize=10) def _fuse_mean_std_and_rescale_factor( do_normalize: bool | None = None, - image_mean: float | list[float] | None = None, - image_std: float | list[float] | None = None, + image_mean: float | tuple[float, ...] | None = None, + image_std: float | tuple[float, ...] | None = None, do_rescale: bool | None = None, rescale_factor: float | None = None, device: Optional["torch.device"] = None, ) -> tuple: if do_rescale and do_normalize: # Fused rescale and normalize + assert rescale_factor is not None + assert image_mean is not None + assert image_std is not None image_mean = torch.tensor(image_mean, device=device) * (1.0 / rescale_factor) image_std = torch.tensor(image_std, device=device) * (1.0 / rescale_factor) do_rescale = False @@ -1300,8 +1327,8 @@ def rescale_and_normalize( do_rescale: bool, rescale_factor: float, do_normalize: bool, - image_mean: float | list[float], - image_std: float | list[float], + image_mean: float | tuple[float, ...], + image_std: float | tuple[float, ...], dtype: torch.dtype = torch.bfloat16, ) -> "torch.Tensor": """Rescale and normalize images.""" diff --git a/vllm/model_executor/models/openvla.py b/vllm/model_executor/models/openvla.py index 77099ffb6e..9242f7e1f5 100644 --- a/vllm/model_executor/models/openvla.py +++ b/vllm/model_executor/models/openvla.py @@ -9,7 +9,7 @@ import torch.nn as nn from transformers import BatchFeature from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.activation import get_act_fn from vllm.model_executor.layers.linear import ( @@ -299,18 +299,16 @@ class OpenVLADummyInputsBuilder(BaseDummyInputsBuilder[OpenVLAProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - image_overrides = mm_options.get("image") image_size = self.info.get_image_size_with_most_features() return { "image": self._get_dummy_images( width=image_size.width, height=image_size.height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ) } @@ -349,6 +347,7 @@ class OpenVLAMultiModalProcessor(BaseMultiModalProcessor[OpenVLAProcessingInfo]) if isinstance(images, ImageEmbeddingItems): num_image_tokens = images.get_feature_size(item_idx) else: + assert isinstance(images, ImageProcessorItems) image_size = images.get_image_size(item_idx) num_image_tokens = self.info.get_num_image_tokens( image_width=image_size.width, diff --git a/vllm/model_executor/models/opt.py b/vllm/model_executor/models/opt.py index 1efe9eb8a8..4726c72b41 100644 --- a/vllm/model_executor/models/opt.py +++ b/vllm/model_executor/models/opt.py @@ -218,6 +218,7 @@ class OPTDecoder(nn.Module): ) # Project out & in will be replicated if they exist. + self.project_out: ReplicatedLinear | None if config.word_embed_proj_dim != config.hidden_size: self.project_out = ReplicatedLinear( config.hidden_size, @@ -229,6 +230,7 @@ class OPTDecoder(nn.Module): else: self.project_out = None + self.project_in: ReplicatedLinear | None if config.word_embed_proj_dim != config.hidden_size: self.project_in = ReplicatedLinear( config.word_embed_proj_dim, diff --git a/vllm/model_executor/models/ovis.py b/vllm/model_executor/models/ovis.py index 5563b9a9ce..ed0ea7ab7b 100644 --- a/vllm/model_executor/models/ovis.py +++ b/vllm/model_executor/models/ovis.py @@ -29,7 +29,7 @@ from torch.nn.functional import gumbel_softmax, pad, softmax from transformers import BatchFeature, PretrainedConfig from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.quantization import QuantizationConfig @@ -59,7 +59,12 @@ from vllm.sequence import IntermediateTensors from vllm.transformers_utils.processors.ovis import OvisProcessor from vllm.utils.tensor_schema import TensorSchema, TensorShape -from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP +from .interfaces import ( + MultiModalEmbeddings, + SupportsMultiModal, + SupportsPP, + supports_pp, +) # Cannot find the following number from hf config. IMAGE_TOKEN = "" @@ -281,7 +286,7 @@ class OvisProcessingInfo(BaseProcessingInfo): # minus 1 for presented image token return (patch_grid_length // hidden_stride) ** 2 - 1 - def get_image_pad_token(self) -> str: + def get_image_pad_token(self) -> str | None: hf_text_config = self.get_hf_config().get_text_config() text_model_type = hf_text_config.model_type return IMAGE_PAD_TOKEN_MAP.get(text_model_type) @@ -306,20 +311,16 @@ class OvisDummyInputsBuilder(BaseDummyInputsBuilder[OvisProcessingInfo]): self, seq_len: int, mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], + mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - target_width, target_height = self.info.get_image_size_with_most_features() - image_overrides = mm_options.get("image") - mm_data = { "image": self._get_dummy_images( width=target_width, height=target_height, - num_images=num_images, - overrides=image_overrides, + num_images=mm_counts.get("image", 0), + overrides=mm_options.get("image"), ), } return mm_data @@ -439,8 +440,10 @@ class Ovis(nn.Module, SupportsMultiModal, SupportsPP): text_model_type = self.config.get_text_config().model_type self.image_pad_token_id = IMAGE_PAD_TOKEN_ID_MAP[text_model_type] + language_model = self.get_language_model() + assert supports_pp(language_model) self.make_empty_intermediate_tensors = ( - self.get_language_model().make_empty_intermediate_tensors + language_model.make_empty_intermediate_tensors ) def _parse_and_validate_image_input( diff --git a/vllm/model_executor/models/ovis2_5.py b/vllm/model_executor/models/ovis2_5.py index 0b8a2c5f84..2520b38ae3 100644 --- a/vllm/model_executor/models/ovis2_5.py +++ b/vllm/model_executor/models/ovis2_5.py @@ -4,14 +4,14 @@ from collections.abc import Iterable, Mapping from functools import partial -from typing import Annotated, Literal +from typing import Annotated, Literal, TypedDict import torch import torch.nn as nn from transformers import BaseImageProcessor, BatchFeature, PretrainedConfig from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.quantization import QuantizationConfig @@ -40,7 +40,12 @@ from vllm.sequence import IntermediateTensors from vllm.transformers_utils.processors.ovis2_5 import Ovis2_5Processor from vllm.utils.tensor_schema import TensorSchema, TensorShape -from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP +from .interfaces import ( + MultiModalEmbeddings, + SupportsMultiModal, + SupportsPP, + supports_pp, +) IMAGE_TOKEN = "" VIDEO_TOKEN = "