diff --git a/.buildkite/intel_jobs/lm_eval_intel.yaml b/.buildkite/intel_jobs/lm_eval_intel.yaml index e1f649ad91..62d99cf769 100644 --- a/.buildkite/intel_jobs/lm_eval_intel.yaml +++ b/.buildkite/intel_jobs/lm_eval_intel.yaml @@ -23,8 +23,7 @@ steps: commands: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'uv pip install "gpt-oss[eval]==0.0.5" && - cd tests && + 'cd tests && pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-xpu.txt' - label: LM Eval Large Models(2 GPUs) key: lm-eval-large-models-2-gpus diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 1b2b854bc4..530fd0d27c 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -1287,7 +1287,6 @@ steps: - vllm/platforms/rocm.py - tests/evals/gpt_oss/ commands: - - uv pip install --system 'gpt-oss[eval]==0.0.5' - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx942.txt - label: ":amd: (MI300) LM Eval Small Models Distributed" @@ -3584,7 +3583,6 @@ steps: - vllm/_aiter_ops.py - vllm/platforms/rocm.py commands: - - uv pip install --system 'gpt-oss[eval]==0.0.5' - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx950.txt - label: ":amd: (MI355 DPX) LM Eval Watermarking" diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index c67cc08b94..9f99e1c53f 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -829,7 +829,6 @@ steps: - vllm/model_executor/layers/quantization - tests/evals/gpt_oss/ commands: - - uv pip install --system 'gpt-oss[eval]==0.0.5' - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-h100.txt - label: ":nvidia: (B200) GPQA Eval (GPT-OSS)" @@ -843,7 +842,6 @@ steps: - vllm/model_executor/layers/quantization - tests/evals/gpt_oss/ commands: - - uv pip install --system 'gpt-oss[eval]==0.0.5' - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-b200.txt mirror: amd: @@ -865,7 +863,6 @@ steps: - vllm/_aiter_ops.py - vllm/platforms/rocm.py commands: - - uv pip install --system 'gpt-oss[eval]==0.0.5' - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx950.txt - label: ":nvidia: (B200) Qwen3.8-Flash-Next-FP8 Accuracy Eval" @@ -940,7 +937,6 @@ steps: - vllm/model_executor/layers/quantization - tests/evals/gpt_oss/ commands: - - uv pip install --system 'gpt-oss[eval]==0.0.5' - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-spark.txt - label: ":nvidia: (H200 MIG 35GB) LM Eval KV-Offload" diff --git a/.buildkite/test_areas/spec_decode.yaml b/.buildkite/test_areas/spec_decode.yaml index 8a41a4c993..73d944a7a4 100644 --- a/.buildkite/test_areas/spec_decode.yaml +++ b/.buildkite/test_areas/spec_decode.yaml @@ -98,9 +98,11 @@ steps: commands: - pytest -v -s v1/e2e/spec_decode/eagle/ -- label: ":nvidia: (H200 MIG 35GB) Spec Decode Speculators + MTP" - key: spec-decode-speculators-mtp - timeout_in_minutes: 50 +# MTP correctness is one parametrized test split into per-model folders; +# each job below runs whole directories. +- label: ":nvidia: (H200 MIG 35GB) Spec Decode Speculators" + key: spec-decode-speculators + timeout_in_minutes: 30 device: h200_35gb source_file_dependencies: - vllm/v1/spec_decode/ @@ -110,13 +112,103 @@ steps: - tests/v1/e2e/spec_decode/ commands: - pytest -v -s v1/e2e/spec_decode/speculators/ - - pytest -v -s v1/e2e/spec_decode/mtp/ mirror: amd: - label: ":amd: (MI300) Spec Decode Speculators + MTP" + label: ":amd: (MI300) Spec Decode Speculators" dind: false device: mi300_1 - timeout_in_minutes: 75 + timeout_in_minutes: 40 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/model_executor/model_loader/ + - vllm/v1/sample/ + - vllm/model_executor/layers/ + - vllm/transformers_utils/configs/speculators/ + - tests/v1/e2e/spec_decode/ + - vllm/platforms/rocm.py + +- label: ":nvidia: (H200 MIG 35GB) Spec Decode MTP Gemma4" + key: spec-decode-mtp-gemma4 + timeout_in_minutes: 30 + device: h200_35gb + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/v1/attention/backends/ + - vllm/transformers_utils/configs/speculators/ + - tests/v1/e2e/spec_decode/ + commands: + - pytest -v -s v1/e2e/spec_decode/mtp/gemma4/ + mirror: + amd: + label: ":amd: (MI300) Spec Decode MTP Gemma4" + dind: false + device: mi300_1 + timeout_in_minutes: 40 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/model_executor/model_loader/ + - vllm/v1/sample/ + - vllm/model_executor/layers/ + - vllm/transformers_utils/configs/speculators/ + - tests/v1/e2e/spec_decode/ + - vllm/platforms/rocm.py + +- label: ":nvidia: (H200 MIG 35GB) Spec Decode MTP DeepSeek + MiMo" + key: spec-decode-mtp-deepseek-mimo + timeout_in_minutes: 30 + device: h200_35gb + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/v1/attention/backends/ + - vllm/transformers_utils/configs/speculators/ + - tests/v1/e2e/spec_decode/ + commands: + - pytest -v -s v1/e2e/spec_decode/mtp/deepseek/ + - pytest -v -s v1/e2e/spec_decode/mtp/mimo/ + mirror: + amd: + label: ":amd: (MI300) Spec Decode MTP DeepSeek + MiMo" + dind: false + device: mi300_1 + timeout_in_minutes: 40 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/model_executor/model_loader/ + - vllm/v1/sample/ + - vllm/model_executor/layers/ + - vllm/transformers_utils/configs/speculators/ + - tests/v1/e2e/spec_decode/ + - vllm/platforms/rocm.py + +- label: ":nvidia: (H200 MIG 35GB) Spec Decode MTP Qwen3.5" + key: spec-decode-mtp-qwen3-5 + timeout_in_minutes: 30 + device: h200_35gb + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/v1/attention/backends/ + - vllm/transformers_utils/configs/speculators/ + - tests/v1/e2e/spec_decode/ + commands: + - pytest -v -s v1/e2e/spec_decode/mtp/qwen3_5/ + mirror: + amd: + label: ":amd: (MI300) Spec Decode MTP Qwen3.5" + dind: false + device: mi300_1 + timeout_in_minutes: 40 depends_on: - image-build-amd source_file_dependencies: diff --git a/cmake/external_projects/flashkda.cmake b/cmake/external_projects/flashkda.cmake index 5ac4f4b415..fadee58efb 100644 --- a/cmake/external_projects/flashkda.cmake +++ b/cmake/external_projects/flashkda.cmake @@ -13,7 +13,7 @@ else() FetchContent_Declare( flashkda GIT_REPOSITORY https://github.com/vllm-project/FlashKDA.git - GIT_TAG b59532f1f464fbd536272780e30df5bf6a2ccc02 + GIT_TAG 17a037d98da546deb4591e967cf961a43c034d8b GIT_PROGRESS TRUE GIT_SUBMODULES cutlass ) diff --git a/docker/Dockerfile b/docker/Dockerfile index 366a9a8752..6609ed7464 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -729,6 +729,8 @@ COPY requirements/lint.txt requirements/lint.txt COPY requirements/test/cuda.in requirements/test/cuda.in COPY requirements/test/cuda.txt requirements/test/cuda.txt COPY requirements/dev.txt requirements/dev.txt +# uv reads `[tool.uv] exclude-dependencies` from here when recompiling below. +COPY pyproject.toml pyproject.toml COPY tools/use_existing_torch.py tools/use_existing_torch.py COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt RUN --mount=type=cache,target=/opt/uv/cache \ @@ -1143,6 +1145,8 @@ COPY requirements/lint.txt requirements/lint.txt COPY requirements/test/cuda.in requirements/test/cuda.in COPY requirements/test/cuda.txt requirements/test/cuda.txt COPY requirements/dev.txt requirements/dev.txt +# uv reads `[tool.uv] exclude-dependencies` from here when recompiling below. +COPY pyproject.toml pyproject.toml COPY tools/use_existing_torch.py tools/use_existing_torch.py COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt RUN --mount=type=cache,target=/opt/uv/cache \ diff --git a/docker/Dockerfile.cpu b/docker/Dockerfile.cpu index 36792698a2..43172fbdcf 100644 --- a/docker/Dockerfile.cpu +++ b/docker/Dockerfile.cpu @@ -27,7 +27,7 @@ # cache mounts (ccache, cargo registry) aren't exported by `--cache-to # type=registry`, so they don't survive a build landing on a different/fresh # builder. sccache's cache lives in S3 instead, so it does. -FROM ubuntu:25.04 AS base +FROM ubuntu:22.04 AS base RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ @@ -52,11 +52,38 @@ RUN if [ "$USE_SCCACHE" = "1" ]; then \ && rm -rf sccache.tar.gz sccache-v0.8.1-${SCCACHE_ARCH}-unknown-linux-musl; \ fi +######################### BINUTILS BUILD IMAGE ######################### +# Build a newer GNU assembler for the AMX-FP8 instruction set. +FROM base AS binutils-build + +ARG BINUTILS_VERSION=2.47 + +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update -y \ + && apt-get install -y --no-install-recommends \ + bison \ + build-essential \ + flex \ + texinfo \ + xz-utils \ + && curl -fsSLO "https://sourceware.org/pub/binutils/releases/binutils-${BINUTILS_VERSION}.tar.xz" \ + && tar -xf "binutils-${BINUTILS_VERSION}.tar.xz" \ + && mkdir binutils-build \ + && cd binutils-build \ + && "../binutils-${BINUTILS_VERSION}/configure" --prefix=/opt/binutils --disable-nls --disable-werror \ + && make -j"$(nproc)" \ + && make install \ + && cd .. \ + && rm -rf binutils-build "/binutils-${BINUTILS_VERSION}" "binutils-${BINUTILS_VERSION}.tar.xz" + ######################### COMMON BASE IMAGE ######################### FROM base AS base-common WORKDIR /workspace +COPY --from=binutils-build /opt/binutils /opt/binutils + ARG PYTHON_VERSION=3.12 ARG max_jobs=32 @@ -93,7 +120,8 @@ ENV CC=/usr/bin/gcc-15 CXX=/usr/bin/g++-15 ENV CCACHE_DIR=/root/.cache/ccache ENV CMAKE_CXX_COMPILER_LAUNCHER=ccache -ENV PATH="/root/.local/bin:$PATH" +ENV PATH="/opt/binutils/bin:/root/.local/bin:$PATH" +ENV COMPILER_PATH="/opt/binutils/bin" ENV VIRTUAL_ENV="/opt/venv" ENV UV_PYTHON_INSTALL_DIR=/opt/uv/python RUN uv venv --python ${PYTHON_VERSION} --seed ${VIRTUAL_ENV} @@ -168,7 +196,7 @@ RUN apt-get update -y \ WORKDIR /workspace COPY requirements/build/rust.txt requirements/build/rust.txt -RUN python3 -m pip install --break-system-packages --no-cache-dir -r requirements/build/rust.txt +RUN python3 -m pip install --no-cache-dir -r requirements/build/rust.txt ARG USE_SCCACHE ARG SCCACHE_ENDPOINT diff --git a/docker/Dockerfile.rock_base b/docker/Dockerfile.rock_base index 79896933e7..d3ba0bc842 100644 --- a/docker/Dockerfile.rock_base +++ b/docker/Dockerfile.rock_base @@ -8,7 +8,7 @@ ARG TRITON_VERSION=3.8.0+gitc0a142ff ARG TRITON_REPO="https://github.com/ROCm/triton.git" ARG FA_BRANCH="0e60e394" ARG FA_REPO="https://github.com/Dao-AILab/flash-attention.git" -ARG AITER_BRANCH="v0.1.22.post1" +ARG AITER_BRANCH="v0.1.23" ARG AITER_REPO="https://github.com/ROCm/aiter.git" ARG MORI_BRANCH="f95cd5e0ca7b730f92f9ffdfb358484d6965d9f7" # TheRock compatible v1.1.0 ARG MORI_REPO="https://github.com/ROCm/mori.git" diff --git a/docker/Dockerfile.rocm_base b/docker/Dockerfile.rocm_base index 750d4ce4d9..1c59968005 100644 --- a/docker/Dockerfile.rocm_base +++ b/docker/Dockerfile.rocm_base @@ -9,7 +9,7 @@ ARG PYTORCH_AUDIO_BRANCH="v2.11.0" ARG PYTORCH_AUDIO_REPO="https://github.com/pytorch/audio.git" ARG FA_BRANCH="0e60e394" ARG FA_REPO="https://github.com/Dao-AILab/flash-attention.git" -ARG AITER_BRANCH="v0.1.22.post1" +ARG AITER_BRANCH="v0.1.23" ARG AITER_REPO="https://github.com/ROCm/aiter.git" ARG MORI_BRANCH="v1.1.0" ARG MORI_REPO="https://github.com/ROCm/mori.git" diff --git a/docs/benchmarking/cli.md b/docs/benchmarking/cli.md index e7ee794da6..d05f0c233e 100644 --- a/docs/benchmarking/cli.md +++ b/docs/benchmarking/cli.md @@ -596,6 +596,46 @@ vllm bench serve \ --max-concurrency 512 ``` +#### Responses API Benchmark + +The `openai-responses` backend benchmarks vLLM's +[Responses API](../serving/online_serving/openai_compatible_server.md#responses-api) +directly, instead of routing the same workload through `/v1/chat/completions`. + +```bash +# Server +vllm serve openai/gpt-oss-20b + +# Client +vllm bench serve \ + --backend openai-responses \ + --endpoint /v1/responses \ + --model openai/gpt-oss-20b \ + --dataset-name random \ + --random-input-len 1024 \ + --random-output-len 1024 \ + --num-prompts 200 +``` + +Reasoning deltas (`response.reasoning_text.delta`) count towards TTFT and ITL, +because the server is already decoding tokens when it emits them. Only output +text (`response.output_text.delta`) is collected as the generated text, which +matches how the `openai-chat` backend treats `DeltaMessage.reasoning`. End-to-end +latency stops at the last token event, so `latency - ttft` equals `sum(itl)` and +TPOT is comparable with the other endpoints. The input and output token counts +come from the usage block on `response.completed`, which is still required: a +stream that ends without a terminal event is reported as a failed request. + +The backend measures one streamed text-generation request per prompt. Built-in +tools, MCP, and multi-turn state via `previous_response_id` are out of scope. +All sampling parameter flags are supported except `--min-p`, which the +Responses API does not accept. + +!!! warning + Do not pass `--extra-body '{"include_reasoning": false}'` when benchmarking + a reasoning model. The server still generates reasoning tokens but emits no + events for them, so the whole reasoning phase is absorbed into TTFT. + #### Running With Sampling Parameters When using OpenAI-compatible backends such as `vllm`, optional sampling diff --git a/docs/features/batch_invariance.md b/docs/features/batch_invariance.md index 7353bd3594..0fad4682fc 100644 --- a/docs/features/batch_invariance.md +++ b/docs/features/batch_invariance.md @@ -144,7 +144,7 @@ When batch invariance is enabled, vLLM: 1. Uses deterministic kernel implementations for attention and other operations 2. Ensures consistent numerical behavior across different batch sizes -3. Disables certain optimizations that may introduce non-determinism (such as custom all-reduce operations in tensor parallel mode) +3. Disables certain optimizations that may introduce non-determinism (such as custom all-reduce operations in tensor parallel mode, and sequence parallelism / async TP, whose reduce-scatter path is not batch-invariant) 4. On CUDA devices with tuned matmul table entries for the model's bf16 unquantized forward linear layers (Ada, Hopper, Blackwell), runs without `torch.compile` using breakable CUDA graphs so tile configs follow the runtime batch size; set `VLLM_USE_BREAKABLE_CUDAGRAPH=0` to opt out (not applied when sequence parallelism / async TP are enabled, since those are `torch.compile` passes). diff --git a/docs/features/mooncake_connector_usage.md b/docs/features/mooncake_connector_usage.md index 6cce042fb6..7c5852a68b 100644 --- a/docs/features/mooncake_connector_usage.md +++ b/docs/features/mooncake_connector_usage.md @@ -46,6 +46,13 @@ Now you can send requests to the proxy server through port 8000. - For headless instances, must be the same as the master instance - Each instance needs a unique port on its host; using the same port number across different hosts is fine +- `VLLM_MOONCAKE_CONNECTOR_TIMEOUT`: Per-request timeout (in seconds) for MooncakeConnector HTTP calls to the bootstrap server, including prefiller worker registration at startup. (Optional) + - Default: 30.0 + - Global rank 0 hosts the bootstrap server in the same process that mounts the Mooncake transfer engine's host segment, so a large `MOONCAKE_GLOBAL_SEGMENT_SIZE` can delay responses for several seconds during startup + - Raise this if bootstrap server timeout warnings appear on hosts with very large host-memory segments + - Bootstrap registration uses up to 3 attempts internally, with exponential backoff capped at 10 seconds; connection errors and timeouts are both retried + - Once the attempts are exhausted the caller raises instead of blocking, so failures surface rather than hanging + - `WITH_NVIDIA_PEERMEM`: Selects how mooncake registers GPU memory for RDMA. Read by mooncake, not vLLM. - Default: 1, which uses `ibv_reg_mr()` and requires the `nvidia-peermem` kernel module to be loaded - Set to 0 to use the DMA-BUF path, which does not need that module. Required on hosts where `nvidia-peermem` is not loaded, such as GB200 diff --git a/docs/mkdocs/gen_files/generate_metrics.py b/docs/mkdocs/gen_files/generate_metrics.py index b952f53977..4d4fa93cc5 100644 --- a/docs/mkdocs/gen_files/generate_metrics.py +++ b/docs/mkdocs/gen_files/generate_metrics.py @@ -23,6 +23,10 @@ METRIC_SOURCE_FILES = [ "path": "vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py", "key": "metrics-nixl", }, + { + "path": "vllm/v1/simple_kv_offload/metrics.py", + "key": "metrics-simple-kv-offload", + }, {"path": "vllm/v1/metrics/perf.py", "key": "metrics-mfu"}, ] diff --git a/docs/models/pooling_models/scoring.md b/docs/models/pooling_models/scoring.md index 58083763cb..1c33c6852d 100644 --- a/docs/models/pooling_models/scoring.md +++ b/docs/models/pooling_models/scoring.md @@ -50,8 +50,8 @@ The three supported scoring functions are as illustrated in the figure below. | `ModernBertForSequenceClassification` | ModernBERT-based | `Alibaba-NLP/gte-reranker-modernbert-base`, etc. | N/A | | | | `Qwen2ForSequenceClassification`C | Qwen2-based | `mixedbread-ai/mxbai-rerank-base-v2`(see note), etc. | [mxbai_rerank_v2.jinja](../../../examples/pooling/score/template/mxbai_rerank_v2.jinja) | ✅︎ | ✅︎ | | `Qwen3ForSequenceClassification`C | Qwen3-based | `tomaarsen/Qwen3-Reranker-0.6B-seq-cls`, `Qwen/Qwen3-Reranker-0.6B`(see note), etc. | [qwen3_reranker.jinja](../../../examples/pooling/score/template/qwen3_reranker.jinja) | ✅︎ | ✅︎ | -| `RobertaForSequenceClassification` | RoBERTa-based | `cross-encoder/quora-roberta-base`, etc. | N/A | | | -| `XLMRobertaForSequenceClassification` | XLM-RoBERTa-based | `BAAI/bge-reranker-v2-m3`, etc. | N/A | | | +| `RobertaForSequenceClassification` | RoBERTa-based | `cross-encoder/quora-roberta-base`, etc. | N/A | ✅︎ | | +| `XLMRobertaForSequenceClassification` | XLM-RoBERTa-based | `BAAI/bge-reranker-v2-m3`, etc. | N/A | ✅︎ | | | `*Model`C, `*ForCausalLM`C, etc. | Generative models | N/A | N/A | \* | \* | C Automatically converted into a classification model via `--convert classify`. ([details](./README.md#model-conversion)) diff --git a/docs/usage/metrics.md b/docs/usage/metrics.md index 305c21478f..8254a7f420 100644 --- a/docs/usage/metrics.md +++ b/docs/usage/metrics.md @@ -45,6 +45,30 @@ The following metrics are exposed: --8<-- "gen:metrics-nixl" +## Simple CPU Offload Connector Metrics + +These metrics are exposed when the `SimpleCPUOffloadConnector` KV connector +is configured (e.g. `--kv-transfer-config='{"kv_connector": +"SimpleCPUOffloadConnector", "kv_role": "kv_both", "kv_connector_extra_config": +{"kv_offload_backend": "disk", "disk_path": "/mnt/nvme/kv"}}'`). They are +updated once per engine step. + +Caveats to keep in mind when interpreting them: + +- A "completed" store means the write syscalls returned; it is **not** + fsync-durable, and the disk backend's file is process-lifetime scratch, + unlinked at startup and shutdown. +- `save_outcomes_total` classifies eager-mode boundary hand-off stores; + lazy-mode stores are not classified. +- `used_blocks` counts blocks pinned by in-flight transfers or cache hits; + warm cached blocks that are evictable are not counted. Use the + `capacity_blocks` label of `simple_kv_offload_info` as the denominator. +- Counters and gauges are quantized to engine steps. They are reported by the + scheduler process and reflect engine-wide logical block counts, not values + pooled from individual tensor-parallel workers. + +--8<-- "gen:metrics-simple-kv-offload" + ## Model Flops Utilization (MFU) Performance Metrics These metrics are available via `--enable-mfu-metrics`: diff --git a/pyproject.toml b/pyproject.toml index 4a7c018929..2098e097bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -258,3 +258,8 @@ CANN = "CANN" [tool.uv] no-build-isolation-package = ["torch"] +# vLLM depends on oss-harmony, a fork that ships the same `openai_harmony` +# module without downloading tiktoken vocabularies at runtime. Installing +# openai-harmony alongside it silently overwrites those files, so keep it out +# of every resolution; gpt-oss pulls it in transitively. +exclude-dependencies = ["openai-harmony"] diff --git a/requirements/common.txt b/requirements/common.txt index b55f680b76..ab0060284c 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -49,7 +49,7 @@ pybase64 # fast base64 implementation cbor2 # Required for cross-language serialization of hashable objects ijson # Required for mistral streaming tool parser setproctitle # Used to set process names for better debugging and monitoring -openai-harmony >= 0.0.3 # Required for gpt-oss +oss-harmony >= 0.0.11 # Required for gpt-oss; includes embedded vocabularies anthropic >= 0.71.0 model-hosting-container-standards >= 0.1.14, < 1.0.0 mcp >= 2.0.0, < 3.0.0 diff --git a/requirements/test/cpu.txt b/requirements/test/cpu.txt index c1d4913953..ab3e42886c 100644 --- a/requirements/test/cpu.txt +++ b/requirements/test/cpu.txt @@ -76,7 +76,9 @@ black==24.10.0 blake3==1.0.9 # via -r requirements/test/../common.txt blobfile==3.0.0 - # via -r requirements/test/cuda.in + # via + # -r requirements/test/cuda.in + # gpt-oss bm25s==0.2.13 # via mteb boto3==1.35.57 @@ -111,7 +113,7 @@ chardet==5.2.0 # via mbstrdecoder charset-normalizer==3.4.0 # via requests -chz==0.3.0 +chz==0.4.0 # via gpt-oss click==8.4.2 # via @@ -190,7 +192,7 @@ distro==1.9.0 # openai dnspython==2.7.0 # via email-validator -docker==7.1.0 +docker==7.2.0 # via gpt-oss docopt==0.6.2 # via num2words @@ -291,7 +293,7 @@ googleapis-common-protos==1.70.0 # google-api-core # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -gpt-oss==0.0.8 +gpt-oss==0.0.9 # via -r requirements/test/cuda.in graphql-core==3.2.6 # via hypothesis-graphql @@ -402,6 +404,7 @@ jinja2==3.1.6 # datamodel-code-generator # fastapi # genai-perf + # gpt-oss # lm-eval # torch jiter==0.15.0 @@ -433,6 +436,10 @@ jsonschema-specifications==2024.10.1 # via jsonschema junit-xml==1.9 # via schemathesis +jupyter-client==8.10.0 + # via gpt-oss +jupyter-core==5.9.1 + # via jupyter-client kaldi-native-fbank==1.22.3 # via -r requirements/test/cuda.in kaleido==0.2.1 @@ -549,6 +556,7 @@ numpy==2.2.6 # evaluate # fastparquet # genai-perf + # gpt-oss # imagehash # imageio # librosa @@ -583,8 +591,6 @@ numpy==2.2.6 open-clip-torch==2.32.0 # via -r requirements/test/cuda.in openai==2.44.0 - # via -r requirements/test/../common.txt -openai-harmony==0.0.4 # via # -r requirements/test/../common.txt # gpt-oss @@ -641,6 +647,8 @@ optuna==3.6.1 # via genai-perf orjson==3.11.5 # via genai-perf +oss-harmony==0.0.11 + # via -r requirements/test/../common.txt outlines-core==0.2.14 # via -r requirements/test/../common.txt packaging==24.2 @@ -672,6 +680,7 @@ pandas==2.2.3 # evaluate # fastparquet # genai-perf + # gpt-oss # statsmodels partial-json-parser==0.2.1.1.post7 # via -r requirements/test/../common.txt @@ -702,6 +711,7 @@ pillow==10.4.0 platformdirs==4.3.6 # via # black + # jupyter-core # pooch # virtualenv plotly==5.24.1 @@ -794,7 +804,7 @@ pydantic==2.12.0 # model-hosting-container-standards # mteb # openai - # openai-harmony + # oss-harmony # pydantic-extra-types # pydantic-settings # ray @@ -855,6 +865,7 @@ pytest-timeout==2.3.1 python-dateutil==2.9.0.post0 # via # botocore + # jupyter-client # matplotlib # pandas # typepy @@ -898,7 +909,9 @@ pyyaml==6.0.2 # uvicorn # vocos pyzmq==27.1.0 - # via -r requirements/test/../common.txt + # via + # -r requirements/test/../common.txt + # jupyter-client rapidfuzz==3.12.1 # via jiwer ray==2.56.1 @@ -1077,7 +1090,7 @@ starlette-testclient==0.4.1 # via schemathesis statsmodels==0.14.4 # via genai-perf -structlog==25.4.0 +structlog==26.1.0 # via gpt-oss supervisor==4.3.0 # via model-hosting-container-standards @@ -1101,7 +1114,7 @@ tenacity==9.1.2 # schemathesis tensorizer==2.10.1 # via -r requirements/test/cuda.in -termcolor==3.1.0 +termcolor==3.3.0 # via gpt-oss threadpoolctl==3.5.0 # via scikit-learn @@ -1157,11 +1170,14 @@ torchvision==0.28.0+cpu # open-clip-torch # segmentation-models-pytorch # timm +tornado==6.5.10 + # via jupyter-client tqdm==4.67.3 # via # -r requirements/test/../common.txt # datasets # evaluate + # gpt-oss # huggingface-hub # lm-eval # mteb @@ -1174,6 +1190,10 @@ tqdm==4.67.3 # segmentation-models-pytorch # sentence-transformers # transformers +traitlets==5.16.1 + # via + # jupyter-client + # jupyter-core transformers==5.17.0 # via # -r requirements/test/../common.txt @@ -1226,6 +1246,7 @@ typing-extensions==4.15.0 # grpcio # httpx2 # huggingface-hub + # jupyter-client # librosa # lm-eval # mcp diff --git a/requirements/test/cuda.in b/requirements/test/cuda.in index 5e3faaebcc..271cdba155 100644 --- a/requirements/test/cuda.in +++ b/requirements/test/cuda.in @@ -66,7 +66,7 @@ decord==0.6.0; platform_machine == "x86_64" imagehash # Required for Prithvi tests segmentation-models-pytorch > 0.4.0 # Required for Prithvi tests -gpt-oss >= 0.0.7; python_version > '3.11' +gpt-oss[eval] >= 0.0.7; python_version > '3.11' perceptron # required for isaac test kaldi-native-fbank >= 1.18.7 # required for fireredasr2 test diff --git a/requirements/test/cuda.txt b/requirements/test/cuda.txt index 3207d395aa..ffbc639c4c 100644 --- a/requirements/test/cuda.txt +++ b/requirements/test/cuda.txt @@ -81,7 +81,9 @@ black==24.10.0 blake3==1.0.9 # via -r requirements/test/../common.txt blobfile==3.0.0 - # via -r requirements/test/cuda.in + # via + # -r requirements/test/cuda.in + # gpt-oss bm25s==0.2.13 # via mteb boto3==1.35.57 @@ -116,7 +118,7 @@ chardet==5.2.0 # via mbstrdecoder charset-normalizer==3.4.0 # via requests -chz==0.3.0 +chz==0.4.0 # via gpt-oss click==8.4.2 # via @@ -205,7 +207,7 @@ distro==1.9.0 # openai dnspython==2.7.0 # via email-validator -docker==7.1.0 +docker==7.2.0 # via gpt-oss docopt==0.6.2 # via num2words @@ -310,7 +312,7 @@ googleapis-common-protos==1.70.0 # google-api-core # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -gpt-oss==0.0.8 +gpt-oss==0.0.9 # via -r requirements/test/cuda.in graphql-core==3.2.6 # via hypothesis-graphql @@ -424,6 +426,7 @@ jinja2==3.1.6 # datamodel-code-generator # fastapi # genai-perf + # gpt-oss # lm-eval # torch jiter==0.15.0 @@ -456,6 +459,10 @@ jsonschema-specifications==2024.10.1 # via jsonschema junit-xml==1.9 # via schemathesis +jupyter-client==8.10.0 + # via gpt-oss +jupyter-core==5.9.1 + # via jupyter-client kaldi-native-fbank==1.22.3 # via -r requirements/test/cuda.in kaleido==0.2.1 @@ -584,6 +591,7 @@ numpy==2.2.6 # evaluate # fastparquet # genai-perf + # gpt-oss # imagehash # imageio # librosa @@ -662,10 +670,6 @@ openai==2.44.0 # via # -c requirements/common.txt # -r requirements/test/../common.txt -openai-harmony==0.0.4 - # via - # -c requirements/common.txt - # -r requirements/test/../common.txt # gpt-oss opencensus==0.11.4 # via ray @@ -727,6 +731,10 @@ optuna==3.6.1 # via genai-perf orjson==3.11.5 # via genai-perf +oss-harmony==0.0.11 + # via + # -c requirements/common.txt + # -r requirements/test/../common.txt outlines-core==0.2.14 # via # -c requirements/common.txt @@ -760,6 +768,7 @@ pandas==2.2.3 # evaluate # fastparquet # genai-perf + # gpt-oss # statsmodels partial-json-parser==0.2.1.1.post7 # via -r requirements/test/../common.txt @@ -790,6 +799,7 @@ pillow==10.4.0 platformdirs==4.3.6 # via # black + # jupyter-core # pooch # virtualenv plotly==5.24.1 @@ -887,7 +897,7 @@ pydantic==2.12.0 # model-hosting-container-standards # mteb # openai - # openai-harmony + # oss-harmony # pydantic-extra-types # pydantic-settings # ray @@ -948,6 +958,7 @@ pytest-timeout==2.3.1 python-dateutil==2.9.0.post0 # via # botocore + # jupyter-client # matplotlib # pandas # typepy @@ -994,6 +1005,7 @@ pyzmq==27.1.0 # via # -c requirements/common.txt # -r requirements/test/../common.txt + # jupyter-client rapidfuzz==3.12.1 # via jiwer ray==2.56.1 @@ -1177,7 +1189,7 @@ starlette-testclient==0.4.1 # via schemathesis statsmodels==0.14.4 # via genai-perf -structlog==25.4.0 +structlog==26.1.0 # via gpt-oss supervisor==4.3.0 # via model-hosting-container-standards @@ -1201,7 +1213,7 @@ tenacity==9.1.2 # schemathesis tensorizer==2.10.1 # via -r requirements/test/cuda.in -termcolor==3.1.0 +termcolor==3.3.0 # via gpt-oss threadpoolctl==3.5.0 # via scikit-learn @@ -1264,11 +1276,14 @@ torchvision==0.28.0+cu130 # open-clip-torch # segmentation-models-pytorch # timm +tornado==6.5.10 + # via jupyter-client tqdm==4.67.3 # via # -r requirements/test/../common.txt # datasets # evaluate + # gpt-oss # huggingface-hub # lm-eval # mteb @@ -1281,6 +1296,10 @@ tqdm==4.67.3 # segmentation-models-pytorch # sentence-transformers # transformers +traitlets==5.16.1 + # via + # jupyter-client + # jupyter-core transformers==5.17.0 # via # -c requirements/common.txt @@ -1337,6 +1356,7 @@ typing-extensions==4.15.0 # grpcio # httpx2 # huggingface-hub + # jupyter-client # librosa # lm-eval # mcp diff --git a/requirements/test/rocm.in b/requirements/test/rocm.in index c8c08fabe5..ffe72643aa 100644 --- a/requirements/test/rocm.in +++ b/requirements/test/rocm.in @@ -65,7 +65,7 @@ decord==0.6.0 imagehash # Required for Prithvi tests segmentation-models-pytorch>0.4.0 # Required for Prithvi tests -gpt-oss>=0.0.7; python_version > '3.11' +gpt-oss[eval]>=0.0.7; python_version > '3.11' perceptron # required for isaac test kaldi-native-fbank>=1.18.7 # required for fireredasr2 test diff --git a/requirements/test/rocm.txt b/requirements/test/rocm.txt index c961b6294a..36e1ba182d 100644 --- a/requirements/test/rocm.txt +++ b/requirements/test/rocm.txt @@ -80,7 +80,9 @@ black==26.3.1 blake3==1.0.8 # via -r requirements/test/../common.txt blobfile==3.0.0 - # via -r requirements/test/rocm.in + # via + # -r requirements/test/rocm.in + # gpt-oss bm25s==0.2.13 # via mteb boto3==1.42.74 @@ -198,7 +200,7 @@ distro==1.9.0 # openai dnspython==2.8.0 # via email-validator -docker==7.1.0 +docker==7.2.0 # via gpt-oss docopt==0.6.2 # via num2words @@ -300,7 +302,7 @@ googleapis-common-protos==1.73.0 # google-api-core # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -gpt-oss==0.0.8 +gpt-oss==0.0.9 # via -r requirements/test/rocm.in graphql-core==3.2.8 # via hypothesis-graphql @@ -415,6 +417,7 @@ jinja2==3.1.6 # datamodel-code-generator # fastapi # genai-perf + # gpt-oss # lm-eval # torch jiter==0.14.0 @@ -447,6 +450,10 @@ jsonschema-specifications==2025.9.1 # via jsonschema junit-xml==1.9 # via schemathesis +jupyter-client==8.10.0 + # via gpt-oss +jupyter-core==5.9.1 + # via jupyter-client kaldi-native-fbank==1.22.3 # via -r requirements/test/rocm.in kaleido==1.0.0 @@ -587,6 +594,7 @@ numpy==2.2.6 # evaluate # fastparquet # genai-perf + # gpt-oss # imagehash # imageio # librosa @@ -629,10 +637,6 @@ openai==2.31.0 # via # -c requirements/common.txt # -r requirements/test/../common.txt -openai-harmony==0.0.8 - # via - # -c requirements/common.txt - # -r requirements/test/../common.txt # gpt-oss opencensus==0.11.4 # via ray @@ -699,6 +703,10 @@ orjson==3.11.7 # via # genai-perf # kaleido +oss-harmony==0.0.11 + # via + # -c requirements/common.txt + # -r requirements/test/../common.txt outlines-core==0.2.14 # via # -c requirements/common.txt @@ -734,6 +742,7 @@ pandas==3.0.1 # evaluate # fastparquet # genai-perf + # gpt-oss # statsmodels partial-json-parser==0.2.1.1.post7 # via -r requirements/test/../common.txt @@ -764,6 +773,7 @@ pillow==12.1.1 platformdirs==4.3.6 # via # black + # jupyter-core # pooch # python-discovery # virtualenv @@ -861,7 +871,7 @@ pydantic==2.12.5 # model-hosting-container-standards # mteb # openai - # openai-harmony + # oss-harmony # pydantic-extra-types # pydantic-settings # ray @@ -922,6 +932,7 @@ pytest-timeout==2.3.1 python-dateutil==2.9.0.post0 # via # botocore + # jupyter-client # matplotlib # pandas # typepy @@ -970,6 +981,7 @@ pyzmq==27.1.0 # via # -c requirements/common.txt # -r requirements/test/../common.txt + # jupyter-client rapidfuzz==3.12.1 # via # -r requirements/test/rocm.in @@ -1159,7 +1171,7 @@ starlette-testclient==0.4.1 # via schemathesis statsmodels==0.14.6 # via genai-perf -structlog==25.5.0 +structlog==26.1.0 # via gpt-oss supervisor==4.3.0 # via model-hosting-container-standards @@ -1215,11 +1227,14 @@ tokenizers==0.23.1 # transformers torch-c-dlpack-ext==0.1.5 # via tilelang +tornado==6.5.10 + # via jupyter-client tqdm==4.67.3 # via # -r requirements/test/../common.txt # datasets # evaluate + # gpt-oss # huggingface-hub # lm-eval # mteb @@ -1233,6 +1248,10 @@ tqdm==4.67.3 # sentence-transformers # tilelang # transformers +traitlets==5.16.1 + # via + # jupyter-client + # jupyter-core transformers==5.17.0 # via # -c requirements/common.txt @@ -1287,6 +1306,7 @@ typing-extensions==4.15.0 # grpcio # httpx2 # huggingface-hub + # jupyter-client # librosa # lm-eval # mcp diff --git a/requirements/test/xpu.in b/requirements/test/xpu.in index d2ad5a0cfd..08a81cd20d 100644 --- a/requirements/test/xpu.in +++ b/requirements/test/xpu.in @@ -32,7 +32,7 @@ soundfile # --- Tool Parsing & Evaluation --- blobfile rapidfuzz -gpt-oss >= 0.0.7; python_version > '3.11' +gpt-oss[eval] >= 0.0.7; python_version > '3.11' schemathesis>=4.0.0 # Required for openai schema test. jiwer bm25s diff --git a/requirements/test/xpu.txt b/requirements/test/xpu.txt index ed8827f5db..e5f898adcc 100644 --- a/requirements/test/xpu.txt +++ b/requirements/test/xpu.txt @@ -67,7 +67,9 @@ azure-storage-blob==12.30.1 blake3==1.0.9 # via -r requirements/test/../common.txt blobfile==3.2.0 - # via -r requirements/test/xpu.in + # via + # -r requirements/test/xpu.in + # gpt-oss bm25s==0.3.10 # via # -r requirements/test/xpu.in @@ -257,7 +259,7 @@ googleapis-common-protos==1.75.1 # google-api-core # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -gpt-oss==0.0.8 +gpt-oss==0.0.9 # via -r requirements/test/xpu.in graphql-core==3.2.11 # via hypothesis-graphql @@ -377,6 +379,7 @@ jinja2==3.1.6 # via # -c requirements/xpu.txt # fastapi + # gpt-oss # lm-eval # torch jiter==0.16.0 @@ -406,6 +409,10 @@ jsonschema-rs==0.49.9 # via schemathesis jsonschema-specifications==2025.9.1 # via jsonschema +jupyter-client==8.10.0 + # via gpt-oss +jupyter-core==5.9.1 + # via jupyter-client kiwisolver==1.5.1 # via matplotlib lark==1.2.2 @@ -526,6 +533,7 @@ numpy==2.2.6 # datasets # decord # evaluate + # gpt-oss # librosa # lm-eval # matplotlib @@ -575,10 +583,6 @@ openai==3.2.0 # via # -c requirements/common.txt # -r requirements/test/../common.txt -openai-harmony==0.0.8 - # via - # -c requirements/common.txt - # -r requirements/test/../common.txt # gpt-oss opencv-python-headless==5.0.0.93 # via @@ -632,6 +636,10 @@ opentelemetry-semantic-conventions-ai==0.5.1 # -c requirements/common.txt # -r requirements/test/../common.txt # -r requirements/test/xpu.in +oss-harmony==0.0.11 + # via + # -c requirements/common.txt + # -r requirements/test/../common.txt outlines-core==0.2.14 # via # -c requirements/common.txt @@ -656,6 +664,7 @@ pandas==3.0.5 # via # datasets # evaluate + # gpt-oss partial-json-parser==0.2.1.1.post7 # via -r requirements/test/../common.txt pathvalidate==3.3.1 @@ -667,7 +676,9 @@ pillow==12.3.0 # mistral-common # torchvision platformdirs==4.11.3 - # via pooch + # via + # jupyter-core + # pooch pluggy==1.6.0 # via # pytest @@ -747,7 +758,7 @@ pydantic==2.13.4 # model-hosting-container-standards # mteb # openai - # openai-harmony + # oss-harmony # pydantic-extra-types # pydantic-settings # xgrammar @@ -806,6 +817,7 @@ pytest-timeout==2.4.0 python-dateutil==2.9.0.post0 # via # botocore + # jupyter-client # matplotlib # pandas # typepy @@ -841,6 +853,7 @@ pyzmq==27.1.0 # via # -c requirements/common.txt # -r requirements/test/../common.txt + # jupyter-client rapidfuzz==3.14.5 # via # -r requirements/test/xpu.in @@ -1066,11 +1079,14 @@ torchvision==0.29.0+xpu # via # open-clip-torch # timm +tornado==6.5.10 + # via jupyter-client tqdm==4.70.0 # via # -r requirements/test/../common.txt # datasets # evaluate + # gpt-oss # huggingface-hub # lm-eval # modelscope @@ -1081,6 +1097,10 @@ tqdm==4.70.0 # pqdm # sentence-transformers # transformers +traitlets==5.16.1 + # via + # jupyter-client + # jupyter-core transformers==5.17.0 # via # -c requirements/common.txt @@ -1132,6 +1152,7 @@ typing-extensions==4.16.0 # grpcio # httpx2 # huggingface-hub + # jupyter-client # lm-eval # mcp # mcp-types diff --git a/tests/benchmarks/test_responses_request_func.py b/tests/benchmarks/test_responses_request_func.py new file mode 100644 index 0000000000..38f1bdc507 --- /dev/null +++ b/tests/benchmarks/test_responses_request_func.py @@ -0,0 +1,455 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for the ``openai-responses`` backend of ``vllm bench serve``. + +The Responses stream differs from Chat Completions in ways the benchmark client +has to get right: every SSE message is labelled with an ``event:`` line, +reasoning and output text arrive as separate token-bearing events, and a +request ends with ``response.completed`` rather than with its last token. These +tests drive the real request function against a local SSE server so the +resulting latency and usage numbers are covered end to end. +""" + +import asyncio +import json +from collections.abc import Sequence + +import aiohttp +import pytest +from aiohttp import web +from aiohttp.test_utils import TestServer + +from vllm.benchmarks.lib.endpoint_request_func import ( + ASYNC_REQUEST_FUNCS, + OPENAI_COMPATIBLE_BACKENDS, + RequestFuncInput, + RequestFuncOutput, + async_request_openai_responses, +) + +# These tests never touch torch, so skip the GPU cleanup fixture. +pytestmark = pytest.mark.skip_global_cleanup + +MODEL = "openai/gpt-oss-20b" + +# Gaps between scripted events. GAP_TOKEN is much larger than GAP_PREFILL so +# that a time to first token measured at the wrong event is unambiguous. +GAP_PREFILL = 0.15 +GAP_TOKEN = 0.5 +GAP_FINALIZE = 0.3 + + +def sse(event_type: str, payload: dict) -> bytes: + """Frame one event the way ``/v1/responses`` does, with an ``event:`` line. + + ``ensure_ascii=False`` matches the server, whose pydantic + ``model_dump_json`` emits non-ASCII characters unescaped. + """ + data = json.dumps({"type": event_type, **payload}, ensure_ascii=False) + return f"event: {event_type}\ndata: {data}\n\n".encode() + + +def usage_event( + input_tokens: int, + output_tokens: int, + event_type: str = "response.completed", + status: str = "completed", +): + return sse( + event_type, + { + "response": { + "status": status, + "usage": { + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "total_tokens": input_tokens + output_tokens, + }, + } + }, + ) + + +def text_delta(delta: str) -> bytes: + return sse("response.output_text.delta", {"delta": delta}) + + +def reasoning_delta(delta: str) -> bytes: + return sse("response.reasoning_text.delta", {"delta": delta}) + + +# The metadata events the server sends before any token. None of these may +# start the TTFT clock or contribute an ITL sample. +PREAMBLE = ( + sse("response.created", {"response": {"status": "in_progress"}}) + + sse("response.in_progress", {"response": {"status": "in_progress"}}) + + sse("response.output_item.added", {"output_index": 0}) + + sse("response.reasoning_part.added", {"output_index": 0}) + + sse("response.content_part.added", {"output_index": 0}) +) + +# The `.done` family the server sends after the last delta. These repeat the +# accumulated text, so timing them would add bogus ITL samples and counting +# their payload would duplicate the output. +DONE_EVENTS = ( + sse("response.reasoning_text.done", {"text": "The user asks "}) + + sse("response.reasoning_part.done", {"output_index": 0}) + + sse("response.output_text.done", {"text": "Rayleigh scattering."}) + + sse("response.content_part.done", {"output_index": 0}) + + sse("response.output_item.done", {"output_index": 0}) +) + + +async def _serve( + script: Sequence[bytes | float], + *, + status: int = 200, + chunk_size: int | None = None, + seen: dict | None = None, + **request_kwargs, +) -> RequestFuncOutput: + """Run the request function against a server replaying ``script``. + + Args: + script: Byte strings to write, interleaved with floats to sleep for. + status: HTTP status to respond with. Non-200 skips the script. + chunk_size: If set, split every write into chunks of this many bytes to + emulate fragmented transport. + seen: If given, populated with the request payload. + request_kwargs: Extra fields for the ``RequestFuncInput``. + + """ + + async def handler(request: web.Request) -> web.StreamResponse: + if seen is not None: + seen["payload"] = await request.json() + seen["headers"] = dict(request.headers) + if status != 200: + return web.Response(status=status, reason="Bad Request") + + response = web.StreamResponse(headers={"Content-Type": "text/event-stream"}) + await response.prepare(request) + for item in script: + if isinstance(item, float): + await asyncio.sleep(item) + elif chunk_size is None: + await response.write(item) + else: + for i in range(0, len(item), chunk_size): + await response.write(item[i : i + chunk_size]) + await response.write_eof() + return response + + app = web.Application() + app.router.add_post("/v1/responses", handler) + server = TestServer(app) + await server.start_server() + try: + request_input = RequestFuncInput( + prompt="Why is the sky blue?", + api_url=str(server.make_url("/v1/responses")), + prompt_len=5, + output_len=16, + model=MODEL, + **request_kwargs, + ) + async with aiohttp.ClientSession() as session: + return await async_request_openai_responses(request_input, session) + finally: + await server.close() + + +def run(script: Sequence[bytes | float], **kwargs) -> RequestFuncOutput: + return asyncio.run(_serve(script, **kwargs)) + + +def test_backend_is_registered_as_openai_compatible(): + assert ASYNC_REQUEST_FUNCS["openai-responses"] is async_request_openai_responses + # Membership enables --ignore-eos on random datasets and the sampling + # parameter flags, all of which ResponsesRequest accepts. + assert "openai-responses" in OPENAI_COMPATIBLE_BACKENDS + + +def test_request_payload_is_accepted_by_the_server_schema(): + """Every field sent must be a real ResponsesRequest field. + + This is what makes the backend safe to list in OPENAI_COMPATIBLE_BACKENDS: + the flags that membership enables have to reach the model, and the server + drops unknown fields without complaining. + """ + from vllm.entrypoints.openai.responses.protocol import ResponsesRequest + + sampling = { + "temperature": 0.7, + "top_p": 0.9, + "top_k": 20, + "frequency_penalty": 0.2, + "presence_penalty": 0.1, + "repetition_penalty": 1.05, + } + seen: dict = {} + output = run( + [PREAMBLE, text_delta("hi"), usage_event(5, 1)], + seen=seen, + extra_body=sampling, + ignore_eos=True, + model_name="served-alias", + request_id="bench-7", + extra_headers={"x-team": "perf"}, + ) + + assert output.success + # --served-model-name wins over --model, as on the other backends. + assert seen["payload"]["model"] == "served-alias" + # The route is behind validate_json_request, which 415s a missing type. + assert seen["headers"]["Content-Type"] == "application/json" + assert seen["headers"]["x-request-id"] == "bench-7" + assert seen["headers"]["x-team"] == "perf" + assert seen["payload"] == { + "model": "served-alias", + "input": "Why is the sky blue?", + "max_output_tokens": 16, + "stream": True, + "store": False, + "ignore_eos": True, + **sampling, + } + assert set(seen["payload"]) <= set(ResponsesRequest.model_fields) + ResponsesRequest.model_validate(seen["payload"]) + + # `vllm bench serve` rejects --min-p for this backend for the same reason. + assert "min_p" not in ResponsesRequest.model_fields + + +def test_latency_accounting_matches_the_other_endpoints(): + """`latency - ttft` must equal `sum(itl)`. + + `serve.py` derives TPOT as `(latency - ttft) / (output_len - 1)`, which only + means average inter-token latency when that identity holds. #55508 fixed the + same accounting on the chat and audio backends; the trailing events after + the last token must not extend the request. + """ + output = run( + [ + PREAMBLE, + GAP_PREFILL / 2, + # A keep-alive comment sent during prefill is not a token. + b": keep-alive\n\n", + GAP_PREFILL / 2, + reasoning_delta("The user asks "), + GAP_TOKEN, + text_delta("Rayleigh"), + text_delta(" scattering."), + DONE_EVENTS, + GAP_FINALIZE, + usage_event(5, 8), + ] + ) + + assert output.success + # Reasoning is timed but not returned, matching how `openai-chat` treats + # `DeltaMessage.reasoning`. + assert output.generated_text == "Rayleigh scattering." + + # Neither the response.created/in_progress/output_item.added preamble nor + # the keep-alive comment may start the clock, and the reasoning delta must, + # so TTFT lands on the prefill gap rather than near zero or out at the + # first output text delta. + assert GAP_PREFILL * 0.8 <= output.ttft < GAP_PREFILL + GAP_TOKEN * 0.5 + + # One ITL per token event after the first: the gap before "Rayleigh" and + # the back-to-back delta that follows it. + assert len(output.itl) == 2 + assert output.itl[0] >= GAP_TOKEN * 0.5 + assert output.itl[1] < GAP_TOKEN * 0.5 + + # The identity is exact, so no wall-clock margin is involved. The .done + # family and the GAP_FINALIZE wait before response.completed sit outside + # the request, exactly as the trailing usage chunk does on chat. + assert output.latency - output.ttft == pytest.approx(sum(output.itl), abs=1e-9) + + assert output.start_time > 0.0 + + +def test_usage_is_taken_from_the_terminal_event(): + output = run([PREAMBLE, text_delta("blue"), usage_event(17, 4)]) + + assert output.success + assert output.prompt_len == 17 + assert output.output_tokens == 4 + + +@pytest.mark.parametrize("event_type", ["response.completed", "response.incomplete"]) +def test_incomplete_response_is_a_successful_request(event_type: str): + """Hitting max_output_tokens is the Responses form of finish_reason=length. + + vLLM reports it as `response.completed` carrying `status: "incomplete"`. + The spec's dedicated `response.incomplete` event is accepted as well. + """ + output = run( + [ + PREAMBLE, + text_delta("truncated"), + usage_event(17, 16, event_type=event_type, status="incomplete"), + ] + ) + + assert output.success + assert output.generated_text == "truncated" + assert output.output_tokens == 16 + + +@pytest.mark.parametrize("chunk_size", [1, 3, 7, 64]) +def test_output_survives_transport_chunk_boundaries(chunk_size: int): + """Splits inside the event line, the JSON, or a multibyte character.""" + text = " leading 中 trailing " + output = run( + [PREAMBLE, text_delta(text), usage_event(5, 6)], + chunk_size=chunk_size, + ) + + assert output.success + assert output.generated_text == text + assert output.output_tokens == 6 + + +@pytest.mark.parametrize( + "char", ["\u2028", "\u2029", "\u0085"], ids=["u2028", "u2029", "u0085"] +) +def test_unicode_line_separators_do_not_truncate_the_payload(char: str): + """The server emits these unescaped and they are legal inside a JSON string. + + `str.splitlines()` treats them as line breaks, which would cut the `data:` + line short and fail the request on valid model output. + """ + text = f"before{char}after" + output = run([PREAMBLE, text_delta(text), usage_event(5, 3)]) + + assert output.success, output.error + assert output.generated_text == text + + +def test_empty_data_line_is_skipped(): + """A bare `data:` line carries no JSON and must not fail the request.""" + output = run([PREAMBLE, b"data:\n\n", text_delta("ok"), usage_event(5, 1)]) + + assert output.success, output.error + assert output.generated_text == "ok" + + +def test_prompt_len_falls_back_when_usage_is_absent(): + """A terminal event without usage keeps the locally computed prompt length.""" + output = run( + [ + PREAMBLE, + text_delta("hi"), + sse("response.completed", {"response": {"status": "completed"}}), + ] + ) + + assert output.success, output.error + assert output.prompt_len == 5 + + +def test_multi_line_data_payload_is_joined_with_newlines(): + """The SSE spec allows one payload to span several `data:` lines.""" + script: list[bytes | float] = [ + PREAMBLE, + b"event: response.output_text.delta\n" + b'data: {"type": "response.output_text.delta",\n' + b'data: "delta": "ok"}\n\n', + usage_event(5, 1), + ] + output = run(script) + + assert output.success, output.error + assert output.generated_text == "ok" + + +def test_done_sentinel_is_skipped(): + """Other Responses servers send `[DONE]`, but vLLM does not.""" + output = run([PREAMBLE, text_delta("ok"), usage_event(5, 1), b"data: [DONE]\n\n"]) + + assert output.success, output.error + assert output.generated_text == "ok" + + +def test_error_before_any_token_reports_the_server_error(): + """The server's message must win over the generic no-TTFT message.""" + output = run( + [PREAMBLE, b'event: error\ndata: {"type": "error", "message": "boom"}\n\n'] + ) + + assert not output.success + assert "boom" in output.error + + +def test_http_error_is_not_reported_as_success(): + output = run([], status=400) + + assert not output.success + assert output.error + + +def test_stream_without_tokens_is_not_reported_as_success(): + """A stream that finishes without generating anything has no TTFT.""" + output = run([PREAMBLE, usage_event(5, 0)]) + + assert not output.success + assert "TTFT" in output.error + + +def test_truncated_stream_is_not_reported_as_success(): + """A stream cut short after a token has no usage and an understated E2EL.""" + output = run([PREAMBLE, text_delta("partial")]) + + assert not output.success + assert "terminal event" in output.error + + +def test_mid_stream_error_is_not_reported_as_success(): + """An error payload mid-stream must not be counted as a success. + + vLLM cannot emit this today: its streaming error path fails to validate + against the event union, so a generation failure truncates the stream + instead. Proxies and other Responses servers do send it. + """ + output = run( + [ + PREAMBLE, + text_delta("partial"), + b'event: error\ndata: {"error": {"message": "boom", "code": 500}}\n\n', + ] + ) + + assert not output.success + assert "boom" in output.error + + +def test_data_only_framing_is_still_parsed(): + """Some proxies drop the ``event:`` line and forward only ``data:``.""" + script: list[bytes | float] = [ + b'data: {"type": "response.output_text.delta", "delta": "ok"}\n\n', + usage_event(5, 1), + ] + output = run(script) + + assert output.success + assert output.generated_text == "ok" + + +def test_endpoint_path_is_validated(): + request_input = RequestFuncInput( + prompt="hi", + api_url="http://localhost:8000/v1/chat/completions", + prompt_len=1, + output_len=1, + model=MODEL, + ) + + async def call() -> None: + async with aiohttp.ClientSession() as session: + await async_request_openai_responses(request_input, session) + + with pytest.raises(ValueError, match="responses"): + asyncio.run(call()) diff --git a/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py b/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py index 467e23e7d3..f1de27bf75 100644 --- a/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py +++ b/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py @@ -3,10 +3,20 @@ import json from collections.abc import AsyncGenerator +from unittest.mock import MagicMock import httpx import pytest +from tests.entrypoints.openai.chat_completion.test_serving_chat import ( + BASE_MODEL_PATHS, + MockHFConfig, + MockModelConfig, + _build_renderer, +) +from tests.entrypoints.openai.chat_completion.test_serving_chat import ( + MODEL_NAME as MOCK_MODEL_NAME, +) from tests.utils import RemoteOpenAIServer from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata from vllm.entrypoints.openai.chat_completion.batch_serving import ( @@ -15,7 +25,10 @@ from vllm.entrypoints.openai.chat_completion.batch_serving import ( from vllm.entrypoints.openai.chat_completion.protocol import ( BatchChatCompletionRequest, ) +from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.outputs import CompletionOutput, RequestOutput +from vllm.renderers.online_renderer import OnlineRenderer +from vllm.v1.engine.async_llm import AsyncLLM # any model with a chat template defined in tokenizer_config should work here MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct" @@ -322,3 +335,53 @@ async def test_batched_echo_prepends_matching_assistant_prefix() -> None: ) assert response.choices[0].message.content == "PREFIX ASSISTANT ANSWER" + + +@pytest.mark.asyncio +@pytest.mark.skip_global_cleanup +async def test_batched_harmony_response_format_uses_structural_tag() -> None: + """Harmony must constrain only the final channel of every conversation.""" + mock_engine = MagicMock(spec=AsyncLLM) + mock_engine.model_config = MockModelConfig() + mock_engine.errored = False + mock_engine.input_processor = MagicMock() + mock_engine.model_config.hf_config = MockHFConfig(model_type="gpt_oss") + mock_engine.renderer = _build_renderer(mock_engine.model_config) + mock_engine.generate.side_effect = lambda *args, **kwargs: _generator(0, "ok") + + models = OpenAIServingModels(mock_engine, BASE_MODEL_PATHS) + online_renderer = OnlineRenderer( + model_config=mock_engine.model_config, + renderer=mock_engine.renderer, + request_logger=None, + chat_template=None, + chat_template_content_format="auto", + reasoning_parser="openai_gptoss", + ) + serving = OpenAIServingChatBatch( + mock_engine, + models, + response_role="assistant", + online_renderer=online_renderer, + chat_template=None, + chat_template_content_format="auto", + request_logger=None, + ) + + request = BatchChatCompletionRequest( + model=MOCK_MODEL_NAME, + messages=[ + [{"role": "user", "content": "France?"}], + [{"role": "user", "content": "Japan?"}], + ], + response_format={ + "type": "json_schema", + "json_schema": {"name": "city", "schema": {"type": "object"}}, + }, + ) + await serving.create_batch_chat_completion(request) + + calls = mock_engine.generate.call_args_list + assert len(calls) == 2 + for call in calls: + assert call.args[1].structured_outputs.structural_tag is not None diff --git a/tests/entrypoints/openai/parser/test_harmony_render_parity.py b/tests/entrypoints/openai/parser/test_harmony_render_parity.py index 5cb4461223..966ac5cf52 100644 --- a/tests/entrypoints/openai/parser/test_harmony_render_parity.py +++ b/tests/entrypoints/openai/parser/test_harmony_render_parity.py @@ -20,6 +20,7 @@ Each test: """ from openai.types.responses import ResponseFunctionToolCall +from openai_harmony import Message, Role from tests.entrypoints.openai.utils import verify_harmony_messages from vllm.entrypoints.openai.parser.harmony_utils import ( @@ -33,6 +34,23 @@ from vllm.entrypoints.openai.responses.harmony import ( response_previous_input_to_harmony, ) + +def test_render_uses_legacy_tool_call_header(): + tool_call = ( + Message.from_role_and_content(Role.ASSISTANT, "print('hello')") + .with_channel("commentary") + .with_recipient("python") + .with_content_type("code") + ) + + rendered = get_encoding().decode(render_for_completion([tool_call])) + + assert ( + "<|start|>assistant to=python<|channel|>commentary code<|message|>" in rendered + ) + assert "commentary to=python <|constrain|>code" not in rendered + + # Use a fixed date so the system message is deterministic across both paths. _DATE = "2025-01-01" diff --git a/tests/entrypoints/openai/responses/test_parsable_context_unit.py b/tests/entrypoints/openai/responses/test_parsable_context_unit.py index 85bf6f2020..0398723184 100644 --- a/tests/entrypoints/openai/responses/test_parsable_context_unit.py +++ b/tests/entrypoints/openai/responses/test_parsable_context_unit.py @@ -84,6 +84,18 @@ class _ReasoningOnlyParser(DelegatingParser): return None +class _CountingParser(_NoOpParser): + """Parser that records the ids it counts; token id 7 is reasoning.""" + + def __init__(self, tokenizer, *args, **kwargs): + super().__init__(tokenizer) + self.counted_ids: list[list[int]] = [] + + def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int: + self.counted_ids.append(list(token_ids)) + return sum(token_id == 7 for token_id in token_ids) + + class _StubToolParser: """Minimal tool parser stub that always returns a hardcoded tool call.""" @@ -339,6 +351,25 @@ def test_multi_turn_accumulation(): assert texts == ["First turn", "Second turn"] +def test_reasoning_tokens_counted_per_round(): + """Each round is counted on its own ids, not the concatenated rounds.""" + ctx = _make_context(_CountingParser) + + ctx.append_output(_make_request_output(token_ids=[7, 7, 1])) + ctx.append_output(_make_request_output(token_ids=[7, 2])) + + assert ctx.response_parser.counted_ids == [[7, 7, 1], [7, 2]] + assert ctx.num_reasoning_tokens == 3 + + +def test_reasoning_tokens_zero_without_parser(): + ctx = _make_context(None) + + ctx.append_output(_make_request_output(token_ids=[7, 7, 1])) + + assert ctx.num_reasoning_tokens == 0 + + def test_num_init_messages_offset(): """Initial messages are preserved and offset works correctly.""" init_messages = [MagicMock(type="message")] diff --git a/tests/entrypoints/openai/responses/test_serving_responses.py b/tests/entrypoints/openai/responses/test_serving_responses.py index 429fcf1d0d..89c188ec8a 100644 --- a/tests/entrypoints/openai/responses/test_serving_responses.py +++ b/tests/entrypoints/openai/responses/test_serving_responses.py @@ -960,24 +960,19 @@ class TestValidateGeneratorInput: assert isinstance(result, ErrorResponse) -@pytest.mark.asyncio -async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch): - """Ensure reasoning_tokens usage is derived from thinking token spans.""" +class _Qwen3FakeTokenizer: + def __init__(self): + self._vocab = {"": 1, "": 2, "reason": 3, "final": 4} - class FakeTokenizer: - def __init__(self): - self._vocab = {"": 1, "": 2, "reason": 3, "final": 4} + def get_vocab(self): + return self._vocab - def get_vocab(self): - return self._vocab + def decode(self, token_ids): + id_to_token = {v: k for k, v in self._vocab.items()} + return "".join(id_to_token.get(token_id, "x") for token_id in token_ids) - def decode(self, token_ids): - id_to_token = {v: k for k, v in self._vocab.items()} - return "".join(id_to_token.get(token_id, "x") for token_id in token_ids) - - # Force non-harmony, SimpleContext path - monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False) +def _make_qwen3_serving() -> tuple[OpenAIServingResponses, _Qwen3FakeTokenizer]: engine_client = MagicMock() model_config = MagicMock() model_config.hf_config.model_type = "test" @@ -987,21 +982,66 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch): engine_client.input_processor = MagicMock() engine_client.renderer = MagicMock() - tokenizer = FakeTokenizer() + tokenizer = _Qwen3FakeTokenizer() engine_client.renderer.get_tokenizer.return_value = tokenizer - models = MagicMock() - serving = OpenAIServingResponses( engine_client=engine_client, - models=models, + models=MagicMock(), online_renderer=MagicMock(), request_logger=None, chat_template=None, chat_template_content_format="auto", reasoning_parser="qwen3", ) + return serving, tokenizer + +def _make_text_request_output(text: str, token_ids: list[int]) -> RequestOutput: + return RequestOutput( + request_id="req", + prompt="hi", + prompt_token_ids=[7, 8], + prompt_logprobs=None, + outputs=[ + CompletionOutput( + index=0, + text=text, + token_ids=token_ids, + cumulative_logprob=0.0, + logprobs=None, + finish_reason="stop", + stop_reason=None, + ) + ], + finished=True, + num_cached_tokens=0, + num_cache_creation_tokens=3, + ) + + +async def _run_full_generator(serving, request, context, tokenizer): + async def dummy_result_generator(): + yield None + + return await serving.responses_full_generator( + request=request, + sampling_params=SamplingParams(max_tokens=16), + result_generator=dummy_result_generator(), + context=context, + model_name="test-model", + tokenizer=tokenizer, + request_metadata=RequestResponseMetadata(request_id="req"), + ) + + +@pytest.mark.asyncio +async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch): + """Ensure reasoning_tokens usage is derived from thinking token spans.""" + # Force non-harmony, SimpleContext path + monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False) + + serving, tokenizer = _make_qwen3_serving() request = ResponsesRequest(input="hi", tools=[], stream=False) response_parser = serving._make_response_parser( request, @@ -1011,43 +1051,12 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch): # Build a SimpleContext with thinking tokens in the output. context = SimpleContext(response_parser=response_parser) - token_ids = [1, 10, 2, 20] # 10 20 -> reasoning token count = 1 - completion = CompletionOutput( - index=0, - text="reasonfinal", - token_ids=token_ids, - cumulative_logprob=0.0, - logprobs=None, - finish_reason="stop", - stop_reason=None, + # 10 20 -> reasoning token count = 1 + context.append_output( + _make_text_request_output("reasonfinal", [1, 10, 2, 20]) ) - req_output = RequestOutput( - request_id="req", - prompt="hi", - prompt_token_ids=[7, 8], - prompt_logprobs=None, - outputs=[completion], - finished=True, - num_cached_tokens=0, - num_cache_creation_tokens=3, - ) - context.append_output(req_output) - async def dummy_result_generator(): - yield None - - sampling_params = SamplingParams(max_tokens=16) - metadata = RequestResponseMetadata(request_id="req") - - response = await serving.responses_full_generator( - request=request, - sampling_params=sampling_params, - result_generator=dummy_result_generator(), - context=context, - model_name="test-model", - tokenizer=tokenizer, - request_metadata=metadata, - ) + response = await _run_full_generator(serving, request, context, tokenizer) assert response.usage.output_tokens_details.reasoning_tokens == 1 assert response.usage.input_tokens_details.cache_write_tokens == 3 diff --git a/tests/evals/gpt_oss/run_gpqa_eval.py b/tests/evals/gpt_oss/run_gpqa_eval.py new file mode 100644 index 0000000000..b9f94a59a4 --- /dev/null +++ b/tests/evals/gpt_oss/run_gpqa_eval.py @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Run the gpt-oss GPQA evaluator with an explicit output-token budget.""" + +import argparse + +from gpt_oss.evals.gpqa_eval import GPQAEval +from gpt_oss.evals.responses_sampler import ResponsesSampler + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--model", required=True) + parser.add_argument("--reasoning-effort", required=True) + parser.add_argument("--base-url", required=True) + parser.add_argument("--n-threads", required=True, type=int) + parser.add_argument("--max-output-tokens", required=True, type=int) + args = parser.parse_args() + + sampler = ResponsesSampler( + model=args.model, + reasoning_model=True, + reasoning_effort=args.reasoning_effort, + base_url=args.base_url, + max_tokens=args.max_output_tokens, + ) + result = GPQAEval(n_repeats=8, n_threads=args.n_threads)(sampler) + # gpt-oss returns a NumPy scalar. Convert it so the parent process gets a + # stable, dependency-independent representation that its regex can parse. + print({"metric": float(result.score)}) + + +if __name__ == "__main__": + main() diff --git a/tests/evals/gpt_oss/test_gpqa_correctness.py b/tests/evals/gpt_oss/test_gpqa_correctness.py index 99d799698a..a57f3a567a 100644 --- a/tests/evals/gpt_oss/test_gpqa_correctness.py +++ b/tests/evals/gpt_oss/test_gpqa_correctness.py @@ -21,6 +21,12 @@ from tests.utils import RemoteOpenAIServer TOL = 0.05 # Absolute tolerance for accuracy comparison +# gpt-oss >= 0.0.7 explicitly sends its 131072-token default as the output +# budget, which leaves no room for the prompt in GPT-OSS's 131072-token context. +# This test originally ran with a 32768-token model context, so retain that +# proven generation budget while using the current evaluator package. +MAX_OUTPUT_TOKENS = 32_768 + # Path to tiktoken encoding files TIKTOKEN_DATA_DIR = Path(__file__).parent / "data" @@ -50,10 +56,7 @@ def run_gpqa_eval(model_name: str, base_url: str, reasoning_effort: str) -> floa # Build the command to run the evaluation cmd = [ sys.executable, - "-m", - "gpt_oss.evals", - "--eval", - "gpqa", + str(Path(__file__).with_name("run_gpqa_eval.py")), "--model", model_name, "--reasoning-effort", @@ -62,6 +65,8 @@ def run_gpqa_eval(model_name: str, base_url: str, reasoning_effort: str) -> floa base_url, "--n-threads", "200", + "--max-output-tokens", + str(MAX_OUTPUT_TOKENS), ] try: diff --git a/tests/kernels/core/test_apply_rotary_emb.py b/tests/kernels/core/test_apply_rotary_emb.py index a576199663..7d63c3499c 100644 --- a/tests/kernels/core/test_apply_rotary_emb.py +++ b/tests/kernels/core/test_apply_rotary_emb.py @@ -225,3 +225,46 @@ def test_apply_rotary_emb_xpu_matches_native( expected = op.forward_native(x, cos, sin) actual = op.forward_xpu(x, cos, sin) torch.testing.assert_close(actual, expected, atol=0, rtol=0) + + +@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="CUDA/ROCm only test.") +@pytest.mark.parametrize("num_tokens", [257, 2304]) +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) +def test_packed_qk_rope_correctness( + num_tokens: int, dtype: torch.dtype, default_vllm_config +): + """packed_qk_rope_ must be bitwise identical to the per-tensor + ApplyRotaryEmb path with enable_fp32_compute=True, and must leave the V + slice untouched.""" + from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb + from vllm.model_executor.layers.rotary_embedding.packed_qk_rope import ( + packed_qk_rope_, + ) + + num_heads, head_dim = 12, 128 + rng = torch.Generator(device="cuda").manual_seed(0) + + angles = ( + torch.rand(num_tokens, head_dim // 2, device="cuda", generator=rng) * torch.pi + ) + freqs_cis = torch.polar(torch.ones_like(angles), angles) + cos = freqs_cis.real.contiguous() + sin = freqs_cis.imag.contiguous() + + xqkv = torch.randn( + num_tokens, 3, num_heads, head_dim, dtype=dtype, device="cuda", generator=rng + ) + xqkv_ref = xqkv.clone() + xq, xk, xv = torch.unbind(xqkv_ref, dim=-3) + + op = ApplyRotaryEmb( + enforce_enable=True, is_neox_style=False, enable_fp32_compute=True + ) + # op() dispatches to forward_cuda / forward_hip per platform. + xq_ref = op(xq, cos, sin) + xk_ref = op(xk, cos, sin) + + packed_qk_rope_(xqkv, freqs_cis) + torch.testing.assert_close(xqkv[:, 0], xq_ref, atol=0, rtol=0) + torch.testing.assert_close(xqkv[:, 1], xk_ref, atol=0, rtol=0) + torch.testing.assert_close(xqkv[:, 2], xv, atol=0, rtol=0) diff --git a/tests/kernels/moe/test_flashinfer.py b/tests/kernels/moe/test_flashinfer.py index 5d55876e81..2a46a0a56a 100644 --- a/tests/kernels/moe/test_flashinfer.py +++ b/tests/kernels/moe/test_flashinfer.py @@ -42,6 +42,9 @@ from vllm.model_executor.layers.fused_moe.oracle.fp8 import ( convert_to_fp8_moe_kernel_format, make_fp8_moe_quant_config, ) +from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import ( + fused_topk_bias, +) from vllm.model_executor.layers.fused_moe.router.fused_topk_router import fused_topk from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( rotate_weights_for_fi_trtllm_fp8_per_tensor_moe, @@ -887,3 +890,97 @@ def test_trtllm_fp8_block_moe_deferred_finalize( ), router_weights=topk_weights, ) + + +@pytest.mark.parametrize("m", [1, 16]) +@pytest.mark.skipif( + not current_platform.is_device_capability_family(100), + reason="Requires TRTLLM-Gen FP8 MoE (SM100)", +) +def test_trtllm_mxfp8_minimax2_routing_applies_routed_scale(m: int, workspace_init): + """FlashInfer's fused MiniMax2 routing must apply routed_scaling_factor, + matching vLLM's own sigmoid+bias top-k routing (MiniMax-M3 uses 2.0).""" + e, topk, n, k, scale = 32, 4, 1024, 1024, 2.0 + block_shape = [1, 32] + set_random_seed(7) + with set_current_vllm_config(vllm_config): + w1, w2, w1_scale, w2_scale = convert_to_fp8_moe_kernel_format( + Fp8MoeBackend.FLASHINFER_TRTLLM, + SimpleNamespace( + weight_block_size=block_shape, + moe_config=SimpleNamespace( + is_act_and_mul=True, intermediate_size_per_partition=n + ), + activation=MoEActivation.SILU, + ), + *_make_mxfp8_moe_weights(e, n, k), + w13_input_scale=None, + w2_input_scale=None, + ) + quant_config = make_fp8_moe_quant_config( + Fp8MoeBackend.FLASHINFER_TRTLLM, + w1_scale, + w2_scale, + a1_scale=None, + a2_scale=None, + block_shape=block_shape, + ) + moe_config = FusedMoEConfig( + num_experts=e, + experts_per_token=topk, + hidden_dim=k, + intermediate_size=n, + num_local_experts=e, + num_logical_experts=e, + activation=MoEActivation.SILU, + device="cuda", + moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(), + in_dtype=torch.bfloat16, + routing_method=RoutingMethodType.MiniMax2, + max_num_tokens=next_power_of_2(m), + ) + + def make_kernel(experts_cls) -> mk.FusedMoEKernel: + return mk.FusedMoEKernel( + maybe_make_prepare_finalize( + moe=moe_config, + quant_config=quant_config, + allow_new_interface=True, + use_monolithic=experts_cls is TrtLlmFp8ExpertsMonolithic, + ), + experts_cls(moe_config=moe_config, quant_config=quant_config), + ) + + a = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) / 10 + logits = torch.randn((m, e), device="cuda", dtype=torch.float32) + bias = torch.randn(e, device="cuda", dtype=torch.float32) / 10 + common = dict( + w1=w1, + w2=w2, + activation=MoEActivation.SILU, + global_num_experts=e, + expert_map=None, + apply_router_weight_on_input=False, + ) + + fused = make_kernel(TrtLlmFp8ExpertsMonolithic).apply_monolithic( + hidden_states=a, + router_logits=logits, + e_score_correction_bias=bias, + routed_scaling_factor=scale, + **common, + ) + topk_weights, topk_ids = fused_topk_bias( + a, + logits, + scoring_func="sigmoid", + e_score_correction_bias=bias, + topk=topk, + renormalize=True, + routed_scaling_factor=scale, + ) + reference = make_kernel(TrtLlmFp8ExpertsModular).apply( + hidden_states=a, topk_weights=topk_weights, topk_ids=topk_ids, **common + ) + + torch.testing.assert_close(fused, reference, atol=2e-2, rtol=2e-2) diff --git a/tests/kernels/moe/test_ocp_mx_moe.py b/tests/kernels/moe/test_ocp_mx_moe.py index e4b11c9548..295a92482f 100644 --- a/tests/kernels/moe/test_ocp_mx_moe.py +++ b/tests/kernels/moe/test_ocp_mx_moe.py @@ -47,7 +47,6 @@ if ROCM_AVAILABLE: ROCM_GFX950 = on_gfx950() if ROCM_AITER_AVAILABLE: - from aiter.ops.triton.moe.quant_moe import upcast_from_mxfp from aiter.ops.triton.quant import dynamic_mxfp4_quant if TRTLLM_GEN_MXFP4_AVAILABLE: @@ -191,11 +190,7 @@ def mxfp8_dequantize(x, scale): def mxfp4_quant_dequant(x: torch.Tensor) -> torch.Tensor: shape = x.shape quantized, scale = dynamic_mxfp4_quant(x.to(torch.bfloat16).flatten(0, -2)) - return ( - upcast_from_mxfp(quantized.view(torch.uint8), scale, torch.bfloat16, axis=-1) - .reshape(shape) - .float() - ) + return mxfp4_dequantize(quantized.view(torch.uint8), scale).reshape(shape) def reference_moe( @@ -1809,12 +1804,8 @@ def test_rocm_mxfp4_moe_oracle( assert out.abs().max() > 0.01, "Output is effectively zero" # Dequantize weights for reference computation - w13_dq = upcast_from_mxfp( - w13_quant_ref.view(torch.uint8), w13_scale_ref, torch.bfloat16, axis=-1 - ) - w2_dq = upcast_from_mxfp( - w2_quant_ref.view(torch.uint8), w2_scale_ref, torch.bfloat16, axis=-1 - ) + w13_dq = mxfp4_dequantize(w13_quant_ref.view(torch.uint8), w13_scale_ref) + w2_dq = mxfp4_dequantize(w2_quant_ref.view(torch.uint8), w2_scale_ref) # Determine activation type and layout # SWIGLUOAI uses interleaved layout (gate/up alternating) diff --git a/tests/kernels/moe/test_rocm_aiter_moe.py b/tests/kernels/moe/test_rocm_aiter_moe.py index 8fa1706f18..fbc8586fd3 100644 --- a/tests/kernels/moe/test_rocm_aiter_moe.py +++ b/tests/kernels/moe/test_rocm_aiter_moe.py @@ -566,72 +566,64 @@ def test_aiter_moe_shared_experts_enablement_follows_env( assert rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() is expected -def test_aiter_moe_situv2_syncs_aiter_a4w4_env( - monkeypatch: pytest.MonkeyPatch, -): - """SiTUv2 MoE should route to AITER a4w4 and clear legacy a8w4 overrides.""" - from vllm._aiter_ops import rocm_aiter_ops - - _assert_aiter_supported() - - with monkeypatch.context() as mp: - mp.delenv("AITER_SITUV2_A8W4", raising=False) - mp.delenv("AITER_SITUV2_A4W4", raising=False) - mp.setenv("VLLM_ROCM_USE_AITER", "1") - mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1") - mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "1") - _reload_envs() - rocm_aiter_ops.refresh_env_variables() - - import os - - assert os.environ.get("AITER_SITUV2_A4W4") == "1" - assert "AITER_SITUV2_A8W4" not in os.environ - - -def test_aiter_moe_situv2_legacy_a8w4_alias_enables_a4w4( - monkeypatch: pytest.MonkeyPatch, -): - """Deprecated VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4 still enables SiTUv2 a4w4.""" - from vllm._aiter_ops import rocm_aiter_ops - - _assert_aiter_supported() - - with monkeypatch.context() as mp: - mp.delenv("AITER_SITUV2_A8W4", raising=False) - mp.delenv("AITER_SITUV2_A4W4", raising=False) - mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", raising=False) - mp.setenv("VLLM_ROCM_USE_AITER", "1") - mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1") - mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", "1") - _reload_envs() - rocm_aiter_ops.refresh_env_variables() - - import os - - assert os.environ.get("AITER_SITUV2_A4W4") == "1" - assert "AITER_SITUV2_A8W4" not in os.environ - - -def test_aiter_moe_situv2_clears_aiter_env_when_disabled( +@pytest.mark.parametrize( + "value,expected_act,expected_env", + [ + # Unset / auto / legacy "1" all mean the a4w4 default. + (None, "a4w4", {"AITER_SITUV2_A4W4": "1"}), + ("auto", "a4w4", {"AITER_SITUV2_A4W4": "1"}), + ("1", "a4w4", {"AITER_SITUV2_A4W4": "1"}), + ("a4w4", "a4w4", {"AITER_SITUV2_A4W4": "1"}), + ("0", "a16w4", {}), + ("a16w4", "a16w4", {}), + ("A8W4", "a8w4", {"AITER_SITUV2_A8W4": "1"}), + ], +) +def test_aiter_moe_situv2_activation_syncs_aiter_env( + value: str | None, + expected_act: str, + expected_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, ): + """Each SiTUv2 activation choice sets exactly its AITER dispatch env.""" + import os + from vllm._aiter_ops import rocm_aiter_ops _assert_aiter_supported() with monkeypatch.context() as mp: + # Stale AITER overrides must be replaced by the vLLM selection. + mp.setenv("AITER_SITUV2_A8W4", "1") mp.setenv("AITER_SITUV2_A4W4", "1") mp.setenv("VLLM_ROCM_USE_AITER", "1") mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1") - mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", raising=False) - mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "0") + if value is None: + mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", raising=False) + else: + mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", value) _reload_envs() rocm_aiter_ops.refresh_env_variables() - import os + assert rocm_aiter_ops.get_fused_moe_situv2_activation() == expected_act + assert rocm_aiter_ops.is_fused_moe_situv2_gate_up_interleaved() is ( + expected_act == "a8w4" + ) + assert rocm_aiter_ops.is_fused_moe_situv2_enabled() is (expected_act != "a16w4") + for name in ("AITER_SITUV2_A8W4", "AITER_SITUV2_A4W4"): + assert os.environ.get(name) == expected_env.get(name) - assert "AITER_SITUV2_A4W4" not in os.environ + +def test_aiter_moe_situv2_rejects_unknown_activation( + monkeypatch: pytest.MonkeyPatch, +): + with monkeypatch.context() as mp: + mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "a8w8") + _reload_envs() + import vllm.envs as envs + + with pytest.raises(ValueError, match="VLLM_ROCM_USE_AITER_MOE_SITUV2"): + _ = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2 @pytest.mark.parametrize("moe_padding", [True, False]) diff --git a/tests/kernels/moe/test_routing.py b/tests/kernels/moe/test_routing.py index 4baad24cc3..b0c3e26db9 100644 --- a/tests/kernels/moe/test_routing.py +++ b/tests/kernels/moe/test_routing.py @@ -167,6 +167,38 @@ def test_single_expert_group_with_non_unit_scale_uses_grouped_topk() -> None: assert isinstance(router, GroupedTopKRouter) +def test_sigmoid_bias_routing_with_routed_scale_is_minimax2() -> None: + """FlashInfer's MiniMax2 routing applies routed_scaling_factor, so a + non-unit scale (MiniMax-M3 uses 2.0) must not block fused routing.""" + router = create_fused_moe_router( + top_k=4, + global_num_experts=128, + scoring_func="sigmoid", + renormalize=True, + routed_scaling_factor=2.0, + e_score_correction_bias=torch.empty(128), + ) + + assert isinstance(router, FusedTopKBiasRouter) + assert router.routing_method_type == RoutingMethodType.MiniMax2 + + +def test_zero_expert_routing_is_unspecified() -> None: + """Zero experts are resolved in the router, so kernels with built-in + routing must never be selected for them.""" + router = create_fused_moe_router( + top_k=4, + global_num_experts=128, + scoring_func="sigmoid", + renormalize=True, + e_score_correction_bias=torch.empty(160), + zero_expert_type="identity", + num_logical_experts=160, + ) + + assert router.routing_method_type == RoutingMethodType.Unspecified + + def setup_eplb_state( enable_eplb: bool, global_num_experts: int ) -> EplbLayerState | None: diff --git a/tests/kernels/test_compressor_kv_cache.py b/tests/kernels/test_compressor_kv_cache.py index 06897f04c0..278c0ef5d0 100644 --- a/tests/kernels/test_compressor_kv_cache.py +++ b/tests/kernels/test_compressor_kv_cache.py @@ -2075,3 +2075,269 @@ def test_fused_kv_insert_split(num_tokens: int, kv_block_size: int): # RoPE (last 64): stored as bf16. The kernel recomputes the rotation, so it # is bf16-close to the reference rather than bit-exact (cf. test_cutedsl). torch.testing.assert_close(recovered[:, NOPE_DIM:], ref[:, NOPE_DIM:]) + + +@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm stream coverage") +def test_v41_rocm_csa2_full_pipeline(): + """_forward_csa2_full: one fork, then K write and q-side after the join. + + The default chain must run first on the current stream, the compressor + chain on aux 0 and the indexer weights projection on aux 1; the join must + publish both cache writes before _produce_k / forward_q run. + """ + from vllm.forward_context import ForwardContext, override_forward_context + from vllm.models.deepseek_v41.amd.rocm import DeepseekV41ROCMAiterMLAAttention + from vllm.models.deepseek_v41.compressor import DeepseekCompressor + + torch.manual_seed(43) + num_tokens = 19 + hidden_states = torch.randn(num_tokens, 1024, device="cuda") + positions = torch.arange(7, 26, device="cuda") + qr_kv = torch.zeros(num_tokens, 1536, device="cuda") + qr = torch.zeros(num_tokens, 4, device="cuda") + kv = torch.zeros(num_tokens, 512, device="cuda") + q = torch.zeros(num_tokens, 512, device="cuda") + latent = torch.ones(num_tokens, 512, dtype=torch.bfloat16, device="cuda") + weights = torch.randn(num_tokens, 32, device="cuda") + index_q = torch.full((num_tokens, 32, 128), 2.0, device="cuda") + index_q_scale = torch.full((num_tokens, 32), 3.0, device="cuda") + index_weights_out = torch.full((num_tokens, 32), 4.0, device="cuda") + main_cache = torch.full((1, 128, 584), 165, dtype=torch.uint8, device="cuda") + index_cache = torch.full((1, 128, 132), 165, dtype=torch.uint8, device="cuda") + + order: list[str] = [] + streams: dict[str, torch.cuda.Stream] = {} + + compressor = DeepseekCompressor.__new__(DeepseekCompressor) + torch.nn.Module.__init__(compressor) + compressor.fused_wkv_wgate = SimpleNamespace( + weight=torch.randn(1024, 1024, device="cuda") + ) + + def compressor_forward(kv_score, positions_): + order.append("compressor") + streams["compressor"] = torch.cuda.current_stream() + return latent + + def compressor_insert(latent_, positions_, rotary_emb): + order.append("insert_cache") + streams["insert_cache"] = torch.cuda.current_stream() + main_cache.fill_(5) + + compressor.forward = compressor_forward + compressor.insert_cache = compressor_insert + + def weights_proj(hidden_states_): + order.append("weights_proj") + streams["weights_proj"] = torch.cuda.current_stream() + return weights, None + + def produce_k(latent_, positions_, rotary_emb): + order.append("produce_k") + streams["produce_k"] = torch.cuda.current_stream() + index_cache.fill_(7) + + def forward_q(qr_, qr_scale, indexer_weights, positions_, rotary_emb): + order.append("forward_q") + streams["forward_q"] = torch.cuda.current_stream() + return index_q, index_q_scale, index_weights_out + + indexer = SimpleNamespace( + weights_proj=weights_proj, + _produce_k=produce_k, + forward_q=forward_q, + ) + rotary = SimpleNamespace(cos_sin_cache=torch.randn(32, 64, device="cuda")) + aux_streams = [torch.cuda.Stream(), torch.cuda.Stream()] + + def wq_b(qr_, qr_scale): + order.append("wq_b") + streams["wq_b"] = torch.cuda.current_stream() + return q + + def qnorm_insert(q_, kv_, positions_, attn_metadata): + order.append("qnorm_insert") + streams["qnorm_insert"] = torch.cuda.current_stream() + return q_ + + attention = SimpleNamespace( + compressor=compressor, + indexer=indexer, + aux_stream_list=aux_streams, + ln_events=[torch.cuda.Event() for _ in range(4)], + rotary_emb=rotary, + indexer_rotary_emb=rotary, + n_local_heads=1, + head_dim=512, + _fused_wqa_wkv_gemm=lambda hidden_states_: qr_kv, + _split_qkv_and_norm=lambda qr_kv_: (qr, None, kv), + _wq_b_proj=wq_b, + _fused_qnorm_rope_kv_insert=qnorm_insert, + ) + + context = ForwardContext({}, {}, {}) + with override_forward_context(context): + result = DeepseekV41ROCMAiterMLAAttention._forward_csa2_full( + attention, hidden_states, positions + ) + + q_out, kv_out, index_q_out, index_q_scale_out, index_weights_out_out = result + assert torch.equal(q_out, q) + assert torch.equal(kv_out, kv) + assert torch.equal(index_q_out, index_q) + assert torch.equal(index_q_scale_out, index_q_scale) + assert torch.equal(index_weights_out_out, index_weights_out) + # Join publishes the aux-0 cache insert and orders the post-join K write. + assert torch.equal(main_cache, torch.full_like(main_cache, 5)) + assert torch.equal(index_cache, torch.full_like(index_cache, 7)) + + assert order == [ + "wq_b", + "qnorm_insert", + "compressor", + "insert_cache", + "weights_proj", + "produce_k", + "forward_q", + ] + current = torch.cuda.current_stream() + for name in ("wq_b", "qnorm_insert", "produce_k", "forward_q"): + assert streams[name] == current, name + for name in ("compressor", "insert_cache"): + assert streams[name] == aux_streams[0], name + assert streams["weights_proj"] == aux_streams[1] + + +@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm stream coverage") +def test_v41_rocm_csa2_reindex_pipeline(): + """_forward_csa2_reindex: serial projections, then SWA q path vs q-side.""" + from vllm.forward_context import ForwardContext, override_forward_context + from vllm.models.deepseek_v41.amd.rocm import DeepseekV41ROCMAiterMLAAttention + + torch.manual_seed(45) + num_tokens = 19 + hidden_states = torch.randn(num_tokens, 1024, device="cuda") + positions = torch.arange(7, 26, device="cuda") + qr_kv = torch.zeros(num_tokens, 1536, device="cuda") + qr = torch.zeros(num_tokens, 4, device="cuda") + kv = torch.zeros(num_tokens, 512, device="cuda") + q = torch.zeros(num_tokens, 512, device="cuda") + indexer_weights = torch.randn(num_tokens, 32, device="cuda") + index_q = torch.full((num_tokens, 32, 128), 2.0, device="cuda") + index_q_scale = torch.full((num_tokens, 32), 3.0, device="cuda") + index_weights_out = torch.full((num_tokens, 32), 4.0, device="cuda") + + order: list[str] = [] + streams: dict[str, torch.cuda.Stream] = {} + + def forward_q(qr_, qr_scale, indexer_weights_, positions_, rotary_emb): + order.append("forward_q") + streams["forward_q"] = torch.cuda.current_stream() + return index_q, index_q_scale, index_weights_out + + indexer = SimpleNamespace(forward_q=forward_q) + rotary = SimpleNamespace(cos_sin_cache=torch.randn(32, 64, device="cuda")) + aux0 = torch.cuda.Stream() + + def wq_b(qr_, qr_scale): + order.append("wq_b") + streams["wq_b"] = torch.cuda.current_stream() + return q + + def qnorm_insert(q_, kv_, positions_, attn_metadata): + order.append("qnorm_insert") + streams["qnorm_insert"] = torch.cuda.current_stream() + return q_ + + attention = SimpleNamespace( + indexer=indexer, + aux_stream_list=[aux0], + ln_events=[torch.cuda.Event() for _ in range(2)], + indexer_rotary_emb=rotary, + n_local_heads=1, + head_dim=512, + _run_parallel_input_projections=lambda hidden_states_: ( + qr_kv, + None, + indexer_weights, + ), + _split_qkv_and_norm=lambda qr_kv_: (qr, None, kv), + _wq_b_proj=wq_b, + _fused_qnorm_rope_kv_insert=qnorm_insert, + ) + + context = ForwardContext({}, {}, {}) + with override_forward_context(context): + result = DeepseekV41ROCMAiterMLAAttention._forward_csa2_reindex( + attention, hidden_states, positions + ) + + q_out, kv_out, index_q_out, index_q_scale_out, index_weights_out_out = result + assert torch.equal(q_out, q) + assert torch.equal(kv_out, kv) + assert torch.equal(index_q_out, index_q) + assert torch.equal(index_q_scale_out, index_q_scale) + assert torch.equal(index_weights_out_out, index_weights_out) + + assert order == ["wq_b", "qnorm_insert", "forward_q"] + current = torch.cuda.current_stream() + for name in ("wq_b", "qnorm_insert"): + assert streams[name] == current, name + assert streams["forward_q"] == aux0 + + +@pytest.mark.parametrize("quant_tuple", [False, True]) +def test_v41_indexer_forward_q(quant_tuple): + """forward_q runs wq_b then the fused RoPE/quant and unpacks both shapes.""" + from unittest.mock import patch + + from vllm.models.deepseek_v41.attention import DeepseekV4Indexer + + num_tokens, n_head, head_dim = 5, 32, 128 + qr = torch.randn(num_tokens, 1280) + q = torch.randn(num_tokens, n_head * head_dim) + positions = torch.arange(num_tokens) + indexer_weights = torch.randn(num_tokens, n_head) + rotary = SimpleNamespace(cos_sin_cache=torch.randn(32, 64)) + fake = SimpleNamespace( + n_head=n_head, + head_dim=head_dim, + softmax_scale=head_dim**-0.5, + use_fp4_kv=False, + indexer_weights_dtype=torch.float32, + _wq_b_proj=lambda qr_, qr_scale: q, + ) + q_quant = torch.randn(num_tokens, n_head, head_dim) + q_scale = torch.randn(num_tokens, n_head) + weights_out = torch.randn(num_tokens, n_head) + if quant_tuple: + fused_return = ((q_quant, q_scale), weights_out) + expected = (q_quant, q_scale, weights_out) + else: + fused_return = (q_quant, weights_out) + expected = (q_quant, None, weights_out) + + with patch( + "vllm.models.deepseek_v41.attention.fused_indexer_q_rope_quant", + return_value=fused_return, + ) as mock_fused: + result = DeepseekV4Indexer.forward_q( + fake, qr, None, indexer_weights, positions, rotary + ) + + assert torch.equal(result[0], expected[0]) + assert (result[1] is None) == (expected[1] is None) + if expected[1] is not None: + assert torch.equal(result[1], expected[1]) + assert torch.equal(result[2], expected[2]) + args = mock_fused.call_args.args + assert args[0] is positions + torch.testing.assert_close(args[1], q.view(-1, n_head, head_dim)) + assert args[2] is rotary.cos_sin_cache + assert args[3] is indexer_weights + assert args[4] == head_dim**-0.5 + assert args[5] == n_head**-0.5 + assert mock_fused.call_args.kwargs == { + "use_fp4": False, + "weights_out_dtype": torch.float32, + } diff --git a/tests/kernels/test_fused_inv_rope_fp8_quant.py b/tests/kernels/test_fused_inv_rope_fp8_quant.py index 03d5b7921f..3b5b24fe49 100644 --- a/tests/kernels/test_fused_inv_rope_fp8_quant.py +++ b/tests/kernels/test_fused_inv_rope_fp8_quant.py @@ -21,7 +21,12 @@ Usage: import pytest import torch +from vllm.model_executor.layers.quantization.utils.fp8_utils import ( + deepgemm_post_process_fp8_weight_block, +) from vllm.models.deepseek_v4.common.ops import fused_inv_rope_fp8_quant +from vllm.platforms import current_platform +from vllm.utils.deep_gemm import fp8_einsum # -- Default dimensions matching DeepSeek V3/V4 -------------------------- HEAD_DIM = 512 @@ -33,6 +38,49 @@ FP8_DTYPE = torch.float8_e4m3fn EPS = 1e-10 +@pytest.mark.skipif( + not current_platform.is_device_capability_family(100), reason="Requires SM100/SM103" +) +@pytest.mark.parametrize("groups", [2, 4]) +@pytest.mark.parametrize("tokens", [1, 6, 18, 32]) +def test_dsv41_fused_wo_a_matches_deepgemm(groups, tokens): + """Fused WO-A matches inverse RoPE + fp8_einsum + MXFP8 quantize.""" + from flashinfer import mxfp8_quantize + + from vllm.models.deepseek_v41.nvidia.ops.fused_wo_a import _FUSED_WO_A_KERNEL + + torch.manual_seed(0) + n = groups * 1024 + x = torch.randn(tokens, 64, 512, device="cuda", dtype=torch.bfloat16)[:, : n // 128] + positions = torch.randint(0, 128, (tokens,), device="cuda") + rope = make_cos_sin_cache(128, device="cuda") + wq = torch.randn(n, 4096, device="cuda").to(FP8_DTYPE) + scale = torch.exp2(torch.randint(-8, 0, (n, 128), device="cuda").float()) + wq, ws = deepgemm_post_process_fp8_weight_block( + wq, scale, (1, 32), False, True, groups + ) + q, sf = _FUSED_WO_A_KERNEL( + x=x, positions=positions, rope=rope, weight=wq, weight_scale=ws + ) + + a, sa = fused_inv_rope_fp8_quant( + x, positions, rope, groups, 8, quant_group_size=32, tma_aligned_scales=True + ) + y = torch.empty(tokens, groups, 1024, device="cuda", dtype=torch.bfloat16) + fp8_einsum("bhr,hdr->bhd", (a, sa), (wq, ws), y, recipe=(1, 1, 32)) + ref, ref_sf = mxfp8_quantize(y.flatten(1), backend="cute-dsl") + + def rows(scales): # F8_128x4 bytes -> [128 rows, n / 32 blocks] + return scales.view(n // 128, 32, 4, 4).permute(2, 1, 0, 3).reshape(128, -1) + + actual, expected = ( + d.float().view(tokens, -1, 32) * torch.exp2(rows(s)[:tokens, :, None] - 127.0) + for d, s in ((q, sf), (ref, ref_sf)) + ) + assert (actual - expected).norm() / expected.norm() < 3e-3 + assert not rows(sf)[tokens:].any(), "padded scale rows must be zero" + + # ========================================================================= # Helpers # ========================================================================= diff --git a/tests/lora/test_olmoe_tp.py b/tests/lora/test_olmoe_tp.py index bbc25cb6b8..c9a4fd6df5 100644 --- a/tests/lora/test_olmoe_tp.py +++ b/tests/lora/test_olmoe_tp.py @@ -139,6 +139,7 @@ def test_olmoe_lora_mixed(olmoe_lora_files): max_loras=4, enforce_eager=True, trust_remote_code=True, + hf_overrides={"head_dtype": "float32"}, enable_chunked_prefill=True, ) diff --git a/tests/model_executor/model_loader/test_weight_cache.py b/tests/model_executor/model_loader/test_weight_cache.py index 2280fff8b7..e4ac86e0eb 100644 --- a/tests/model_executor/model_loader/test_weight_cache.py +++ b/tests/model_executor/model_loader/test_weight_cache.py @@ -12,6 +12,8 @@ import subprocess import sys import tempfile import threading +import urllib.error +import urllib.request from dataclasses import dataclass, field from typing import Any @@ -20,6 +22,7 @@ import pytest from vllm import SamplingParams from vllm.assets.image import ImageAsset from vllm.platforms import current_platform +from vllm.utils.network_utils import get_open_port class WeightCacheDaemon: @@ -33,6 +36,7 @@ class WeightCacheDaemon: ): # Short base path: Unix socket paths are limited to ~107 characters. self.socket_dir = tempfile.mkdtemp(prefix="vllm_ipc_") + self.health_port = get_open_port() self._cmd = [ sys.executable, "-m", @@ -45,6 +49,10 @@ class WeightCacheDaemon: "--weight-cache-socket-dir", self.socket_dir, "--enforce-eager", + "--weight-cache-health-host", + "127.0.0.1", + "--weight-cache-health-port", + str(self.health_port), *(extra_args or []), ] self._proc: subprocess.Popen | None = None @@ -63,6 +71,16 @@ class WeightCacheDaemon: assert self._proc is not None and self._proc.stderr is not None self._proc.stderr.read() + def check_health(self) -> None: + url = f"http://127.0.0.1:{self.health_port}/health" + try: + with urllib.request.urlopen(url, timeout=5) as response: + assert response.status == 200 + except urllib.error.HTTPError as error: + raise AssertionError( + f"Weight cache daemon is not ready: HTTP {error.code}" + ) from error + def _stop(self) -> None: assert self._proc is not None self._proc.terminate() @@ -200,6 +218,7 @@ def test_ipc_cache_cold_start_and_warm_restart(vllm_runner, case: ModelCase): extra_args=case.daemon_args, ) as d: warm_outputs = generate(vllm_runner, case, d.socket_dir, fallback=False) + d.check_health() # Warm restart: a second engine lifetime against the same daemon. restart_outputs = generate(vllm_runner, case, d.socket_dir, fallback=False) diff --git a/tests/models/kimi_k3/test_kda_metadata.py b/tests/models/kimi_k3/test_kda_metadata.py index 6268cb9bcc..6b3c71bdc6 100644 --- a/tests/models/kimi_k3/test_kda_metadata.py +++ b/tests/models/kimi_k3/test_kda_metadata.py @@ -49,6 +49,7 @@ PRUNED_METADATA_FIELDS = { "prefill_state_indices", "prefill_has_initial_state", "spec_sequence_masks", + "spec_sequence_masks_cpu", "uniform_spec_sequence_length", "flashinfer_prefill_query_start_loc", "flashinfer_prefill_seq_order", diff --git a/tests/models/multimodal/generation/test_vit_cudagraph.py b/tests/models/multimodal/generation/test_vit_cudagraph.py index da472a164d..8c7cf1a68e 100644 --- a/tests/models/multimodal/generation/test_vit_cudagraph.py +++ b/tests/models/multimodal/generation/test_vit_cudagraph.py @@ -175,6 +175,24 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = { }, marks=[pytest.mark.core_model], ), + "minimax_m3": VitCudagraphTestConfig( + model="MiniMaxAI/MiniMax-M3", + modalities=["image"], + image_prompt=("]~b]user\n]<]image[>[What is in this image?[e~[\n]~b]ai\n"), + compilation_config_overrides={ + "encoder_cudagraph_token_budgets": [512], + }, + vllm_runner_kwargs={ + "trust_remote_code": True, + "load_format": "dummy", + "hf_overrides": partial( + dummy_hf_overrides, + model_arch="MiniMaxM3SparseForConditionalGeneration", + ), + "mm_processor_kwargs": {"max_long_side_pixel": 336}, + }, + marks=[pytest.mark.core_model], + ), "qwen3_vl": VitCudagraphTestConfig( model="Qwen/Qwen3-VL-2B-Instruct", image_prompt=qwen_vl_chat_template( diff --git a/tests/models/multimodal/test_mimo_v2_omni.py b/tests/models/multimodal/test_mimo_v2_omni.py index d4e1d9ec18..deccace50d 100644 --- a/tests/models/multimodal/test_mimo_v2_omni.py +++ b/tests/models/multimodal/test_mimo_v2_omni.py @@ -5,13 +5,7 @@ import pytest import torch -from tests.utils import ensure_current_vllm_config -from vllm.distributed.parallel_state import ( - init_distributed_environment, - initialize_model_parallel, -) from vllm.platforms import current_platform -from vllm.utils.network_utils import get_open_port EMBED_DIM = 256 NUM_HEADS = 4 @@ -21,20 +15,11 @@ WINDOW = 8 SEQ_LENS = [5, 37] -@pytest.fixture(scope="module") -def vision_attn_env(): - init_distributed_environment( - world_size=1, - rank=0, - local_rank=0, - distributed_init_method=f"tcp://127.0.0.1:{get_open_port()}", - backend="nccl", - ) +@pytest.fixture +def vision_attn_env(dist_init): default_dtype = torch.get_default_dtype() torch.set_default_dtype(torch.bfloat16) - with ensure_current_vllm_config(): - initialize_model_parallel(tensor_model_parallel_size=1) - yield + yield torch.set_default_dtype(default_dtype) diff --git a/tests/parser/test_abstract_parser.py b/tests/parser/test_abstract_parser.py new file mode 100644 index 0000000000..56d20417d1 --- /dev/null +++ b/tests/parser/test_abstract_parser.py @@ -0,0 +1,305 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import json +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from xgrammar import Grammar +from xgrammar.testing import _is_grammar_accept_string + +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionRequest, + ChatCompletionToolsParam, +) +from vllm.parser.abstract_parser import DelegatingParser +from vllm.sampling_params import StructuredOutputsParams +from vllm.tool_parsers.abstract_tool_parser import ToolParser +from vllm.tool_parsers.qwen3_engine_tool_parser import Qwen3EngineToolParser +from vllm.tool_parsers.structural_tag_registry import ToolChoice + + +class TestToolChoice_Plus_ResponseFormat: + """Note(arpera): + Test cases for tool_choice={auto,required} + response_format + To keep it short: + DelegatingParser.adjust_request behavior in some corner cases is checked there + + Initial bug report: + https://github.com/vllm-project/vllm/issues/39929 + And PR that fixed this: + https://github.com/vllm-project/vllm/pull/56086 + """ + + # ================================ + # Helper methods + # ================================ + + @staticmethod + def _tools(strict: bool) -> list[ChatCompletionToolsParam]: + """Single get_weather tool, optionally marked as strict""" + function: dict[str, Any] = { + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + } + if strict: + function["strict"] = True + return [ChatCompletionToolsParam(type="function", function=function)] + + @staticmethod + def _qwen_tool_call() -> str: + """Tool call for Qwen model that do support structural tag""" + return ( + "\n\n\nParis\n" + "\n\n" + ) + + @staticmethod + def _json_schema_response_format() -> dict: + return { + "type": "json_schema", + "json_schema": { + "name": "answer", + "schema": { + "type": "object", + "properties": {"text": {"type": "string"}}, + "required": ["text"], + }, + }, + } + + @staticmethod + def _setup_request( + tools: list[ChatCompletionToolsParam], + tool_choice: ToolChoice, + response_format: dict, + ) -> ChatCompletionRequest: + request = ChatCompletionRequest( + messages=[], # for our test cases it's always empty + model="m", # Just a placeholder, don't pay much attention + tools=tools, + tool_choice=tool_choice, + response_format=response_format, + ) + return request + + @staticmethod + def _setup_abstract_parser( + tools: list[ChatCompletionToolsParam], + ) -> DelegatingParser: + """Construct parser that does NOT support structural tag""" + + class TestParser(DelegatingParser): + tool_parser_cls = ToolParser + + return TestParser(MagicMock(), tools=tools) + + @staticmethod + def _setup_qwen_parser( + tools: list[ChatCompletionToolsParam], + ) -> DelegatingParser: + """Construct parser that supports structural tag""" + + class TestParser(DelegatingParser): + tool_parser_cls = Qwen3EngineToolParser + + return TestParser(MagicMock(), tools=tools) + + # ================================ + # Test cases + # tool_choice=auto + response_format + # ================================ + + @pytest.mark.parametrize( + # In this test we check that for response_format + # resulting grammar accepts @compliant_output and rejects @non_compliant_output + # You can add more examples here if you see some corner cases not covered + ("response_format", "compliant_output", "non_compliant_output"), + [ + (_json_schema_response_format(), '{"text": "hi"}', '{"foo": 1}'), + ({"type": "json_object"}, '{"any": 1}', "[1, 2]"), + ], + # We test here two different response_format types: + ids=["json_schema", "json_object"], + ) + def test_auto_with_strict_tools( + self, + response_format: dict, + compliant_output: str, + non_compliant_output: str, + ): + tools = self._tools(strict=True) + request = self._setup_request( + tools=tools, + tool_choice="auto", + response_format=response_format, + ) + parser = self._setup_qwen_parser(tools) + out = parser.adjust_request(request) + + # Now check that request does not have response_format anymore + # but instead has structured_outputs set as structural tag + # And this structural tag is OR operation + assert out.tool_choice == "auto" + assert out.response_format is None + assert out.structured_outputs is not None + tag = json.loads(out.structured_outputs.structural_tag) + assert tag["format"]["type"] == "or" + grammar = Grammar.from_structural_tag(out.structured_outputs.structural_tag) + + assert _is_grammar_accept_string(grammar, compliant_output) + assert not _is_grammar_accept_string(grammar, non_compliant_output) + + # Also tool call must be accepted by grammar + assert _is_grammar_accept_string(grammar, self._qwen_tool_call()) + + # IMPORTANT(arpera): Regression test + # If we in adjust_request implementation by mistake + # construct structural tag using tool_choice=auto + # then such a structural tag would allow plain text as well + # We need to be sure that plain text is NOT accepted in our case + assert not _is_grammar_accept_string(grammar, "Hello") + + def test_auto_without_strict_tools(self): + tools = self._tools(strict=False) + request = self._setup_request( + tools=tools, + tool_choice="auto", + response_format=self._json_schema_response_format(), + ) + parser = self._setup_qwen_parser(tools) + + # There must be a warning that tool calls are disabled + # Consume that warning + with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn: + out = parser.adjust_request(request) + + assert out.response_format is not None + assert out.structured_outputs is None + mock_warn.assert_called_once() + + def test_when_model_does_not_have_structural_tag(self): + """Note(arpera): + When model does NOT have structural tag support + we apply constraint only for response_format + """ + tools = self._tools(strict=True) + request = self._setup_request( + tools=tools, + tool_choice="auto", + response_format=self._json_schema_response_format(), + ) + # SIC! use parser whose model does NOT support structural tag + parser = self._setup_abstract_parser(tools) + + with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn: + out = parser.adjust_request(request) + + assert out.response_format is not None + assert out.structured_outputs is None + mock_warn.assert_called_once() + + def test_auto_with_xgrammar_unsupported_schema(self): + """Conrer case based on Vadim's feedback in PR #56086 + https://github.com/vllm-project/vllm/pull/56086#pullrequestreview-5329203459 + """ + tools = self._tools(strict=True) + request = self._setup_request( + tools=tools, + tool_choice="auto", + response_format={ + "type": "json_schema", + "json_schema": { + "name": "answer", + "schema": {"type": "integer", "multipleOf": 2}, + }, + }, + ) + parser = self._setup_qwen_parser(tools) + + with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn: + out = parser.adjust_request(request) + + assert out.response_format is not None + assert out.structured_outputs is None + mock_warn.assert_called_once() + + def test_auto_with_xgrammar_unsupported_nested_schema(self): + """Conrer case based on Vadim's feedback in PR #56086 + https://github.com/vllm-project/vllm/pull/56086#pullrequestreview-5329203459 + """ + tools = self._tools(strict=True) + request = self._setup_request( + tools=tools, + tool_choice="auto", + response_format={ + "type": "json_schema", + "json_schema": { + "name": "answer", + "schema": { + "type": "object", + "properties": {"value": {"type": "number", "multipleOf": 0.5}}, + "required": ["value"], + }, + }, + }, + ) + parser = self._setup_qwen_parser(tools) + + with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn: + out = parser.adjust_request(request) + + assert out.response_format is not None + assert out.structured_outputs is None + mock_warn.assert_called_once() + + def test_auto_with_lark_grammar(self): + """Conrer case based on Vadim's feedback in PR #56086 + https://github.com/vllm-project/vllm/pull/56086#pullrequestreview-5329203459 + Lark grammars are converted to EBNF before being merged into the tag. + """ + tools = self._tools(strict=True) + request = ChatCompletionRequest( + messages=[], + model="m", + tools=tools, + tool_choice="auto", + structured_outputs=StructuredOutputsParams(grammar='start: "ok"'), + ) + parser = self._setup_qwen_parser(tools) + out = parser.adjust_request(request) + + assert out.structured_outputs is not None + grammar = Grammar.from_structural_tag(out.structured_outputs.structural_tag) + assert _is_grammar_accept_string(grammar, "ok") + assert not _is_grammar_accept_string(grammar, "bad") + assert _is_grammar_accept_string(grammar, self._qwen_tool_call()) + + # ================================ + # Test cases + # tool_choice=required + response_format + # ================================ + + def test_required(self): + tools = self._tools(strict=True) + request = self._setup_request( + tools=tools, + tool_choice="required", + response_format=self._json_schema_response_format(), + ) + parser = self._setup_qwen_parser(tools) + + with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn: + out = parser.adjust_request(request) + + assert out.response_format is None + assert out.structured_outputs is not None + grammar = Grammar.from_structural_tag(out.structured_outputs.structural_tag) + assert _is_grammar_accept_string(grammar, self._qwen_tool_call()) + assert not _is_grammar_accept_string(grammar, '{"text": "hi"}') + mock_warn.assert_called_once() diff --git a/tests/parser/test_harmony.py b/tests/parser/test_harmony.py index 3d6fff04bf..d8db7f4057 100644 --- a/tests/parser/test_harmony.py +++ b/tests/parser/test_harmony.py @@ -1196,6 +1196,21 @@ class TestAdjustRequest: "TOOL_CALL_2_FUNCTION_FIRST", ], ), + ( + { + "tool_choice": "auto", + "strict_tools": True, + "response_format_type": "json_schema", + }, + [ + "COMMENTARY", + "TOOL_CALL_1_CHANNEL_FIRST", + "TOOL_CALL_1_FUNCTION_FIRST", + "TOOL_CALL_2_CHANNEL_FIRST", + "TOOL_CALL_2_FUNCTION_FIRST", + "FINAL_JSON_SCHEMA", + ], + ), ( {"response_format_type": "json_schema"}, ["FINAL_JSON_SCHEMA"], @@ -1281,6 +1296,7 @@ class TestAdjustRequest: "tool_auto_strict", "tool_required", "tool_named", + "pr56086_tool_auto_strict_response_format_json_schema", "response_format_json_schema", "response_format_json_object", "structured_outputs_json", diff --git a/tests/v1/attention/test_attention_backends.py b/tests/v1/attention/test_attention_backends.py index 2f8b80a7c1..77366c02ae 100644 --- a/tests/v1/attention/test_attention_backends.py +++ b/tests/v1/attention/test_attention_backends.py @@ -19,7 +19,12 @@ from tests.v1.attention.utils import ( try_backend_includes_kv_cache_update, try_get_attention_backend, ) -from vllm.config import ModelConfig, set_current_vllm_config +from vllm.config import ( + DiffusionConfig, + ModelConfig, + SpeculativeConfig, + set_current_vllm_config, +) from vllm.platforms import current_platform from vllm.utils.math_utils import cdiv from vllm.utils.torch_utils import ( @@ -942,8 +947,6 @@ def test_flashinfer_xqa_single_token_decode_preserves_cudagraph_padding(monkeypa impl.o_sf_scale = None impl.window_left = -1 impl.sinks = None - impl.cache_config = unittest.mock.Mock() - impl.cache_config.get_resolved_kv_cache_layout.return_value = KVCacheLayout.LBHNC layer = unittest.mock.Mock( _q_scale=torch.tensor(1.0), @@ -959,6 +962,7 @@ def test_flashinfer_xqa_single_token_decode_preserves_cudagraph_padding(monkeypa q_len_per_req=1, ) attn_metadata = flashinfer_backend.FlashInferMetadata( + kv_cache_layout=KVCacheLayout.LBHNC, num_actual_tokens=2, slot_mapping=torch.empty(0, dtype=torch.int64), q_data_type_prefill=torch.bfloat16, @@ -1070,6 +1074,87 @@ def test_flashinfer_trtllm_gen_padded_decode_uses_varlen_offsets( assert q_cu_seq_lens.tolist() == qo_indptr_values +@pytest.mark.skipif( + not current_platform.is_cuda() + or AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST, + reason="FlashInfer with CUDA is required.", +) +@pytest.mark.parametrize( + "runner_type,spec_kind,has_upper_bound,expected_copies", + [ + ("generate", None, True, 0), + ("pooling", None, True, 0), + ("generate", None, False, 1), + ("generate", "spec_decode", True, 1), + ("generate", "diffusion", True, 1), + ], +) +def test_flashinfer_avoids_seq_lens_copy_without_spec_tokens( + monkeypatch, runner_type, spec_kind, has_upper_bound, expected_copies +): + """Planning reuses the host seq_lens bound only when it is exact. + + Without speculative token accounting (no spec-decode drafts, no dLLM + canvas tokens) scheduled tokens always land in KV, so the CPU upper bound + is exact regardless of runner type. Any speculative accounting keeps the + bound optimistic (rejected tokens are in-flight) and forces the copy. + """ + import unittest.mock + + from vllm.v1.attention.backends import flashinfer as flashinfer_backend + from vllm.v1.attention.backends.utils import PerLayerParameters + + monkeypatch.setattr( + flashinfer_backend, + "get_per_layer_parameters", + lambda *args: { + "layer.0": PerLayerParameters( + window_left=-1, logits_soft_cap=0.0, sm_scale=0.125 + ) + }, + ) + + config = create_vllm_config( + model_name="Qwen/Qwen3-0.6B", max_model_len=64, max_num_seqs=3 + ) + config.model_config.runner_type = runner_type + config.attention_config.use_trtllm_attention = False + if spec_kind == "spec_decode": + config.speculative_config = SpeculativeConfig( + method="ngram", num_speculative_tokens=3 + ) + elif spec_kind == "diffusion": + config.diffusion_config = DiffusionConfig(canvas_length=8) + device = torch.device(f"{DEVICE_TYPE}:0") + # Include a one-token final chunk and both partial and full KV pages. + common_attn_metadata = create_common_attn_metadata( + BatchSpec(seq_lens=[17, 20, 32], query_lens=[1, 4, 16]), 16, device + ) + if not has_upper_bound: + common_attn_metadata.seq_lens_cpu_upper_bound = None + elif spec_kind is not None: + # Simulate rejected speculative tokens: CPU says 18, GPU says 17. + assert common_attn_metadata.seq_lens_cpu_upper_bound is not None + common_attn_metadata.seq_lens_cpu_upper_bound[0] += 1 + + with set_current_vllm_config(config): + builder = flashinfer_backend.FlashInferMetadataBuilder( + create_standard_kv_cache_spec(config), ["layer.0"], config, device + ) + with unittest.mock.patch.object( + common_attn_metadata.seq_lens, + "cpu", + wraps=common_attn_metadata.seq_lens.cpu, + ) as copy_to_cpu: + metadata = builder.build(0, common_attn_metadata) + + assert copy_to_cpu.call_count == expected_copies + assert metadata.num_decodes == 1 + assert metadata.num_prefills == 2 + assert builder.paged_kv_indptr.cpu[:4].tolist() == [0, 2, 4, 6] + assert builder.paged_kv_last_page_len.cpu[:3].tolist() == [1, 4, 16] + + @pytest.mark.skipif( AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST, reason="FlashInfer is not available.", diff --git a/tests/v1/attention/test_cpu_mla_backend.py b/tests/v1/attention/test_cpu_mla_backend.py index fc3867099e..938babb262 100644 --- a/tests/v1/attention/test_cpu_mla_backend.py +++ b/tests/v1/attention/test_cpu_mla_backend.py @@ -25,6 +25,9 @@ from vllm.v1.attention.backends.mla.prefill.cpu_sdpa import ( from vllm.v1.attention.backends.mla.prefill.selector import ( get_mla_prefill_backend, ) +from vllm.v1.attention.backends.mla.prefill.zen_cpu_sdpa import ( + ZenCPUSDPAMLAPrefillBackend, +) def _reference_ragged_attention( @@ -157,8 +160,10 @@ def test_cpu_mla_backend_smoke(tmp_path) -> None: def test_cpu_mla_prefill_backend_selected() -> None: # On CPU (no device capability) the MLA prefill backend must be the # SDPA-based CPU backend, not flash-attn which is unavailable on CPU. + # Accelerator-specific subclasses (e.g. the zentorch-backed Zen CPU + # backend) are also valid selections. backend_cls = get_mla_prefill_backend(None) - assert backend_cls is CPUSDPAMLAPrefillBackend + assert issubclass(backend_cls, CPUSDPAMLAPrefillBackend) @pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only") @@ -248,3 +253,64 @@ def test_cpu_mla_prefill_context_chunk() -> None: assert lse.float().allclose(ref_lse.float(), atol=2e-2) assert out.shape == (5, 4, 16) assert lse.shape == (4, 5) + + +@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only") +@pytest.mark.skipif( + not ZenCPUSDPAMLAPrefillBackend.is_available(), + reason="requires a Zen CPU with the zentorch_sdpa op registered", +) +def test_zen_cpu_mla_prefill_new_tokens() -> None: + # Runs the zentorch kernel itself. The head dims are deliberately + # asymmetric (v_head_dim < qk_head_dim, as in real MLA) so that the V + # padding and output slicing are exercised; the generic-CPU cases above + # use equal dims and never reach that code. + qk_head_dim = 24 + v_head_dim = 16 + backend = ZenCPUSDPAMLAPrefillBackend( + num_heads=4, + scale=0.25, + kv_lora_rank=16, + qk_nope_head_dim=16, + qk_rope_head_dim=8, + v_head_dim=v_head_dim, + vllm_config=None, + ) + + class _PrefillMeta: + query_start_loc = torch.tensor([0, 3, 6], dtype=torch.int32) + + backend.prepare_metadata(_PrefillMeta()) + q = torch.randn(6, 4, qk_head_dim, dtype=torch.bfloat16) + k = torch.randn(6, 4, qk_head_dim, dtype=torch.bfloat16) + v = torch.randn(6, 4, v_head_dim, dtype=torch.bfloat16) + + out = backend.run_prefill_new_tokens(q, k, v, return_softmax_lse=False) + ref, _ = _reference_ragged_attention( + q, + k, + v, + _PrefillMeta.query_start_loc, + _PrefillMeta.query_start_loc, + scale=0.25, + causal=True, + ) + assert out.shape == (6, 4, v_head_dim) + assert out.float().allclose(ref.float(), atol=2e-2) + + # zentorch_sdpa does not return a usable LSE, so that path must fall back + # to the fp32 reference and match the generic backend exactly. + generic = CPUSDPAMLAPrefillBackend( + num_heads=4, + scale=0.25, + kv_lora_rank=16, + qk_nope_head_dim=16, + qk_rope_head_dim=8, + v_head_dim=v_head_dim, + vllm_config=None, + ) + generic.prepare_metadata(_PrefillMeta()) + zen_out, zen_lse = backend.run_prefill_new_tokens(q, k, v, return_softmax_lse=True) + gen_out, gen_lse = generic.run_prefill_new_tokens(q, k, v, return_softmax_lse=True) + assert torch.equal(zen_out, gen_out) + assert torch.equal(zen_lse, gen_lse) diff --git a/tests/v1/attention/test_flashinfer_mla_sparse_sm90.py b/tests/v1/attention/test_flashinfer_mla_sparse_sm90.py index 3428625e96..b25656cf6e 100644 --- a/tests/v1/attention/test_flashinfer_mla_sparse_sm90.py +++ b/tests/v1/attention/test_flashinfer_mla_sparse_sm90.py @@ -9,6 +9,7 @@ causality), ckv/kpe cache splitting, and the backend's model-shape gates. """ from types import SimpleNamespace +from typing import Any import pytest import torch @@ -50,6 +51,8 @@ class FakeWrapper: def __init__(self): self.plan_args = None self.run_args = None + self._plan_info: Any = None + self._planned_backend: Any = None def plan(self, *args, **kwargs): self.plan_args = (args, kwargs) @@ -69,8 +72,11 @@ class FakeState: self.kv_indptr = torch.zeros(max_tokens + 1, dtype=torch.int32) self.wrapper = FakeWrapper() self.plan_calls = [] + self.index_topk = 2048 + self.index_kpool = 4 + self.max_valid = 2051 - def plan(self, num_tokens, kv_lens): + def plan(self, num_tokens, kv_lens, cam, req_id_per_token): self.plan_calls.append((num_tokens, kv_lens)) def pack_indices(self, slots): @@ -168,13 +174,12 @@ def test_forward_wiring(monkeypatch, qk_rope, kv_dtype): def test_builder_plans_only_rows_dispatched_to_mqa(monkeypatch, use_mha, num_decodes): """MHA prefill rows must not make the MQA kernel read beyond its query.""" builder = object.__new__(FlashInferMLASparseSM90Builder) - builder._index_topk = 2048 - builder._index_kpool = 4 - builder._async_scheduling = False + builder._adaptive_verification = False builder.state = FakeState(TOPK) builder._attention_layer = SimpleNamespace(_use_sparse_mha=lambda _: use_mha) metadata = object.__new__(sm90_mod.FlashInferMLASparseSM90Metadata) metadata.state = None + metadata.req_id_per_token = None metadata.num_prefills = 1 metadata.num_decode_tokens = num_decodes monkeypatch.setattr( @@ -206,12 +211,17 @@ def test_builder_plans_only_rows_dispatched_to_mqa(monkeypatch, use_mha, num_dec def test_plan_uses_state_params(monkeypatch): """The NoPE/rope dims and scale live on the builder state, not the layer. - plan() takes exact per-row KV lengths; the schedule is rebuilt on every - call (contexts grow between steps) and the indptrs are always full-size + Without a device batch layout plan() takes exact per-row KV lengths and + rebuilds the schedule on every call; the indptrs are always full-size with zero-query padding rows past num_tokens. """ - impl, rows = make_impl(64, "auto") wrapper = FakeWrapper() + plan_info = [0] * sm90_mod._PLAN_INFO_LEN + plan_info[sm90_mod._PI_KV_INDPTR] = 64 * 4 + wrapper._plan_info = torch.tensor(plan_info) + wrapper._planned_backend = SimpleNamespace( + _int_workspace_buffer=torch.zeros(64, dtype=torch.int32) + ) state = sm90_mod._SM90State.__new__(sm90_mod._SM90State) state.device = torch.device("cpu") state.wrapper = wrapper @@ -227,8 +237,9 @@ def test_plan_uses_state_params(monkeypatch): state._qo_cpu = torch.empty(5, dtype=torch.int32) state._kv_cpu = torch.empty(5, dtype=torch.int32) state._lens_cpu = torch.full((4,), TOPK, dtype=torch.int32) + state._saved_kv_end = torch.empty(0, dtype=torch.int32) - state.plan(3, torch.tensor([2, 5, 7], dtype=torch.int32)) + state.plan(3, torch.tensor([2, 5, 7], dtype=torch.int32), None, None) assert wrapper.plan_args is not None args, kwargs = wrapper.plan_args (qo, kv, indices, kv_len, heads, ckv, kpe, page, causal, scale) = args @@ -241,10 +252,10 @@ def test_plan_uses_state_params(monkeypatch): assert kwargs["kv_data_type"] == torch.bfloat16 # Replanning a smaller batch must clear the previous rows' lengths. - state.plan(1, torch.tensor([TOPK], dtype=torch.int32)) + state.plan(1, torch.tensor([TOPK], dtype=torch.int32), None, None) assert state._kv_cpu.tolist() == [0, TOPK, TOPK, TOPK, TOPK] assert state._lens_cpu.tolist() == [TOPK, 0, 0, 0] - state.plan(0, torch.empty(0, dtype=torch.int32)) + state.plan(0, torch.empty(0, dtype=torch.int32), None, None) assert state._kv_cpu.tolist() == [0, 0, 0, 0, 0] assert state._lens_cpu.tolist() == [0, 0, 0, 0] @@ -279,38 +290,49 @@ def test_pack_indices_replays_with_updated_offsets(width): assert (state.kv_indices[expected.numel() :] == -99).all() -def test_kv_lens_host_formula(): +@pytest.mark.parametrize("adaptive", [False, True]) +def test_kv_lens_host_formula(adaptive): """Per-row host lengths: context == position + 1; capped at - index_topk + trailing-pool remainder past the sparse threshold.""" + index_topk + trailing-pool remainder past the sparse threshold. + + Exact (synced) without a host upper bound; with one, a sync-free bound + capped at the largest valid count, which the plan clamps on device. + """ builder = object.__new__(FlashInferMLASparseSM90Builder) - builder._index_topk = 2048 - builder._index_kpool = 4 - builder._async_scheduling = False + builder.state = FakeState(TOPK) + builder._adaptive_verification = adaptive + seq_lens = torch.tensor([100, 9, 3000], dtype=torch.int32) cam = SimpleNamespace( num_reqs=3, query_start_loc_cpu=torch.tensor([0, 5, 7, 10], dtype=torch.int32), - seq_lens=torch.tensor([100, 9, 3000], dtype=torch.int32), - seq_lens_cpu_upper_bound=torch.tensor([100, 9, 3000], dtype=torch.int32), - positions=None, + seq_lens=seq_lens, + seq_lens_cpu_upper_bound=None, ) - num_rows, lens = builder._kv_lens_host(cam) - assert num_rows == 10 + num_rows, lens, exact = builder._kv_lens_host(cam) + assert num_rows == 10 and exact # req0: positions 95..99 -> ctx 96..100 (all <= 2048: full context) # req1: positions 7,8 -> ctx 8,9 # req2: positions 2997..2999 -> ctx 2998..3000 (> 2048: topk + ctx%4) assert lens.tolist() == [96, 97, 98, 99, 100, 8, 9, 2050, 2051, 2048] + cam.seq_lens_cpu_upper_bound = seq_lens + num_rows, lens, exact = builder._kv_lens_host(cam) + assert num_rows == 10 and not exact + if adaptive: + # Host query split may differ from the device one: batch-wide bound. + assert lens.tolist() == [2051] * 10 + else: + assert lens.tolist() == [96, 97, 98, 99, 100, 8, 9, 2051, 2051, 2051] + def test_kv_lens_host_empty(): builder = object.__new__(FlashInferMLASparseSM90Builder) - builder._index_topk = 2048 - builder._index_kpool = 4 cam = SimpleNamespace( num_reqs=0, query_start_loc_cpu=torch.tensor([0], dtype=torch.int32), seq_lens=torch.zeros(0, dtype=torch.int32), ) - num_rows, lens = builder._kv_lens_host(cam) + num_rows, lens, _ = builder._kv_lens_host(cam) assert num_rows == 0 and lens.numel() == 0 @@ -334,7 +356,13 @@ def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens) "__init__", lambda *_args, **_kwargs: None, ) - monkeypatch.setattr(sm90_mod, "_SM90State", lambda *_args, **_kwargs: None) + + class _RecordingState: + def __init__(self, *_args, **kwargs): + self.index_topk = kwargs["index_topk"] + self.index_kpool = kwargs["index_kpool"] + + monkeypatch.setattr(sm90_mod, "_SM90State", _RecordingState) impl, _ = make_impl(64) spec = MLAAttentionSpec( block_size=BLOCK_SIZE, num_kv_heads=1, head_size=576, dtype=torch.bfloat16 @@ -351,6 +379,7 @@ def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens) max_num_batched_tokens=64, async_scheduling=False ), model_config=SimpleNamespace(hf_text_config=hf_config), + speculative_config=None, ) builder = FlashInferMLASparseSM90Builder( spec, ["attn"], vllm_config, torch.device("cpu") @@ -359,14 +388,138 @@ def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens) cam = SimpleNamespace( num_reqs=2, query_start_loc_cpu=torch.tensor([0, 4, 6], dtype=torch.int32), - seq_lens_cpu_upper_bound=torch.tensor([42295, 100], dtype=torch.int32), - positions=None, + seq_lens=torch.tensor([42295, 100], dtype=torch.int32), + seq_lens_cpu_upper_bound=None, ) - num_rows, lens = builder._kv_lens_host(cam) - assert num_rows == 6 + num_rows, lens, exact = builder._kv_lens_host(cam) + assert num_rows == 6 and exact assert lens.tolist() == prefill_lens + [99, 100] +def _sm90_flashinfer_available(): + return ( + torch.cuda.is_available() + and torch.cuda.get_device_capability()[0] == 9 + and sm90_mod.has_flashinfer_sm90_nope_mla() + ) + + +SM90_HEADS, SM90_TOPK, SM90_KPOOL, SM90_WIDTH = 16, 2048, 4, 2176 + + +def _real_state(rows): + return sm90_mod._SM90State( + torch.device("cuda"), + SM90_HEADS, + torch.bfloat16, + rows, + SM90_WIDTH, + HEAD, + 0, + HEAD**-0.5, + SM90_TOPK, + SM90_KPOOL, + ) + + +def _clamp_batch(num_reqs, q_per_req, lo, hi, topk, kpool, width): + """Random batch: per-row contexts, device layout, and top-k slots.""" + last = torch.randint(lo + q_per_req, hi, (num_reqs,)) + ctx = (last[:, None] + torch.arange(1 - q_per_req, 1)).reshape(-1) + valid = torch.where(ctx <= topk, ctx, topk + ctx % kpool) + cam = SimpleNamespace( + seq_lens=last.to(torch.int32).cuda(), + query_start_loc=(torch.arange(num_reqs + 1) * q_per_req).to(torch.int32).cuda(), + ) + req_id = torch.arange(num_reqs, dtype=torch.int32).repeat_interleave(q_per_req) + slots = torch.randint(0, 1 << 16, (ctx.numel(), width), dtype=torch.int32) + slots[torch.arange(width) >= valid[:, None]] = -1 + return ctx, valid.to(torch.int32), cam, req_id.cuda(), slots.cuda() + + +@pytest.mark.skipif( + not _sm90_flashinfer_available(), reason="Requires SM90 FlashInfer MLA" +) +@pytest.mark.parametrize( + "num_reqs,q_per_req,lo,hi", + [ + (64, 1, 1, 3000), + (16, 6, 2030, 2070), # MTP-style rows straddling index_topk + (4096, 1, 1, 3000), # clamp spans several program blocks + ], +) +def test_upper_bound_plan_matches_exact(num_reqs, q_per_req, lo, hi): + """Sync-free planning from optimistic host bounds (async spec decode) + must match exact planning: the device clamp trims every work item back + to its row's valid count, so no -1 slot is read.""" + torch.manual_seed(0) + heads, topk, kpool, width = SM90_HEADS, SM90_TOPK, SM90_KPOOL, SM90_WIDTH + ctx, valid, cam, req_id, slots = _clamp_batch( + num_reqs, q_per_req, lo, hi, topk, kpool, width + ) + rows = ctx.numel() + + kv = torch.randn(1 << 16, 1, HEAD, device="cuda", dtype=torch.bfloat16) + q = torch.randn(rows, heads, HEAD, device="cuda", dtype=torch.bfloat16) + q_pe = q.new_empty(rows, heads, 0) + + def run(state): + state.pack_indices(slots) + return state.wrapper.run(q, q_pe, kv, kv[..., :0]) + + exact = _real_state(rows) + exact.plan(rows, valid, None, None) + ub = _real_state(rows) + host_bound = torch.clamp(ctx + torch.randint(0, 7, (rows,)), max=ub.max_valid) + ub.plan(rows, host_bound, cam, req_id) + out = run(ub) + assert not out.isnan().any() + torch.testing.assert_close(out, run(exact), atol=1e-2, rtol=1e-2) + + +@pytest.mark.skipif( + not _sm90_flashinfer_available(), reason="Requires SM90 FlashInfer MLA" +) +def test_upper_bound_plan_reused_across_steps(monkeypatch): + """Growing contexts (draft/decode steps) reuse the padded plan and only + re-clamp on device, matching an exact replan every step.""" + torch.manual_seed(0) + heads, topk, kpool, width = SM90_HEADS, SM90_TOPK, SM90_KPOOL, SM90_WIDTH + rows = 48 + ctx, _, _, req_id, _ = _clamp_batch(rows, 1, 1, 3000, topk, kpool, width) + + exact, ub = _real_state(rows), _real_state(rows) + replans: list[int] = [] + plan_fn = ub._plan + + def counting_plan(num_tokens, kv_lens): + replans.append(num_tokens) + plan_fn(num_tokens, kv_lens) + + monkeypatch.setattr(ub, "_plan", counting_plan) + kv = torch.randn(1 << 16, 1, HEAD, device="cuda", dtype=torch.bfloat16) + for step in range(5): + step_ctx = ctx + step + valid = torch.where(step_ctx <= topk, step_ctx, topk + step_ctx % kpool) + slots = torch.randint(0, 1 << 16, (rows, width), dtype=torch.int32) + slots[torch.arange(width) >= valid[:, None]] = -1 + slots = slots.cuda() + cam = SimpleNamespace( + seq_lens=step_ctx.to(torch.int32).cuda(), + query_start_loc=torch.arange(rows + 1, dtype=torch.int32).cuda(), + ) + num_plans = len(replans) + ub.plan(rows, torch.clamp(step_ctx + 5, max=ub.max_valid), cam, req_id) + assert len(replans) - num_plans == (1 if step == 0 else 0) + exact.plan(rows, valid.to(torch.int32), None, None) + q = torch.randn(rows, heads, HEAD, device="cuda", dtype=torch.bfloat16) + outs = [] + for state in (ub, exact): + state.pack_indices(slots) + outs.append(state.wrapper.run(q, q[..., :0], kv, kv[..., :0])) + torch.testing.assert_close(outs[0], outs[1], atol=1e-2, rtol=1e-2) + + def test_supports_combination_gates(monkeypatch, default_vllm_config): monkeypatch.setattr(sm90_mod, "has_flashinfer_sm90_nope_mla", lambda: True) call = lambda **kw: FlashInferMLASparseSM90Backend.supports_combination( diff --git a/tests/v1/attention/test_gdn_metadata_builder.py b/tests/v1/attention/test_gdn_metadata_builder.py index b4d3acf07c..44dee49a4e 100644 --- a/tests/v1/attention/test_gdn_metadata_builder.py +++ b/tests/v1/attention/test_gdn_metadata_builder.py @@ -21,6 +21,7 @@ from vllm.v1.attention.backends.gdn_attn import ( GDNAttentionMetadata, GDNAttentionMetadataBuilder, ) +from vllm.v1.attention.backends.utils import mamba_get_block_table_tensor from vllm.v1.kv_cache_interface import MambaSpec BLOCK_SIZE = 16 @@ -75,6 +76,17 @@ GDN_BUILD_TEST_CASES = { expected_num_prefill_tokens=0, expected_num_spec_decodes=0, ), + # No speculative config, decode alongside prefill + "regular_decode_with_prefill": GDNBuildTestCase( + seq_lens=[40, 100], + query_lens=[1, 50], + num_decode_draft_tokens=None, + num_speculative_tokens=0, + expected_num_decodes=1, + expected_num_prefills=1, + expected_num_prefill_tokens=50, + expected_num_spec_decodes=0, + ), # Multi-token prefill alongside spec decode — no decode to reclassify "spec_decode_with_real_prefill": GDNBuildTestCase( seq_lens=[100, 20], @@ -155,9 +167,12 @@ def _build( builder: GDNAttentionMetadataBuilder, batch_spec: BatchSpec, num_decode_draft_tokens: list[int] | None = None, + block_table: torch.Tensor | None = None, ) -> GDNAttentionMetadata: """Build GDN attention metadata, optionally with spec-decode kwargs.""" common = create_common_attn_metadata(batch_spec, BLOCK_SIZE, DEVICE) + if block_table is not None: + common = common.replace(block_table_tensor=block_table) kwargs: dict = {} if num_decode_draft_tokens is not None: kwargs["num_decode_draft_tokens_cpu"] = torch.tensor( @@ -184,6 +199,54 @@ def test_gdn_build_classification(test_case: GDNBuildTestCase): assert meta.num_spec_decodes == test_case.expected_num_spec_decodes +@pytest.mark.parametrize("mamba_cache_mode", ["none", "align"]) +@pytest.mark.parametrize("full_cuda_graph", [False, True]) +@pytest.mark.parametrize( + "test_case", GDN_BUILD_TEST_CASES.values(), ids=GDN_BUILD_TEST_CASES.keys() +) +def test_update_block_table_matches_build( + test_case: GDNBuildTestCase, full_cuda_graph: bool, mamba_cache_mode: str +): + """update_block_table() on another group's metadata matches build().""" + batch = BatchSpec(seq_lens=test_case.seq_lens, query_lens=test_case.query_lens) + src, dst, ref = ( + _create_gdn_builder(test_case.num_speculative_tokens, full_cuda_graph) + for _ in range(3) + ) + for builder in (src, dst, ref): + builder.vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode + common = create_common_attn_metadata(batch, BLOCK_SIZE, DEVICE) + if mamba_cache_mode == "align": + # MRV2 precomputes these for every Mamba group each step. + dst.mamba_aligned_state_indices = ref.mamba_aligned_state_indices = ( + mamba_get_block_table_tensor( + common.block_table_tensor, common.seq_lens, ref.kv_cache_spec, "align" + ) + ) + draft_tokens = test_case.num_decode_draft_tokens + expected = _build(ref, batch, draft_tokens, common.block_table_tensor) + source = _build(src, batch, draft_tokens) + fields = ( + "spec_state_indices_tensor", + "non_spec_state_indices_tensor", + "prefill_state_indices", + ) + source_indices = [getattr(source, f) for f in fields] + source_indices = [t if t is None else t.clone() for t in source_indices] + meta = dst.update_block_table( + source, common.block_table_tensor, common.slot_mapping + ) + + for field, source_index in zip(fields, source_indices): + actual = getattr(meta, field) + torch.testing.assert_close(actual, getattr(expected, field)) + # The source group's indices are untouched. + torch.testing.assert_close(getattr(source, field), source_index) + # FULL graph state indices land in this group's own buffers. + if full_cuda_graph and meta.num_prefills == 0 and actual is not None: + assert actual.data_ptr() == getattr(dst, field).data_ptr() + + def test_has_initial_state_after_reclassification(): """After reclassification, num_prefills > 0 so the prefill kernel path should compute has_initial_state. For the reclassified request with diff --git a/tests/v1/attention/test_group_head_counts.py b/tests/v1/attention/test_group_head_counts.py index 54fe3cfdbc..40d9e2d00f 100644 --- a/tests/v1/attention/test_group_head_counts.py +++ b/tests/v1/attention/test_group_head_counts.py @@ -101,6 +101,7 @@ def test_flash_attention_geometry_comes_from_the_group(): num_kv_heads=2, head_size=64, dtype=torch.bfloat16, + dcp_sharded=True, ) with ( diff --git a/tests/v1/attention/test_mla_prefill_selector.py b/tests/v1/attention/test_mla_prefill_selector.py index 098eddfbf6..87d188c6a0 100644 --- a/tests/v1/attention/test_mla_prefill_selector.py +++ b/tests/v1/attention/test_mla_prefill_selector.py @@ -60,10 +60,43 @@ class TestGetMLAPrefillBackend: def test_cpu_uses_sdpa_prefill(self): vllm_config = _make_vllm_config() + zen_cls = MLAPrefillBackendEnum.ZEN_CPU.get_class() + + # A Zen CPU with zentorch selects the Zen backend, so expect whichever + # one this host offers. + expected = ( + zen_cls if zen_cls.is_available() else MLAPrefillBackendEnum.CPU.get_class() + ) with patch("vllm.platforms.current_platform") as mock_platform: mock_platform.is_cpu.return_value = True + backend = get_mla_prefill_backend(vllm_config) + assert backend is expected + + def test_zen_cpu_prefers_zentorch_prefill(self): + vllm_config = _make_vllm_config() + zen_cls = MLAPrefillBackendEnum.ZEN_CPU.get_class() + + with ( + patch("vllm.platforms.current_platform") as mock_platform, + patch.object(zen_cls, "is_available", return_value=True), + ): + mock_platform.is_cpu.return_value = True + + backend = get_mla_prefill_backend(vllm_config) + assert backend is zen_cls + + def test_zen_cpu_falls_back_when_zentorch_unavailable(self): + vllm_config = _make_vllm_config() + zen_cls = MLAPrefillBackendEnum.ZEN_CPU.get_class() + + with ( + patch("vllm.platforms.current_platform") as mock_platform, + patch.object(zen_cls, "is_available", return_value=False), + ): + mock_platform.is_cpu.return_value = True + backend = get_mla_prefill_backend(vllm_config) assert backend is MLAPrefillBackendEnum.CPU.get_class() diff --git a/tests/v1/core/test_kv_cache_utils.py b/tests/v1/core/test_kv_cache_utils.py index 24be16e4a5..c2c86053f0 100644 --- a/tests/v1/core/test_kv_cache_utils.py +++ b/tests/v1/core/test_kv_cache_utils.py @@ -1599,19 +1599,142 @@ def test_project_kv_cache_groups_to_worker(): assert set(proj_spec.kv_cache_specs.keys()) == {"layer1", "layer3"} -def test_dcp_world_size_for_kv_cache_spec_shards_full_attention_only(): - dcp = 8 - full = FullAttentionSpec( - block_size=16, num_kv_heads=1, head_size=1, dtype=torch.float32 +@pytest.mark.parametrize("sliding_window", [None, 256]) +@pytest.mark.parametrize("disable_hybrid", [False, True]) +@pytest.mark.parametrize("pcp_size", [1, 4]) +def test_dcp_target_allocates_replicated_draft_independently( + monkeypatch, sliding_window, disable_hybrid, pcp_size +): + """A draft must retain all positions even when the target shards them.""" + from vllm.v1.attention.backends.utils import resolve_kv_cache_layout + + monkeypatch.delenv("VLLM_KV_CACHE_LAYOUT", raising=False) + config = VllmConfig(model_config=ModelConfig(max_model_len=1024)) + config.parallel_config.decode_context_parallel_size = 4 + config.parallel_config.prefill_context_parallel_size = pcp_size + config.scheduler_config.disable_hybrid_kv_cache_manager = disable_hybrid + config.cache_config.block_size = 16 + config.cache_config.kv_cache_layout = None + draft_args = dict( + block_size=16, + num_kv_heads=8, + head_size=64, + dtype=torch.bfloat16, + dcp_sharded=False, ) - mla = new_mla_spec() - mamba = new_mamba_spec() - uniform_mla = UniformTypeKVCacheSpecs(block_size=16, kv_cache_specs={"layer": mla}) - assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(full, dcp) == dcp - assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(mla, dcp) == dcp - assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(uniform_mla, dcp) == dcp - assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(mamba, dcp) == 1 - assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(full, 1) == 1 + draft = ( + FullAttentionSpec(**draft_args) + if sliding_window is None + else SlidingWindowSpec(**draft_args, sliding_window=sliding_window) + ) + specs = {"target": new_mla_spec(), "draft": draft} + layout = resolve_kv_cache_layout(config, [["LBHNC", "BLHNC"]], specs.values()) + assert layout == KVCacheLayout.BLHNC + groups = get_kv_cache_groups(config, specs) + assert [group.layer_names for group in groups] == [["target"], ["draft"]] + widths = [g.kv_cache_spec.max_num_blocks_per_req(config, 1024) for g in groups] + assert widths == [16, 64] + + cache_config = kv_cache_utils.get_kv_cache_config_from_groups( + config, groups, available_memory=16 * 1024 * 1024 + ) + scheduler_config = generate_scheduler_kv_cache_config([cache_config]) + assert get_max_concurrency_for_kv_cache_config(config, cache_config) > 0 + assert get_max_concurrency_for_kv_cache_config(config, cache_config) == ( + get_max_concurrency_for_kv_cache_config(config, scheduler_config) + ) + manager = KVCacheManager( + scheduler_config, + max_model_len=1024, + hash_block_size=16, + scheduler_block_size=64, + dcp_world_size=4, + pcp_world_size=pcp_size, + enable_caching=True, + ) + assert manager.coordinator.group_block_sizes == (64, 16) + request = make_request("replicated-draft", [1] * 65, block_size=16, hash_fn=sha256) + blocks = manager.allocate_slots(request, 65) + assert blocks is not None + assert [len(group) for group in blocks.blocks] == [2, 5] + manager.cache_blocks(request, 64) + cached_request = make_request( + "cached-draft", [1] * 65, block_size=16, hash_fn=sha256 + ) + cached_blocks, num_cached, _ = manager.get_computed_blocks(cached_request) + assert num_cached == 64 + assert [len(group) for group in cached_blocks.blocks] == [1, 4] + + +@pytest.mark.parametrize("use_mla", [False, True]) +def test_full_attention_merge_preserves_replicated_cache_geometry(use_mla): + config = VllmConfig(model_config=ModelConfig(max_model_len=1024)) + config.parallel_config.decode_context_parallel_size = 4 + draft = replace( + new_mla_spec() if use_mla else new_kv_cache_spec(), + block_size=16, + dcp_sharded=False, + ) + merged = type(draft).merge([draft, draft]) + wrapped = UniformTypeKVCacheSpecs.from_specs({"draft": merged}) + assert wrapped is not None and not wrapped.dcp_sharded + assert merged.max_num_blocks_per_req(config, 1024) == 64 + assert merged.max_memory_usage_bytes(config) == 64 * draft.page_size_bytes + assert kv_cache_utils.resolve_dcp_kv_block_size(merged, 4) == 16 + with pytest.raises(AssertionError): + type(draft).merge([draft, replace(draft, dcp_sharded=True)]) + + +@pytest.mark.parametrize("with_draft", [False, True]) +@pytest.mark.parametrize("indexer_alignment", [None, 512]) +def test_sparse_mla_preserves_physical_row_addressing(with_draft, indexer_alignment): + from vllm.v1.attention.backends.mla.sparse_utils import flat_kv_row_view + from vllm.v1.worker.utils import allocate_kv_cache + + config = VllmConfig(model_config=ModelConfig(max_model_len=1024)) + config.cache_config.kv_cache_layout = "BLHNC" + config.parallel_config.decode_context_parallel_size = 4 + common = dict(block_size=64, num_kv_heads=1, dtype=torch.uint8) + specs = { + "target": MLAAttentionSpec( + **common, + head_size=576, + state_content_bytes=656, + block_stride_alignment=656, + cache_dtype_str="fp8_ds_mla", + is_index_group_leader=True, + ), + "indexer": MLAAttentionSpec( + **common, + head_size=132, + cache_role=SparseCacheRole.INDEXER, + block_stride_alignment=indexer_alignment, + ), + "draft": SlidingWindowSpec( + block_size=64, + num_kv_heads=64, + head_size=64, + dtype=torch.bfloat16, + sliding_window=2048, + dcp_sharded=False, + ), + } + if not with_draft: + del specs["draft"] + groups = get_kv_cache_groups(config, specs) + cache_config = kv_cache_utils.get_kv_cache_config_from_groups( + config, groups, available_memory=8 * 1024 * 1024 + ) + caches = allocate_kv_cache( + cache_config, torch.device("cpu"), KVCacheLayout.BLHNC, [64] * len(groups) + ) + cache = caches["target"].squeeze(1) + if indexer_alignment is not None: + assert caches["indexer"].stride(0) % indexer_alignment == 0 + rows, stride_rows = flat_kv_row_view(cache, 64) + cache[1, 0].fill_(7) + torch.testing.assert_close(rows[stride_rows], cache[1, 0]) + assert torch.all(rows[stride_rows] == 7) @pytest.mark.parametrize( @@ -3815,6 +3938,8 @@ def test_unify_hybrid_kv_cache_specs(): "layer_1": before_spec_1, "layer_2": before_spec_2, } + kv_cache_spec["draft_layer_1"] = replace(before_spec_1, dcp_sharded=False) + kv_cache_spec["draft_layer_2"] = replace(before_spec_2, dcp_sharded=False) kv_cache_utils.unify_hybrid_kv_cache_specs(kv_cache_spec) expected_spec_1 = new_kv_cache_spec(block_size=64) expected_spec_2 = new_kv_cache_spec( @@ -3823,6 +3948,7 @@ def test_unify_hybrid_kv_cache_specs(): assert kv_cache_spec["layer_1"] == expected_spec_1 assert kv_cache_spec["layer_2"] == expected_spec_2 assert kv_cache_spec["layer_2"].page_size_bytes == 64 * 1024 + assert kv_cache_spec["draft_layer_2"] == replace(expected_spec_2, dcp_sharded=False) # 2. has_full_attention and has_chunked_local_attention before_spec_1 = new_kv_cache_spec() @@ -3874,6 +4000,11 @@ def test_unify_hybrid_kv_cache_specs(): with pytest.raises(ValueError): kv_cache_utils.unify_hybrid_kv_cache_specs(kv_cache_spec) + # Replicated Mamba state still requires the hybrid cache manager. + kv_cache_spec = {"attention": new_kv_cache_spec(), "mamba": new_mamba_spec()} + with pytest.raises(ValueError): + kv_cache_utils.unify_hybrid_kv_cache_specs(kv_cache_spec) + def test_unify_kv_cache_spec_page_size_mamba(): """Regression test for https://github.com/vllm-project/vllm/issues/43626. diff --git a/tests/v1/core/test_prefix_caching.py b/tests/v1/core/test_prefix_caching.py index 0a34a6131d..99d3235006 100644 --- a/tests/v1/core/test_prefix_caching.py +++ b/tests/v1/core/test_prefix_caching.py @@ -1336,12 +1336,12 @@ def make_kv_cache_config_three_types( ) -def test_prefix_cache_hit_uses_per_group_dcp_geometry(): +@pytest.mark.parametrize("draft_sharded", [True, False]) +def test_prefix_cache_hit_uses_per_group_dcp_geometry(draft_sharded): """Prefix lookup must use each group's DCP size, not the process-wide one. - Target and draft MLA are both sharded (DCP=8); Mamba stays replicated - (DCP=1). Hits then align to the sharded full-attention block, not the - unsharded page size. + Target MLA is sharded; draft MLA can opt out and Mamba stays replicated. + Hits must align to the target block in either case. """ block_size = 16 dcp = 8 @@ -1362,6 +1362,7 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry(): KVCacheGroupSpec( ["draft_mla"], MLAAttentionSpec( + dcp_sharded=draft_sharded, block_size=block_size, num_kv_heads=1, head_size=1, @@ -1388,10 +1389,10 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry(): ) target_mgr, draft_mgr, mamba_mgr = manager.coordinator.single_type_managers assert target_mgr.dcp_world_size == dcp - assert draft_mgr.dcp_world_size == dcp + assert draft_mgr.dcp_world_size == (dcp if draft_sharded else 1) assert mamba_mgr.dcp_world_size == 1 assert target_mgr.block_size == sharded_block - assert draft_mgr.block_size == sharded_block + assert draft_mgr.block_size == (sharded_block if draft_sharded else block_size) assert mamba_mgr.block_size == block_size hash_fn = sha256 @@ -1407,7 +1408,11 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry(): req1 = make_request("1", common_token_ids + [100] * 5, block_size, hash_fn) computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1) assert num_computed_tokens == 2 * sharded_block - assert [len(group) for group in computed_blocks.blocks] == [2, 2, 16] + assert [len(group) for group in computed_blocks.blocks] == [ + 2, + 2 if draft_sharded else 16, + 16, + ] manager.free(req0) manager.free(req1) diff --git a/tests/v1/e2e/spec_decode/draft_model/test_draft_model.py b/tests/v1/e2e/spec_decode/draft_model/test_draft_model.py index 3a474ac203..d2c2ac87b7 100644 --- a/tests/v1/e2e/spec_decode/draft_model/test_draft_model.py +++ b/tests/v1/e2e/spec_decode/draft_model/test_draft_model.py @@ -11,6 +11,7 @@ from vllm.config import CompilationConfig, VllmConfig, replace from vllm.config.kernel import MoEBackend from vllm.engine.arg_utils import EngineArgs from vllm.platforms import current_platform +from vllm.v1.spec_decode.draft_model import DraftModelProposer from ..utils import ( Messages, @@ -196,6 +197,38 @@ def test_draft_model_engine_args_tensor_parallelism(): assert draft_config.quant_config is None +@pytest.mark.cpu_test +@pytest.mark.parametrize("dcp_size", [1, 2]) +def test_draft_model_inherits_target_dcp_config(monkeypatch, dcp_size): + monkeypatch.setattr(current_platform, "device_count", lambda: 16) + target_config = EngineArgs( + model="Qwen/Qwen3-1.7B", + tensor_parallel_size=16, + decode_context_parallel_size=dcp_size, + cp_kv_cache_interleave_size=16, + dcp_comm_backend="ag_rs", + distributed_executor_backend="mp", + speculative_config={ + "model": "Qwen/Qwen3-0.6B", + "method": "draft_model", + "num_speculative_tokens": 3, + }, + ).create_engine_config() + proposer = object.__new__(DraftModelProposer) + proposer.vllm_config = target_config + proposer.speculative_config = target_config.speculative_config + draft_config = proposer._create_draft_vllm_config() + draft_parallel = draft_config.parallel_config + assert draft_parallel.decode_context_parallel_size == dcp_size + assert draft_parallel.cp_kv_cache_interleave_size == 16 + assert draft_parallel.dcp_comm_backend == "ag_rs" + assert ( + draft_parallel.dcp_q_replicate == target_config.parallel_config.dcp_q_replicate + ) + assert draft_parallel is not target_config.parallel_config + assert target_config.parallel_config.decode_context_parallel_size == dcp_size + + def _apply_draft_moe_backend(vllm_config: VllmConfig) -> VllmConfig: """Replicate SpecDecodeBaseProposer._create_draft_vllm_config logic so we can test it without instantiating a full proposer.""" diff --git a/tests/v1/e2e/spec_decode/mtp/_correctness.py b/tests/v1/e2e/spec_decode/mtp/_correctness.py new file mode 100644 index 0000000000..b88c1cce7a --- /dev/null +++ b/tests/v1/e2e/spec_decode/mtp/_correctness.py @@ -0,0 +1,120 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from typing import Any + +import pytest + +from vllm import SamplingParams +from vllm.config import CompilationConfig +from vllm.platforms import current_platform + +from ..utils import ( + _skip_if_insufficient_gpus_for_tp, + assert_request_outputs_match, + evaluate_llm_for_gsm8k, + get_test_prompts, +) + + +def check_mtp_correctness( + monkeypatch: pytest.MonkeyPatch, + sampling_config: SamplingParams, + model_setup: tuple[str, str, int, str | None], + mm_enabled: bool, + expected_accuracy_threshold: float, + vllm_runner, +): + """Compare the outputs of a original LLM and a speculative LLM + which should be the same when using MTP speculative decoding. Due to some variance + in the engine, it is possible for some outputs to differ, so we expect that at least + 6/10 output tokens match exactly, and that the GSM8k accuracy is above a precomputed + reference threshold for each model. + """ + method, model_name, tp_size, draft_model = model_setup + _skip_if_insufficient_gpus_for_tp(tp_size) + + # Generate test prompts inside the function instead of using fixture + test_prompts = get_test_prompts(mm_enabled) + with monkeypatch.context() as m: + m.setenv("VLLM_MLA_DISABLE", "1") + + attn_backend = "TRITON_ATTN" if current_platform.is_rocm() else "auto" + + # Skip multimodal profiling for models that don't need it in this test. + extra_kwargs: dict[str, Any] = {} + if "Qwen3.5" in model_name: + extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "video": 0} + elif "gemma-4" in model_name: + extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "audio": 0} + + if draft_model is not None and "gemma-4" in draft_model: + import transformers + from packaging.version import Version + + if Version(transformers.__version__) < Version("5.8.0"): + pytest.skip( + "Gemma4 MTP assistant requires transformers>=5.8.0, " + f"got {transformers.__version__}" + ) + + with vllm_runner( + model_name, + block_size=None, + max_model_len=2048, + tensor_parallel_size=tp_size, + trust_remote_code=True, + attention_backend=attn_backend, + enable_chunked_prefill=None, + compilation_config=CompilationConfig(), + **extra_kwargs, + ) as ref_runner: + ref_outputs = ref_runner.llm.chat(test_prompts, sampling_config) + evaluate_llm_for_gsm8k( + ref_runner.llm, + expected_accuracy_threshold=expected_accuracy_threshold, + ) + + speculative_config: dict[str, Any] = { + "method": method, + "num_speculative_tokens": 1, + "max_model_len": 2048, + } + if draft_model is not None: + speculative_config["model"] = draft_model + speculative_config["num_speculative_tokens"] = 2 + + with vllm_runner( + model_name, + block_size=None, + trust_remote_code=True, + tensor_parallel_size=tp_size, + speculative_config=speculative_config, + max_model_len=2048, + attention_backend=attn_backend, + enable_chunked_prefill=None, + compilation_config=CompilationConfig(), + **extra_kwargs, + ) as spec_runner: + # MTP supports async scheduling by default. + has_async = ( + spec_runner.llm.llm_engine.vllm_config.scheduler_config.async_scheduling + ) + assert has_async, ( + f"Expected async scheduling for {method}: target={model_name}, " + f"draft={draft_model}; got {has_async}" + ) + evaluate_llm_for_gsm8k( + spec_runner.llm, + expected_accuracy_threshold=expected_accuracy_threshold, + ) + spec_outputs = spec_runner.llm.chat(test_prompts, sampling_config) + + # Heuristic: expect at least 80% of the prompts to match exactly + # Upon failure, inspect the outputs to check for inaccuracy. + assert_request_outputs_match( + ref_outputs, + spec_outputs, + required_matches=int(0.8 * len(ref_outputs)) + 1, + context=f"{method} target={model_name}, draft={draft_model}", + ) diff --git a/tests/v1/e2e/spec_decode/mtp/deepseek/__init__.py b/tests/v1/e2e/spec_decode/mtp/deepseek/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/tests/v1/e2e/spec_decode/mtp/deepseek/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/tests/v1/e2e/spec_decode/mtp/deepseek/test_mtp.py b/tests/v1/e2e/spec_decode/mtp/deepseek/test_mtp.py new file mode 100644 index 0000000000..a23c3ee26e --- /dev/null +++ b/tests/v1/e2e/spec_decode/mtp/deepseek/test_mtp.py @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest + +from tests.utils import single_gpu_only +from vllm import SamplingParams +from vllm.platforms import current_platform + +from .._correctness import check_mtp_correctness + + +@pytest.mark.parametrize( + ["model_setup", "mm_enabled", "expected_accuracy_threshold"], + [ + pytest.param( + ("mtp", "ZixiQi/DeepSeek-V3-4layers-MTP-FP8", 1, None), + False, + 0.0, + marks=pytest.mark.skipif( + current_platform.is_device_capability_family(100), + reason="DeepSeek MTP: TRTLLM MoE top_k check fails on Blackwell", + ), + ), # dummy model + ], + ids=["deepseek"], +) +@single_gpu_only +def test_mtp_correctness( + monkeypatch: pytest.MonkeyPatch, + sampling_config: SamplingParams, + model_setup: tuple[str, str, int, str | None], + mm_enabled: bool, + expected_accuracy_threshold: float, + vllm_runner, +): + check_mtp_correctness( + monkeypatch, + sampling_config, + model_setup, + mm_enabled, + expected_accuracy_threshold, + vllm_runner, + ) diff --git a/tests/v1/e2e/spec_decode/mtp/gemma4/__init__.py b/tests/v1/e2e/spec_decode/mtp/gemma4/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/tests/v1/e2e/spec_decode/mtp/gemma4/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/tests/v1/e2e/spec_decode/mtp/gemma4/test_mtp.py b/tests/v1/e2e/spec_decode/mtp/gemma4/test_mtp.py new file mode 100644 index 0000000000..66fb49fcef --- /dev/null +++ b/tests/v1/e2e/spec_decode/mtp/gemma4/test_mtp.py @@ -0,0 +1,39 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest + +from tests.utils import single_gpu_only +from vllm import SamplingParams + +from .._correctness import check_mtp_correctness + + +@pytest.mark.parametrize( + ["model_setup", "mm_enabled", "expected_accuracy_threshold"], + [ + ( + ("mtp", "google/gemma-4-E4B-it", 1, "google/gemma-4-E4B-it-assistant"), + False, + 0.50, + ), # gemma4 MTP with assistant model, ref: ~62% + ], + ids=["gemma4-e4b"], +) +@single_gpu_only +def test_mtp_correctness( + monkeypatch: pytest.MonkeyPatch, + sampling_config: SamplingParams, + model_setup: tuple[str, str, int, str | None], + mm_enabled: bool, + expected_accuracy_threshold: float, + vllm_runner, +): + check_mtp_correctness( + monkeypatch, + sampling_config, + model_setup, + mm_enabled, + expected_accuracy_threshold, + vllm_runner, + ) diff --git a/tests/v1/e2e/spec_decode/mtp/mimo/__init__.py b/tests/v1/e2e/spec_decode/mtp/mimo/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/tests/v1/e2e/spec_decode/mtp/mimo/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/tests/v1/e2e/spec_decode/mtp/mimo/test_mtp.py b/tests/v1/e2e/spec_decode/mtp/mimo/test_mtp.py new file mode 100644 index 0000000000..8a4e7e1236 --- /dev/null +++ b/tests/v1/e2e/spec_decode/mtp/mimo/test_mtp.py @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest + +from tests.utils import single_gpu_only +from vllm import SamplingParams + +from .._correctness import check_mtp_correctness + + +@pytest.mark.parametrize( + ["model_setup", "mm_enabled", "expected_accuracy_threshold"], + [ + # Reference accuracy: 65%-70%. + (("mtp", "XiaomiMiMo/MiMo-7B-Base", 1, None), False, 0.5), + ], + ids=["mimo"], +) +@single_gpu_only +def test_mtp_correctness( + monkeypatch: pytest.MonkeyPatch, + sampling_config: SamplingParams, + model_setup: tuple[str, str, int, str | None], + mm_enabled: bool, + expected_accuracy_threshold: float, + vllm_runner, +): + check_mtp_correctness( + monkeypatch, + sampling_config, + model_setup, + mm_enabled, + expected_accuracy_threshold, + vllm_runner, + ) diff --git a/tests/v1/e2e/spec_decode/mtp/qwen3_5/__init__.py b/tests/v1/e2e/spec_decode/mtp/qwen3_5/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/tests/v1/e2e/spec_decode/mtp/qwen3_5/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/tests/v1/e2e/spec_decode/mtp/qwen3_5/test_mtp.py b/tests/v1/e2e/spec_decode/mtp/qwen3_5/test_mtp.py new file mode 100644 index 0000000000..c61838a655 --- /dev/null +++ b/tests/v1/e2e/spec_decode/mtp/qwen3_5/test_mtp.py @@ -0,0 +1,39 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest + +from tests.utils import single_gpu_only +from vllm import SamplingParams + +from .._correctness import check_mtp_correctness + + +@pytest.mark.parametrize( + ["model_setup", "mm_enabled", "expected_accuracy_threshold"], + [ + ( + ("mtp", "Qwen/Qwen3.5-0.8B-Base", 1, None), + False, + 0.20, + ), # hybrid + MTP, ref: ~34%-35% + ], + ids=["qwen3_5-hybrid"], +) +@single_gpu_only +def test_mtp_correctness( + monkeypatch: pytest.MonkeyPatch, + sampling_config: SamplingParams, + model_setup: tuple[str, str, int, str | None], + mm_enabled: bool, + expected_accuracy_threshold: float, + vllm_runner, +): + check_mtp_correctness( + monkeypatch, + sampling_config, + model_setup, + mm_enabled, + expected_accuracy_threshold, + vllm_runner, + ) diff --git a/tests/v1/e2e/spec_decode/mtp/qwen3_5/test_rejected_placeholders.py b/tests/v1/e2e/spec_decode/mtp/qwen3_5/test_rejected_placeholders.py new file mode 100644 index 0000000000..5116f7db27 --- /dev/null +++ b/tests/v1/e2e/spec_decode/mtp/qwen3_5/test_rejected_placeholders.py @@ -0,0 +1,115 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from typing import Any + +import pytest + +from tests.utils import single_gpu_only +from vllm import SamplingParams +from vllm.config import CompilationConfig, CUDAGraphMode +from vllm.platforms import current_platform + +from ...utils import get_spec_decode_metric_value + +PLACEHOLDER_REGRESSION_MODEL = "Qwen/Qwen3.5-0.8B-Base" +PLACEHOLDER_REGRESSION_PROMPT = "The capital of France is" +PLACEHOLDER_REGRESSION_MAX_TOKENS = 16 +PLACEHOLDER_REGRESSION_ALLOWED_TOKEN_ID = 42 +PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS = 3 + + +@pytest.mark.parametrize( + "cudagraph_mode", + [CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL_AND_PIECEWISE], + ids=["piecewise", "full-and-piecewise"], +) +@single_gpu_only +def test_mtp_rejected_placeholders_match_greedy_token_ids( + cudagraph_mode: CUDAGraphMode, + vllm_runner, +): + """Rejected MTP padding must not alter the non-speculative token stream.""" + common_kwargs: dict[str, Any] = { + "block_size": None, + "max_model_len": 256, + "max_num_seqs": 1, + "trust_remote_code": True, + "enable_chunked_prefill": None, + "limit_mm_per_prompt": {"image": 0, "video": 0}, + "attention_backend": ("TRITON_ATTN" if current_platform.is_rocm() else "auto"), + } + sampling_params = SamplingParams( + temperature=0.0, + max_tokens=PLACEHOLDER_REGRESSION_MAX_TOKENS, + ignore_eos=True, + allowed_token_ids=[PLACEHOLDER_REGRESSION_ALLOWED_TOKEN_ID], + logprobs=1, + ) + + def generate_token_ids(runner) -> tuple[int, ...]: + outputs = runner.llm.generate([PLACEHOLDER_REGRESSION_PROMPT], sampling_params) + assert outputs and outputs[0].outputs, "Expected one completion" + token_ids = tuple(outputs[0].outputs[0].token_ids) + assert len(token_ids) == PLACEHOLDER_REGRESSION_MAX_TOKENS + return token_ids + + def compilation_config() -> CompilationConfig: + return CompilationConfig( + cudagraph_mode=cudagraph_mode, + cudagraph_capture_sizes=[1, PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS + 1], + ) + + with vllm_runner( + PLACEHOLDER_REGRESSION_MODEL, + compilation_config=compilation_config(), + **common_kwargs, + ) as ref_runner: + ref_mode = ( + ref_runner.llm.llm_engine.vllm_config.compilation_config.cudagraph_mode + ) + assert ref_mode == cudagraph_mode + ref_token_ids = generate_token_ids(ref_runner) + expected_token_ids = (PLACEHOLDER_REGRESSION_ALLOWED_TOKEN_ID,) * ( + PLACEHOLDER_REGRESSION_MAX_TOKENS + ) + assert ref_token_ids == expected_token_ids + + with vllm_runner( + PLACEHOLDER_REGRESSION_MODEL, + speculative_config={ + "method": "mtp", + "num_speculative_tokens": PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS, + "max_model_len": 256, + "rejection_sample_method": "synthetic", + "synthetic_acceptance_rates": [0.0] + * PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS, + }, + disable_log_stats=False, + compilation_config=compilation_config(), + **common_kwargs, + ) as spec_runner: + spec_mode = ( + spec_runner.llm.llm_engine.vllm_config.compilation_config.cudagraph_mode + ) + assert spec_mode == cudagraph_mode + spec_token_ids = generate_token_ids(spec_runner) + metrics = spec_runner.llm.get_metrics() + num_drafts = get_spec_decode_metric_value( + metrics, "vllm:spec_decode_num_drafts" + ) + num_draft_tokens = get_spec_decode_metric_value( + metrics, "vllm:spec_decode_num_draft_tokens" + ) + num_accepted_tokens = get_spec_decode_metric_value( + metrics, "vllm:spec_decode_num_accepted_tokens" + ) + + assert num_drafts > 0, "MTP drafter did not run" + assert num_draft_tokens > 0, "MTP drafter did not produce draft tokens" + assert num_accepted_tokens == 0, "Synthetic sampler accepted a draft token" + assert spec_token_ids == ref_token_ids, ( + f"MTP changed greedy token ids under {cudagraph_mode.name}:\n" + f" reference={ref_token_ids}\n" + f" speculative={spec_token_ids}" + ) diff --git a/tests/v1/e2e/spec_decode/mtp/test_mtp.py b/tests/v1/e2e/spec_decode/mtp/test_mtp.py deleted file mode 100644 index 92f4a2a773..0000000000 --- a/tests/v1/e2e/spec_decode/mtp/test_mtp.py +++ /dev/null @@ -1,252 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from typing import Any - -import pytest - -from tests.utils import single_gpu_only -from vllm import SamplingParams -from vllm.config import CompilationConfig, CUDAGraphMode -from vllm.platforms import current_platform - -from ..utils import ( - _skip_if_insufficient_gpus_for_tp, - assert_request_outputs_match, - evaluate_llm_for_gsm8k, - get_spec_decode_metric_value, - get_test_prompts, -) - -PLACEHOLDER_REGRESSION_MODEL = "Qwen/Qwen3.5-0.8B-Base" -PLACEHOLDER_REGRESSION_PROMPT = "The capital of France is" -PLACEHOLDER_REGRESSION_MAX_TOKENS = 16 -PLACEHOLDER_REGRESSION_ALLOWED_TOKEN_ID = 42 -PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS = 3 - - -@pytest.mark.parametrize( - "cudagraph_mode", - [CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL_AND_PIECEWISE], - ids=["piecewise", "full-and-piecewise"], -) -@single_gpu_only -def test_mtp_rejected_placeholders_match_greedy_token_ids( - cudagraph_mode: CUDAGraphMode, - vllm_runner, -): - """Rejected MTP padding must not alter the non-speculative token stream.""" - common_kwargs: dict[str, Any] = { - "block_size": None, - "max_model_len": 256, - "max_num_seqs": 1, - "trust_remote_code": True, - "enable_chunked_prefill": None, - "limit_mm_per_prompt": {"image": 0, "video": 0}, - "attention_backend": ("TRITON_ATTN" if current_platform.is_rocm() else "auto"), - } - sampling_params = SamplingParams( - temperature=0.0, - max_tokens=PLACEHOLDER_REGRESSION_MAX_TOKENS, - ignore_eos=True, - allowed_token_ids=[PLACEHOLDER_REGRESSION_ALLOWED_TOKEN_ID], - logprobs=1, - ) - - def generate_token_ids(runner) -> tuple[int, ...]: - outputs = runner.llm.generate([PLACEHOLDER_REGRESSION_PROMPT], sampling_params) - assert outputs and outputs[0].outputs, "Expected one completion" - token_ids = tuple(outputs[0].outputs[0].token_ids) - assert len(token_ids) == PLACEHOLDER_REGRESSION_MAX_TOKENS - return token_ids - - def compilation_config() -> CompilationConfig: - return CompilationConfig( - cudagraph_mode=cudagraph_mode, - cudagraph_capture_sizes=[1, PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS + 1], - ) - - with vllm_runner( - PLACEHOLDER_REGRESSION_MODEL, - compilation_config=compilation_config(), - **common_kwargs, - ) as ref_runner: - ref_mode = ( - ref_runner.llm.llm_engine.vllm_config.compilation_config.cudagraph_mode - ) - assert ref_mode == cudagraph_mode - ref_token_ids = generate_token_ids(ref_runner) - expected_token_ids = (PLACEHOLDER_REGRESSION_ALLOWED_TOKEN_ID,) * ( - PLACEHOLDER_REGRESSION_MAX_TOKENS - ) - assert ref_token_ids == expected_token_ids - - with vllm_runner( - PLACEHOLDER_REGRESSION_MODEL, - speculative_config={ - "method": "mtp", - "num_speculative_tokens": PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS, - "max_model_len": 256, - "rejection_sample_method": "synthetic", - "synthetic_acceptance_rates": [0.0] - * PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS, - }, - disable_log_stats=False, - compilation_config=compilation_config(), - **common_kwargs, - ) as spec_runner: - spec_mode = ( - spec_runner.llm.llm_engine.vllm_config.compilation_config.cudagraph_mode - ) - assert spec_mode == cudagraph_mode - spec_token_ids = generate_token_ids(spec_runner) - metrics = spec_runner.llm.get_metrics() - num_drafts = get_spec_decode_metric_value( - metrics, "vllm:spec_decode_num_drafts" - ) - num_draft_tokens = get_spec_decode_metric_value( - metrics, "vllm:spec_decode_num_draft_tokens" - ) - num_accepted_tokens = get_spec_decode_metric_value( - metrics, "vllm:spec_decode_num_accepted_tokens" - ) - - assert num_drafts > 0, "MTP drafter did not run" - assert num_draft_tokens > 0, "MTP drafter did not produce draft tokens" - assert num_accepted_tokens == 0, "Synthetic sampler accepted a draft token" - assert spec_token_ids == ref_token_ids, ( - f"MTP changed greedy token ids under {cudagraph_mode.name}:\n" - f" reference={ref_token_ids}\n" - f" speculative={spec_token_ids}" - ) - - -@pytest.mark.parametrize( - ["model_setup", "mm_enabled", "expected_accuracy_threshold"], - [ - # Reference accuracy: 65%-70%. - (("mtp", "XiaomiMiMo/MiMo-7B-Base", 1, None), False, 0.5), - pytest.param( - ("mtp", "ZixiQi/DeepSeek-V3-4layers-MTP-FP8", 1, None), - False, - 0.0, - marks=pytest.mark.skipif( - current_platform.is_device_capability_family(100), - reason="DeepSeek MTP: TRTLLM MoE top_k check fails on Blackwell", - ), - ), # dummy model - ( - ("mtp", "Qwen/Qwen3.5-0.8B-Base", 1, None), - False, - 0.20, - ), # hybrid + MTP, ref: ~34%-35% - ( - ("mtp", "google/gemma-4-E4B-it", 1, "google/gemma-4-E4B-it-assistant"), - False, - 0.50, - ), # gemma4 MTP with assistant model, ref: ~62% - ], - ids=["mimo", "deepseek", "qwen3_5-hybrid", "gemma4-e4b"], -) -@single_gpu_only -def test_mtp_correctness( - monkeypatch: pytest.MonkeyPatch, - sampling_config: SamplingParams, - model_setup: tuple[str, str, int, str | None], - mm_enabled: bool, - expected_accuracy_threshold: float, - vllm_runner, -): - """Compare the outputs of a original LLM and a speculative LLM - which should be the same when using MTP speculative decoding. Due to some variance - in the engine, it is possible for some outputs to differ, so we expect that at least - 6/10 output tokens match exactly, and that the GSM8k accuracy is above a precomputed - reference threshold for each model. - """ - method, model_name, tp_size, draft_model = model_setup - _skip_if_insufficient_gpus_for_tp(tp_size) - - # Generate test prompts inside the function instead of using fixture - test_prompts = get_test_prompts(mm_enabled) - with monkeypatch.context() as m: - m.setenv("VLLM_MLA_DISABLE", "1") - - attn_backend = "TRITON_ATTN" if current_platform.is_rocm() else "auto" - - # Skip multimodal profiling for models that don't need it in this test. - extra_kwargs: dict[str, Any] = {} - if "Qwen3.5" in model_name: - extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "video": 0} - elif "gemma-4" in model_name: - extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "audio": 0} - - if draft_model is not None and "gemma-4" in draft_model: - import transformers - from packaging.version import Version - - if Version(transformers.__version__) < Version("5.8.0"): - pytest.skip( - "Gemma4 MTP assistant requires transformers>=5.8.0, " - f"got {transformers.__version__}" - ) - - with vllm_runner( - model_name, - block_size=None, - max_model_len=2048, - tensor_parallel_size=tp_size, - trust_remote_code=True, - attention_backend=attn_backend, - enable_chunked_prefill=None, - compilation_config=CompilationConfig(), - **extra_kwargs, - ) as ref_runner: - ref_outputs = ref_runner.llm.chat(test_prompts, sampling_config) - evaluate_llm_for_gsm8k( - ref_runner.llm, - expected_accuracy_threshold=expected_accuracy_threshold, - ) - - speculative_config: dict[str, Any] = { - "method": method, - "num_speculative_tokens": 1, - "max_model_len": 2048, - } - if draft_model is not None: - speculative_config["model"] = draft_model - speculative_config["num_speculative_tokens"] = 2 - - with vllm_runner( - model_name, - block_size=None, - trust_remote_code=True, - tensor_parallel_size=tp_size, - speculative_config=speculative_config, - max_model_len=2048, - attention_backend=attn_backend, - enable_chunked_prefill=None, - compilation_config=CompilationConfig(), - **extra_kwargs, - ) as spec_runner: - # MTP supports async scheduling by default. - has_async = ( - spec_runner.llm.llm_engine.vllm_config.scheduler_config.async_scheduling - ) - assert has_async, ( - f"Expected async scheduling for {method}: target={model_name}, " - f"draft={draft_model}; got {has_async}" - ) - evaluate_llm_for_gsm8k( - spec_runner.llm, - expected_accuracy_threshold=expected_accuracy_threshold, - ) - spec_outputs = spec_runner.llm.chat(test_prompts, sampling_config) - - # Heuristic: expect at least 80% of the prompts to match exactly - # Upon failure, inspect the outputs to check for inaccuracy. - assert_request_outputs_match( - ref_outputs, - spec_outputs, - required_matches=int(0.8 * len(ref_outputs)) + 1, - context=f"{method} target={model_name}, draft={draft_model}", - ) diff --git a/tests/v1/engine/test_engine_core.py b/tests/v1/engine/test_engine_core.py index e16e0550c2..2d9f7e8632 100644 --- a/tests/v1/engine/test_engine_core.py +++ b/tests/v1/engine/test_engine_core.py @@ -4,6 +4,7 @@ import copy import time import uuid +from collections import deque from concurrent.futures import Future, ThreadPoolExecutor from unittest.mock import MagicMock, PropertyMock, patch @@ -395,6 +396,38 @@ def test_engine_core_concurrent_batches(): req_id = (req_id + 1) % 2 +@pytest.mark.parametrize("encoder_only", [True, False]) +def test_async_step_skips_sampling_for_mm_encoder_only(encoder_only): + """Encoder outputs are complete; only language-model steps need sampling.""" + core = MagicMock() + core.batch_queue = deque() + core.batch_queue_size = 2 + core.is_mm_encoder_only = encoder_only + core.is_pooling_model = False + core.scheduler.has_requests.side_effect = [True, False, False] + scheduled = core.scheduler.schedule.return_value + scheduled.total_num_scheduled_tokens = 16 + scheduled.pending_structured_output_tokens = False + + output = MagicMock() + executed: Future[ModelRunnerOutput | None] = Future() + executed.set_result(output if encoder_only else None) + sampled: Future[ModelRunnerOutput | None] = Future() + sampled.set_result(None if encoder_only else output) + core.model_executor.execute_model.return_value = executed + core.model_executor.sample_tokens.return_value = sampled + + EngineCore.step_with_batch_queue(core) + if core.batch_queue: + EngineCore.step_with_batch_queue(core) + + core.scheduler.update_from_output.assert_called_once_with(scheduled, output) + if encoder_only: + core.model_executor.sample_tokens.assert_not_called() + else: + core.model_executor.sample_tokens.assert_called_once() + + @multi_gpu_test(num_gpus=2) def test_engine_core_tp(): """Test engine can initialize worker in tp properly.""" diff --git a/tests/v1/kv_connector/unit/test_decode_bench_connector.py b/tests/v1/kv_connector/unit/test_decode_bench_connector.py index 693c10d40a..6f518ab53a 100644 --- a/tests/v1/kv_connector/unit/test_decode_bench_connector.py +++ b/tests/v1/kv_connector/unit/test_decode_bench_connector.py @@ -500,7 +500,7 @@ def test_decode_bench_connector_skips_null_padding(): def test_decode_bench_connector_uses_per_group_dcp_block_sizes(): - """DCP scales full-attention blocks but not sliding-window blocks.""" + """DCP scales target blocks but not the replicated draft's window.""" block_size = 16 dcp_world_size = 2 vllm_config = create_vllm_config( @@ -515,6 +515,7 @@ def test_decode_bench_connector_uses_per_group_dcp_block_sizes(): dtype=torch.float32, ) sliding_window = SlidingWindowSpec( + dcp_sharded=False, block_size=block_size, num_kv_heads=1, head_size=1, diff --git a/tests/v1/kv_connector/unit/test_handshake_pp_aggregation.py b/tests/v1/kv_connector/unit/test_handshake_pp_aggregation.py index 509d3d3234..045f6cb425 100644 --- a/tests/v1/kv_connector/unit/test_handshake_pp_aggregation.py +++ b/tests/v1/kv_connector/unit/test_handshake_pp_aggregation.py @@ -99,6 +99,7 @@ def _run_engine_core_handshake( ), speculative_config=None, ec_transfer_config=None, + is_mm_encoder_only=False, max_concurrent_batches=1, model_config=SimpleNamespace( runner_type="generate", diff --git a/tests/v1/kv_connector/unit/test_mooncake_connector.py b/tests/v1/kv_connector/unit/test_mooncake_connector.py index db9cf94171..5a5fffac3d 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_connector.py +++ b/tests/v1/kv_connector/unit/test_mooncake_connector.py @@ -3,10 +3,12 @@ import asyncio import contextlib +import threading import time from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest import torch import zmq.asyncio @@ -14,6 +16,7 @@ import zmq.asyncio from tests.v1.attention.utils import dense_kv_cache_views from vllm import envs from vllm.config import set_current_vllm_config +from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import mooncake_connector from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector import ( KVConnectorRole, MooncakeConnector, @@ -33,6 +36,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector im ) from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_utils import ( MooncakeBootstrapServer, + RegisterWorkerPayload, ) from vllm.utils.network_utils import get_open_port from vllm.v1.kv_cache_interface import ( @@ -538,6 +542,46 @@ def bootstrap_server(): server.shutdown() +@pytest.mark.asyncio +async def test_register_worker_recovers_from_slow_bootstrap_server(monkeypatch): + """End-to-end: a real HTTP server that responds slower than the configured + timeout must be retried on the wire, not treated as fatal.""" + original = MooncakeBootstrapServer.register_worker + state = {"done": False, "calls": 0} + + async def slow_register(self, payload: RegisterWorkerPayload): + state["calls"] += 1 + if not state["done"]: + state["done"] = True + # Record the worker first, then stall past the client timeout, so + # the retry exercises the duplicate-registration path. + response = await original(self, payload) + await asyncio.sleep(1.5) + return response + return await original(self, payload) + + monkeypatch.setattr(MooncakeBootstrapServer, "register_worker", slow_register) + + port = get_open_port() + monkeypatch.setenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", str(port)) + monkeypatch.setenv("VLLM_MOONCAKE_CONNECTOR_TIMEOUT", "0.5") + monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 5) + + server = MooncakeBootstrapServer("127.0.0.1", port) + server.start() + try: + worker = _make_local_register_worker_stub() + await MooncakeConnectorWorker.register_worker_with_bootstrap(worker) + assert state["calls"] == 2 + + async with httpx.AsyncClient() as client: + response = await client.get(f"http://127.0.0.1:{port}/query") + assert response.status_code == 200 + assert response.json()["0"]["engine_id"] == "eng-1" + finally: + server.shutdown() + + @pytest.mark.asyncio async def test_bootstrap_server(bootstrap_server: MooncakeBootstrapServer): """Tests the bootstrap server's api for worker registration and querying. @@ -589,9 +633,16 @@ async def test_bootstrap_server(bootstrap_server: MooncakeBootstrapServer): assert data["0"]["worker_addr"]["0"]["0"] == "tcp://1.1.1.1:1111" assert data["0"]["worker_addr"]["0"]["1"] == "tcp://2.2.2.2:2222" - # Test failure: re-registering the same worker + # Re-registering the identical payload is idempotent. async with httpx.AsyncClient() as client: response = await client.post(f"{base_url}/register", json=payload1) + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + + # Test failure: same ranks, conflicting address + payload1_conflict = dict(payload1, addr="tcp://9.9.9.9:9999") + async with httpx.AsyncClient() as client: + response = await client.post(f"{base_url}/register", json=payload1_conflict) assert response.status_code == 400 assert "is already registered" in response.text @@ -628,6 +679,137 @@ def _make_bootstrap_vllm_config( ) +def _make_local_register_worker_stub(): + return SimpleNamespace( + vllm_config=SimpleNamespace( + parallel_config=SimpleNamespace( + local_engines_only=False, + data_parallel_rank_local=0, + data_parallel_index=0, + nnodes_within_dp=1, + master_addr="127.0.0.1", + data_parallel_master_ip="127.0.0.1", + ) + ), + hostname="127.0.0.1", + side_channel_port=1234, + engine_id="eng-1", + dp_rank=0, + tp_rank=0, + pp_rank=0, + ) + + +def _make_register_worker_stub(): + return SimpleNamespace( + vllm_config=_make_bootstrap_vllm_config(), + hostname="127.0.0.1", + side_channel_port=1234, + engine_id="eng-1", + dp_rank=0, + tp_rank=0, + pp_rank=0, + ) + + +class _FlakyAsyncClient: + """httpx.AsyncClient stub that read-times-out `failures` times first.""" + + def __init__(self, failures: int, calls: list[int], **kwargs): + self.failures = failures + self.calls = calls + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc_info): + return False + + async def post(self, url, json=None): + self.calls.append(1) + if len(self.calls) <= self.failures: + raise httpx.ReadTimeout("simulated slow bootstrap server") + response = MagicMock() + response.raise_for_status = MagicMock() + return response + + +@pytest.mark.asyncio +async def test_register_worker_retries_on_read_timeout(monkeypatch): + """A slow rank-0 bootstrap response must be retried, not treated as fatal.""" + monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 5) + monkeypatch.setattr(asyncio, "sleep", AsyncMock()) + + calls: list[int] = [] + monkeypatch.setattr( + httpx, + "AsyncClient", + lambda **kwargs: _FlakyAsyncClient(2, calls), + ) + + worker = _make_register_worker_stub() + await MooncakeConnectorWorker.register_worker_with_bootstrap(worker) + + assert len(calls) == 3 + + +@pytest.mark.asyncio +async def test_register_worker_raises_after_max_attempts(monkeypatch): + """Terminal registration failure must raise, not loop forever.""" + monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 3) + monkeypatch.setattr(asyncio, "sleep", AsyncMock()) + + calls: list[int] = [] + monkeypatch.setattr( + httpx, + "AsyncClient", + lambda **kwargs: _FlakyAsyncClient(99, calls), + ) + + worker = _make_register_worker_stub() + with pytest.raises(RuntimeError, match="after 3 attempts"): + await MooncakeConnectorWorker.register_worker_with_bootstrap(worker) + + assert len(calls) == 3 + + +@pytest.mark.asyncio +async def test_sender_listener_failure_propagates_to_caller(monkeypatch): + """A terminal registration failure must surface via the Future rather than + leaving register_kv_caches blocked on its ready event.""" + monkeypatch.setenv("VLLM_MOONCAKE_CONNECTOR_TIMEOUT", "0.1") + monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 1) + + loop = asyncio.new_event_loop() + thread = threading.Thread(target=loop.run_forever, daemon=True) + thread.start() + + async def failing_listener(ready_event): + raise RuntimeError("simulated terminal registration failure") + + try: + ready_event = threading.Event() + fut = asyncio.run_coroutine_threadsafe(failing_listener(ready_event), loop) + + deadline = time.monotonic() + 10.0 + raised = None + while not ready_event.wait(timeout=0.1): + if fut.done(): + try: + fut.result() + except RuntimeError as e: + raised = e + break + assert time.monotonic() < deadline, "caller blocked past its deadline" + + assert raised is not None + assert "simulated terminal registration failure" in str(raised) + finally: + loop.call_soon_threadsafe(loop.stop) + thread.join(timeout=5) + loop.close() + + @pytest.mark.parametrize( ( "tp_rank", diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py index 947e9f6a20..f08213444e 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py @@ -290,6 +290,8 @@ class _FakeKVTransferConfig: class _FakeModelConfig: model = "test-model" use_mla = False + enable_sleep_mode = False + enable_cumem_allocator = False def get_num_layers(self, parallel_config) -> int: return 1 @@ -432,6 +434,64 @@ def _patch_worker_runtime( monkeypatch.setattr(worker, "LookupKeyServer", MagicMock()) +def test_create_mem_pool_returns_none_when_unconfigured(): + assert worker.MooncakeStoreWorker._create_mem_pool({}, _FakeModelConfig()) is None + + +def test_create_mem_pool_rejects_unknown_pool(): + with pytest.raises( + ValueError, + match="Unsupported custom_mem_pool='UNKNOWN'", + ): + worker.MooncakeStoreWorker._create_mem_pool( + {"custom_mem_pool": "unknown"}, _FakeModelConfig() + ) + + +@pytest.mark.parametrize( + "conflicting_option", ["enable_sleep_mode", "enable_cumem_allocator"] +) +def test_create_mem_pool_rejects_cumem_conflicts(conflicting_option): + model_config = _FakeModelConfig() + setattr(model_config, conflicting_option, True) + + with pytest.raises(ValueError, match="custom_mem_pool is incompatible"): + worker.MooncakeStoreWorker._create_mem_pool( + {"custom_mem_pool": "NVLINK"}, model_config + ) + + +@pytest.mark.parametrize( + ("pool_type", "allocator_name"), + [("nvlink", "NVLinkAllocator"), ("barex", "BarexAllocator")], +) +def test_create_mem_pool_uses_mooncake_allocator( + monkeypatch, pool_type, allocator_name +): + allocator_cls = MagicMock() + allocator = allocator_cls.get_allocator.return_value + allocator_module = types.ModuleType("mooncake.allocator") + setattr(allocator_module, allocator_name, allocator_cls) + mooncake_module = types.ModuleType("mooncake") + mooncake_module.allocator = allocator_module # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "mooncake", mooncake_module) + monkeypatch.setitem(sys.modules, "mooncake.allocator", allocator_module) + + monkeypatch.setattr(torch.accelerator, "current_device_index", lambda: 2) + mem_pool = MagicMock() + mem_pool_ctor = MagicMock(return_value=mem_pool) + monkeypatch.setattr(torch.cuda, "MemPool", mem_pool_ctor) + + result = worker.MooncakeStoreWorker._create_mem_pool( + {"custom_mem_pool": pool_type}, _FakeModelConfig() + ) + + assert result is mem_pool + allocator_cls.get_allocator.assert_called_once_with(torch.device("cuda", 2)) + allocator.allocator.assert_called_once_with() + mem_pool_ctor.assert_called_once_with(allocator.allocator.return_value) + + def test_pool_key_to_string_without_prefix_is_unchanged(): """Default (empty) cache_prefix keeps keys byte-identical to the historical unprefixed format so existing deployments keep their hits.""" diff --git a/tests/v1/kv_connector/unit/test_multi_connector.py b/tests/v1/kv_connector/unit/test_multi_connector.py index 1a85865560..9d04a2ea53 100644 --- a/tests/v1/kv_connector/unit/test_multi_connector.py +++ b/tests/v1/kv_connector/unit/test_multi_connector.py @@ -68,6 +68,7 @@ class MockConnector(KVConnectorBase_V1): # Override just build_kv_connector_stats mock.build_kv_connector_stats = cls.build_kv_connector_stats mock.get_kv_connector_stats.return_value = None + mock.get_mem_pool_context.return_value = None return mock @classmethod @@ -201,6 +202,36 @@ def mc() -> MultiConnector: return mc +def test_multi_connector_mem_pool_context_none(mc: MultiConnector): + assert mc.get_mem_pool_context() is None + + +def test_multi_connector_forwards_mem_pool_context(mc: MultiConnector): + context = MagicMock() + provider = MagicMock(spec_set=KVConnectorBase_V1) + provider.get_mem_pool_context.return_value = context + mc._connectors = [mc._connectors[0], provider] + + assert mc.get_mem_pool_context() is context + provider.get_mem_pool_context.assert_called_once_with() + + +def test_multi_connector_rejects_multiple_mem_pool_contexts(mc: MultiConnector): + providers = [ + MagicMock(spec_set=KVConnectorBase_V1), + MagicMock(spec_set=KVConnectorBase_V1), + ] + for provider in providers: + provider.get_mem_pool_context.return_value = MagicMock() + mc._connectors = providers + + with pytest.raises( + ValueError, + match="Multiple connectors provide a KV cache memory pool", + ): + mc.get_mem_pool_context() + + # Helper function to compare directories recursively def _compare_directories(dir1: Path, dir2: Path) -> bool: """Compares two directories recursively for identical content.""" @@ -320,6 +351,7 @@ def test_multi_example_connector_consistency(): # First three events are from initialization. Layer hooks run before the # deferred load starts after the forward pass. expected_worker_prefix = [ + "get_mem_pool_context", "register_kv_caches", "set_host_xfer_buffer_ops", "get_handshake_metadata", diff --git a/tests/v1/kv_connector/unit/test_nixl_connector.py b/tests/v1/kv_connector/unit/test_nixl_connector.py index 39f8f88d5c..d8b6fcb5a0 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector.py @@ -7,6 +7,7 @@ import os import queue import tempfile import textwrap +import threading import time import uuid from collections import defaultdict @@ -49,6 +50,8 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl import ( NixlKVConnectorStats, ) from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( + RemoteMeta, + ReqMeta, compute_nixl_compatibility_hash, ) from vllm.distributed.kv_transfer.kv_transfer_state import ( @@ -2087,6 +2090,10 @@ def recv_worker(): worker._recving_transfers = defaultdict(list) worker._failed_recv_reqs = queue.Queue() worker._recv_failures = set() + worker._handshake_lock = threading.RLock() + worker._handshake_futures = {} + worker._remote_agents = {} + worker._engine_by_address = {} worker._replicated_pcp_done_sending = set() worker._invalid_block_ids = queue.Queue() worker._pending_recv_notifs = {} @@ -2681,6 +2688,174 @@ def test_engine_ttl_disabled(default_vllm_config, dist_init): assert engine_id in worker.dst_xfer_side_handles +@pytest.mark.cpu_test +class TestPeerReplacement: + @pytest.fixture(autouse=True) + def setup(self, monkeypatch): + from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker as bw + from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend + + config = create_vllm_config(kv_connector_extra_config={"engine_ttl": 0}) + config.kv_transfer_config.kv_buffer_device = "cpu" + platform = SimpleNamespace( + device_type="cpu", + discover_numa_topology=lambda: [], + get_nixl_memory_type=lambda: "DRAM", + is_rocm=lambda: False, + ) + with ( + patch.object(bw, "NixlWrapper", FakeNixlWrapper), + patch.object(bw, "current_platform", platform), + patch.object(bw, "get_tensor_model_parallel_rank", return_value=0), + patch.object(bw, "get_tensor_model_parallel_world_size", return_value=1), + patch.object( + bw, "get_current_attn_backends", return_value=[FlashAttentionBackend] + ), + ): + self.worker = FakeNixlConnectorWorker(config, "local", hand_shake_latency=0) + self.nixl = self.worker.nixl_wrapper + self.transport = MagicMock(wraps=self.nixl) + self.worker.nixl_wrapper = self.transport + monkeypatch.setattr( + self.worker._handshake_initiation_executor, + "submit", + lambda *args: Future(), + ) + self._connect("old") + yield + self.worker.shutdown() + + def _connect(self, engine_id="new", host="localhost", port=1234): + future = self.worker._ensure_handshake(engine_id, host, port, 1) + self.worker.REMOTE_ENGINE_ID = engine_id + self.nixl.REMOTE_AGENT_NAME = engine_id + future.set_result(self.worker._nixl_handshake(host, port, 1, engine_id)) + + def _request(self, engine_id="new", blocks=(1,), awaiting_kvs=True): + metadata = NixlConnectorMetadata() + metadata.reqs_to_recv[f"{engine_id}-req"] = ReqMeta( + local_block_ids=(blocks,), + local_physical_block_ids=(blocks,), + tp_size=1, + remote=RemoteMeta( + ([2],), "localhost", 1234, engine_id, f"{engine_id}-prefill" + ), + awaiting_kvs=awaiting_kvs, + ) + return metadata + + def test_cleans_old_peer_after_confirmed_handshake(self): + self._connect("healthy", host="other-host") + old_handle = self.worker.dst_xfer_side_handles["old"][0] + self.worker.start_load_kv(self._request()) + self.worker.get_transfer_results() + self.transport.remove_remote_agent.assert_not_called() + + self._connect() + self.worker.get_transfer_results() + self.transport.release_dlist_handle.assert_called_once_with(old_handle) + self.transport.remove_remote_agent.assert_called_once_with("old") + assert set(self.worker._remote_agents) == {"new", "healthy"} + assert self.worker._engine_by_address == { + ("localhost", 1234): "new", + ("other-host", 1234): "healthy", + } + with pytest.raises(KeyError): + self.worker.transfer_topo.get_engine_info("old") + + self.worker.start_load_kv(NixlConnectorMetadata()) + assert self.worker.get_transfer_results().finished_recving == {"new-req"} + self.transport.remove_remote_agent.assert_called_once_with("old") + + @pytest.mark.parametrize("host,port", [("other-host", 1234), ("localhost", 5678)]) + def test_preserves_other_addresses(self, host, port): + self._connect(host=host, port=port) + self.worker.get_transfer_results() + assert "old" in self.worker._remote_agents + self.transport.remove_remote_agent.assert_not_called() + + def test_preserves_old_peer_when_handshake_fails(self): + self.worker.start_load_kv(self._request()) + self.worker._handshake_futures["new"].set_exception( + RuntimeError("handshake failed") + ) + assert self.worker.get_transfer_results().failed_recving == {"new-req"} + assert "old" in self.worker._remote_agents + assert self.worker._engine_by_address == {("localhost", 1234): "old"} + self.transport.remove_remote_agent.assert_not_called() + + @pytest.mark.parametrize( + "states,failed", + [ + (["PROC", "DONE"], False), + (["ERR", "PROC", "DONE"], True), + (["ERR", "PROC", "ERR"], True), + ], + ids=["healthy", "failed-sibling-completes", "failed-sibling-fails"], + ) + def test_waits_for_reads_before_cleanup_and_failure_reporting(self, states, failed): + self.worker._recving_metadata.update(self._request("old").reqs_to_recv) + self.worker._recving_transfers["old-req"] = [1, 2] if failed else [1] + self.transport.check_xfer_state.side_effect = states + self._connect() + + result = self.worker.get_transfer_results() + assert result.finished_recving == result.failed_recving == set() + assert self.worker.get_block_ids_with_load_errors() == set() + self.transport.remove_remote_agent.assert_not_called() + + result = self.worker.get_transfer_results() + assert result.finished_recving == {"old-req"} + assert result.failed_recving == ({"old-req"} if failed else set()) + assert self.worker.get_block_ids_with_load_errors() == ( + {1} if failed else set() + ) + assert self.transport.release_xfer_handle.call_count == (2 if failed else 1) + self.transport.remove_remote_agent.assert_called_once_with("old") + + def test_waits_for_queued_notification(self): + metadata = self._request("old", blocks=()) + self.worker._recving_metadata.update(metadata.reqs_to_recv) + self.worker._background_nixl_handshake( + "old-req", "old", metadata.reqs_to_recv["old-req"] + ) + self._connect() + self.worker.get_transfer_results() + self.transport.remove_remote_agent.assert_not_called() + + self.worker.start_load_kv(NixlConnectorMetadata()) + self.transport.send_notif.assert_called_once_with( + "old", notif_msg=b"old-prefill:1" + ) + result = self.worker.get_transfer_results() + assert result.finished_recving == {"old-req"} + assert result.failed_recving == set() + self.transport.remove_remote_agent.assert_called_once_with("old") + + def test_waits_for_other_handshakes(self): + self.worker._ensure_handshake("other", "other-host", 1234, 1) + self._connect() + self.worker.get_transfer_results() + self.transport.remove_remote_agent.assert_not_called() + + self._connect("other", host="other-host") + self.worker.get_transfer_results() + self.transport.remove_remote_agent.assert_called_once_with("old") + + def test_rehandshakes_queued_request_for_released_engine(self): + # A request queued after its engine's handshake, but drained only after + # that engine was replaced and released, must not read from it. + metadata = self._request("old", blocks=(), awaiting_kvs=False) + self.worker._ready_requests.put(("old-req", metadata.reqs_to_recv["old-req"])) + self._connect() + self.worker.get_transfer_results() + self.transport.remove_remote_agent.assert_called_once_with("old") + + self.worker.start_load_kv(NixlConnectorMetadata()) + assert "old" in self.worker._handshake_futures + self.transport.send_notif.assert_not_called() + + def test_transfer_topology_unregister(): """TransferTopology.unregister_remote_engine removes the engine.""" from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend diff --git a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py index 3be1730ec3..125dfa6143 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py @@ -4,6 +4,7 @@ import gc import queue +import threading from types import SimpleNamespace from unittest.mock import MagicMock, patch @@ -54,6 +55,9 @@ def region_pull_worker(): worker.enable_permute_local_kv = False worker.enable_heterogeneous_attn_post_process = False worker._engine_last_active = {} + worker._handshake_lock = threading.RLock() + worker._handshake_futures = {} + worker._transfer_layer_group_ids = () worker._bidirectional_kv_xfer_enabled = False worker._recving_transfers = {} worker.use_mla, worker._has_mamba = True, False @@ -94,6 +98,7 @@ def region_pull_worker(): worker.dst_num_blocks = {"P": 100, "D": 100} worker.dst_region_num_blocks = {"P": [100, 100], "D": [100, 100]} worker._remote_agents = {"P": {(0, 0): "P-rank0"}} + worker._engine_by_address = {("localhost", 1): "P"} worker._read_blocks_mixed = MagicMock() worker.nixl_wrapper = MagicMock() return worker diff --git a/tests/v1/kv_connector/unit/test_nixl_push_connector.py b/tests/v1/kv_connector/unit/test_nixl_push_connector.py index 231343de37..4d8ead28f3 100644 --- a/tests/v1/kv_connector/unit/test_nixl_push_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_push_connector.py @@ -395,6 +395,7 @@ class _StubWriterWorker(NixlPushConnectorWorker): w._group_spec_types = (FullAttentionSpec,) w._engine_ttl = 0.0 w._engine_last_active = {} + w._engine_by_address = {} # Track _do_start_push_kv invocations. calls: list[tuple[str, Any, dict[str, Any]]] = [] diff --git a/tests/v1/kv_connector/unit/test_simple_cpu_offload_metrics.py b/tests/v1/kv_connector/unit/test_simple_cpu_offload_metrics.py new file mode 100644 index 0000000000..16358dec7e --- /dev/null +++ b/tests/v1/kv_connector/unit/test_simple_cpu_offload_metrics.py @@ -0,0 +1,466 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for SimpleCPUOffloadConnector metrics.""" + +from __future__ import annotations + +import dataclasses +from types import SimpleNamespace +from typing import Any + +import pytest +from prometheus_client import Counter, Gauge, Histogram + +from tests.v1.kv_connector.unit.test_simple_cpu_offload_connector import ( + _make_connector, +) +from tests.v1.simple_kv_offload.test_scheduler import ( + make_request, + make_scheduler, + make_scheduler_output, +) +from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector import ( + SimpleCPUOffloadConnector, +) +from vllm.v1.outputs import KVConnectorOutput +from vllm.v1.simple_kv_offload.manager import ( + BoundaryStoreStats, + LoadRequestState, + TransferMeta, +) +from vllm.v1.simple_kv_offload.metrics import ( + INFO_LABELS, + OUTCOME_TO_FIELD, + MetricName, + SimpleCPUOffloadStats, + _MetricType, + _StatsKey, +) + +SAVE_OUTCOMES = MetricName.SAVE_OUTCOMES +LOAD_BLOCKS = MetricName.LOAD_BLOCKS +USED_BLOCKS = MetricName.USED_BLOCKS +PENDING_STORE_BLOCKS = MetricName.PENDING_STORE_BLOCKS +INFO = MetricName.INFO + + +class _FakeMetric: + def __init__(self, **kwargs: Any): + self.kwargs = kwargs + self.children: list[_FakeMetric] = [] + self.observed: list[int | float] = [] + self.increments: list[int | float] = [] + self.set_values: list[int | float] = [] + self.labelvalues: tuple[object, ...] = () + + def labels(self, *labelvalues: object) -> _FakeMetric: + child = _FakeMetric(**self.kwargs) + child.labelvalues = labelvalues + self.children.append(child) + return child + + def observe(self, value: int | float) -> None: + self.observed.append(value) + + def inc(self, value: int | float) -> None: + self.increments.append(value) + + def set(self, value: int | float) -> None: + self.set_values.append(value) + + +def _fake_metric_types() -> dict[type, type]: + return {Gauge: _FakeMetric, Counter: _FakeMetric, Histogram: _FakeMetric} + + +def _make_prom_metrics(per_engine: dict[int, list[object]] | None = None) -> Any: + from vllm.v1.simple_kv_offload.metrics import SimpleCPUOffloadPromMetrics + + return SimpleCPUOffloadPromMetrics( + vllm_config=SimpleNamespace(kv_transfer_config=None), + metric_types=_fake_metric_types(), + labelnames=["model_name", "engine"], + per_engine_labelvalues=per_engine or {0: ["model", "0"]}, + ) + + +def _stats_payload(types: dict, data: dict) -> dict[str, Any]: + return {_StatsKey.TYPES: types, _StatsKey.DATA: data} + + +def _values(stats: SimpleCPUOffloadStats, name: str) -> dict: + return stats.data[_StatsKey.DATA][name] + + +# --------------------------------------------------------------------------- +# Stats class +# --------------------------------------------------------------------------- + + +def test_build_kv_connector_stats_none_and_empty() -> None: + stats = SimpleCPUOffloadConnector.build_kv_connector_stats(data=None) + assert isinstance(stats, SimpleCPUOffloadStats) + assert stats.is_empty() + + stats = SimpleCPUOffloadConnector.build_kv_connector_stats(data={}) + assert isinstance(stats, SimpleCPUOffloadStats) + assert stats.is_empty() + + +def test_stats_round_trip_through_serialized_dict() -> None: + stats = SimpleCPUOffloadStats() + stats.increase_counter(SAVE_OUTCOMES, 3, ("stored",)) + stats.increase_counter(LOAD_BLOCKS, 4) + stats.set_gauge(USED_BLOCKS, 7) + stats.set_gauge(INFO, 1, ("cpu", "false", "false", "64")) + + rebuilt = SimpleCPUOffloadConnector.build_kv_connector_stats(data=stats.to_dict()) + assert isinstance(rebuilt, SimpleCPUOffloadStats) + assert _values(rebuilt, SAVE_OUTCOMES) == {("stored",): 3} + assert _values(rebuilt, LOAD_BLOCKS) == {(): 4} + assert _values(rebuilt, USED_BLOCKS) == {(): 7} + assert _values(rebuilt, INFO) == {("cpu", "false", "false", "64"): 1} + + +def test_stats_aggregate_sums_counters_and_keeps_latest_gauge() -> None: + stats1 = SimpleCPUOffloadStats() + stats1.increase_counter(SAVE_OUTCOMES, 3, ("stored",)) + stats1.increase_counter(SAVE_OUTCOMES, 1, ("dropped_cpu_full",)) + stats1.set_gauge(USED_BLOCKS, 7) + + stats2 = SimpleCPUOffloadStats() + stats2.increase_counter(SAVE_OUTCOMES, 4, ("stored",)) + stats2.set_gauge(USED_BLOCKS, 2) + + result = stats1.aggregate(stats2) + assert result is stats1 + outcomes = _values(stats1, SAVE_OUTCOMES) + assert outcomes == {("stored",): 7, ("dropped_cpu_full",): 1} + assert _values(stats1, USED_BLOCKS) == {(): 2} + + +def test_stats_aggregate_merges_types_from_other() -> None: + stats1 = SimpleCPUOffloadStats() + stats1.set_gauge(USED_BLOCKS, 1) + + stats2 = SimpleCPUOffloadStats() + stats2.increase_counter(LOAD_BLOCKS, 2) + + stats1.aggregate(stats2) + assert stats1.data[_StatsKey.TYPES][LOAD_BLOCKS] == _MetricType.COUNTER + assert _values(stats1, LOAD_BLOCKS) == {(): 2} + + +def test_stats_reduce_and_reset() -> None: + stats = SimpleCPUOffloadStats() + stats.increase_counter(SAVE_OUTCOMES, 3, ("stored",)) + stats.set_gauge(PENDING_STORE_BLOCKS, 4) + stats.increase_counter(LOAD_BLOCKS, 9) + stats.set_gauge(INFO, 1, ("cpu", "false", "false", "64")) + + reduced = stats.reduce() + assert reduced[f"{SAVE_OUTCOMES}:{('stored',)}"] == 3 + assert reduced[PENDING_STORE_BLOCKS] == 4 + assert reduced[LOAD_BLOCKS] == 9 + assert not any(key.startswith(INFO) for key in reduced) + + stats.reset() + assert stats.is_empty() + + +def test_outcome_labels_match_boundary_store_stats_fields() -> None: + """Every outcome label maps 1:1 to a BoundaryStoreStats counter field.""" + boundary_fields = { + f.name for f in dataclasses.fields(BoundaryStoreStats) if f.name != "published" + } + assert set(OUTCOME_TO_FIELD.values()) == boundary_fields + assert len(set(OUTCOME_TO_FIELD.values())) == len(OUTCOME_TO_FIELD) + + +# --------------------------------------------------------------------------- +# Prom metrics class +# --------------------------------------------------------------------------- + + +def test_prom_metrics_registers_tier1_metrics() -> None: + prom = _make_prom_metrics() + defs = prom._defs + assert set(defs) == { + SAVE_OUTCOMES, + LOAD_BLOCKS, + USED_BLOCKS, + PENDING_STORE_BLOCKS, + INFO, + } + # The prom class registers literal names (required by the docs + # generator); they must stay in sync with the emission-side constants. + assert {m.kwargs["name"] for m in defs.values()} == { + SAVE_OUTCOMES, + LOAD_BLOCKS, + USED_BLOCKS, + PENDING_STORE_BLOCKS, + INFO, + } + for metric in defs.values(): + assert metric.kwargs["documentation"] + assert metric.kwargs["labelnames"][:2] == ["model_name", "engine"] + + assert defs[SAVE_OUTCOMES].kwargs["labelnames"] == [ + "model_name", + "engine", + "outcome", + ] + assert defs[LOAD_BLOCKS].kwargs["labelnames"] == ["model_name", "engine"] + assert defs[INFO].kwargs["labelnames"] == [ + "model_name", + "engine", + *INFO_LABELS, + ] + assert defs[USED_BLOCKS].kwargs["labelnames"] == ["model_name", "engine"] + + # mostrecent: multiprocess scrapes expose one series per label set, not + # one pid-tagged series per writer process. + for gauge in (USED_BLOCKS, PENDING_STORE_BLOCKS, INFO): + assert defs[gauge].kwargs["multiprocess_mode"] == "mostrecent" + + +def test_prom_metrics_observe_routes_each_series() -> None: + prom = _make_prom_metrics() + prom.observe( + _stats_payload( + types={ + SAVE_OUTCOMES: _MetricType.COUNTER, + LOAD_BLOCKS: _MetricType.COUNTER, + USED_BLOCKS: _MetricType.GAUGE, + PENDING_STORE_BLOCKS: _MetricType.GAUGE, + INFO: _MetricType.GAUGE, + }, + data={ + SAVE_OUTCOMES: {("stored",): 3}, + LOAD_BLOCKS: {(): 4}, + USED_BLOCKS: {(): 5}, + PENDING_STORE_BLOCKS: {(): 1}, + INFO: {("cpu", "false", "false", "64"): 1}, + }, + ) + ) + + assert prom._metrics[(0, SAVE_OUTCOMES, ("stored",))].increments == [3] + assert prom._metrics[(0, LOAD_BLOCKS, ())].increments == [4] + assert prom._metrics[(0, USED_BLOCKS, ())].set_values == [5] + assert prom._metrics[(0, PENDING_STORE_BLOCKS, ())].set_values == [1] + assert prom._metrics[(0, INFO, ("cpu", "false", "false", "64"))].set_values == [1] + assert prom._metrics[(0, SAVE_OUTCOMES, ("stored",))].labelvalues == ( + "model", + "0", + "stored", + ) + assert prom._metrics[(0, INFO, ("cpu", "false", "false", "64"))].labelvalues == ( + "model", + "0", + "cpu", + "false", + "false", + "64", + ) + + +def test_prom_metrics_reuses_bound_children() -> None: + prom = _make_prom_metrics() + payload = _stats_payload( + types={SAVE_OUTCOMES: _MetricType.COUNTER}, + data={SAVE_OUTCOMES: {("stored",): 1}}, + ) + prom.observe(payload) + prom.observe(payload) + + child = prom._metrics[(0, SAVE_OUTCOMES, ("stored",))] + assert child.increments == [1, 1] + assert len(prom._defs[SAVE_OUTCOMES].children) == 1 + + +def test_prom_metrics_rejects_unknown_metric() -> None: + prom = _make_prom_metrics() + with pytest.raises(AssertionError, match="Unknown"): + prom.observe( + _stats_payload( + types={"vllm:not_a_metric": _MetricType.COUNTER}, + data={"vllm:not_a_metric": {(): 1}}, + ) + ) + + +def test_prom_metrics_rejects_wrong_label_count() -> None: + prom = _make_prom_metrics() + with pytest.raises(AssertionError, match="labels"): + prom.observe( + _stats_payload( + types={SAVE_OUTCOMES: _MetricType.COUNTER}, + data={SAVE_OUTCOMES: {("stored", "extra"): 1}}, + ) + ) + + +def test_prom_metrics_routes_per_engine() -> None: + prom = _make_prom_metrics(per_engine={0: ["model", "0"], 1: ["model", "1"]}) + prom.observe( + _stats_payload( + types={USED_BLOCKS: _MetricType.GAUGE}, data={USED_BLOCKS: {(): 3}} + ), + engine_idx=1, + ) + assert (0, USED_BLOCKS, ()) not in prom._metrics + assert prom._metrics[(1, USED_BLOCKS, ())].set_values == [3] + + +def test_connector_build_prom_metrics() -> None: + from vllm.v1.simple_kv_offload.metrics import SimpleCPUOffloadPromMetrics + + prom = SimpleCPUOffloadConnector.build_prom_metrics( + vllm_config=SimpleNamespace(kv_transfer_config=None), + metric_types=_fake_metric_types(), + labelnames=["model_name", "engine"], + per_engine_labelvalues={0: ["model", "0"]}, + ) + assert isinstance(prom, SimpleCPUOffloadPromMetrics) + + +# --------------------------------------------------------------------------- +# Manager: interval stats +# --------------------------------------------------------------------------- + + +def test_manager_get_stats_reports_outcome_deltas_per_interval() -> None: + fixture = make_scheduler() + sched = fixture.scheduler + + sched.boundary_store_stats.stored += 3 + sched.boundary_store_stats.dropped_cpu_full += 2 + sched.boundary_store_stats.skipped_in_flight += 1 + + stats = sched.get_stats() + assert _values(stats, SAVE_OUTCOMES) == { + ("stored",): 3, + ("dropped_cpu_full",): 2, + ("skipped_in_flight",): 1, + } + + # Deltas are per-interval: a second drain reports no new outcomes. + stats2 = sched.get_stats() + assert SAVE_OUTCOMES not in stats2.data[_StatsKey.DATA] + + +def test_manager_reset_preserves_undrained_outcome_deltas() -> None: + fixture = make_scheduler() + sched = fixture.scheduler + + sched.boundary_store_stats.stored += 3 + sched.reset() + + stats = sched.get_stats() + assert _values(stats, SAVE_OUTCOMES) == {("stored",): 3} + + +def test_manager_get_stats_gauges_and_info_labels() -> None: + fixture = make_scheduler(num_cpu_blocks=8) + sched = fixture.scheduler + + sched._store_event_to_blocks[0] = TransferMeta([10], [11]) + sched._pending_finished_stores.append(TransferMeta([12], [13])) + sched._abandoned_store_event_to_blocks[1] = TransferMeta([14], [15]) + + stats = sched.get_stats() + assert _values(stats, PENDING_STORE_BLOCKS) == {(): 3} + used = _values(stats, USED_BLOCKS)[()] + assert used == sched.num_cpu_blocks - sched.cpu_block_pool.get_num_free_blocks() + + info = _values(stats, INFO) + assert info == {("cpu", "false", "false", str(sched.num_cpu_blocks)): 1} + + # Gauges are re-reported every drain. + stats2 = sched.get_stats() + assert _values(stats2, PENDING_STORE_BLOCKS) == {(): 3} + + +def test_manager_info_labels_reflect_disk_mode() -> None: + connector = _make_connector( + extra_config={ + "kv_offload_backend": "disk", + "disk_path": "/tmp/fake-disk-metrics", + "disk_capacity_bytes": 1024**3, + "use_page_cache": True, + } + ) + sched = connector.scheduler_manager + assert sched is not None + assert sched._info_labelvalues[0] == "disk" + assert sched._info_labelvalues[1] == "true" + stats = sched.get_stats() + assert set(_values(stats, INFO)) == {sched._info_labelvalues} + + +def test_manager_info_labels_page_cache_false_for_cpu_backend() -> None: + """CPU backend ignores use_page_cache; the info gauge must not claim it.""" + connector = _make_connector(extra_config={"use_page_cache": True}) + sched = connector.scheduler_manager + assert sched is not None + assert sched._info_labelvalues[:2] == ("cpu", "false") + stats = sched.get_stats() + assert set(_values(stats, INFO)) == {sched._info_labelvalues} + + +def test_manager_counts_load_blocks_completed() -> None: + fixture = make_scheduler(num_cpu_blocks=8) + sched = fixture.scheduler + + request = make_request(num_blocks=2) + gpu_blocks = fixture.gpu_block_pool.get_new_blocks(2) + cpu_blocks = sched.cpu_block_pool.get_new_blocks(2) + sched._reqs_to_load[request.request_id] = LoadRequestState( + request=request, + transfer_meta=TransferMeta( + [b.block_id for b in gpu_blocks], [b.block_id for b in cpu_blocks] + ), + ) + + sched.build_connector_meta(make_scheduler_output({})) + stats = sched.get_stats() + assert LOAD_BLOCKS not in stats.data[_StatsKey.DATA] + + sched.update_connector_output( + KVConnectorOutput(finished_recving={request.request_id}) + ) + stats = sched.get_stats() + assert _values(stats, LOAD_BLOCKS) == {(): 2} + + +def test_manager_abandoned_load_counts_as_completed_after_reset() -> None: + """Loads abandoned by reset() that still finish on the worker are + counted as loaded blocks. + """ + fixture = make_scheduler(num_cpu_blocks=8) + sched = fixture.scheduler + + request = make_request(num_blocks=2) + gpu_blocks = fixture.gpu_block_pool.get_new_blocks(2) + cpu_blocks = sched.cpu_block_pool.get_new_blocks(2) + sched._reqs_to_load[request.request_id] = LoadRequestState( + request=request, + transfer_meta=TransferMeta( + [b.block_id for b in gpu_blocks], [b.block_id for b in cpu_blocks] + ), + ) + + sched.build_connector_meta(make_scheduler_output({})) + stats = sched.get_stats() + assert LOAD_BLOCKS not in stats.data[_StatsKey.DATA] + + assert sched.reset() is False + assert set(sched._abandoned_reqs_to_load) == {request.request_id} + + sched.update_connector_output( + KVConnectorOutput(finished_recving={request.request_id}) + ) + assert sched._abandoned_reqs_to_load == {} + stats = sched.get_stats() + assert _values(stats, LOAD_BLOCKS) == {(): 2} diff --git a/tests/v1/simple_kv_offload/test_scheduler.py b/tests/v1/simple_kv_offload/test_scheduler.py index ed07718de6..3817a15e3e 100644 --- a/tests/v1/simple_kv_offload/test_scheduler.py +++ b/tests/v1/simple_kv_offload/test_scheduler.py @@ -2316,8 +2316,8 @@ def _make_hybrid_attention_mamba_scheduler( def test_hybrid_store_uses_resolved_group_block_sizes() -> None: """Replicated groups must not be scaled by the DCP world size. - ``dcp_world_size_for_kv_cache_spec`` gives full attention the process DCP - size and every other spec 1. Reconstructing a group's geometry as + The spec's ``dcp_sharded`` flag controls its effective block size. + Reconstructing a group's geometry as ``spec.block_size * cp_world_size`` over-scales the replicated groups, so the eager store scan believes far fewer of their blocks are ready and silently offloads only a fraction of them. diff --git a/tests/v1/spec_decode/test_dflash_prepare_inputs.py b/tests/v1/spec_decode/test_dflash_prepare_inputs.py index 16d6d8e516..1d004c3667 100644 --- a/tests/v1/spec_decode/test_dflash_prepare_inputs.py +++ b/tests/v1/spec_decode/test_dflash_prepare_inputs.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from types import SimpleNamespace +from unittest.mock import patch import numpy as np import pytest @@ -9,7 +10,7 @@ import torch from vllm.v1.attention.backends.utils import PAD_SLOT_ID from vllm.v1.worker.gpu.spec_decode.dflash.speculator import ( - prepare_dflash_inputs, + DFlashSpeculator, ) pytestmark = pytest.mark.skipif( @@ -24,6 +25,7 @@ def _run_prepare( cp_rank: int = 0, cp_size: int = 1, cp_interleave: int = 1, + draft_dcp_size: int | None = None, ): device = torch.device("cuda") max_num_reqs = 4 @@ -40,7 +42,9 @@ def _run_prepare( ) input_batch = SimpleNamespace( num_reqs=1, + num_tokens=4, num_scheduled_tokens=np.array([4], dtype=np.int32), + seq_lens_cpu_upper_bound=torch.tensor([target_positions[-1] + 1]), positions=torch.tensor(target_positions, dtype=torch.int64, device=device), query_start_loc=torch.tensor([0, 4], dtype=torch.int32, device=device), idx_mapping=torch.tensor([2], dtype=torch.int32, device=device), @@ -74,36 +78,65 @@ def _run_prepare( next_prefill_tokens = torch.zeros_like(last_sampled) block_table = torch.tensor([block_table_values], dtype=torch.int32, device=device) - prepare_dflash_inputs( - input_buffers, - query_slot_mapping, - context_positions, - context_slot_mapping, - sample_indices, - sample_pos, - sample_idx_mapping, - temperature, - seeds, - input_batch, - torch.tensor([1], dtype=torch.int32, device=device), - torch.tensor([2], dtype=torch.int32, device=device), - last_sampled, - next_prefill_tokens, - input_temperature, - input_seeds, - block_table, - 4, - cp_rank, - cp_size, - cp_interleave, - 123, - num_speculative_steps, - num_speculative_steps, - max_num_reqs, - max_num_tokens, - 128, + class InputsPrepared(Exception): + pass + + draft = SimpleNamespace( + input_buffers=input_buffers, + context_positions=context_positions, + _context_slot_mappings=context_slot_mapping.unsqueeze(0), + sample_indices=sample_indices, + sample_pos=sample_pos, + sample_idx_mapping=sample_idx_mapping, + temperature=temperature, + seeds=seeds, + hidden_states=torch.zeros(4, 1, device=device), + prepare_context_anchor=lambda *args: None, + query_cudagraph_manager=None, + dp_size=1, + dp_rank=0, + pcp_manager=None, + draft_kv_cache_group_id=0, + draft_kv_cache_group_ids=[0], + _layer_group_idx=None, + block_tables=SimpleNamespace( + slot_mappings=query_slot_mapping.unsqueeze(0), + input_block_tables=[block_table], + kernel_block_sizes=[4], + cp_rank=cp_rank, + cp_size=cp_size, + cp_interleave=cp_interleave, + ), + dcp_size=cp_size if draft_dcp_size is None else draft_dcp_size, + parallel_drafting_token_id=123, + num_query_per_req=num_speculative_steps, + num_speculative_steps=num_speculative_steps, + max_num_reqs=max_num_reqs, + max_num_tokens=max_num_tokens, + max_model_len=128, sample_from_anchor=True, ) + with ( + patch( + "vllm.v1.worker.gpu.spec_decode.dflash.speculator.dispatch_cg_and_sync_dp", + side_effect=InputsPrepared, + ), + pytest.raises(InputsPrepared), + ): + DFlashSpeculator.propose( + draft, + input_batch, + {}, + {}, + draft.hidden_states, + None, + torch.tensor([1], dtype=torch.int32, device=device), + torch.tensor([2], dtype=torch.int32, device=device), + last_sampled, + next_prefill_tokens, + input_temperature, + input_seeds, + ) torch.accelerator.synchronize() return SimpleNamespace( input_buffers=input_buffers, @@ -118,12 +151,17 @@ def _run_prepare( ) -def test_prepare_dflash_inputs_excludes_rejected_context_suffix(): +@pytest.mark.parametrize("cp_rank,cp_size", [(0, 1), (0, 2), (1, 2)]) +def test_prepare_dflash_inputs_excludes_rejected_context_suffix(cp_rank, cp_size): # Positions 10/11 use physical block 7. Rejected positions 12/13 would use # block 8, but must be PAD context rather than contaminating draft KV. out = _run_prepare( target_positions=[10, 11, 12, 13], block_table_values=[0, 0, 7, 8, 9, 10, 11, 12], + cp_rank=cp_rank, + cp_size=cp_size, + cp_interleave=2, + draft_dcp_size=1, ) assert out.context_positions[:4].tolist() == [10, 11, 0, 0] diff --git a/tests/v1/spec_decode/test_eagle_draft_attn_metadata.py b/tests/v1/spec_decode/test_eagle_draft_attn_metadata.py index b542388952..2c4b951be7 100644 --- a/tests/v1/spec_decode/test_eagle_draft_attn_metadata.py +++ b/tests/v1/spec_decode/test_eagle_draft_attn_metadata.py @@ -17,6 +17,7 @@ from types import MethodType, SimpleNamespace from unittest.mock import patch import numpy as np +import pytest import torch from vllm.config.compilation import CUDAGraphMode @@ -53,6 +54,7 @@ def _make_fake_speculator( arange_np=np.arange(max_num_reqs + 1, dtype=np.int32), draft_is_prefilling=torch.zeros(max_num_reqs, dtype=torch.bool), block_tables=fake_block_tables, + dcp_size=1, input_buffers=fake_input_buffers, attn_groups=[], kv_cache_config=SimpleNamespace(kv_cache_groups=[]), @@ -152,14 +154,17 @@ def test_build_draft_attn_metadata_clamps_to_max_model_len(): assert torch.equal(bound, torch.tensor([1024, 503], dtype=torch.int32)) -def test_build_draft_attn_metadata_recomputes_dcp_local_seq_lens(): +@pytest.mark.parametrize("draft_dcp_size", [1, 2]) +def test_build_draft_attn_metadata_recomputes_dcp_local_seq_lens(draft_dcp_size): fake = _make_fake_speculator() + fake.dcp_size = draft_dcp_size fake.block_tables.cp_size = 2 fake.block_tables.cp_rank = 1 fake.block_tables.cp_interleave = 4 fake.input_buffers.seq_lens[:3] = torch.tensor([5, 9, 16]) def fake_prepare(out, seq_lens, num_reqs, dcp_size, dcp_rank, cp_interleave): + assert draft_dcp_size > 1 assert seq_lens is fake.input_buffers.seq_lens assert (num_reqs, dcp_size, dcp_rank, cp_interleave) == (3, 2, 1, 4) out[:num_reqs].copy_(torch.tensor([1, 4, 8], dtype=torch.int32)) @@ -176,6 +181,9 @@ def test_build_draft_attn_metadata_recomputes_dcp_local_seq_lens(): ) local = captured["dcp_local_seq_lens"] + if draft_dcp_size == 1: + assert local is None + return assert isinstance(local, torch.Tensor) assert local.data_ptr() == fake.input_buffers.dcp_local_seq_lens.data_ptr() assert local.tolist() == [1, 4, 8, 0] diff --git a/tests/v1/worker/test_cp_utils.py b/tests/v1/worker/test_cp_utils.py index b38ae4d063..d477ace479 100644 --- a/tests/v1/worker/test_cp_utils.py +++ b/tests/v1/worker/test_cp_utils.py @@ -1,5 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from types import SimpleNamespace + import pytest import torch @@ -7,6 +9,38 @@ from vllm.v1.attention.backends.utils import get_dcp_local_seq_lens from vllm.v1.worker.cp_utils import should_skip_dcp_context_attention +def test_cp_checks_allow_a_replicated_draft_backend_without_dcp(monkeypatch): + from vllm.v1.worker import cp_utils + + config = SimpleNamespace( + parallel_config=SimpleNamespace( + prefill_context_parallel_size=1, + decode_context_parallel_size=4, + cp_kv_cache_interleave_size=16, + ), + speculative_config=object(), + ) + target = SimpleNamespace( + impl=SimpleNamespace( + dcp_world_size=4, + supports_mtp_with_cp_non_trivial_interleave_size=True, + need_to_return_lse_for_decode=True, + ) + ) + draft = SimpleNamespace( + impl=SimpleNamespace( + dcp_world_size=1, + supports_mtp_with_cp_non_trivial_interleave_size=False, + need_to_return_lse_for_decode=False, + ) + ) + layers = {"target": target, "draft": draft} + monkeypatch.setattr(cp_utils, "get_layers_from_vllm_config", lambda *args: layers) + cp_utils.check_attention_cp_compatibility(config, target_layer_names={"target"}) + with pytest.raises(AssertionError, match="MTP with cp_kv_cache_interleave_size"): + cp_utils.check_attention_cp_compatibility(config) + + def test_skip_gate_only_for_zero_context(): assert should_skip_dcp_context_attention(torch.zeros(3, dtype=torch.int32)) assert not should_skip_dcp_context_attention( diff --git a/tests/v1/worker/test_dspark_utils.py b/tests/v1/worker/test_dspark_utils.py index 94655f192c..1014051d64 100644 --- a/tests/v1/worker/test_dspark_utils.py +++ b/tests/v1/worker/test_dspark_utils.py @@ -2,8 +2,17 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from dataclasses import dataclass, field +from types import SimpleNamespace +import pytest +import torch + +from vllm.config import ParallelConfig +from vllm.v1.kv_cache_interface import FullAttentionSpec +from vllm.v1.worker.gpu import model_runner +from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator from vllm.v1.worker.gpu.spec_decode.dspark.utils import _get_dspark_parallel_config +from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator @dataclass @@ -49,3 +58,86 @@ def test_dspark_parallel_config_disables_eplb_atomically(): assert draft_config.eplb_config.num_redundant_experts == 0 assert not draft_config.enable_elastic_ep assert draft_config.eplb_config is not target_config.eplb_config + + +@pytest.mark.parametrize("pcp_size", [1, 4]) +@pytest.mark.parametrize("dcp_size", [1, 4]) +@pytest.mark.parametrize("use_mla", [False, True]) +def test_draft_context_parallelism_without_changing_target( + monkeypatch, pcp_size, dcp_size, use_mla +): + target_parallel = ParallelConfig( + tensor_parallel_size=4, + prefill_context_parallel_size=pcp_size, + decode_context_parallel_size=dcp_size, + cp_kv_cache_interleave_size=16, + distributed_executor_backend="mp", + ) + target_config = SimpleNamespace( + parallel_config=target_parallel, + speculative_config=SimpleNamespace( + draft_model_config=SimpleNamespace(use_mla=use_mla) + ), + ) + + class CapturedConfig(Exception): + pass + + def capture_init(self, config, device): + raise CapturedConfig(config) + + monkeypatch.setattr(DraftModelSpeculator, "__init__", capture_init) + with pytest.raises(CapturedConfig) as captured: + DFlashSpeculator(target_config, device=None) + draft_parallel = captured.value.args[0].parallel_config + assert draft_parallel.tensor_parallel_size == 4 + assert draft_parallel.prefill_context_parallel_size == 1 + assert draft_parallel.decode_context_parallel_size == (dcp_size if use_mla else 1) + assert draft_parallel.cp_kv_cache_interleave_size == 16 + assert target_parallel.prefill_context_parallel_size == pcp_size + assert target_parallel.decode_context_parallel_size == dcp_size + assert target_parallel.cp_kv_cache_interleave_size == 16 + + +@pytest.mark.parametrize("configured_dcp", [1, 4]) +def test_attention_uses_draft_dcp_setting_inside_target_process_group( + monkeypatch, configured_dcp +): + import vllm.config as config_module + from vllm.distributed import parallel_state + from vllm.v1.attention.backend import AttentionImplBase + + config = SimpleNamespace( + parallel_config=SimpleNamespace(decode_context_parallel_size=configured_dcp) + ) + monkeypatch.setattr( + config_module, "get_current_vllm_config_or_none", lambda: config + ) + + monkeypatch.setattr( + parallel_state, "_DCP", SimpleNamespace(world_size=4, rank_in_group=2) + ) + impl = AttentionImplBase() + assert impl.dcp_world_size == configured_dcp + assert impl.dcp_rank == (0 if configured_dcp == 1 else 2) + + +@pytest.mark.parametrize("draft_dcp_size", [1, 4]) +def test_runner_marks_only_replicated_draft_caches(monkeypatch, draft_dcp_size): + spec = FullAttentionSpec( + block_size=16, num_kv_heads=1, head_size=64, dtype=torch.bfloat16 + ) + monkeypatch.setattr( + model_runner, "get_kv_cache_spec", lambda _: {"target": spec, "draft": spec} + ) + runner = object.__new__(model_runner.GPUModelRunner) + runner.vllm_config = None + runner.dcp_size = 4 + runner.speculator = object.__new__(DFlashSpeculator) + runner.speculator.dcp_size = draft_dcp_size + runner.speculator.draft_attn_layer_names = {"draft"} + + specs = runner.get_kv_cache_spec() + + assert specs["target"].dcp_sharded + assert specs["draft"].dcp_sharded == (draft_dcp_size == 4) diff --git a/tests/v1/worker/test_gpu_autoregressive_speculator.py b/tests/v1/worker/test_gpu_autoregressive_speculator.py index 4fc7d4d2e5..fde3605a20 100644 --- a/tests/v1/worker/test_gpu_autoregressive_speculator.py +++ b/tests/v1/worker/test_gpu_autoregressive_speculator.py @@ -88,6 +88,7 @@ def test_pcp_draft_metadata_keeps_graph_padding_in_decode(cg_mode): query_start_loc=torch.tensor([0, 1, 2, 2, 2], dtype=torch.int32), seq_lens=torch.tensor([11, 21, 0, 0], dtype=torch.int32), ) + speculator.dcp_size = 1 speculator.block_tables = SimpleNamespace( cp_size=1, input_block_tables=[torch.zeros(4, 1, dtype=torch.int32)], @@ -97,7 +98,9 @@ def test_pcp_draft_metadata_keeps_graph_padding_in_decode(cg_mode): speculator.attn_groups = [ [ SimpleNamespace( - get_metadata_builder=lambda _: SimpleNamespace(build=build), + get_metadata_builder=lambda _: SimpleNamespace( + build=build, supports_update_block_table=False + ), layer_names=["draft"], ) ] @@ -183,6 +186,7 @@ def test_speculator_uses_draft_model_hidden_size(monkeypatch, hc_mult, expected) parallel_config=SimpleNamespace( data_parallel_size=1, data_parallel_rank=0, + decode_context_parallel_size=1, ), ) diff --git a/tests/v1/worker/test_gpu_block_table.py b/tests/v1/worker/test_gpu_block_table.py index 5c245d1c4d..5c2231a1f2 100644 --- a/tests/v1/worker/test_gpu_block_table.py +++ b/tests/v1/worker/test_gpu_block_table.py @@ -171,22 +171,23 @@ def test_block_tables_skip_custom_slot_mapping_groups(): @pytest.mark.parametrize("cp_rank", range(4)) def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int): - """DCP interleave is expressed in logical-block token coordinates.""" + """Only sharded groups use DCP interleave in logical-block coordinates.""" device = torch.device("cuda") block_tables = BlockTables( - block_sizes=[128], + block_sizes=[128, 128], max_num_reqs=1, max_num_batched_tokens=1024, - max_num_blocks_per_group=[2], + max_num_blocks_per_group=[2, 8], device=device, - kernel_block_sizes=[64], + kernel_block_sizes=[64, 64], + dcp_sharded=[True, False], cp_size=4, cp_rank=cp_rank, cp_interleave=128, ) block_tables.append_block_ids( req_index=0, - new_block_ids=([5, 9],), + new_block_ids=([5, 9], list(range(10, 18))), overwrite=True, ) block_tables.apply_staged_writes() @@ -199,7 +200,7 @@ def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int): query_start_loc, positions, num_tokens_padded=1024, - )[0] + ) expected = torch.full((1024,), -1, dtype=torch.int64, device=device) first_start = cp_rank * 128 @@ -210,7 +211,8 @@ def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int): expected[second_start : second_start + 128] = torch.arange( 9 * 128, 10 * 128, dtype=torch.int64, device=device ) - assert torch.equal(actual, expected) + assert torch.equal(actual[0], expected) + assert torch.equal(actual[1], positions + 10 * 128) def test_v1_block_table_move_row_clears_vacated_row(): diff --git a/tests/v1/worker/test_gpu_input_batch.py b/tests/v1/worker/test_gpu_input_batch.py index 7c6c9ba417..f37e824e6c 100644 --- a/tests/v1/worker/test_gpu_input_batch.py +++ b/tests/v1/worker/test_gpu_input_batch.py @@ -372,6 +372,62 @@ def test_swap_states_in_input_batch(device: str, batch_size: int, swap_list: lis _compare_objs(input_batch, ref_input_batch) +@pytest.mark.parametrize("device", DEVICES) +def test_condense_clears_stale_allowed_token_ids_mask(device: str): + """condense() must clear the mask row a constrained request is moved out + of. Otherwise a later request that reuses that row without + allowed_token_ids inherits the stale whitelist. + See https://github.com/vllm-project/vllm/issues/43894. + """ + batch_size = 4 + allowed_token_id = 13 + input_batch = InputBatch( + max_num_reqs=batch_size, + max_model_len=1024, + max_num_batched_tokens=1024, + device=torch.device(device), + vocab_size=VOCAB_SIZE, + block_sizes=[1], + kernel_block_sizes=[1], + max_num_blocks_per_req=[1024], + ) + + def _make_req(suffix: int, allowed_token_ids=None) -> CachedRequestState: + return CachedRequestState( + req_id=f"req_id_{suffix}", + prompt_token_ids=[1, 2, 3], + sampling_params=SamplingParams(allowed_token_ids=allowed_token_ids), + pooling_params=None, + mm_features=[], + block_ids=([],), + generator=None, + num_computed_tokens=0, + output_token_ids=[], + ) + + # Only req_id_2 is constrained, and it lands on the highest row. + assert input_batch.add_request(_make_req(0)) == 0 + assert input_batch.add_request(_make_req(1)) == 1 + assert input_batch.add_request(_make_req(2, [allowed_token_id])) == 2 + + mask = input_batch.allowed_token_ids_mask_cpu_tensor + assert mask is not None + # The constrained row masks every token except the single allowed id. + assert not mask[2][allowed_token_id].item() + assert int(mask[2].sum().item()) == VOCAB_SIZE - 1 + + # Free row 0, then condense: req_id_2 slides from row 2 down into row 0. + input_batch.remove_request("req_id_0") + input_batch.condense() + assert input_batch.req_id_to_index["req_id_2"] == 0 + assert int(mask[2].sum().item()) == 0 + + # A new unrestricted request reuses row 2 and must stay unconstrained. + assert input_batch.add_request(_make_req(3)) == 2 + assert "req_id_3" not in input_batch.has_allowed_token_ids + assert int(mask[2].sum().item()) == 0 + + @pytest.mark.parametrize("device", DEVICES) def test_swap_states_preserves_allowed_token_ids_mask(device: str): """swap_states() must exchange the two allowed_token_ids_mask rows. diff --git a/tests/v1/worker/test_gpu_pcp_manager.py b/tests/v1/worker/test_gpu_pcp_manager.py index 0c4b5aff68..4612d49d92 100644 --- a/tests/v1/worker/test_gpu_pcp_manager.py +++ b/tests/v1/worker/test_gpu_pcp_manager.py @@ -75,6 +75,24 @@ def test_validate_config_rejects_full_graph_for_prefills(): ) +@pytest.mark.parametrize("draft_uses_mla", [False, True]) +@pytest.mark.parametrize("dcp_size", [1, 2, 4]) +def test_validate_config_dcp_for_dspark(draft_uses_mla, dcp_size): + config = _make_config(CUDAGraphMode.NONE) + config.parallel_config.prefill_context_parallel_size = 4 + config.parallel_config.decode_context_parallel_size = dcp_size + config.model_config.hf_text_config.index_topk = 1 + config.speculative_config = SimpleNamespace( + use_dspark=lambda: True, + draft_model_config=SimpleNamespace(use_mla=draft_uses_mla), + ) + if draft_uses_mla and dcp_size == 2: + with pytest.raises(NotImplementedError, match="requires DCP=1 or DCP=PCP"): + PCPManager.validate_config(config, supports_mm_inputs=False) + return + PCPManager.validate_config(config, supports_mm_inputs=False) + + def test_replicated_decode_piecewise_graph_padding(monkeypatch): manager = PCPManager( pcp_world_size=2, diff --git a/tools/pre_commit/mypy.py b/tools/pre_commit/mypy.py index f9d064a499..7262c1f2a9 100755 --- a/tools/pre_commit/mypy.py +++ b/tools/pre_commit/mypy.py @@ -113,7 +113,6 @@ SEPARATE_GROUPS = [ EXCLUDE = [ r"vllm/model_executor/models/[kK]", - r"vllm/model_executor/models/[uU]", r"vllm/model_executor/models/[vV]", r"vllm/model_executor/models/[wW]", r"vllm/model_executor/models/[zZ]", diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index af0213361e..0e89b39818 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -1870,28 +1870,39 @@ def _mhc_delayed_pre_tail( _OPS_REGISTERED = False +_AITER_SITUV2_ACT_ENV = { + "a8w4": "AITER_SITUV2_A8W4", + "a4w4": "AITER_SITUV2_A4W4", +} + + +def _resolve_situv2_activation() -> str: + """VLLM_ROCM_USE_AITER_MOE_SITUV2 -> a4w4 | a8w4 | a16w4. + + auto (default) and the legacy 1 mean a4w4; legacy 0 means a16w4. + """ + value = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2.lower() + if value in ("auto", "1"): + return "a4w4" + if value == "0": + return "a16w4" + return value + + def _sync_aiter_situv2_moe_env() -> None: - """Mirror the SiTUv2 MoE toggle into AITER's a4w4 dispatch env. + """Set the AITER_SITUV2_* env matching VLLM_ROCM_USE_AITER_MOE_SITUV2. - AITER selects afp8 vs afp4 activation kernels via AITER_SITUV2_A8W4 / - AITER_SITUV2_A4W4 (see ROCm/aiter fused_moe.py, A8W4 checked first). - When VLLM_ROCM_USE_AITER_MOE_SITUV2 is enabled we route to a4w4 - (afp4_wfp4_fp4 kernels) and clear any legacy AITER_SITUV2_A8W4 override. - - Requires AITER with ROCm/aiter#4463 (first tagged in v0.1.20): a4w4 - dispatch plus kimik3_a4w4_{un,}tuned_fmoe.csv. Older AITER still runs - afp4 FlyDSL but falls back to heuristic configs, which is not the - tuned a4w4 path this recipe is meant to use. + AITER reads AITER_SITUV2_A8W4 / AITER_SITUV2_A4W4 and defaults to a16w4, + so exactly one is set for a8w4/a4w4 and both are cleared for a16w4. """ import os - import vllm.envs as envs - - if envs.VLLM_ROCM_USE_AITER_MOE_SITUV2: - os.environ["AITER_SITUV2_A4W4"] = "1" - os.environ.pop("AITER_SITUV2_A8W4", None) - else: - os.environ.pop("AITER_SITUV2_A4W4", None) + selected = _AITER_SITUV2_ACT_ENV.get(_resolve_situv2_activation()) + for name in _AITER_SITUV2_ACT_ENV.values(): + if name == selected: + os.environ[name] = "1" + else: + os.environ.pop(name, None) class rocm_aiter_ops: @@ -1917,7 +1928,8 @@ class rocm_aiter_ops: VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: Controls FP4 assembly GEMM. VLLM_ROCM_USE_AITER_TRITON_ROPE: Controls Triton rotary embeddings. VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: Controls shared expert fusion. - VLLM_ROCM_USE_AITER_MOE_SITUV2: Controls SiTUv2 FlyDSL MoE (a4w4). + VLLM_ROCM_USE_AITER_MOE_SITUV2: SiTUv2 FlyDSL MoE activation + dtype (a16w4 | a8w4 | a4w4). VLLM_ROCM_USE_AITER_TRITON_GEMM: Controls Triton unquantized GEMM. Note: @@ -1987,7 +1999,7 @@ class rocm_aiter_ops: # TODO: Consolidate under VLLM_ROCM_USE_AITER_ROPE _TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE _MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS - _MOE_SITUV2 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2 + _MOE_SITUV2 = _resolve_situv2_activation() # TODO: Consolidate under _LINEAR_ENABLED _TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM # Lazily probed: whether aiter.topk_softmax supports the @@ -2016,7 +2028,7 @@ class rocm_aiter_ops: cls._FP4_GEMM_DYNAMIC_QUANT_ASM = envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM cls._TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE cls._MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS - cls._MOE_SITUV2 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2 + cls._MOE_SITUV2 = _resolve_situv2_activation() _sync_aiter_situv2_moe_env() cls._TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM cls._MOE_DISPATCH_POLICY = envs.VLLM_ROCM_AITER_MOE_DISPATCH_POLICY @@ -2132,9 +2144,25 @@ class rocm_aiter_ops: @classmethod @if_aiter_supported def is_fused_moe_situv2_enabled(cls) -> bool: - # _MOE_SITUV2 is a variant of aiter fused moe, so aiter - # fused moe must be enabled as well. - return cls.is_fused_moe_enabled() and cls._MOE_SITUV2 + """True when a low-precision (a8w4/a4w4) SiTUv2 activation is selected.""" + return cls.is_fused_moe_enabled() and cls._MOE_SITUV2 != "a16w4" + + @classmethod + @if_aiter_supported + def get_fused_moe_situv2_activation(cls) -> str: + """SiTUv2 activation dtype name: a16w4, a8w4 or a4w4.""" + return cls._MOE_SITUV2 + + @classmethod + @if_aiter_supported + def is_fused_moe_situv2_gate_up_interleaved(cls) -> bool: + """Whether the SiTUv2 kernels expect gate/up-interleaved w13. + + The a8w4 FlyDSL stage-1 kernels (_gui_) are interleaved; a16w4 and + a4w4 are separated. Weight shuffle (oracle/mxfp4.py) and gate_mode + (experts/rocm_aiter_moe.py) must both use this. + """ + return cls._MOE_SITUV2 == "a8w4" @classmethod @if_aiter_supported diff --git a/vllm/benchmarks/datasets/datasets.py b/vllm/benchmarks/datasets/datasets.py index 8f9e87cf83..76de75a66f 100644 --- a/vllm/benchmarks/datasets/datasets.py +++ b/vllm/benchmarks/datasets/datasets.py @@ -2148,7 +2148,9 @@ def get_samples( dataset_path=args.dataset_path, disable_shuffle=args.disable_shuffle ) # For the "sonnet" dataset, formatting depends on the backend. - if args.backend == "openai-chat": + # Chat-style backends leave templating to the server; completions + # backends need the prompt rendered client side. + if args.backend in ("openai-chat", "openai-responses"): input_requests = sonnet_dataset.sample( num_requests=args.num_prompts, input_len=args.sonnet_input_len, diff --git a/vllm/benchmarks/lib/endpoint_request_func.py b/vllm/benchmarks/lib/endpoint_request_func.py index 2b4027b502..14ff6c1ab1 100644 --- a/vllm/benchmarks/lib/endpoint_request_func.py +++ b/vllm/benchmarks/lib/endpoint_request_func.py @@ -447,6 +447,178 @@ async def async_request_openai_chat_completions( return output +def _extract_sse_data(message: str) -> str | None: + """Extract the ``data`` payload of a single SSE message. + + The Responses API stream labels every message with an ``event:`` line + before its ``data:`` line, so the payload cannot be recovered by + stripping a leading ``data: `` the way the Completions and Chat + Completions streams allow. + + Args: + message: One complete SSE message, without its trailing blank line. + + Returns: + The payload, with multi-line payloads joined by newlines as required + by the SSE spec, or None if the message carries no ``data:`` line. + + """ + # NOTE: split on "\n" rather than using splitlines(), which also breaks on + # U+2028, U+2029 and U+0085. Those are legal inside a JSON string and the + # server emits them unescaped, so splitlines() would truncate the payload. + data_lines = [ + line.removeprefix("data:").removeprefix(" ") + for line in message.split("\n") + if line.startswith("data:") + ] + if not data_lines: + return None + + return "\n".join(data_lines) + + +# Responses events that carry generated tokens. Reasoning deltas are included +# because the server is already decoding when it emits them, so excluding them +# would report the whole reasoning phase as time to first token. +_RESPONSES_TOKEN_DELTA_EVENTS = frozenset( + { + "response.output_text.delta", + "response.reasoning_text.delta", + } +) + +# Responses events that end a request. Truncation at the output length limit +# is a normal benchmark outcome: vLLM reports it as `response.completed` with +# `status: "incomplete"`, while the spec also defines a dedicated +# `response.incomplete` terminal event. +_RESPONSES_TERMINAL_EVENTS = frozenset( + { + "response.completed", + "response.incomplete", + } +) + +# The spec's failure terminals. vLLM does not emit them today, but +# `vllm bench serve` can be pointed at other Responses servers. +_RESPONSES_ERROR_EVENTS = frozenset({"error", "response.failed"}) + + +async def async_request_openai_responses( + request_func_input: RequestFuncInput, + session: aiohttp.ClientSession, + pbar: tqdm | None = None, +) -> RequestFuncOutput: + api_url = request_func_input.api_url + _validate_api_url(api_url, "OpenAI Responses API", "responses") + + payload = { + "model": request_func_input.model_name + if request_func_input.model_name + else request_func_input.model, + "input": request_func_input.prompt, + "max_output_tokens": request_func_input.output_len, + "stream": True, + # `store` defaults to true. A server started with + # VLLM_ENABLE_RESPONSES_API_STORE=1 keeps every stored response for + # the lifetime of the process, which would grow its heap over a long + # benchmark run. + "store": False, + } + _update_payload_common(payload, request_func_input) + + headers = _get_headers("application/json") + _update_headers_common(headers, request_func_input) + + output = RequestFuncOutput() + output.prompt_len = request_func_input.prompt_len + + # Only output text is accumulated. Reasoning text is timed but not + # returned, matching how `openai-chat` handles `DeltaMessage.reasoning`. + generated_text = "" + stream_error: str | None = None + saw_terminal_event = False + first_chunk_received = False + st = time.perf_counter() + output.start_time = st + most_recent_timestamp = st + try: + async with session.post(url=api_url, json=payload, headers=headers) as response: + if response.status == 200: + handler = StreamedResponseHandler() + + async for chunk_bytes in response.content.iter_any(): + for message in handler.add_chunk(chunk_bytes): + # NOTE: `_extract_sse_data` returns None for SSE + # comments, which servers may send as keep-alives. + chunk = _extract_sse_data(message) + if not chunk or chunk == "[DONE]": + continue + + timestamp = time.perf_counter() + data = json.loads(chunk) + event_type = data.get("type") + + if event_type in _RESPONSES_TOKEN_DELTA_EVENTS: + # First token + if not first_chunk_received: + first_chunk_received = True + output.ttft = timestamp - st + + # Decoding phase + else: + output.itl.append(timestamp - most_recent_timestamp) + + if event_type == "response.output_text.delta": + generated_text += data.get("delta") or "" + + # Only token events advance the end of the + # request, so `latency - ttft` stays equal to + # `sum(itl)` and TPOT means what serve.py says. + most_recent_timestamp = timestamp + elif event_type in _RESPONSES_TERMINAL_EVENTS: + saw_terminal_event = True + usage = (data.get("response") or {}).get("usage") or {} + if (n_output := usage.get("output_tokens")) is not None: + output.output_tokens = n_output + if (n_input := usage.get("input_tokens")) is not None: + output.prompt_len = n_input + elif event_type in _RESPONSES_ERROR_EVENTS or data.get("error"): + stream_error = json.dumps(data.get("error") or data) + + output.generated_text = generated_text + output.latency = most_recent_timestamp - st + if stream_error is not None: + output.success = False + output.error = stream_error + elif not first_chunk_received: + output.success = False + output.error = ( + "Never received a token delta to calculate TTFT." + "This response will be marked as failed!" + ) + elif not saw_terminal_event: + # Without the terminal event the response is truncated and + # its usage block is missing. + output.success = False + output.error = ( + "Stream ended without a terminal event." + "This response will be marked as failed!" + ) + else: + output.success = True + else: + output.error = response.reason or "" + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + if pbar: + pbar.update(1) + return output + + async def async_request_openai_audio( request_func_input: RequestFuncInput, session: aiohttp.ClientSession, @@ -920,6 +1092,7 @@ ASYNC_REQUEST_FUNCS: dict[str, RequestFunc] = { "vllm": async_request_openai_completions, "openai": async_request_openai_completions, "openai-chat": async_request_openai_chat_completions, + "openai-responses": async_request_openai_responses, "openai-audio": async_request_openai_audio, "openai-embeddings": async_request_openai_embeddings, "openai-embeddings-chat": async_request_openai_embeddings_chat, @@ -947,5 +1120,10 @@ POOLING_BACKENDS = { OPENAI_COMPATIBLE_BACKENDS = [ k for k, v in ASYNC_REQUEST_FUNCS.items() - if v in (async_request_openai_completions, async_request_openai_chat_completions) + if v + in ( + async_request_openai_completions, + async_request_openai_chat_completions, + async_request_openai_responses, + ) ] diff --git a/vllm/benchmarks/serve.py b/vllm/benchmarks/serve.py index d751f7f22d..19130716f4 100644 --- a/vllm/benchmarks/serve.py +++ b/vllm/benchmarks/serve.py @@ -2216,6 +2216,11 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]: "Sampling parameters are only supported by openai-compatible backends." ) + # The Responses API accepts every sampling parameter above except + # min_p, which it would silently drop as an unknown field. + if args.backend == "openai-responses" and "min_p" in sampling_params: + raise ValueError("--min-p is not supported by the Responses API.") + if "temperature" not in sampling_params: print( "WARNING: vllm bench serve no longer sets temperature==0 (greedy) " diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index c50edf4363..220da0c72b 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -1718,6 +1718,26 @@ class VllmConfig: ) self.compilation_config.mode = CompilationMode.NONE + # TODO: This is a stopgap that turns sequence parallelism / async TP + # off under batch invariance. Make the sequence-parallel reduce-scatter + # path batch-invariant instead so they can stay enabled (#56370). + # Sequence parallelism / async TP rewrite all_reduce + rms_norm into a + # reduce-scatter whose reduction order depends on the batch, so they + # are not batch-invariant. Decide this before the breakable-CUDA-graph + # default below, which declines to auto-enable when they are on. + pass_config = self.compilation_config.pass_config + if envs.VLLM_BATCH_INVARIANT and ( + pass_config.enable_sp or pass_config.fuse_gemm_comms + ): + logger.warning_once( + "Disabling sequence parallelism and async TP " + "(pass_config.enable_sp / fuse_gemm_comms) when " + "VLLM_BATCH_INVARIANT is enabled: the reduce-scatter path " + "is not batch-invariant (see vllm-project/vllm#56370)." + ) + pass_config.enable_sp = False + pass_config.fuse_gemm_comms = False + breakable_cudagraph_enabled = self._maybe_enable_breakable_cudagraph() if not breakable_cudagraph_enabled and ( diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/base.py b/vllm/distributed/kv_transfer/kv_connector/v1/base.py index bc3c32ba4c..6df9d1cfbe 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/base.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/base.py @@ -42,6 +42,7 @@ The class provides the following primitives: import enum from abc import ABC, abstractmethod from collections.abc import Callable, Iterable +from contextlib import AbstractContextManager from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Literal @@ -227,6 +228,13 @@ class KVConnectorBase_V1(ABC): # Worker-side methods # ============================== + def get_mem_pool_context(self) -> AbstractContextManager | None: + """Return a custom KV cache allocation context, if configured. + + Returning None uses the engine's default memory pool. + """ + return None + def bind_connector_metadata(self, connector_metadata: KVConnectorMetadata) -> None: """Set the connector metadata from the scheduler. diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py index a9255f7cda..cbb7339c91 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py @@ -59,7 +59,6 @@ from vllm.platforms import current_platform from vllm.utils.math_utils import cdiv from vllm.v1.attention.backend import AttentionMetadata from vllm.v1.core.kv_cache_utils import ( - dcp_world_size_for_kv_cache_spec, resolve_dcp_kv_block_size, ) from vllm.v1.kv_cache_interface import ( @@ -259,13 +258,7 @@ class DecodeBenchConnectorScheduler: dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size self.kv_cache_groups = kv_cache_config.kv_cache_groups self.group_block_sizes = tuple( - resolve_dcp_kv_block_size( - group.kv_cache_spec, - dcp_world_size_for_kv_cache_spec( - group.kv_cache_spec, - dcp_world_size, - ), - ) + resolve_dcp_kv_block_size(group.kv_cache_spec, dcp_world_size) for group in self.kv_cache_groups ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py index beabf157cc..5ef4f8f670 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py @@ -10,7 +10,7 @@ from collections.abc import Collection from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from enum import IntEnum -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Final import msgspec import numpy as np @@ -70,6 +70,8 @@ from vllm.v1.worker.utils import select_common_block_size logger = init_logger(__name__) +_BOOTSTRAP_MAX_ATTEMPTS: Final[int] = 3 + try: from mooncake.engine import TransferEngine except ImportError: @@ -1147,24 +1149,44 @@ class MooncakeConnectorWorker: pp_rank=self.pp_rank, addr=worker_addr, ) - while True: + timeout = envs.VLLM_MOONCAKE_CONNECTOR_TIMEOUT + max_attempts = _BOOTSTRAP_MAX_ATTEMPTS + backoff = 1.0 + for attempt in range(1, max_attempts + 1): try: - async with httpx.AsyncClient() as client: + async with httpx.AsyncClient(timeout=timeout) as client: response = await client.post(url, json=payload.model_dump()) response.raise_for_status() logger.debug("Successfully registered with bootstrap server at %s", url) - break - except httpx.ConnectError: - # Bootstrap server not ready, wait for a while and retry. - await asyncio.sleep(1) + return + except (httpx.ConnectError, httpx.TimeoutException) as e: + # Rank 0 may be busy registering a large memory segment. + if attempt == max_attempts: + break + logger.warning( + "Bootstrap registration attempt %d/%d for %s failed with %s: %s. " + "Retrying in %.1fs.", + attempt, + max_attempts, + payload, + type(e).__name__, + e, + backoff, + ) + await asyncio.sleep(backoff) + backoff = min(backoff * 2, 10.0) except Exception as e: err_msg = ( e.response.text if isinstance(e, httpx.HTTPStatusError) else str(e) ) - logger.error( + logger.exception( "Error registering %s with bootstrap server: %s", payload, err_msg ) - raise e + raise + raise RuntimeError( + f"Failed to register {payload} with the Mooncake bootstrap server at " + f"{url} after {max_attempts} attempts." + ) async def _mooncake_sender_listener(self, ready_event: threading.Event): """Background thread that listens for Mooncake requests, dispatches them @@ -1800,11 +1822,29 @@ class MooncakeConnectorWorker: if self.is_kv_consumer: return + # httpx applies the timeout to each phase (connect, read, write, pool) + # rather than to the whole request, so budget for all four. + register_timeout = envs.VLLM_MOONCAKE_CONNECTOR_TIMEOUT + max_attempts = _BOOTSTRAP_MAX_ATTEMPTS + backoff_total = sum(min(2.0**i, 10.0) for i in range(max_attempts - 1)) + ready_timeout = max_attempts * 4 * register_timeout + backoff_total + 30.0 ready_event = threading.Event() - asyncio.run_coroutine_threadsafe( + fut = asyncio.run_coroutine_threadsafe( self._mooncake_sender_listener(ready_event), self.sender_loop ) - ready_event.wait() # Wait for listener ZMQ socket to be ready. + deadline = time.monotonic() + ready_timeout + while not ready_event.wait(timeout=1.0): + if fut.done(): + fut.result() + raise RuntimeError( + "Mooncake sender listener exited before becoming ready." + ) + if time.monotonic() > deadline: + fut.cancel() + raise RuntimeError( + "Mooncake sender listener did not become ready within " + f"{ready_timeout:.0f}s." + ) async def fetch_finished_recving_reqs(self) -> set[ReqId]: finished_recving_reqs = self.finished_recving_reqs diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_utils.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_utils.py index df47f9fdb7..dfe0641b36 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_utils.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_utils.py @@ -97,14 +97,19 @@ class MooncakeBootstrapServer: dp_entry.worker_addr[payload.tp_rank] = {} tp_entry = dp_entry.worker_addr[payload.tp_rank] - if payload.pp_rank in tp_entry: + existing = tp_entry.get(payload.pp_rank) + if existing is not None: + # A client timeout can fire after the server recorded the + # registration so an identical retry must not be an error. + if existing == payload.addr: + return {"status": "ok"} raise HTTPException( status_code=400, detail=( f"Worker with dp_rank={payload.dp_rank}, " f"tp_rank={payload.tp_rank}, pp_rank={payload.pp_rank} " f"is already registered at " - f"{tp_entry[payload.pp_rank]}, " + f"{existing}, " f"but still want to register at {payload.addr}" ), ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py index eb4a7fafe9..dd64dd671e 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py @@ -13,6 +13,7 @@ enabling prefix caching via hash-based deduplication. from collections import Counter from collections.abc import Iterable, Sequence +from contextlib import AbstractContextManager from typing import Any import torch @@ -322,6 +323,17 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA): # Worker-side methods # ============================================================ + def get_mem_pool_context(self) -> AbstractContextManager | None: + """Return a context manager for the custom MemPool, or None. + + Called by the Worker before ``initialize_kv_cache`` so that KV + cache is allocated from the Mooncake-managed pool when + ``custom_mem_pool`` is set in ``kv_connector_extra_config``. + """ + if self.connector_worker is None: + return None + return self.connector_worker.get_mem_pool_context() + def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]): assert self.connector_worker is not None self.connector_worker.register_kv_caches(kv_caches) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py index baf8967bc8..173aa324e7 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py @@ -19,13 +19,14 @@ import threading import time from collections.abc import Callable, Sequence from concurrent.futures import Future, ThreadPoolExecutor +from contextlib import AbstractContextManager from typing import Any, TypeVar import torch import zmq import vllm.envs as envs -from vllm.config import VllmConfig +from vllm.config import ModelConfig, VllmConfig from vllm.distributed import ( get_dcp_group, get_pcp_group, @@ -1414,6 +1415,41 @@ class KVCacheStoreRecvingThread(KVTransferThread): class MooncakeStoreWorker: """Worker-side component for MooncakeStoreConnector.""" + @staticmethod + def _create_mem_pool( + extra_config: dict[str, Any], model_config: ModelConfig + ) -> torch.cuda.MemPool | None: + pool_type = str(extra_config.get("custom_mem_pool") or "").upper() + if not pool_type: + return None + if pool_type not in ("NVLINK", "BAREX"): + raise ValueError( + f"Unsupported custom_mem_pool={pool_type!r}, " + "expected 'NVLINK' or 'BAREX'" + ) + if model_config.enable_sleep_mode or model_config.enable_cumem_allocator: + raise ValueError( + "custom_mem_pool is incompatible with enable_sleep_mode " + "or enable_cumem_allocator; CuMemAllocator cannot manage " + "allocations from the custom pool." + ) + try: + if pool_type == "NVLINK": + from mooncake.allocator import NVLinkAllocator as allocator_cls + else: # pool_type == "BAREX" + from mooncake.allocator import BarexAllocator as allocator_cls + except ImportError as e: + raise ImportError( + f"custom_mem_pool={pool_type!r} requires " + "mooncake-transfer-engine>=0.3.8. Please upgrade Mooncake." + ) from e + + device = torch.device("cuda", torch.accelerator.current_device_index()) + allocator = allocator_cls.get_allocator(device) + mem_pool = torch.cuda.MemPool(allocator.allocator()) + logger.info("Using Mooncake custom memory pool: %s", pool_type) + return mem_pool + def __init__( self, vllm_config: VllmConfig, @@ -1471,6 +1507,10 @@ class MooncakeStoreWorker: # Initialize MooncakeDistributedStore with its own TransferEngine store_config = MooncakeStoreConfig.load_from_config() + # Validate and create the custom memory pool before initializing the + # store so failures cannot leak a partially initialized store handle. + self._mem_pool = self._create_mem_pool(extra_config, model_config) + self.store = MooncakeDistributedStore() local_ip = get_ip() local_hostname = rdma_utils.get_requester_local_hostname(local_ip) @@ -1741,6 +1781,13 @@ class MooncakeStoreWorker: ) return token_dbs + def get_mem_pool_context(self) -> AbstractContextManager | None: + """Return a context manager for the custom MemPool, or None if + no custom pool is configured.""" + if self._mem_pool is None: + return None + return torch.cuda.use_mem_pool(self._mem_pool) + def _spec_tp_replication_factor(self, spec: KVCacheSpec) -> int: if self.dcp_size > 1: return 1 diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py index a6f0d4cafb..73b6b0ccf6 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import copy from collections.abc import Callable, Iterable +from contextlib import AbstractContextManager from dataclasses import dataclass from typing import TYPE_CHECKING, Any, cast @@ -294,6 +295,26 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA): # ============================== # Worker-side methods # ============================== + def get_mem_pool_context(self) -> AbstractContextManager | None: + """Forward a custom KV cache memory pool from a child connector. + + KV cache is allocated once and can only live in one pool, so at + most one child may provide a context. + """ + found: list[tuple[str, AbstractContextManager]] = [] + for connector in self._connectors: + if (ctx := connector.get_mem_pool_context()) is not None: + found.append((type(connector).__name__, ctx)) + + if len(found) > 1: + names = [name for name, _ in found] + raise ValueError( + f"Multiple connectors provide a KV cache memory pool {names}; " + "KV cache is allocated once and can only live in one pool. " + "Configure custom_mem_pool on at most one connector." + ) + return found[0][1] if found else None + def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None: for c in self._connectors: c.start_load_kv(forward_context, **kwargs) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py index b630dedfc3..f4a643d2b7 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py @@ -839,6 +839,8 @@ class NixlBaseConnectorWorker: self._engine_ttl: float = vllm_config.kv_transfer_config.get_from_extra_config( "engine_ttl", 3600.0 ) + # Map of pull peer (host, port) -> engine_id currently serving it. + self._engine_by_address: dict[tuple[str, int], EngineId] = {} self.model_config = vllm_config.model_config @@ -1300,6 +1302,7 @@ class NixlBaseConnectorWorker: self._remote_agents[eid] = remote_agents self._engine_clock_offset[eid] = clock_offset self._engine_last_active[eid] = time.perf_counter() + self._track_remote_engine_replacement(eid, host, port) except Exception as e: self._log_failure( failure_type="handshake_setup_failed", @@ -2953,6 +2956,7 @@ class NixlBaseConnectorWorker: # Handle timeout to avoid stranding blocks on remote. self._reap_expired_send_leases(done_sending) + self._cleanup_replaced_remote_engines() return KVConnectorTransferResults( finished_sending=done_sending, finished_recving=done_recving, @@ -3541,14 +3545,53 @@ class NixlBaseConnectorWorker: and meta.remote is not None } + def _track_remote_engine_replacement( + self, engine_id: EngineId, host: str, port: int + ) -> None: + """Record the engine currently serving a pull peer address.""" + # TODO: Also handle push mode, which handshakes in both directions. + if self._TRANSFER_MODE != "pull": + return + previous = self._engine_by_address.get((host, port)) + self._engine_by_address[(host, port)] = engine_id + if previous is None or previous == engine_id: + return + logger.info( + "Remote engine at %s:%d changed from %s to %s.", + host, + port, + previous, + engine_id, + ) + + def _cleanup_replaced_remote_engines(self) -> None: + """Release replaced pull peers once requests and handshakes have drained.""" + if self._TRANSFER_MODE != "pull": + return + with self._handshake_lock: + if self._handshake_futures: + return + replaced = self._remote_agents.keys() - self._engine_by_address.values() + if not replaced: + return + # _recving_metadata is only accessed from this (main) thread. + busy = { + meta.remote.engine_id + for meta in self._recving_metadata.values() + if meta.remote is not None + } + for engine_id in replaced - busy: + self._cleanup_remote_engine(engine_id, log_eviction=False) + logger.info("Released NIXL state for replaced remote engine %s.", engine_id) + def _cleanup_remote_engine( self, engine_id: EngineId, *, log_eviction: bool = True ) -> None: """Remove all state for a single remote engine. Releases NIXL resources (dlist handles, remote agents) and clears - all per-engine data structures. Used by both TTL eviction and - shutdown. + all per-engine data structures. Used by TTL eviction, replaced peer + cleanup and shutdown. """ assert engine_id in self._remote_agents @@ -3558,6 +3601,9 @@ class NixlBaseConnectorWorker: # Pop under the handshake lock; NIXL teardown stays outside it. with self._handshake_lock: agents = self._remote_agents.pop(engine_id) + for address, eid in list(self._engine_by_address.items()): + if eid == engine_id: + del self._engine_by_address[address] for agent_name in agents.values(): self.nixl_wrapper.remove_remote_agent(agent_name) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py index 6539b7e57e..11b7215950 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py @@ -84,7 +84,14 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): # Start transfers for requests whose handshakes have now finished. while not self._ready_requests.empty(): - self._read_blocks_for_req(*self._ready_requests.get_nowait()) + req_id, meta = self._ready_requests.get_nowait() + assert meta.remote is not None + if meta.remote.engine_id not in self._remote_agents: + # The engine was released after its handshake completed, so + # handshake again. This fails if the engine is gone. + self._background_nixl_handshake(req_id, meta.remote.engine_id, meta) + continue + self._read_blocks_for_req(req_id, meta) if self.pcp_rank > 0 and not self.pcp_dcp_sharded: # Replicated-KV PCP: only PCP rank 0 serves the KV, so this rank diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py index 64bf8146c9..9124d9eac1 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py @@ -15,6 +15,12 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import ( KVConnectorRole, SupportsHMA, ) +from vllm.distributed.kv_transfer.kv_connector.v1.metrics import ( + KVConnectorPromMetrics, + KVConnectorStats, + PromMetric, + PromMetricT, +) from vllm.logger import init_logger from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.outputs import KVConnectorOutput @@ -25,6 +31,10 @@ from vllm.v1.simple_kv_offload.manager import ( from vllm.v1.simple_kv_offload.metadata import ( SimpleCPUOffloadMetadata, ) +from vllm.v1.simple_kv_offload.metrics import ( + SimpleCPUOffloadPromMetrics, + SimpleCPUOffloadStats, +) from vllm.v1.simple_kv_offload.worker import ( SimpleCPUOffloadWorker, ) @@ -162,6 +172,7 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA): hash_block_size=hash_block_size, lazy_offload=lazy_offload, disk_capacity_bytes=disk_capacity_bytes if disk_mode else 0, + use_page_cache=use_page_cache if disk_mode else False, ) elif role == KVConnectorRole.WORKER: self.worker_handler = SimpleCPUOffloadWorker( @@ -307,6 +318,38 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA): return self.scheduler_manager.get_boundary_store_stats() return None + def get_kv_connector_stats(self) -> KVConnectorStats | None: + # Worker-side callers may invoke this hook on connectors constructed + # via __new__ (as in test_worker.py), which lack scheduler_manager. + scheduler_manager: SimpleCPUOffloadScheduler | None = getattr( + self, "scheduler_manager", None + ) + if scheduler_manager is not None: + return scheduler_manager.get_stats() + return None + + @classmethod + def build_kv_connector_stats( + cls, data: dict[str, Any] | None = None + ) -> KVConnectorStats | None: + return ( + SimpleCPUOffloadStats(data=data) + if data is not None + else SimpleCPUOffloadStats() + ) + + @classmethod + def build_prom_metrics( + cls, + vllm_config: VllmConfig, + metric_types: dict[type[PromMetric], type[PromMetricT]], + labelnames: list[str], + per_engine_labelvalues: dict[int, list[object]], + ) -> KVConnectorPromMetrics: + return SimpleCPUOffloadPromMetrics( + vllm_config, metric_types, labelnames, per_engine_labelvalues + ) + def take_events(self) -> Iterable[KVCacheEvent]: if self.scheduler_manager is not None: return self.scheduler_manager.take_events() diff --git a/vllm/distributed/parallel_state.py b/vllm/distributed/parallel_state.py index 20118649e1..88bb3d7c5a 100644 --- a/vllm/distributed/parallel_state.py +++ b/vllm/distributed/parallel_state.py @@ -1583,6 +1583,14 @@ def get_dcp_group() -> GroupCoordinator: return _DCP +def get_dcp_world_size_and_rank(enabled: bool = True) -> tuple[int, int]: + """Return ``(world_size, rank)`` in the DCP group, or ``(1, 0)`` when disabled + (e.g. a replicated draft cache) or the group is uninitialized (unit tests).""" + if not enabled or _DCP is None: + return 1, 0 + return _DCP.world_size, _DCP.rank_in_group + + _PP: GroupCoordinator | None = None diff --git a/vllm/entrypoints/cli/preload.py b/vllm/entrypoints/cli/preload.py index d312b6c36a..50734a052b 100644 --- a/vllm/entrypoints/cli/preload.py +++ b/vllm/entrypoints/cli/preload.py @@ -23,8 +23,14 @@ import multiprocessing import queue import signal import sys +import threading import typing +from collections.abc import Sequence from itertools import product +from multiprocessing.process import BaseProcess + +from fastapi import FastAPI +from fastapi.responses import Response from vllm.engine.arg_utils import EngineArgs from vllm.entrypoints.cli.types import CLISubcommand @@ -44,6 +50,55 @@ else: logger = init_logger(__name__) +class _HealthState: + def __init__(self, processes: Sequence[BaseProcess]) -> None: + self.processes = processes + self._ready = False + self._lock = threading.Lock() + + def mark_ready(self) -> None: + with self._lock: + self._ready = True + + @property + def is_ready(self) -> bool: + with self._lock: + ready = self._ready + return ready and not any(proc.exitcode is not None for proc in self.processes) + + +def _build_health_app(state: _HealthState) -> FastAPI: + app = FastAPI(openapi_url=None, docs_url=None, redoc_url=None) + + @app.get("/health", include_in_schema=False) + async def health() -> Response: + return Response(status_code=200 if state.is_ready else 503) + + return app + + +def _start_health_server( + host: str, port: int, state: _HealthState +) -> tuple[typing.Any, threading.Thread]: + import uvicorn + + config = uvicorn.Config( + _build_health_app(state), + host=host, + port=port, + log_level="error", + access_log=False, + ) + server = uvicorn.Server(config) + thread = threading.Thread( + target=server.run, + name="vllm-weight-cache-health", + daemon=True, + ) + thread.start() + return server, thread + + class PreloadSubcommand(CLISubcommand): """The `preload` subcommand for the vLLM CLI.""" @@ -75,6 +130,18 @@ class PreloadSubcommand(CLISubcommand): "--weight-cache-master-port + 1 for multi-node, or a free port " "for single-node.", ) + parser.add_argument( + "--weight-cache-health-port", + type=int, + default=None, + help="Port for the parent /health endpoint; disabled by default.", + ) + parser.add_argument( + "--weight-cache-health-host", + type=str, + default="0.0.0.0", + help="Host for the parent /health endpoint (default: 0.0.0.0).", + ) @staticmethod def cmd(args: argparse.Namespace) -> None: @@ -190,10 +257,21 @@ class PreloadSubcommand(CLISubcommand): tp_rank, ) in product(groups, placements) ] + health_state = _HealthState(procs) + health_server = None + health_thread = None + if args.weight_cache_health_port is not None: + health_server, health_thread = _start_health_server( + args.weight_cache_health_host, + args.weight_cache_health_port, + health_state, + ) for proc in procs: proc.start() def _shutdown(signum, frame): + if health_server is not None: + health_server.should_exit = True for proc in procs: proc.terminate() @@ -216,7 +294,12 @@ class PreloadSubcommand(CLISubcommand): proc.terminate() for proc in procs: proc.join() + if health_server is not None: + health_server.should_exit = True + if health_thread is not None: + health_thread.join() sys.exit(max((p.exitcode or 0) for p in procs)) + health_state.mark_ready() socket_dir_msg = args.weight_cache_socket_dir or "the default socket dir" logger.info_once( "===== Weight cache daemon READY: node %d/%d serving %d local " @@ -230,6 +313,10 @@ class PreloadSubcommand(CLISubcommand): for proc in procs: proc.join() + if health_server is not None: + health_server.should_exit = True + if health_thread is not None: + health_thread.join() sys.exit(max(proc.exitcode or 0 for proc in procs)) def subparser_init( diff --git a/vllm/entrypoints/openai/chat_completion/batch_serving.py b/vllm/entrypoints/openai/chat_completion/batch_serving.py index ea6601bff1..3fde958a3d 100644 --- a/vllm/entrypoints/openai/chat_completion/batch_serving.py +++ b/vllm/entrypoints/openai/chat_completion/batch_serving.py @@ -12,6 +12,7 @@ from vllm.entrypoints.chat_utils import ConversationMessage from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata from vllm.entrypoints.openai.chat_completion.protocol import ( BatchChatCompletionRequest, + ChatCompletionRequest, ChatCompletionResponse, ChatCompletionResponseChoice, ChatMessage, @@ -40,7 +41,14 @@ class OpenAIServingChatBatch(OpenAIServingChat): async def render_batch_chat_request( self, request: BatchChatCompletionRequest, - ) -> tuple[list[list[ConversationMessage]], list[EngineInput]] | ErrorResponse: + ) -> ( + tuple[ + list[ChatCompletionRequest], + list[list[ConversationMessage]], + list[EngineInput], + ] + | ErrorResponse + ): """Validate the model and preprocess a batched chat completion request. Performs engine-aware checks then delegates per-conversation @@ -48,8 +56,10 @@ class OpenAIServingChatBatch(OpenAIServingChat): once for the whole batch. Returns: - A tuple of (all_conversations, engine_prompts) on success — one - entry per conversation — or an ErrorResponse on failure. + A tuple of (single_requests, all_conversations, engine_prompts) on + success — one entry per conversation — or an ErrorResponse on + failure. The renderer adjusts ``single_requests`` in place, so + sampling must read them rather than rebuilding from ``request``. """ error_check_ret = await self._check_model(request) @@ -74,12 +84,14 @@ class OpenAIServingChatBatch(OpenAIServingChat): parser = renderer.parser tool_dicts: list[dict] | None = None + single_requests: list[ChatCompletionRequest] = [] all_conversations: list[list[ConversationMessage]] = [] all_engine_prompts: list[EngineInput] = [] for messages in request.messages: single_request = request.to_chat_completion_request(messages) if renderer.use_harmony: + renderer.adjust_harmony_request(single_request) conversation, engine_prompts = renderer._make_request_with_harmony( single_request, should_include_tools=tool_dicts is not None ) @@ -93,10 +105,11 @@ class OpenAIServingChatBatch(OpenAIServingChat): tool_dicts=tool_dicts, parser=parser, ) + single_requests.append(single_request) all_conversations.append(conversation) all_engine_prompts.append(engine_prompts[0]) - return all_conversations, all_engine_prompts + return single_requests, all_conversations, all_engine_prompts async def create_batch_chat_completion( self, @@ -111,10 +124,10 @@ class OpenAIServingChatBatch(OpenAIServingChat): """ tokenizer = self.renderer.tokenizer assert tokenizer is not None - single_requests = [ - request.to_chat_completion_request(messages) - for messages in request.messages - ] + render_result = await self.render_batch_chat_request(request) + if isinstance(render_result, ErrorResponse): + return render_result + single_requests, all_conversations, engine_prompts = render_result parser: Parser | None = None if self.parser_cls is not None: @@ -127,11 +140,6 @@ class OpenAIServingChatBatch(OpenAIServingChat): chat_template_kwargs=chat_template_kwargs, ) - render_result = await self.render_batch_chat_request(request) - if isinstance(render_result, ErrorResponse): - return render_result - all_conversations, engine_prompts = render_result - request_id = ( f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}" ) diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index be35415538..f22fed1e75 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -200,6 +200,20 @@ class OpenAIServingChat(GenerateBaseServing): .chat_template_kwargs ) + def _make_parser( + self, + request: ChatCompletionRequest, + tokenizer: TokenizerLike, + chat_template_kwargs: dict[str, Any] | None, + ) -> Parser: + assert self.parser_cls is not None + return self.parser_cls( + tokenizer, + request.tools, + chat_template_kwargs=chat_template_kwargs, + model_config=self.model_config, + ) + def _engine_chat_template_kwargs( self, chat_template_kwargs: dict[str, Any] ) -> dict[str, Any]: @@ -267,12 +281,7 @@ class OpenAIServingChat(GenerateBaseServing): chat_template_kwargs = self._effective_chat_template_kwargs(request) parser: Parser | None = None if self.parser_cls is not None: - parser = self.parser_cls( - tokenizer, - request.tools, - chat_template_kwargs=chat_template_kwargs, - model_config=self.model_config, - ) + parser = self._make_parser(request, tokenizer, chat_template_kwargs) result = await self.render_chat_request(request) if isinstance(result, ErrorResponse): return result @@ -490,12 +499,7 @@ class OpenAIServingChat(GenerateBaseServing): "Tokenizer not available when `skip_tokenizer_init=True`" ) parsers: list[Parser | None] = [ - self.parser_cls( - tokenizer, - request.tools, - chat_template_kwargs=chat_template_kwargs, - model_config=self.model_config, - ) + self._make_parser(request, tokenizer, chat_template_kwargs) for _ in range(num_choices) ] else: @@ -961,10 +965,15 @@ class OpenAIServingChat(GenerateBaseServing): tool_parser_cls = ( self.parser_cls.tool_parser_cls if self.parser_cls is not None else None ) - for output in final_res.outputs: + for i, output in enumerate(final_res.outputs): # check for error finish reason and raise GenerationError # finish_reason='error' indicates a retryable request-level internal error self._raise_if_error(output.finish_reason, request_id) + if i > 0 and parser is not None: + # Parsers are stateful: like streaming, use one per choice. + parser = self._make_parser( + request, tokenizer, self._effective_chat_template_kwargs(request) + ) token_ids = output.token_ids out_logprobs = output.logprobs diff --git a/vllm/entrypoints/openai/parser/harmony_utils.py b/vllm/entrypoints/openai/parser/harmony_utils.py index 873b9e92bd..dc3aef4a95 100644 --- a/vllm/entrypoints/openai/parser/harmony_utils.py +++ b/vllm/entrypoints/openai/parser/harmony_utils.py @@ -449,12 +449,80 @@ def parse_chat_input_to_harmony_message( def render_for_completion(messages: list[Message]) -> list[int]: messages = auto_drop_analysis_messages(messages) conversation = Conversation.from_messages(messages) - token_ids = get_encoding().render_conversation_for_completion( + encoding = get_encoding() + token_ids = encoding.render_conversation_for_completion( conversation, Role.ASSISTANT, config=RenderConversationConfig(auto_drop_analysis=False), ) - return token_ids + return _use_legacy_tool_call_headers(token_ids) + + +def _use_legacy_tool_call_headers(token_ids: list[int]) -> list[int]: + """Keep prior assistant tool calls in the format expected by GPT-OSS. + + oss-harmony 0.0.10 changed a rendered tool-call header from:: + + assistant to=python<|channel|>commentary code + + to:: + + assistant<|channel|>commentary to=python <|constrain|>code + + GPT-OSS can echo the latter as a malformed header with the recipient in + both positions. Rewrite only that unambiguous token-level header shape, + while retaining oss-harmony's embedded vocabulary and parser. + """ + encoding = get_encoding() + start = encoding.encode("<|start|>", allowed_special="all")[0] + channel_token = encoding.encode("<|channel|>", allowed_special="all")[0] + constrain = encoding.encode("<|constrain|>", allowed_special="all")[0] + message = encoding.encode("<|message|>", allowed_special="all")[0] + + result = token_ids.copy() + index = 0 + while index < len(result): + if result[index] != start: + index += 1 + continue + + try: + header_end = result.index(message, index + 1) + channel_index = result.index(channel_token, index + 1, header_end) + constrain_index = result.index(constrain, channel_index + 1, header_end) + except ValueError: + index += 1 + continue + + author = encoding.decode(result[index + 1 : channel_index]) + channel_and_recipient = encoding.decode( + result[channel_index + 1 : constrain_index] + ) + content_type = encoding.decode(result[constrain_index + 1 : header_end]) + if ( + author != "assistant" + or " to=" not in channel_and_recipient + or not channel_and_recipient.endswith(" ") + ): + index = header_end + 1 + continue + + channel, recipient = channel_and_recipient[:-1].split(" to=", 1) + if not channel or not recipient or not content_type: + index = header_end + 1 + continue + + legacy_header = ( + [start] + + encoding.encode(f"assistant to={recipient}") + + [channel_token] + + encoding.encode(f"{channel} {content_type}") + + [message] + ) + result[index : header_end + 1] = legacy_header + index += len(legacy_header) + + return result def get_streamable_parser_for_assistant() -> StreamableParser: diff --git a/vllm/entrypoints/openai/responses/context.py b/vllm/entrypoints/openai/responses/context.py index 59b42f96a3..fa233dcce1 100644 --- a/vllm/entrypoints/openai/responses/context.py +++ b/vllm/entrypoints/openai/responses/context.py @@ -359,6 +359,10 @@ class ParsableContext(ConversationContext): enable_auto_tools=self.enable_auto_tools, model_output_token_ids=completion.token_ids, ) + # Each round is a separate generation, so count it on its own ids. + self.num_reasoning_tokens += self.response_parser.count_reasoning_tokens( + completion.token_ids or [] + ) if not self.request.include_reasoning: reasoning = None self.response_messages.extend( diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py index 9fba86b80c..f18c493259 100644 --- a/vllm/entrypoints/openai/responses/serving.py +++ b/vllm/entrypoints/openai/responses/serving.py @@ -832,18 +832,12 @@ class OpenAIServingResponses(GenerateBaseServing): num_generated_tokens = context.num_output_tokens num_cached_tokens = context.num_cached_tokens num_reasoning_tokens = context.num_reasoning_tokens - # For text-based reasoning parsers (e.g., ...), - # HarmonyContext already counts reasoning tokens via channels. - # For Simple/Parsable contexts, derive reasoning_tokens from - # accumulated output token IDs using the parser if not already set. - if ( - num_reasoning_tokens == 0 - and isinstance(context, (SimpleContext, ParsableContext)) - and context.response_parser is not None - ): - accumulated = getattr(context, "_accumulated_token_ids", []) or [] + # HarmonyContext and ParsableContext count reasoning tokens as each + # round is appended. SimpleContext is single-round but accumulates + # streaming deltas, so count its full output once here. + if isinstance(context, SimpleContext) and context.response_parser is not None: num_reasoning_tokens = context.response_parser.count_reasoning_tokens( - accumulated + context._accumulated_token_ids ) usage = ResponseUsage( diff --git a/vllm/envs.py b/vllm/envs.py index 78f99ea0ac..f663414a91 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -141,7 +141,7 @@ if TYPE_CHECKING: VLLM_ROCM_USE_AITER_LINEAR_HIPBMM: bool = False VLLM_ROCM_USE_AITER_MOE: bool = True VLLM_ROCM_AITER_MOE_DISPATCH_POLICY: int = 0 - VLLM_ROCM_USE_AITER_MOE_SITUV2: bool = False + VLLM_ROCM_USE_AITER_MOE_SITUV2: Literal["auto", "a4w4", "a8w4", "a16w4"] = "auto" VLLM_ROCM_USE_AITER_RMSNORM: bool = True VLLM_ROCM_USE_AITER_MLA: bool = True VLLM_ROCM_AITER_MLA_ASM_PADDING: Literal["auto", "gluon", "asm"] = "auto" @@ -260,6 +260,7 @@ if TYPE_CHECKING: VLLM_ROCM_QUICK_REDUCE_MIN_SIZE_BYTES_MB: int | None = None VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB: int | None = None VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT: int = 480 + VLLM_MOONCAKE_CONNECTOR_TIMEOUT: float = 30.0 VLLM_ENABLE_CUDAGRAPH_GC: bool = False VLLM_LOOPBACK_IP: str = "" VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE: bool = True @@ -1280,18 +1281,13 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_ROCM_USE_AITER_MOE": lambda: ( os.getenv("VLLM_ROCM_USE_AITER_MOE", "True").lower() in ("true", "1") ), - # Route K3 SiTU MXFP4 MoE through the FlyDSL SiTUv2 path (a4w4 fp4 - # activations, separated gate/up layout) instead of default a16w4. vLLM - # sets AITER_SITUV2_A4W4 at init when this flag is on and clears any - # legacy AITER_SITUV2_A8W4 override (AITER checks A8W4 first). - # VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4 is a deprecated alias for existing - # recipes; it does not select a8w4 kernels. - # Needs AITER >= v0.1.20 (ROCm/aiter#4463) for the a4w4 dispatch flag - # and tuned kimik3_a4w4_*_fmoe.csv rows; otherwise FlyDSL uses heuristics. - "VLLM_ROCM_USE_AITER_MOE_SITUV2": lambda: ( - os.getenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "0").lower() in ("true", "1") - or os.getenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", "0").lower() - in ("true", "1") + # Activation dtype for the Kimi-K3 SiTU MXFP4 MoE (AITER FlyDSL SiTUv2): + # auto (= a4w4), a4w4, a8w4 or a16w4. Legacy 1/0 mean a4w4/a16w4. + "VLLM_ROCM_USE_AITER_MOE_SITUV2": env_with_choices( + "VLLM_ROCM_USE_AITER_MOE_SITUV2", + "auto", + ["auto", "a4w4", "a8w4", "a16w4", "0", "1"], + case_sensitive=False, ), # MoE sorting dispatch policy for AITER fused MoE kernels. # 0 = auto (default): single-pass for small batches, multi-pass @@ -1718,6 +1714,11 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_EC_SIDE_CHANNEL_PORT": lambda: int( os.getenv("VLLM_EC_SIDE_CHANNEL_PORT", "5601") ), + # Per-request timeout (seconds) when a prefiller worker registers with + # the Mooncake bootstrap server. + "VLLM_MOONCAKE_CONNECTOR_TIMEOUT": lambda: float( + os.getenv("VLLM_MOONCAKE_CONNECTOR_TIMEOUT", "30.0") + ), # Port used for Mooncake handshake between remote agents. "VLLM_MOONCAKE_BOOTSTRAP_PORT": lambda: int( os.getenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", "8998") diff --git a/vllm/model_executor/layers/fused_moe/config.py b/vllm/model_executor/layers/fused_moe/config.py index 9221e93824..11e2a57520 100644 --- a/vllm/model_executor/layers/fused_moe/config.py +++ b/vllm/model_executor/layers/fused_moe/config.py @@ -118,7 +118,7 @@ class RoutingMethodType(IntEnum): # SigmoidRenorm: Sigmoid -> TopK -> Renormalize (divide by sum of top-K) SigmoidRenorm = (6,) # MiniMax2: Sigmoid + Bias -> TopK -> ScaledSumNormalize - # (routeScale=1.0, epsilon=1e-20) + # (routeScale=routed_scaling_factor, epsilon=1e-20) MiniMax2 = (7,) # Sigmoid: Sigmoid -> TopK (no renormalization) Sigmoid = (8,) @@ -137,7 +137,6 @@ def get_routing_method_type( renormalize: bool, num_expert_group: int | None, has_e_score_bias: bool, - routed_scaling_factor: float | None = 1.0, ) -> RoutingMethodType: if scoring_func == "sqrtsoftplus": # DeepSeek V4 uses sqrtsoftplus routing with optional routing bias @@ -153,9 +152,7 @@ def get_routing_method_type( return RoutingMethodType.Unspecified if (num_expert_group or 0) > 0: return RoutingMethodType.DeepSeekV3 - if routed_scaling_factor in (None, 1.0): - return RoutingMethodType.MiniMax2 - return RoutingMethodType.Unspecified + return RoutingMethodType.MiniMax2 else: return RoutingMethodType.Unspecified diff --git a/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a16_moe.py b/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a16_moe.py index fb3256e1a4..13c34698c8 100644 --- a/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a16_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a16_moe.py @@ -139,7 +139,7 @@ def aiter_triton_kernel_w4a16_moe_forward( from vllm.model_executor.layers.quantization.utils.mxfp4_utils import ( should_use_cdna4_mx_scale_swizzle, ) - from vllm.platforms.rocm import on_gfx1250 + from vllm.platforms.rocm import on_gfx950, on_gfx1250 try: from aiter.ops.triton.moe.moe_op_gemm_a16w4 import moe_gemm_a16w4 @@ -236,6 +236,12 @@ def aiter_triton_kernel_w4a16_moe_forward( # kernel indexes a swizzled buffer as if it were linear. swz = "CDNA4_SCALE" if should_use_cdna4_mx_scale_swizzle() else None + # AITER's gfx950 Gluon a16w4 kernel (default backend="None"/auto-detect + # picks it) calls a Triton-Gluon AMD API that isn't available in vLLM's + # pinned Triton, so force the Triton backend on gfx950. gfx1250 already + # used Gluon before this and is unaffected, so leave it on auto-detect. + a16w4_backend = "triton" if on_gfx950() else None + intermediate = moe_gemm_a16w4( hidden_states, w1_data, @@ -254,6 +260,7 @@ def aiter_triton_kernel_w4a16_moe_forward( swiglu_add_residual=swiglu_add_residual, unpadded_N=unpadded_N_w1, unpadded_K=unpadded_K_w1, + backend=a16w4_backend, ) out = moe_gemm_a16w4( @@ -270,6 +277,7 @@ def aiter_triton_kernel_w4a16_moe_forward( swizzle_mx_scale=swz, unpadded_N=unpadded_N_w2, unpadded_K=unpadded_K_w2, + backend=a16w4_backend, ) return out diff --git a/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py b/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py index 62df395f69..4f2f760ad3 100644 --- a/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py @@ -385,10 +385,11 @@ def rocm_aiter_fused_experts( gate_mode = "" if activation == MoEActivation.SITU: - # SiTUv2 flydsl (VLLM_ROCM_USE_AITER_MOE_SITUV2=1) uses a4w4 - # fp4 activations with separated gate/up weights (AITER #4463); - # default a16w4 SiTU also stays separated. - gate_mode = GateMode.SEPARATED.value + gate_mode = ( + GateMode.INTERLEAVE.value + if rocm_aiter_ops.is_fused_moe_situv2_gate_up_interleaved() + else GateMode.SEPARATED.value + ) elif quant_config.use_mxfp4_w4a16: gate_mode = GateMode.INTERLEAVE.value elif activation_interleave is not None: diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py index 5450f73f85..56ba3981b7 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py @@ -1596,8 +1596,7 @@ def convert_weight_to_mxfp4_moe_kernel_format( fp4_dtype = torch.float4_e2m1fn_x2 e8m0_dtype = torch.float8_e8m0fnu - # SiTUv2 flydsl uses separated gate/up layout (a4w4). - guinterleave = False + guinterleave = rocm_aiter_ops.is_fused_moe_situv2_gate_up_interleaved() w13 = rocm_aiter_ops.shuffle_weight_a16w4( w13_weight.data.view(fp4_dtype), 16, guinterleave ) diff --git a/vllm/model_executor/layers/fused_moe/router/cpu_router.py b/vllm/model_executor/layers/fused_moe/router/cpu_router.py index 1f96e9107a..3d034efdf0 100644 --- a/vllm/model_executor/layers/fused_moe/router/cpu_router.py +++ b/vllm/model_executor/layers/fused_moe/router/cpu_router.py @@ -198,7 +198,6 @@ class CPURouter(BaseRouter): renormalize=self.renormalize, num_expert_group=self.num_expert_group, has_e_score_bias=self.e_score_correction_bias is not None, - routed_scaling_factor=self.routed_scaling_factor, ) def _compute_routing( diff --git a/vllm/model_executor/layers/fused_moe/router/fused_topk_bias_router.py b/vllm/model_executor/layers/fused_moe/router/fused_topk_bias_router.py index 257ede27d5..2df66d063a 100644 --- a/vllm/model_executor/layers/fused_moe/router/fused_topk_bias_router.py +++ b/vllm/model_executor/layers/fused_moe/router/fused_topk_bias_router.py @@ -391,7 +391,6 @@ class FusedTopKBiasRouter(BaseRouter): renormalize=self.renormalize, num_expert_group=None, has_e_score_bias=True, - routed_scaling_factor=self.routed_scaling_factor, ) def _compute_routing( diff --git a/vllm/model_executor/layers/fused_moe/router/grouped_topk_router.py b/vllm/model_executor/layers/fused_moe/router/grouped_topk_router.py index 9b134d385a..64a12e57cd 100644 --- a/vllm/model_executor/layers/fused_moe/router/grouped_topk_router.py +++ b/vllm/model_executor/layers/fused_moe/router/grouped_topk_router.py @@ -283,7 +283,6 @@ class GroupedTopKRouter(BaseRouter): renormalize=self.renormalize, num_expert_group=self.num_expert_group, has_e_score_bias=self.e_score_correction_bias is not None, - routed_scaling_factor=self.routed_scaling_factor, ) def _compute_routing( diff --git a/vllm/model_executor/layers/fused_moe/router/router_factory.py b/vllm/model_executor/layers/fused_moe/router/router_factory.py index f03fb106f7..96ccc853b8 100644 --- a/vllm/model_executor/layers/fused_moe/router/router_factory.py +++ b/vllm/model_executor/layers/fused_moe/router/router_factory.py @@ -188,7 +188,6 @@ def create_fused_moe_router( renormalize=renormalize, num_expert_group=groups, has_e_score_bias=e_score_correction_bias is not None, - routed_scaling_factor=routed_scaling_factor, ) routing_method_preserved = advertised_routing_method( diff --git a/vllm/model_executor/layers/fused_moe/router/zero_expert_router.py b/vllm/model_executor/layers/fused_moe/router/zero_expert_router.py index 0107f88156..4e4a562c2b 100644 --- a/vllm/model_executor/layers/fused_moe/router/zero_expert_router.py +++ b/vllm/model_executor/layers/fused_moe/router/zero_expert_router.py @@ -4,10 +4,7 @@ import torch from vllm.distributed.eplb.eplb_state import EplbLayerState -from vllm.model_executor.layers.fused_moe.config import ( - RoutingMethodType, - get_routing_method_type, -) +from vllm.model_executor.layers.fused_moe.config import RoutingMethodType from vllm.model_executor.layers.fused_moe.fused_moe import ( zero_experts_compute_triton, ) @@ -53,14 +50,9 @@ class ZeroExpertRouter(BaseRouter): @property def routing_method_type(self) -> RoutingMethodType: - return get_routing_method_type( - scoring_func=self.scoring_func, - top_k=self.top_k, - renormalize=self.renormalize, - num_expert_group=None, - has_e_score_bias=True, - routed_scaling_factor=self.routed_scaling_factor, - ) + # Zero experts are resolved by this router, which kernels with + # built-in routing (e.g. FlashInfer TRT-LLM) cannot reproduce. + return RoutingMethodType.Unspecified def _compute_routing( self, diff --git a/vllm/model_executor/layers/rotary_embedding/packed_qk_rope.py b/vllm/model_executor/layers/rotary_embedding/packed_qk_rope.py new file mode 100644 index 0000000000..e7d916e48f --- /dev/null +++ b/vllm/model_executor/layers/rotary_embedding/packed_qk_rope.py @@ -0,0 +1,81 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""In-place interleaved RoPE on the Q/K slices of a packed QKV buffer.""" + +import torch + +from vllm.triton_utils import tl, triton + + +# seqlen is not specialized: image sizes vary per request. +@triton.jit(do_not_specialize=["seqlen"]) +def _packed_qk_rope_kernel( + qk_ptr, # (seqlen, 2 * nheads, headdim) strided view of the packed QKV + freqs_ptr, # (seqlen, rotary_dim) fp32 re/im view of the complex freqs + seqlen, + qk_row_stride, + qk_head_stride, + rotary_dim: tl.constexpr, + BLOCK_M: tl.constexpr, +): + BLOCK_K: tl.constexpr = triton.next_power_of_2(rotary_dim) + pid_m = tl.program_id(0) + pid_h = tl.program_id(1) + rm = pid_m * BLOCK_M + tl.arange(0, BLOCK_M) + rk = tl.arange(0, BLOCK_K) + mask_m = rm < seqlen + + base = qk_ptr + pid_h * qk_head_stride + rm[:, None] * qk_row_stride + mask_x = mask_m[:, None] & (rk < rotary_dim)[None, :] + x = tl.load(base + rk[None, :], mask=mask_x, other=0.0).to(tl.float32) + f = tl.load(freqs_ptr + rm[:, None] * rotary_dim + rk[None, :], mask=mask_x) + cos, sin = tl.split(tl.reshape(f, (BLOCK_M, BLOCK_K // 2, 2))) + + x0, x1 = tl.split(tl.reshape(x, (BLOCK_M, BLOCK_K // 2, 2))) + o0 = x0 * cos - x1 * sin + o1 = x1 * cos + x0 * sin + tl.store(base + rk[None, :], tl.interleave(o0, o1), mask=mask_x) + + +def packed_qk_rope_(xqkv: torch.Tensor, freqs_cis: torch.Tensor) -> None: + """Rotate Q and K in place inside a packed QKV buffer. + + Args: + xqkv: contiguous (seqlen, 3, nheads, headdim); only the Q and K + slices are rotated. + freqs_cis: contiguous (seqlen, headdim // 2) complex64 rotary freqs, + read through its interleaved re/im fp32 view. + + Bitwise identical to ``ApplyRotaryEmb(enable_fp32_compute=True)`` applied + to Q and K separately, but in one kernel with ~5x less memory traffic. + Requires triton: callers must check HAS_TRITON and use the unfused path + otherwise. Contract violations raise. + + """ + assert xqkv.ndim == 4 and xqkv.size(1) == 3, xqkv.shape + assert xqkv.is_contiguous(), xqkv.shape + assert freqs_cis.ndim == 2 and freqs_cis.is_contiguous(), freqs_cis.shape + assert freqs_cis.dtype == torch.complex64, freqs_cis.dtype + + seq_length, _, num_heads, head_dim = xqkv.shape + rotary_dim = freqs_cis.size(-1) * 2 + assert freqs_cis.size(0) == seq_length, (freqs_cis.shape, xqkv.shape) + assert rotary_dim == head_dim, (freqs_cis.shape, xqkv.shape) # full rotation + qk = xqkv.as_strided( + (seq_length, 2 * num_heads, head_dim), + (3 * num_heads * head_dim, head_dim, 1), + ) + + BLOCK_M = 4 + # seqlen on grid.x: HIP caps gridDim.y at 65535. + grid = (triton.cdiv(seq_length, BLOCK_M), 2 * num_heads) + _packed_qk_rope_kernel[grid]( + qk, + freqs_cis.view(torch.float32), + seq_length, + qk.stride(0), + qk.stride(1), + rotary_dim=rotary_dim, + BLOCK_M=BLOCK_M, + num_warps=2 if rotary_dim <= 64 else 4, + ) diff --git a/vllm/model_executor/layers/utils.py b/vllm/model_executor/layers/utils.py index aec4ebdd44..af8b7d4d5c 100644 --- a/vllm/model_executor/layers/utils.py +++ b/vllm/model_executor/layers/utils.py @@ -377,6 +377,13 @@ def rocm_unquantized_gemm( weight: torch.Tensor, bias: torch.Tensor | None = None, ) -> torch.Tensor: + # dispatch_unquantized_gemm() picks this backend for any ROCm build, + # regardless of the tensor's actual device; the custom op below is only + # registered for the CUDA/HIP dispatch key, so route CPU tensors (e.g. + # from unit tests that build layers without moving them to the GPU) + # through the device-agnostic fallback instead of hard-crashing. + if not x.is_cuda: + return default_unquantized_gemm(layer, x, weight, bias) return torch.ops.vllm.rocm_unquantized_gemm(x, weight, bias) diff --git a/vllm/model_executor/model_loader/reload/torchao_decorator.py b/vllm/model_executor/model_loader/reload/torchao_decorator.py index dc8e0acc6e..10ced70d31 100644 --- a/vllm/model_executor/model_loader/reload/torchao_decorator.py +++ b/vllm/model_executor/model_loader/reload/torchao_decorator.py @@ -1,10 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from collections.abc import Iterable +from collections.abc import Callable from functools import wraps -from types import FunctionType -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Concatenate, ParamSpec, TypeVar import torch @@ -20,13 +19,18 @@ if TYPE_CHECKING: __all__ = ["set_torchao_reload_attrs", "support_quantized_model_reload_from_hp_weights"] +_P = ParamSpec("_P") +_R = TypeVar("_R") + def set_torchao_reload_attrs(model: torch.nn.Module, model_config: ModelConfig): model._do_torchao_reload = True model._model_config = model_config -def support_quantized_model_reload_from_hp_weights(original_load_weights: FunctionType): +def support_quantized_model_reload_from_hp_weights( + original_load_weights: Callable[Concatenate["AutoWeightsLoader", _P], _R], +) -> Callable[Concatenate["AutoWeightsLoader", _P], _R]: """Decorator for `load_weights` method for AutoWeightsLoader.load_weights to support reloading high precision (bfloat16/float16/float32) weight for an already quantized model, this involves restoring the weights to a high precision weights and @@ -39,17 +43,16 @@ def support_quantized_model_reload_from_hp_weights(original_load_weights: Functi @wraps(original_load_weights) def patched_model_load_weights( self: "AutoWeightsLoader", - weights: Iterable[tuple[str, torch.Tensor]], - *args, - **kwargs, - ): + *args: _P.args, + **kwargs: _P.kwargs, + ) -> _R: model = self.module if not getattr(model, "_do_torchao_reload", False): - return original_load_weights(self, weights, *args, **kwargs) + return original_load_weights(self, *args, **kwargs) initialize_layerwise_reload(model) - loaded_weights = original_load_weights(self, weights, *args, **kwargs) + loaded_weights = original_load_weights(self, *args, **kwargs) finalize_layerwise_reload(model, model._model_config) return loaded_weights diff --git a/vllm/model_executor/models/kimi_k25_vit.py b/vllm/model_executor/models/kimi_k25_vit.py index 1b05f1f77b..72e45d82e6 100644 --- a/vllm/model_executor/models/kimi_k25_vit.py +++ b/vllm/model_executor/models/kimi_k25_vit.py @@ -30,6 +30,9 @@ from vllm.model_executor.layers.linear import ( ) from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb +from vllm.model_executor.layers.rotary_embedding.packed_qk_rope import ( + packed_qk_rope_, +) from vllm.model_executor.models.utils import maybe_prefix from vllm.model_executor.models.vision import ( is_vit_use_data_parallel, @@ -37,6 +40,7 @@ from vllm.model_executor.models.vision import ( ) from vllm.platforms import current_platform from vllm.transformers_utils.configs.kimi_k25 import KimiK25VisionConfig +from vllm.triton_utils import HAS_TRITON from vllm.utils.torch_utils import async_tensor_h2d logger = init_logger(__name__) @@ -451,14 +455,19 @@ class MoonViTEncoderLayer(nn.Module): ) # xqkv: (seqlen, 3, nheads, headdim) xqkv = xqkv.view(*qkv_shape) - xq, xk, xv = torch.unbind(xqkv, dim=-3) - _apply_rope_input_validation(xq, rope_freqs_cis) - _apply_rope_input_validation(xk, rope_freqs_cis) - rope_cos = rope_freqs_cis.real.contiguous() - rope_sin = rope_freqs_cis.imag.contiguous() - xq = self.apply_rotary_emb(xq, rope_cos, rope_sin) - xk = self.apply_rotary_emb(xk, rope_cos, rope_sin) + # xq/xk alias xqkv, so the in-place rotation below is visible through + # them; only the fallback needs to rebind. + xq, xk, xv = torch.unbind(xqkv, dim=-3) + if HAS_TRITON: + packed_qk_rope_(xqkv, rope_freqs_cis) + else: + _apply_rope_input_validation(xq, rope_freqs_cis) + _apply_rope_input_validation(xk, rope_freqs_cis) + rope_cos = rope_freqs_cis.real.contiguous() + rope_sin = rope_freqs_cis.imag.contiguous() + xq = self.apply_rotary_emb(xq, rope_cos, rope_sin) + xk = self.apply_rotary_emb(xk, rope_cos, rope_sin) if max_seqlen is None: max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max() diff --git a/vllm/model_executor/models/openpangu_vl.py b/vllm/model_executor/models/openpangu_vl.py index b7d5c81032..8a62aa6ba6 100644 --- a/vllm/model_executor/models/openpangu_vl.py +++ b/vllm/model_executor/models/openpangu_vl.py @@ -1055,15 +1055,18 @@ class OpenPanguVLForConditionalGeneration( input_ids: torch.Tensor, multimodal_embeddings: MultiModalEmbeddings | None = None, ) -> torch.Tensor: - inputs_embeds = self.language_model.embed_input_ids(input_ids) - if multimodal_embeddings is not None: - inputs_embeds = self.embed_input_ids( # type: ignore[call-overload] - input_ids, - inputs_embeds, - multimodal_embeddings, - [self.config.image_token_id, self.config.video_token_id], - ) - return inputs_embeds + if multimodal_embeddings is None: + return self.language_model.embed_input_ids(input_ids) + + mm_token_ids = input_ids.new_tensor( + [self.config.image_token_id, self.config.video_token_id] + ) + is_multimodal = torch.isin(input_ids, mm_token_ids) + return self.embed_input_ids( + input_ids, + multimodal_embeddings, + is_multimodal=is_multimodal, + ) def _process_image_input( self, image_input: OpenPanguVLImageInputs diff --git a/vllm/model_executor/models/phi4mm.py b/vllm/model_executor/models/phi4mm.py index 8524455975..7f223b1ccc 100644 --- a/vllm/model_executor/models/phi4mm.py +++ b/vllm/model_executor/models/phi4mm.py @@ -1297,7 +1297,7 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal): logits = self.logits_processor(self.lm_head, hidden_states) return logits - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> None: + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: loader = AutoWeightsLoader(self) return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) diff --git a/vllm/model_executor/models/roberta.py b/vllm/model_executor/models/roberta.py index 6a65b3a7e9..8e141229f0 100644 --- a/vllm/model_executor/models/roberta.py +++ b/vllm/model_executor/models/roberta.py @@ -41,7 +41,7 @@ from vllm.model_executor.models.utils import ( from vllm.sequence import IntermediateTensors from .bert_with_rope import BertWithRope, JinaRobertaModel -from .interfaces import SupportsCrossEncoding +from .interfaces import SupportsCrossEncoding, SupportsLoRA from .interfaces_base import attn_type, default_pooling_type @@ -266,7 +266,7 @@ class BgeM3EmbeddingModel(RobertaEmbeddingModel): @default_pooling_type(seq_pooling_type="CLS") -class RobertaForSequenceClassification(nn.Module, SupportsCrossEncoding): +class RobertaForSequenceClassification(nn.Module, SupportsCrossEncoding, SupportsLoRA): """A model that uses Roberta to provide embedding functionalities. This class encapsulates the BertModel and provides an interface for @@ -279,6 +279,7 @@ class RobertaForSequenceClassification(nn.Module, SupportsCrossEncoding): """ is_pooling_model = True + packed_modules_mapping = {"qkv_proj": ["query", "key", "value"]} jina_to_vllm_mapper = WeightsMapper( orig_to_new_substr={ "emb_ln": "embeddings.LayerNorm", @@ -295,7 +296,8 @@ class RobertaForSequenceClassification(nn.Module, SupportsCrossEncoding): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() config = vllm_config.model_config.hf_config - self.padding_idx: int = vllm_config.model_config.hf_config.pad_token_id + self.config = config + self.padding_idx: int = config.pad_token_id self.num_labels = config.num_labels self.roberta = BertModel( diff --git a/vllm/model_executor/models/ultravox.py b/vllm/model_executor/models/ultravox.py index a454ba5962..72ce26f21c 100644 --- a/vllm/model_executor/models/ultravox.py +++ b/vllm/model_executor/models/ultravox.py @@ -282,10 +282,14 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo def get_replacement_ultravox(item_idx: int): if "audio_num_tokens" in out_mm_data: - return [replacement_id] * int(out_mm_data["audio_num_tokens"][item_idx]) + audio_num_tokens = out_mm_data["audio_num_tokens"] + assert isinstance(audio_num_tokens, torch.Tensor) + return [replacement_id] * int(audio_num_tokens[item_idx]) start = chunks_start_idx[item_idx] end = chunks_start_idx[item_idx + 1] - audio_token_len = out_mm_data["audio_token_len"][start:end].sum() + audio_token_lens = out_mm_data["audio_token_len"] + assert isinstance(audio_token_lens, torch.Tensor) + audio_token_len = audio_token_lens[start:end].sum() return [replacement_id] * int(audio_token_len) # type: ignore return [ @@ -566,7 +570,8 @@ class UltravoxWhisperEncoder(WhisperEncoder): ) -> torch.Tensor: return (input_lengths - 1) // 2 + 1 - def forward( + # Ultravox requires chunk lengths in addition to Whisper's input features. + def forward( # type: ignore[override] self, input_features: torch.Tensor, audio_lens: torch.Tensor, @@ -960,12 +965,12 @@ def pad_and_concat_to_dim3( return features - features = [pad_and_concat_to_dim3(f) for f in features] + tensors = [pad_and_concat_to_dim3(f) for f in features] - max_len = max(f.shape[-1] for f in features) + max_len = max(f.shape[-1] for f in tensors) # Ensure all features have dim=3 - features = [f.view(-1, *f.shape[-2:]) for f in features] + tensors = [f.view(-1, *f.shape[-2:]) for f in tensors] # Pad and concatenate: # [[B1, 80, M1], [B2, 80, M2]] -> [B1+B2, 80, max(M1, M2)] - features = [F.pad(f, (0, max_len - f.shape[-1])) for f in features] - return torch.cat(features) + tensors = [F.pad(f, (0, max_len - f.shape[-1])) for f in tensors] + return torch.cat(tensors) diff --git a/vllm/model_executor/models/unlimited_ocr.py b/vllm/model_executor/models/unlimited_ocr.py index d248e5c8e0..27e5503d2c 100644 --- a/vllm/model_executor/models/unlimited_ocr.py +++ b/vllm/model_executor/models/unlimited_ocr.py @@ -199,6 +199,7 @@ class UnlimitedOCRMultiModalProcessor(DeepseekOCRMultiModalProcessor): if isinstance(images, ImageEmbeddingItems): num_image_tokens = images.get_feature_size(item_idx) else: + assert isinstance(images, ImageProcessorItems) size = images.get_image_size(item_idx) # Disable crop mode for multi-image input. diff --git a/vllm/model_executor/models/utils.py b/vllm/model_executor/models/utils.py index 6375e0572e..e762ad1c47 100644 --- a/vllm/model_executor/models/utils.py +++ b/vllm/model_executor/models/utils.py @@ -35,7 +35,6 @@ if TYPE_CHECKING: from transformers import PreTrainedConfig from transformers.conversion_mapping import WeightRenaming - from vllm.config.model import ModelConfig from vllm.model_executor.layers.quantization import QuantizationConfig logger = init_logger(__name__) @@ -337,7 +336,7 @@ class AutoWeightsLoader: """ # Add persistent registered buffers. # Non-persistent buffers are excluded, matching PyTorch state_dict(). - non_persistent = getattr(module, "_non_persistent_buffers_set", set()) + non_persistent: set[str] = getattr(module, "_non_persistent_buffers_set", set()) for buf_name, buf in module.named_buffers(recurse=False): if buf_name not in child_params and buf_name not in non_persistent: child_params[buf_name] = buf @@ -558,7 +557,7 @@ def maybe_fuse_shared_experts( def get_spec_layer_idx_from_weight_name( - config: "ModelConfig", weight_name: str + config: "PreTrainedConfig", weight_name: str ) -> int | None: """Return the MTP layer index a weight belongs to, or None. @@ -587,7 +586,7 @@ def get_spec_layer_idx_from_weight_name( def skip_spec_layers( - weights: Iterable[tuple[str, torch.Tensor]], config: "ModelConfig" + weights: Iterable[tuple[str, torch.Tensor]], config: "PreTrainedConfig" ) -> Iterable[tuple[str, torch.Tensor]]: """Drop MTP spec-layer weights (loaded by the MTP head, not the base model). @@ -629,11 +628,11 @@ def init_vllm_registered_model( @overload -def flatten_bn(x: torch.Tensor) -> torch.Tensor: ... +def flatten_bn(x: list[torch.Tensor]) -> list[torch.Tensor]: ... @overload -def flatten_bn(x: list[torch.Tensor]) -> list[torch.Tensor]: ... +def flatten_bn(x: torch.Tensor) -> torch.Tensor: ... @overload @@ -973,7 +972,9 @@ def get_draft_quant_config(vllm_config: VllmConfig) -> "QuantizationConfig | Non The draft model's config if available, None otherwise. """ - draft_model_config = vllm_config.speculative_config.draft_model_config + speculative_config = vllm_config.speculative_config + assert speculative_config is not None + draft_model_config = speculative_config.draft_model_config draft_load_config = vllm_config.load_config return ( diff --git a/vllm/models/deepseek_v41/amd/rocm.py b/vllm/models/deepseek_v41/amd/rocm.py index e1588b55e7..9de09bad8c 100644 --- a/vllm/models/deepseek_v41/amd/rocm.py +++ b/vllm/models/deepseek_v41/amd/rocm.py @@ -29,6 +29,7 @@ from vllm.models.deepseek_v41.sparse_mla import ( from vllm.platforms import current_platform from vllm.platforms.rocm import _ON_GFX950 from vllm.triton_utils import tl, triton +from vllm.utils.multi_stream_utils import execute_in_parallel from vllm.v1.attention.backend import ( CommonAttentionMetadata, ) @@ -529,6 +530,9 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): "this rank; PP splits inside a v4.1 index-sharing group " "are not supported." ) + if self.compressor is None and self.indexer is None: + # Dense layers have nothing to overlap; keep the base serial path. + self.aux_stream_list = None @classmethod def get_padded_num_q_heads(cls, num_heads: int) -> int: @@ -600,8 +604,193 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): def _run_parallel_input_projections( self, hidden_states: torch.Tensor ) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None]: + if self.indexer is not None and self.compressor is None: + # Serial: qr must sit on the default stream before the q-side + # fork below can consume it (ROCm). + aux_streams = self.aux_stream_list + self.aux_stream_list = None + try: + return super()._run_parallel_input_projections(hidden_states) + finally: + self.aux_stream_list = aux_streams return super()._run_parallel_input_projections(hidden_states) + def _enable_multi_stream_overlap(self) -> bool: + """ROCm multi-stream gates: streams and capture region. + + Dict metadata marks piecewise cudagraph, whose eager breaks rebuild + the attention inputs on the owning stream. Forking side streams + there would rely on runtime HIP event sync, which is unreliable in + this overlap on ROCm (event waits can hang), so multi-stream only + runs where the fork/join becomes static graph edges: inside capture, + or with non-dict metadata (full cudagraph or the profile run), which + has no eager breaks. + """ + attn_metadata = get_forward_context().attn_metadata + return self.aux_stream_list is not None and ( + torch.cuda.is_current_stream_capturing() + or not isinstance(attn_metadata, dict) + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + llama_4_scaling: torch.Tensor | None = None, + ) -> torch.Tensor: + if ( + self.indexer is None + or self._prepare_and_attn_fn == self._prepare_and_attn_eager + or not self._enable_multi_stream_overlap() + ): + # Sequential fallback: no forks outside the capture region, where + # HIP event sync is unreliable; MRV1 keeps the input prep in its + # wide eager region (#51430). + aux_streams = self.aux_stream_list + self.aux_stream_list = None + try: + return super().forward(positions, hidden_states, llama_4_scaling) + finally: + self.aux_stream_list = aux_streams + + attn_out = self._alloc_attn_out(hidden_states.shape[0], hidden_states) + if self.compressor is not None: + q, kv, index_q, index_q_scale, index_weights = self._forward_csa2_full( + hidden_states, positions + ) + else: + q, kv, index_q, index_q_scale, index_weights = self._forward_csa2_reindex( + hidden_states, positions + ) + self._sparse_indexer_and_attn( + hidden_states, + index_q, + index_q_scale, + index_weights, + q, + kv, + positions, + attn_out, + ) + return self._o_proj(attn_out, positions) + + def _forward_csa2_full( + self, hidden_states: torch.Tensor, positions: torch.Tensor + ) -> tuple[ + torch.Tensor, + torch.Tensor, + torch.Tensor | None, + torch.Tensor | None, + torch.Tensor | None, + ]: + """CSA layers: one fork replaces the base pipeline's three. + + Each fork/join is a HIP event pair per side stream on ROCm, so the + base three-fork pipeline is expensive there; the compressor chain and + the indexer weights projection run fully in parallel with the default + chain. The indexer K write and q-side stay serial after the join: + ``wk`` consumes the aux-0 latent and the q-side's wq_b consumes the + default-stream qr. + """ + attn_metadata = get_forward_context().attn_metadata + compressor = self.compressor + indexer = self.indexer + assert compressor is not None and indexer is not None + aux_streams = self.aux_stream_list + assert aux_streams is not None and len(aux_streams) >= 2 + + def default_chain(): + qr_kv = self._fused_wqa_wkv_gemm(hidden_states) + qr, qr_scale, kv = self._split_qkv_and_norm(qr_kv) + q = self._wq_b_proj(qr, qr_scale).view( + -1, self.n_local_heads, self.head_dim + ) + return ( + self._fused_qnorm_rope_kv_insert(q, kv, positions, attn_metadata), + qr, + qr_scale, + kv, + ) + + def compressor_chain(): + kv_score = torch.mm( + hidden_states, + compressor.fused_wkv_wgate.weight.T, + out_dtype=torch.float32, + ) + latent = compressor(kv_score, positions) + compressor.insert_cache(latent, positions, self.rotary_emb) + return latent + + def indexer_weights_chain(): + # ReplicatedLinear returns (output, bias); bias is None. + weights, _ = indexer.weights_proj(hidden_states) + return weights + + (q, qr, qr_scale, kv), (latent, indexer_weights) = execute_in_parallel( + default_chain, + [compressor_chain, indexer_weights_chain], + self.ln_events[0], + self.ln_events[1:3], + aux_streams[:2], + enable=True, + default_first=True, + ) + + indexer._produce_k(latent, positions, self.indexer_rotary_emb) + index_q, index_q_scale, index_weights_out = indexer.forward_q( + qr, qr_scale, indexer_weights, positions, self.indexer_rotary_emb + ) + return q, kv, index_q, index_q_scale, index_weights_out + + def _forward_csa2_reindex( + self, hidden_states: torch.Tensor, positions: torch.Tensor + ) -> tuple[ + torch.Tensor, + torch.Tensor, + torch.Tensor | None, + torch.Tensor | None, + torch.Tensor | None, + ]: + """Indexer-only layers: the q-side overlaps the SWA q path. + + The input projections run serial first: the q-side consumes qr, so + it can only overlap the SWA q projection and KV insert. + """ + indexer = self.indexer + assert indexer is not None + attn_metadata = get_forward_context().attn_metadata + # Serial on these layers via the _run_parallel_input_projections + # override: qr must sit on the default stream before the fork. + qr_kv, _, indexer_weights = self._run_parallel_input_projections(hidden_states) + assert indexer_weights is not None + qr, qr_scale, kv = self._split_qkv_and_norm(qr_kv) + + def swa_q_chain(): + q = self._wq_b_proj(qr, qr_scale).view( + -1, self.n_local_heads, self.head_dim + ) + return self._fused_qnorm_rope_kv_insert(q, kv, positions, attn_metadata) + + def indexer_q_chain(): + return indexer.forward_q( + qr, qr_scale, indexer_weights, positions, self.indexer_rotary_emb + ) + + aux_streams = self.aux_stream_list + assert aux_streams is not None + q, aux_results = execute_in_parallel( + swa_q_chain, + [indexer_q_chain], + self.ln_events[0], + self.ln_events[1:2], + aux_streams[:1], + enable=True, + default_first=True, + ) + index_q, index_q_scale, index_weights_out = aux_results[0] + return q, kv, index_q, index_q_scale, index_weights_out + @functools.cached_property def _wq_b_uses_aiter_block_scaled(self) -> bool: """True when both wq_b GEMMs run the aiter block-scaled fp8 kernel. diff --git a/vllm/models/deepseek_v41/attention.py b/vllm/models/deepseek_v41/attention.py index d0e69fb358..c27a8d9baf 100644 --- a/vllm/models/deepseek_v41/attention.py +++ b/vllm/models/deepseek_v41/attention.py @@ -478,7 +478,6 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): # graph and MRV1 produces garbage (#51430). self._prepare_and_attn_fn = self._prepare_and_attn_eager - # Will be None on ROCm for now. self.aux_stream_list = aux_stream_list # [0]: GEMM start / post-GEMM event0. [1..3]: GEMM done events; # [1] doubles as post-GEMM event1. Reuse is safe: GEMM fully joins @@ -697,7 +696,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): else: gemm_rs.warn_incompatible_projection() - def _wo_b_proj(self, z: torch.Tensor) -> torch.Tensor: + def _wo_b_proj(self, z: torch.Tensor | QuantizedActivation) -> torch.Tensor: """Apply ``wo_b``; with GEMM-RS bound, also reduce-scatter the result. Every ``_o_proj`` implementation projects through this so the decoder @@ -707,7 +706,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): """ if self.gemm_rs is None: return self.wo_b(z) - if self.gemm_rs.should_run(z): + if isinstance(z, torch.Tensor) and self.gemm_rs.should_run(z): return self.gemm_rs.apply(z, self.wo_b) # Small batches stay on the unfused path, which is faster there. return sp_reduce_scatter(self.wo_b(z)) @@ -1381,6 +1380,37 @@ class DeepseekV4Indexer(nn.Module): self.use_fp4_kv, ) + def forward_q( + self, + qr: torch.Tensor | QuantizedActivation, + qr_scale: torch.Tensor | None, + indexer_weights: torch.Tensor, + positions: torch.Tensor, + rotary_emb: nn.Module, + ) -> tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]: + """Build the indexer queries: wq_b over qr plus fused RoPE/quant. + + Split out so the ROCm layer can schedule it apart from + ``_produce_k`` (e.g. on an aux stream once ``qr`` is ready). + """ + q = self._wq_b_proj(qr, qr_scale) + q = q.view(-1, self.n_head, self.head_dim) + q_quant, weights = fused_indexer_q_rope_quant( + positions, + q, + rotary_emb.cos_sin_cache, + indexer_weights, + self.softmax_scale, + self.n_head**-0.5, + use_fp4=self.use_fp4_kv, + weights_out_dtype=self.indexer_weights_dtype, + ) + if isinstance(q_quant, tuple): + q, q_scale = q_quant + else: + q, q_scale = q_quant, None + return q, q_scale, weights + def forward( self, qr: torch.Tensor | QuantizedActivation, @@ -1422,23 +1452,7 @@ class DeepseekV4Indexer(nn.Module): # (skip_k_cache_insert=True). self._produce_k(latent, positions, rotary_emb) - q = self._wq_b_proj(qr, qr_scale) - q = q.view(-1, self.n_head, self.head_dim) - q_quant, weights = fused_indexer_q_rope_quant( - positions, - q, - rotary_emb.cos_sin_cache, - indexer_weights, - self.softmax_scale, - self.n_head**-0.5, - use_fp4=self.use_fp4_kv, - weights_out_dtype=self.indexer_weights_dtype, - ) - if isinstance(q_quant, tuple): - q, q_scale = q_quant - else: - q, q_scale = q_quant, None - return q, q_scale, weights + return self.forward_q(qr, qr_scale, indexer_weights, positions, rotary_emb) def _wq_b_proj( self, diff --git a/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py b/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py index 423849dcf1..0e02b486a5 100644 --- a/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py +++ b/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py @@ -9,15 +9,16 @@ import torch from vllm.config import VllmConfig from vllm.config.cache import CacheDType from vllm.forward_context import get_forward_context -from vllm.models.deepseek_v4.nvidia.ops.o_proj import ( - compute_fp8_einsum_recipe, - deep_gemm_fp8_o_proj, -) +from vllm.models.deepseek_v4.nvidia.ops.o_proj import compute_fp8_einsum_recipe from vllm.models.deepseek_v41.attention import DeepseekV4Attention from vllm.models.deepseek_v41.common.ops import ( build_flashinfer_mixed_sparse_indices, compute_global_topk_indices_and_lens, ) +from vllm.models.deepseek_v41.nvidia.ops.o_proj import ( + dsv41_o_proj, + register_dsv41_o_proj_warmup, +) from vllm.models.deepseek_v41.sparse_mla import ( DeepseekV4FlashMLAMetadata, DeepseekV4SparseMLABackend, @@ -242,27 +243,14 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention): return _pad_to_supported_q_heads(num_heads) def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: - o = attn_out[:, : self.n_local_heads, :] - return deep_gemm_fp8_o_proj( - o, - positions, - self.rotary_emb.cos_sin_cache, - self.wo_a, - self._wo_b_proj, - n_groups=self.n_local_groups, - heads_per_group=self.n_local_heads // self.n_local_groups, - nope_dim=self.nope_head_dim, - rope_dim=self.rope_head_dim, - o_lora_rank=self.o_lora_rank, - einsum_recipe=self._einsum_recipe, - tma_aligned_scales=self._tma_aligned_scales, - ) + return dsv41_o_proj(self, attn_out, positions) def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self._einsum_recipe, self._tma_aligned_scales = compute_fp8_einsum_recipe( self._o_proj_block_size ) + register_dsv41_o_proj_warmup(self) # Per-tensor FP8 scale buffers + precomputed scalar BMM scales. Only the # per-tensor FP8 cache path consumes these; bf16 reads ``self.scale``. if self.kv_cache_torch_dtype != torch.float8_e4m3fn: @@ -623,21 +611,7 @@ class DeepseekV4FlashInferSM120Attention(DeepseekV4Attention): return _pad_to_supported_q_heads(num_heads) def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: - o = attn_out[:, : self.n_local_heads, :] - return deep_gemm_fp8_o_proj( - o, - positions, - self.rotary_emb.cos_sin_cache, - self.wo_a, - self._wo_b_proj, - n_groups=self.n_local_groups, - heads_per_group=self.n_local_heads // self.n_local_groups, - nope_dim=self.nope_head_dim, - rope_dim=self.rope_head_dim, - o_lora_rank=self.o_lora_rank, - einsum_recipe=self._einsum_recipe, - tma_aligned_scales=self._tma_aligned_scales, - ) + return dsv41_o_proj(self, attn_out, positions) def __init__(self, vllm_config: VllmConfig, *args, **kwargs) -> None: super().__init__(vllm_config, *args, **kwargs) diff --git a/vllm/models/deepseek_v41/nvidia/flashmla.py b/vllm/models/deepseek_v41/nvidia/flashmla.py index 96dde36c61..ce3047ceca 100644 --- a/vllm/models/deepseek_v41/nvidia/flashmla.py +++ b/vllm/models/deepseek_v41/nvidia/flashmla.py @@ -6,16 +6,17 @@ from typing import TYPE_CHECKING, ClassVar, cast import torch from vllm.forward_context import get_forward_context -from vllm.models.deepseek_v4.nvidia.ops.o_proj import ( - compute_fp8_einsum_recipe, - deep_gemm_fp8_o_proj, -) +from vllm.models.deepseek_v4.nvidia.ops.o_proj import compute_fp8_einsum_recipe from vllm.models.deepseek_v41.attention import DeepseekV4Attention from vllm.models.deepseek_v41.common.ops import ( combine_topk_swa_indices, compute_global_topk_indices_and_lens, dequantize_and_gather_k_cache, ) +from vllm.models.deepseek_v41.nvidia.ops.o_proj import ( + dsv41_o_proj, + register_dsv41_o_proj_warmup, +) from vllm.models.deepseek_v41.sparse_mla import ( DeepseekV4FlashMLABackend, DeepseekV4FlashMLAMetadata, @@ -57,23 +58,10 @@ class DeepseekV4FlashMLAAttention(DeepseekV4Attention): self._einsum_recipe, self._tma_aligned_scales = compute_fp8_einsum_recipe( self._o_proj_block_size ) + register_dsv41_o_proj_warmup(self) def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: - o = attn_out[:, : self.n_local_heads, :] - return deep_gemm_fp8_o_proj( - o, - positions, - self.rotary_emb.cos_sin_cache, - self.wo_a, - self._wo_b_proj, - n_groups=self.n_local_groups, - heads_per_group=self.n_local_heads // self.n_local_groups, - nope_dim=self.nope_head_dim, - rope_dim=self.rope_head_dim, - o_lora_rank=self.o_lora_rank, - einsum_recipe=self._einsum_recipe, - tma_aligned_scales=self._tma_aligned_scales, - ) + return dsv41_o_proj(self, attn_out, positions) @classmethod def get_padded_num_q_heads(cls, num_heads: int) -> int: diff --git a/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py b/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py new file mode 100644 index 0000000000..b195d4b849 --- /dev/null +++ b/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py @@ -0,0 +1,599 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""DSV4.1 small-batch MXFP8 WO-A with inverse RoPE and quantization.""" + +from dataclasses import dataclass +from typing import Any + +import cutlass +import cutlass.cute as cute +import cutlass.utils as utils +import torch +from cuda.bindings.driver import CUstream +from cutlass import ( + BFloat16, + Float8E4M3FN, + Float32, + Int32, + Int64, + Uint8, + Uint32, + Uint64, +) +from cutlass._mlir.dialects import llvm +from cutlass.cute.nvgpu import cpasync, tcgen05 +from cutlass.cute.runtime import make_fake_tensor +from cutlass.cutlass_dsl import dsl_user_op + +from vllm.cute_utils import _tcgen05, mbarrier, recast_val, simple_tma_copy +from vllm.model_executor.warmup.jit_warmup import WarmupIntRange, kernel_launcher +from vllm.model_executor.warmup.jit_warmup_cutedsl_helper import ( + CuTeDSLLaunchSpec, + VllmCuTeDSLJitKernel, +) + +# Each head is [nope | rope] as in fused_inv_rope_fp8_quant: only the last +# _ROPE_DIM columns carry RoPE. +_HEAD_DIM = 512 +_ROPE_DIM = 64 + + +@dsl_user_op +def _ue8m0_rp(value, *, loc=None, ip=None): + result = llvm.inline_asm( + Uint32.mlir_type, + [value.ir_value(loc=loc, ip=ip)], + "{ .reg .b16 sf; cvt.rp.satfinite.ue8m0x2.f32 sf, $1, $1; " + "cvt.u32.u16 $0, sf; and.b32 $0, $0, 255; }", + "=r,f", + has_side_effects=False, + loc=loc, + ip=ip, + ) + return Uint32(result) + + +@dsl_user_op +def _rope_fma(value, cos, other, sin, *, subtract=False, loc=None, ip=None): + """Preserve multiply/FMA rounding when RoPE loads are hoisted.""" + result = llvm.inline_asm( + Float32.mlir_type, + [v.ir_value(loc=loc, ip=ip) for v in (value, cos, other, sin)], + "{ .reg .f32 product; mul.rn.f32 product, $3, $4; " + + ( + "neg.f32 product, product; fma.rn.f32 $0, $1, $2, product; }" + if subtract + else "fma.rn.f32 $0, $1, $2, product; }" + ), + "=f,f,f,f,f", + has_side_effects=False, + loc=loc, + ip=ip, + ) + return Float32(result) + + +@cute.jit +def _scale(amax: Float32): + exponent = _ue8m0_rp(amax * Float32(1.0 / 448.0)) + inv = recast_val((Uint32(254) - exponent) << 23, Float32) + if exponent == 0: + inv = Float32(0) + return exponent, inv + + +class FusedWoAKernel(VllmCuTeDSLJitKernel["FusedWoAKernel.CompileKey"]): + @dataclass(frozen=True) + class CompileKey: + # T=1..96 uses at most 46 keys per (n_groups, heads_per_group, o_lora_rank). + tokens_per_tile: int + token_tiles: int + full_tiles: bool + n_groups: int + heads_per_group: int + o_lora_rank: int + + @staticmethod + def kernel(compile_key: CompileKey) -> Any: + tokens = compile_key.tokens_per_tile + token_tiles = compile_key.token_tiles + full_tiles = compile_key.full_tiles + acc_cols = 1 << (tokens - 1).bit_length() + tile_m = max(8, acc_cols) + tile_n = 128 + tile_k = 128 + # tcgen05 MXFP8 MMA consumes K = 32, one MX block, per instruction. + mma_k = 32 + # Each group's heads split K across one cluster, one head per CTA. + heads_per_group = compile_key.heads_per_group + tiles_per_group = compile_key.o_lora_rank // tile_n + n_tiles = compile_key.n_groups * tiles_per_group + # Each CTA keeps one whole head of K in SMEM, one TMA box per stage. + num_stages = _HEAD_DIM // tile_k + w_stage_bytes = tile_n * tile_k + nope_dim = _HEAD_DIM - _ROPE_DIM + threads_per_token = _HEAD_DIM // 4 + sf_col = max(16, tile_m) + tmem_cols = sf_col * 2 + threads = min(acc_cols * threads_per_token, 512) + tokens_per_iter = threads // threads_per_token + + @cute.kernel + def device_kernel( + x: cute.Tensor, + positions: cute.Tensor, + rope: cute.Tensor, + w: cpasync.TmaInfo, + ws: cpasync.TmaInfo, + q: cute.Tensor, + qs: cute.Tensor, + num_tokens: Int32, + ): + total_tokens = num_tokens + if cutlass.const_expr(full_tiles): + total_tokens = Int32(tokens * token_tiles) + tid, _, _ = cute.arch.thread_idx() + bid, token_tile, _ = cute.arch.block_idx() + token_start = Int32(0) + if cutlass.const_expr(token_tiles > 1): + token_start = token_tile * tokens + warp = cute.arch.make_warp_uniform(tid // 32) + lane = tid % 32 + split = bid % heads_per_group + tile = bid // heads_per_group + group = tile // tiles_per_group + smem = utils.SmemAllocator() + sW = smem.allocate_tensor( + Float8E4M3FN, + w.smem_layout.outer, + byte_alignment=128, + swizzle=w.smem_layout.inner, + ) + sX = smem.allocate_tensor( + Float8E4M3FN, + cute.make_layout( + (tile_m, tile_k, num_stages), stride=(tile_k, 1, tile_m * tile_k) + ), + byte_alignment=1024, + swizzle=cute.make_swizzle(3, 4, 3), + ) + # tcgen05.cp 32x128b.warpx4 order: row r at byte (r % 32) * 16 + + # (r // 32) * 4, one 4-byte cell of MX-block scales per row. + sW_SF = smem.allocate_tensor( + Int32, + cute.make_layout(((32, 4), num_stages), stride=((4, 1), tile_n)), + 128, + ) + sW_SF_raw = smem.allocate_tensor( + Int32, cute.make_layout((tile_n, num_stages)), 128 + ) + sX_SF = smem.allocate_tensor( + Uint8, + cute.make_layout( + ((32, 4), tile_k // mma_k, num_stages), + stride=((16, 4), 1, tile_n * 4), + ), + 128, + ) + copy_fp8x4 = cute.make_copy_atom( + cute.nvgpu.CopyUniversalOp(), Float8E4M3FN, num_bits_per_copy=32 + ) + partial = smem.allocate_tensor( + Float32, + cute.make_layout( + (tile_n, cute.ceil_div(tokens, heads_per_group), heads_per_group) + ), + 128, + ) + mbar_loaded = smem.allocate_array(Int64, num_stages) + mbar_ws_loaded = smem.allocate_array(Int64, 1) + mbar_done = smem.allocate_array(Int64, 1) + mbar_reduced = smem.allocate_array(Int64, 1) + taddr = smem.allocate(Int32, 4) + if tid == 0: + for stage in cutlass.range_constexpr(num_stages): + # Wait for weight TMA and this stage's quantization threads. + cute.arch.mbarrier_init(mbar_loaded + stage, 1 + threads // 4) + cute.arch.mbarrier_init(mbar_ws_loaded, 1) + cute.arch.mbarrier_init(mbar_done, 1) + cute.arch.mbarrier_init(mbar_reduced, 1) + cute.arch.mbarrier_init_fence() + # Local users of the barriers (warp 0's TMA below) need the init too. + cute.arch.sync_threads() + # Paired with cluster_wait() below: peers must see mbar_reduced + # initialized before their st.async; the split hides the barrier. + cute.arch.cluster_arrive_relaxed() + if warp == 0: + # Raw DeepGEMM MN-major scales; the MMA warp transposes them to + # the UTCCP order, which TMA cannot scatter at 4-byte granularity. + ws_tiles = cute.zipped_divide(ws.tma_tensor, (tile_n, num_stages)) + with cute.arch.elect_one(): + mbarrier.arrive_expect_tx(mbar_ws_loaded, tile_n * num_stages * 4) + simple_tma_copy( + ws.atom, + ws_tiles[None, (tile % tiles_per_group, split, group)], + sW_SF_raw, + mbar_ws_loaded, + ) + tiles = cute.zipped_divide(w.tma_tensor, (tile_n, tile_k)) + for stage in cutlass.range_constexpr(num_stages): + with cute.arch.elect_one(): + mbarrier.arrive_expect_tx(mbar_loaded + stage, w_stage_bytes) + simple_tma_copy( + w.atom, + tiles[None, (tile, split * num_stages + stage)], + sW[None, None, stage], + mbar_loaded + stage, + ) + # x and positions come from the PDL predecessor; everything above + # only touches weights. Padded sX rows only feed unread MMA columns. + cute.arch.griddepcontrol_wait() + # TMEM is allocated at runtime, and a PDL predecessor on this SM may + # hold it until it exits; allocating earlier can deadlock. + if warp == 0: + cute.arch.alloc_tmem(tmem_cols, taddr) + cute.arch.relinquish_tmem_alloc_permit() + copy_x = cute.make_copy_atom( + cute.nvgpu.CopyUniversalOp(), BFloat16, num_bits_per_copy=64 + ) + n_iters = cute.ceil_div(tokens, tokens_per_iter) + qid = (tid + 32) % threads_per_token + k = qid * 4 + # Tail rows repeat the last input; their output stores are masked. + # Issue every token's x load before any math so their latencies overlap. + x_bf16 = cute.make_rmem_tensor((4, n_iters), BFloat16) + for iteration in cutlass.range_constexpr(n_iters): + token = iteration * tokens_per_iter + tid // threads_per_token + if cutlass.const_expr(tokens % tokens_per_iter == 0) or token < tokens: + x_src = cute.local_tile( + x[ + cute.min(token_start + token, total_tokens - 1), + group * heads_per_group + split, + None, + ], + (4,), + (qid,), + ) + cute.copy(copy_x, x_src, x_bf16[None, iteration]) + # Overlap the RoPE loads across tokens before consuming any of them. + cos_reg = cute.make_rmem_tensor((2, n_iters), Float32) + sin_reg = cute.make_rmem_tensor((2, n_iters), Float32) + if cutlass.const_expr(n_iters > 1): + copy_rope = cute.make_copy_atom( + cute.nvgpu.CopyUniversalOp(), Float32, num_bits_per_copy=32 + ) + if k >= nope_dim: + for iteration in cutlass.range_constexpr(n_iters): + token = iteration * tokens_per_iter + tid // threads_per_token + if ( + cutlass.const_expr(tokens % tokens_per_iter == 0) + or token < tokens + ): + src_token = cute.min(token_start + token, total_tokens - 1) + pos = positions[src_token] + freq = (k - nope_dim) // 2 + rope_row = rope[pos, None] + cos_src = cute.local_tile(rope_row, (2,), (freq // 2,)) + cute.copy(copy_rope, cos_src, cos_reg[None, iteration]) + sin_tile = (freq + _ROPE_DIM // 2) // 2 + sin_src = cute.local_tile(rope_row, (2,), (sin_tile,)) + cute.copy(copy_rope, sin_src, sin_reg[None, iteration]) + for iteration in cutlass.range_constexpr(n_iters): + token = iteration * tokens_per_iter + tid // threads_per_token + if cutlass.const_expr(tokens % tokens_per_iter == 0) or token < tokens: + x_f32 = cute.make_rmem_tensor((4,), Float32) + x_f32.store(x_bf16[None, iteration].load().to(Float32)) + if k >= nope_dim: + # Inverse RoPE on the interleaved pairs (k, k+1) and + # (k+2, k+3); each rope row is cos || sin. + for pair in cutlass.range_constexpr(2): + if cutlass.const_expr(n_iters > 1): + cos = cos_reg[pair, iteration] + sin = sin_reg[pair, iteration] + else: + pos = positions[ + cute.min(token_start + token, total_tokens - 1) + ] + freq = (k - nope_dim) // 2 + pair + cos = Float32(rope[pos, freq]) + sin = Float32(rope[pos, freq + _ROPE_DIM // 2]) + even, odd = x_f32[2 * pair], x_f32[2 * pair + 1] + x_f32[2 * pair] = _rope_fma(even, cos, odd, sin) + x_f32[2 * pair + 1] = _rope_fma( + odd, cos, even, sin, subtract=True + ) + amax = cute.arch.fmax( + cute.arch.fmax(cute.abs(x_f32[0]), cute.abs(x_f32[1])), + cute.arch.fmax(cute.abs(x_f32[2]), cute.abs(x_f32[3])), + ) + amax = cute.arch.warp_reduction_max(amax, threads_in_group=8) + exponent, inv = _scale(cute.arch.fmax(amax, Float32(1e-10))) + x_fp8 = cute.make_rmem_tensor((4,), Float8E4M3FN) + x_fp8.store((x_f32.load() * inv).to(Float8E4M3FN)) + cute.copy( + copy_fp8x4, + x_fp8, + cute.local_tile( + sX[token, None, k // tile_k], (4,), (qid % 32,) + ), + ) + if lane % 8 == 0: + sX_SF[token, k % tile_k // mma_k, k // tile_k] = Uint8(exponent) + cute.arch.fence_proxy("async.shared", space="cta") + mbarrier.arrive(mbar_loaded + qid // 32, order="release") + if warp == 0: + base = cute.make_tensor(taddr, cute.make_layout(1))[0] + # sW / sX are 128B-swizzled K-major; one stage's K fills the + # swizzle atom, so the leading byte offset is unused. + sdesc = _tcgen05.make_sdesc_128B_swizzle(LBO=0) + # Unswizzled 32 x 16 B scale tiles: SBO = one 8-row core matrix + # (16 B units at bit 32); bit 46 is the SM100 descriptor version. + sf_sbo = 8 * 16 + sfdesc = Uint64((sf_sbo >> 4 << 32) | (1 << 46)) + # M = tile_n weight rows (A), N = tile_m token rows (B). + idesc = _tcgen05.make_mxfp8_idesc(tile_n, tile_m) + # sW_SF's layout is the UTCCP order, so a plain copy transposes; + # fence the generic-proxy writes before tcgen05.cp reads them. + cute.arch.mbarrier_wait(mbar_ws_loaded, 0) + for stage in cutlass.range_constexpr(num_stages): + for i in cutlass.range_constexpr(4): + sW_SF[i * 32 + lane, stage] = sW_SF_raw[i * 32 + lane, stage] + cute.arch.fence_proxy("async.shared", space="cta") + cute.arch.sync_warp() + for stage in cutlass.range_constexpr(num_stages): + cute.arch.mbarrier_wait(mbar_loaded + stage, 0) + _tcgen05.fence_after_thread_sync() + if cutlass.const_expr(stage == num_stages - 1): + with cute.arch.elect_one(): + cute.arch.griddepcontrol_launch_dependents() + adesc = sdesc | (sW[None, None, stage].iterator.toint() >> 4) + bdesc = sdesc | (sX[None, None, stage].iterator.toint() >> 4) + _tcgen05.cp( + base + sf_col, + sfdesc | (sW_SF[None, stage].iterator.toint() >> 4), + "32x128b", + "warpx4", + ) + _tcgen05.cp( + base + sf_col + 4, + sfdesc | (sX_SF[None, None, stage].iterator.toint() >> 4), + "32x128b", + "warpx4", + ) + # Advance mma_k FP8 bytes (16 B units) and pick MX block kk of + # each 4-byte scale (B sf_id at bit 4, A sf_id at bit 29). + for kk in cutlass.range_constexpr(tile_k // mma_k): + _tcgen05.mma_mxfp8( + base, + adesc + kk * mma_k // 16, + bdesc + kk * mma_k // 16, + idesc + ((kk << 4) | (kk << 29)), + base + sf_col, + base + sf_col + 4, + stage > 0 or kk > 0, + ) + _tcgen05.commit(mbar_done) + # Only the MMA warp waits; barrier 2 releases the TMEM readers. + cute.arch.mbarrier_wait(mbar_done, 0) + if tid < tile_n: + cute.arch.barrier(barrier_id=2, number_of_threads=tile_n) + # Tokens t with t % heads_per_group == split arrive from every peer + # (this CTA included), tile_n Float32 rows each. + if tid == 0: + owned = cute.ceil_div(tokens - split, heads_per_group) + mbarrier.arrive_expect_tx( + mbar_reduced, owned * tile_n * heads_per_group * 4 + ) + # Pairs with cluster_arrive_relaxed() above: every peer's + # mbar_reduced is initialized before the st.async below. + cute.arch.cluster_wait() + # quack's process-wide const_expr-if rewrite can leave these unbound + # after the dynamic path above; bind them so the regions join. + amax, exponent, inv = Float32(0), Uint32(0), Float32(0) + if tid < tile_n: + base = cute.make_tensor(taddr, cute.make_layout(1))[0] + _tcgen05.fence_after_thread_sync() + acc = cute.make_rmem_tensor(acc_cols, Float32) + if cutlass.const_expr(tokens == 1): + acc[0] = _tcgen05.ld(warp * 32, base, "32x32b", 1) + else: + acc.store(_tcgen05.ld(warp * 32, base, "32x32b", acc_cols)) + _tcgen05.wait_ld() + for token in cutlass.range_constexpr(tokens): + ptr = cute.domain_offset( + (tid, token // heads_per_group, split), partial + ).iterator + cute.arch.store_async_dsmem( + ptr, + recast_val(acc[token], Int32), + mbar_reduced, + token % heads_per_group, + ) + if split < tokens: + # One warp waits for the peers' bytes; named barrier 1 (0 is + # sync_threads) releases the other tile_n epilogue threads. + if warp == 0: + cute.arch.mbarrier_wait(mbar_reduced, 0) + cute.arch.barrier(barrier_id=1, number_of_threads=tile_n) + for i in cutlass.range_constexpr( + cute.ceil_div(tokens, heads_per_group) + ): + token = split + i * heads_per_group + if cutlass.const_expr( + full_tiles + and ( + tokens <= heads_per_group + or tokens % heads_per_group == 0 + ) + ) or (token < tokens and token_start + token < total_tokens): + value = Float32(0) + for peer in cutlass.range_constexpr(heads_per_group): + value += partial[tid, i, peer] + value = Float32(BFloat16(value)) + amax = cute.arch.warp_redux_sync(value, "max", abs=True) + exponent, inv = _scale(amax) + q[token_start + token, tile * tile_n + tid] = Float8E4M3FN( + value * inv + ) + if lane == 0: + row = token_start + token + offset = row % 32 * 16 + row // 32 * 4 + warp + qs[tile * 512 + offset] = Uint8(exponent) + # Every scale tile owns its padding, with no second memset kernel. + for i in cutlass.range_constexpr(4): + offset = i * tile_n + tid + row = offset // 16 + (offset % 16 // 4) * 32 + if token_tile == 0 and split == 0 and row >= total_tokens: + qs[tile * 512 + offset] = Uint8(0) + # Each receiver drained all writes into its own inbox. Only local + # TMEM readers must finish before deallocation; no peer reads SMEM. + if tid < tile_n: + cute.arch.barrier(barrier_id=2, number_of_threads=tile_n) + if warp == 0: + cute.arch.dealloc_tmem( + cute.make_ptr( + Float32, + cute.make_tensor(taddr, cute.make_layout(1))[0], + cute.AddressSpace.tmem, + ), + tmem_cols, + ) + + @cute.jit + def host_entrypoint( + x: cute.Tensor, + positions: cute.Tensor, + rope: cute.Tensor, + w: cute.Tensor, + ws: cute.Tensor, + q: cute.Tensor, + qs: cute.Tensor, + num_tokens: Int32, + stream: CUstream, + ): + layout = cute.make_composed_layout( + cute.make_swizzle(3, 4, 3), + 0, + cute.make_layout( + (tile_n, tile_k, num_stages), stride=(tile_k, 1, w_stage_bytes) + ), + ) + tma = cpasync.make_tiled_tma_atom( + cpasync.CopyBulkTensorTileG2SOp(cta_group=tcgen05.CtaGroup.ONE), + w, + layout, + (tile_n, tile_k), + ) + ws_tma = cpasync.make_tiled_tma_atom( + cpasync.CopyBulkTensorTileG2SOp(cta_group=tcgen05.CtaGroup.ONE), + cute.make_tensor(ws.iterator, cute.select(ws.layout, mode=[1, 2, 0])), + cute.make_layout((tile_n, num_stages)), + (tile_n, num_stages), + ) + device_kernel(x, positions, rope, tma, ws_tma, q, qs, num_tokens).launch( + grid=(n_tiles * heads_per_group, token_tiles, 1), + block=(threads, 1, 1), + cluster=(heads_per_group, 1, 1), + stream=stream, + use_pdl=True, + ) + + return host_entrypoint + + def dispatch( # type: ignore[override] + self, *, tokens: int, n_groups: int, heads_per_group: int, o_lora_rank: int + ) -> CompileKey: + token_tiles = (tokens + 31) // 32 + # Host integer arithmetic: balance tiles in multiples of four tokens. + tile_tokens = ( + tokens + if token_tiles == 1 + else (tokens + 4 * token_tiles - 1) // (4 * token_tiles) * 4 + ) + return self.CompileKey( + tokens_per_tile=tile_tokens, + token_tiles=token_tiles, + full_tiles=tokens % tile_tokens == 0, + n_groups=n_groups, + heads_per_group=heads_per_group, + o_lora_rank=o_lora_rank, + ) + + def get_warmup_keys( + self, *, max_tokens: int, n_groups: int, heads_per_group: int, o_lora_rank: int + ) -> list[CompileKey]: + # Trace all accepted token counts and deduplicate shared tile shapes. + return self._trace_dispatch(self.dispatch)( + tokens=WarmupIntRange(1, max_tokens + 1), + n_groups=n_groups, + heads_per_group=heads_per_group, + o_lora_rank=o_lora_rank, + ) + + def warmup_inputs(self, compile_key: CompileKey) -> tuple[Any, ...]: + tokens = ( + compile_key.tokens_per_tile * compile_key.token_tiles + if compile_key.full_tiles + else cute.sym_int() + ) + groups = compile_key.n_groups + rank = compile_key.o_lora_rank + n = groups * rank + k = compile_key.heads_per_group * _HEAD_DIM + return ( + make_fake_tensor( + BFloat16, + (tokens, groups * compile_key.heads_per_group, _HEAD_DIM), + (cute.sym_int(divisibility=8), _HEAD_DIM, 1), + assumed_align=16, + ), + make_fake_tensor(Int64, (tokens,), (1,)), + make_fake_tensor(Float32, (cute.sym_int(), _ROPE_DIM), (_ROPE_DIM, 1)), + make_fake_tensor(Float8E4M3FN, (n, k), (k, 1), assumed_align=16), + make_fake_tensor( + Int32, + (groups, rank, k // 128), + (rank * k // 128, 1, rank), + assumed_align=16, + ), + make_fake_tensor(Float8E4M3FN, (tokens, n), (n, 1)), + make_fake_tensor(Uint8, (128 * n // 32,), (1,)), + Int32(0), + ) + + @kernel_launcher + def __call__( + self, + *, + x: torch.Tensor, + positions: torch.Tensor, + rope: torch.Tensor, + weight: torch.Tensor, + weight_scale: torch.Tensor, + ) -> CuTeDSLLaunchSpec["FusedWoAKernel.CompileKey"]: + """Return FP8 WO-A output and FlashInfer F8_128x4 scale bytes.""" + tokens = x.shape[0] + groups, rank = weight_scale.shape[:2] + n = groups * rank + q = torch.empty((tokens, n), device=x.device, dtype=torch.float8_e4m3fn) + scales = torch.empty(128 * n // 32, device=x.device, dtype=torch.uint8) + launch_args = ( + x, + positions, + rope, + weight.view(n, -1), + weight_scale, + q, + scales, + tokens, + ) + compile_key = self.dispatch( + tokens=tokens, + n_groups=groups, + heads_per_group=x.shape[1] // groups, + o_lora_rank=rank, + ) + return compile_key, launch_args, (q, scales) + + +_FUSED_WO_A_KERNEL = FusedWoAKernel() diff --git a/vllm/models/deepseek_v41/nvidia/ops/o_proj.py b/vllm/models/deepseek_v41/nvidia/ops/o_proj.py new file mode 100644 index 0000000000..4a315ddc9b --- /dev/null +++ b/vllm/models/deepseek_v41/nvidia/ops/o_proj.py @@ -0,0 +1,95 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""DSV4.1 output projection with a small-batch SM100/SM103 fusion.""" + +import torch +from torch import nn + +from vllm.model_executor.kernels.linear.mxfp8.flashinfer import ( + FlashInferCutedslMxfp8LinearKernel, +) +from vllm.model_executor.layers.fusion.quant_activation import ( + QuantizedActivation, + get_input_quant_key, +) +from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Dynamic +from vllm.models.deepseek_v4.nvidia.ops.o_proj import deep_gemm_fp8_o_proj +from vllm.platforms import current_platform + + +def _fused_wo_a_max_tokens(layer: nn.Module) -> int: + max_tokens = getattr(layer, "_fused_wo_a_token_limit", None) + if max_tokens is None: + # Additional token tiles only pay off for smaller grids. + ctas_per_tile = layer.n_local_heads * (layer.o_lora_rank // 128) + max_tokens = 96 if ctas_per_tile <= 128 else 32 + layer._fused_wo_a_token_limit = max_tokens + return max_tokens + + +def _can_fuse_wo_a(layer: nn.Module) -> bool: + """Whether the attention layer matches the fused WO-A kernel's layout.""" + method = getattr(layer.wo_b, "scheme", layer.wo_b.quant_method) + return ( + current_platform.is_device_capability_family(100) + # WO-A is FP8 with per-32 scales, and WO-B takes MXFP8 input directly. + and layer._einsum_recipe == (1, 1, 32) + and get_input_quant_key(layer.wo_b) == kMxfp8Dynamic + and isinstance( + getattr(method, "kernel", None), FlashInferCutedslMxfp8LinearKernel + ) + # A group's heads form one portable (<= 8 CTA) cluster. + and layer.n_local_heads // layer.n_local_groups <= 8 + and (layer.nope_head_dim, layer.rope_head_dim) == (448, 64) + and layer.o_lora_rank % 128 == 0 + ) + + +def register_dsv41_o_proj_warmup(layer: nn.Module) -> None: + """Warm the fused WO-A token counts ``dsv41_o_proj`` can dispatch.""" + # Draft models load outside the warmup registry; their layers share the + # target's attention shapes, so the target's registration covers them too. + if _can_fuse_wo_a(layer): + from .fused_wo_a import _FUSED_WO_A_KERNEL + + _FUSED_WO_A_KERNEL.register_warmup( + max_tokens=_fused_wo_a_max_tokens(layer), + n_groups=layer.n_local_groups, + heads_per_group=layer.n_local_heads // layer.n_local_groups, + o_lora_rank=layer.o_lora_rank, + ) + + +def dsv41_o_proj( + layer: nn.Module, attn_out: torch.Tensor, positions: torch.Tensor +) -> torch.Tensor: + """``deep_gemm_fp8_o_proj`` with WO-A fused for small SM100/SM103 batches.""" + o = attn_out[:, : layer.n_local_heads, :] + cos_sin_cache = layer.rotary_emb.cos_sin_cache + if 1 <= o.shape[0] <= _fused_wo_a_max_tokens(layer) and _can_fuse_wo_a(layer): + from .fused_wo_a import _FUSED_WO_A_KERNEL + + q, scales = _FUSED_WO_A_KERNEL( + x=o, + positions=positions, + rope=cos_sin_cache, + weight=layer.wo_a.weight, + weight_scale=layer.wo_a.weight_scale, + ) + return layer._wo_b_proj( + QuantizedActivation(q, scales, o.dtype, q.shape, kMxfp8Dynamic) + ) + return deep_gemm_fp8_o_proj( + o, + positions, + cos_sin_cache, + layer.wo_a, + layer._wo_b_proj, + n_groups=layer.n_local_groups, + heads_per_group=layer.n_local_heads // layer.n_local_groups, + nope_dim=layer.nope_head_dim, + rope_dim=layer.rope_head_dim, + o_lora_rank=layer.o_lora_rank, + einsum_recipe=layer._einsum_recipe, + tma_aligned_scales=layer._tma_aligned_scales, + ) diff --git a/vllm/models/minimax_m3/amd/model.py b/vllm/models/minimax_m3/amd/model.py index 4871e66a9a..40daa4a6ee 100644 --- a/vllm/models/minimax_m3/amd/model.py +++ b/vllm/models/minimax_m3/amd/model.py @@ -106,6 +106,9 @@ from vllm.models.minimax_m3.amd.sparse_attention_msa import ( MiniMaxM3SparseAiterPAImpl, MiniMaxM3SparseAiterPAPrefillMetadata, ) +from vllm.models.minimax_m3.common.encoder_cudagraph import ( + MiniMaxM3EncoderCudaGraphMixin, +) from vllm.models.minimax_m3.common.indexer import MiniMaxM3Indexer from vllm.models.minimax_m3.common.mm_preprocess import ( MiniMaxM3VLDummyInputsBuilder, @@ -1685,7 +1688,11 @@ class MiniMaxM3SparseForCausalLM(nn.Module, SupportsPP, SupportsEagle3): dummy_inputs=MiniMaxM3VLDummyInputsBuilder, ) class MiniMaxM3SparseForConditionalGeneration( - nn.Module, SupportsMultiModal, SupportsPP, SupportsEagle3 + nn.Module, + SupportsMultiModal, + MiniMaxM3EncoderCudaGraphMixin, + SupportsPP, + SupportsEagle3, ): """Top-level (VL) entry point for MiniMax M3. diff --git a/vllm/models/minimax_m3/common/encoder_cudagraph.py b/vllm/models/minimax_m3/common/encoder_cudagraph.py new file mode 100644 index 0000000000..7c55cde946 --- /dev/null +++ b/vllm/models/minimax_m3/common/encoder_cudagraph.py @@ -0,0 +1,281 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Encoder CUDA graph support for the MiniMax M3 vision tower. + +Mixin implementing the SupportsEncoderCudaGraph protocol for any host class +that holds the vision tower as ``self.vision_tower`` (a +``MiniMaxVLVisionModel``). The manager routes image items through this mixin; +video items use the eager encoder path. +""" + +from collections.abc import Hashable +from typing import TYPE_CHECKING, Any + +import torch + +from vllm.model_executor.models.interfaces import SupportsEncoderCudaGraph + +if TYPE_CHECKING: + from vllm.config import VllmConfig + from vllm.config.multimodal import MultiModalConfig + from vllm.models.minimax_m3.common.vision_tower import MiniMaxVLVisionModel + from vllm.v1.worker.encoder_cudagraph_defs import ( + EncoderCudaGraphCaptureInputs, + EncoderCudaGraphConfig, + EncoderCudaGraphReplayBuffers, + EncoderItemSpec, + ) + + +class MiniMaxM3EncoderCudaGraphMixin(SupportsEncoderCudaGraph): + """SupportsEncoderCudaGraph for hosts with ``self.vision_tower``.""" + + vision_tower: "MiniMaxVLVisionModel" + multimodal_config: "MultiModalConfig | None" + _encoder_cg_pad_totals: dict[int, int] = {} + + def get_encoder_cudagraph_config(self) -> "EncoderCudaGraphConfig": + from vllm.v1.attention.backends.registry import AttentionBackendEnum + from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphConfig + + if ( + self.multimodal_config is not None + and self.multimodal_config.get_limit_per_prompt("image") == 0 + ): + raise ValueError( + "cudagraph_mm_encoder requires the MiniMax M3 vision tower, " + "but the image limit is 0. Disable cudagraph_mm_encoder." + ) + + mm_cfg = self.multimodal_config + if ( + mm_cfg is not None + and mm_cfg.mm_encoder_attn_dtype == "fp8" + and mm_cfg.mm_encoder_fp8_scale_path is None + ): + raise ValueError( + "cudagraph_mm_encoder requires static FP8 scales for the " + "MiniMax M3 vision encoder. Set --mm-encoder-fp8-scale-path " + "or disable cudagraph_mm_encoder." + ) + + # FlashInfer reads max_seqlen on the host and TORCH_SDPA calls + # .tolist() on CUDA cu_seqlens; neither can be captured. + backend = self.vision_tower.vision_model.attn_backend + if backend not in ( + AttentionBackendEnum.FLASH_ATTN, + AttentionBackendEnum.ROCM_AITER_FA, + ): + raise ValueError( + f"cudagraph_mm_encoder is not supported with the " + f"{backend.name} ViT attention backend for MiniMax M3. Set " + "--mm-encoder-attn-backend FLASH_ATTN or ROCM_AITER_FA " + "(ROCm), or disable cudagraph_mm_encoder." + ) + + pad_totals = self._encoder_cg_pad_totals + + def pad_cu_seqlens(dst: torch.Tensor, src: torch.Tensor) -> None: + # Varlen attention requires cu_seqlens[-1] to equal the number of + # rows actually passed in. The captured buffers are sized for the + # full token budget, so a smaller real batch has to be completed + # with one trailing padding sequence; declaring fewer rows than the + # buffer holds is undefined behaviour and returns NaN on FlashAttn. + total = pad_totals.get(dst.data_ptr()) + if total is None: + raise RuntimeError("cu_seqlens replay buffer was not registered") + n = src.shape[0] + dst[:n].copy_(src[:n]) + dst[n:] = total + + def pad_rope_planes(dst: torch.Tensor, src: torch.Tensor) -> None: + # cos/sin planes are (3, N, half_rot_dim): the row axis is dim 1, + # not dim 0 as the manager's default slice-copy assumes. + dst.zero_() + dst[:, : src.shape[1]].copy_(src) + + return EncoderCudaGraphConfig( + modalities=["image"], + buffer_keys=[ + "pixel_values", + "rotary_cos", + "rotary_sin", + "cu_seqlens", + "max_seqlen", + ], + out_hidden_size=self.vision_tower.out_hidden_size, + padding_logics={ + "cu_seqlens": pad_cu_seqlens, + "rotary_cos": pad_rope_planes, + "rotary_sin": pad_rope_planes, + }, + ) + + def get_encoder_cudagraph_budget_range( + self, + vllm_config: "VllmConfig", + ) -> tuple[int, int]: + min_budget = 64 + max_budget = min( + vllm_config.scheduler_config.max_num_batched_tokens, + vllm_config.model_config.max_model_len, + ) + return (min_budget, max_budget) + + @staticmethod + def _get_grid_thw_list(mm_kwargs: dict[str, Any]) -> list[list[int]]: + grid_thw = mm_kwargs["image_grid_thw"] + if isinstance(grid_thw, torch.Tensor): + return [[int(x) for x in row] for row in grid_thw.tolist()] + return [[int(x) for x in row] for row in grid_thw] + + @staticmethod + def _get_pixel_values(mm_kwargs: dict[str, Any]) -> torch.Tensor: + pixel_values = mm_kwargs["pixel_values"] + if isinstance(pixel_values, list): + pixel_values = torch.cat(pixel_values) + return pixel_values + + def get_encoder_cudagraph_item_specs( + self, mm_kwargs: dict[str, Any] + ) -> list["EncoderItemSpec"]: + from vllm.v1.worker.encoder_cudagraph_defs import EncoderItemSpec + + merge = self.vision_tower.spatial_merge_size + return [ + EncoderItemSpec( + input_size=t * h * w, + output_tokens=t * h * w // (merge * merge), + ) + for t, h, w in self._get_grid_thw_list(mm_kwargs) + ] + + def select_encoder_cudagraph_items( + self, mm_kwargs: dict[str, Any], indices: list[int] + ) -> dict[str, Any]: + grid_thw_list = self._get_grid_thw_list(mm_kwargs) + pixel_values = self._get_pixel_values(mm_kwargs) + source_grid = mm_kwargs["image_grid_thw"] + + if len(indices) == 0: + empty_grid = ( + source_grid[:0] + if isinstance(source_grid, torch.Tensor) + else pixel_values.new_zeros((0, 3), dtype=torch.long) + ) + return {"pixel_values": pixel_values[:0], "image_grid_thw": empty_grid} + + patch_counts = [t * h * w for t, h, w in grid_thw_list] + cum = [0] + for pc in patch_counts: + cum.append(cum[-1] + pc) + selected_pv = torch.cat( + [pixel_values[cum[i] : cum[i + 1]] for i in indices], dim=0 + ) + grid_device = ( + source_grid.device if isinstance(source_grid, torch.Tensor) else None + ) + selected_grid = torch.tensor( + [grid_thw_list[i] for i in indices], + dtype=torch.long, + device=grid_device, + ) + return {"pixel_values": selected_pv, "image_grid_thw": selected_grid} + + def prepare_encoder_cudagraph_capture_inputs( + self, + token_budget: int, + max_batch_size: int, + max_frames_per_batch: int, + device: torch.device, + dtype: torch.dtype, + path: str = "default", + axis_keys: tuple[Hashable, ...] | None = None, + ) -> "EncoderCudaGraphCaptureInputs": + from vllm.v1.worker.encoder_cudagraph_defs import ( + EncoderCudaGraphCaptureInputs, + ) + + merge = self.vision_tower.spatial_merge_size + # Output tokens per item in the dummy grid (ceiling so total >= budget). + per_item_out = (token_budget + max_batch_size - 1) // max_batch_size + grid_thw_list = [ + [1, merge, per_item_out * merge] for _ in range(max_batch_size) + ] + + embeddings = self.vision_tower.vision_model.embeddings + patch_dim = ( + embeddings.num_channels + * embeddings.temporal_patch_size + * embeddings.patch_size**2 + ) + total_patches = sum(t * h * w for t, h, w in grid_thw_list) + # Match the tower dtype exactly: MiniMaxVLPatchEmbed.forward mutates + # the module on dtype mismatch, which must not happen during capture. + dummy_pixel_values = torch.zeros( + total_patches, + patch_dim, + device=device, + dtype=self.vision_tower.dtype, + ) + + # The padding sequence can contain nearly all captured rows when a + # small image replays this graph. Leave one spare cu_seqlens slot. + metadata = self.vision_tower.vision_model.prepare_encoder_metadata( + grid_thw_list, + device=device, + max_batch_size=max_batch_size + 1, + max_seqlen_override=total_patches, + ) + + values: dict[str, torch.Tensor] = {"pixel_values": dummy_pixel_values} + values.update({k: v for k, v in metadata.items() if v is not None}) + self._encoder_cg_pad_totals[values["cu_seqlens"].data_ptr()] = total_patches + return EncoderCudaGraphCaptureInputs(values=values) + + def prepare_encoder_cudagraph_replay_buffers( + self, + mm_kwargs: dict[str, Any], + max_batch_size: int, + max_frames_per_batch: int, + path: str = "default", + ) -> "EncoderCudaGraphReplayBuffers": + from vllm.v1.worker.encoder_cudagraph_defs import ( + EncoderCudaGraphReplayBuffers, + ) + + pixel_values = self._get_pixel_values(mm_kwargs) + # Unpadded: pad_cu_seqlens completes the tail with the padding sequence + # so cu_seqlens[-1] matches the captured buffer's row count. + metadata = self.vision_tower.vision_model.prepare_encoder_metadata( + self._get_grid_thw_list(mm_kwargs), + device=pixel_values.device, + ) + # FlashAttention reads this CPU scalar during capture, so the captured + # launch must retain the bound for the padded buffer. + metadata.pop("max_seqlen") + + values: dict[str, torch.Tensor | None] = {"pixel_values": pixel_values} + values.update(metadata) + return EncoderCudaGraphReplayBuffers(values=values) + + def encoder_cudagraph_forward( + self, + inputs: dict[str, torch.Tensor], + path: str = "default", + ) -> torch.Tensor: + pixel_values = inputs.pop("pixel_values") + # Remaining keys are consumed as encoder_metadata. + return self.vision_tower(pixel_values, grid_thw=None, encoder_metadata=inputs) + + def encoder_eager_forward( + self, + mm_kwargs: dict[str, Any], + path: str = "default", + ) -> torch.Tensor: + return self.vision_tower( + pixel_values=self._get_pixel_values(mm_kwargs).type( + self.vision_tower.dtype + ), + grid_thw=self._get_grid_thw_list(mm_kwargs), + ) diff --git a/vllm/models/minimax_m3/common/vision_tower.py b/vllm/models/minimax_m3/common/vision_tower.py index ce56a179d0..fd9cff2c6d 100644 --- a/vllm/models/minimax_m3/common/vision_tower.py +++ b/vllm/models/minimax_m3/common/vision_tower.py @@ -392,19 +392,33 @@ class MiniMaxVLVisionTransformer(nn.Module): out.append([min(max_f, t - i), h, w]) return out - # ── Forward ────────────────────────────────────────────────────────── + # ── Encoder metadata (shared by eager forward and CUDA graph replay) ── - def forward( + def prepare_encoder_metadata( self, - pixel_values: torch.Tensor, grid_thw: list[list[int]], - ) -> torch.Tensor: - # pixel_values: (total_N, C * temporal_patch_size * patch_size²) - # Output: (total_N, hidden_size) + *, + device: torch.device, + max_batch_size: int | None = None, + max_seqlen_override: int | None = None, + ) -> dict[str, torch.Tensor | None]: + """Build all grid-dependent forward inputs as standalone tensors. - hidden = self.embeddings(pixel_values) # (total_N, hidden_size) - hidden = self.pre_layrnorm(hidden) + The eager path calls this per forward; encoder CUDA graphs call it at + capture (dummy grids, padded to fixed buffer sizes) and before each + replay (real grids, unpadded — the manager's padding logic completes + the captured buffers). + Args: + grid_thw: per-item [t, h, w] patch grids (host metadata). + device: device for the attention metadata tensors. + max_batch_size: pad cu_seqlens to this many segments with empty + (zero-length) trailing segments; None leaves it unpadded. + max_seqlen_override: bake this max_seqlen instead of deriving it + from the grids; CUDA graph capture passes a worst-case value + because max_seqlen is a CPU scalar baked into the graph. + + """ limited = self._apply_max_frames_limit(grid_thw) # Token-level cumulative sequence lengths (one segment per limited grid). @@ -412,15 +426,23 @@ class MiniMaxVLVisionTransformer(nn.Module): cu_seqlens_np = np.zeros(len(lens) + 1, dtype=np.int32) np.cumsum(np.array(lens, dtype=np.int32), out=cu_seqlens_np[1:]) + if max_batch_size is not None and len(lens) < max_batch_size: + pad = np.full(max_batch_size - len(lens), cu_seqlens_np[-1], dtype=np.int32) + cu_seqlens_np = np.concatenate([cu_seqlens_np, pad]) + # Backend-specific encoder metadata. For FLASH_ATTN this returns the raw # token cu_seqlens, the max segment length, and sequence_lengths=None; # for FLASHINFER (cuDNN) it repacks cu_seqlens into element-offset # indptrs, buckets max_seqlen, and builds padded per-sequence lengths. sequence_lengths = MMEncoderAttention.maybe_compute_seq_lens( - self.attn_backend, cu_seqlens_np, hidden.device + self.attn_backend, cu_seqlens_np, device ) max_seqlen = torch.tensor( - MMEncoderAttention.compute_max_seqlen(self.attn_backend, cu_seqlens_np), + max_seqlen_override + if max_seqlen_override is not None + else MMEncoderAttention.compute_max_seqlen( + self.attn_backend, cu_seqlens_np + ), dtype=torch.int32, ) cu_seqlens = MMEncoderAttention.maybe_recompute_cu_seqlens( @@ -428,7 +450,7 @@ class MiniMaxVLVisionTransformer(nn.Module): cu_seqlens_np, self.hidden_size, self.tp_size, - hidden.device, + device, ) # 3D RoPE cos/sin: (3, total_N, half_rot_dim) fp32 t/h/w planes for @@ -442,6 +464,45 @@ class MiniMaxVLVisionTransformer(nn.Module): self.spatial_merge_size, ) + return { + "rotary_cos": rotary_cos, + "rotary_sin": rotary_sin, + "cu_seqlens": cu_seqlens, + "max_seqlen": max_seqlen, + "sequence_lengths": sequence_lengths, + } + + # ── Forward ────────────────────────────────────────────────────────── + + def forward( + self, + pixel_values: torch.Tensor, + grid_thw: list[list[int]] | None = None, + *, + encoder_metadata: dict[str, torch.Tensor | None] | None = None, + ) -> torch.Tensor: + # pixel_values: (total_N, C * temporal_patch_size * patch_size²) + # Output: (total_N, hidden_size) + + hidden = self.embeddings(pixel_values) # (total_N, hidden_size) + hidden = self.pre_layrnorm(hidden) + + if encoder_metadata is None: + assert grid_thw is not None + encoder_metadata = self.prepare_encoder_metadata( + grid_thw, device=hidden.device + ) + + cu_seqlens = encoder_metadata["cu_seqlens"] + rotary_cos = encoder_metadata["rotary_cos"] + rotary_sin = encoder_metadata["rotary_sin"] + max_seqlen = encoder_metadata["max_seqlen"] + sequence_lengths = encoder_metadata.get("sequence_lengths") + assert cu_seqlens is not None + assert rotary_cos is not None + assert rotary_sin is not None + assert max_seqlen is not None + # Encoder expects (N, 1, hidden_size) — add batch dim hidden = hidden.unsqueeze(1) @@ -600,9 +661,15 @@ class MiniMaxVLVisionModel(nn.Module): def forward( self, pixel_values: torch.Tensor, - grid_thw: list[list[int]], + grid_thw: list[list[int]] | None = None, + *, + encoder_metadata: dict[str, torch.Tensor | None] | None = None, ) -> torch.Tensor: - hidden = self.vision_model(pixel_values=pixel_values, grid_thw=grid_thw) + hidden = self.vision_model( + pixel_values=pixel_values, + grid_thw=grid_thw, + encoder_metadata=encoder_metadata, + ) if hidden.dim() == 3: hidden = hidden.squeeze(0) hidden = self.multi_modal_projector(hidden) diff --git a/vllm/models/minimax_m3/nvidia/model.py b/vllm/models/minimax_m3/nvidia/model.py index 4fb260330f..c16918bd37 100644 --- a/vllm/models/minimax_m3/nvidia/model.py +++ b/vllm/models/minimax_m3/nvidia/model.py @@ -70,6 +70,9 @@ from vllm.model_executor.models.utils import ( maybe_prefix, ) from vllm.model_executor.models.vision import run_dp_sharded_mrope_vision_model +from vllm.models.minimax_m3.common.encoder_cudagraph import ( + MiniMaxM3EncoderCudaGraphMixin, +) from vllm.models.minimax_m3.common.indexer import MiniMaxM3Indexer from vllm.models.minimax_m3.common.mm_preprocess import ( MiniMaxM3VLDummyInputsBuilder, @@ -1076,7 +1079,11 @@ class MiniMaxM3SparseForCausalLM(nn.Module, SupportsPP, SupportsEagle3): dummy_inputs=MiniMaxM3VLDummyInputsBuilder, ) class MiniMaxM3SparseForConditionalGeneration( - nn.Module, SupportsMultiModal, SupportsPP, SupportsEagle3 + nn.Module, + SupportsMultiModal, + MiniMaxM3EncoderCudaGraphMixin, + SupportsPP, + SupportsEagle3, ): """Top-level (VL) entry point for MiniMax M3. diff --git a/vllm/models/qwen4_exp/nvidia/ops/qsa_indexer.py b/vllm/models/qwen4_exp/nvidia/ops/qsa_indexer.py index 077dea91f0..9b28ed6dca 100644 --- a/vllm/models/qwen4_exp/nvidia/ops/qsa_indexer.py +++ b/vllm/models/qwen4_exp/nvidia/ops/qsa_indexer.py @@ -383,6 +383,7 @@ def _prefill_logits( page_table: torch.Tensor, query_start_loc: torch.Tensor, visible_blocks: torch.Tensor, + logits_workspace: torch.Tensor, max_query_len: int, logits_width: int, query_offset: int, @@ -392,10 +393,14 @@ def _prefill_logits( assert visible_blocks.shape == (q.shape[0],) assert 0 <= query_offset <= query_offset + num_queries <= q.shape[0] assert 0 < logits_width <= page_table.shape[1] * k_cache.shape[1] + assert logits_workspace.is_contiguous() + assert logits_workspace.numel() >= num_queries * logits_width - logits = torch.empty( - (num_queries, logits_width), dtype=torch.float32, device=q.device + # slice from pre-allocated workspace + logits = logits_workspace[: num_queries * logits_width].view( + num_queries, logits_width ) + # tuned on GB300 if k_cache.dtype == torch.float8_e4m3fn: TILE_R, STAGES, num_warps = 32, 2, 8 @@ -610,8 +615,13 @@ def qsa_select_paged_prefill( logits_width = min(max(64, logits_width), page_table.shape[1] * k_cache.shape[1]) # chunk the inputs to keep temp logits below VLLM_SPARSE_INDEXER_MAX_LOGITS_MB + # always allocate the worst case to avoid memory fragmentation. max_logits_bytes = envs.VLLM_SPARSE_INDEXER_MAX_LOGITS_MB * 1024 * 1024 rows_per_chunk = max(1, max_logits_bytes // (logits_width * 4)) + + budget_bytes = max(max_logits_bytes, page_table.shape[1] * k_cache.shape[1] * 4) + logits_workspace = q.new_empty(budget_bytes // 4, dtype=torch.float32) + topk_workspace = torch.empty( (_TOPK_WORKSPACE_BYTES,), dtype=torch.uint8, device=q.device ) @@ -625,6 +635,7 @@ def qsa_select_paged_prefill( page_table, query_start_loc, visible_blocks, + logits_workspace, max_query_len, logits_width, query_offset=query_start, diff --git a/vllm/parser/abstract_parser.py b/vllm/parser/abstract_parser.py index 0e84e710ed..20f22bd0e3 100644 --- a/vllm/parser/abstract_parser.py +++ b/vllm/parser/abstract_parser.py @@ -5,10 +5,21 @@ import contextlib import json from abc import abstractmethod from collections.abc import Sequence -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from openai.types.responses import ToolChoiceFunction from pydantic import TypeAdapter, ValidationError +from xgrammar import Grammar, StructuralTag +from xgrammar.structural_tag import ( + ConstStringFormat, + Format, + GrammarFormat, + JSONSchemaFormat, + OrFormat, + RegexFormat, + TagFormat, + TriggeredTagsFormat, +) from vllm.entrypoints.chat_utils import ( get_tool_call_id_type, @@ -25,17 +36,19 @@ from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionRequest, ) from vllm.entrypoints.openai.responses.protocol import ResponsesRequest +from vllm.exceptions import VLLMValidationError from vllm.logger import init_logger from vllm.parser.metrics import record_tool_parser_invocation from vllm.parser.utils import count_history_tool_calls from vllm.reasoning.abs_reasoning_parsers import ReasoningParser -from vllm.sampling_params import StructuredOutputsParams +from vllm.sampling_params import SamplingParams, StructuredOutputsParams from vllm.tokenizers import TokenizerLike from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser from vllm.tool_parsers.streaming import ( extract_named_tool_call_streaming, extract_required_tool_call_streaming, ) +from vllm.tool_parsers.structural_tag_registry import resolve_tool_strictness from vllm.tool_parsers.tool_strict_level import ToolStrictLevel logger = init_logger(__name__) @@ -236,6 +249,67 @@ class Parser: return 0 +def structured_outputs_to_format(params: StructuredOutputsParams) -> Format | None: + """Map StructuredOutputsParams in a XGrammar Format.""" + if params.json_object: + return JSONSchemaFormat(json_schema={"type": "object"}) + if params.json is not None: + schema = params.json + if isinstance(schema, str): + schema = json.loads(schema) + return JSONSchemaFormat(json_schema=schema) + if params.regex is not None: + return RegexFormat(pattern=params.regex) + if params.choice is not None: + return OrFormat( + elements=[ConstStringFormat(value=choice) for choice in params.choice] + ) + if params.grammar is not None: + from vllm.v1.structured_output.utils import grammar_is_likely_lark + + grammar = params.grammar + # IMPORTANT(arpera): + # GrammarFormat only accepts EBNF. + if grammar_is_likely_lark(grammar): + try: + grammar = str(Grammar.from_lark(grammar)) + except Exception as e: + raise VLLMValidationError("Invalid grammar specification.") from e + return GrammarFormat(grammar=grammar) + if params.structural_tag is not None: + s_tag = json.loads(params.structural_tag) + if "structures" in s_tag: + # LegacyStructuralTagResponseFormat + return TriggeredTagsFormat( + triggers=s_tag["triggers"], + tags=[ + TagFormat( + begin=structure["begin"], + content=JSONSchemaFormat(json_schema=structure["schema"]), + end=structure["end"], + ) + for structure in s_tag["structures"] + ], + ) + # StructuralTagResponseFormat + return StructuralTag.model_validate(s_tag).format + return None + + +def _xgrammar_supports(structured_outputs: StructuredOutputsParams) -> bool: + """Whether the constraint passes xgrammar validation on its own.""" + from vllm.v1.structured_output.backend_xgrammar import validate_xgrammar_grammar + + try: + # validate_xgrammar_grammar rewrites `choice` into `grammar` in place. + validate_xgrammar_grammar( + SamplingParams(structured_outputs=replace(structured_outputs)) + ) + except (TypeError, ValueError, VLLMValidationError): + return False + return True + + class DelegatingParser(Parser): """A Parser implementation that delegates to separate ReasoningParser and ToolParser instances. @@ -390,18 +464,14 @@ class DelegatingParser(Parser): request = self._reasoning_parser.adjust_request(request) if self._tool_parser is not None: request = self._apply_structural_tag(request) - if self._tool_parser is not None: request = self._tool_parser.adjust_request(request) return request def _apply_structural_tag( self, request: ChatCompletionRequest | ResponsesRequest ) -> ChatCompletionRequest | ResponsesRequest: - if ( - self._tool_parser is None - or self._tool_parser.structural_tag_model is None - or not request.tools - ): + tool_parser = self._tool_parser + if tool_parser is None or not request.tools: return request need_tool_calling = ( @@ -415,17 +485,69 @@ class DelegatingParser(Parser): if not need_tool_calling: return request - structure_tag = self._tool_parser.get_structural_tag( - request, - reasoning=False, - strict_level=self.tool_strict_level, - ) - if structure_tag is None: + structured_outputs = request.extract_structured_outputs() + is_auto = request.tool_choice == "auto" + + resolved_tools = None + if tool_parser.structural_tag_model is not None: + resolved_tools = resolve_tool_strictness( + request.tools, + request.tool_choice, + self.tool_strict_level, + ) + + output_format = None + if resolved_tools and is_auto and structured_outputs: + if not _xgrammar_supports(structured_outputs): + logger.warning_once( + "Tool calls are not constrained for tool_choice=auto with " + "structured outputs because xgrammar does not support the " + "structured output constraint; the structured output " + "constraint applies.", + scope="local", + ) + return request + output_format = structured_outputs_to_format(structured_outputs) + + if resolved_tools is not None: + tag_request = request + if output_format is not None: + # IMPORTANT(arpera): + # The "auto" tag allows plain text in response. + # Use tag "required" here to ensure structured-output branch + # remains meaningful. + tag_request = request.model_copy(update={"tool_choice": "required"}) + tools_structural_tag = tool_parser.get_structural_tag( + tag_request, + reasoning=False, + strict_level=self.tool_strict_level, + ) + else: + tools_structural_tag = None + + if tools_structural_tag is None: + if is_auto and structured_outputs is not None: + logger.warning_once( + "Tool calls are not constrained for tool_choice=auto with " + "structured outputs because structural tags are unavailable; " + "the structured output constraint applies.", + scope="local", + ) return request - structural_tag = json.dumps(structure_tag.model_dump()) + structural_tag = tools_structural_tag + if output_format is not None: + structural_tag = StructuralTag( + format=OrFormat(elements=[tools_structural_tag.format, output_format]) + ) + elif structured_outputs is not None and not is_auto: + logger.warning_once( + "structured outputs are ignored because tool_choice forces tool call.", + scope="local", + ) + request.structured_outputs = StructuredOutputsParams( - structural_tag=structural_tag, + structural_tag=json.dumps(structural_tag.model_dump()), ) if isinstance(request, ResponsesRequest): request.text = None diff --git a/vllm/parser/cohere_command.py b/vllm/parser/cohere_command.py index e5ed7aed33..c37cc569ac 100644 --- a/vllm/parser/cohere_command.py +++ b/vllm/parser/cohere_command.py @@ -563,6 +563,15 @@ class CohereCommandParser(DelegatingParser): request.response_format = None return request + def _apply_structural_tag( + self, request: ChatCompletionRequest | ResponsesRequest + ) -> ChatCompletionRequest | ResponsesRequest: + # Note(arpera): + # No need to go through _apply_structural_tag in abstract_parser.py + # since we have already go through _apply_structural_tags here + # So, override this method to return request as is + return request + def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int: count = 0 depth = 0 diff --git a/vllm/parser/harmony.py b/vllm/parser/harmony.py index 5748bd6653..26f311c7e9 100644 --- a/vllm/parser/harmony.py +++ b/vllm/parser/harmony.py @@ -16,14 +16,11 @@ from xgrammar.structural_tag import ( AnyTextFormat, ConstStringFormat, Format, - GrammarFormat, JSONSchemaFormat, OptionalFormat, OrFormat, - RegexFormat, SequenceFormat, TagFormat, - TriggeredTagsFormat, ) from vllm.entrypoints.chat_utils import make_tool_call_id @@ -44,9 +41,8 @@ from vllm.entrypoints.openai.parser.harmony_utils import ( ) from vllm.entrypoints.openai.responses.protocol import ResponsesRequest from vllm.logger import init_logger -from vllm.parser.abstract_parser import DelegatingParser +from vllm.parser.abstract_parser import DelegatingParser, structured_outputs_to_format from vllm.reasoning.gptoss_reasoning_parser import GptOssReasoningParser -from vllm.sampling_params import StructuredOutputsParams from vllm.tool_parsers.gptoss_tool_parser import GptOssToolParser from vllm.tool_parsers.structural_tag_registry import ( SimplifiedToolChoice, @@ -428,7 +424,6 @@ _FUNCTION_CALL_BEGINS = [ " to=functions.{name}{channel}{constrain}<|message|>", "{channel} to=functions.{name}{constrain}<|message|>", ] -_JSON_CONTENT = JSONSchemaFormat(json_schema={"type": "object"}) _ANY_CONTENT = AnyTextFormat() @@ -531,43 +526,6 @@ def get_harmony_structural_tag( ) -def _params_to_final_content(params: StructuredOutputsParams) -> Format | None: - """Map StructuredOutputsParams in a XGrammar Format.""" - if params.json_object: - return _JSON_CONTENT - if params.json is not None: - schema = params.json - if isinstance(schema, str): - schema = json.loads(schema) - return JSONSchemaFormat(json_schema=schema) - if params.regex is not None: - return RegexFormat(pattern=params.regex) - if params.choice is not None: - return OrFormat( - elements=[ConstStringFormat(value=choice) for choice in params.choice] - ) - if params.grammar is not None: - return GrammarFormat(grammar=params.grammar) - if params.structural_tag is not None: - s_tag = json.loads(params.structural_tag) - if "structures" in s_tag: - # LegacyStructuralTagResponseFormat - return TriggeredTagsFormat( - triggers=s_tag["triggers"], - tags=[ - TagFormat( - begin=structure["begin"], - content=JSONSchemaFormat(json_schema=structure["schema"]), - end=structure["end"], - ) - for structure in s_tag["structures"] - ], - ) - # StructuralTagResponseFormat - return StructuralTag.model_validate(s_tag).format - return None - - def _adjust_output_format( request: ChatCompletionRequest | ResponsesRequest, ) -> ChatCompletionRequest | ResponsesRequest: @@ -576,7 +534,7 @@ def _adjust_output_format( if params is None: return request - final_content = _params_to_final_content(params) + final_content = structured_outputs_to_format(params) if final_content is None: return request diff --git a/vllm/renderers/online_renderer.py b/vllm/renderers/online_renderer.py index 139b783aaa..be0056a350 100644 --- a/vllm/renderers/online_renderer.py +++ b/vllm/renderers/online_renderer.py @@ -259,15 +259,7 @@ class OnlineRenderer: ) else: # For GPT-OSS. - if self.parser is not None: - # HarmonyParser doesn't need chat_template_kwargs - # TODO: Unify adjust_request() call with non-harmony branch - self.parser( - self.renderer.get_tokenizer(), - request.tools, - model_config=self.model_config, - ).adjust_request(request=request) - + self.adjust_harmony_request(request) should_include_tools = tool_dicts is not None conversation, engine_inputs = self._make_request_with_harmony( request, should_include_tools @@ -355,14 +347,7 @@ class OnlineRenderer: previous_response_outputs: list[ResponseOutputItem] | None, tool_server: "ToolServer | None", ) -> ResponsesRenderResult | ErrorResponse: - if self.parser is not None: - # HarmonyParser doesn't need chat_template_kwargs - # TODO: Unify adjust_request() call with non-harmony branch - self.parser( - self.renderer.get_tokenizer(), - request.tools, - model_config=self.model_config, - ).adjust_request(request=request) + self.adjust_harmony_request(request) if previous_messages is not None and any( not isinstance(message, OpenAIMessage) for message in previous_messages @@ -488,6 +473,24 @@ class OnlineRenderer: engine_input=engine_inputs[0], ) + def adjust_harmony_request( + self, request: ChatCompletionRequest | ResponsesRequest + ) -> None: + """Apply the Harmony parser's ``adjust_request`` to ``request`` in place. + + Every Harmony render path must call this before sampling params are + built from ``request``. + """ + if self.parser is None: + return + # HarmonyParser doesn't need chat_template_kwargs + # TODO: Unify adjust_request() call with non-harmony branch + self.parser( + self.renderer.get_tokenizer(), + request.tools, + model_config=self.model_config, + ).adjust_request(request=request) + def _make_request_with_harmony( self, request: ChatCompletionRequest, diff --git a/vllm/tool_parsers/structural_tag_registry.py b/vllm/tool_parsers/structural_tag_registry.py index ba86395b7a..f3e5c9ec98 100644 --- a/vllm/tool_parsers/structural_tag_registry.py +++ b/vllm/tool_parsers/structural_tag_registry.py @@ -121,7 +121,7 @@ def _with_tool_strict( return tool -def _resolve_tool_strictness( +def resolve_tool_strictness( tools: Sequence[ChatCompletionToolsParam | ResponsesTool], tool_choice: ToolChoice, strict_level: ToolStrictLevel, @@ -158,7 +158,7 @@ def get_model_structural_tag( if not tools or tool_choice == "none": return None - tools = _resolve_tool_strictness(tools, tool_choice, strict_level) + tools = resolve_tool_strictness(tools, tool_choice, strict_level) if tools is None: return None diff --git a/vllm/utils/multi_stream_utils.py b/vllm/utils/multi_stream_utils.py index 6190b76991..5cb7f00f76 100644 --- a/vllm/utils/multi_stream_utils.py +++ b/vllm/utils/multi_stream_utils.py @@ -70,6 +70,7 @@ def execute_in_parallel( done_events: list[torch.cuda.Event], aux_streams: list[torch.cuda.Stream] | None = None, enable: bool = False, + default_first: bool = False, ) -> tuple[Any, list[Any]]: """Run default_fn on the current stream and aux_fns concurrently on aux_streams. @@ -97,6 +98,10 @@ def execute_in_parallel( so callers that pass aux_streams must also pass enable=True (typically gated by an env var) to actually overlap. When False, execution falls back to sequential on the current stream. + default_first: When True, enqueue the default chain before the aux + chains. The default chain is typically the longest, so it starts + first; the aux streams still only wait on start_event, so they + overlap the default chain. Defaults to False. Returns: Tuple of (default_result, aux_results) where aux_results[i] is the @@ -117,6 +122,8 @@ def execute_in_parallel( pending: list[torch.cuda.Event] = [] start_event.record() + if default_first: + default_result = default_fn() for i, fn in enumerate(aux_fns): if fn is None: continue @@ -125,8 +132,8 @@ def execute_in_parallel( aux_results[i] = fn() done_events[i].record() pending.append(done_events[i]) - - default_result = default_fn() + if not default_first: + default_result = default_fn() for ev in pending: ev.wait() diff --git a/vllm/v1/attention/backend.py b/vllm/v1/attention/backend.py index b5cd6e4b32..5d661634d7 100644 --- a/vllm/v1/attention/backend.py +++ b/vllm/v1/attention/backend.py @@ -870,15 +870,14 @@ class AttentionImplBase(ABC, Generic[T]): def __new__(cls, *args, **kwargs): # use __new__ so that all subclasses will call this self = super().__new__(cls) - try: - from vllm.distributed.parallel_state import get_dcp_group + from vllm.config import get_current_vllm_config_or_none + from vllm.distributed.parallel_state import get_dcp_world_size_and_rank - self.dcp_world_size = get_dcp_group().world_size - self.dcp_rank = get_dcp_group().rank_in_group - except AssertionError: - # DCP might not be initialized in testing - self.dcp_world_size = 1 - self.dcp_rank = 0 + # Replicated drafts run at DCP=1 inside a DCP target's process group. + config = get_current_vllm_config_or_none() + self.dcp_world_size, self.dcp_rank = get_dcp_world_size_and_rank( + config is None or config.parallel_config.decode_context_parallel_size > 1 + ) try: from vllm.distributed.parallel_state import get_pcp_group diff --git a/vllm/v1/attention/backends/flash_attn.py b/vllm/v1/attention/backends/flash_attn.py index 129b3e39c3..6035678394 100755 --- a/vllm/v1/attention/backends/flash_attn.py +++ b/vllm/v1/attention/backends/flash_attn.py @@ -60,7 +60,10 @@ from vllm.config import ( get_layers_from_vllm_config, ) from vllm.config.cache import CacheDType -from vllm.distributed.parallel_state import get_dcp_group +from vllm.distributed.parallel_state import ( + get_dcp_group, + get_dcp_world_size_and_rank, +) from vllm.logger import init_logger from vllm.platforms.interface import DeviceCapability from vllm.utils.math_utils import cdiv, round_up @@ -726,15 +729,9 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad num_qo_heads=self.num_heads_q, ) - try: - from vllm.distributed.parallel_state import get_dcp_group - - self.dcp_world_size = get_dcp_group().world_size - self.dcp_rank = get_dcp_group().rank_in_group - except AssertionError: - # DCP might not be initialized in testing - self.dcp_world_size = 1 - self.dcp_rank = 0 + self.dcp_world_size, self.dcp_rank = get_dcp_world_size_and_rank( + kv_cache_spec.dcp_sharded + ) # Fused draft decode reuses the captured metadata object across draft # steps. For DCP, build-time host-side decisions such as diff --git a/vllm/v1/attention/backends/flashinfer.py b/vllm/v1/attention/backends/flashinfer.py index f40c75b474..ecc66a75ca 100755 --- a/vllm/v1/attention/backends/flashinfer.py +++ b/vllm/v1/attention/backends/flashinfer.py @@ -30,7 +30,10 @@ from vllm.config import ( get_current_vllm_config_or_none, ) from vllm.config.cache import CacheDType -from vllm.distributed.parallel_state import get_dcp_group +from vllm.distributed.parallel_state import ( + get_dcp_group, + get_dcp_world_size_and_rank, +) from vllm.logger import init_logger from vllm.model_executor.layers.quantization.utils.quant_utils import ( QuantKey, @@ -641,6 +644,7 @@ class FlashInferTrtllmAPIDecode: @dataclass class FlashInferMetadata: + kv_cache_layout: KVCacheLayout num_actual_tokens: int """Total number of tokens in the batch (excluding padding).""" @@ -696,6 +700,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): self.cache_config = vllm_config.cache_config self.model_config = vllm_config.model_config self.attention_config = vllm_config.attention_config + self._num_speculative_tokens = vllm_config.num_speculative_tokens self._workspace_buffer = None self._prefill_wrapper: ( BatchPrefillWithPagedKVCacheWrapper | BatchDCPPrefillWrapper | None @@ -747,17 +752,12 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): self._decode_cudagraph_max_bs, self.compilation_config.max_cudagraph_capture_size, ) - try: - self.dcp_world_size = get_dcp_group().world_size - self.dcp_rank = get_dcp_group().rank_in_group - self.dcp_kv_cache_interleave_size = ( - vllm_config.parallel_config.cp_kv_cache_interleave_size - ) - except AssertionError: - # DCP might not be initialized in testing - self.dcp_world_size = 1 - self.dcp_rank = 0 - self.dcp_kv_cache_interleave_size = 1 + self.dcp_world_size, self.dcp_rank = get_dcp_world_size_and_rank( + kv_cache_spec.dcp_sharded + ) + self.dcp_kv_cache_interleave_size = ( + vllm_config.parallel_config.cp_kv_cache_interleave_size + ) self.use_dcp = self.dcp_world_size > 1 self.dcp_a2a = ( self.use_dcp and vllm_config.parallel_config.dcp_comm_backend == "a2a" @@ -1424,6 +1424,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): # Leave prefill/decode/cascade_wrapper empty, to be populated # case by case depending on the batch contents and backend selection. attn_metadata = FlashInferMetadata( + kv_cache_layout=self.kv_cache_layout, num_actual_tokens=num_actual_tokens, slot_mapping=common_attn_metadata.slot_mapping, q_data_type_prefill=self.q_data_type_prefill, @@ -1446,8 +1447,15 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): # (block_tables, seq_lens) directly. needs_seq_lens_cpu = self.use_dcp or use_cascade or not all_uses_trtllm if needs_seq_lens_cpu: - with gpu_sync_allowed(): - seq_lens_cpu = common_attn_metadata.seq_lens.cpu() + if ( + self._num_speculative_tokens == 0 + and common_attn_metadata.seq_lens_cpu_upper_bound is not None + ): + # No speculative token accounting, so the upper bound is exact. + seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound + else: + with gpu_sync_allowed(): + seq_lens_cpu = common_attn_metadata.seq_lens.cpu() else: seq_lens_cpu = None @@ -1904,8 +1912,6 @@ class FlashInferImpl(AttentionImpl): num_heads, num_kv_heads, is_prefill=False ) vllm_config = get_current_vllm_config_or_none() - # The layout is resolved after model construction, so read it lazily. - self.cache_config = vllm_config.cache_config if vllm_config else None # Query pre-quantization needs a single dtype for the whole query tensor. # SM90 XQA needs BF16/FP16-Q for decode and FP8 for prefill, # so only enable this for SM100 trtllm-gen where both use FP8-Q. @@ -1941,11 +1947,6 @@ class FlashInferImpl(AttentionImpl): else: self.dcp_combine = partial(cp_lse_ag_out_rs, is_lse_base_on_e=False) - @property - def kv_cache_layout(self) -> KVCacheLayout: - assert self.cache_config is not None - return self.cache_config.get_resolved_kv_cache_layout() - def fused_output_quant_supported(self, quant_key: QuantKey): if quant_key == kNvfp4Dynamic and self.is_kvcache_nvfp4: logger.warning_once( @@ -2111,6 +2112,7 @@ class FlashInferImpl(AttentionImpl): # performance to make sure it does not introduce any overhead. num_actual_tokens = attn_metadata.num_actual_tokens + kv_cache_layout = attn_metadata.kv_cache_layout # FlashInfer treats uint8 KV cache as NVFP4. vLLM stores FP8 KV cache # as uint8 bytes, so pass FP8 caches with their logical dtype. @@ -2134,7 +2136,7 @@ class FlashInferImpl(AttentionImpl): if attn_metadata.use_cascade: # Cascade attention (rare case). assert attn_metadata.cascade_wrapper is not None - stride_order = self.kv_cache_layout.layer_view_order + stride_order = kv_cache_layout.layer_view_order if self.is_kvcache_nvfp4: kv_cache_views = tuple( cache.permute(*stride_order) @@ -2154,7 +2156,7 @@ class FlashInferImpl(AttentionImpl): num_decode_tokens = attn_metadata.num_decode_tokens num_prefill_tokens = attn_metadata.num_prefill_tokens - stride_order = self.kv_cache_layout.layer_view_order + stride_order = kv_cache_layout.layer_view_order kv_cache_permute = kv_cache.permute(*stride_order) # HND and contiguous # Fix degenerate strides on any size-1 dimension (e.g. num_kv_heads=1 # with TP=8). PyTorch permits non-canonical strides on size-1 dims; @@ -2311,7 +2313,7 @@ class FlashInferImpl(AttentionImpl): seq_lens_prefill = attn_metadata.prefill.seq_lens # This path needs to be enabled with VLLM_KV_CACHE_LAYOUT = HND - assert get_flashinfer_layout_string(self.kv_cache_layout) == "HND" + assert get_flashinfer_layout_string(kv_cache_layout) == "HND" assert is_strictly_contiguous(prefill_query) assert is_strictly_contiguous(workspace_buffer) assert is_strictly_contiguous(block_tables_prefill) @@ -2501,7 +2503,7 @@ class FlashInferImpl(AttentionImpl): # trtllm-gen needs HND layout on SM100. XQA is selected # separately on SM90 and does not use this SM100 layout gate. if decode_with_trtllm_gen: - assert get_flashinfer_layout_string(self.kv_cache_layout) == "HND" + assert get_flashinfer_layout_string(kv_cache_layout) == "HND" else: assert decode_with_xqa assert is_strictly_contiguous(decode_query) @@ -2547,7 +2549,7 @@ class FlashInferImpl(AttentionImpl): window_left=self.window_left, out=output_padded[:decode_query_tokens], sinks=self.sinks, - kv_layout=get_flashinfer_layout_string(self.kv_cache_layout), + kv_layout=get_flashinfer_layout_string(kv_cache_layout), q_len_per_req=q_len_per_req, mask=attn_metadata.decode.mask, q_cu_seq_lens=attn_metadata.decode.q_cu_seq_lens, @@ -2614,7 +2616,7 @@ class FlashInferImpl(AttentionImpl): sinks=self.sinks, o_sf_scale=self.o_sf_scale, out=out, - kv_layout=get_flashinfer_layout_string(self.kv_cache_layout), + kv_layout=get_flashinfer_layout_string(kv_cache_layout), backend=attn_metadata.decode.kernel.value, q_len_per_req=q_len_per_req, max_q_len=max_q_len, diff --git a/vllm/v1/attention/backends/gdn_attn.py b/vllm/v1/attention/backends/gdn_attn.py index 85173dc2e4..ae8a9437d5 100644 --- a/vllm/v1/attention/backends/gdn_attn.py +++ b/vllm/v1/attention/backends/gdn_attn.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Backend for GatedDeltaNet attention.""" -from dataclasses import dataclass +from dataclasses import dataclass, replace from typing import Literal import torch @@ -60,6 +60,7 @@ class GDNAttentionMetadata: None # shape: [batch - num_spec_decodes,] ) spec_sequence_masks: torch.Tensor | None = None # shape: [batch,] + spec_sequence_masks_cpu: torch.Tensor | None = None # shape: [batch,] spec_token_indx: torch.Tensor | None = None non_spec_token_indx: torch.Tensor | None = None @@ -114,6 +115,18 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] self.use_full_cuda_graph: bool = ( self.compilation_config.cudagraph_mode.has_full_cudagraphs() ) + # update_block_table() keeps the source group's batch-level FULL graph + # buffers, so only MRV2, which also reuses metadata at capture, may use + # it. + self.supports_update_block_table = ( + vllm_config.use_v2_model_runner + and device.type == "cuda" + # Not isinstance: KDA's RecoverSSM/checkpoint metadata is per group. + and type(self) is GDNAttentionMetadataBuilder + ) + if self.supports_update_block_table: + # Opts into MRV2's CUDA-only aligned-index precompute. + self.mamba_aligned_state_indices: torch.Tensor | None = None self.decode_cudagraph_max_bs: int = ( self.vllm_config.scheduler_config.max_num_seqs * (self.num_spec + 1) @@ -220,12 +233,16 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] query_start_loc = m.query_start_loc query_start_loc_cpu = m.query_start_loc_cpu nums_dict, batch_ptr, token_chunk_offset_ptr = None, None, None - block_table_tensor = mamba_get_block_table_tensor( - m.block_table_tensor, - m.seq_lens, - self.kv_cache_spec, - self.vllm_config.cache_config.mamba_cache_mode, - ) + aligned_state_indices = getattr(self, "mamba_aligned_state_indices", None) + if aligned_state_indices is not None: + block_table_tensor = aligned_state_indices[: m.num_reqs] + else: + block_table_tensor = mamba_get_block_table_tensor( + m.block_table_tensor, + m.seq_lens, + self.kv_cache_spec, + self.vllm_config.cache_config.mamba_cache_mode, + ) uniform_spec_sequence_length = None spec_sequence_masks_cpu: torch.Tensor | None = None @@ -460,12 +477,8 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] # metadata below is indexed by request. batch_size = m.num_reqs - if ( - self.use_full_cuda_graph - and num_prefills == 0 - and num_decodes == 0 - and num_spec_decodes <= self.decode_cudagraph_max_bs - and num_spec_decode_tokens <= self.decode_cudagraph_max_bs + if self._stage_spec_decode( + num_prefills, num_decodes, num_spec_decodes, num_spec_decode_tokens ): assert spec_sequence_masks is not None self.spec_state_indices_tensor[:num_spec_decodes].copy_( @@ -506,12 +519,7 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] num_accepted_tokens = self.num_accepted_tokens[:batch_size] num_accepted_tokens[num_spec_decodes:].fill_(1) - if ( - self.use_full_cuda_graph - and num_prefills == 0 - and num_spec_decodes == 0 - and num_decodes <= self.decode_cudagraph_max_bs - ): + if self._stage_decode(num_prefills, num_decodes, num_spec_decodes): self.non_spec_state_indices_tensor[:num_decodes].copy_( non_spec_state_indices_tensor, non_blocking=True ) @@ -546,6 +554,7 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] spec_state_indices_tensor=spec_state_indices_tensor, non_spec_state_indices_tensor=non_spec_state_indices_tensor, spec_sequence_masks=spec_sequence_masks, + spec_sequence_masks_cpu=spec_sequence_masks_cpu, spec_token_indx=spec_token_indx, non_spec_token_indx=non_spec_token_indx, num_accepted_tokens=num_accepted_tokens, @@ -556,6 +565,83 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] ) return attn_metadata + def _stage_spec_decode( + self, + num_prefills: int, + num_decodes: int, + num_spec_decodes: int, + num_spec_decode_tokens: int, + ) -> bool: + """Whether spec-decode metadata goes into the FULL cudagraph buffers.""" + return ( + self.use_full_cuda_graph + and num_prefills == 0 + and num_decodes == 0 + and num_spec_decodes <= self.decode_cudagraph_max_bs + and num_spec_decode_tokens <= self.decode_cudagraph_max_bs + ) + + def _stage_decode( + self, num_prefills: int, num_decodes: int, num_spec_decodes: int + ) -> bool: + """Whether decode metadata goes into the FULL cudagraph buffers.""" + return ( + self.use_full_cuda_graph + and num_prefills == 0 + and num_spec_decodes == 0 + and num_decodes <= self.decode_cudagraph_max_bs + ) + + def update_block_table( + self, + metadata: GDNAttentionMetadata, + blk_table: torch.Tensor, + slot_mapping: torch.Tensor, + ) -> GDNAttentionMetadata: + """Re-gather this group's state indices. The other fields are + batch-level and stay shared with ``metadata``.""" + m = metadata + if self.vllm_config.cache_config.mamba_cache_mode == "align": + assert self.mamba_aligned_state_indices is not None + blk_table = self.mamba_aligned_state_indices + masks = m.spec_sequence_masks_cpu + spec_indices = non_spec_indices = prefill_indices = None + if masks is None: + non_spec_indices = blk_table[:, 0] + if m.num_prefills > 0: + prefill_indices = non_spec_indices[m.num_decodes :] + else: + spec_indices = blk_table[masks, : self.num_spec + 1] + if m.num_prefills > 0: + non_spec_indices = prefill_indices = blk_table[~masks, 0] + + if self._stage_spec_decode( + m.num_prefills, m.num_decodes, m.num_spec_decodes, m.num_spec_decode_tokens + ): + assert m.spec_state_indices_tensor is not None + batch_size = m.spec_state_indices_tensor.shape[0] + self.spec_state_indices_tensor[: m.num_spec_decodes].copy_( + spec_indices, non_blocking=True + ) + spec_indices = self.spec_state_indices_tensor[:batch_size] + spec_indices[m.num_spec_decodes :].fill_(NULL_BLOCK_ID) + + if self._stage_decode(m.num_prefills, m.num_decodes, m.num_spec_decodes): + assert m.non_spec_state_indices_tensor is not None + batch_size = m.non_spec_state_indices_tensor.shape[0] + self.non_spec_state_indices_tensor[: m.num_decodes].copy_( + non_spec_indices, non_blocking=True + ) + non_spec_indices = self.non_spec_state_indices_tensor[:batch_size] + non_spec_indices[m.num_decodes :].fill_(NULL_BLOCK_ID) + + return replace( + m, + spec_state_indices_tensor=spec_indices, + non_spec_state_indices_tensor=non_spec_indices, + prefill_state_indices=prefill_indices, + ) + def build_for_cudagraph_capture( self, common_attn_metadata: CommonAttentionMetadata ): diff --git a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse_sm90.py b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse_sm90.py index 9dc899c903..a5f7757d38 100644 --- a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse_sm90.py +++ b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse_sm90.py @@ -17,14 +17,14 @@ encoded by the indexer's selection, so ``causal=False``. CUDA-graph handling: ``plan()`` copies its inputs to host unconditionally, so it must stay outside graph capture. Each metadata builder owns a wrapper, reserved capture-stable device buffers, and the plan parameters. The wrapper -bakes the per-row ``kv_len`` into its int schedule -at plan() time — ``run()`` never reads the device-side buffer — so the -metadata builder replans every step (outside capture) with exact host-side -lengths derived from the batch's sequence lengths; a full-width schedule -would send the kernel past each row's valid count into the -1 tail of the -converted index buffer (illegal address). Per-step content (top-k slots) -is written into the reserved buffers by kernels inside the captured -forward, and captured runs read the refreshed plan buffers on replay. +bakes the per-row ``kv_len`` into its int schedule at plan() time — ``run()`` +never reads the device-side buffer. The builder plans (outside capture) from +sync-free host upper bounds and clamps each scheduled work item's ``kv_end`` +on device to the exact valid count, so the kernel never reads past a row's +valid prefix (the -1 tail of the converted index buffer) and async scheduling +needs no D2H sync. Per-step content (top-k slots) is written into the +reserved buffers by kernels inside the captured forward, and captured runs +read the refreshed plan buffers on replay. KV cache format: plain contiguous E4M3 ``[num_blocks, block_size, 512]`` (uint8 storage) with a per-tensor ``k_scale``; BF16 caches also work. The @@ -63,6 +63,26 @@ from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheLayout _FP8_KV_DTYPES = ("fp8", "fp8_e4m3") _WORKSPACE_BYTES = 128 * 1024 * 1024 +# TODO: FlashInfer bakes per-row kv_len into the plan on the host, which +# forces a D2H sync under async scheduling. The plan-info layout constants +# below and the int-workspace clamp in _SM90State work around this by +# planning from sync-free upper bounds; delete them once FlashInfer accepts +# an upper-bound plan() that clamps kv_end internally, or exposes a +# GPU-side plan API (upstream ask: +# https://github.com/flashinfer-ai/flashinfer/issues/5559, see +# vllm-project/vllm#58684). +# FlashInfer MLAPlanInfo layout (scheduler.cuh) field indices. +_PLAN_INFO_LEN = 18 +_PI_NUM_BLKS_Y = 1 +_PI_Q_INDPTR = 2 +_PI_KV_INDPTR = 3 +_PI_KV_START = 13 +_PI_KV_END = 14 +_PI_WORK_INDPTR = 15 +# Planned lengths exceed the host upper bound by up to this many tokens so a +# plan stays reusable across draft steps and consecutive decode steps; the +# device-side clamp restores the exact lengths. +_PLAN_SLACK = 32 @triton.jit @@ -82,6 +102,40 @@ def _pack_topk_indices( tl.store(indices + start + cols, tl.maximum(values, 0), mask=mask) +@triton.jit +def _clamp_work_kv_end( + int_ws, + saved_kv_end, + seq_lens, + query_start_loc, + req_id_per_token, + num_works_idx, + q_indptr_off, + kv_start_off, + kv_end_off, + TOPK: tl.constexpr, + KPOOL: tl.constexpr, + BLOCK_SIZE: tl.constexpr, +): + # Clamp each planned work item to its row's exact valid count, derived + # from the device batch layout exactly as the convert kernel counts it. + works = tl.program_id(0) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) + mask = works < tl.load(int_ws + num_works_idx) + row = tl.load(int_ws + q_indptr_off + works, mask=mask, other=0) + req = tl.load(req_id_per_token + row, mask=mask, other=0) + seq_len = tl.load(seq_lens + req, mask=mask, other=0) + query_end = tl.load(query_start_loc + req + 1, mask=mask, other=0) + ctx = seq_len - query_end + row + 1 + valid = tl.where(ctx <= TOPK, ctx, TOPK + ctx % KPOOL) + start = tl.load(int_ws + kv_start_off + works, mask=mask, other=0) + end = tl.load(saved_kv_end + works, mask=mask, other=0) + tl.store( + int_ws + kv_end_off + works, + tl.maximum(tl.minimum(end, valid), start), + mask=mask, + ) + + class FlashInferMLASparseSM90Backend(AttentionBackend): supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16] supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [ @@ -190,6 +244,8 @@ class _SM90State: kv_lora_rank: int, qk_rope_head_dim: int, sm_scale: float, + index_topk: int, + index_kpool: int, ) -> None: from flashinfer.mla import BatchMLAPagedAttentionWrapper @@ -202,6 +258,10 @@ class _SM90State: self.kv_lora_rank = kv_lora_rank self.qk_rope_head_dim = qk_rope_head_dim self.sm_scale = sm_scale + self.index_topk = index_topk + self.index_kpool = max(index_kpool, 1) + # Largest valid count the convert kernel can produce for any context. + self.max_valid = min(self.index_topk + self.index_kpool - 1, topk_width) # User-reserved buffers: with use_cuda_graph=True plan() refreshes # these in place, so run()'s captured kernels always read them. self.kv_indices = torch.zeros( @@ -226,24 +286,54 @@ class _SM90State: self._lens_cpu = torch.full( (self.max_tokens,), self.topk_width, dtype=torch.int32 ) + # Planned per-work kv_end of the current plan, restored by each clamp; + # sized on first plan to the planner's work capacity. + self._saved_kv_end = torch.empty(0, dtype=torch.int32, device=device) + self._planned_num_tokens = -1 + self._int_ws: torch.Tensor | None = None + self._plan_offsets: tuple[int, int, int, int] | None = None - def plan(self, num_tokens: int, kv_lens: torch.Tensor) -> None: - """Replan with exact per-row KV lengths (CPU int32, ``[num_tokens]``). + def plan( + self, + num_tokens: int, + kv_lens: torch.Tensor, + cam: CommonAttentionMetadata | None, + req_id_per_token: torch.Tensor | None, + ) -> None: + """Plan per-row KV lengths (CPU int32, ``[num_tokens]``). - The wrapper bakes kv_len into its int schedule from host values; - run() never reads the device kv_len_arr buffer. Scheduling with the - full buffer width while kv_indices rows carry a -1 tail past each - row's valid count makes the kernel compute ``-1 * ckv_stride_page`` - (illegal address), so the lengths must be exact at plan time and - replanned every step as contexts grow. Must run outside CUDA graph - capture: the in-place refreshed plan_info/indptr buffers are what - captured runs read. + The wrapper bakes kv_len into its int schedule from host values, so + rows past their valid count would send the kernel into the -1 tail of + the converted index buffer. With ``cam`` and ``req_id_per_token`` + (device batch layout), ``kv_lens`` only needs to upper-bound the + valid counts: the plan is padded by ``_PLAN_SLACK`` and reused while + the bound still fits, and every call clamps each work item's kv_end + on device to the exact count, so no D2H sync is needed. Without them + the lengths must be exact. Must run outside CUDA graph capture: the + in-place refreshed plan_info/indptr buffers are what captured runs + read. """ if torch.cuda.is_current_stream_capturing(): raise RuntimeError( "FlashInferMLASparseSM90 plan() called inside CUDA graph " "capture; lengths must be planned host-side before capture." ) + kv_lens = kv_lens.to(torch.int32) + if cam is None or req_id_per_token is None: + self._plan(num_tokens, kv_lens) + self._planned_num_tokens = -1 + return + planned = self._lens_cpu[:num_tokens] + if num_tokens != self._planned_num_tokens or not bool( + ((planned >= kv_lens) & (planned <= kv_lens + _PLAN_SLACK)).all() + ): + self._plan( + num_tokens, torch.clamp(kv_lens + _PLAN_SLACK, max=self.max_valid) + ) + self._planned_num_tokens = num_tokens + self._clamp_kv_end(cam, req_id_per_token) + + def _plan(self, num_tokens: int, kv_lens: torch.Tensor) -> None: # CPU staging buffers are filled in place: plan() runs per step # (once per draft/verify metadata build), so per-call allocations and # device round trips are on the hot path. Passing CPU tensors lets @@ -257,13 +347,16 @@ class _SM90State: # matching the exact lengths, including zero-length padded rows. torch.clamp(self._arange_cpu, max=num_tokens, out=self._qo_cpu) self._lens_cpu.zero_() - self._lens_cpu[:num_tokens] = kv_lens.to(torch.int32) + self._lens_cpu[:num_tokens] = kv_lens self._kv_cpu[0] = 0 torch.cumsum(self._lens_cpu, dim=0, out=self._kv_cpu[1:]) + # Only the live prefix of kv_indices: the wrapper snapshots and + # copies whatever is passed. + num_indices = max(int(self._kv_cpu[-1]), 1) self.wrapper.plan( self._qo_cpu, self._kv_cpu, - self.kv_indices, + self.kv_indices[:num_indices], self._lens_cpu, self.num_heads, self.kv_lora_rank, # head_dim_ckv @@ -274,6 +367,42 @@ class _SM90State: q_data_type=torch.bfloat16, kv_data_type=self.kv_dtype, ) + plan_info = [int(x) for x in self.wrapper._plan_info] + assert len(plan_info) == _PLAN_INFO_LEN, plan_info + # Per-work arrays are laid out back to back at the planner's capacity. + max_works = (plan_info[_PI_KV_INDPTR] - plan_info[_PI_Q_INDPTR]) // 4 + if self._saved_kv_end.numel() != max_works: + self._saved_kv_end = torch.empty( + max_works, dtype=torch.int32, device=self.device + ) + self._int_ws = self.wrapper._planned_backend._int_workspace_buffer.view( + torch.int32 + ) + self._plan_offsets = ( + plan_info[_PI_WORK_INDPTR] // 4 + plan_info[_PI_NUM_BLKS_Y], + plan_info[_PI_Q_INDPTR] // 4, + plan_info[_PI_KV_START] // 4, + plan_info[_PI_KV_END] // 4, + ) + kv_end_off = self._plan_offsets[3] + self._saved_kv_end.copy_(self._int_ws[kv_end_off : kv_end_off + max_works]) + + def _clamp_kv_end( + self, cam: CommonAttentionMetadata, req_id_per_token: torch.Tensor + ) -> None: + assert self._int_ws is not None and self._plan_offsets is not None + block = 1024 + _clamp_work_kv_end[(triton.cdiv(self._saved_kv_end.numel(), block),)]( + self._int_ws, + self._saved_kv_end, + cam.seq_lens, + cam.query_start_loc, + req_id_per_token, + *self._plan_offsets, + TOPK=self.index_topk, + KPOOL=self.index_kpool, + BLOCK_SIZE=block, + ) def pack_indices(self, topk_slots: torch.Tensor) -> None: num_tokens, width = topk_slots.shape @@ -312,6 +441,8 @@ class FlashInferMLASparseSM90Builder(FlashInferMLASparseMetadataBuilder): ) topk_indices_buffer = impl.topk_indices_buffer assert topk_indices_buffer is not None + hf_config = vllm_config.model_config.hf_text_config + assert hf_config.index_topk is not None self.state = _SM90State( device, impl.num_heads, @@ -321,20 +452,18 @@ class FlashInferMLASparseSM90Builder(FlashInferMLASparseMetadataBuilder): kv_lora_rank=impl.kv_lora_rank, qk_rope_head_dim=impl.qk_rope_head_dim, sm_scale=impl.scale, + index_topk=int(hf_config.index_topk), + index_kpool=int(getattr(hf_config, "index_kpool", 1) or 1), + ) + spec_config = vllm_config.speculative_config + self._adaptive_verification = bool( + spec_config is not None and spec_config.enable_adaptive_verification ) - # seq_lens_cpu_upper_bound is optimistic on decode rows under async - # spec decode, so the fast sync-free path is only safe without it; - # under async scheduling the exact (device) positions are used at - # the cost of one D2H sync per metadata build. - self._async_scheduling = bool(vllm_config.scheduler_config.async_scheduling) - hf_config = vllm_config.model_config.hf_text_config - assert hf_config.index_topk is not None - self._index_topk = int(hf_config.index_topk) - self._index_kpool = int(getattr(hf_config, "index_kpool", 1) or 1) - def _kv_lens_host(self, cam: CommonAttentionMetadata) -> tuple[int, torch.Tensor]: - """Exact per-row KV lengths, host-side (the flashinfer wrapper bakes - them into its schedule at plan time; there is no device-side path). + def _kv_lens_host( + self, cam: CommonAttentionMetadata + ) -> tuple[int, torch.Tensor, bool]: + """Host per-row KV lengths for the plan and whether they are exact. A row for the j-th query token of request i attends ``seq_lens[i] - q_len[i] + j + 1`` tokens. The indexer's selection @@ -343,53 +472,41 @@ class FlashInferMLASparseSM90Builder(FlashInferMLASparseMetadataBuilder): ``index_topk`` pool-expanded tokens plus the trailing incomplete pool (valid == ``index_topk + context % index_kpool``). Both match the count of non -1 entries the convert kernel produces. + + With ``seq_lens_cpu_upper_bound`` (optimistic under async spec + decode) returns sync-free upper bounds, which the plan clamps on + device. Otherwise returns exact lengths via a D2H copy. """ num_reqs = cam.num_reqs qsl = cam.query_start_loc_cpu[: num_reqs + 1] num_rows = int(qsl[-1]) if num_rows == 0: - return 0, torch.zeros(0, dtype=torch.int32) - # Row context == position + 1. Without async scheduling the - # maintained host upper bound is exact, giving a sync-free path; - # under async scheduling it is optimistic on decode rows, so fall - # back to the exact device positions (one D2H sync per build). The - # seq_lens derivation equals position + 1 because positions are - # contiguous per request. + return 0, torch.zeros(0, dtype=torch.int32), True + state = self.state sl_host = cam.seq_lens_cpu_upper_bound - positions = cam.positions - if not self._async_scheduling and sl_host is not None: - seq_lens = sl_host[:num_reqs].to(torch.int32) - q_lens = qsl[1:] - qsl[:-1] - first_pos = seq_lens - q_lens - req_of_row = torch.repeat_interleave( - torch.arange(num_reqs, dtype=torch.int64), q_lens.to(torch.int64) - ) - rows = torch.arange(num_rows, dtype=torch.int32) - ctx = ( - first_pos.to(torch.int64)[req_of_row] - + rows.to(torch.int64) - - qsl.to(torch.int64)[req_of_row] - + 1 - ) - elif positions is not None and num_rows <= positions.shape[0]: - ctx = positions[:num_rows].cpu().to(torch.int64) + 1 + if sl_host is None: + ctx = self._row_contexts(qsl, cam.seq_lens[:num_reqs].cpu()) + topk, kpool = state.index_topk, state.index_kpool + lens = torch.where(ctx <= topk, ctx, topk + ctx % kpool) + return num_rows, lens.to(torch.int32), True + if self._adaptive_verification: + # The host query split may not match the device one, so bound + # every row by the longest request. + ctx = torch.full((num_rows,), int(sl_host[:num_reqs].max())) else: - seq_lens = cam.seq_lens[:num_reqs].cpu().to(torch.int32) - q_lens = qsl[1:] - qsl[:-1] - first_pos = seq_lens - q_lens - req_of_row = torch.repeat_interleave( - torch.arange(num_reqs, dtype=torch.int64), q_lens.to(torch.int64) - ) - rows = torch.arange(num_rows, dtype=torch.int32) - ctx = ( - first_pos.to(torch.int64)[req_of_row] - + rows.to(torch.int64) - - qsl.to(torch.int64)[req_of_row] - + 1 - ) - topk = self._index_topk - lens = torch.where(ctx <= topk, ctx, topk + ctx % self._index_kpool) - return num_rows, lens.to(torch.int32) + ctx = self._row_contexts(qsl, sl_host[:num_reqs]) + return num_rows, torch.clamp(ctx, max=state.max_valid), False + + @staticmethod + def _row_contexts(qsl: torch.Tensor, seq_lens: torch.Tensor) -> torch.Tensor: + num_reqs = qsl.shape[0] - 1 + q_lens = (qsl[1:] - qsl[:-1]).to(torch.int64) + first_ctx = seq_lens.to(torch.int64) - q_lens + 1 + req_of_row = torch.repeat_interleave( + torch.arange(num_reqs, dtype=torch.int64), q_lens + ) + rows = torch.arange(int(qsl[-1]), dtype=torch.int64) + return first_ctx[req_of_row] + rows - qsl.to(torch.int64)[req_of_row] def build( self, @@ -399,9 +516,9 @@ class FlashInferMLASparseSM90Builder(FlashInferMLASparseMetadataBuilder): ) -> FlashInferMLASparseSM90Metadata: metadata = super().build(common_prefix_len, common_attn_metadata, fast_build) assert isinstance(metadata, FlashInferMLASparseSM90Metadata) - # Replan every step outside any CUDA graph capture with this step's - # exact per-row lengths; captured runs read the refreshed buffers. - num_rows, kv_lens = self._kv_lens_host(common_attn_metadata) + # Plan outside any CUDA graph capture; captured runs read the + # refreshed buffers. + num_rows, kv_lens, exact = self._kv_lens_host(common_attn_metadata) # MHA prefills are removed from q before forward_mqa. Plan only the # decode prefix in that case, using the same routing decision as MLA. if metadata.num_prefills > 0 and self._attention_layer._use_sparse_mha( @@ -409,7 +526,12 @@ class FlashInferMLASparseSM90Builder(FlashInferMLASparseMetadataBuilder): ): num_rows = metadata.num_decode_tokens kv_lens = kv_lens[:num_rows] - self.state.plan(num_rows, kv_lens) + if exact: + self.state.plan(num_rows, kv_lens, None, None) + else: + self.state.plan( + num_rows, kv_lens, common_attn_metadata, metadata.req_id_per_token + ) metadata.state = self.state return metadata diff --git a/vllm/v1/attention/backends/mla/prefill/registry.py b/vllm/v1/attention/backends/mla/prefill/registry.py index 1cf1761a77..f7ec31e153 100644 --- a/vllm/v1/attention/backends/mla/prefill/registry.py +++ b/vllm/v1/attention/backends/mla/prefill/registry.py @@ -53,6 +53,10 @@ class MLAPrefillBackendEnum(Enum, metaclass=_MLAPrefillBackendEnumMeta): "AiterFlashAttnPrefillBackend" ) CPU = "vllm.v1.attention.backends.mla.prefill.cpu_sdpa.CPUSDPAMLAPrefillBackend" + ZEN_CPU = ( + "vllm.v1.attention.backends.mla.prefill.zen_cpu_sdpa." + "ZenCPUSDPAMLAPrefillBackend" + ) # Placeholder for third-party/custom backends - must be registered before use # set to None to avoid alias with other backend, whose value is an empty string CUSTOM = None diff --git a/vllm/v1/attention/backends/mla/prefill/selector.py b/vllm/v1/attention/backends/mla/prefill/selector.py index a2b74e4327..0cb29f51b4 100644 --- a/vllm/v1/attention/backends/mla/prefill/selector.py +++ b/vllm/v1/attention/backends/mla/prefill/selector.py @@ -110,8 +110,18 @@ def get_mla_prefill_backend( from vllm.platforms import current_platform if current_platform.is_cpu(): - logger.info_once("Using CPU SDPA MLA prefill backend.") - return MLAPrefillBackendEnum.CPU.get_class() + # CPUs have no compute capability, so the capability-driven priority + # path below does not apply. Prefer an accelerator-specific CPU backend + # when its kernels are present, else the generic SDPA one. + for backend_enum in (MLAPrefillBackendEnum.ZEN_CPU, MLAPrefillBackendEnum.CPU): + try: + cpu_backend_cls = backend_enum.get_class() + except ImportError: + continue + if cpu_backend_cls.is_available(): + logger.info_once("Using %s MLA prefill backend.", backend_enum.name) + return cpu_backend_cls + raise ValueError("No valid CPU MLA prefill backend found.") device_capability = current_platform.get_device_capability() if device_capability is None: diff --git a/vllm/v1/attention/backends/mla/prefill/zen_cpu_sdpa.py b/vllm/v1/attention/backends/mla/prefill/zen_cpu_sdpa.py new file mode 100644 index 0000000000..88c6ce98a0 --- /dev/null +++ b/vllm/v1/attention/backends/mla/prefill/zen_cpu_sdpa.py @@ -0,0 +1,106 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""CPU MLA prefill backend using zentorch SDPA on AMD Zen CPUs. + +Subclasses the generic CPU SDPA backend and replaces the per-request attention +with ``zentorch_sdpa``, which fuses the scale, causal mask, softmax and both +matmuls of the base implementation into a single call. + +``zentorch_sdpa`` does not return a usable log-sum-exp, so requests that need +one (context chunks, prefix/suffix merge) are delegated to the base backend. + +Availability is reported through :meth:`is_available`, so the prefill selector +gates this backend once at selection time rather than per attention call. +""" + +import torch + +from vllm.model_executor.kernels.linear.zentorch_utils import has_zentorch_op +from vllm.v1.attention.backends.mla.prefill.cpu_sdpa import CPUSDPAMLAPrefillBackend + + +class ZenCPUSDPAMLAPrefillBackend(CPUSDPAMLAPrefillBackend): + """MLA prefill backend for AMD Zen CPUs backed by ``zentorch_sdpa``.""" + + @staticmethod + def get_name() -> str: + return "ZEN_CPU_SDPA_MLA" + + @classmethod + def is_available(cls) -> bool: + return has_zentorch_op(["zentorch_sdpa"]) + + @staticmethod + def _sdpa_layout( + q_r: torch.Tensor, + k_r: torch.Tensor, + v_r: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, int]: + """Convert one request's [S, NH, D] tensors to the [1, NH, S, D] layout. + + MLA has ``v_head_dim < qk_head_dim``, which the fused kernel does not + accept, so V is padded up to the query head dim. The original V head + dim is returned so the caller can slice the padding off afterwards. + """ + h_q = q_r.shape[-1] + h_v = v_r.shape[-1] + pad = h_q - h_v + + qs = q_r.transpose(0, 1).unsqueeze(0).contiguous() + ks = k_r.transpose(0, 1).unsqueeze(0).contiguous() + vs = v_r.transpose(0, 1).unsqueeze(0).contiguous() + if pad > 0: + vs = torch.nn.functional.pad(vs, (0, pad)) + return qs, ks, vs, h_v + + def _ragged_sdpa( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + q_cu_seq_lens: torch.Tensor, + kv_cu_seq_lens: torch.Tensor, + causal: bool, + return_softmax_lse: bool, + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + if return_softmax_lse: + # zentorch_sdpa's second return is not a usable LSE, so fall back + # to the base fp32 implementation whenever one is required. + return super()._ragged_sdpa( + q, + k, + v, + q_cu_seq_lens, + kv_cu_seq_lens, + causal, + return_softmax_lse, + ) + + num_reqs = q_cu_seq_lens.numel() - 1 + assert num_reqs == kv_cu_seq_lens.numel() - 1 + + out_chunks: list[torch.Tensor] = [] + for r in range(num_reqs): + q_start = int(q_cu_seq_lens[r]) + q_end = int(q_cu_seq_lens[r + 1]) + kv_start = int(kv_cu_seq_lens[r]) + kv_end = int(kv_cu_seq_lens[r + 1]) + + qs, ks, vs, h_v = self._sdpa_layout( + q[q_start:q_end], + k[kv_start:kv_end], + v[kv_start:kv_end], + ) + + attn, _ = torch.ops.zentorch.zentorch_sdpa( + qs, + ks, + vs, + 0.0, + causal, + attn_mask=None, + scale=self.scale, + ) + out_chunks.append(attn[0].transpose(0, 1)[..., :h_v].to(v.dtype)) + + return torch.cat(out_chunks, dim=0) diff --git a/vllm/v1/attention/backends/utils.py b/vllm/v1/attention/backends/utils.py index e918774088..c97483912d 100644 --- a/vllm/v1/attention/backends/utils.py +++ b/vllm/v1/attention/backends/utils.py @@ -21,7 +21,12 @@ from vllm.config.cache import _layout_from_name from vllm.utils.gpu_sync_debug import gpu_sync_allowed from vllm.utils.math_utils import cdiv from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d, np_to_pinned_tensor -from vllm.v1.kv_cache_interface import KVCacheLayout, KVCacheSpec, MambaSpec +from vllm.v1.kv_cache_interface import ( + AttentionSpec, + KVCacheLayout, + KVCacheSpec, + MambaSpec, +) if TYPE_CHECKING: from vllm.v1.core.sched.output import SchedulerOutput @@ -286,9 +291,10 @@ def resolve_kv_cache_layout( # specs can re-interpret HNC with different sizes as long as the total number of # bytes is the same. If not block-compact, each spec must agree on HNC to alias # the same page (this aliasing is done by the Hybrid Memory Allocator, HMA). + kv_cache_specs = tuple(kv_cache_specs or ()) hnc_shapes = { (spec.num_heads, spec.num_states, spec.page_size_bytes) - for spec in kv_cache_specs or () + for spec in kv_cache_specs } if len(hnc_shapes) > 1: candidates = [m for m in candidates if m.is_block_compact] @@ -298,6 +304,20 @@ def resolve_kv_cache_layout( f"none is in every supported set: {supported_layouts}." ) + dcp_sharding = { + spec.dcp_sharded for spec in kv_cache_specs if isinstance(spec, AttentionSpec) + } + page_sizes = {spec.page_size_bytes for spec in kv_cache_specs} + if len(dcp_sharding) > 1 and len(page_sizes) > 1: + # Sharded target and replicated draft caches need independent groups. + # Block-outer layouts can pack those groups without equalizing pages. + candidates = [layout for layout in candidates if layout.is_block_outermost] + if not candidates: + raise ValueError( + "DCP with a replicated draft and mixed KV page sizes requires " + f"a block-outer KV cache layout; supported sets: {supported_layouts}." + ) + if (requested := envs.VLLM_KV_CACHE_LAYOUT) is not None: layout = _layout_from_name(requested) if layout not in candidates: diff --git a/vllm/v1/core/kv_cache_coordinator.py b/vllm/v1/core/kv_cache_coordinator.py index 8061999409..d69b2f5fb3 100644 --- a/vllm/v1/core/kv_cache_coordinator.py +++ b/vllm/v1/core/kv_cache_coordinator.py @@ -11,7 +11,6 @@ from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector from vllm.v1.core.kv_cache_utils import ( BlockHash, KVCacheBlock, - dcp_world_size_for_kv_cache_spec, ) from vllm.v1.core.single_type_kv_cache_manager import ( CrossAttentionManager, @@ -20,6 +19,7 @@ from vllm.v1.core.single_type_kv_cache_manager import ( get_manager_for_kv_cache_spec, ) from vllm.v1.kv_cache_interface import ( + AttentionSpec, FullAttentionSpec, KVCacheConfig, KVCacheSpec, @@ -140,9 +140,7 @@ class KVCacheCoordinator(ABC): role=kv_cache_group.role, enable_caching=enable_caching, kv_cache_group_id=i, - dcp_world_size=dcp_world_size_for_kv_cache_spec( - kv_cache_group.kv_cache_spec, dcp_world_size - ), + dcp_world_size=dcp_world_size, pcp_world_size=pcp_world_size, scheduler_block_size=self.scheduler_block_size, needs_kv_cache_zeroing=self.kv_cache_config.needs_kv_cache_zeroing, @@ -664,16 +662,18 @@ class HybridKVCacheCoordinator(KVCacheCoordinator): f"hash_block_size. block_sizes={cacheable_block_sizes}, " f"hash_block_size={hash_block_size}" ) - assert pcp_world_size == 1, "PCP not support hybrid attn now." - if dcp_world_size > 1: - # DCP shards full-attention KV across ranks and replicates Mamba - # state; other spec types (e.g. sliding window) have no DCP-aware - # handling yet, so reject them explicitly. - for g in kv_cache_config.kv_cache_groups: - assert isinstance(g.kv_cache_spec, (FullAttentionSpec, MambaSpec)), ( + for g in kv_cache_config.kv_cache_groups: + spec = g.kv_cache_spec + replicated = not spec.dcp_sharded + if pcp_world_size > 1: + assert isinstance(spec, FullAttentionSpec) or ( + isinstance(spec, AttentionSpec) and replicated + ), "PCP only supports full attention and replicated draft groups." + if dcp_world_size > 1: + assert isinstance(spec, FullAttentionSpec) or replicated, ( "DCP with hybrid KV cache layouts only supports " - "full-attention and Mamba groups, got: " - f"{type(g.kv_cache_spec).__name__}." + "full-attention, Mamba, and replicated draft groups, got: " + f"{type(spec).__name__}." ) # Fine-grained hash hits require Mamba "align" and compatible cache # managers in every group. TP needs hashing finer than the Mamba block; diff --git a/vllm/v1/core/kv_cache_utils.py b/vllm/v1/core/kv_cache_utils.py index ee0957839e..3f4f4e7bb1 100644 --- a/vllm/v1/core/kv_cache_utils.py +++ b/vllm/v1/core/kv_cache_utils.py @@ -682,12 +682,7 @@ def hash_block_tokens( def resolve_dcp_kv_block_size(spec: KVCacheSpec, dcp_world_size: int) -> int: """Return the token span of a cache block under DCP.""" - layer_specs = iter_layer_specs(spec) - if len(layer_specs) > 0 and all( - isinstance(layer_spec, AttentionSpec) for layer_spec in layer_specs - ): - return spec.block_size * dcp_world_size - return spec.block_size + return spec.block_size * (dcp_world_size if spec.dcp_sharded else 1) def resolve_dcp_kv_cache_spec(spec: KVCacheSpec, dcp_world_size: int) -> KVCacheSpec: @@ -707,28 +702,6 @@ def resolve_dcp_kv_cache_spec(spec: KVCacheSpec, dcp_world_size: int) -> KVCache return replace(spec, block_size=block_size) -def dcp_world_size_for_kv_cache_spec(spec: KVCacheSpec, dcp_world_size: int) -> int: - """Return the DCP size that owns this group's block geometry. - - Full-attention KV (including MLA) is sharded across DCP ranks, so prefix - hashing and manager ``block_size`` use the process DCP size. Other specs - keep replicated per-rank state (Mamba, sliding window, chunked-local) and - must keep ``dcp_world_size=1`` even when the process runs with DCP > 1. - - Draft MLA groups on the sharded DSpark path are ``FullAttentionSpec`` / - ``MLAAttentionSpec`` and therefore keep the process DCP size. A replicated - draft group would need a different spec, not this helper. - """ - if dcp_world_size <= 1: - return 1 - inner = spec - if isinstance(spec, UniformTypeKVCacheSpecs): - inner = next(iter(spec.kv_cache_specs.values())) - if isinstance(inner, FullAttentionSpec): - return dcp_world_size - return 1 - - def resolve_kv_cache_block_sizes( kv_cache_config: KVCacheConfig, vllm_config: VllmConfig, @@ -752,6 +725,8 @@ def resolve_kv_cache_block_sizes( groups = kv_cache_config.kv_cache_groups if len(groups) <= 1: + if groups and not groups[0].kv_cache_spec.dcp_sharded: + dcp = 1 bs = cache_config.block_size * dcp return bs, bs @@ -1935,18 +1910,21 @@ def unify_hybrid_kv_cache_specs(kv_cache_spec: dict[str, KVCacheSpec]): kv_cache_spec: The kv cache spec of each attention layer in the model """ - if is_kv_cache_spec_uniform( - kv_cache_spec - ) or UniformTypeKVCacheSpecs.is_uniform_type(kv_cache_spec): - return - - logger.warning( - "Hybrid KV cache manager is disabled for this hybrid model, " - "This means we do not enable any optimizations for saving KV cache " - "memory (e.g., dropping the KV cache outside the sliding window). " - "The compute of layers like sliding window is still saved." - ) - kv_cache_spec.update(_promote_local_kv_cache_specs(kv_cache_spec)) + groups: defaultdict[bool, dict[str, KVCacheSpec]] = defaultdict(dict) + for name, spec in kv_cache_spec.items(): + replicated = isinstance(spec, AttentionSpec) and not spec.dcp_sharded + groups[replicated][name] = spec + for specs in groups.values(): + promoted_specs = _promote_local_kv_cache_specs(specs) + if promoted_specs == specs: + continue + logger.warning( + "Hybrid KV cache manager is disabled for this hybrid model, " + "This means we do not enable any optimizations for saving KV cache " + "memory (e.g., dropping the KV cache outside the sliding window). " + "The compute of layers like sliding window is still saved." + ) + kv_cache_spec.update(promoted_specs) def _approximate_gcd(values: Sequence[int], *, lower_bound: int | None = None) -> int: @@ -2276,7 +2254,7 @@ def _ensure_min_page_size( scaled: list[KVCacheGroupSpec] = [] for g in groups: s = g.kv_cache_spec - kw: dict[str, int] = {"block_size": s.block_size * scale} + kw: dict[str, Any] = {"block_size": s.block_size * scale} if isinstance(s, (AttentionSpec, MambaSpec)) and s.page_size_padded is not None: kw["page_size_padded"] = s.page_size_padded * scale scaled.append(KVCacheGroupSpec(g.layer_names, replace(s, **kw))) diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index 4c56ba8d81..7c0c7e7fff 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -106,11 +106,9 @@ class SingleTypeKVCacheManager(ABC): # managers have been validated by the coordinator. self.cache_hit_alignment_tokens = scheduler_block_size # The block size for this manager; used for actual block allocation. - self.block_size = kv_cache_spec.block_size - self.dcp_world_size = dcp_world_size - self.pcp_world_size = pcp_world_size - if dcp_world_size > 1: - self.block_size *= dcp_world_size + self.dcp_world_size = dcp_world_size if kv_cache_spec.dcp_sharded else 1 + self.pcp_world_size = pcp_world_size if kv_cache_spec.dcp_sharded else 1 + self.block_size = kv_cache_spec.block_size * self.dcp_world_size self.kv_cache_spec = kv_cache_spec self.block_pool = block_pool self.enable_caching = enable_caching @@ -1455,10 +1453,6 @@ class MambaManager(SingleTypeKVCacheManager): self, kv_cache_spec: MambaSpec, block_pool: BlockPool, **kwargs ) -> None: super().__init__(kv_cache_spec, block_pool, **kwargs) - # Mamba layers use TP instead of DCP, so each rank holds the full - # recurrent state. Undo the DCP/PCP block_size scaling that the base - # class applies for attention groups whose KV cache is partitioned. - self.block_size = kv_cache_spec.block_size self.mamba_cache_mode = kv_cache_spec.mamba_cache_mode self.num_speculative_blocks: int = kv_cache_spec.num_speculative_blocks self.has_prefill_checkpoint_blocks = ( diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py index b36caf092c..5c09a0059e 100644 --- a/vllm/v1/engine/core.py +++ b/vllm/v1/engine/core.py @@ -219,10 +219,7 @@ class EngineCore: logger.debug("Batch queue is enabled with size %d", self.batch_queue_size) self.batch_queue = deque(maxlen=self.batch_queue_size) - self.is_ec_consumer = ( - vllm_config.ec_transfer_config is None - or vllm_config.ec_transfer_config.is_ec_consumer - ) + self.is_mm_encoder_only = vllm_config.is_mm_encoder_only self.is_pooling_model = vllm_config.model_config.runner_type == "pooling" self.request_block_hasher: Callable[[Request], list[BlockHash]] | None = None @@ -702,7 +699,7 @@ class EngineCore: exec_future = self.model_executor.execute_model( scheduler_output, non_block=True ) - if self.is_ec_consumer: + if not self.is_mm_encoder_only: model_executed = scheduler_output.total_num_scheduled_tokens > 0 if self.is_pooling_model or not model_executed: diff --git a/vllm/v1/kv_cache_interface.py b/vllm/v1/kv_cache_interface.py index 499b50a3a2..5791704b2b 100644 --- a/vllm/v1/kv_cache_interface.py +++ b/vllm/v1/kv_cache_interface.py @@ -158,6 +158,8 @@ class KVCacheSpec: # number of tokens in a block block_size: int + dcp_sharded: bool = field(default=False, kw_only=True) + """Whether DCP shards this cache's token positions across ranks.""" block_stride_alignment: int | None = field(default=None, kw_only=True) """Required byte alignment between physical blocks, including packed layers.""" @@ -481,6 +483,7 @@ class HiSparseResidentSpec(KVCacheSpec): @dataclass(frozen=True, kw_only=True) class AttentionSpec(KVCacheSpec): + dcp_sharded: bool = True num_kv_heads: int head_size: int dtype: torch.dtype @@ -536,7 +539,9 @@ class AttentionSpec(KVCacheSpec): def max_num_blocks_per_req(self, vllm_config: VllmConfig, max_len: int) -> int: parallel_config = vllm_config.parallel_config - kv_shard_count = parallel_config.decode_context_parallel_size + kv_shard_count = ( + parallel_config.decode_context_parallel_size if self.dcp_sharded else 1 + ) return cdiv(max_len, self.block_size * kv_shard_count) @@ -568,7 +573,7 @@ class FullAttentionSpec(AttentionSpec): def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int: max_model_len = vllm_config.model_config.max_model_len dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size - if dcp_world_size > 1: + if self.dcp_sharded and dcp_world_size > 1: max_model_len = cdiv(max_model_len, dcp_world_size) return cdiv(max_model_len, self.block_size) * self.page_size_bytes @@ -612,6 +617,7 @@ class FullAttentionSpec(AttentionSpec): head_size_v=specs[0].head_size_v, dtype=specs[0].dtype, kv_quant_mode=specs[0].kv_quant_mode, + dcp_sharded=specs[0].dcp_sharded, page_size_padded=specs[0].page_size_padded, num_head_slots=specs[0].num_head_slots, state_content_bytes=specs[0].state_content_bytes, @@ -698,6 +704,7 @@ class MLAAttentionSpec(FullAttentionSpec): head_size=specs[0].head_size, dtype=specs[0].dtype, kv_quant_mode=specs[0].kv_quant_mode, + dcp_sharded=specs[0].dcp_sharded, page_size_padded=specs[0].page_size_padded, num_head_slots=specs[0].num_head_slots, state_content_bytes=specs[0].state_content_bytes, @@ -760,6 +767,7 @@ class RSWASpec(FullAttentionSpec): head_size_v=base.head_size_v, dtype=base.dtype, kv_quant_mode=base.kv_quant_mode, + dcp_sharded=base.dcp_sharded, page_size_padded=base.page_size_padded, num_head_slots=base.num_head_slots, state_content_bytes=base.state_content_bytes, @@ -850,9 +858,10 @@ class SlidingWindowSpec(AttentionSpec): return cdiv(num_tokens, self.block_size) + 1 def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int: - assert vllm_config.parallel_config.decode_context_parallel_size == 1, ( - "DCP not support sliding window." - ) + assert ( + not self.dcp_sharded + or vllm_config.parallel_config.decode_context_parallel_size == 1 + ), "DCP not support sliding window." max_blocks = self.max_admission_blocks_per_request( max_in_flight_tokens=vllm_config.max_in_flight_tokens, max_model_len=vllm_config.model_config.max_model_len, @@ -950,6 +959,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec): extra_retained_set = set(spec.extra_retained_tokens for spec in specs) bounded_replay_set = set(spec.bounded_replay for spec in specs) block_stride_alignment_set = {spec.block_stride_alignment for spec in specs} + assert len({spec.dcp_sharded for spec in specs}) == 1 assert ( len(cache_dtype_str_set) == 1 and len(tokens_per_state_set) == 1 @@ -969,6 +979,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec): num_kv_heads=specs[0].num_kv_heads, head_size=specs[0].head_size, dtype=specs[0].dtype, + dcp_sharded=specs[0].dcp_sharded, page_size_padded=specs[0].page_size_padded, num_head_slots=specs[0].num_head_slots, state_content_bytes=specs[0].state_content_bytes, @@ -1185,6 +1196,7 @@ class SinkFullAttentionSpec(FullAttentionSpec): block_stride_alignment=specs[0].block_stride_alignment, dtype=specs[0].dtype, kv_quant_mode=specs[0].kv_quant_mode, + dcp_sharded=specs[0].dcp_sharded, page_size_padded=specs[0].page_size_padded, num_head_slots=specs[0].num_head_slots, state_content_bytes=specs[0].state_content_bytes, @@ -1217,6 +1229,11 @@ class UniformTypeKVCacheSpecs(KVCacheSpec): kv_cache_specs: dict[str, KVCacheSpec] + def __post_init__(self): + super().__post_init__() + if self.kv_cache_specs: + object.__setattr__(self, "dcp_sharded", self.first_spec.dcp_sharded) + @property def prefix_cacheable(self) -> bool: return all(spec.prefix_cacheable for spec in self.kv_cache_specs.values()) @@ -1270,6 +1287,8 @@ class UniformTypeKVCacheSpecs(KVCacheSpec): if len(block_sizes) > 1: # Different block sizes, not uniform. return False + if len({spec.dcp_sharded for spec in kv_cache_specs.values()}) > 1: + return False first_spec = next(iter(kv_cache_specs.values())) return first_spec.is_uniform_with_collection(kv_cache_specs) diff --git a/vllm/v1/simple_kv_offload/manager.py b/vllm/v1/simple_kv_offload/manager.py index 96bfb35a67..f70afaa343 100644 --- a/vllm/v1/simple_kv_offload/manager.py +++ b/vllm/v1/simple_kv_offload/manager.py @@ -4,7 +4,7 @@ import contextlib from collections.abc import Iterable -from dataclasses import dataclass, field, replace +from dataclasses import dataclass, field, fields, replace from enum import Enum, auto from typing import TYPE_CHECKING, Any @@ -27,13 +27,13 @@ from vllm.v1.core.kv_cache_coordinator import ( from vllm.v1.core.kv_cache_utils import ( BlockHashWithGroupId, ExternalBlockHash, - dcp_world_size_for_kv_cache_spec, generate_block_hash_extra_keys, get_block_hash, get_group_id, make_block_hash_with_group_id, maybe_convert_block_hash, resolve_block_hashes, + resolve_dcp_kv_block_size, ) from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.kv_cache_interface import ( @@ -48,6 +48,11 @@ from vllm.v1.simple_kv_offload.metadata import ( SimpleCPUOffloadMetadata, SimpleCPUOffloadWorkerMetadata, ) +from vllm.v1.simple_kv_offload.metrics import ( + OUTCOME_TO_FIELD, + MetricName, + SimpleCPUOffloadStats, +) if TYPE_CHECKING: from vllm.v1.core.kv_cache_manager import KVCacheBlocks @@ -118,7 +123,8 @@ class BoundaryStoreStats: symptom is a lower cache hit rate with nothing in the logs, so each decline reason is counted and exposed through ``SimpleCPUOffloadConnector.get_boundary_store_stats()``. Reset by - ``reset()``; not otherwise cleared. + ``reset()``, which carries any not-yet-drained deltas; not otherwise + cleared. """ published: int = 0 @@ -143,6 +149,7 @@ class SimpleCPUOffloadScheduler: hash_block_size: int, lazy_offload: bool = False, disk_capacity_bytes: int = 0, + use_page_cache: bool = False, ): self.vllm_config = vllm_config self.kv_cache_config = kv_cache_config @@ -169,6 +176,13 @@ class SimpleCPUOffloadScheduler: self.cpu_kv_cache_config.prefix_cacheable_group_ids ) self.kv_event_medium = MEDIUM_STORAGE if disk_capacity_bytes > 0 else MEDIUM_CPU + # Must track kv_event_medium. + self._info_labelvalues: tuple[str, ...] = ( + "disk" if disk_capacity_bytes > 0 else "cpu", + str(use_page_cache).lower(), + str(lazy_offload).lower(), + str(self.num_cpu_blocks), + ) # Find the full attention kv group for prefix cache matching. self.fa_gidx = -1 for g_idx, g in enumerate(self.cpu_kv_cache_config.kv_cache_groups): @@ -251,6 +265,9 @@ class SimpleCPUOffloadScheduler: self._load_event_counter: int = 0 self._store_event_counter: int = 0 self.boundary_store_stats = BoundaryStoreStats() + # Interval stats state drained by get_stats() + self._boundary_stats_snapshot = BoundaryStoreStats() + self._interval_load_blocks_completed = 0 # For TP/PP: track partial store completions across steps. # Events must be reported by all world_size workers before considered complete. @@ -302,11 +319,7 @@ class SimpleCPUOffloadScheduler: target = 0 for g in kv_cache_config.prefix_cacheable_groups: spec = g.kv_cache_spec - # Only full attention is sharded across DCP ranks; replicated specs - # (mamba, sliding window, chunked-local) keep their own block size. - block_size = spec.block_size * dcp_world_size_for_kv_cache_spec( - spec, cp_world_size - ) + block_size = resolve_dcp_kv_block_size(spec, cp_world_size) if isinstance(spec, MambaSpec): target += 2 elif isinstance(spec, SlidingWindowSpec): @@ -955,6 +968,42 @@ class SimpleCPUOffloadScheduler: def get_boundary_store_stats(self) -> BoundaryStoreStats: return replace(self.boundary_store_stats) + def get_stats(self) -> SimpleCPUOffloadStats: + """Drain per-step stats for the connector's stats hooks.""" + stats = SimpleCPUOffloadStats() + + current = self.boundary_store_stats + for outcome, field_name in OUTCOME_TO_FIELD.items(): + delta = getattr(current, field_name) - getattr( + self._boundary_stats_snapshot, field_name + ) + if delta > 0: + stats.increase_counter(MetricName.SAVE_OUTCOMES, delta, (outcome,)) + self._boundary_stats_snapshot = replace(current) + + if self._interval_load_blocks_completed: + stats.increase_counter( + MetricName.LOAD_BLOCKS, + self._interval_load_blocks_completed, + ) + self._interval_load_blocks_completed = 0 + + stats.set_gauge( + MetricName.USED_BLOCKS, + self.num_cpu_blocks - self.cpu_block_pool.get_num_free_blocks(), + ) + pending = sum( + len(t.cpu_block_ids) + for t in ( + *self._store_event_to_blocks.values(), + *self._pending_finished_stores, + *self._abandoned_store_event_to_blocks.values(), + ) + ) + stats.set_gauge(MetricName.PENDING_STORE_BLOCKS, pending) + stats.set_gauge(MetricName.INFO, 1, self._info_labelvalues) + return stats + def update_connector_output(self, connector_output: KVConnectorOutput) -> None: """Handle async transfer completions from worker. @@ -965,7 +1014,9 @@ class SimpleCPUOffloadScheduler: """ # --- Load completions --- for req_id in list(connector_output.finished_recving or []): - self._cleanup_load_request(req_id) + completed_blocks = self._cleanup_load_request(req_id) + if completed_blocks: + self._interval_load_blocks_completed += completed_blocks # --- Store completions --- meta = connector_output.kv_connector_worker_meta @@ -1270,18 +1321,22 @@ class SimpleCPUOffloadScheduler: if blocks_to_free: self.cpu_block_pool.free_blocks(blocks_to_free) - def _cleanup_load_request(self, req_id: str) -> None: + def _cleanup_load_request(self, req_id: str) -> int: """Release all load resources for a request. Shared between request_finished() and update_connector_output() paths. Removes the request from _reqs_to_load, cleans up event mappings, and frees CPU/GPU touch refs. + + Returns the number of blocks in the load if it had been issued to + the worker, else 0 (never-issued loads are not counted as completed + by the caller). """ state = self._reqs_to_load.pop(req_id, None) if state is None: state = self._abandoned_reqs_to_load.pop(req_id, None) if state is None: - return + return 0 # Remove from load event mapping (only this req, not whole event) if state.load_event is not None: reqs = self._load_event_to_reqs.get(state.load_event) @@ -1304,6 +1359,10 @@ class SimpleCPUOffloadScheduler: for bid in state.transfer_meta.gpu_block_ids ) + if state.load_event is None: + return 0 + return len(state.transfer_meta.gpu_block_ids) + def _cleanup_store_request(self, req_id: str) -> None: """Release store metadata for a request. @@ -1363,7 +1422,20 @@ class SimpleCPUOffloadScheduler: if event_idx in self._abandoned_store_event_to_blocks } self._cursor = None - self.boundary_store_stats = BoundaryStoreStats() + # Seed the fresh counters with outcomes since the last drain so a + # mid-interval reset() does not drop them from the next report. + undrained = BoundaryStoreStats( + **{ + f.name: max( + 0, + getattr(self.boundary_store_stats, f.name) + - getattr(self._boundary_stats_snapshot, f.name), + ) + for f in fields(BoundaryStoreStats) + } + ) + self.boundary_store_stats = undrained + self._boundary_stats_snapshot = BoundaryStoreStats() # NOTE: _load_event_counter / _store_event_counter are not # reset as they are monotonic and must stay ahead of the workers # high-water marks to avoid event index collisions diff --git a/vllm/v1/simple_kv_offload/metrics.py b/vllm/v1/simple_kv_offload/metrics.py new file mode 100644 index 0000000000..9017d40cfd --- /dev/null +++ b/vllm/v1/simple_kv_offload/metrics.py @@ -0,0 +1,284 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Prometheus metrics for SimpleCPUOffloadConnector. + +The stats payload uses the same self-describing wire format as the +offloading connector, so it survives IPC serialization without the +metric classes on the receiving side. +""" + +from dataclasses import dataclass +from typing import Any + +from vllm.config import VllmConfig +from vllm.distributed.kv_transfer.kv_connector.v1.metrics import ( + KVConnectorPromMetrics, + KVConnectorStats, + PromMetric, + PromMetricT, +) + + +class MetricName: + """Metric names for SimpleCPUOffloadConnector.""" + + SAVE_OUTCOMES = "vllm:simple_kv_offload_save_outcomes_total" + LOAD_BLOCKS = "vllm:simple_kv_offload_load_blocks_total" + USED_BLOCKS = "vllm:simple_kv_offload_used_blocks" + PENDING_STORE_BLOCKS = "vllm:simple_kv_offload_pending_store_blocks" + INFO = "vllm:simple_kv_offload_info" + + +# outcome label value -> BoundaryStoreStats field name. +# ``published`` is the sum of all outcomes and is not labeled. +OUTCOME_TO_FIELD = { + "stored": "stored", + "dropped_cpu_full": "dropped_cpu_full", + "dropped_request_gone": "dropped_request_gone", + "dropped_null_block": "dropped_null_block", + "dropped_not_hashed": "dropped_not_hashed", + "skipped_already_cached": "skipped_already_cached", + "skipped_in_flight": "skipped_in_flight", +} + +INFO_LABELS = ["backend", "page_cache", "lazy_offload", "capacity_blocks"] + + +class _MetricType: + """Type tags embedded in the serialized stats payload.""" + + COUNTER = "counter" + GAUGE = "gauge" + + +class _StatsKey: + """Top-level keys in the serialized stats dict.""" + + # Maps metric name -> _MetricType value + TYPES = "types" + # Maps metric name -> {label values tuple -> observed value} + DATA = "data" + + +@dataclass +class SimpleCPUOffloadStats(KVConnectorStats): + """Stats payload for SimpleCPUOffloadConnector. + + The ``data`` dict is structured as:: + + { + _StatsKey.TYPES: {name: _MetricType.*, ...}, + _StatsKey.DATA: {name: {labelvalues: value, ...}, ...}, + } + + Counter values are summed when aggregating, gauge values keep the + latest snapshot, and unlabeled metrics use ``()`` as their + labelvalues tuple. + """ + + def __post_init__(self): + if _StatsKey.DATA not in self.data: + self.reset() + + def reset(self): + self.data: dict[str, Any] = { + _StatsKey.TYPES: {}, + _StatsKey.DATA: {}, + } + + @property + def _types(self) -> dict[str, str]: + return self.data[_StatsKey.TYPES] + + @property + def _values(self) -> dict[str, Any]: + return self.data[_StatsKey.DATA] + + def aggregate(self, other: "KVConnectorStats") -> "KVConnectorStats": + if other.is_empty(): + return self + assert isinstance(other, SimpleCPUOffloadStats) + for key, other_label_values in other._values.items(): + type_str = other._types.get(key) + if type_str is None: + raise AssertionError(f"Unknown simple KV offload stats key: {key}") + self._types.setdefault(key, type_str) + current_label_values = self._values.setdefault(key, {}) + for labelvalues, value in other_label_values.items(): + if type_str == _MetricType.COUNTER: + assert isinstance(value, int | float) + current_label_values[labelvalues] = ( + current_label_values.get(labelvalues, 0) + value + ) + elif type_str == _MetricType.GAUGE: + assert isinstance(value, int | float) + current_label_values[labelvalues] = value + else: + raise AssertionError( + f"Unknown metric type '{type_str}' for key: {key}" + ) + return self + + def reduce(self) -> dict[str, int | float]: + """Reduce the interval observations to a flat summary for logging.""" + return_dict: dict[str, int | float] = {} + for key, label_value_map in self._values.items(): + if key == MetricName.INFO: + # Constant config fingerprint; exclude from the log summary. + continue + type_str = self._types.get(key) + if type_str is None: + raise AssertionError(f"Unknown simple KV offload stats key: {key}") + for labelvalues, value in label_value_map.items(): + key_with_labels = f"{key}:{labelvalues}" if labelvalues else key + if type_str in (_MetricType.COUNTER, _MetricType.GAUGE): + assert isinstance(value, int | float) + return_dict[key_with_labels] = value + else: + raise AssertionError( + f"Unknown metric type '{type_str}' for key: {key}" + ) + return return_dict + + def is_empty(self) -> bool: + return not self.data.get(_StatsKey.DATA) + + def increase_counter( + self, + counter_name: str, + counter_increase_value: int | float = 1, + labelvalues: tuple[str, ...] = (), + ) -> None: + """Increase a counter on the stats payload.""" + self._types.setdefault(counter_name, _MetricType.COUNTER) + counter_values = self._values.setdefault(counter_name, {}) + counter_values[labelvalues] = ( + counter_values.get(labelvalues, 0) + counter_increase_value + ) + + def set_gauge( + self, + gauge_name: str, + gauge_value: int | float, + labelvalues: tuple[str, ...] = (), + ) -> None: + """Set a gauge snapshot on the stats payload.""" + self._types.setdefault(gauge_name, _MetricType.GAUGE) + gauge_values = self._values.setdefault(gauge_name, {}) + gauge_values[labelvalues] = gauge_value + + +class SimpleCPUOffloadPromMetrics(KVConnectorPromMetrics): + """Prometheus registration and recording for SimpleCPUOffloadConnector.""" + + def __init__( + self, + vllm_config: VllmConfig, + metric_types: dict[type[PromMetric], type[PromMetricT]], + labelnames: list[str], + per_engine_labelvalues: dict[int, list[object]], + ): + super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues) + # (engine_idx, metric_name, labelvalues) -> bound child metric + self._metrics: dict[tuple[int, str, tuple[str, ...]], PromMetricT] = {} + + self._counter_save_outcomes = self._counter_cls( + name="vllm:simple_kv_offload_save_outcomes_total", + documentation=( + "Store admission decisions in SimpleCPUOffloadConnector, by " + "outcome. Outcomes classify eager-mode boundary hand-off " + "stores; lazy-mode stores are not classified." + ), + labelnames=self._labelnames + ["outcome"], + ) + self._counter_load_blocks = self._counter_cls( + name="vllm:simple_kv_offload_load_blocks_total", + documentation=( + "KV blocks restored to GPU from the offload pool, counted " + "when a load finishes. This is prefill work avoided " + "through offload cache hits." + ), + labelnames=self._labelnames, + ) + self._gauge_used_blocks = self._gauge_cls( + name="vllm:simple_kv_offload_used_blocks", + documentation=( + "Offload-pool blocks currently pinned by in-flight " + "transfers or cache hits (capacity minus free). Evictable " + "cached blocks are not counted; capacity_blocks is on " + "vllm:simple_kv_offload_info." + ), + labelnames=self._labelnames, + multiprocess_mode="mostrecent", + ) + self._gauge_pending_store_blocks = self._gauge_cls( + name="vllm:simple_kv_offload_pending_store_blocks", + documentation=( + "Offload blocks in pending stores: queued for dispatch, " + "issued to workers, or awaiting release after a cache " + "reset. A persistently growing value indicates a stuck " + "transfer." + ), + labelnames=self._labelnames, + multiprocess_mode="mostrecent", + ) + self._gauge_info = self._gauge_cls( + name="vllm:simple_kv_offload_info", + documentation=( + "SimpleCPUOffloadConnector deployment facts. Value is " + "always 1. backend is cpu or disk; page_cache and " + "lazy_offload are true/false; capacity_blocks is the " + "offload pool size." + ), + labelnames=self._labelnames + INFO_LABELS, + multiprocess_mode="mostrecent", + ) + + self._defs: dict[str, PromMetricT] = { + MetricName.SAVE_OUTCOMES: self._counter_save_outcomes, + MetricName.LOAD_BLOCKS: self._counter_load_blocks, + MetricName.USED_BLOCKS: self._gauge_used_blocks, + MetricName.PENDING_STORE_BLOCKS: self._gauge_pending_store_blocks, + MetricName.INFO: self._gauge_info, + } + self._num_label_values: dict[str, int] = { + MetricName.SAVE_OUTCOMES: 1, + MetricName.LOAD_BLOCKS: 0, + MetricName.USED_BLOCKS: 0, + MetricName.PENDING_STORE_BLOCKS: 0, + MetricName.INFO: len(INFO_LABELS), + } + + def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0): + """Observe transfer statistics.""" + metric_types = transfer_stats_data.get(_StatsKey.TYPES, {}) + metric_data = transfer_stats_data.get(_StatsKey.DATA, {}) + for key, label_value_map in metric_data.items(): + prom_metric = self._defs.get(key) + if prom_metric is None: + raise AssertionError(f"Unknown simple KV offload stats key: {key}") + type_str = metric_types.get(key) + for labelvalues, value in label_value_map.items(): + if len(labelvalues) != self._num_label_values[key]: + raise AssertionError( + f"Metric {key} expects " + f"{self._num_label_values[key]} labels, " + f"got {len(labelvalues)}" + ) + child = self._metrics.get((engine_idx, key, labelvalues)) + if child is None: + engine_labelvalues = self.per_engine_labelvalues[engine_idx] + child = prom_metric.labels( + *(engine_labelvalues + list(labelvalues)) + ) + self._metrics[(engine_idx, key, labelvalues)] = child + if type_str == _MetricType.COUNTER: + assert isinstance(value, int | float) + child.inc(value) + elif type_str == _MetricType.GAUGE: + assert isinstance(value, int | float) + child.set(value) + else: + raise AssertionError( + f"Unknown metric type '{type_str}' for key: {key}" + ) diff --git a/vllm/v1/spec_decode/draft_model.py b/vllm/v1/spec_decode/draft_model.py index 08542f03a1..17be2732e6 100644 --- a/vllm/v1/spec_decode/draft_model.py +++ b/vllm/v1/spec_decode/draft_model.py @@ -81,13 +81,20 @@ class DraftModelProposer(SpecDecodeBaseProposer): def _create_draft_vllm_config(self) -> VllmConfig: base = super()._create_draft_vllm_config() spec = self.speculative_config + target_parallel = base.parallel_config return replace( base, quant_config=None, parallel_config=replace( spec.draft_parallel_config, - rank=self.vllm_config.parallel_config.rank, + rank=target_parallel.rank, + decode_context_parallel_size=( + target_parallel.decode_context_parallel_size + ), + cp_kv_cache_interleave_size=target_parallel.cp_kv_cache_interleave_size, + dcp_comm_backend=target_parallel.dcp_comm_backend, + dcp_q_replicate=target_parallel.dcp_q_replicate, ), model_config=spec.draft_model_config, ) diff --git a/vllm/v1/worker/cp_utils.py b/vllm/v1/worker/cp_utils.py index 79651759aa..bcf56dc0bf 100644 --- a/vllm/v1/worker/cp_utils.py +++ b/vllm/v1/worker/cp_utils.py @@ -40,6 +40,8 @@ def check_attention_cp_compatibility( layer_impl = getattr(layer, "impl", None) if layer_impl is None: continue + if not check_pcp and layer_impl.dcp_world_size == 1: + continue if vllm_config.speculative_config is not None and interleave_size > 1: assert layer_impl.supports_mtp_with_cp_non_trivial_interleave_size, ( "MTP with cp_kv_cache_interleave_size > 1 is not " diff --git a/vllm/v1/worker/gpu/attn_utils.py b/vllm/v1/worker/gpu/attn_utils.py index 423bcebb2d..19de125a9b 100644 --- a/vllm/v1/worker/gpu/attn_utils.py +++ b/vllm/v1/worker/gpu/attn_utils.py @@ -29,6 +29,7 @@ from vllm.v1.kv_cache_interface import ( AttentionSpec, KVCacheConfig, KVCacheSpec, + MambaSpec, UniformTypeKVCacheSpecs, ) from vllm.v1.worker.gpu.model_states.interface import ModelSpecificAttnMetadata @@ -462,6 +463,10 @@ def build_attn_metadata( attn_metadata: dict[str, Any] = {} token_to_req_indices: torch.Tensor | None = None + # Mamba groups with the same spec and builder differ only in their state + # indices, so later groups re-gather those from the first group's metadata. + # Also at capture, so FULL graphs share the batch-level buffers. + cached_metadata: dict[tuple[KVCacheSpec, type], Any] = {} num_kv_cache_groups = len(kv_cache_config.kv_cache_groups) for i in range(num_kv_cache_groups): if not attn_groups[i]: @@ -514,7 +519,16 @@ def build_attn_metadata( for attn_group in attn_groups[i]: attn_metadata_builder = attn_group.get_metadata_builder(ubatch_idx) - if for_cudagraph_capture: + reuse_key = None + if attn_metadata_builder.supports_update_block_table and isinstance( + attn_group.kv_cache_spec, MambaSpec + ): + reuse_key = (attn_group.kv_cache_spec, type(attn_metadata_builder)) + if reuse_key in cached_metadata: + metadata = attn_metadata_builder.update_block_table( + cached_metadata[reuse_key], block_table, slot_mapping + ) + elif for_cudagraph_capture: metadata = attn_metadata_builder.build_for_cudagraph_capture( common_attn_metadata ) @@ -532,6 +546,8 @@ def build_attn_metadata( common_attn_metadata=common_attn_metadata, **attn_metadata_extra_kwargs, ) + if reuse_key is not None and reuse_key not in cached_metadata: + cached_metadata[reuse_key] = metadata for layer_name in attn_group.layer_names: attn_metadata[layer_name] = metadata token_to_req_indices = common_attn_metadata._token_to_req_indices_cache diff --git a/vllm/v1/worker/gpu/block_table.py b/vllm/v1/worker/gpu/block_table.py index 096d9b99aa..8cf3ba98dd 100644 --- a/vllm/v1/worker/gpu/block_table.py +++ b/vllm/v1/worker/gpu/block_table.py @@ -27,6 +27,7 @@ class BlockTables: cp_rank: int = 0, cp_interleave: int = 1, slot_mapping_enabled: list[bool] | None = None, + dcp_sharded: list[bool] | None = None, ): self.block_sizes = block_sizes self.kernel_block_sizes = kernel_block_sizes @@ -44,6 +45,10 @@ class BlockTables: slot_mapping_enabled = [True] * self.num_kv_cache_groups assert len(slot_mapping_enabled) == self.num_kv_cache_groups self._slot_mapping_enabled = slot_mapping_enabled + if dcp_sharded is None: + dcp_sharded = [True] * self.num_kv_cache_groups + assert len(dcp_sharded) == self.num_kv_cache_groups + self.dcp_sharded = torch.tensor(dcp_sharded, dtype=torch.bool, device=device) self.blocks_per_kv_block = [ bs // kbs for bs, kbs in zip(block_sizes, kernel_block_sizes) @@ -210,6 +215,7 @@ class BlockTables: self.block_sizes_tensor, self.kernel_block_sizes_tensor, self.slot_mapping_enabled, + self.dcp_sharded, slot_mappings, slot_mappings.stride(0), self.cp_rank, @@ -283,6 +289,7 @@ def _compute_slot_mappings_kernel( block_sizes, # [num_kv_cache_groups] kernel_block_sizes, # [num_kv_cache_groups] slot_mapping_enabled, # [num_kv_cache_groups] + dcp_sharded, # [num_kv_cache_groups] slot_mappings_ptr, # [num_kv_cache_groups, max_num_tokens] slot_mappings_stride, cp_rank, @@ -312,6 +319,8 @@ def _compute_slot_mappings_kernel( kv_block_size = tl.load(block_sizes + group_id) kernel_block_size = tl.load(kernel_block_sizes + group_id) mapping_enabled = tl.load(slot_mapping_enabled + group_id) + if CP_SIZE != 1: + sharded = tl.load(dcp_sharded + group_id) req_state_idx = tl.load(idx_mapping + batch_idx) # idx_mapping == -1 marks a dummy (or CUDA-graph padding) request that owns @@ -337,6 +346,8 @@ def _compute_slot_mappings_kernel( remainder = virtual_block_offsets % CP_INTERLEAVE local_offsets = rounds * CP_INTERLEAVE + remainder local_positions = virtual_block_indices * kv_block_size + local_offsets + local_positions = tl.where(sharded, local_positions, positions) + is_local = ~sharded | is_local block_indices = tl.where( mapping_enabled, local_positions // kernel_block_size, 0 diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 61ceaeef2e..9f25425ab6 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -21,6 +21,7 @@ import gc import time from contextlib import AbstractContextManager from copy import deepcopy +from dataclasses import replace from typing import Any, NamedTuple import numpy as np @@ -561,7 +562,16 @@ class GPUModelRunner(LoRAModelRunnerMixin): return encoder_runner.get_encoder_timing_stats() def get_kv_cache_spec(self): - return get_kv_cache_spec(self.vllm_config) + kv_cache_spec = get_kv_cache_spec(self.vllm_config) + # A draft running at DCP=1 under a DCP target keeps a replicated cache. + if ( + isinstance(self.speculator, DraftModelSpeculator) + and self.speculator.dcp_size == 1 + and self.dcp_size > 1 + ): + for name in self.speculator.draft_attn_layer_names & kv_cache_spec.keys(): + kv_cache_spec[name] = replace(kv_cache_spec[name], dcp_sharded=False) + return kv_cache_spec def initialize_kv_cache( self, @@ -587,6 +597,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): block_sizes = [] max_num_blocks_per_group = [] slot_mapping_enabled = [] + dcp_sharded = [] for kv_cache_group in kv_cache_config.kv_cache_groups: spec = kv_cache_group.kv_cache_spec block_sizes.append(spec.block_size) @@ -594,6 +605,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): spec.first_spec if isinstance(spec, UniformTypeKVCacheSpecs) else spec ) slot_mapping_enabled.append(layer_spec.uses_slot_mapping) + dcp_sharded.append(spec.dcp_sharded) # Let each cache type account for CP. Attention KV is DCP-sharded, # while Mamba/GDN recurrent state is replicated across DCP ranks. max_num_blocks = spec.max_num_blocks_per_req( @@ -657,6 +669,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): device=self.device, kernel_block_sizes=self.kernel_block_sizes, slot_mapping_enabled=slot_mapping_enabled, + dcp_sharded=dcp_sharded, cp_size=self.dcp_size, cp_rank=self.dcp_rank, cp_interleave=self.cp_interleave, diff --git a/vllm/v1/worker/gpu/pcp_manager.py b/vllm/v1/worker/gpu/pcp_manager.py index aff8f04cf3..de49fcffb9 100644 --- a/vllm/v1/worker/gpu/pcp_manager.py +++ b/vllm/v1/worker/gpu/pcp_manager.py @@ -145,7 +145,10 @@ class PCPManager: if speculative_config is not None: if speculative_config.use_dspark(): dcp_size = parallel_config.decode_context_parallel_size - if dcp_size not in (1, pcp_size): + if ( + dcp_size not in (1, pcp_size) + and speculative_config.draft_model_config.use_mla + ): raise NotImplementedError( "MRV2 PCP DSpark requires DCP=1 or DCP=PCP; got " f"DCP={dcp_size}, PCP={pcp_size}." diff --git a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py index cdcdd5fcba..57110e0b05 100644 --- a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py @@ -37,12 +37,18 @@ class DFlashSpeculator(DraftModelSpeculator): def __init__(self, vllm_config: VllmConfig, device: torch.device): parallel_config = vllm_config.parallel_config - if parallel_config.prefill_context_parallel_size > 1: - vllm_config = copy.copy(vllm_config) - vllm_config.parallel_config = replace( - parallel_config, - prefill_context_parallel_size=1, - ) + speculative_config = vllm_config.speculative_config + assert speculative_config is not None + vllm_config = copy.copy(vllm_config) + vllm_config.parallel_config = replace( + parallel_config, + prefill_context_parallel_size=1, + decode_context_parallel_size=( + parallel_config.decode_context_parallel_size + if speculative_config.draft_model_config.use_mla + else 1 + ), + ) super().__init__(vllm_config, device) self.hidden_states = torch.zeros( @@ -427,7 +433,7 @@ class DFlashSpeculator(DraftModelSpeculator): self.block_tables.input_block_tables[gid], self.block_tables.kernel_block_sizes[gid], self.block_tables.cp_rank, - self.block_tables.cp_size, + self.dcp_size, self.block_tables.cp_interleave, self.parallel_drafting_token_id, self.num_query_per_req, diff --git a/vllm/v1/worker/gpu/spec_decode/speculator.py b/vllm/v1/worker/gpu/spec_decode/speculator.py index d1d6d888aa..6f88febb66 100644 --- a/vllm/v1/worker/gpu/spec_decode/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/speculator.py @@ -138,6 +138,7 @@ class DraftModelSpeculator(BaseSpeculator): # DP configuration self.dp_size = vllm_config.parallel_config.data_parallel_size self.dp_rank = vllm_config.parallel_config.data_parallel_rank + self.dcp_size = vllm_config.parallel_config.decode_context_parallel_size self.eplb_state: EplbState | None = None @@ -332,14 +333,14 @@ class DraftModelSpeculator(BaseSpeculator): out=draft_seq_lens_cpu_upper_bound[:num_reqs], ) draft_seq_lens_cpu_upper_bound[:num_reqs].clamp_(max=self.max_model_len) - if dcp_local_seq_lens is None and self.block_tables.cp_size > 1: + if dcp_local_seq_lens is None and self.dcp_size > 1: # Draft steps advance and rewind their own global sequence lengths, # so the target model's DCP-local lengths may already be stale. dcp_local_seq_lens = prepare_dcp_local_seq_lens( self.input_buffers.dcp_local_seq_lens, self.input_buffers.seq_lens, num_reqs, - self.block_tables.cp_size, + self.dcp_size, self.block_tables.cp_rank, self.block_tables.cp_interleave, ) diff --git a/vllm/v1/worker/gpu_input_batch.py b/vllm/v1/worker/gpu_input_batch.py index 2a1a3534c9..4707e4f2ca 100644 --- a/vllm/v1/worker/gpu_input_batch.py +++ b/vllm/v1/worker/gpu_input_batch.py @@ -817,6 +817,7 @@ class InputBatch: self.allowed_token_ids_mask_cpu_tensor[empty_index] = ( self.allowed_token_ids_mask_cpu_tensor[last_req_index] ) + self.allowed_token_ids_mask_cpu_tensor[last_req_index].fill_(False) bad_words_token_ids = self.bad_words_token_ids.pop(last_req_index, None) if bad_words_token_ids is not None: diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index def92f601d..778d71bdbc 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -801,11 +801,20 @@ class Worker(WorkerBase): # related to kv cache connector (e.g. kv cache sharing layers). ensure_kv_transfer_initialized(self.vllm_config, kv_cache_config) + # If the connector provides a custom memory pool (e.g. Mooncake + # NVLink/BAREX), use it for KV cache allocation; otherwise fall + # back to the standard CuMem pool. + mem_pool_context = ( + get_kv_transfer_group().get_mem_pool_context() + if has_kv_transfer_group() + else None + ) + if mem_pool_context is None: + mem_pool_context = self._maybe_get_memory_pool_context(tag="kv_cache") + self.model_runner.initialize_kv_cache( kv_cache_config, - kv_cache_allocation_context=self._maybe_get_memory_pool_context( - tag="kv_cache" - ), + kv_cache_allocation_context=mem_pool_context, ) # Build KV-zero metadata outside the CuMem pool so the bookkeeping