diff --git a/.buildkite/intel_jobs/lm_eval_intel.yaml b/.buildkite/intel_jobs/lm_eval_intel.yaml
index e1f649ad91..62d99cf769 100644
--- a/.buildkite/intel_jobs/lm_eval_intel.yaml
+++ b/.buildkite/intel_jobs/lm_eval_intel.yaml
@@ -23,8 +23,7 @@ steps:
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
- 'uv pip install "gpt-oss[eval]==0.0.5" &&
- cd tests &&
+ 'cd tests &&
pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-xpu.txt'
- label: LM Eval Large Models(2 GPUs)
key: lm-eval-large-models-2-gpus
diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml
index 1b2b854bc4..530fd0d27c 100644
--- a/.buildkite/test-amd.yaml
+++ b/.buildkite/test-amd.yaml
@@ -1287,7 +1287,6 @@ steps:
- vllm/platforms/rocm.py
- tests/evals/gpt_oss/
commands:
- - uv pip install --system 'gpt-oss[eval]==0.0.5'
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx942.txt
- label: ":amd: (MI300) LM Eval Small Models Distributed"
@@ -3584,7 +3583,6 @@ steps:
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- - uv pip install --system 'gpt-oss[eval]==0.0.5'
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx950.txt
- label: ":amd: (MI355 DPX) LM Eval Watermarking"
diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml
index c67cc08b94..9f99e1c53f 100644
--- a/.buildkite/test_areas/lm_eval.yaml
+++ b/.buildkite/test_areas/lm_eval.yaml
@@ -829,7 +829,6 @@ steps:
- vllm/model_executor/layers/quantization
- tests/evals/gpt_oss/
commands:
- - uv pip install --system 'gpt-oss[eval]==0.0.5'
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-h100.txt
- label: ":nvidia: (B200) GPQA Eval (GPT-OSS)"
@@ -843,7 +842,6 @@ steps:
- vllm/model_executor/layers/quantization
- tests/evals/gpt_oss/
commands:
- - uv pip install --system 'gpt-oss[eval]==0.0.5'
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-b200.txt
mirror:
amd:
@@ -865,7 +863,6 @@ steps:
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- - uv pip install --system 'gpt-oss[eval]==0.0.5'
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx950.txt
- label: ":nvidia: (B200) Qwen3.8-Flash-Next-FP8 Accuracy Eval"
@@ -940,7 +937,6 @@ steps:
- vllm/model_executor/layers/quantization
- tests/evals/gpt_oss/
commands:
- - uv pip install --system 'gpt-oss[eval]==0.0.5'
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-spark.txt
- label: ":nvidia: (H200 MIG 35GB) LM Eval KV-Offload"
diff --git a/.buildkite/test_areas/spec_decode.yaml b/.buildkite/test_areas/spec_decode.yaml
index 8a41a4c993..73d944a7a4 100644
--- a/.buildkite/test_areas/spec_decode.yaml
+++ b/.buildkite/test_areas/spec_decode.yaml
@@ -98,9 +98,11 @@ steps:
commands:
- pytest -v -s v1/e2e/spec_decode/eagle/
-- label: ":nvidia: (H200 MIG 35GB) Spec Decode Speculators + MTP"
- key: spec-decode-speculators-mtp
- timeout_in_minutes: 50
+# MTP correctness is one parametrized test split into per-model folders;
+# each job below runs whole directories.
+- label: ":nvidia: (H200 MIG 35GB) Spec Decode Speculators"
+ key: spec-decode-speculators
+ timeout_in_minutes: 30
device: h200_35gb
source_file_dependencies:
- vllm/v1/spec_decode/
@@ -110,13 +112,103 @@ steps:
- tests/v1/e2e/spec_decode/
commands:
- pytest -v -s v1/e2e/spec_decode/speculators/
- - pytest -v -s v1/e2e/spec_decode/mtp/
mirror:
amd:
- label: ":amd: (MI300) Spec Decode Speculators + MTP"
+ label: ":amd: (MI300) Spec Decode Speculators"
dind: false
device: mi300_1
- timeout_in_minutes: 75
+ timeout_in_minutes: 40
+ depends_on:
+ - image-build-amd
+ source_file_dependencies:
+ - vllm/v1/spec_decode/
+ - vllm/v1/worker/gpu/spec_decode/
+ - vllm/model_executor/model_loader/
+ - vllm/v1/sample/
+ - vllm/model_executor/layers/
+ - vllm/transformers_utils/configs/speculators/
+ - tests/v1/e2e/spec_decode/
+ - vllm/platforms/rocm.py
+
+- label: ":nvidia: (H200 MIG 35GB) Spec Decode MTP Gemma4"
+ key: spec-decode-mtp-gemma4
+ timeout_in_minutes: 30
+ device: h200_35gb
+ source_file_dependencies:
+ - vllm/v1/spec_decode/
+ - vllm/v1/worker/gpu/spec_decode/
+ - vllm/v1/attention/backends/
+ - vllm/transformers_utils/configs/speculators/
+ - tests/v1/e2e/spec_decode/
+ commands:
+ - pytest -v -s v1/e2e/spec_decode/mtp/gemma4/
+ mirror:
+ amd:
+ label: ":amd: (MI300) Spec Decode MTP Gemma4"
+ dind: false
+ device: mi300_1
+ timeout_in_minutes: 40
+ depends_on:
+ - image-build-amd
+ source_file_dependencies:
+ - vllm/v1/spec_decode/
+ - vllm/v1/worker/gpu/spec_decode/
+ - vllm/model_executor/model_loader/
+ - vllm/v1/sample/
+ - vllm/model_executor/layers/
+ - vllm/transformers_utils/configs/speculators/
+ - tests/v1/e2e/spec_decode/
+ - vllm/platforms/rocm.py
+
+- label: ":nvidia: (H200 MIG 35GB) Spec Decode MTP DeepSeek + MiMo"
+ key: spec-decode-mtp-deepseek-mimo
+ timeout_in_minutes: 30
+ device: h200_35gb
+ source_file_dependencies:
+ - vllm/v1/spec_decode/
+ - vllm/v1/worker/gpu/spec_decode/
+ - vllm/v1/attention/backends/
+ - vllm/transformers_utils/configs/speculators/
+ - tests/v1/e2e/spec_decode/
+ commands:
+ - pytest -v -s v1/e2e/spec_decode/mtp/deepseek/
+ - pytest -v -s v1/e2e/spec_decode/mtp/mimo/
+ mirror:
+ amd:
+ label: ":amd: (MI300) Spec Decode MTP DeepSeek + MiMo"
+ dind: false
+ device: mi300_1
+ timeout_in_minutes: 40
+ depends_on:
+ - image-build-amd
+ source_file_dependencies:
+ - vllm/v1/spec_decode/
+ - vllm/v1/worker/gpu/spec_decode/
+ - vllm/model_executor/model_loader/
+ - vllm/v1/sample/
+ - vllm/model_executor/layers/
+ - vllm/transformers_utils/configs/speculators/
+ - tests/v1/e2e/spec_decode/
+ - vllm/platforms/rocm.py
+
+- label: ":nvidia: (H200 MIG 35GB) Spec Decode MTP Qwen3.5"
+ key: spec-decode-mtp-qwen3-5
+ timeout_in_minutes: 30
+ device: h200_35gb
+ source_file_dependencies:
+ - vllm/v1/spec_decode/
+ - vllm/v1/worker/gpu/spec_decode/
+ - vllm/v1/attention/backends/
+ - vllm/transformers_utils/configs/speculators/
+ - tests/v1/e2e/spec_decode/
+ commands:
+ - pytest -v -s v1/e2e/spec_decode/mtp/qwen3_5/
+ mirror:
+ amd:
+ label: ":amd: (MI300) Spec Decode MTP Qwen3.5"
+ dind: false
+ device: mi300_1
+ timeout_in_minutes: 40
depends_on:
- image-build-amd
source_file_dependencies:
diff --git a/cmake/external_projects/flashkda.cmake b/cmake/external_projects/flashkda.cmake
index 5ac4f4b415..fadee58efb 100644
--- a/cmake/external_projects/flashkda.cmake
+++ b/cmake/external_projects/flashkda.cmake
@@ -13,7 +13,7 @@ else()
FetchContent_Declare(
flashkda
GIT_REPOSITORY https://github.com/vllm-project/FlashKDA.git
- GIT_TAG b59532f1f464fbd536272780e30df5bf6a2ccc02
+ GIT_TAG 17a037d98da546deb4591e967cf961a43c034d8b
GIT_PROGRESS TRUE
GIT_SUBMODULES cutlass
)
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 366a9a8752..6609ed7464 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -729,6 +729,8 @@ COPY requirements/lint.txt requirements/lint.txt
COPY requirements/test/cuda.in requirements/test/cuda.in
COPY requirements/test/cuda.txt requirements/test/cuda.txt
COPY requirements/dev.txt requirements/dev.txt
+# uv reads `[tool.uv] exclude-dependencies` from here when recompiling below.
+COPY pyproject.toml pyproject.toml
COPY tools/use_existing_torch.py tools/use_existing_torch.py
COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt
RUN --mount=type=cache,target=/opt/uv/cache \
@@ -1143,6 +1145,8 @@ COPY requirements/lint.txt requirements/lint.txt
COPY requirements/test/cuda.in requirements/test/cuda.in
COPY requirements/test/cuda.txt requirements/test/cuda.txt
COPY requirements/dev.txt requirements/dev.txt
+# uv reads `[tool.uv] exclude-dependencies` from here when recompiling below.
+COPY pyproject.toml pyproject.toml
COPY tools/use_existing_torch.py tools/use_existing_torch.py
COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt
RUN --mount=type=cache,target=/opt/uv/cache \
diff --git a/docker/Dockerfile.cpu b/docker/Dockerfile.cpu
index 36792698a2..43172fbdcf 100644
--- a/docker/Dockerfile.cpu
+++ b/docker/Dockerfile.cpu
@@ -27,7 +27,7 @@
# cache mounts (ccache, cargo registry) aren't exported by `--cache-to
# type=registry`, so they don't survive a build landing on a different/fresh
# builder. sccache's cache lives in S3 instead, so it does.
-FROM ubuntu:25.04 AS base
+FROM ubuntu:22.04 AS base
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
--mount=type=cache,target=/var/lib/apt,sharing=locked \
@@ -52,11 +52,38 @@ RUN if [ "$USE_SCCACHE" = "1" ]; then \
&& rm -rf sccache.tar.gz sccache-v0.8.1-${SCCACHE_ARCH}-unknown-linux-musl; \
fi
+######################### BINUTILS BUILD IMAGE #########################
+# Build a newer GNU assembler for the AMX-FP8 instruction set.
+FROM base AS binutils-build
+
+ARG BINUTILS_VERSION=2.47
+
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+ --mount=type=cache,target=/var/lib/apt,sharing=locked \
+ apt-get update -y \
+ && apt-get install -y --no-install-recommends \
+ bison \
+ build-essential \
+ flex \
+ texinfo \
+ xz-utils \
+ && curl -fsSLO "https://sourceware.org/pub/binutils/releases/binutils-${BINUTILS_VERSION}.tar.xz" \
+ && tar -xf "binutils-${BINUTILS_VERSION}.tar.xz" \
+ && mkdir binutils-build \
+ && cd binutils-build \
+ && "../binutils-${BINUTILS_VERSION}/configure" --prefix=/opt/binutils --disable-nls --disable-werror \
+ && make -j"$(nproc)" \
+ && make install \
+ && cd .. \
+ && rm -rf binutils-build "/binutils-${BINUTILS_VERSION}" "binutils-${BINUTILS_VERSION}.tar.xz"
+
######################### COMMON BASE IMAGE #########################
FROM base AS base-common
WORKDIR /workspace
+COPY --from=binutils-build /opt/binutils /opt/binutils
+
ARG PYTHON_VERSION=3.12
ARG max_jobs=32
@@ -93,7 +120,8 @@ ENV CC=/usr/bin/gcc-15 CXX=/usr/bin/g++-15
ENV CCACHE_DIR=/root/.cache/ccache
ENV CMAKE_CXX_COMPILER_LAUNCHER=ccache
-ENV PATH="/root/.local/bin:$PATH"
+ENV PATH="/opt/binutils/bin:/root/.local/bin:$PATH"
+ENV COMPILER_PATH="/opt/binutils/bin"
ENV VIRTUAL_ENV="/opt/venv"
ENV UV_PYTHON_INSTALL_DIR=/opt/uv/python
RUN uv venv --python ${PYTHON_VERSION} --seed ${VIRTUAL_ENV}
@@ -168,7 +196,7 @@ RUN apt-get update -y \
WORKDIR /workspace
COPY requirements/build/rust.txt requirements/build/rust.txt
-RUN python3 -m pip install --break-system-packages --no-cache-dir -r requirements/build/rust.txt
+RUN python3 -m pip install --no-cache-dir -r requirements/build/rust.txt
ARG USE_SCCACHE
ARG SCCACHE_ENDPOINT
diff --git a/docker/Dockerfile.rock_base b/docker/Dockerfile.rock_base
index 79896933e7..d3ba0bc842 100644
--- a/docker/Dockerfile.rock_base
+++ b/docker/Dockerfile.rock_base
@@ -8,7 +8,7 @@ ARG TRITON_VERSION=3.8.0+gitc0a142ff
ARG TRITON_REPO="https://github.com/ROCm/triton.git"
ARG FA_BRANCH="0e60e394"
ARG FA_REPO="https://github.com/Dao-AILab/flash-attention.git"
-ARG AITER_BRANCH="v0.1.22.post1"
+ARG AITER_BRANCH="v0.1.23"
ARG AITER_REPO="https://github.com/ROCm/aiter.git"
ARG MORI_BRANCH="f95cd5e0ca7b730f92f9ffdfb358484d6965d9f7" # TheRock compatible v1.1.0
ARG MORI_REPO="https://github.com/ROCm/mori.git"
diff --git a/docker/Dockerfile.rocm_base b/docker/Dockerfile.rocm_base
index 750d4ce4d9..1c59968005 100644
--- a/docker/Dockerfile.rocm_base
+++ b/docker/Dockerfile.rocm_base
@@ -9,7 +9,7 @@ ARG PYTORCH_AUDIO_BRANCH="v2.11.0"
ARG PYTORCH_AUDIO_REPO="https://github.com/pytorch/audio.git"
ARG FA_BRANCH="0e60e394"
ARG FA_REPO="https://github.com/Dao-AILab/flash-attention.git"
-ARG AITER_BRANCH="v0.1.22.post1"
+ARG AITER_BRANCH="v0.1.23"
ARG AITER_REPO="https://github.com/ROCm/aiter.git"
ARG MORI_BRANCH="v1.1.0"
ARG MORI_REPO="https://github.com/ROCm/mori.git"
diff --git a/docs/benchmarking/cli.md b/docs/benchmarking/cli.md
index e7ee794da6..d05f0c233e 100644
--- a/docs/benchmarking/cli.md
+++ b/docs/benchmarking/cli.md
@@ -596,6 +596,46 @@ vllm bench serve \
--max-concurrency 512
```
+#### Responses API Benchmark
+
+The `openai-responses` backend benchmarks vLLM's
+[Responses API](../serving/online_serving/openai_compatible_server.md#responses-api)
+directly, instead of routing the same workload through `/v1/chat/completions`.
+
+```bash
+# Server
+vllm serve openai/gpt-oss-20b
+
+# Client
+vllm bench serve \
+ --backend openai-responses \
+ --endpoint /v1/responses \
+ --model openai/gpt-oss-20b \
+ --dataset-name random \
+ --random-input-len 1024 \
+ --random-output-len 1024 \
+ --num-prompts 200
+```
+
+Reasoning deltas (`response.reasoning_text.delta`) count towards TTFT and ITL,
+because the server is already decoding tokens when it emits them. Only output
+text (`response.output_text.delta`) is collected as the generated text, which
+matches how the `openai-chat` backend treats `DeltaMessage.reasoning`. End-to-end
+latency stops at the last token event, so `latency - ttft` equals `sum(itl)` and
+TPOT is comparable with the other endpoints. The input and output token counts
+come from the usage block on `response.completed`, which is still required: a
+stream that ends without a terminal event is reported as a failed request.
+
+The backend measures one streamed text-generation request per prompt. Built-in
+tools, MCP, and multi-turn state via `previous_response_id` are out of scope.
+All sampling parameter flags are supported except `--min-p`, which the
+Responses API does not accept.
+
+!!! warning
+ Do not pass `--extra-body '{"include_reasoning": false}'` when benchmarking
+ a reasoning model. The server still generates reasoning tokens but emits no
+ events for them, so the whole reasoning phase is absorbed into TTFT.
+
#### Running With Sampling Parameters
When using OpenAI-compatible backends such as `vllm`, optional sampling
diff --git a/docs/features/batch_invariance.md b/docs/features/batch_invariance.md
index 7353bd3594..0fad4682fc 100644
--- a/docs/features/batch_invariance.md
+++ b/docs/features/batch_invariance.md
@@ -144,7 +144,7 @@ When batch invariance is enabled, vLLM:
1. Uses deterministic kernel implementations for attention and other operations
2. Ensures consistent numerical behavior across different batch sizes
-3. Disables certain optimizations that may introduce non-determinism (such as custom all-reduce operations in tensor parallel mode)
+3. Disables certain optimizations that may introduce non-determinism (such as custom all-reduce operations in tensor parallel mode, and sequence parallelism / async TP, whose reduce-scatter path is not batch-invariant)
4. On CUDA devices with tuned matmul table entries for the model's bf16 unquantized forward linear layers (Ada, Hopper, Blackwell),
runs without `torch.compile` using breakable CUDA graphs so tile configs follow the runtime batch size; set `VLLM_USE_BREAKABLE_CUDAGRAPH=0` to opt out
(not applied when sequence parallelism / async TP are enabled, since those are `torch.compile` passes).
diff --git a/docs/features/mooncake_connector_usage.md b/docs/features/mooncake_connector_usage.md
index 6cce042fb6..7c5852a68b 100644
--- a/docs/features/mooncake_connector_usage.md
+++ b/docs/features/mooncake_connector_usage.md
@@ -46,6 +46,13 @@ Now you can send requests to the proxy server through port 8000.
- For headless instances, must be the same as the master instance
- Each instance needs a unique port on its host; using the same port number across different hosts is fine
+- `VLLM_MOONCAKE_CONNECTOR_TIMEOUT`: Per-request timeout (in seconds) for MooncakeConnector HTTP calls to the bootstrap server, including prefiller worker registration at startup. (Optional)
+ - Default: 30.0
+ - Global rank 0 hosts the bootstrap server in the same process that mounts the Mooncake transfer engine's host segment, so a large `MOONCAKE_GLOBAL_SEGMENT_SIZE` can delay responses for several seconds during startup
+ - Raise this if bootstrap server timeout warnings appear on hosts with very large host-memory segments
+ - Bootstrap registration uses up to 3 attempts internally, with exponential backoff capped at 10 seconds; connection errors and timeouts are both retried
+ - Once the attempts are exhausted the caller raises instead of blocking, so failures surface rather than hanging
+
- `WITH_NVIDIA_PEERMEM`: Selects how mooncake registers GPU memory for RDMA. Read by mooncake, not vLLM.
- Default: 1, which uses `ibv_reg_mr()` and requires the `nvidia-peermem` kernel module to be loaded
- Set to 0 to use the DMA-BUF path, which does not need that module. Required on hosts where `nvidia-peermem` is not loaded, such as GB200
diff --git a/docs/mkdocs/gen_files/generate_metrics.py b/docs/mkdocs/gen_files/generate_metrics.py
index b952f53977..4d4fa93cc5 100644
--- a/docs/mkdocs/gen_files/generate_metrics.py
+++ b/docs/mkdocs/gen_files/generate_metrics.py
@@ -23,6 +23,10 @@ METRIC_SOURCE_FILES = [
"path": "vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py",
"key": "metrics-nixl",
},
+ {
+ "path": "vllm/v1/simple_kv_offload/metrics.py",
+ "key": "metrics-simple-kv-offload",
+ },
{"path": "vllm/v1/metrics/perf.py", "key": "metrics-mfu"},
]
diff --git a/docs/models/pooling_models/scoring.md b/docs/models/pooling_models/scoring.md
index 58083763cb..1c33c6852d 100644
--- a/docs/models/pooling_models/scoring.md
+++ b/docs/models/pooling_models/scoring.md
@@ -50,8 +50,8 @@ The three supported scoring functions are as illustrated in the figure below.
| `ModernBertForSequenceClassification` | ModernBERT-based | `Alibaba-NLP/gte-reranker-modernbert-base`, etc. | N/A | | |
| `Qwen2ForSequenceClassification`C | Qwen2-based | `mixedbread-ai/mxbai-rerank-base-v2`(see note), etc. | [mxbai_rerank_v2.jinja](../../../examples/pooling/score/template/mxbai_rerank_v2.jinja) | ✅︎ | ✅︎ |
| `Qwen3ForSequenceClassification`C | Qwen3-based | `tomaarsen/Qwen3-Reranker-0.6B-seq-cls`, `Qwen/Qwen3-Reranker-0.6B`(see note), etc. | [qwen3_reranker.jinja](../../../examples/pooling/score/template/qwen3_reranker.jinja) | ✅︎ | ✅︎ |
-| `RobertaForSequenceClassification` | RoBERTa-based | `cross-encoder/quora-roberta-base`, etc. | N/A | | |
-| `XLMRobertaForSequenceClassification` | XLM-RoBERTa-based | `BAAI/bge-reranker-v2-m3`, etc. | N/A | | |
+| `RobertaForSequenceClassification` | RoBERTa-based | `cross-encoder/quora-roberta-base`, etc. | N/A | ✅︎ | |
+| `XLMRobertaForSequenceClassification` | XLM-RoBERTa-based | `BAAI/bge-reranker-v2-m3`, etc. | N/A | ✅︎ | |
| `*Model`C, `*ForCausalLM`C, etc. | Generative models | N/A | N/A | \* | \* |
C Automatically converted into a classification model via `--convert classify`. ([details](./README.md#model-conversion))
diff --git a/docs/usage/metrics.md b/docs/usage/metrics.md
index 305c21478f..8254a7f420 100644
--- a/docs/usage/metrics.md
+++ b/docs/usage/metrics.md
@@ -45,6 +45,30 @@ The following metrics are exposed:
--8<-- "gen:metrics-nixl"
+## Simple CPU Offload Connector Metrics
+
+These metrics are exposed when the `SimpleCPUOffloadConnector` KV connector
+is configured (e.g. `--kv-transfer-config='{"kv_connector":
+"SimpleCPUOffloadConnector", "kv_role": "kv_both", "kv_connector_extra_config":
+{"kv_offload_backend": "disk", "disk_path": "/mnt/nvme/kv"}}'`). They are
+updated once per engine step.
+
+Caveats to keep in mind when interpreting them:
+
+- A "completed" store means the write syscalls returned; it is **not**
+ fsync-durable, and the disk backend's file is process-lifetime scratch,
+ unlinked at startup and shutdown.
+- `save_outcomes_total` classifies eager-mode boundary hand-off stores;
+ lazy-mode stores are not classified.
+- `used_blocks` counts blocks pinned by in-flight transfers or cache hits;
+ warm cached blocks that are evictable are not counted. Use the
+ `capacity_blocks` label of `simple_kv_offload_info` as the denominator.
+- Counters and gauges are quantized to engine steps. They are reported by the
+ scheduler process and reflect engine-wide logical block counts, not values
+ pooled from individual tensor-parallel workers.
+
+--8<-- "gen:metrics-simple-kv-offload"
+
## Model Flops Utilization (MFU) Performance Metrics
These metrics are available via `--enable-mfu-metrics`:
diff --git a/pyproject.toml b/pyproject.toml
index 4a7c018929..2098e097bf 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -258,3 +258,8 @@ CANN = "CANN"
[tool.uv]
no-build-isolation-package = ["torch"]
+# vLLM depends on oss-harmony, a fork that ships the same `openai_harmony`
+# module without downloading tiktoken vocabularies at runtime. Installing
+# openai-harmony alongside it silently overwrites those files, so keep it out
+# of every resolution; gpt-oss pulls it in transitively.
+exclude-dependencies = ["openai-harmony"]
diff --git a/requirements/common.txt b/requirements/common.txt
index b55f680b76..ab0060284c 100644
--- a/requirements/common.txt
+++ b/requirements/common.txt
@@ -49,7 +49,7 @@ pybase64 # fast base64 implementation
cbor2 # Required for cross-language serialization of hashable objects
ijson # Required for mistral streaming tool parser
setproctitle # Used to set process names for better debugging and monitoring
-openai-harmony >= 0.0.3 # Required for gpt-oss
+oss-harmony >= 0.0.11 # Required for gpt-oss; includes embedded vocabularies
anthropic >= 0.71.0
model-hosting-container-standards >= 0.1.14, < 1.0.0
mcp >= 2.0.0, < 3.0.0
diff --git a/requirements/test/cpu.txt b/requirements/test/cpu.txt
index c1d4913953..ab3e42886c 100644
--- a/requirements/test/cpu.txt
+++ b/requirements/test/cpu.txt
@@ -76,7 +76,9 @@ black==24.10.0
blake3==1.0.9
# via -r requirements/test/../common.txt
blobfile==3.0.0
- # via -r requirements/test/cuda.in
+ # via
+ # -r requirements/test/cuda.in
+ # gpt-oss
bm25s==0.2.13
# via mteb
boto3==1.35.57
@@ -111,7 +113,7 @@ chardet==5.2.0
# via mbstrdecoder
charset-normalizer==3.4.0
# via requests
-chz==0.3.0
+chz==0.4.0
# via gpt-oss
click==8.4.2
# via
@@ -190,7 +192,7 @@ distro==1.9.0
# openai
dnspython==2.7.0
# via email-validator
-docker==7.1.0
+docker==7.2.0
# via gpt-oss
docopt==0.6.2
# via num2words
@@ -291,7 +293,7 @@ googleapis-common-protos==1.70.0
# google-api-core
# opentelemetry-exporter-otlp-proto-grpc
# opentelemetry-exporter-otlp-proto-http
-gpt-oss==0.0.8
+gpt-oss==0.0.9
# via -r requirements/test/cuda.in
graphql-core==3.2.6
# via hypothesis-graphql
@@ -402,6 +404,7 @@ jinja2==3.1.6
# datamodel-code-generator
# fastapi
# genai-perf
+ # gpt-oss
# lm-eval
# torch
jiter==0.15.0
@@ -433,6 +436,10 @@ jsonschema-specifications==2024.10.1
# via jsonschema
junit-xml==1.9
# via schemathesis
+jupyter-client==8.10.0
+ # via gpt-oss
+jupyter-core==5.9.1
+ # via jupyter-client
kaldi-native-fbank==1.22.3
# via -r requirements/test/cuda.in
kaleido==0.2.1
@@ -549,6 +556,7 @@ numpy==2.2.6
# evaluate
# fastparquet
# genai-perf
+ # gpt-oss
# imagehash
# imageio
# librosa
@@ -583,8 +591,6 @@ numpy==2.2.6
open-clip-torch==2.32.0
# via -r requirements/test/cuda.in
openai==2.44.0
- # via -r requirements/test/../common.txt
-openai-harmony==0.0.4
# via
# -r requirements/test/../common.txt
# gpt-oss
@@ -641,6 +647,8 @@ optuna==3.6.1
# via genai-perf
orjson==3.11.5
# via genai-perf
+oss-harmony==0.0.11
+ # via -r requirements/test/../common.txt
outlines-core==0.2.14
# via -r requirements/test/../common.txt
packaging==24.2
@@ -672,6 +680,7 @@ pandas==2.2.3
# evaluate
# fastparquet
# genai-perf
+ # gpt-oss
# statsmodels
partial-json-parser==0.2.1.1.post7
# via -r requirements/test/../common.txt
@@ -702,6 +711,7 @@ pillow==10.4.0
platformdirs==4.3.6
# via
# black
+ # jupyter-core
# pooch
# virtualenv
plotly==5.24.1
@@ -794,7 +804,7 @@ pydantic==2.12.0
# model-hosting-container-standards
# mteb
# openai
- # openai-harmony
+ # oss-harmony
# pydantic-extra-types
# pydantic-settings
# ray
@@ -855,6 +865,7 @@ pytest-timeout==2.3.1
python-dateutil==2.9.0.post0
# via
# botocore
+ # jupyter-client
# matplotlib
# pandas
# typepy
@@ -898,7 +909,9 @@ pyyaml==6.0.2
# uvicorn
# vocos
pyzmq==27.1.0
- # via -r requirements/test/../common.txt
+ # via
+ # -r requirements/test/../common.txt
+ # jupyter-client
rapidfuzz==3.12.1
# via jiwer
ray==2.56.1
@@ -1077,7 +1090,7 @@ starlette-testclient==0.4.1
# via schemathesis
statsmodels==0.14.4
# via genai-perf
-structlog==25.4.0
+structlog==26.1.0
# via gpt-oss
supervisor==4.3.0
# via model-hosting-container-standards
@@ -1101,7 +1114,7 @@ tenacity==9.1.2
# schemathesis
tensorizer==2.10.1
# via -r requirements/test/cuda.in
-termcolor==3.1.0
+termcolor==3.3.0
# via gpt-oss
threadpoolctl==3.5.0
# via scikit-learn
@@ -1157,11 +1170,14 @@ torchvision==0.28.0+cpu
# open-clip-torch
# segmentation-models-pytorch
# timm
+tornado==6.5.10
+ # via jupyter-client
tqdm==4.67.3
# via
# -r requirements/test/../common.txt
# datasets
# evaluate
+ # gpt-oss
# huggingface-hub
# lm-eval
# mteb
@@ -1174,6 +1190,10 @@ tqdm==4.67.3
# segmentation-models-pytorch
# sentence-transformers
# transformers
+traitlets==5.16.1
+ # via
+ # jupyter-client
+ # jupyter-core
transformers==5.17.0
# via
# -r requirements/test/../common.txt
@@ -1226,6 +1246,7 @@ typing-extensions==4.15.0
# grpcio
# httpx2
# huggingface-hub
+ # jupyter-client
# librosa
# lm-eval
# mcp
diff --git a/requirements/test/cuda.in b/requirements/test/cuda.in
index 5e3faaebcc..271cdba155 100644
--- a/requirements/test/cuda.in
+++ b/requirements/test/cuda.in
@@ -66,7 +66,7 @@ decord==0.6.0; platform_machine == "x86_64"
imagehash # Required for Prithvi tests
segmentation-models-pytorch > 0.4.0 # Required for Prithvi tests
-gpt-oss >= 0.0.7; python_version > '3.11'
+gpt-oss[eval] >= 0.0.7; python_version > '3.11'
perceptron # required for isaac test
kaldi-native-fbank >= 1.18.7 # required for fireredasr2 test
diff --git a/requirements/test/cuda.txt b/requirements/test/cuda.txt
index 3207d395aa..ffbc639c4c 100644
--- a/requirements/test/cuda.txt
+++ b/requirements/test/cuda.txt
@@ -81,7 +81,9 @@ black==24.10.0
blake3==1.0.9
# via -r requirements/test/../common.txt
blobfile==3.0.0
- # via -r requirements/test/cuda.in
+ # via
+ # -r requirements/test/cuda.in
+ # gpt-oss
bm25s==0.2.13
# via mteb
boto3==1.35.57
@@ -116,7 +118,7 @@ chardet==5.2.0
# via mbstrdecoder
charset-normalizer==3.4.0
# via requests
-chz==0.3.0
+chz==0.4.0
# via gpt-oss
click==8.4.2
# via
@@ -205,7 +207,7 @@ distro==1.9.0
# openai
dnspython==2.7.0
# via email-validator
-docker==7.1.0
+docker==7.2.0
# via gpt-oss
docopt==0.6.2
# via num2words
@@ -310,7 +312,7 @@ googleapis-common-protos==1.70.0
# google-api-core
# opentelemetry-exporter-otlp-proto-grpc
# opentelemetry-exporter-otlp-proto-http
-gpt-oss==0.0.8
+gpt-oss==0.0.9
# via -r requirements/test/cuda.in
graphql-core==3.2.6
# via hypothesis-graphql
@@ -424,6 +426,7 @@ jinja2==3.1.6
# datamodel-code-generator
# fastapi
# genai-perf
+ # gpt-oss
# lm-eval
# torch
jiter==0.15.0
@@ -456,6 +459,10 @@ jsonschema-specifications==2024.10.1
# via jsonschema
junit-xml==1.9
# via schemathesis
+jupyter-client==8.10.0
+ # via gpt-oss
+jupyter-core==5.9.1
+ # via jupyter-client
kaldi-native-fbank==1.22.3
# via -r requirements/test/cuda.in
kaleido==0.2.1
@@ -584,6 +591,7 @@ numpy==2.2.6
# evaluate
# fastparquet
# genai-perf
+ # gpt-oss
# imagehash
# imageio
# librosa
@@ -662,10 +670,6 @@ openai==2.44.0
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
-openai-harmony==0.0.4
- # via
- # -c requirements/common.txt
- # -r requirements/test/../common.txt
# gpt-oss
opencensus==0.11.4
# via ray
@@ -727,6 +731,10 @@ optuna==3.6.1
# via genai-perf
orjson==3.11.5
# via genai-perf
+oss-harmony==0.0.11
+ # via
+ # -c requirements/common.txt
+ # -r requirements/test/../common.txt
outlines-core==0.2.14
# via
# -c requirements/common.txt
@@ -760,6 +768,7 @@ pandas==2.2.3
# evaluate
# fastparquet
# genai-perf
+ # gpt-oss
# statsmodels
partial-json-parser==0.2.1.1.post7
# via -r requirements/test/../common.txt
@@ -790,6 +799,7 @@ pillow==10.4.0
platformdirs==4.3.6
# via
# black
+ # jupyter-core
# pooch
# virtualenv
plotly==5.24.1
@@ -887,7 +897,7 @@ pydantic==2.12.0
# model-hosting-container-standards
# mteb
# openai
- # openai-harmony
+ # oss-harmony
# pydantic-extra-types
# pydantic-settings
# ray
@@ -948,6 +958,7 @@ pytest-timeout==2.3.1
python-dateutil==2.9.0.post0
# via
# botocore
+ # jupyter-client
# matplotlib
# pandas
# typepy
@@ -994,6 +1005,7 @@ pyzmq==27.1.0
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
+ # jupyter-client
rapidfuzz==3.12.1
# via jiwer
ray==2.56.1
@@ -1177,7 +1189,7 @@ starlette-testclient==0.4.1
# via schemathesis
statsmodels==0.14.4
# via genai-perf
-structlog==25.4.0
+structlog==26.1.0
# via gpt-oss
supervisor==4.3.0
# via model-hosting-container-standards
@@ -1201,7 +1213,7 @@ tenacity==9.1.2
# schemathesis
tensorizer==2.10.1
# via -r requirements/test/cuda.in
-termcolor==3.1.0
+termcolor==3.3.0
# via gpt-oss
threadpoolctl==3.5.0
# via scikit-learn
@@ -1264,11 +1276,14 @@ torchvision==0.28.0+cu130
# open-clip-torch
# segmentation-models-pytorch
# timm
+tornado==6.5.10
+ # via jupyter-client
tqdm==4.67.3
# via
# -r requirements/test/../common.txt
# datasets
# evaluate
+ # gpt-oss
# huggingface-hub
# lm-eval
# mteb
@@ -1281,6 +1296,10 @@ tqdm==4.67.3
# segmentation-models-pytorch
# sentence-transformers
# transformers
+traitlets==5.16.1
+ # via
+ # jupyter-client
+ # jupyter-core
transformers==5.17.0
# via
# -c requirements/common.txt
@@ -1337,6 +1356,7 @@ typing-extensions==4.15.0
# grpcio
# httpx2
# huggingface-hub
+ # jupyter-client
# librosa
# lm-eval
# mcp
diff --git a/requirements/test/rocm.in b/requirements/test/rocm.in
index c8c08fabe5..ffe72643aa 100644
--- a/requirements/test/rocm.in
+++ b/requirements/test/rocm.in
@@ -65,7 +65,7 @@ decord==0.6.0
imagehash # Required for Prithvi tests
segmentation-models-pytorch>0.4.0 # Required for Prithvi tests
-gpt-oss>=0.0.7; python_version > '3.11'
+gpt-oss[eval]>=0.0.7; python_version > '3.11'
perceptron # required for isaac test
kaldi-native-fbank>=1.18.7 # required for fireredasr2 test
diff --git a/requirements/test/rocm.txt b/requirements/test/rocm.txt
index c961b6294a..36e1ba182d 100644
--- a/requirements/test/rocm.txt
+++ b/requirements/test/rocm.txt
@@ -80,7 +80,9 @@ black==26.3.1
blake3==1.0.8
# via -r requirements/test/../common.txt
blobfile==3.0.0
- # via -r requirements/test/rocm.in
+ # via
+ # -r requirements/test/rocm.in
+ # gpt-oss
bm25s==0.2.13
# via mteb
boto3==1.42.74
@@ -198,7 +200,7 @@ distro==1.9.0
# openai
dnspython==2.8.0
# via email-validator
-docker==7.1.0
+docker==7.2.0
# via gpt-oss
docopt==0.6.2
# via num2words
@@ -300,7 +302,7 @@ googleapis-common-protos==1.73.0
# google-api-core
# opentelemetry-exporter-otlp-proto-grpc
# opentelemetry-exporter-otlp-proto-http
-gpt-oss==0.0.8
+gpt-oss==0.0.9
# via -r requirements/test/rocm.in
graphql-core==3.2.8
# via hypothesis-graphql
@@ -415,6 +417,7 @@ jinja2==3.1.6
# datamodel-code-generator
# fastapi
# genai-perf
+ # gpt-oss
# lm-eval
# torch
jiter==0.14.0
@@ -447,6 +450,10 @@ jsonschema-specifications==2025.9.1
# via jsonschema
junit-xml==1.9
# via schemathesis
+jupyter-client==8.10.0
+ # via gpt-oss
+jupyter-core==5.9.1
+ # via jupyter-client
kaldi-native-fbank==1.22.3
# via -r requirements/test/rocm.in
kaleido==1.0.0
@@ -587,6 +594,7 @@ numpy==2.2.6
# evaluate
# fastparquet
# genai-perf
+ # gpt-oss
# imagehash
# imageio
# librosa
@@ -629,10 +637,6 @@ openai==2.31.0
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
-openai-harmony==0.0.8
- # via
- # -c requirements/common.txt
- # -r requirements/test/../common.txt
# gpt-oss
opencensus==0.11.4
# via ray
@@ -699,6 +703,10 @@ orjson==3.11.7
# via
# genai-perf
# kaleido
+oss-harmony==0.0.11
+ # via
+ # -c requirements/common.txt
+ # -r requirements/test/../common.txt
outlines-core==0.2.14
# via
# -c requirements/common.txt
@@ -734,6 +742,7 @@ pandas==3.0.1
# evaluate
# fastparquet
# genai-perf
+ # gpt-oss
# statsmodels
partial-json-parser==0.2.1.1.post7
# via -r requirements/test/../common.txt
@@ -764,6 +773,7 @@ pillow==12.1.1
platformdirs==4.3.6
# via
# black
+ # jupyter-core
# pooch
# python-discovery
# virtualenv
@@ -861,7 +871,7 @@ pydantic==2.12.5
# model-hosting-container-standards
# mteb
# openai
- # openai-harmony
+ # oss-harmony
# pydantic-extra-types
# pydantic-settings
# ray
@@ -922,6 +932,7 @@ pytest-timeout==2.3.1
python-dateutil==2.9.0.post0
# via
# botocore
+ # jupyter-client
# matplotlib
# pandas
# typepy
@@ -970,6 +981,7 @@ pyzmq==27.1.0
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
+ # jupyter-client
rapidfuzz==3.12.1
# via
# -r requirements/test/rocm.in
@@ -1159,7 +1171,7 @@ starlette-testclient==0.4.1
# via schemathesis
statsmodels==0.14.6
# via genai-perf
-structlog==25.5.0
+structlog==26.1.0
# via gpt-oss
supervisor==4.3.0
# via model-hosting-container-standards
@@ -1215,11 +1227,14 @@ tokenizers==0.23.1
# transformers
torch-c-dlpack-ext==0.1.5
# via tilelang
+tornado==6.5.10
+ # via jupyter-client
tqdm==4.67.3
# via
# -r requirements/test/../common.txt
# datasets
# evaluate
+ # gpt-oss
# huggingface-hub
# lm-eval
# mteb
@@ -1233,6 +1248,10 @@ tqdm==4.67.3
# sentence-transformers
# tilelang
# transformers
+traitlets==5.16.1
+ # via
+ # jupyter-client
+ # jupyter-core
transformers==5.17.0
# via
# -c requirements/common.txt
@@ -1287,6 +1306,7 @@ typing-extensions==4.15.0
# grpcio
# httpx2
# huggingface-hub
+ # jupyter-client
# librosa
# lm-eval
# mcp
diff --git a/requirements/test/xpu.in b/requirements/test/xpu.in
index d2ad5a0cfd..08a81cd20d 100644
--- a/requirements/test/xpu.in
+++ b/requirements/test/xpu.in
@@ -32,7 +32,7 @@ soundfile
# --- Tool Parsing & Evaluation ---
blobfile
rapidfuzz
-gpt-oss >= 0.0.7; python_version > '3.11'
+gpt-oss[eval] >= 0.0.7; python_version > '3.11'
schemathesis>=4.0.0 # Required for openai schema test.
jiwer
bm25s
diff --git a/requirements/test/xpu.txt b/requirements/test/xpu.txt
index ed8827f5db..e5f898adcc 100644
--- a/requirements/test/xpu.txt
+++ b/requirements/test/xpu.txt
@@ -67,7 +67,9 @@ azure-storage-blob==12.30.1
blake3==1.0.9
# via -r requirements/test/../common.txt
blobfile==3.2.0
- # via -r requirements/test/xpu.in
+ # via
+ # -r requirements/test/xpu.in
+ # gpt-oss
bm25s==0.3.10
# via
# -r requirements/test/xpu.in
@@ -257,7 +259,7 @@ googleapis-common-protos==1.75.1
# google-api-core
# opentelemetry-exporter-otlp-proto-grpc
# opentelemetry-exporter-otlp-proto-http
-gpt-oss==0.0.8
+gpt-oss==0.0.9
# via -r requirements/test/xpu.in
graphql-core==3.2.11
# via hypothesis-graphql
@@ -377,6 +379,7 @@ jinja2==3.1.6
# via
# -c requirements/xpu.txt
# fastapi
+ # gpt-oss
# lm-eval
# torch
jiter==0.16.0
@@ -406,6 +409,10 @@ jsonschema-rs==0.49.9
# via schemathesis
jsonschema-specifications==2025.9.1
# via jsonschema
+jupyter-client==8.10.0
+ # via gpt-oss
+jupyter-core==5.9.1
+ # via jupyter-client
kiwisolver==1.5.1
# via matplotlib
lark==1.2.2
@@ -526,6 +533,7 @@ numpy==2.2.6
# datasets
# decord
# evaluate
+ # gpt-oss
# librosa
# lm-eval
# matplotlib
@@ -575,10 +583,6 @@ openai==3.2.0
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
-openai-harmony==0.0.8
- # via
- # -c requirements/common.txt
- # -r requirements/test/../common.txt
# gpt-oss
opencv-python-headless==5.0.0.93
# via
@@ -632,6 +636,10 @@ opentelemetry-semantic-conventions-ai==0.5.1
# -c requirements/common.txt
# -r requirements/test/../common.txt
# -r requirements/test/xpu.in
+oss-harmony==0.0.11
+ # via
+ # -c requirements/common.txt
+ # -r requirements/test/../common.txt
outlines-core==0.2.14
# via
# -c requirements/common.txt
@@ -656,6 +664,7 @@ pandas==3.0.5
# via
# datasets
# evaluate
+ # gpt-oss
partial-json-parser==0.2.1.1.post7
# via -r requirements/test/../common.txt
pathvalidate==3.3.1
@@ -667,7 +676,9 @@ pillow==12.3.0
# mistral-common
# torchvision
platformdirs==4.11.3
- # via pooch
+ # via
+ # jupyter-core
+ # pooch
pluggy==1.6.0
# via
# pytest
@@ -747,7 +758,7 @@ pydantic==2.13.4
# model-hosting-container-standards
# mteb
# openai
- # openai-harmony
+ # oss-harmony
# pydantic-extra-types
# pydantic-settings
# xgrammar
@@ -806,6 +817,7 @@ pytest-timeout==2.4.0
python-dateutil==2.9.0.post0
# via
# botocore
+ # jupyter-client
# matplotlib
# pandas
# typepy
@@ -841,6 +853,7 @@ pyzmq==27.1.0
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
+ # jupyter-client
rapidfuzz==3.14.5
# via
# -r requirements/test/xpu.in
@@ -1066,11 +1079,14 @@ torchvision==0.29.0+xpu
# via
# open-clip-torch
# timm
+tornado==6.5.10
+ # via jupyter-client
tqdm==4.70.0
# via
# -r requirements/test/../common.txt
# datasets
# evaluate
+ # gpt-oss
# huggingface-hub
# lm-eval
# modelscope
@@ -1081,6 +1097,10 @@ tqdm==4.70.0
# pqdm
# sentence-transformers
# transformers
+traitlets==5.16.1
+ # via
+ # jupyter-client
+ # jupyter-core
transformers==5.17.0
# via
# -c requirements/common.txt
@@ -1132,6 +1152,7 @@ typing-extensions==4.16.0
# grpcio
# httpx2
# huggingface-hub
+ # jupyter-client
# lm-eval
# mcp
# mcp-types
diff --git a/tests/benchmarks/test_responses_request_func.py b/tests/benchmarks/test_responses_request_func.py
new file mode 100644
index 0000000000..38f1bdc507
--- /dev/null
+++ b/tests/benchmarks/test_responses_request_func.py
@@ -0,0 +1,455 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Tests for the ``openai-responses`` backend of ``vllm bench serve``.
+
+The Responses stream differs from Chat Completions in ways the benchmark client
+has to get right: every SSE message is labelled with an ``event:`` line,
+reasoning and output text arrive as separate token-bearing events, and a
+request ends with ``response.completed`` rather than with its last token. These
+tests drive the real request function against a local SSE server so the
+resulting latency and usage numbers are covered end to end.
+"""
+
+import asyncio
+import json
+from collections.abc import Sequence
+
+import aiohttp
+import pytest
+from aiohttp import web
+from aiohttp.test_utils import TestServer
+
+from vllm.benchmarks.lib.endpoint_request_func import (
+ ASYNC_REQUEST_FUNCS,
+ OPENAI_COMPATIBLE_BACKENDS,
+ RequestFuncInput,
+ RequestFuncOutput,
+ async_request_openai_responses,
+)
+
+# These tests never touch torch, so skip the GPU cleanup fixture.
+pytestmark = pytest.mark.skip_global_cleanup
+
+MODEL = "openai/gpt-oss-20b"
+
+# Gaps between scripted events. GAP_TOKEN is much larger than GAP_PREFILL so
+# that a time to first token measured at the wrong event is unambiguous.
+GAP_PREFILL = 0.15
+GAP_TOKEN = 0.5
+GAP_FINALIZE = 0.3
+
+
+def sse(event_type: str, payload: dict) -> bytes:
+ """Frame one event the way ``/v1/responses`` does, with an ``event:`` line.
+
+ ``ensure_ascii=False`` matches the server, whose pydantic
+ ``model_dump_json`` emits non-ASCII characters unescaped.
+ """
+ data = json.dumps({"type": event_type, **payload}, ensure_ascii=False)
+ return f"event: {event_type}\ndata: {data}\n\n".encode()
+
+
+def usage_event(
+ input_tokens: int,
+ output_tokens: int,
+ event_type: str = "response.completed",
+ status: str = "completed",
+):
+ return sse(
+ event_type,
+ {
+ "response": {
+ "status": status,
+ "usage": {
+ "input_tokens": input_tokens,
+ "output_tokens": output_tokens,
+ "total_tokens": input_tokens + output_tokens,
+ },
+ }
+ },
+ )
+
+
+def text_delta(delta: str) -> bytes:
+ return sse("response.output_text.delta", {"delta": delta})
+
+
+def reasoning_delta(delta: str) -> bytes:
+ return sse("response.reasoning_text.delta", {"delta": delta})
+
+
+# The metadata events the server sends before any token. None of these may
+# start the TTFT clock or contribute an ITL sample.
+PREAMBLE = (
+ sse("response.created", {"response": {"status": "in_progress"}})
+ + sse("response.in_progress", {"response": {"status": "in_progress"}})
+ + sse("response.output_item.added", {"output_index": 0})
+ + sse("response.reasoning_part.added", {"output_index": 0})
+ + sse("response.content_part.added", {"output_index": 0})
+)
+
+# The `.done` family the server sends after the last delta. These repeat the
+# accumulated text, so timing them would add bogus ITL samples and counting
+# their payload would duplicate the output.
+DONE_EVENTS = (
+ sse("response.reasoning_text.done", {"text": "The user asks "})
+ + sse("response.reasoning_part.done", {"output_index": 0})
+ + sse("response.output_text.done", {"text": "Rayleigh scattering."})
+ + sse("response.content_part.done", {"output_index": 0})
+ + sse("response.output_item.done", {"output_index": 0})
+)
+
+
+async def _serve(
+ script: Sequence[bytes | float],
+ *,
+ status: int = 200,
+ chunk_size: int | None = None,
+ seen: dict | None = None,
+ **request_kwargs,
+) -> RequestFuncOutput:
+ """Run the request function against a server replaying ``script``.
+
+ Args:
+ script: Byte strings to write, interleaved with floats to sleep for.
+ status: HTTP status to respond with. Non-200 skips the script.
+ chunk_size: If set, split every write into chunks of this many bytes to
+ emulate fragmented transport.
+ seen: If given, populated with the request payload.
+ request_kwargs: Extra fields for the ``RequestFuncInput``.
+
+ """
+
+ async def handler(request: web.Request) -> web.StreamResponse:
+ if seen is not None:
+ seen["payload"] = await request.json()
+ seen["headers"] = dict(request.headers)
+ if status != 200:
+ return web.Response(status=status, reason="Bad Request")
+
+ response = web.StreamResponse(headers={"Content-Type": "text/event-stream"})
+ await response.prepare(request)
+ for item in script:
+ if isinstance(item, float):
+ await asyncio.sleep(item)
+ elif chunk_size is None:
+ await response.write(item)
+ else:
+ for i in range(0, len(item), chunk_size):
+ await response.write(item[i : i + chunk_size])
+ await response.write_eof()
+ return response
+
+ app = web.Application()
+ app.router.add_post("/v1/responses", handler)
+ server = TestServer(app)
+ await server.start_server()
+ try:
+ request_input = RequestFuncInput(
+ prompt="Why is the sky blue?",
+ api_url=str(server.make_url("/v1/responses")),
+ prompt_len=5,
+ output_len=16,
+ model=MODEL,
+ **request_kwargs,
+ )
+ async with aiohttp.ClientSession() as session:
+ return await async_request_openai_responses(request_input, session)
+ finally:
+ await server.close()
+
+
+def run(script: Sequence[bytes | float], **kwargs) -> RequestFuncOutput:
+ return asyncio.run(_serve(script, **kwargs))
+
+
+def test_backend_is_registered_as_openai_compatible():
+ assert ASYNC_REQUEST_FUNCS["openai-responses"] is async_request_openai_responses
+ # Membership enables --ignore-eos on random datasets and the sampling
+ # parameter flags, all of which ResponsesRequest accepts.
+ assert "openai-responses" in OPENAI_COMPATIBLE_BACKENDS
+
+
+def test_request_payload_is_accepted_by_the_server_schema():
+ """Every field sent must be a real ResponsesRequest field.
+
+ This is what makes the backend safe to list in OPENAI_COMPATIBLE_BACKENDS:
+ the flags that membership enables have to reach the model, and the server
+ drops unknown fields without complaining.
+ """
+ from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
+
+ sampling = {
+ "temperature": 0.7,
+ "top_p": 0.9,
+ "top_k": 20,
+ "frequency_penalty": 0.2,
+ "presence_penalty": 0.1,
+ "repetition_penalty": 1.05,
+ }
+ seen: dict = {}
+ output = run(
+ [PREAMBLE, text_delta("hi"), usage_event(5, 1)],
+ seen=seen,
+ extra_body=sampling,
+ ignore_eos=True,
+ model_name="served-alias",
+ request_id="bench-7",
+ extra_headers={"x-team": "perf"},
+ )
+
+ assert output.success
+ # --served-model-name wins over --model, as on the other backends.
+ assert seen["payload"]["model"] == "served-alias"
+ # The route is behind validate_json_request, which 415s a missing type.
+ assert seen["headers"]["Content-Type"] == "application/json"
+ assert seen["headers"]["x-request-id"] == "bench-7"
+ assert seen["headers"]["x-team"] == "perf"
+ assert seen["payload"] == {
+ "model": "served-alias",
+ "input": "Why is the sky blue?",
+ "max_output_tokens": 16,
+ "stream": True,
+ "store": False,
+ "ignore_eos": True,
+ **sampling,
+ }
+ assert set(seen["payload"]) <= set(ResponsesRequest.model_fields)
+ ResponsesRequest.model_validate(seen["payload"])
+
+ # `vllm bench serve` rejects --min-p for this backend for the same reason.
+ assert "min_p" not in ResponsesRequest.model_fields
+
+
+def test_latency_accounting_matches_the_other_endpoints():
+ """`latency - ttft` must equal `sum(itl)`.
+
+ `serve.py` derives TPOT as `(latency - ttft) / (output_len - 1)`, which only
+ means average inter-token latency when that identity holds. #55508 fixed the
+ same accounting on the chat and audio backends; the trailing events after
+ the last token must not extend the request.
+ """
+ output = run(
+ [
+ PREAMBLE,
+ GAP_PREFILL / 2,
+ # A keep-alive comment sent during prefill is not a token.
+ b": keep-alive\n\n",
+ GAP_PREFILL / 2,
+ reasoning_delta("The user asks "),
+ GAP_TOKEN,
+ text_delta("Rayleigh"),
+ text_delta(" scattering."),
+ DONE_EVENTS,
+ GAP_FINALIZE,
+ usage_event(5, 8),
+ ]
+ )
+
+ assert output.success
+ # Reasoning is timed but not returned, matching how `openai-chat` treats
+ # `DeltaMessage.reasoning`.
+ assert output.generated_text == "Rayleigh scattering."
+
+ # Neither the response.created/in_progress/output_item.added preamble nor
+ # the keep-alive comment may start the clock, and the reasoning delta must,
+ # so TTFT lands on the prefill gap rather than near zero or out at the
+ # first output text delta.
+ assert GAP_PREFILL * 0.8 <= output.ttft < GAP_PREFILL + GAP_TOKEN * 0.5
+
+ # One ITL per token event after the first: the gap before "Rayleigh" and
+ # the back-to-back delta that follows it.
+ assert len(output.itl) == 2
+ assert output.itl[0] >= GAP_TOKEN * 0.5
+ assert output.itl[1] < GAP_TOKEN * 0.5
+
+ # The identity is exact, so no wall-clock margin is involved. The .done
+ # family and the GAP_FINALIZE wait before response.completed sit outside
+ # the request, exactly as the trailing usage chunk does on chat.
+ assert output.latency - output.ttft == pytest.approx(sum(output.itl), abs=1e-9)
+
+ assert output.start_time > 0.0
+
+
+def test_usage_is_taken_from_the_terminal_event():
+ output = run([PREAMBLE, text_delta("blue"), usage_event(17, 4)])
+
+ assert output.success
+ assert output.prompt_len == 17
+ assert output.output_tokens == 4
+
+
+@pytest.mark.parametrize("event_type", ["response.completed", "response.incomplete"])
+def test_incomplete_response_is_a_successful_request(event_type: str):
+ """Hitting max_output_tokens is the Responses form of finish_reason=length.
+
+ vLLM reports it as `response.completed` carrying `status: "incomplete"`.
+ The spec's dedicated `response.incomplete` event is accepted as well.
+ """
+ output = run(
+ [
+ PREAMBLE,
+ text_delta("truncated"),
+ usage_event(17, 16, event_type=event_type, status="incomplete"),
+ ]
+ )
+
+ assert output.success
+ assert output.generated_text == "truncated"
+ assert output.output_tokens == 16
+
+
+@pytest.mark.parametrize("chunk_size", [1, 3, 7, 64])
+def test_output_survives_transport_chunk_boundaries(chunk_size: int):
+ """Splits inside the event line, the JSON, or a multibyte character."""
+ text = " leading 中 trailing "
+ output = run(
+ [PREAMBLE, text_delta(text), usage_event(5, 6)],
+ chunk_size=chunk_size,
+ )
+
+ assert output.success
+ assert output.generated_text == text
+ assert output.output_tokens == 6
+
+
+@pytest.mark.parametrize(
+ "char", ["\u2028", "\u2029", "\u0085"], ids=["u2028", "u2029", "u0085"]
+)
+def test_unicode_line_separators_do_not_truncate_the_payload(char: str):
+ """The server emits these unescaped and they are legal inside a JSON string.
+
+ `str.splitlines()` treats them as line breaks, which would cut the `data:`
+ line short and fail the request on valid model output.
+ """
+ text = f"before{char}after"
+ output = run([PREAMBLE, text_delta(text), usage_event(5, 3)])
+
+ assert output.success, output.error
+ assert output.generated_text == text
+
+
+def test_empty_data_line_is_skipped():
+ """A bare `data:` line carries no JSON and must not fail the request."""
+ output = run([PREAMBLE, b"data:\n\n", text_delta("ok"), usage_event(5, 1)])
+
+ assert output.success, output.error
+ assert output.generated_text == "ok"
+
+
+def test_prompt_len_falls_back_when_usage_is_absent():
+ """A terminal event without usage keeps the locally computed prompt length."""
+ output = run(
+ [
+ PREAMBLE,
+ text_delta("hi"),
+ sse("response.completed", {"response": {"status": "completed"}}),
+ ]
+ )
+
+ assert output.success, output.error
+ assert output.prompt_len == 5
+
+
+def test_multi_line_data_payload_is_joined_with_newlines():
+ """The SSE spec allows one payload to span several `data:` lines."""
+ script: list[bytes | float] = [
+ PREAMBLE,
+ b"event: response.output_text.delta\n"
+ b'data: {"type": "response.output_text.delta",\n'
+ b'data: "delta": "ok"}\n\n',
+ usage_event(5, 1),
+ ]
+ output = run(script)
+
+ assert output.success, output.error
+ assert output.generated_text == "ok"
+
+
+def test_done_sentinel_is_skipped():
+ """Other Responses servers send `[DONE]`, but vLLM does not."""
+ output = run([PREAMBLE, text_delta("ok"), usage_event(5, 1), b"data: [DONE]\n\n"])
+
+ assert output.success, output.error
+ assert output.generated_text == "ok"
+
+
+def test_error_before_any_token_reports_the_server_error():
+ """The server's message must win over the generic no-TTFT message."""
+ output = run(
+ [PREAMBLE, b'event: error\ndata: {"type": "error", "message": "boom"}\n\n']
+ )
+
+ assert not output.success
+ assert "boom" in output.error
+
+
+def test_http_error_is_not_reported_as_success():
+ output = run([], status=400)
+
+ assert not output.success
+ assert output.error
+
+
+def test_stream_without_tokens_is_not_reported_as_success():
+ """A stream that finishes without generating anything has no TTFT."""
+ output = run([PREAMBLE, usage_event(5, 0)])
+
+ assert not output.success
+ assert "TTFT" in output.error
+
+
+def test_truncated_stream_is_not_reported_as_success():
+ """A stream cut short after a token has no usage and an understated E2EL."""
+ output = run([PREAMBLE, text_delta("partial")])
+
+ assert not output.success
+ assert "terminal event" in output.error
+
+
+def test_mid_stream_error_is_not_reported_as_success():
+ """An error payload mid-stream must not be counted as a success.
+
+ vLLM cannot emit this today: its streaming error path fails to validate
+ against the event union, so a generation failure truncates the stream
+ instead. Proxies and other Responses servers do send it.
+ """
+ output = run(
+ [
+ PREAMBLE,
+ text_delta("partial"),
+ b'event: error\ndata: {"error": {"message": "boom", "code": 500}}\n\n',
+ ]
+ )
+
+ assert not output.success
+ assert "boom" in output.error
+
+
+def test_data_only_framing_is_still_parsed():
+ """Some proxies drop the ``event:`` line and forward only ``data:``."""
+ script: list[bytes | float] = [
+ b'data: {"type": "response.output_text.delta", "delta": "ok"}\n\n',
+ usage_event(5, 1),
+ ]
+ output = run(script)
+
+ assert output.success
+ assert output.generated_text == "ok"
+
+
+def test_endpoint_path_is_validated():
+ request_input = RequestFuncInput(
+ prompt="hi",
+ api_url="http://localhost:8000/v1/chat/completions",
+ prompt_len=1,
+ output_len=1,
+ model=MODEL,
+ )
+
+ async def call() -> None:
+ async with aiohttp.ClientSession() as session:
+ await async_request_openai_responses(request_input, session)
+
+ with pytest.raises(ValueError, match="responses"):
+ asyncio.run(call())
diff --git a/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py b/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py
index 467e23e7d3..f1de27bf75 100644
--- a/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py
+++ b/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py
@@ -3,10 +3,20 @@
import json
from collections.abc import AsyncGenerator
+from unittest.mock import MagicMock
import httpx
import pytest
+from tests.entrypoints.openai.chat_completion.test_serving_chat import (
+ BASE_MODEL_PATHS,
+ MockHFConfig,
+ MockModelConfig,
+ _build_renderer,
+)
+from tests.entrypoints.openai.chat_completion.test_serving_chat import (
+ MODEL_NAME as MOCK_MODEL_NAME,
+)
from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata
from vllm.entrypoints.openai.chat_completion.batch_serving import (
@@ -15,7 +25,10 @@ from vllm.entrypoints.openai.chat_completion.batch_serving import (
from vllm.entrypoints.openai.chat_completion.protocol import (
BatchChatCompletionRequest,
)
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.outputs import CompletionOutput, RequestOutput
+from vllm.renderers.online_renderer import OnlineRenderer
+from vllm.v1.engine.async_llm import AsyncLLM
# any model with a chat template defined in tokenizer_config should work here
MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
@@ -322,3 +335,53 @@ async def test_batched_echo_prepends_matching_assistant_prefix() -> None:
)
assert response.choices[0].message.content == "PREFIX ASSISTANT ANSWER"
+
+
+@pytest.mark.asyncio
+@pytest.mark.skip_global_cleanup
+async def test_batched_harmony_response_format_uses_structural_tag() -> None:
+ """Harmony must constrain only the final channel of every conversation."""
+ mock_engine = MagicMock(spec=AsyncLLM)
+ mock_engine.model_config = MockModelConfig()
+ mock_engine.errored = False
+ mock_engine.input_processor = MagicMock()
+ mock_engine.model_config.hf_config = MockHFConfig(model_type="gpt_oss")
+ mock_engine.renderer = _build_renderer(mock_engine.model_config)
+ mock_engine.generate.side_effect = lambda *args, **kwargs: _generator(0, "ok")
+
+ models = OpenAIServingModels(mock_engine, BASE_MODEL_PATHS)
+ online_renderer = OnlineRenderer(
+ model_config=mock_engine.model_config,
+ renderer=mock_engine.renderer,
+ request_logger=None,
+ chat_template=None,
+ chat_template_content_format="auto",
+ reasoning_parser="openai_gptoss",
+ )
+ serving = OpenAIServingChatBatch(
+ mock_engine,
+ models,
+ response_role="assistant",
+ online_renderer=online_renderer,
+ chat_template=None,
+ chat_template_content_format="auto",
+ request_logger=None,
+ )
+
+ request = BatchChatCompletionRequest(
+ model=MOCK_MODEL_NAME,
+ messages=[
+ [{"role": "user", "content": "France?"}],
+ [{"role": "user", "content": "Japan?"}],
+ ],
+ response_format={
+ "type": "json_schema",
+ "json_schema": {"name": "city", "schema": {"type": "object"}},
+ },
+ )
+ await serving.create_batch_chat_completion(request)
+
+ calls = mock_engine.generate.call_args_list
+ assert len(calls) == 2
+ for call in calls:
+ assert call.args[1].structured_outputs.structural_tag is not None
diff --git a/tests/entrypoints/openai/parser/test_harmony_render_parity.py b/tests/entrypoints/openai/parser/test_harmony_render_parity.py
index 5cb4461223..966ac5cf52 100644
--- a/tests/entrypoints/openai/parser/test_harmony_render_parity.py
+++ b/tests/entrypoints/openai/parser/test_harmony_render_parity.py
@@ -20,6 +20,7 @@ Each test:
"""
from openai.types.responses import ResponseFunctionToolCall
+from openai_harmony import Message, Role
from tests.entrypoints.openai.utils import verify_harmony_messages
from vllm.entrypoints.openai.parser.harmony_utils import (
@@ -33,6 +34,23 @@ from vllm.entrypoints.openai.responses.harmony import (
response_previous_input_to_harmony,
)
+
+def test_render_uses_legacy_tool_call_header():
+ tool_call = (
+ Message.from_role_and_content(Role.ASSISTANT, "print('hello')")
+ .with_channel("commentary")
+ .with_recipient("python")
+ .with_content_type("code")
+ )
+
+ rendered = get_encoding().decode(render_for_completion([tool_call]))
+
+ assert (
+ "<|start|>assistant to=python<|channel|>commentary code<|message|>" in rendered
+ )
+ assert "commentary to=python <|constrain|>code" not in rendered
+
+
# Use a fixed date so the system message is deterministic across both paths.
_DATE = "2025-01-01"
diff --git a/tests/entrypoints/openai/responses/test_parsable_context_unit.py b/tests/entrypoints/openai/responses/test_parsable_context_unit.py
index 85bf6f2020..0398723184 100644
--- a/tests/entrypoints/openai/responses/test_parsable_context_unit.py
+++ b/tests/entrypoints/openai/responses/test_parsable_context_unit.py
@@ -84,6 +84,18 @@ class _ReasoningOnlyParser(DelegatingParser):
return None
+class _CountingParser(_NoOpParser):
+ """Parser that records the ids it counts; token id 7 is reasoning."""
+
+ def __init__(self, tokenizer, *args, **kwargs):
+ super().__init__(tokenizer)
+ self.counted_ids: list[list[int]] = []
+
+ def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
+ self.counted_ids.append(list(token_ids))
+ return sum(token_id == 7 for token_id in token_ids)
+
+
class _StubToolParser:
"""Minimal tool parser stub that always returns a hardcoded tool call."""
@@ -339,6 +351,25 @@ def test_multi_turn_accumulation():
assert texts == ["First turn", "Second turn"]
+def test_reasoning_tokens_counted_per_round():
+ """Each round is counted on its own ids, not the concatenated rounds."""
+ ctx = _make_context(_CountingParser)
+
+ ctx.append_output(_make_request_output(token_ids=[7, 7, 1]))
+ ctx.append_output(_make_request_output(token_ids=[7, 2]))
+
+ assert ctx.response_parser.counted_ids == [[7, 7, 1], [7, 2]]
+ assert ctx.num_reasoning_tokens == 3
+
+
+def test_reasoning_tokens_zero_without_parser():
+ ctx = _make_context(None)
+
+ ctx.append_output(_make_request_output(token_ids=[7, 7, 1]))
+
+ assert ctx.num_reasoning_tokens == 0
+
+
def test_num_init_messages_offset():
"""Initial messages are preserved and offset works correctly."""
init_messages = [MagicMock(type="message")]
diff --git a/tests/entrypoints/openai/responses/test_serving_responses.py b/tests/entrypoints/openai/responses/test_serving_responses.py
index 429fcf1d0d..89c188ec8a 100644
--- a/tests/entrypoints/openai/responses/test_serving_responses.py
+++ b/tests/entrypoints/openai/responses/test_serving_responses.py
@@ -960,24 +960,19 @@ class TestValidateGeneratorInput:
assert isinstance(result, ErrorResponse)
-@pytest.mark.asyncio
-async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
- """Ensure reasoning_tokens usage is derived from thinking token spans."""
+class _Qwen3FakeTokenizer:
+ def __init__(self):
+ self._vocab = {"": 1, "": 2, "reason": 3, "final": 4}
- class FakeTokenizer:
- def __init__(self):
- self._vocab = {"": 1, "": 2, "reason": 3, "final": 4}
+ def get_vocab(self):
+ return self._vocab
- def get_vocab(self):
- return self._vocab
+ def decode(self, token_ids):
+ id_to_token = {v: k for k, v in self._vocab.items()}
+ return "".join(id_to_token.get(token_id, "x") for token_id in token_ids)
- def decode(self, token_ids):
- id_to_token = {v: k for k, v in self._vocab.items()}
- return "".join(id_to_token.get(token_id, "x") for token_id in token_ids)
-
- # Force non-harmony, SimpleContext path
- monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
+def _make_qwen3_serving() -> tuple[OpenAIServingResponses, _Qwen3FakeTokenizer]:
engine_client = MagicMock()
model_config = MagicMock()
model_config.hf_config.model_type = "test"
@@ -987,21 +982,66 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
engine_client.input_processor = MagicMock()
engine_client.renderer = MagicMock()
- tokenizer = FakeTokenizer()
+ tokenizer = _Qwen3FakeTokenizer()
engine_client.renderer.get_tokenizer.return_value = tokenizer
- models = MagicMock()
-
serving = OpenAIServingResponses(
engine_client=engine_client,
- models=models,
+ models=MagicMock(),
online_renderer=MagicMock(),
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
reasoning_parser="qwen3",
)
+ return serving, tokenizer
+
+def _make_text_request_output(text: str, token_ids: list[int]) -> RequestOutput:
+ return RequestOutput(
+ request_id="req",
+ prompt="hi",
+ prompt_token_ids=[7, 8],
+ prompt_logprobs=None,
+ outputs=[
+ CompletionOutput(
+ index=0,
+ text=text,
+ token_ids=token_ids,
+ cumulative_logprob=0.0,
+ logprobs=None,
+ finish_reason="stop",
+ stop_reason=None,
+ )
+ ],
+ finished=True,
+ num_cached_tokens=0,
+ num_cache_creation_tokens=3,
+ )
+
+
+async def _run_full_generator(serving, request, context, tokenizer):
+ async def dummy_result_generator():
+ yield None
+
+ return await serving.responses_full_generator(
+ request=request,
+ sampling_params=SamplingParams(max_tokens=16),
+ result_generator=dummy_result_generator(),
+ context=context,
+ model_name="test-model",
+ tokenizer=tokenizer,
+ request_metadata=RequestResponseMetadata(request_id="req"),
+ )
+
+
+@pytest.mark.asyncio
+async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
+ """Ensure reasoning_tokens usage is derived from thinking token spans."""
+ # Force non-harmony, SimpleContext path
+ monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
+
+ serving, tokenizer = _make_qwen3_serving()
request = ResponsesRequest(input="hi", tools=[], stream=False)
response_parser = serving._make_response_parser(
request,
@@ -1011,43 +1051,12 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
# Build a SimpleContext with thinking tokens in the output.
context = SimpleContext(response_parser=response_parser)
- token_ids = [1, 10, 2, 20] # 10 20 -> reasoning token count = 1
- completion = CompletionOutput(
- index=0,
- text="reasonfinal",
- token_ids=token_ids,
- cumulative_logprob=0.0,
- logprobs=None,
- finish_reason="stop",
- stop_reason=None,
+ # 10 20 -> reasoning token count = 1
+ context.append_output(
+ _make_text_request_output("reasonfinal", [1, 10, 2, 20])
)
- req_output = RequestOutput(
- request_id="req",
- prompt="hi",
- prompt_token_ids=[7, 8],
- prompt_logprobs=None,
- outputs=[completion],
- finished=True,
- num_cached_tokens=0,
- num_cache_creation_tokens=3,
- )
- context.append_output(req_output)
- async def dummy_result_generator():
- yield None
-
- sampling_params = SamplingParams(max_tokens=16)
- metadata = RequestResponseMetadata(request_id="req")
-
- response = await serving.responses_full_generator(
- request=request,
- sampling_params=sampling_params,
- result_generator=dummy_result_generator(),
- context=context,
- model_name="test-model",
- tokenizer=tokenizer,
- request_metadata=metadata,
- )
+ response = await _run_full_generator(serving, request, context, tokenizer)
assert response.usage.output_tokens_details.reasoning_tokens == 1
assert response.usage.input_tokens_details.cache_write_tokens == 3
diff --git a/tests/evals/gpt_oss/run_gpqa_eval.py b/tests/evals/gpt_oss/run_gpqa_eval.py
new file mode 100644
index 0000000000..b9f94a59a4
--- /dev/null
+++ b/tests/evals/gpt_oss/run_gpqa_eval.py
@@ -0,0 +1,34 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Run the gpt-oss GPQA evaluator with an explicit output-token budget."""
+
+import argparse
+
+from gpt_oss.evals.gpqa_eval import GPQAEval
+from gpt_oss.evals.responses_sampler import ResponsesSampler
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--model", required=True)
+ parser.add_argument("--reasoning-effort", required=True)
+ parser.add_argument("--base-url", required=True)
+ parser.add_argument("--n-threads", required=True, type=int)
+ parser.add_argument("--max-output-tokens", required=True, type=int)
+ args = parser.parse_args()
+
+ sampler = ResponsesSampler(
+ model=args.model,
+ reasoning_model=True,
+ reasoning_effort=args.reasoning_effort,
+ base_url=args.base_url,
+ max_tokens=args.max_output_tokens,
+ )
+ result = GPQAEval(n_repeats=8, n_threads=args.n_threads)(sampler)
+ # gpt-oss returns a NumPy scalar. Convert it so the parent process gets a
+ # stable, dependency-independent representation that its regex can parse.
+ print({"metric": float(result.score)})
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tests/evals/gpt_oss/test_gpqa_correctness.py b/tests/evals/gpt_oss/test_gpqa_correctness.py
index 99d799698a..a57f3a567a 100644
--- a/tests/evals/gpt_oss/test_gpqa_correctness.py
+++ b/tests/evals/gpt_oss/test_gpqa_correctness.py
@@ -21,6 +21,12 @@ from tests.utils import RemoteOpenAIServer
TOL = 0.05 # Absolute tolerance for accuracy comparison
+# gpt-oss >= 0.0.7 explicitly sends its 131072-token default as the output
+# budget, which leaves no room for the prompt in GPT-OSS's 131072-token context.
+# This test originally ran with a 32768-token model context, so retain that
+# proven generation budget while using the current evaluator package.
+MAX_OUTPUT_TOKENS = 32_768
+
# Path to tiktoken encoding files
TIKTOKEN_DATA_DIR = Path(__file__).parent / "data"
@@ -50,10 +56,7 @@ def run_gpqa_eval(model_name: str, base_url: str, reasoning_effort: str) -> floa
# Build the command to run the evaluation
cmd = [
sys.executable,
- "-m",
- "gpt_oss.evals",
- "--eval",
- "gpqa",
+ str(Path(__file__).with_name("run_gpqa_eval.py")),
"--model",
model_name,
"--reasoning-effort",
@@ -62,6 +65,8 @@ def run_gpqa_eval(model_name: str, base_url: str, reasoning_effort: str) -> floa
base_url,
"--n-threads",
"200",
+ "--max-output-tokens",
+ str(MAX_OUTPUT_TOKENS),
]
try:
diff --git a/tests/kernels/core/test_apply_rotary_emb.py b/tests/kernels/core/test_apply_rotary_emb.py
index a576199663..7d63c3499c 100644
--- a/tests/kernels/core/test_apply_rotary_emb.py
+++ b/tests/kernels/core/test_apply_rotary_emb.py
@@ -225,3 +225,46 @@ def test_apply_rotary_emb_xpu_matches_native(
expected = op.forward_native(x, cos, sin)
actual = op.forward_xpu(x, cos, sin)
torch.testing.assert_close(actual, expected, atol=0, rtol=0)
+
+
+@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="CUDA/ROCm only test.")
+@pytest.mark.parametrize("num_tokens", [257, 2304])
+@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
+def test_packed_qk_rope_correctness(
+ num_tokens: int, dtype: torch.dtype, default_vllm_config
+):
+ """packed_qk_rope_ must be bitwise identical to the per-tensor
+ ApplyRotaryEmb path with enable_fp32_compute=True, and must leave the V
+ slice untouched."""
+ from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb
+ from vllm.model_executor.layers.rotary_embedding.packed_qk_rope import (
+ packed_qk_rope_,
+ )
+
+ num_heads, head_dim = 12, 128
+ rng = torch.Generator(device="cuda").manual_seed(0)
+
+ angles = (
+ torch.rand(num_tokens, head_dim // 2, device="cuda", generator=rng) * torch.pi
+ )
+ freqs_cis = torch.polar(torch.ones_like(angles), angles)
+ cos = freqs_cis.real.contiguous()
+ sin = freqs_cis.imag.contiguous()
+
+ xqkv = torch.randn(
+ num_tokens, 3, num_heads, head_dim, dtype=dtype, device="cuda", generator=rng
+ )
+ xqkv_ref = xqkv.clone()
+ xq, xk, xv = torch.unbind(xqkv_ref, dim=-3)
+
+ op = ApplyRotaryEmb(
+ enforce_enable=True, is_neox_style=False, enable_fp32_compute=True
+ )
+ # op() dispatches to forward_cuda / forward_hip per platform.
+ xq_ref = op(xq, cos, sin)
+ xk_ref = op(xk, cos, sin)
+
+ packed_qk_rope_(xqkv, freqs_cis)
+ torch.testing.assert_close(xqkv[:, 0], xq_ref, atol=0, rtol=0)
+ torch.testing.assert_close(xqkv[:, 1], xk_ref, atol=0, rtol=0)
+ torch.testing.assert_close(xqkv[:, 2], xv, atol=0, rtol=0)
diff --git a/tests/kernels/moe/test_flashinfer.py b/tests/kernels/moe/test_flashinfer.py
index 5d55876e81..2a46a0a56a 100644
--- a/tests/kernels/moe/test_flashinfer.py
+++ b/tests/kernels/moe/test_flashinfer.py
@@ -42,6 +42,9 @@ from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
convert_to_fp8_moe_kernel_format,
make_fp8_moe_quant_config,
)
+from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import (
+ fused_topk_bias,
+)
from vllm.model_executor.layers.fused_moe.router.fused_topk_router import fused_topk
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
rotate_weights_for_fi_trtllm_fp8_per_tensor_moe,
@@ -887,3 +890,97 @@ def test_trtllm_fp8_block_moe_deferred_finalize(
),
router_weights=topk_weights,
)
+
+
+@pytest.mark.parametrize("m", [1, 16])
+@pytest.mark.skipif(
+ not current_platform.is_device_capability_family(100),
+ reason="Requires TRTLLM-Gen FP8 MoE (SM100)",
+)
+def test_trtllm_mxfp8_minimax2_routing_applies_routed_scale(m: int, workspace_init):
+ """FlashInfer's fused MiniMax2 routing must apply routed_scaling_factor,
+ matching vLLM's own sigmoid+bias top-k routing (MiniMax-M3 uses 2.0)."""
+ e, topk, n, k, scale = 32, 4, 1024, 1024, 2.0
+ block_shape = [1, 32]
+ set_random_seed(7)
+ with set_current_vllm_config(vllm_config):
+ w1, w2, w1_scale, w2_scale = convert_to_fp8_moe_kernel_format(
+ Fp8MoeBackend.FLASHINFER_TRTLLM,
+ SimpleNamespace(
+ weight_block_size=block_shape,
+ moe_config=SimpleNamespace(
+ is_act_and_mul=True, intermediate_size_per_partition=n
+ ),
+ activation=MoEActivation.SILU,
+ ),
+ *_make_mxfp8_moe_weights(e, n, k),
+ w13_input_scale=None,
+ w2_input_scale=None,
+ )
+ quant_config = make_fp8_moe_quant_config(
+ Fp8MoeBackend.FLASHINFER_TRTLLM,
+ w1_scale,
+ w2_scale,
+ a1_scale=None,
+ a2_scale=None,
+ block_shape=block_shape,
+ )
+ moe_config = FusedMoEConfig(
+ num_experts=e,
+ experts_per_token=topk,
+ hidden_dim=k,
+ intermediate_size=n,
+ num_local_experts=e,
+ num_logical_experts=e,
+ activation=MoEActivation.SILU,
+ device="cuda",
+ moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
+ in_dtype=torch.bfloat16,
+ routing_method=RoutingMethodType.MiniMax2,
+ max_num_tokens=next_power_of_2(m),
+ )
+
+ def make_kernel(experts_cls) -> mk.FusedMoEKernel:
+ return mk.FusedMoEKernel(
+ maybe_make_prepare_finalize(
+ moe=moe_config,
+ quant_config=quant_config,
+ allow_new_interface=True,
+ use_monolithic=experts_cls is TrtLlmFp8ExpertsMonolithic,
+ ),
+ experts_cls(moe_config=moe_config, quant_config=quant_config),
+ )
+
+ a = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) / 10
+ logits = torch.randn((m, e), device="cuda", dtype=torch.float32)
+ bias = torch.randn(e, device="cuda", dtype=torch.float32) / 10
+ common = dict(
+ w1=w1,
+ w2=w2,
+ activation=MoEActivation.SILU,
+ global_num_experts=e,
+ expert_map=None,
+ apply_router_weight_on_input=False,
+ )
+
+ fused = make_kernel(TrtLlmFp8ExpertsMonolithic).apply_monolithic(
+ hidden_states=a,
+ router_logits=logits,
+ e_score_correction_bias=bias,
+ routed_scaling_factor=scale,
+ **common,
+ )
+ topk_weights, topk_ids = fused_topk_bias(
+ a,
+ logits,
+ scoring_func="sigmoid",
+ e_score_correction_bias=bias,
+ topk=topk,
+ renormalize=True,
+ routed_scaling_factor=scale,
+ )
+ reference = make_kernel(TrtLlmFp8ExpertsModular).apply(
+ hidden_states=a, topk_weights=topk_weights, topk_ids=topk_ids, **common
+ )
+
+ torch.testing.assert_close(fused, reference, atol=2e-2, rtol=2e-2)
diff --git a/tests/kernels/moe/test_ocp_mx_moe.py b/tests/kernels/moe/test_ocp_mx_moe.py
index e4b11c9548..295a92482f 100644
--- a/tests/kernels/moe/test_ocp_mx_moe.py
+++ b/tests/kernels/moe/test_ocp_mx_moe.py
@@ -47,7 +47,6 @@ if ROCM_AVAILABLE:
ROCM_GFX950 = on_gfx950()
if ROCM_AITER_AVAILABLE:
- from aiter.ops.triton.moe.quant_moe import upcast_from_mxfp
from aiter.ops.triton.quant import dynamic_mxfp4_quant
if TRTLLM_GEN_MXFP4_AVAILABLE:
@@ -191,11 +190,7 @@ def mxfp8_dequantize(x, scale):
def mxfp4_quant_dequant(x: torch.Tensor) -> torch.Tensor:
shape = x.shape
quantized, scale = dynamic_mxfp4_quant(x.to(torch.bfloat16).flatten(0, -2))
- return (
- upcast_from_mxfp(quantized.view(torch.uint8), scale, torch.bfloat16, axis=-1)
- .reshape(shape)
- .float()
- )
+ return mxfp4_dequantize(quantized.view(torch.uint8), scale).reshape(shape)
def reference_moe(
@@ -1809,12 +1804,8 @@ def test_rocm_mxfp4_moe_oracle(
assert out.abs().max() > 0.01, "Output is effectively zero"
# Dequantize weights for reference computation
- w13_dq = upcast_from_mxfp(
- w13_quant_ref.view(torch.uint8), w13_scale_ref, torch.bfloat16, axis=-1
- )
- w2_dq = upcast_from_mxfp(
- w2_quant_ref.view(torch.uint8), w2_scale_ref, torch.bfloat16, axis=-1
- )
+ w13_dq = mxfp4_dequantize(w13_quant_ref.view(torch.uint8), w13_scale_ref)
+ w2_dq = mxfp4_dequantize(w2_quant_ref.view(torch.uint8), w2_scale_ref)
# Determine activation type and layout
# SWIGLUOAI uses interleaved layout (gate/up alternating)
diff --git a/tests/kernels/moe/test_rocm_aiter_moe.py b/tests/kernels/moe/test_rocm_aiter_moe.py
index 8fa1706f18..fbc8586fd3 100644
--- a/tests/kernels/moe/test_rocm_aiter_moe.py
+++ b/tests/kernels/moe/test_rocm_aiter_moe.py
@@ -566,72 +566,64 @@ def test_aiter_moe_shared_experts_enablement_follows_env(
assert rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() is expected
-def test_aiter_moe_situv2_syncs_aiter_a4w4_env(
- monkeypatch: pytest.MonkeyPatch,
-):
- """SiTUv2 MoE should route to AITER a4w4 and clear legacy a8w4 overrides."""
- from vllm._aiter_ops import rocm_aiter_ops
-
- _assert_aiter_supported()
-
- with monkeypatch.context() as mp:
- mp.delenv("AITER_SITUV2_A8W4", raising=False)
- mp.delenv("AITER_SITUV2_A4W4", raising=False)
- mp.setenv("VLLM_ROCM_USE_AITER", "1")
- mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
- mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "1")
- _reload_envs()
- rocm_aiter_ops.refresh_env_variables()
-
- import os
-
- assert os.environ.get("AITER_SITUV2_A4W4") == "1"
- assert "AITER_SITUV2_A8W4" not in os.environ
-
-
-def test_aiter_moe_situv2_legacy_a8w4_alias_enables_a4w4(
- monkeypatch: pytest.MonkeyPatch,
-):
- """Deprecated VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4 still enables SiTUv2 a4w4."""
- from vllm._aiter_ops import rocm_aiter_ops
-
- _assert_aiter_supported()
-
- with monkeypatch.context() as mp:
- mp.delenv("AITER_SITUV2_A8W4", raising=False)
- mp.delenv("AITER_SITUV2_A4W4", raising=False)
- mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", raising=False)
- mp.setenv("VLLM_ROCM_USE_AITER", "1")
- mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
- mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", "1")
- _reload_envs()
- rocm_aiter_ops.refresh_env_variables()
-
- import os
-
- assert os.environ.get("AITER_SITUV2_A4W4") == "1"
- assert "AITER_SITUV2_A8W4" not in os.environ
-
-
-def test_aiter_moe_situv2_clears_aiter_env_when_disabled(
+@pytest.mark.parametrize(
+ "value,expected_act,expected_env",
+ [
+ # Unset / auto / legacy "1" all mean the a4w4 default.
+ (None, "a4w4", {"AITER_SITUV2_A4W4": "1"}),
+ ("auto", "a4w4", {"AITER_SITUV2_A4W4": "1"}),
+ ("1", "a4w4", {"AITER_SITUV2_A4W4": "1"}),
+ ("a4w4", "a4w4", {"AITER_SITUV2_A4W4": "1"}),
+ ("0", "a16w4", {}),
+ ("a16w4", "a16w4", {}),
+ ("A8W4", "a8w4", {"AITER_SITUV2_A8W4": "1"}),
+ ],
+)
+def test_aiter_moe_situv2_activation_syncs_aiter_env(
+ value: str | None,
+ expected_act: str,
+ expected_env: dict[str, str],
monkeypatch: pytest.MonkeyPatch,
):
+ """Each SiTUv2 activation choice sets exactly its AITER dispatch env."""
+ import os
+
from vllm._aiter_ops import rocm_aiter_ops
_assert_aiter_supported()
with monkeypatch.context() as mp:
+ # Stale AITER overrides must be replaced by the vLLM selection.
+ mp.setenv("AITER_SITUV2_A8W4", "1")
mp.setenv("AITER_SITUV2_A4W4", "1")
mp.setenv("VLLM_ROCM_USE_AITER", "1")
mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
- mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", raising=False)
- mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "0")
+ if value is None:
+ mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", raising=False)
+ else:
+ mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", value)
_reload_envs()
rocm_aiter_ops.refresh_env_variables()
- import os
+ assert rocm_aiter_ops.get_fused_moe_situv2_activation() == expected_act
+ assert rocm_aiter_ops.is_fused_moe_situv2_gate_up_interleaved() is (
+ expected_act == "a8w4"
+ )
+ assert rocm_aiter_ops.is_fused_moe_situv2_enabled() is (expected_act != "a16w4")
+ for name in ("AITER_SITUV2_A8W4", "AITER_SITUV2_A4W4"):
+ assert os.environ.get(name) == expected_env.get(name)
- assert "AITER_SITUV2_A4W4" not in os.environ
+
+def test_aiter_moe_situv2_rejects_unknown_activation(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ with monkeypatch.context() as mp:
+ mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "a8w8")
+ _reload_envs()
+ import vllm.envs as envs
+
+ with pytest.raises(ValueError, match="VLLM_ROCM_USE_AITER_MOE_SITUV2"):
+ _ = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2
@pytest.mark.parametrize("moe_padding", [True, False])
diff --git a/tests/kernels/moe/test_routing.py b/tests/kernels/moe/test_routing.py
index 4baad24cc3..b0c3e26db9 100644
--- a/tests/kernels/moe/test_routing.py
+++ b/tests/kernels/moe/test_routing.py
@@ -167,6 +167,38 @@ def test_single_expert_group_with_non_unit_scale_uses_grouped_topk() -> None:
assert isinstance(router, GroupedTopKRouter)
+def test_sigmoid_bias_routing_with_routed_scale_is_minimax2() -> None:
+ """FlashInfer's MiniMax2 routing applies routed_scaling_factor, so a
+ non-unit scale (MiniMax-M3 uses 2.0) must not block fused routing."""
+ router = create_fused_moe_router(
+ top_k=4,
+ global_num_experts=128,
+ scoring_func="sigmoid",
+ renormalize=True,
+ routed_scaling_factor=2.0,
+ e_score_correction_bias=torch.empty(128),
+ )
+
+ assert isinstance(router, FusedTopKBiasRouter)
+ assert router.routing_method_type == RoutingMethodType.MiniMax2
+
+
+def test_zero_expert_routing_is_unspecified() -> None:
+ """Zero experts are resolved in the router, so kernels with built-in
+ routing must never be selected for them."""
+ router = create_fused_moe_router(
+ top_k=4,
+ global_num_experts=128,
+ scoring_func="sigmoid",
+ renormalize=True,
+ e_score_correction_bias=torch.empty(160),
+ zero_expert_type="identity",
+ num_logical_experts=160,
+ )
+
+ assert router.routing_method_type == RoutingMethodType.Unspecified
+
+
def setup_eplb_state(
enable_eplb: bool, global_num_experts: int
) -> EplbLayerState | None:
diff --git a/tests/kernels/test_compressor_kv_cache.py b/tests/kernels/test_compressor_kv_cache.py
index 06897f04c0..278c0ef5d0 100644
--- a/tests/kernels/test_compressor_kv_cache.py
+++ b/tests/kernels/test_compressor_kv_cache.py
@@ -2075,3 +2075,269 @@ def test_fused_kv_insert_split(num_tokens: int, kv_block_size: int):
# RoPE (last 64): stored as bf16. The kernel recomputes the rotation, so it
# is bf16-close to the reference rather than bit-exact (cf. test_cutedsl).
torch.testing.assert_close(recovered[:, NOPE_DIM:], ref[:, NOPE_DIM:])
+
+
+@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm stream coverage")
+def test_v41_rocm_csa2_full_pipeline():
+ """_forward_csa2_full: one fork, then K write and q-side after the join.
+
+ The default chain must run first on the current stream, the compressor
+ chain on aux 0 and the indexer weights projection on aux 1; the join must
+ publish both cache writes before _produce_k / forward_q run.
+ """
+ from vllm.forward_context import ForwardContext, override_forward_context
+ from vllm.models.deepseek_v41.amd.rocm import DeepseekV41ROCMAiterMLAAttention
+ from vllm.models.deepseek_v41.compressor import DeepseekCompressor
+
+ torch.manual_seed(43)
+ num_tokens = 19
+ hidden_states = torch.randn(num_tokens, 1024, device="cuda")
+ positions = torch.arange(7, 26, device="cuda")
+ qr_kv = torch.zeros(num_tokens, 1536, device="cuda")
+ qr = torch.zeros(num_tokens, 4, device="cuda")
+ kv = torch.zeros(num_tokens, 512, device="cuda")
+ q = torch.zeros(num_tokens, 512, device="cuda")
+ latent = torch.ones(num_tokens, 512, dtype=torch.bfloat16, device="cuda")
+ weights = torch.randn(num_tokens, 32, device="cuda")
+ index_q = torch.full((num_tokens, 32, 128), 2.0, device="cuda")
+ index_q_scale = torch.full((num_tokens, 32), 3.0, device="cuda")
+ index_weights_out = torch.full((num_tokens, 32), 4.0, device="cuda")
+ main_cache = torch.full((1, 128, 584), 165, dtype=torch.uint8, device="cuda")
+ index_cache = torch.full((1, 128, 132), 165, dtype=torch.uint8, device="cuda")
+
+ order: list[str] = []
+ streams: dict[str, torch.cuda.Stream] = {}
+
+ compressor = DeepseekCompressor.__new__(DeepseekCompressor)
+ torch.nn.Module.__init__(compressor)
+ compressor.fused_wkv_wgate = SimpleNamespace(
+ weight=torch.randn(1024, 1024, device="cuda")
+ )
+
+ def compressor_forward(kv_score, positions_):
+ order.append("compressor")
+ streams["compressor"] = torch.cuda.current_stream()
+ return latent
+
+ def compressor_insert(latent_, positions_, rotary_emb):
+ order.append("insert_cache")
+ streams["insert_cache"] = torch.cuda.current_stream()
+ main_cache.fill_(5)
+
+ compressor.forward = compressor_forward
+ compressor.insert_cache = compressor_insert
+
+ def weights_proj(hidden_states_):
+ order.append("weights_proj")
+ streams["weights_proj"] = torch.cuda.current_stream()
+ return weights, None
+
+ def produce_k(latent_, positions_, rotary_emb):
+ order.append("produce_k")
+ streams["produce_k"] = torch.cuda.current_stream()
+ index_cache.fill_(7)
+
+ def forward_q(qr_, qr_scale, indexer_weights, positions_, rotary_emb):
+ order.append("forward_q")
+ streams["forward_q"] = torch.cuda.current_stream()
+ return index_q, index_q_scale, index_weights_out
+
+ indexer = SimpleNamespace(
+ weights_proj=weights_proj,
+ _produce_k=produce_k,
+ forward_q=forward_q,
+ )
+ rotary = SimpleNamespace(cos_sin_cache=torch.randn(32, 64, device="cuda"))
+ aux_streams = [torch.cuda.Stream(), torch.cuda.Stream()]
+
+ def wq_b(qr_, qr_scale):
+ order.append("wq_b")
+ streams["wq_b"] = torch.cuda.current_stream()
+ return q
+
+ def qnorm_insert(q_, kv_, positions_, attn_metadata):
+ order.append("qnorm_insert")
+ streams["qnorm_insert"] = torch.cuda.current_stream()
+ return q_
+
+ attention = SimpleNamespace(
+ compressor=compressor,
+ indexer=indexer,
+ aux_stream_list=aux_streams,
+ ln_events=[torch.cuda.Event() for _ in range(4)],
+ rotary_emb=rotary,
+ indexer_rotary_emb=rotary,
+ n_local_heads=1,
+ head_dim=512,
+ _fused_wqa_wkv_gemm=lambda hidden_states_: qr_kv,
+ _split_qkv_and_norm=lambda qr_kv_: (qr, None, kv),
+ _wq_b_proj=wq_b,
+ _fused_qnorm_rope_kv_insert=qnorm_insert,
+ )
+
+ context = ForwardContext({}, {}, {})
+ with override_forward_context(context):
+ result = DeepseekV41ROCMAiterMLAAttention._forward_csa2_full(
+ attention, hidden_states, positions
+ )
+
+ q_out, kv_out, index_q_out, index_q_scale_out, index_weights_out_out = result
+ assert torch.equal(q_out, q)
+ assert torch.equal(kv_out, kv)
+ assert torch.equal(index_q_out, index_q)
+ assert torch.equal(index_q_scale_out, index_q_scale)
+ assert torch.equal(index_weights_out_out, index_weights_out)
+ # Join publishes the aux-0 cache insert and orders the post-join K write.
+ assert torch.equal(main_cache, torch.full_like(main_cache, 5))
+ assert torch.equal(index_cache, torch.full_like(index_cache, 7))
+
+ assert order == [
+ "wq_b",
+ "qnorm_insert",
+ "compressor",
+ "insert_cache",
+ "weights_proj",
+ "produce_k",
+ "forward_q",
+ ]
+ current = torch.cuda.current_stream()
+ for name in ("wq_b", "qnorm_insert", "produce_k", "forward_q"):
+ assert streams[name] == current, name
+ for name in ("compressor", "insert_cache"):
+ assert streams[name] == aux_streams[0], name
+ assert streams["weights_proj"] == aux_streams[1]
+
+
+@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm stream coverage")
+def test_v41_rocm_csa2_reindex_pipeline():
+ """_forward_csa2_reindex: serial projections, then SWA q path vs q-side."""
+ from vllm.forward_context import ForwardContext, override_forward_context
+ from vllm.models.deepseek_v41.amd.rocm import DeepseekV41ROCMAiterMLAAttention
+
+ torch.manual_seed(45)
+ num_tokens = 19
+ hidden_states = torch.randn(num_tokens, 1024, device="cuda")
+ positions = torch.arange(7, 26, device="cuda")
+ qr_kv = torch.zeros(num_tokens, 1536, device="cuda")
+ qr = torch.zeros(num_tokens, 4, device="cuda")
+ kv = torch.zeros(num_tokens, 512, device="cuda")
+ q = torch.zeros(num_tokens, 512, device="cuda")
+ indexer_weights = torch.randn(num_tokens, 32, device="cuda")
+ index_q = torch.full((num_tokens, 32, 128), 2.0, device="cuda")
+ index_q_scale = torch.full((num_tokens, 32), 3.0, device="cuda")
+ index_weights_out = torch.full((num_tokens, 32), 4.0, device="cuda")
+
+ order: list[str] = []
+ streams: dict[str, torch.cuda.Stream] = {}
+
+ def forward_q(qr_, qr_scale, indexer_weights_, positions_, rotary_emb):
+ order.append("forward_q")
+ streams["forward_q"] = torch.cuda.current_stream()
+ return index_q, index_q_scale, index_weights_out
+
+ indexer = SimpleNamespace(forward_q=forward_q)
+ rotary = SimpleNamespace(cos_sin_cache=torch.randn(32, 64, device="cuda"))
+ aux0 = torch.cuda.Stream()
+
+ def wq_b(qr_, qr_scale):
+ order.append("wq_b")
+ streams["wq_b"] = torch.cuda.current_stream()
+ return q
+
+ def qnorm_insert(q_, kv_, positions_, attn_metadata):
+ order.append("qnorm_insert")
+ streams["qnorm_insert"] = torch.cuda.current_stream()
+ return q_
+
+ attention = SimpleNamespace(
+ indexer=indexer,
+ aux_stream_list=[aux0],
+ ln_events=[torch.cuda.Event() for _ in range(2)],
+ indexer_rotary_emb=rotary,
+ n_local_heads=1,
+ head_dim=512,
+ _run_parallel_input_projections=lambda hidden_states_: (
+ qr_kv,
+ None,
+ indexer_weights,
+ ),
+ _split_qkv_and_norm=lambda qr_kv_: (qr, None, kv),
+ _wq_b_proj=wq_b,
+ _fused_qnorm_rope_kv_insert=qnorm_insert,
+ )
+
+ context = ForwardContext({}, {}, {})
+ with override_forward_context(context):
+ result = DeepseekV41ROCMAiterMLAAttention._forward_csa2_reindex(
+ attention, hidden_states, positions
+ )
+
+ q_out, kv_out, index_q_out, index_q_scale_out, index_weights_out_out = result
+ assert torch.equal(q_out, q)
+ assert torch.equal(kv_out, kv)
+ assert torch.equal(index_q_out, index_q)
+ assert torch.equal(index_q_scale_out, index_q_scale)
+ assert torch.equal(index_weights_out_out, index_weights_out)
+
+ assert order == ["wq_b", "qnorm_insert", "forward_q"]
+ current = torch.cuda.current_stream()
+ for name in ("wq_b", "qnorm_insert"):
+ assert streams[name] == current, name
+ assert streams["forward_q"] == aux0
+
+
+@pytest.mark.parametrize("quant_tuple", [False, True])
+def test_v41_indexer_forward_q(quant_tuple):
+ """forward_q runs wq_b then the fused RoPE/quant and unpacks both shapes."""
+ from unittest.mock import patch
+
+ from vllm.models.deepseek_v41.attention import DeepseekV4Indexer
+
+ num_tokens, n_head, head_dim = 5, 32, 128
+ qr = torch.randn(num_tokens, 1280)
+ q = torch.randn(num_tokens, n_head * head_dim)
+ positions = torch.arange(num_tokens)
+ indexer_weights = torch.randn(num_tokens, n_head)
+ rotary = SimpleNamespace(cos_sin_cache=torch.randn(32, 64))
+ fake = SimpleNamespace(
+ n_head=n_head,
+ head_dim=head_dim,
+ softmax_scale=head_dim**-0.5,
+ use_fp4_kv=False,
+ indexer_weights_dtype=torch.float32,
+ _wq_b_proj=lambda qr_, qr_scale: q,
+ )
+ q_quant = torch.randn(num_tokens, n_head, head_dim)
+ q_scale = torch.randn(num_tokens, n_head)
+ weights_out = torch.randn(num_tokens, n_head)
+ if quant_tuple:
+ fused_return = ((q_quant, q_scale), weights_out)
+ expected = (q_quant, q_scale, weights_out)
+ else:
+ fused_return = (q_quant, weights_out)
+ expected = (q_quant, None, weights_out)
+
+ with patch(
+ "vllm.models.deepseek_v41.attention.fused_indexer_q_rope_quant",
+ return_value=fused_return,
+ ) as mock_fused:
+ result = DeepseekV4Indexer.forward_q(
+ fake, qr, None, indexer_weights, positions, rotary
+ )
+
+ assert torch.equal(result[0], expected[0])
+ assert (result[1] is None) == (expected[1] is None)
+ if expected[1] is not None:
+ assert torch.equal(result[1], expected[1])
+ assert torch.equal(result[2], expected[2])
+ args = mock_fused.call_args.args
+ assert args[0] is positions
+ torch.testing.assert_close(args[1], q.view(-1, n_head, head_dim))
+ assert args[2] is rotary.cos_sin_cache
+ assert args[3] is indexer_weights
+ assert args[4] == head_dim**-0.5
+ assert args[5] == n_head**-0.5
+ assert mock_fused.call_args.kwargs == {
+ "use_fp4": False,
+ "weights_out_dtype": torch.float32,
+ }
diff --git a/tests/kernels/test_fused_inv_rope_fp8_quant.py b/tests/kernels/test_fused_inv_rope_fp8_quant.py
index 03d5b7921f..3b5b24fe49 100644
--- a/tests/kernels/test_fused_inv_rope_fp8_quant.py
+++ b/tests/kernels/test_fused_inv_rope_fp8_quant.py
@@ -21,7 +21,12 @@ Usage:
import pytest
import torch
+from vllm.model_executor.layers.quantization.utils.fp8_utils import (
+ deepgemm_post_process_fp8_weight_block,
+)
from vllm.models.deepseek_v4.common.ops import fused_inv_rope_fp8_quant
+from vllm.platforms import current_platform
+from vllm.utils.deep_gemm import fp8_einsum
# -- Default dimensions matching DeepSeek V3/V4 --------------------------
HEAD_DIM = 512
@@ -33,6 +38,49 @@ FP8_DTYPE = torch.float8_e4m3fn
EPS = 1e-10
+@pytest.mark.skipif(
+ not current_platform.is_device_capability_family(100), reason="Requires SM100/SM103"
+)
+@pytest.mark.parametrize("groups", [2, 4])
+@pytest.mark.parametrize("tokens", [1, 6, 18, 32])
+def test_dsv41_fused_wo_a_matches_deepgemm(groups, tokens):
+ """Fused WO-A matches inverse RoPE + fp8_einsum + MXFP8 quantize."""
+ from flashinfer import mxfp8_quantize
+
+ from vllm.models.deepseek_v41.nvidia.ops.fused_wo_a import _FUSED_WO_A_KERNEL
+
+ torch.manual_seed(0)
+ n = groups * 1024
+ x = torch.randn(tokens, 64, 512, device="cuda", dtype=torch.bfloat16)[:, : n // 128]
+ positions = torch.randint(0, 128, (tokens,), device="cuda")
+ rope = make_cos_sin_cache(128, device="cuda")
+ wq = torch.randn(n, 4096, device="cuda").to(FP8_DTYPE)
+ scale = torch.exp2(torch.randint(-8, 0, (n, 128), device="cuda").float())
+ wq, ws = deepgemm_post_process_fp8_weight_block(
+ wq, scale, (1, 32), False, True, groups
+ )
+ q, sf = _FUSED_WO_A_KERNEL(
+ x=x, positions=positions, rope=rope, weight=wq, weight_scale=ws
+ )
+
+ a, sa = fused_inv_rope_fp8_quant(
+ x, positions, rope, groups, 8, quant_group_size=32, tma_aligned_scales=True
+ )
+ y = torch.empty(tokens, groups, 1024, device="cuda", dtype=torch.bfloat16)
+ fp8_einsum("bhr,hdr->bhd", (a, sa), (wq, ws), y, recipe=(1, 1, 32))
+ ref, ref_sf = mxfp8_quantize(y.flatten(1), backend="cute-dsl")
+
+ def rows(scales): # F8_128x4 bytes -> [128 rows, n / 32 blocks]
+ return scales.view(n // 128, 32, 4, 4).permute(2, 1, 0, 3).reshape(128, -1)
+
+ actual, expected = (
+ d.float().view(tokens, -1, 32) * torch.exp2(rows(s)[:tokens, :, None] - 127.0)
+ for d, s in ((q, sf), (ref, ref_sf))
+ )
+ assert (actual - expected).norm() / expected.norm() < 3e-3
+ assert not rows(sf)[tokens:].any(), "padded scale rows must be zero"
+
+
# =========================================================================
# Helpers
# =========================================================================
diff --git a/tests/lora/test_olmoe_tp.py b/tests/lora/test_olmoe_tp.py
index bbc25cb6b8..c9a4fd6df5 100644
--- a/tests/lora/test_olmoe_tp.py
+++ b/tests/lora/test_olmoe_tp.py
@@ -139,6 +139,7 @@ def test_olmoe_lora_mixed(olmoe_lora_files):
max_loras=4,
enforce_eager=True,
trust_remote_code=True,
+ hf_overrides={"head_dtype": "float32"},
enable_chunked_prefill=True,
)
diff --git a/tests/model_executor/model_loader/test_weight_cache.py b/tests/model_executor/model_loader/test_weight_cache.py
index 2280fff8b7..e4ac86e0eb 100644
--- a/tests/model_executor/model_loader/test_weight_cache.py
+++ b/tests/model_executor/model_loader/test_weight_cache.py
@@ -12,6 +12,8 @@ import subprocess
import sys
import tempfile
import threading
+import urllib.error
+import urllib.request
from dataclasses import dataclass, field
from typing import Any
@@ -20,6 +22,7 @@ import pytest
from vllm import SamplingParams
from vllm.assets.image import ImageAsset
from vllm.platforms import current_platform
+from vllm.utils.network_utils import get_open_port
class WeightCacheDaemon:
@@ -33,6 +36,7 @@ class WeightCacheDaemon:
):
# Short base path: Unix socket paths are limited to ~107 characters.
self.socket_dir = tempfile.mkdtemp(prefix="vllm_ipc_")
+ self.health_port = get_open_port()
self._cmd = [
sys.executable,
"-m",
@@ -45,6 +49,10 @@ class WeightCacheDaemon:
"--weight-cache-socket-dir",
self.socket_dir,
"--enforce-eager",
+ "--weight-cache-health-host",
+ "127.0.0.1",
+ "--weight-cache-health-port",
+ str(self.health_port),
*(extra_args or []),
]
self._proc: subprocess.Popen | None = None
@@ -63,6 +71,16 @@ class WeightCacheDaemon:
assert self._proc is not None and self._proc.stderr is not None
self._proc.stderr.read()
+ def check_health(self) -> None:
+ url = f"http://127.0.0.1:{self.health_port}/health"
+ try:
+ with urllib.request.urlopen(url, timeout=5) as response:
+ assert response.status == 200
+ except urllib.error.HTTPError as error:
+ raise AssertionError(
+ f"Weight cache daemon is not ready: HTTP {error.code}"
+ ) from error
+
def _stop(self) -> None:
assert self._proc is not None
self._proc.terminate()
@@ -200,6 +218,7 @@ def test_ipc_cache_cold_start_and_warm_restart(vllm_runner, case: ModelCase):
extra_args=case.daemon_args,
) as d:
warm_outputs = generate(vllm_runner, case, d.socket_dir, fallback=False)
+ d.check_health()
# Warm restart: a second engine lifetime against the same daemon.
restart_outputs = generate(vllm_runner, case, d.socket_dir, fallback=False)
diff --git a/tests/models/kimi_k3/test_kda_metadata.py b/tests/models/kimi_k3/test_kda_metadata.py
index 6268cb9bcc..6b3c71bdc6 100644
--- a/tests/models/kimi_k3/test_kda_metadata.py
+++ b/tests/models/kimi_k3/test_kda_metadata.py
@@ -49,6 +49,7 @@ PRUNED_METADATA_FIELDS = {
"prefill_state_indices",
"prefill_has_initial_state",
"spec_sequence_masks",
+ "spec_sequence_masks_cpu",
"uniform_spec_sequence_length",
"flashinfer_prefill_query_start_loc",
"flashinfer_prefill_seq_order",
diff --git a/tests/models/multimodal/generation/test_vit_cudagraph.py b/tests/models/multimodal/generation/test_vit_cudagraph.py
index da472a164d..8c7cf1a68e 100644
--- a/tests/models/multimodal/generation/test_vit_cudagraph.py
+++ b/tests/models/multimodal/generation/test_vit_cudagraph.py
@@ -175,6 +175,24 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
},
marks=[pytest.mark.core_model],
),
+ "minimax_m3": VitCudagraphTestConfig(
+ model="MiniMaxAI/MiniMax-M3",
+ modalities=["image"],
+ image_prompt=("]~b]user\n]<]image[>[What is in this image?[e~[\n]~b]ai\n"),
+ compilation_config_overrides={
+ "encoder_cudagraph_token_budgets": [512],
+ },
+ vllm_runner_kwargs={
+ "trust_remote_code": True,
+ "load_format": "dummy",
+ "hf_overrides": partial(
+ dummy_hf_overrides,
+ model_arch="MiniMaxM3SparseForConditionalGeneration",
+ ),
+ "mm_processor_kwargs": {"max_long_side_pixel": 336},
+ },
+ marks=[pytest.mark.core_model],
+ ),
"qwen3_vl": VitCudagraphTestConfig(
model="Qwen/Qwen3-VL-2B-Instruct",
image_prompt=qwen_vl_chat_template(
diff --git a/tests/models/multimodal/test_mimo_v2_omni.py b/tests/models/multimodal/test_mimo_v2_omni.py
index d4e1d9ec18..deccace50d 100644
--- a/tests/models/multimodal/test_mimo_v2_omni.py
+++ b/tests/models/multimodal/test_mimo_v2_omni.py
@@ -5,13 +5,7 @@
import pytest
import torch
-from tests.utils import ensure_current_vllm_config
-from vllm.distributed.parallel_state import (
- init_distributed_environment,
- initialize_model_parallel,
-)
from vllm.platforms import current_platform
-from vllm.utils.network_utils import get_open_port
EMBED_DIM = 256
NUM_HEADS = 4
@@ -21,20 +15,11 @@ WINDOW = 8
SEQ_LENS = [5, 37]
-@pytest.fixture(scope="module")
-def vision_attn_env():
- init_distributed_environment(
- world_size=1,
- rank=0,
- local_rank=0,
- distributed_init_method=f"tcp://127.0.0.1:{get_open_port()}",
- backend="nccl",
- )
+@pytest.fixture
+def vision_attn_env(dist_init):
default_dtype = torch.get_default_dtype()
torch.set_default_dtype(torch.bfloat16)
- with ensure_current_vllm_config():
- initialize_model_parallel(tensor_model_parallel_size=1)
- yield
+ yield
torch.set_default_dtype(default_dtype)
diff --git a/tests/parser/test_abstract_parser.py b/tests/parser/test_abstract_parser.py
new file mode 100644
index 0000000000..56d20417d1
--- /dev/null
+++ b/tests/parser/test_abstract_parser.py
@@ -0,0 +1,305 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import json
+from typing import Any
+from unittest.mock import MagicMock, patch
+
+import pytest
+from xgrammar import Grammar
+from xgrammar.testing import _is_grammar_accept_string
+
+from vllm.entrypoints.openai.chat_completion.protocol import (
+ ChatCompletionRequest,
+ ChatCompletionToolsParam,
+)
+from vllm.parser.abstract_parser import DelegatingParser
+from vllm.sampling_params import StructuredOutputsParams
+from vllm.tool_parsers.abstract_tool_parser import ToolParser
+from vllm.tool_parsers.qwen3_engine_tool_parser import Qwen3EngineToolParser
+from vllm.tool_parsers.structural_tag_registry import ToolChoice
+
+
+class TestToolChoice_Plus_ResponseFormat:
+ """Note(arpera):
+ Test cases for tool_choice={auto,required} + response_format
+ To keep it short:
+ DelegatingParser.adjust_request behavior in some corner cases is checked there
+
+ Initial bug report:
+ https://github.com/vllm-project/vllm/issues/39929
+ And PR that fixed this:
+ https://github.com/vllm-project/vllm/pull/56086
+ """
+
+ # ================================
+ # Helper methods
+ # ================================
+
+ @staticmethod
+ def _tools(strict: bool) -> list[ChatCompletionToolsParam]:
+ """Single get_weather tool, optionally marked as strict"""
+ function: dict[str, Any] = {
+ "name": "get_weather",
+ "parameters": {
+ "type": "object",
+ "properties": {"city": {"type": "string"}},
+ "required": ["city"],
+ },
+ }
+ if strict:
+ function["strict"] = True
+ return [ChatCompletionToolsParam(type="function", function=function)]
+
+ @staticmethod
+ def _qwen_tool_call() -> str:
+ """Tool call for Qwen model that do support structural tag"""
+ return (
+ "\n\n\nParis\n"
+ "\n\n"
+ )
+
+ @staticmethod
+ def _json_schema_response_format() -> dict:
+ return {
+ "type": "json_schema",
+ "json_schema": {
+ "name": "answer",
+ "schema": {
+ "type": "object",
+ "properties": {"text": {"type": "string"}},
+ "required": ["text"],
+ },
+ },
+ }
+
+ @staticmethod
+ def _setup_request(
+ tools: list[ChatCompletionToolsParam],
+ tool_choice: ToolChoice,
+ response_format: dict,
+ ) -> ChatCompletionRequest:
+ request = ChatCompletionRequest(
+ messages=[], # for our test cases it's always empty
+ model="m", # Just a placeholder, don't pay much attention
+ tools=tools,
+ tool_choice=tool_choice,
+ response_format=response_format,
+ )
+ return request
+
+ @staticmethod
+ def _setup_abstract_parser(
+ tools: list[ChatCompletionToolsParam],
+ ) -> DelegatingParser:
+ """Construct parser that does NOT support structural tag"""
+
+ class TestParser(DelegatingParser):
+ tool_parser_cls = ToolParser
+
+ return TestParser(MagicMock(), tools=tools)
+
+ @staticmethod
+ def _setup_qwen_parser(
+ tools: list[ChatCompletionToolsParam],
+ ) -> DelegatingParser:
+ """Construct parser that supports structural tag"""
+
+ class TestParser(DelegatingParser):
+ tool_parser_cls = Qwen3EngineToolParser
+
+ return TestParser(MagicMock(), tools=tools)
+
+ # ================================
+ # Test cases
+ # tool_choice=auto + response_format
+ # ================================
+
+ @pytest.mark.parametrize(
+ # In this test we check that for response_format
+ # resulting grammar accepts @compliant_output and rejects @non_compliant_output
+ # You can add more examples here if you see some corner cases not covered
+ ("response_format", "compliant_output", "non_compliant_output"),
+ [
+ (_json_schema_response_format(), '{"text": "hi"}', '{"foo": 1}'),
+ ({"type": "json_object"}, '{"any": 1}', "[1, 2]"),
+ ],
+ # We test here two different response_format types:
+ ids=["json_schema", "json_object"],
+ )
+ def test_auto_with_strict_tools(
+ self,
+ response_format: dict,
+ compliant_output: str,
+ non_compliant_output: str,
+ ):
+ tools = self._tools(strict=True)
+ request = self._setup_request(
+ tools=tools,
+ tool_choice="auto",
+ response_format=response_format,
+ )
+ parser = self._setup_qwen_parser(tools)
+ out = parser.adjust_request(request)
+
+ # Now check that request does not have response_format anymore
+ # but instead has structured_outputs set as structural tag
+ # And this structural tag is OR operation
+ assert out.tool_choice == "auto"
+ assert out.response_format is None
+ assert out.structured_outputs is not None
+ tag = json.loads(out.structured_outputs.structural_tag)
+ assert tag["format"]["type"] == "or"
+ grammar = Grammar.from_structural_tag(out.structured_outputs.structural_tag)
+
+ assert _is_grammar_accept_string(grammar, compliant_output)
+ assert not _is_grammar_accept_string(grammar, non_compliant_output)
+
+ # Also tool call must be accepted by grammar
+ assert _is_grammar_accept_string(grammar, self._qwen_tool_call())
+
+ # IMPORTANT(arpera): Regression test
+ # If we in adjust_request implementation by mistake
+ # construct structural tag using tool_choice=auto
+ # then such a structural tag would allow plain text as well
+ # We need to be sure that plain text is NOT accepted in our case
+ assert not _is_grammar_accept_string(grammar, "Hello")
+
+ def test_auto_without_strict_tools(self):
+ tools = self._tools(strict=False)
+ request = self._setup_request(
+ tools=tools,
+ tool_choice="auto",
+ response_format=self._json_schema_response_format(),
+ )
+ parser = self._setup_qwen_parser(tools)
+
+ # There must be a warning that tool calls are disabled
+ # Consume that warning
+ with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn:
+ out = parser.adjust_request(request)
+
+ assert out.response_format is not None
+ assert out.structured_outputs is None
+ mock_warn.assert_called_once()
+
+ def test_when_model_does_not_have_structural_tag(self):
+ """Note(arpera):
+ When model does NOT have structural tag support
+ we apply constraint only for response_format
+ """
+ tools = self._tools(strict=True)
+ request = self._setup_request(
+ tools=tools,
+ tool_choice="auto",
+ response_format=self._json_schema_response_format(),
+ )
+ # SIC! use parser whose model does NOT support structural tag
+ parser = self._setup_abstract_parser(tools)
+
+ with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn:
+ out = parser.adjust_request(request)
+
+ assert out.response_format is not None
+ assert out.structured_outputs is None
+ mock_warn.assert_called_once()
+
+ def test_auto_with_xgrammar_unsupported_schema(self):
+ """Conrer case based on Vadim's feedback in PR #56086
+ https://github.com/vllm-project/vllm/pull/56086#pullrequestreview-5329203459
+ """
+ tools = self._tools(strict=True)
+ request = self._setup_request(
+ tools=tools,
+ tool_choice="auto",
+ response_format={
+ "type": "json_schema",
+ "json_schema": {
+ "name": "answer",
+ "schema": {"type": "integer", "multipleOf": 2},
+ },
+ },
+ )
+ parser = self._setup_qwen_parser(tools)
+
+ with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn:
+ out = parser.adjust_request(request)
+
+ assert out.response_format is not None
+ assert out.structured_outputs is None
+ mock_warn.assert_called_once()
+
+ def test_auto_with_xgrammar_unsupported_nested_schema(self):
+ """Conrer case based on Vadim's feedback in PR #56086
+ https://github.com/vllm-project/vllm/pull/56086#pullrequestreview-5329203459
+ """
+ tools = self._tools(strict=True)
+ request = self._setup_request(
+ tools=tools,
+ tool_choice="auto",
+ response_format={
+ "type": "json_schema",
+ "json_schema": {
+ "name": "answer",
+ "schema": {
+ "type": "object",
+ "properties": {"value": {"type": "number", "multipleOf": 0.5}},
+ "required": ["value"],
+ },
+ },
+ },
+ )
+ parser = self._setup_qwen_parser(tools)
+
+ with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn:
+ out = parser.adjust_request(request)
+
+ assert out.response_format is not None
+ assert out.structured_outputs is None
+ mock_warn.assert_called_once()
+
+ def test_auto_with_lark_grammar(self):
+ """Conrer case based on Vadim's feedback in PR #56086
+ https://github.com/vllm-project/vllm/pull/56086#pullrequestreview-5329203459
+ Lark grammars are converted to EBNF before being merged into the tag.
+ """
+ tools = self._tools(strict=True)
+ request = ChatCompletionRequest(
+ messages=[],
+ model="m",
+ tools=tools,
+ tool_choice="auto",
+ structured_outputs=StructuredOutputsParams(grammar='start: "ok"'),
+ )
+ parser = self._setup_qwen_parser(tools)
+ out = parser.adjust_request(request)
+
+ assert out.structured_outputs is not None
+ grammar = Grammar.from_structural_tag(out.structured_outputs.structural_tag)
+ assert _is_grammar_accept_string(grammar, "ok")
+ assert not _is_grammar_accept_string(grammar, "bad")
+ assert _is_grammar_accept_string(grammar, self._qwen_tool_call())
+
+ # ================================
+ # Test cases
+ # tool_choice=required + response_format
+ # ================================
+
+ def test_required(self):
+ tools = self._tools(strict=True)
+ request = self._setup_request(
+ tools=tools,
+ tool_choice="required",
+ response_format=self._json_schema_response_format(),
+ )
+ parser = self._setup_qwen_parser(tools)
+
+ with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn:
+ out = parser.adjust_request(request)
+
+ assert out.response_format is None
+ assert out.structured_outputs is not None
+ grammar = Grammar.from_structural_tag(out.structured_outputs.structural_tag)
+ assert _is_grammar_accept_string(grammar, self._qwen_tool_call())
+ assert not _is_grammar_accept_string(grammar, '{"text": "hi"}')
+ mock_warn.assert_called_once()
diff --git a/tests/parser/test_harmony.py b/tests/parser/test_harmony.py
index 3d6fff04bf..d8db7f4057 100644
--- a/tests/parser/test_harmony.py
+++ b/tests/parser/test_harmony.py
@@ -1196,6 +1196,21 @@ class TestAdjustRequest:
"TOOL_CALL_2_FUNCTION_FIRST",
],
),
+ (
+ {
+ "tool_choice": "auto",
+ "strict_tools": True,
+ "response_format_type": "json_schema",
+ },
+ [
+ "COMMENTARY",
+ "TOOL_CALL_1_CHANNEL_FIRST",
+ "TOOL_CALL_1_FUNCTION_FIRST",
+ "TOOL_CALL_2_CHANNEL_FIRST",
+ "TOOL_CALL_2_FUNCTION_FIRST",
+ "FINAL_JSON_SCHEMA",
+ ],
+ ),
(
{"response_format_type": "json_schema"},
["FINAL_JSON_SCHEMA"],
@@ -1281,6 +1296,7 @@ class TestAdjustRequest:
"tool_auto_strict",
"tool_required",
"tool_named",
+ "pr56086_tool_auto_strict_response_format_json_schema",
"response_format_json_schema",
"response_format_json_object",
"structured_outputs_json",
diff --git a/tests/v1/attention/test_attention_backends.py b/tests/v1/attention/test_attention_backends.py
index 2f8b80a7c1..77366c02ae 100644
--- a/tests/v1/attention/test_attention_backends.py
+++ b/tests/v1/attention/test_attention_backends.py
@@ -19,7 +19,12 @@ from tests.v1.attention.utils import (
try_backend_includes_kv_cache_update,
try_get_attention_backend,
)
-from vllm.config import ModelConfig, set_current_vllm_config
+from vllm.config import (
+ DiffusionConfig,
+ ModelConfig,
+ SpeculativeConfig,
+ set_current_vllm_config,
+)
from vllm.platforms import current_platform
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import (
@@ -942,8 +947,6 @@ def test_flashinfer_xqa_single_token_decode_preserves_cudagraph_padding(monkeypa
impl.o_sf_scale = None
impl.window_left = -1
impl.sinks = None
- impl.cache_config = unittest.mock.Mock()
- impl.cache_config.get_resolved_kv_cache_layout.return_value = KVCacheLayout.LBHNC
layer = unittest.mock.Mock(
_q_scale=torch.tensor(1.0),
@@ -959,6 +962,7 @@ def test_flashinfer_xqa_single_token_decode_preserves_cudagraph_padding(monkeypa
q_len_per_req=1,
)
attn_metadata = flashinfer_backend.FlashInferMetadata(
+ kv_cache_layout=KVCacheLayout.LBHNC,
num_actual_tokens=2,
slot_mapping=torch.empty(0, dtype=torch.int64),
q_data_type_prefill=torch.bfloat16,
@@ -1070,6 +1074,87 @@ def test_flashinfer_trtllm_gen_padded_decode_uses_varlen_offsets(
assert q_cu_seq_lens.tolist() == qo_indptr_values
+@pytest.mark.skipif(
+ not current_platform.is_cuda()
+ or AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
+ reason="FlashInfer with CUDA is required.",
+)
+@pytest.mark.parametrize(
+ "runner_type,spec_kind,has_upper_bound,expected_copies",
+ [
+ ("generate", None, True, 0),
+ ("pooling", None, True, 0),
+ ("generate", None, False, 1),
+ ("generate", "spec_decode", True, 1),
+ ("generate", "diffusion", True, 1),
+ ],
+)
+def test_flashinfer_avoids_seq_lens_copy_without_spec_tokens(
+ monkeypatch, runner_type, spec_kind, has_upper_bound, expected_copies
+):
+ """Planning reuses the host seq_lens bound only when it is exact.
+
+ Without speculative token accounting (no spec-decode drafts, no dLLM
+ canvas tokens) scheduled tokens always land in KV, so the CPU upper bound
+ is exact regardless of runner type. Any speculative accounting keeps the
+ bound optimistic (rejected tokens are in-flight) and forces the copy.
+ """
+ import unittest.mock
+
+ from vllm.v1.attention.backends import flashinfer as flashinfer_backend
+ from vllm.v1.attention.backends.utils import PerLayerParameters
+
+ monkeypatch.setattr(
+ flashinfer_backend,
+ "get_per_layer_parameters",
+ lambda *args: {
+ "layer.0": PerLayerParameters(
+ window_left=-1, logits_soft_cap=0.0, sm_scale=0.125
+ )
+ },
+ )
+
+ config = create_vllm_config(
+ model_name="Qwen/Qwen3-0.6B", max_model_len=64, max_num_seqs=3
+ )
+ config.model_config.runner_type = runner_type
+ config.attention_config.use_trtllm_attention = False
+ if spec_kind == "spec_decode":
+ config.speculative_config = SpeculativeConfig(
+ method="ngram", num_speculative_tokens=3
+ )
+ elif spec_kind == "diffusion":
+ config.diffusion_config = DiffusionConfig(canvas_length=8)
+ device = torch.device(f"{DEVICE_TYPE}:0")
+ # Include a one-token final chunk and both partial and full KV pages.
+ common_attn_metadata = create_common_attn_metadata(
+ BatchSpec(seq_lens=[17, 20, 32], query_lens=[1, 4, 16]), 16, device
+ )
+ if not has_upper_bound:
+ common_attn_metadata.seq_lens_cpu_upper_bound = None
+ elif spec_kind is not None:
+ # Simulate rejected speculative tokens: CPU says 18, GPU says 17.
+ assert common_attn_metadata.seq_lens_cpu_upper_bound is not None
+ common_attn_metadata.seq_lens_cpu_upper_bound[0] += 1
+
+ with set_current_vllm_config(config):
+ builder = flashinfer_backend.FlashInferMetadataBuilder(
+ create_standard_kv_cache_spec(config), ["layer.0"], config, device
+ )
+ with unittest.mock.patch.object(
+ common_attn_metadata.seq_lens,
+ "cpu",
+ wraps=common_attn_metadata.seq_lens.cpu,
+ ) as copy_to_cpu:
+ metadata = builder.build(0, common_attn_metadata)
+
+ assert copy_to_cpu.call_count == expected_copies
+ assert metadata.num_decodes == 1
+ assert metadata.num_prefills == 2
+ assert builder.paged_kv_indptr.cpu[:4].tolist() == [0, 2, 4, 6]
+ assert builder.paged_kv_last_page_len.cpu[:3].tolist() == [1, 4, 16]
+
+
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
reason="FlashInfer is not available.",
diff --git a/tests/v1/attention/test_cpu_mla_backend.py b/tests/v1/attention/test_cpu_mla_backend.py
index fc3867099e..938babb262 100644
--- a/tests/v1/attention/test_cpu_mla_backend.py
+++ b/tests/v1/attention/test_cpu_mla_backend.py
@@ -25,6 +25,9 @@ from vllm.v1.attention.backends.mla.prefill.cpu_sdpa import (
from vllm.v1.attention.backends.mla.prefill.selector import (
get_mla_prefill_backend,
)
+from vllm.v1.attention.backends.mla.prefill.zen_cpu_sdpa import (
+ ZenCPUSDPAMLAPrefillBackend,
+)
def _reference_ragged_attention(
@@ -157,8 +160,10 @@ def test_cpu_mla_backend_smoke(tmp_path) -> None:
def test_cpu_mla_prefill_backend_selected() -> None:
# On CPU (no device capability) the MLA prefill backend must be the
# SDPA-based CPU backend, not flash-attn which is unavailable on CPU.
+ # Accelerator-specific subclasses (e.g. the zentorch-backed Zen CPU
+ # backend) are also valid selections.
backend_cls = get_mla_prefill_backend(None)
- assert backend_cls is CPUSDPAMLAPrefillBackend
+ assert issubclass(backend_cls, CPUSDPAMLAPrefillBackend)
@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only")
@@ -248,3 +253,64 @@ def test_cpu_mla_prefill_context_chunk() -> None:
assert lse.float().allclose(ref_lse.float(), atol=2e-2)
assert out.shape == (5, 4, 16)
assert lse.shape == (4, 5)
+
+
+@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only")
+@pytest.mark.skipif(
+ not ZenCPUSDPAMLAPrefillBackend.is_available(),
+ reason="requires a Zen CPU with the zentorch_sdpa op registered",
+)
+def test_zen_cpu_mla_prefill_new_tokens() -> None:
+ # Runs the zentorch kernel itself. The head dims are deliberately
+ # asymmetric (v_head_dim < qk_head_dim, as in real MLA) so that the V
+ # padding and output slicing are exercised; the generic-CPU cases above
+ # use equal dims and never reach that code.
+ qk_head_dim = 24
+ v_head_dim = 16
+ backend = ZenCPUSDPAMLAPrefillBackend(
+ num_heads=4,
+ scale=0.25,
+ kv_lora_rank=16,
+ qk_nope_head_dim=16,
+ qk_rope_head_dim=8,
+ v_head_dim=v_head_dim,
+ vllm_config=None,
+ )
+
+ class _PrefillMeta:
+ query_start_loc = torch.tensor([0, 3, 6], dtype=torch.int32)
+
+ backend.prepare_metadata(_PrefillMeta())
+ q = torch.randn(6, 4, qk_head_dim, dtype=torch.bfloat16)
+ k = torch.randn(6, 4, qk_head_dim, dtype=torch.bfloat16)
+ v = torch.randn(6, 4, v_head_dim, dtype=torch.bfloat16)
+
+ out = backend.run_prefill_new_tokens(q, k, v, return_softmax_lse=False)
+ ref, _ = _reference_ragged_attention(
+ q,
+ k,
+ v,
+ _PrefillMeta.query_start_loc,
+ _PrefillMeta.query_start_loc,
+ scale=0.25,
+ causal=True,
+ )
+ assert out.shape == (6, 4, v_head_dim)
+ assert out.float().allclose(ref.float(), atol=2e-2)
+
+ # zentorch_sdpa does not return a usable LSE, so that path must fall back
+ # to the fp32 reference and match the generic backend exactly.
+ generic = CPUSDPAMLAPrefillBackend(
+ num_heads=4,
+ scale=0.25,
+ kv_lora_rank=16,
+ qk_nope_head_dim=16,
+ qk_rope_head_dim=8,
+ v_head_dim=v_head_dim,
+ vllm_config=None,
+ )
+ generic.prepare_metadata(_PrefillMeta())
+ zen_out, zen_lse = backend.run_prefill_new_tokens(q, k, v, return_softmax_lse=True)
+ gen_out, gen_lse = generic.run_prefill_new_tokens(q, k, v, return_softmax_lse=True)
+ assert torch.equal(zen_out, gen_out)
+ assert torch.equal(zen_lse, gen_lse)
diff --git a/tests/v1/attention/test_flashinfer_mla_sparse_sm90.py b/tests/v1/attention/test_flashinfer_mla_sparse_sm90.py
index 3428625e96..b25656cf6e 100644
--- a/tests/v1/attention/test_flashinfer_mla_sparse_sm90.py
+++ b/tests/v1/attention/test_flashinfer_mla_sparse_sm90.py
@@ -9,6 +9,7 @@ causality), ckv/kpe cache splitting, and the backend's model-shape gates.
"""
from types import SimpleNamespace
+from typing import Any
import pytest
import torch
@@ -50,6 +51,8 @@ class FakeWrapper:
def __init__(self):
self.plan_args = None
self.run_args = None
+ self._plan_info: Any = None
+ self._planned_backend: Any = None
def plan(self, *args, **kwargs):
self.plan_args = (args, kwargs)
@@ -69,8 +72,11 @@ class FakeState:
self.kv_indptr = torch.zeros(max_tokens + 1, dtype=torch.int32)
self.wrapper = FakeWrapper()
self.plan_calls = []
+ self.index_topk = 2048
+ self.index_kpool = 4
+ self.max_valid = 2051
- def plan(self, num_tokens, kv_lens):
+ def plan(self, num_tokens, kv_lens, cam, req_id_per_token):
self.plan_calls.append((num_tokens, kv_lens))
def pack_indices(self, slots):
@@ -168,13 +174,12 @@ def test_forward_wiring(monkeypatch, qk_rope, kv_dtype):
def test_builder_plans_only_rows_dispatched_to_mqa(monkeypatch, use_mha, num_decodes):
"""MHA prefill rows must not make the MQA kernel read beyond its query."""
builder = object.__new__(FlashInferMLASparseSM90Builder)
- builder._index_topk = 2048
- builder._index_kpool = 4
- builder._async_scheduling = False
+ builder._adaptive_verification = False
builder.state = FakeState(TOPK)
builder._attention_layer = SimpleNamespace(_use_sparse_mha=lambda _: use_mha)
metadata = object.__new__(sm90_mod.FlashInferMLASparseSM90Metadata)
metadata.state = None
+ metadata.req_id_per_token = None
metadata.num_prefills = 1
metadata.num_decode_tokens = num_decodes
monkeypatch.setattr(
@@ -206,12 +211,17 @@ def test_builder_plans_only_rows_dispatched_to_mqa(monkeypatch, use_mha, num_dec
def test_plan_uses_state_params(monkeypatch):
"""The NoPE/rope dims and scale live on the builder state, not the layer.
- plan() takes exact per-row KV lengths; the schedule is rebuilt on every
- call (contexts grow between steps) and the indptrs are always full-size
+ Without a device batch layout plan() takes exact per-row KV lengths and
+ rebuilds the schedule on every call; the indptrs are always full-size
with zero-query padding rows past num_tokens.
"""
- impl, rows = make_impl(64, "auto")
wrapper = FakeWrapper()
+ plan_info = [0] * sm90_mod._PLAN_INFO_LEN
+ plan_info[sm90_mod._PI_KV_INDPTR] = 64 * 4
+ wrapper._plan_info = torch.tensor(plan_info)
+ wrapper._planned_backend = SimpleNamespace(
+ _int_workspace_buffer=torch.zeros(64, dtype=torch.int32)
+ )
state = sm90_mod._SM90State.__new__(sm90_mod._SM90State)
state.device = torch.device("cpu")
state.wrapper = wrapper
@@ -227,8 +237,9 @@ def test_plan_uses_state_params(monkeypatch):
state._qo_cpu = torch.empty(5, dtype=torch.int32)
state._kv_cpu = torch.empty(5, dtype=torch.int32)
state._lens_cpu = torch.full((4,), TOPK, dtype=torch.int32)
+ state._saved_kv_end = torch.empty(0, dtype=torch.int32)
- state.plan(3, torch.tensor([2, 5, 7], dtype=torch.int32))
+ state.plan(3, torch.tensor([2, 5, 7], dtype=torch.int32), None, None)
assert wrapper.plan_args is not None
args, kwargs = wrapper.plan_args
(qo, kv, indices, kv_len, heads, ckv, kpe, page, causal, scale) = args
@@ -241,10 +252,10 @@ def test_plan_uses_state_params(monkeypatch):
assert kwargs["kv_data_type"] == torch.bfloat16
# Replanning a smaller batch must clear the previous rows' lengths.
- state.plan(1, torch.tensor([TOPK], dtype=torch.int32))
+ state.plan(1, torch.tensor([TOPK], dtype=torch.int32), None, None)
assert state._kv_cpu.tolist() == [0, TOPK, TOPK, TOPK, TOPK]
assert state._lens_cpu.tolist() == [TOPK, 0, 0, 0]
- state.plan(0, torch.empty(0, dtype=torch.int32))
+ state.plan(0, torch.empty(0, dtype=torch.int32), None, None)
assert state._kv_cpu.tolist() == [0, 0, 0, 0, 0]
assert state._lens_cpu.tolist() == [0, 0, 0, 0]
@@ -279,38 +290,49 @@ def test_pack_indices_replays_with_updated_offsets(width):
assert (state.kv_indices[expected.numel() :] == -99).all()
-def test_kv_lens_host_formula():
+@pytest.mark.parametrize("adaptive", [False, True])
+def test_kv_lens_host_formula(adaptive):
"""Per-row host lengths: context == position + 1; capped at
- index_topk + trailing-pool remainder past the sparse threshold."""
+ index_topk + trailing-pool remainder past the sparse threshold.
+
+ Exact (synced) without a host upper bound; with one, a sync-free bound
+ capped at the largest valid count, which the plan clamps on device.
+ """
builder = object.__new__(FlashInferMLASparseSM90Builder)
- builder._index_topk = 2048
- builder._index_kpool = 4
- builder._async_scheduling = False
+ builder.state = FakeState(TOPK)
+ builder._adaptive_verification = adaptive
+ seq_lens = torch.tensor([100, 9, 3000], dtype=torch.int32)
cam = SimpleNamespace(
num_reqs=3,
query_start_loc_cpu=torch.tensor([0, 5, 7, 10], dtype=torch.int32),
- seq_lens=torch.tensor([100, 9, 3000], dtype=torch.int32),
- seq_lens_cpu_upper_bound=torch.tensor([100, 9, 3000], dtype=torch.int32),
- positions=None,
+ seq_lens=seq_lens,
+ seq_lens_cpu_upper_bound=None,
)
- num_rows, lens = builder._kv_lens_host(cam)
- assert num_rows == 10
+ num_rows, lens, exact = builder._kv_lens_host(cam)
+ assert num_rows == 10 and exact
# req0: positions 95..99 -> ctx 96..100 (all <= 2048: full context)
# req1: positions 7,8 -> ctx 8,9
# req2: positions 2997..2999 -> ctx 2998..3000 (> 2048: topk + ctx%4)
assert lens.tolist() == [96, 97, 98, 99, 100, 8, 9, 2050, 2051, 2048]
+ cam.seq_lens_cpu_upper_bound = seq_lens
+ num_rows, lens, exact = builder._kv_lens_host(cam)
+ assert num_rows == 10 and not exact
+ if adaptive:
+ # Host query split may differ from the device one: batch-wide bound.
+ assert lens.tolist() == [2051] * 10
+ else:
+ assert lens.tolist() == [96, 97, 98, 99, 100, 8, 9, 2051, 2051, 2051]
+
def test_kv_lens_host_empty():
builder = object.__new__(FlashInferMLASparseSM90Builder)
- builder._index_topk = 2048
- builder._index_kpool = 4
cam = SimpleNamespace(
num_reqs=0,
query_start_loc_cpu=torch.tensor([0], dtype=torch.int32),
seq_lens=torch.zeros(0, dtype=torch.int32),
)
- num_rows, lens = builder._kv_lens_host(cam)
+ num_rows, lens, _ = builder._kv_lens_host(cam)
assert num_rows == 0 and lens.numel() == 0
@@ -334,7 +356,13 @@ def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens)
"__init__",
lambda *_args, **_kwargs: None,
)
- monkeypatch.setattr(sm90_mod, "_SM90State", lambda *_args, **_kwargs: None)
+
+ class _RecordingState:
+ def __init__(self, *_args, **kwargs):
+ self.index_topk = kwargs["index_topk"]
+ self.index_kpool = kwargs["index_kpool"]
+
+ monkeypatch.setattr(sm90_mod, "_SM90State", _RecordingState)
impl, _ = make_impl(64)
spec = MLAAttentionSpec(
block_size=BLOCK_SIZE, num_kv_heads=1, head_size=576, dtype=torch.bfloat16
@@ -351,6 +379,7 @@ def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens)
max_num_batched_tokens=64, async_scheduling=False
),
model_config=SimpleNamespace(hf_text_config=hf_config),
+ speculative_config=None,
)
builder = FlashInferMLASparseSM90Builder(
spec, ["attn"], vllm_config, torch.device("cpu")
@@ -359,14 +388,138 @@ def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens)
cam = SimpleNamespace(
num_reqs=2,
query_start_loc_cpu=torch.tensor([0, 4, 6], dtype=torch.int32),
- seq_lens_cpu_upper_bound=torch.tensor([42295, 100], dtype=torch.int32),
- positions=None,
+ seq_lens=torch.tensor([42295, 100], dtype=torch.int32),
+ seq_lens_cpu_upper_bound=None,
)
- num_rows, lens = builder._kv_lens_host(cam)
- assert num_rows == 6
+ num_rows, lens, exact = builder._kv_lens_host(cam)
+ assert num_rows == 6 and exact
assert lens.tolist() == prefill_lens + [99, 100]
+def _sm90_flashinfer_available():
+ return (
+ torch.cuda.is_available()
+ and torch.cuda.get_device_capability()[0] == 9
+ and sm90_mod.has_flashinfer_sm90_nope_mla()
+ )
+
+
+SM90_HEADS, SM90_TOPK, SM90_KPOOL, SM90_WIDTH = 16, 2048, 4, 2176
+
+
+def _real_state(rows):
+ return sm90_mod._SM90State(
+ torch.device("cuda"),
+ SM90_HEADS,
+ torch.bfloat16,
+ rows,
+ SM90_WIDTH,
+ HEAD,
+ 0,
+ HEAD**-0.5,
+ SM90_TOPK,
+ SM90_KPOOL,
+ )
+
+
+def _clamp_batch(num_reqs, q_per_req, lo, hi, topk, kpool, width):
+ """Random batch: per-row contexts, device layout, and top-k slots."""
+ last = torch.randint(lo + q_per_req, hi, (num_reqs,))
+ ctx = (last[:, None] + torch.arange(1 - q_per_req, 1)).reshape(-1)
+ valid = torch.where(ctx <= topk, ctx, topk + ctx % kpool)
+ cam = SimpleNamespace(
+ seq_lens=last.to(torch.int32).cuda(),
+ query_start_loc=(torch.arange(num_reqs + 1) * q_per_req).to(torch.int32).cuda(),
+ )
+ req_id = torch.arange(num_reqs, dtype=torch.int32).repeat_interleave(q_per_req)
+ slots = torch.randint(0, 1 << 16, (ctx.numel(), width), dtype=torch.int32)
+ slots[torch.arange(width) >= valid[:, None]] = -1
+ return ctx, valid.to(torch.int32), cam, req_id.cuda(), slots.cuda()
+
+
+@pytest.mark.skipif(
+ not _sm90_flashinfer_available(), reason="Requires SM90 FlashInfer MLA"
+)
+@pytest.mark.parametrize(
+ "num_reqs,q_per_req,lo,hi",
+ [
+ (64, 1, 1, 3000),
+ (16, 6, 2030, 2070), # MTP-style rows straddling index_topk
+ (4096, 1, 1, 3000), # clamp spans several program blocks
+ ],
+)
+def test_upper_bound_plan_matches_exact(num_reqs, q_per_req, lo, hi):
+ """Sync-free planning from optimistic host bounds (async spec decode)
+ must match exact planning: the device clamp trims every work item back
+ to its row's valid count, so no -1 slot is read."""
+ torch.manual_seed(0)
+ heads, topk, kpool, width = SM90_HEADS, SM90_TOPK, SM90_KPOOL, SM90_WIDTH
+ ctx, valid, cam, req_id, slots = _clamp_batch(
+ num_reqs, q_per_req, lo, hi, topk, kpool, width
+ )
+ rows = ctx.numel()
+
+ kv = torch.randn(1 << 16, 1, HEAD, device="cuda", dtype=torch.bfloat16)
+ q = torch.randn(rows, heads, HEAD, device="cuda", dtype=torch.bfloat16)
+ q_pe = q.new_empty(rows, heads, 0)
+
+ def run(state):
+ state.pack_indices(slots)
+ return state.wrapper.run(q, q_pe, kv, kv[..., :0])
+
+ exact = _real_state(rows)
+ exact.plan(rows, valid, None, None)
+ ub = _real_state(rows)
+ host_bound = torch.clamp(ctx + torch.randint(0, 7, (rows,)), max=ub.max_valid)
+ ub.plan(rows, host_bound, cam, req_id)
+ out = run(ub)
+ assert not out.isnan().any()
+ torch.testing.assert_close(out, run(exact), atol=1e-2, rtol=1e-2)
+
+
+@pytest.mark.skipif(
+ not _sm90_flashinfer_available(), reason="Requires SM90 FlashInfer MLA"
+)
+def test_upper_bound_plan_reused_across_steps(monkeypatch):
+ """Growing contexts (draft/decode steps) reuse the padded plan and only
+ re-clamp on device, matching an exact replan every step."""
+ torch.manual_seed(0)
+ heads, topk, kpool, width = SM90_HEADS, SM90_TOPK, SM90_KPOOL, SM90_WIDTH
+ rows = 48
+ ctx, _, _, req_id, _ = _clamp_batch(rows, 1, 1, 3000, topk, kpool, width)
+
+ exact, ub = _real_state(rows), _real_state(rows)
+ replans: list[int] = []
+ plan_fn = ub._plan
+
+ def counting_plan(num_tokens, kv_lens):
+ replans.append(num_tokens)
+ plan_fn(num_tokens, kv_lens)
+
+ monkeypatch.setattr(ub, "_plan", counting_plan)
+ kv = torch.randn(1 << 16, 1, HEAD, device="cuda", dtype=torch.bfloat16)
+ for step in range(5):
+ step_ctx = ctx + step
+ valid = torch.where(step_ctx <= topk, step_ctx, topk + step_ctx % kpool)
+ slots = torch.randint(0, 1 << 16, (rows, width), dtype=torch.int32)
+ slots[torch.arange(width) >= valid[:, None]] = -1
+ slots = slots.cuda()
+ cam = SimpleNamespace(
+ seq_lens=step_ctx.to(torch.int32).cuda(),
+ query_start_loc=torch.arange(rows + 1, dtype=torch.int32).cuda(),
+ )
+ num_plans = len(replans)
+ ub.plan(rows, torch.clamp(step_ctx + 5, max=ub.max_valid), cam, req_id)
+ assert len(replans) - num_plans == (1 if step == 0 else 0)
+ exact.plan(rows, valid.to(torch.int32), None, None)
+ q = torch.randn(rows, heads, HEAD, device="cuda", dtype=torch.bfloat16)
+ outs = []
+ for state in (ub, exact):
+ state.pack_indices(slots)
+ outs.append(state.wrapper.run(q, q[..., :0], kv, kv[..., :0]))
+ torch.testing.assert_close(outs[0], outs[1], atol=1e-2, rtol=1e-2)
+
+
def test_supports_combination_gates(monkeypatch, default_vllm_config):
monkeypatch.setattr(sm90_mod, "has_flashinfer_sm90_nope_mla", lambda: True)
call = lambda **kw: FlashInferMLASparseSM90Backend.supports_combination(
diff --git a/tests/v1/attention/test_gdn_metadata_builder.py b/tests/v1/attention/test_gdn_metadata_builder.py
index b4d3acf07c..44dee49a4e 100644
--- a/tests/v1/attention/test_gdn_metadata_builder.py
+++ b/tests/v1/attention/test_gdn_metadata_builder.py
@@ -21,6 +21,7 @@ from vllm.v1.attention.backends.gdn_attn import (
GDNAttentionMetadata,
GDNAttentionMetadataBuilder,
)
+from vllm.v1.attention.backends.utils import mamba_get_block_table_tensor
from vllm.v1.kv_cache_interface import MambaSpec
BLOCK_SIZE = 16
@@ -75,6 +76,17 @@ GDN_BUILD_TEST_CASES = {
expected_num_prefill_tokens=0,
expected_num_spec_decodes=0,
),
+ # No speculative config, decode alongside prefill
+ "regular_decode_with_prefill": GDNBuildTestCase(
+ seq_lens=[40, 100],
+ query_lens=[1, 50],
+ num_decode_draft_tokens=None,
+ num_speculative_tokens=0,
+ expected_num_decodes=1,
+ expected_num_prefills=1,
+ expected_num_prefill_tokens=50,
+ expected_num_spec_decodes=0,
+ ),
# Multi-token prefill alongside spec decode — no decode to reclassify
"spec_decode_with_real_prefill": GDNBuildTestCase(
seq_lens=[100, 20],
@@ -155,9 +167,12 @@ def _build(
builder: GDNAttentionMetadataBuilder,
batch_spec: BatchSpec,
num_decode_draft_tokens: list[int] | None = None,
+ block_table: torch.Tensor | None = None,
) -> GDNAttentionMetadata:
"""Build GDN attention metadata, optionally with spec-decode kwargs."""
common = create_common_attn_metadata(batch_spec, BLOCK_SIZE, DEVICE)
+ if block_table is not None:
+ common = common.replace(block_table_tensor=block_table)
kwargs: dict = {}
if num_decode_draft_tokens is not None:
kwargs["num_decode_draft_tokens_cpu"] = torch.tensor(
@@ -184,6 +199,54 @@ def test_gdn_build_classification(test_case: GDNBuildTestCase):
assert meta.num_spec_decodes == test_case.expected_num_spec_decodes
+@pytest.mark.parametrize("mamba_cache_mode", ["none", "align"])
+@pytest.mark.parametrize("full_cuda_graph", [False, True])
+@pytest.mark.parametrize(
+ "test_case", GDN_BUILD_TEST_CASES.values(), ids=GDN_BUILD_TEST_CASES.keys()
+)
+def test_update_block_table_matches_build(
+ test_case: GDNBuildTestCase, full_cuda_graph: bool, mamba_cache_mode: str
+):
+ """update_block_table() on another group's metadata matches build()."""
+ batch = BatchSpec(seq_lens=test_case.seq_lens, query_lens=test_case.query_lens)
+ src, dst, ref = (
+ _create_gdn_builder(test_case.num_speculative_tokens, full_cuda_graph)
+ for _ in range(3)
+ )
+ for builder in (src, dst, ref):
+ builder.vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
+ common = create_common_attn_metadata(batch, BLOCK_SIZE, DEVICE)
+ if mamba_cache_mode == "align":
+ # MRV2 precomputes these for every Mamba group each step.
+ dst.mamba_aligned_state_indices = ref.mamba_aligned_state_indices = (
+ mamba_get_block_table_tensor(
+ common.block_table_tensor, common.seq_lens, ref.kv_cache_spec, "align"
+ )
+ )
+ draft_tokens = test_case.num_decode_draft_tokens
+ expected = _build(ref, batch, draft_tokens, common.block_table_tensor)
+ source = _build(src, batch, draft_tokens)
+ fields = (
+ "spec_state_indices_tensor",
+ "non_spec_state_indices_tensor",
+ "prefill_state_indices",
+ )
+ source_indices = [getattr(source, f) for f in fields]
+ source_indices = [t if t is None else t.clone() for t in source_indices]
+ meta = dst.update_block_table(
+ source, common.block_table_tensor, common.slot_mapping
+ )
+
+ for field, source_index in zip(fields, source_indices):
+ actual = getattr(meta, field)
+ torch.testing.assert_close(actual, getattr(expected, field))
+ # The source group's indices are untouched.
+ torch.testing.assert_close(getattr(source, field), source_index)
+ # FULL graph state indices land in this group's own buffers.
+ if full_cuda_graph and meta.num_prefills == 0 and actual is not None:
+ assert actual.data_ptr() == getattr(dst, field).data_ptr()
+
+
def test_has_initial_state_after_reclassification():
"""After reclassification, num_prefills > 0 so the prefill kernel path
should compute has_initial_state. For the reclassified request with
diff --git a/tests/v1/attention/test_group_head_counts.py b/tests/v1/attention/test_group_head_counts.py
index 54fe3cfdbc..40d9e2d00f 100644
--- a/tests/v1/attention/test_group_head_counts.py
+++ b/tests/v1/attention/test_group_head_counts.py
@@ -101,6 +101,7 @@ def test_flash_attention_geometry_comes_from_the_group():
num_kv_heads=2,
head_size=64,
dtype=torch.bfloat16,
+ dcp_sharded=True,
)
with (
diff --git a/tests/v1/attention/test_mla_prefill_selector.py b/tests/v1/attention/test_mla_prefill_selector.py
index 098eddfbf6..87d188c6a0 100644
--- a/tests/v1/attention/test_mla_prefill_selector.py
+++ b/tests/v1/attention/test_mla_prefill_selector.py
@@ -60,10 +60,43 @@ class TestGetMLAPrefillBackend:
def test_cpu_uses_sdpa_prefill(self):
vllm_config = _make_vllm_config()
+ zen_cls = MLAPrefillBackendEnum.ZEN_CPU.get_class()
+
+ # A Zen CPU with zentorch selects the Zen backend, so expect whichever
+ # one this host offers.
+ expected = (
+ zen_cls if zen_cls.is_available() else MLAPrefillBackendEnum.CPU.get_class()
+ )
with patch("vllm.platforms.current_platform") as mock_platform:
mock_platform.is_cpu.return_value = True
+ backend = get_mla_prefill_backend(vllm_config)
+ assert backend is expected
+
+ def test_zen_cpu_prefers_zentorch_prefill(self):
+ vllm_config = _make_vllm_config()
+ zen_cls = MLAPrefillBackendEnum.ZEN_CPU.get_class()
+
+ with (
+ patch("vllm.platforms.current_platform") as mock_platform,
+ patch.object(zen_cls, "is_available", return_value=True),
+ ):
+ mock_platform.is_cpu.return_value = True
+
+ backend = get_mla_prefill_backend(vllm_config)
+ assert backend is zen_cls
+
+ def test_zen_cpu_falls_back_when_zentorch_unavailable(self):
+ vllm_config = _make_vllm_config()
+ zen_cls = MLAPrefillBackendEnum.ZEN_CPU.get_class()
+
+ with (
+ patch("vllm.platforms.current_platform") as mock_platform,
+ patch.object(zen_cls, "is_available", return_value=False),
+ ):
+ mock_platform.is_cpu.return_value = True
+
backend = get_mla_prefill_backend(vllm_config)
assert backend is MLAPrefillBackendEnum.CPU.get_class()
diff --git a/tests/v1/core/test_kv_cache_utils.py b/tests/v1/core/test_kv_cache_utils.py
index 24be16e4a5..c2c86053f0 100644
--- a/tests/v1/core/test_kv_cache_utils.py
+++ b/tests/v1/core/test_kv_cache_utils.py
@@ -1599,19 +1599,142 @@ def test_project_kv_cache_groups_to_worker():
assert set(proj_spec.kv_cache_specs.keys()) == {"layer1", "layer3"}
-def test_dcp_world_size_for_kv_cache_spec_shards_full_attention_only():
- dcp = 8
- full = FullAttentionSpec(
- block_size=16, num_kv_heads=1, head_size=1, dtype=torch.float32
+@pytest.mark.parametrize("sliding_window", [None, 256])
+@pytest.mark.parametrize("disable_hybrid", [False, True])
+@pytest.mark.parametrize("pcp_size", [1, 4])
+def test_dcp_target_allocates_replicated_draft_independently(
+ monkeypatch, sliding_window, disable_hybrid, pcp_size
+):
+ """A draft must retain all positions even when the target shards them."""
+ from vllm.v1.attention.backends.utils import resolve_kv_cache_layout
+
+ monkeypatch.delenv("VLLM_KV_CACHE_LAYOUT", raising=False)
+ config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
+ config.parallel_config.decode_context_parallel_size = 4
+ config.parallel_config.prefill_context_parallel_size = pcp_size
+ config.scheduler_config.disable_hybrid_kv_cache_manager = disable_hybrid
+ config.cache_config.block_size = 16
+ config.cache_config.kv_cache_layout = None
+ draft_args = dict(
+ block_size=16,
+ num_kv_heads=8,
+ head_size=64,
+ dtype=torch.bfloat16,
+ dcp_sharded=False,
)
- mla = new_mla_spec()
- mamba = new_mamba_spec()
- uniform_mla = UniformTypeKVCacheSpecs(block_size=16, kv_cache_specs={"layer": mla})
- assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(full, dcp) == dcp
- assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(mla, dcp) == dcp
- assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(uniform_mla, dcp) == dcp
- assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(mamba, dcp) == 1
- assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(full, 1) == 1
+ draft = (
+ FullAttentionSpec(**draft_args)
+ if sliding_window is None
+ else SlidingWindowSpec(**draft_args, sliding_window=sliding_window)
+ )
+ specs = {"target": new_mla_spec(), "draft": draft}
+ layout = resolve_kv_cache_layout(config, [["LBHNC", "BLHNC"]], specs.values())
+ assert layout == KVCacheLayout.BLHNC
+ groups = get_kv_cache_groups(config, specs)
+ assert [group.layer_names for group in groups] == [["target"], ["draft"]]
+ widths = [g.kv_cache_spec.max_num_blocks_per_req(config, 1024) for g in groups]
+ assert widths == [16, 64]
+
+ cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
+ config, groups, available_memory=16 * 1024 * 1024
+ )
+ scheduler_config = generate_scheduler_kv_cache_config([cache_config])
+ assert get_max_concurrency_for_kv_cache_config(config, cache_config) > 0
+ assert get_max_concurrency_for_kv_cache_config(config, cache_config) == (
+ get_max_concurrency_for_kv_cache_config(config, scheduler_config)
+ )
+ manager = KVCacheManager(
+ scheduler_config,
+ max_model_len=1024,
+ hash_block_size=16,
+ scheduler_block_size=64,
+ dcp_world_size=4,
+ pcp_world_size=pcp_size,
+ enable_caching=True,
+ )
+ assert manager.coordinator.group_block_sizes == (64, 16)
+ request = make_request("replicated-draft", [1] * 65, block_size=16, hash_fn=sha256)
+ blocks = manager.allocate_slots(request, 65)
+ assert blocks is not None
+ assert [len(group) for group in blocks.blocks] == [2, 5]
+ manager.cache_blocks(request, 64)
+ cached_request = make_request(
+ "cached-draft", [1] * 65, block_size=16, hash_fn=sha256
+ )
+ cached_blocks, num_cached, _ = manager.get_computed_blocks(cached_request)
+ assert num_cached == 64
+ assert [len(group) for group in cached_blocks.blocks] == [1, 4]
+
+
+@pytest.mark.parametrize("use_mla", [False, True])
+def test_full_attention_merge_preserves_replicated_cache_geometry(use_mla):
+ config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
+ config.parallel_config.decode_context_parallel_size = 4
+ draft = replace(
+ new_mla_spec() if use_mla else new_kv_cache_spec(),
+ block_size=16,
+ dcp_sharded=False,
+ )
+ merged = type(draft).merge([draft, draft])
+ wrapped = UniformTypeKVCacheSpecs.from_specs({"draft": merged})
+ assert wrapped is not None and not wrapped.dcp_sharded
+ assert merged.max_num_blocks_per_req(config, 1024) == 64
+ assert merged.max_memory_usage_bytes(config) == 64 * draft.page_size_bytes
+ assert kv_cache_utils.resolve_dcp_kv_block_size(merged, 4) == 16
+ with pytest.raises(AssertionError):
+ type(draft).merge([draft, replace(draft, dcp_sharded=True)])
+
+
+@pytest.mark.parametrize("with_draft", [False, True])
+@pytest.mark.parametrize("indexer_alignment", [None, 512])
+def test_sparse_mla_preserves_physical_row_addressing(with_draft, indexer_alignment):
+ from vllm.v1.attention.backends.mla.sparse_utils import flat_kv_row_view
+ from vllm.v1.worker.utils import allocate_kv_cache
+
+ config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
+ config.cache_config.kv_cache_layout = "BLHNC"
+ config.parallel_config.decode_context_parallel_size = 4
+ common = dict(block_size=64, num_kv_heads=1, dtype=torch.uint8)
+ specs = {
+ "target": MLAAttentionSpec(
+ **common,
+ head_size=576,
+ state_content_bytes=656,
+ block_stride_alignment=656,
+ cache_dtype_str="fp8_ds_mla",
+ is_index_group_leader=True,
+ ),
+ "indexer": MLAAttentionSpec(
+ **common,
+ head_size=132,
+ cache_role=SparseCacheRole.INDEXER,
+ block_stride_alignment=indexer_alignment,
+ ),
+ "draft": SlidingWindowSpec(
+ block_size=64,
+ num_kv_heads=64,
+ head_size=64,
+ dtype=torch.bfloat16,
+ sliding_window=2048,
+ dcp_sharded=False,
+ ),
+ }
+ if not with_draft:
+ del specs["draft"]
+ groups = get_kv_cache_groups(config, specs)
+ cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
+ config, groups, available_memory=8 * 1024 * 1024
+ )
+ caches = allocate_kv_cache(
+ cache_config, torch.device("cpu"), KVCacheLayout.BLHNC, [64] * len(groups)
+ )
+ cache = caches["target"].squeeze(1)
+ if indexer_alignment is not None:
+ assert caches["indexer"].stride(0) % indexer_alignment == 0
+ rows, stride_rows = flat_kv_row_view(cache, 64)
+ cache[1, 0].fill_(7)
+ torch.testing.assert_close(rows[stride_rows], cache[1, 0])
+ assert torch.all(rows[stride_rows] == 7)
@pytest.mark.parametrize(
@@ -3815,6 +3938,8 @@ def test_unify_hybrid_kv_cache_specs():
"layer_1": before_spec_1,
"layer_2": before_spec_2,
}
+ kv_cache_spec["draft_layer_1"] = replace(before_spec_1, dcp_sharded=False)
+ kv_cache_spec["draft_layer_2"] = replace(before_spec_2, dcp_sharded=False)
kv_cache_utils.unify_hybrid_kv_cache_specs(kv_cache_spec)
expected_spec_1 = new_kv_cache_spec(block_size=64)
expected_spec_2 = new_kv_cache_spec(
@@ -3823,6 +3948,7 @@ def test_unify_hybrid_kv_cache_specs():
assert kv_cache_spec["layer_1"] == expected_spec_1
assert kv_cache_spec["layer_2"] == expected_spec_2
assert kv_cache_spec["layer_2"].page_size_bytes == 64 * 1024
+ assert kv_cache_spec["draft_layer_2"] == replace(expected_spec_2, dcp_sharded=False)
# 2. has_full_attention and has_chunked_local_attention
before_spec_1 = new_kv_cache_spec()
@@ -3874,6 +4000,11 @@ def test_unify_hybrid_kv_cache_specs():
with pytest.raises(ValueError):
kv_cache_utils.unify_hybrid_kv_cache_specs(kv_cache_spec)
+ # Replicated Mamba state still requires the hybrid cache manager.
+ kv_cache_spec = {"attention": new_kv_cache_spec(), "mamba": new_mamba_spec()}
+ with pytest.raises(ValueError):
+ kv_cache_utils.unify_hybrid_kv_cache_specs(kv_cache_spec)
+
def test_unify_kv_cache_spec_page_size_mamba():
"""Regression test for https://github.com/vllm-project/vllm/issues/43626.
diff --git a/tests/v1/core/test_prefix_caching.py b/tests/v1/core/test_prefix_caching.py
index 0a34a6131d..99d3235006 100644
--- a/tests/v1/core/test_prefix_caching.py
+++ b/tests/v1/core/test_prefix_caching.py
@@ -1336,12 +1336,12 @@ def make_kv_cache_config_three_types(
)
-def test_prefix_cache_hit_uses_per_group_dcp_geometry():
+@pytest.mark.parametrize("draft_sharded", [True, False])
+def test_prefix_cache_hit_uses_per_group_dcp_geometry(draft_sharded):
"""Prefix lookup must use each group's DCP size, not the process-wide one.
- Target and draft MLA are both sharded (DCP=8); Mamba stays replicated
- (DCP=1). Hits then align to the sharded full-attention block, not the
- unsharded page size.
+ Target MLA is sharded; draft MLA can opt out and Mamba stays replicated.
+ Hits must align to the target block in either case.
"""
block_size = 16
dcp = 8
@@ -1362,6 +1362,7 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry():
KVCacheGroupSpec(
["draft_mla"],
MLAAttentionSpec(
+ dcp_sharded=draft_sharded,
block_size=block_size,
num_kv_heads=1,
head_size=1,
@@ -1388,10 +1389,10 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry():
)
target_mgr, draft_mgr, mamba_mgr = manager.coordinator.single_type_managers
assert target_mgr.dcp_world_size == dcp
- assert draft_mgr.dcp_world_size == dcp
+ assert draft_mgr.dcp_world_size == (dcp if draft_sharded else 1)
assert mamba_mgr.dcp_world_size == 1
assert target_mgr.block_size == sharded_block
- assert draft_mgr.block_size == sharded_block
+ assert draft_mgr.block_size == (sharded_block if draft_sharded else block_size)
assert mamba_mgr.block_size == block_size
hash_fn = sha256
@@ -1407,7 +1408,11 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry():
req1 = make_request("1", common_token_ids + [100] * 5, block_size, hash_fn)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
assert num_computed_tokens == 2 * sharded_block
- assert [len(group) for group in computed_blocks.blocks] == [2, 2, 16]
+ assert [len(group) for group in computed_blocks.blocks] == [
+ 2,
+ 2 if draft_sharded else 16,
+ 16,
+ ]
manager.free(req0)
manager.free(req1)
diff --git a/tests/v1/e2e/spec_decode/draft_model/test_draft_model.py b/tests/v1/e2e/spec_decode/draft_model/test_draft_model.py
index 3a474ac203..d2c2ac87b7 100644
--- a/tests/v1/e2e/spec_decode/draft_model/test_draft_model.py
+++ b/tests/v1/e2e/spec_decode/draft_model/test_draft_model.py
@@ -11,6 +11,7 @@ from vllm.config import CompilationConfig, VllmConfig, replace
from vllm.config.kernel import MoEBackend
from vllm.engine.arg_utils import EngineArgs
from vllm.platforms import current_platform
+from vllm.v1.spec_decode.draft_model import DraftModelProposer
from ..utils import (
Messages,
@@ -196,6 +197,38 @@ def test_draft_model_engine_args_tensor_parallelism():
assert draft_config.quant_config is None
+@pytest.mark.cpu_test
+@pytest.mark.parametrize("dcp_size", [1, 2])
+def test_draft_model_inherits_target_dcp_config(monkeypatch, dcp_size):
+ monkeypatch.setattr(current_platform, "device_count", lambda: 16)
+ target_config = EngineArgs(
+ model="Qwen/Qwen3-1.7B",
+ tensor_parallel_size=16,
+ decode_context_parallel_size=dcp_size,
+ cp_kv_cache_interleave_size=16,
+ dcp_comm_backend="ag_rs",
+ distributed_executor_backend="mp",
+ speculative_config={
+ "model": "Qwen/Qwen3-0.6B",
+ "method": "draft_model",
+ "num_speculative_tokens": 3,
+ },
+ ).create_engine_config()
+ proposer = object.__new__(DraftModelProposer)
+ proposer.vllm_config = target_config
+ proposer.speculative_config = target_config.speculative_config
+ draft_config = proposer._create_draft_vllm_config()
+ draft_parallel = draft_config.parallel_config
+ assert draft_parallel.decode_context_parallel_size == dcp_size
+ assert draft_parallel.cp_kv_cache_interleave_size == 16
+ assert draft_parallel.dcp_comm_backend == "ag_rs"
+ assert (
+ draft_parallel.dcp_q_replicate == target_config.parallel_config.dcp_q_replicate
+ )
+ assert draft_parallel is not target_config.parallel_config
+ assert target_config.parallel_config.decode_context_parallel_size == dcp_size
+
+
def _apply_draft_moe_backend(vllm_config: VllmConfig) -> VllmConfig:
"""Replicate SpecDecodeBaseProposer._create_draft_vllm_config logic
so we can test it without instantiating a full proposer."""
diff --git a/tests/v1/e2e/spec_decode/mtp/_correctness.py b/tests/v1/e2e/spec_decode/mtp/_correctness.py
new file mode 100644
index 0000000000..b88c1cce7a
--- /dev/null
+++ b/tests/v1/e2e/spec_decode/mtp/_correctness.py
@@ -0,0 +1,120 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from typing import Any
+
+import pytest
+
+from vllm import SamplingParams
+from vllm.config import CompilationConfig
+from vllm.platforms import current_platform
+
+from ..utils import (
+ _skip_if_insufficient_gpus_for_tp,
+ assert_request_outputs_match,
+ evaluate_llm_for_gsm8k,
+ get_test_prompts,
+)
+
+
+def check_mtp_correctness(
+ monkeypatch: pytest.MonkeyPatch,
+ sampling_config: SamplingParams,
+ model_setup: tuple[str, str, int, str | None],
+ mm_enabled: bool,
+ expected_accuracy_threshold: float,
+ vllm_runner,
+):
+ """Compare the outputs of a original LLM and a speculative LLM
+ which should be the same when using MTP speculative decoding. Due to some variance
+ in the engine, it is possible for some outputs to differ, so we expect that at least
+ 6/10 output tokens match exactly, and that the GSM8k accuracy is above a precomputed
+ reference threshold for each model.
+ """
+ method, model_name, tp_size, draft_model = model_setup
+ _skip_if_insufficient_gpus_for_tp(tp_size)
+
+ # Generate test prompts inside the function instead of using fixture
+ test_prompts = get_test_prompts(mm_enabled)
+ with monkeypatch.context() as m:
+ m.setenv("VLLM_MLA_DISABLE", "1")
+
+ attn_backend = "TRITON_ATTN" if current_platform.is_rocm() else "auto"
+
+ # Skip multimodal profiling for models that don't need it in this test.
+ extra_kwargs: dict[str, Any] = {}
+ if "Qwen3.5" in model_name:
+ extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "video": 0}
+ elif "gemma-4" in model_name:
+ extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "audio": 0}
+
+ if draft_model is not None and "gemma-4" in draft_model:
+ import transformers
+ from packaging.version import Version
+
+ if Version(transformers.__version__) < Version("5.8.0"):
+ pytest.skip(
+ "Gemma4 MTP assistant requires transformers>=5.8.0, "
+ f"got {transformers.__version__}"
+ )
+
+ with vllm_runner(
+ model_name,
+ block_size=None,
+ max_model_len=2048,
+ tensor_parallel_size=tp_size,
+ trust_remote_code=True,
+ attention_backend=attn_backend,
+ enable_chunked_prefill=None,
+ compilation_config=CompilationConfig(),
+ **extra_kwargs,
+ ) as ref_runner:
+ ref_outputs = ref_runner.llm.chat(test_prompts, sampling_config)
+ evaluate_llm_for_gsm8k(
+ ref_runner.llm,
+ expected_accuracy_threshold=expected_accuracy_threshold,
+ )
+
+ speculative_config: dict[str, Any] = {
+ "method": method,
+ "num_speculative_tokens": 1,
+ "max_model_len": 2048,
+ }
+ if draft_model is not None:
+ speculative_config["model"] = draft_model
+ speculative_config["num_speculative_tokens"] = 2
+
+ with vllm_runner(
+ model_name,
+ block_size=None,
+ trust_remote_code=True,
+ tensor_parallel_size=tp_size,
+ speculative_config=speculative_config,
+ max_model_len=2048,
+ attention_backend=attn_backend,
+ enable_chunked_prefill=None,
+ compilation_config=CompilationConfig(),
+ **extra_kwargs,
+ ) as spec_runner:
+ # MTP supports async scheduling by default.
+ has_async = (
+ spec_runner.llm.llm_engine.vllm_config.scheduler_config.async_scheduling
+ )
+ assert has_async, (
+ f"Expected async scheduling for {method}: target={model_name}, "
+ f"draft={draft_model}; got {has_async}"
+ )
+ evaluate_llm_for_gsm8k(
+ spec_runner.llm,
+ expected_accuracy_threshold=expected_accuracy_threshold,
+ )
+ spec_outputs = spec_runner.llm.chat(test_prompts, sampling_config)
+
+ # Heuristic: expect at least 80% of the prompts to match exactly
+ # Upon failure, inspect the outputs to check for inaccuracy.
+ assert_request_outputs_match(
+ ref_outputs,
+ spec_outputs,
+ required_matches=int(0.8 * len(ref_outputs)) + 1,
+ context=f"{method} target={model_name}, draft={draft_model}",
+ )
diff --git a/tests/v1/e2e/spec_decode/mtp/deepseek/__init__.py b/tests/v1/e2e/spec_decode/mtp/deepseek/__init__.py
new file mode 100644
index 0000000000..208f01a7cb
--- /dev/null
+++ b/tests/v1/e2e/spec_decode/mtp/deepseek/__init__.py
@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
diff --git a/tests/v1/e2e/spec_decode/mtp/deepseek/test_mtp.py b/tests/v1/e2e/spec_decode/mtp/deepseek/test_mtp.py
new file mode 100644
index 0000000000..a23c3ee26e
--- /dev/null
+++ b/tests/v1/e2e/spec_decode/mtp/deepseek/test_mtp.py
@@ -0,0 +1,44 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import pytest
+
+from tests.utils import single_gpu_only
+from vllm import SamplingParams
+from vllm.platforms import current_platform
+
+from .._correctness import check_mtp_correctness
+
+
+@pytest.mark.parametrize(
+ ["model_setup", "mm_enabled", "expected_accuracy_threshold"],
+ [
+ pytest.param(
+ ("mtp", "ZixiQi/DeepSeek-V3-4layers-MTP-FP8", 1, None),
+ False,
+ 0.0,
+ marks=pytest.mark.skipif(
+ current_platform.is_device_capability_family(100),
+ reason="DeepSeek MTP: TRTLLM MoE top_k check fails on Blackwell",
+ ),
+ ), # dummy model
+ ],
+ ids=["deepseek"],
+)
+@single_gpu_only
+def test_mtp_correctness(
+ monkeypatch: pytest.MonkeyPatch,
+ sampling_config: SamplingParams,
+ model_setup: tuple[str, str, int, str | None],
+ mm_enabled: bool,
+ expected_accuracy_threshold: float,
+ vllm_runner,
+):
+ check_mtp_correctness(
+ monkeypatch,
+ sampling_config,
+ model_setup,
+ mm_enabled,
+ expected_accuracy_threshold,
+ vllm_runner,
+ )
diff --git a/tests/v1/e2e/spec_decode/mtp/gemma4/__init__.py b/tests/v1/e2e/spec_decode/mtp/gemma4/__init__.py
new file mode 100644
index 0000000000..208f01a7cb
--- /dev/null
+++ b/tests/v1/e2e/spec_decode/mtp/gemma4/__init__.py
@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
diff --git a/tests/v1/e2e/spec_decode/mtp/gemma4/test_mtp.py b/tests/v1/e2e/spec_decode/mtp/gemma4/test_mtp.py
new file mode 100644
index 0000000000..66fb49fcef
--- /dev/null
+++ b/tests/v1/e2e/spec_decode/mtp/gemma4/test_mtp.py
@@ -0,0 +1,39 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import pytest
+
+from tests.utils import single_gpu_only
+from vllm import SamplingParams
+
+from .._correctness import check_mtp_correctness
+
+
+@pytest.mark.parametrize(
+ ["model_setup", "mm_enabled", "expected_accuracy_threshold"],
+ [
+ (
+ ("mtp", "google/gemma-4-E4B-it", 1, "google/gemma-4-E4B-it-assistant"),
+ False,
+ 0.50,
+ ), # gemma4 MTP with assistant model, ref: ~62%
+ ],
+ ids=["gemma4-e4b"],
+)
+@single_gpu_only
+def test_mtp_correctness(
+ monkeypatch: pytest.MonkeyPatch,
+ sampling_config: SamplingParams,
+ model_setup: tuple[str, str, int, str | None],
+ mm_enabled: bool,
+ expected_accuracy_threshold: float,
+ vllm_runner,
+):
+ check_mtp_correctness(
+ monkeypatch,
+ sampling_config,
+ model_setup,
+ mm_enabled,
+ expected_accuracy_threshold,
+ vllm_runner,
+ )
diff --git a/tests/v1/e2e/spec_decode/mtp/mimo/__init__.py b/tests/v1/e2e/spec_decode/mtp/mimo/__init__.py
new file mode 100644
index 0000000000..208f01a7cb
--- /dev/null
+++ b/tests/v1/e2e/spec_decode/mtp/mimo/__init__.py
@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
diff --git a/tests/v1/e2e/spec_decode/mtp/mimo/test_mtp.py b/tests/v1/e2e/spec_decode/mtp/mimo/test_mtp.py
new file mode 100644
index 0000000000..8a4e7e1236
--- /dev/null
+++ b/tests/v1/e2e/spec_decode/mtp/mimo/test_mtp.py
@@ -0,0 +1,36 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import pytest
+
+from tests.utils import single_gpu_only
+from vllm import SamplingParams
+
+from .._correctness import check_mtp_correctness
+
+
+@pytest.mark.parametrize(
+ ["model_setup", "mm_enabled", "expected_accuracy_threshold"],
+ [
+ # Reference accuracy: 65%-70%.
+ (("mtp", "XiaomiMiMo/MiMo-7B-Base", 1, None), False, 0.5),
+ ],
+ ids=["mimo"],
+)
+@single_gpu_only
+def test_mtp_correctness(
+ monkeypatch: pytest.MonkeyPatch,
+ sampling_config: SamplingParams,
+ model_setup: tuple[str, str, int, str | None],
+ mm_enabled: bool,
+ expected_accuracy_threshold: float,
+ vllm_runner,
+):
+ check_mtp_correctness(
+ monkeypatch,
+ sampling_config,
+ model_setup,
+ mm_enabled,
+ expected_accuracy_threshold,
+ vllm_runner,
+ )
diff --git a/tests/v1/e2e/spec_decode/mtp/qwen3_5/__init__.py b/tests/v1/e2e/spec_decode/mtp/qwen3_5/__init__.py
new file mode 100644
index 0000000000..208f01a7cb
--- /dev/null
+++ b/tests/v1/e2e/spec_decode/mtp/qwen3_5/__init__.py
@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
diff --git a/tests/v1/e2e/spec_decode/mtp/qwen3_5/test_mtp.py b/tests/v1/e2e/spec_decode/mtp/qwen3_5/test_mtp.py
new file mode 100644
index 0000000000..c61838a655
--- /dev/null
+++ b/tests/v1/e2e/spec_decode/mtp/qwen3_5/test_mtp.py
@@ -0,0 +1,39 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import pytest
+
+from tests.utils import single_gpu_only
+from vllm import SamplingParams
+
+from .._correctness import check_mtp_correctness
+
+
+@pytest.mark.parametrize(
+ ["model_setup", "mm_enabled", "expected_accuracy_threshold"],
+ [
+ (
+ ("mtp", "Qwen/Qwen3.5-0.8B-Base", 1, None),
+ False,
+ 0.20,
+ ), # hybrid + MTP, ref: ~34%-35%
+ ],
+ ids=["qwen3_5-hybrid"],
+)
+@single_gpu_only
+def test_mtp_correctness(
+ monkeypatch: pytest.MonkeyPatch,
+ sampling_config: SamplingParams,
+ model_setup: tuple[str, str, int, str | None],
+ mm_enabled: bool,
+ expected_accuracy_threshold: float,
+ vllm_runner,
+):
+ check_mtp_correctness(
+ monkeypatch,
+ sampling_config,
+ model_setup,
+ mm_enabled,
+ expected_accuracy_threshold,
+ vllm_runner,
+ )
diff --git a/tests/v1/e2e/spec_decode/mtp/qwen3_5/test_rejected_placeholders.py b/tests/v1/e2e/spec_decode/mtp/qwen3_5/test_rejected_placeholders.py
new file mode 100644
index 0000000000..5116f7db27
--- /dev/null
+++ b/tests/v1/e2e/spec_decode/mtp/qwen3_5/test_rejected_placeholders.py
@@ -0,0 +1,115 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from typing import Any
+
+import pytest
+
+from tests.utils import single_gpu_only
+from vllm import SamplingParams
+from vllm.config import CompilationConfig, CUDAGraphMode
+from vllm.platforms import current_platform
+
+from ...utils import get_spec_decode_metric_value
+
+PLACEHOLDER_REGRESSION_MODEL = "Qwen/Qwen3.5-0.8B-Base"
+PLACEHOLDER_REGRESSION_PROMPT = "The capital of France is"
+PLACEHOLDER_REGRESSION_MAX_TOKENS = 16
+PLACEHOLDER_REGRESSION_ALLOWED_TOKEN_ID = 42
+PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS = 3
+
+
+@pytest.mark.parametrize(
+ "cudagraph_mode",
+ [CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL_AND_PIECEWISE],
+ ids=["piecewise", "full-and-piecewise"],
+)
+@single_gpu_only
+def test_mtp_rejected_placeholders_match_greedy_token_ids(
+ cudagraph_mode: CUDAGraphMode,
+ vllm_runner,
+):
+ """Rejected MTP padding must not alter the non-speculative token stream."""
+ common_kwargs: dict[str, Any] = {
+ "block_size": None,
+ "max_model_len": 256,
+ "max_num_seqs": 1,
+ "trust_remote_code": True,
+ "enable_chunked_prefill": None,
+ "limit_mm_per_prompt": {"image": 0, "video": 0},
+ "attention_backend": ("TRITON_ATTN" if current_platform.is_rocm() else "auto"),
+ }
+ sampling_params = SamplingParams(
+ temperature=0.0,
+ max_tokens=PLACEHOLDER_REGRESSION_MAX_TOKENS,
+ ignore_eos=True,
+ allowed_token_ids=[PLACEHOLDER_REGRESSION_ALLOWED_TOKEN_ID],
+ logprobs=1,
+ )
+
+ def generate_token_ids(runner) -> tuple[int, ...]:
+ outputs = runner.llm.generate([PLACEHOLDER_REGRESSION_PROMPT], sampling_params)
+ assert outputs and outputs[0].outputs, "Expected one completion"
+ token_ids = tuple(outputs[0].outputs[0].token_ids)
+ assert len(token_ids) == PLACEHOLDER_REGRESSION_MAX_TOKENS
+ return token_ids
+
+ def compilation_config() -> CompilationConfig:
+ return CompilationConfig(
+ cudagraph_mode=cudagraph_mode,
+ cudagraph_capture_sizes=[1, PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS + 1],
+ )
+
+ with vllm_runner(
+ PLACEHOLDER_REGRESSION_MODEL,
+ compilation_config=compilation_config(),
+ **common_kwargs,
+ ) as ref_runner:
+ ref_mode = (
+ ref_runner.llm.llm_engine.vllm_config.compilation_config.cudagraph_mode
+ )
+ assert ref_mode == cudagraph_mode
+ ref_token_ids = generate_token_ids(ref_runner)
+ expected_token_ids = (PLACEHOLDER_REGRESSION_ALLOWED_TOKEN_ID,) * (
+ PLACEHOLDER_REGRESSION_MAX_TOKENS
+ )
+ assert ref_token_ids == expected_token_ids
+
+ with vllm_runner(
+ PLACEHOLDER_REGRESSION_MODEL,
+ speculative_config={
+ "method": "mtp",
+ "num_speculative_tokens": PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS,
+ "max_model_len": 256,
+ "rejection_sample_method": "synthetic",
+ "synthetic_acceptance_rates": [0.0]
+ * PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS,
+ },
+ disable_log_stats=False,
+ compilation_config=compilation_config(),
+ **common_kwargs,
+ ) as spec_runner:
+ spec_mode = (
+ spec_runner.llm.llm_engine.vllm_config.compilation_config.cudagraph_mode
+ )
+ assert spec_mode == cudagraph_mode
+ spec_token_ids = generate_token_ids(spec_runner)
+ metrics = spec_runner.llm.get_metrics()
+ num_drafts = get_spec_decode_metric_value(
+ metrics, "vllm:spec_decode_num_drafts"
+ )
+ num_draft_tokens = get_spec_decode_metric_value(
+ metrics, "vllm:spec_decode_num_draft_tokens"
+ )
+ num_accepted_tokens = get_spec_decode_metric_value(
+ metrics, "vllm:spec_decode_num_accepted_tokens"
+ )
+
+ assert num_drafts > 0, "MTP drafter did not run"
+ assert num_draft_tokens > 0, "MTP drafter did not produce draft tokens"
+ assert num_accepted_tokens == 0, "Synthetic sampler accepted a draft token"
+ assert spec_token_ids == ref_token_ids, (
+ f"MTP changed greedy token ids under {cudagraph_mode.name}:\n"
+ f" reference={ref_token_ids}\n"
+ f" speculative={spec_token_ids}"
+ )
diff --git a/tests/v1/e2e/spec_decode/mtp/test_mtp.py b/tests/v1/e2e/spec_decode/mtp/test_mtp.py
deleted file mode 100644
index 92f4a2a773..0000000000
--- a/tests/v1/e2e/spec_decode/mtp/test_mtp.py
+++ /dev/null
@@ -1,252 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-from typing import Any
-
-import pytest
-
-from tests.utils import single_gpu_only
-from vllm import SamplingParams
-from vllm.config import CompilationConfig, CUDAGraphMode
-from vllm.platforms import current_platform
-
-from ..utils import (
- _skip_if_insufficient_gpus_for_tp,
- assert_request_outputs_match,
- evaluate_llm_for_gsm8k,
- get_spec_decode_metric_value,
- get_test_prompts,
-)
-
-PLACEHOLDER_REGRESSION_MODEL = "Qwen/Qwen3.5-0.8B-Base"
-PLACEHOLDER_REGRESSION_PROMPT = "The capital of France is"
-PLACEHOLDER_REGRESSION_MAX_TOKENS = 16
-PLACEHOLDER_REGRESSION_ALLOWED_TOKEN_ID = 42
-PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS = 3
-
-
-@pytest.mark.parametrize(
- "cudagraph_mode",
- [CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL_AND_PIECEWISE],
- ids=["piecewise", "full-and-piecewise"],
-)
-@single_gpu_only
-def test_mtp_rejected_placeholders_match_greedy_token_ids(
- cudagraph_mode: CUDAGraphMode,
- vllm_runner,
-):
- """Rejected MTP padding must not alter the non-speculative token stream."""
- common_kwargs: dict[str, Any] = {
- "block_size": None,
- "max_model_len": 256,
- "max_num_seqs": 1,
- "trust_remote_code": True,
- "enable_chunked_prefill": None,
- "limit_mm_per_prompt": {"image": 0, "video": 0},
- "attention_backend": ("TRITON_ATTN" if current_platform.is_rocm() else "auto"),
- }
- sampling_params = SamplingParams(
- temperature=0.0,
- max_tokens=PLACEHOLDER_REGRESSION_MAX_TOKENS,
- ignore_eos=True,
- allowed_token_ids=[PLACEHOLDER_REGRESSION_ALLOWED_TOKEN_ID],
- logprobs=1,
- )
-
- def generate_token_ids(runner) -> tuple[int, ...]:
- outputs = runner.llm.generate([PLACEHOLDER_REGRESSION_PROMPT], sampling_params)
- assert outputs and outputs[0].outputs, "Expected one completion"
- token_ids = tuple(outputs[0].outputs[0].token_ids)
- assert len(token_ids) == PLACEHOLDER_REGRESSION_MAX_TOKENS
- return token_ids
-
- def compilation_config() -> CompilationConfig:
- return CompilationConfig(
- cudagraph_mode=cudagraph_mode,
- cudagraph_capture_sizes=[1, PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS + 1],
- )
-
- with vllm_runner(
- PLACEHOLDER_REGRESSION_MODEL,
- compilation_config=compilation_config(),
- **common_kwargs,
- ) as ref_runner:
- ref_mode = (
- ref_runner.llm.llm_engine.vllm_config.compilation_config.cudagraph_mode
- )
- assert ref_mode == cudagraph_mode
- ref_token_ids = generate_token_ids(ref_runner)
- expected_token_ids = (PLACEHOLDER_REGRESSION_ALLOWED_TOKEN_ID,) * (
- PLACEHOLDER_REGRESSION_MAX_TOKENS
- )
- assert ref_token_ids == expected_token_ids
-
- with vllm_runner(
- PLACEHOLDER_REGRESSION_MODEL,
- speculative_config={
- "method": "mtp",
- "num_speculative_tokens": PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS,
- "max_model_len": 256,
- "rejection_sample_method": "synthetic",
- "synthetic_acceptance_rates": [0.0]
- * PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS,
- },
- disable_log_stats=False,
- compilation_config=compilation_config(),
- **common_kwargs,
- ) as spec_runner:
- spec_mode = (
- spec_runner.llm.llm_engine.vllm_config.compilation_config.cudagraph_mode
- )
- assert spec_mode == cudagraph_mode
- spec_token_ids = generate_token_ids(spec_runner)
- metrics = spec_runner.llm.get_metrics()
- num_drafts = get_spec_decode_metric_value(
- metrics, "vllm:spec_decode_num_drafts"
- )
- num_draft_tokens = get_spec_decode_metric_value(
- metrics, "vllm:spec_decode_num_draft_tokens"
- )
- num_accepted_tokens = get_spec_decode_metric_value(
- metrics, "vllm:spec_decode_num_accepted_tokens"
- )
-
- assert num_drafts > 0, "MTP drafter did not run"
- assert num_draft_tokens > 0, "MTP drafter did not produce draft tokens"
- assert num_accepted_tokens == 0, "Synthetic sampler accepted a draft token"
- assert spec_token_ids == ref_token_ids, (
- f"MTP changed greedy token ids under {cudagraph_mode.name}:\n"
- f" reference={ref_token_ids}\n"
- f" speculative={spec_token_ids}"
- )
-
-
-@pytest.mark.parametrize(
- ["model_setup", "mm_enabled", "expected_accuracy_threshold"],
- [
- # Reference accuracy: 65%-70%.
- (("mtp", "XiaomiMiMo/MiMo-7B-Base", 1, None), False, 0.5),
- pytest.param(
- ("mtp", "ZixiQi/DeepSeek-V3-4layers-MTP-FP8", 1, None),
- False,
- 0.0,
- marks=pytest.mark.skipif(
- current_platform.is_device_capability_family(100),
- reason="DeepSeek MTP: TRTLLM MoE top_k check fails on Blackwell",
- ),
- ), # dummy model
- (
- ("mtp", "Qwen/Qwen3.5-0.8B-Base", 1, None),
- False,
- 0.20,
- ), # hybrid + MTP, ref: ~34%-35%
- (
- ("mtp", "google/gemma-4-E4B-it", 1, "google/gemma-4-E4B-it-assistant"),
- False,
- 0.50,
- ), # gemma4 MTP with assistant model, ref: ~62%
- ],
- ids=["mimo", "deepseek", "qwen3_5-hybrid", "gemma4-e4b"],
-)
-@single_gpu_only
-def test_mtp_correctness(
- monkeypatch: pytest.MonkeyPatch,
- sampling_config: SamplingParams,
- model_setup: tuple[str, str, int, str | None],
- mm_enabled: bool,
- expected_accuracy_threshold: float,
- vllm_runner,
-):
- """Compare the outputs of a original LLM and a speculative LLM
- which should be the same when using MTP speculative decoding. Due to some variance
- in the engine, it is possible for some outputs to differ, so we expect that at least
- 6/10 output tokens match exactly, and that the GSM8k accuracy is above a precomputed
- reference threshold for each model.
- """
- method, model_name, tp_size, draft_model = model_setup
- _skip_if_insufficient_gpus_for_tp(tp_size)
-
- # Generate test prompts inside the function instead of using fixture
- test_prompts = get_test_prompts(mm_enabled)
- with monkeypatch.context() as m:
- m.setenv("VLLM_MLA_DISABLE", "1")
-
- attn_backend = "TRITON_ATTN" if current_platform.is_rocm() else "auto"
-
- # Skip multimodal profiling for models that don't need it in this test.
- extra_kwargs: dict[str, Any] = {}
- if "Qwen3.5" in model_name:
- extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "video": 0}
- elif "gemma-4" in model_name:
- extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "audio": 0}
-
- if draft_model is not None and "gemma-4" in draft_model:
- import transformers
- from packaging.version import Version
-
- if Version(transformers.__version__) < Version("5.8.0"):
- pytest.skip(
- "Gemma4 MTP assistant requires transformers>=5.8.0, "
- f"got {transformers.__version__}"
- )
-
- with vllm_runner(
- model_name,
- block_size=None,
- max_model_len=2048,
- tensor_parallel_size=tp_size,
- trust_remote_code=True,
- attention_backend=attn_backend,
- enable_chunked_prefill=None,
- compilation_config=CompilationConfig(),
- **extra_kwargs,
- ) as ref_runner:
- ref_outputs = ref_runner.llm.chat(test_prompts, sampling_config)
- evaluate_llm_for_gsm8k(
- ref_runner.llm,
- expected_accuracy_threshold=expected_accuracy_threshold,
- )
-
- speculative_config: dict[str, Any] = {
- "method": method,
- "num_speculative_tokens": 1,
- "max_model_len": 2048,
- }
- if draft_model is not None:
- speculative_config["model"] = draft_model
- speculative_config["num_speculative_tokens"] = 2
-
- with vllm_runner(
- model_name,
- block_size=None,
- trust_remote_code=True,
- tensor_parallel_size=tp_size,
- speculative_config=speculative_config,
- max_model_len=2048,
- attention_backend=attn_backend,
- enable_chunked_prefill=None,
- compilation_config=CompilationConfig(),
- **extra_kwargs,
- ) as spec_runner:
- # MTP supports async scheduling by default.
- has_async = (
- spec_runner.llm.llm_engine.vllm_config.scheduler_config.async_scheduling
- )
- assert has_async, (
- f"Expected async scheduling for {method}: target={model_name}, "
- f"draft={draft_model}; got {has_async}"
- )
- evaluate_llm_for_gsm8k(
- spec_runner.llm,
- expected_accuracy_threshold=expected_accuracy_threshold,
- )
- spec_outputs = spec_runner.llm.chat(test_prompts, sampling_config)
-
- # Heuristic: expect at least 80% of the prompts to match exactly
- # Upon failure, inspect the outputs to check for inaccuracy.
- assert_request_outputs_match(
- ref_outputs,
- spec_outputs,
- required_matches=int(0.8 * len(ref_outputs)) + 1,
- context=f"{method} target={model_name}, draft={draft_model}",
- )
diff --git a/tests/v1/engine/test_engine_core.py b/tests/v1/engine/test_engine_core.py
index e16e0550c2..2d9f7e8632 100644
--- a/tests/v1/engine/test_engine_core.py
+++ b/tests/v1/engine/test_engine_core.py
@@ -4,6 +4,7 @@
import copy
import time
import uuid
+from collections import deque
from concurrent.futures import Future, ThreadPoolExecutor
from unittest.mock import MagicMock, PropertyMock, patch
@@ -395,6 +396,38 @@ def test_engine_core_concurrent_batches():
req_id = (req_id + 1) % 2
+@pytest.mark.parametrize("encoder_only", [True, False])
+def test_async_step_skips_sampling_for_mm_encoder_only(encoder_only):
+ """Encoder outputs are complete; only language-model steps need sampling."""
+ core = MagicMock()
+ core.batch_queue = deque()
+ core.batch_queue_size = 2
+ core.is_mm_encoder_only = encoder_only
+ core.is_pooling_model = False
+ core.scheduler.has_requests.side_effect = [True, False, False]
+ scheduled = core.scheduler.schedule.return_value
+ scheduled.total_num_scheduled_tokens = 16
+ scheduled.pending_structured_output_tokens = False
+
+ output = MagicMock()
+ executed: Future[ModelRunnerOutput | None] = Future()
+ executed.set_result(output if encoder_only else None)
+ sampled: Future[ModelRunnerOutput | None] = Future()
+ sampled.set_result(None if encoder_only else output)
+ core.model_executor.execute_model.return_value = executed
+ core.model_executor.sample_tokens.return_value = sampled
+
+ EngineCore.step_with_batch_queue(core)
+ if core.batch_queue:
+ EngineCore.step_with_batch_queue(core)
+
+ core.scheduler.update_from_output.assert_called_once_with(scheduled, output)
+ if encoder_only:
+ core.model_executor.sample_tokens.assert_not_called()
+ else:
+ core.model_executor.sample_tokens.assert_called_once()
+
+
@multi_gpu_test(num_gpus=2)
def test_engine_core_tp():
"""Test engine can initialize worker in tp properly."""
diff --git a/tests/v1/kv_connector/unit/test_decode_bench_connector.py b/tests/v1/kv_connector/unit/test_decode_bench_connector.py
index 693c10d40a..6f518ab53a 100644
--- a/tests/v1/kv_connector/unit/test_decode_bench_connector.py
+++ b/tests/v1/kv_connector/unit/test_decode_bench_connector.py
@@ -500,7 +500,7 @@ def test_decode_bench_connector_skips_null_padding():
def test_decode_bench_connector_uses_per_group_dcp_block_sizes():
- """DCP scales full-attention blocks but not sliding-window blocks."""
+ """DCP scales target blocks but not the replicated draft's window."""
block_size = 16
dcp_world_size = 2
vllm_config = create_vllm_config(
@@ -515,6 +515,7 @@ def test_decode_bench_connector_uses_per_group_dcp_block_sizes():
dtype=torch.float32,
)
sliding_window = SlidingWindowSpec(
+ dcp_sharded=False,
block_size=block_size,
num_kv_heads=1,
head_size=1,
diff --git a/tests/v1/kv_connector/unit/test_handshake_pp_aggregation.py b/tests/v1/kv_connector/unit/test_handshake_pp_aggregation.py
index 509d3d3234..045f6cb425 100644
--- a/tests/v1/kv_connector/unit/test_handshake_pp_aggregation.py
+++ b/tests/v1/kv_connector/unit/test_handshake_pp_aggregation.py
@@ -99,6 +99,7 @@ def _run_engine_core_handshake(
),
speculative_config=None,
ec_transfer_config=None,
+ is_mm_encoder_only=False,
max_concurrent_batches=1,
model_config=SimpleNamespace(
runner_type="generate",
diff --git a/tests/v1/kv_connector/unit/test_mooncake_connector.py b/tests/v1/kv_connector/unit/test_mooncake_connector.py
index db9cf94171..5a5fffac3d 100644
--- a/tests/v1/kv_connector/unit/test_mooncake_connector.py
+++ b/tests/v1/kv_connector/unit/test_mooncake_connector.py
@@ -3,10 +3,12 @@
import asyncio
import contextlib
+import threading
import time
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
+import httpx
import pytest
import torch
import zmq.asyncio
@@ -14,6 +16,7 @@ import zmq.asyncio
from tests.v1.attention.utils import dense_kv_cache_views
from vllm import envs
from vllm.config import set_current_vllm_config
+from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import mooncake_connector
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector import (
KVConnectorRole,
MooncakeConnector,
@@ -33,6 +36,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector im
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_utils import (
MooncakeBootstrapServer,
+ RegisterWorkerPayload,
)
from vllm.utils.network_utils import get_open_port
from vllm.v1.kv_cache_interface import (
@@ -538,6 +542,46 @@ def bootstrap_server():
server.shutdown()
+@pytest.mark.asyncio
+async def test_register_worker_recovers_from_slow_bootstrap_server(monkeypatch):
+ """End-to-end: a real HTTP server that responds slower than the configured
+ timeout must be retried on the wire, not treated as fatal."""
+ original = MooncakeBootstrapServer.register_worker
+ state = {"done": False, "calls": 0}
+
+ async def slow_register(self, payload: RegisterWorkerPayload):
+ state["calls"] += 1
+ if not state["done"]:
+ state["done"] = True
+ # Record the worker first, then stall past the client timeout, so
+ # the retry exercises the duplicate-registration path.
+ response = await original(self, payload)
+ await asyncio.sleep(1.5)
+ return response
+ return await original(self, payload)
+
+ monkeypatch.setattr(MooncakeBootstrapServer, "register_worker", slow_register)
+
+ port = get_open_port()
+ monkeypatch.setenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", str(port))
+ monkeypatch.setenv("VLLM_MOONCAKE_CONNECTOR_TIMEOUT", "0.5")
+ monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 5)
+
+ server = MooncakeBootstrapServer("127.0.0.1", port)
+ server.start()
+ try:
+ worker = _make_local_register_worker_stub()
+ await MooncakeConnectorWorker.register_worker_with_bootstrap(worker)
+ assert state["calls"] == 2
+
+ async with httpx.AsyncClient() as client:
+ response = await client.get(f"http://127.0.0.1:{port}/query")
+ assert response.status_code == 200
+ assert response.json()["0"]["engine_id"] == "eng-1"
+ finally:
+ server.shutdown()
+
+
@pytest.mark.asyncio
async def test_bootstrap_server(bootstrap_server: MooncakeBootstrapServer):
"""Tests the bootstrap server's api for worker registration and querying.
@@ -589,9 +633,16 @@ async def test_bootstrap_server(bootstrap_server: MooncakeBootstrapServer):
assert data["0"]["worker_addr"]["0"]["0"] == "tcp://1.1.1.1:1111"
assert data["0"]["worker_addr"]["0"]["1"] == "tcp://2.2.2.2:2222"
- # Test failure: re-registering the same worker
+ # Re-registering the identical payload is idempotent.
async with httpx.AsyncClient() as client:
response = await client.post(f"{base_url}/register", json=payload1)
+ assert response.status_code == 200
+ assert response.json() == {"status": "ok"}
+
+ # Test failure: same ranks, conflicting address
+ payload1_conflict = dict(payload1, addr="tcp://9.9.9.9:9999")
+ async with httpx.AsyncClient() as client:
+ response = await client.post(f"{base_url}/register", json=payload1_conflict)
assert response.status_code == 400
assert "is already registered" in response.text
@@ -628,6 +679,137 @@ def _make_bootstrap_vllm_config(
)
+def _make_local_register_worker_stub():
+ return SimpleNamespace(
+ vllm_config=SimpleNamespace(
+ parallel_config=SimpleNamespace(
+ local_engines_only=False,
+ data_parallel_rank_local=0,
+ data_parallel_index=0,
+ nnodes_within_dp=1,
+ master_addr="127.0.0.1",
+ data_parallel_master_ip="127.0.0.1",
+ )
+ ),
+ hostname="127.0.0.1",
+ side_channel_port=1234,
+ engine_id="eng-1",
+ dp_rank=0,
+ tp_rank=0,
+ pp_rank=0,
+ )
+
+
+def _make_register_worker_stub():
+ return SimpleNamespace(
+ vllm_config=_make_bootstrap_vllm_config(),
+ hostname="127.0.0.1",
+ side_channel_port=1234,
+ engine_id="eng-1",
+ dp_rank=0,
+ tp_rank=0,
+ pp_rank=0,
+ )
+
+
+class _FlakyAsyncClient:
+ """httpx.AsyncClient stub that read-times-out `failures` times first."""
+
+ def __init__(self, failures: int, calls: list[int], **kwargs):
+ self.failures = failures
+ self.calls = calls
+
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, *exc_info):
+ return False
+
+ async def post(self, url, json=None):
+ self.calls.append(1)
+ if len(self.calls) <= self.failures:
+ raise httpx.ReadTimeout("simulated slow bootstrap server")
+ response = MagicMock()
+ response.raise_for_status = MagicMock()
+ return response
+
+
+@pytest.mark.asyncio
+async def test_register_worker_retries_on_read_timeout(monkeypatch):
+ """A slow rank-0 bootstrap response must be retried, not treated as fatal."""
+ monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 5)
+ monkeypatch.setattr(asyncio, "sleep", AsyncMock())
+
+ calls: list[int] = []
+ monkeypatch.setattr(
+ httpx,
+ "AsyncClient",
+ lambda **kwargs: _FlakyAsyncClient(2, calls),
+ )
+
+ worker = _make_register_worker_stub()
+ await MooncakeConnectorWorker.register_worker_with_bootstrap(worker)
+
+ assert len(calls) == 3
+
+
+@pytest.mark.asyncio
+async def test_register_worker_raises_after_max_attempts(monkeypatch):
+ """Terminal registration failure must raise, not loop forever."""
+ monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 3)
+ monkeypatch.setattr(asyncio, "sleep", AsyncMock())
+
+ calls: list[int] = []
+ monkeypatch.setattr(
+ httpx,
+ "AsyncClient",
+ lambda **kwargs: _FlakyAsyncClient(99, calls),
+ )
+
+ worker = _make_register_worker_stub()
+ with pytest.raises(RuntimeError, match="after 3 attempts"):
+ await MooncakeConnectorWorker.register_worker_with_bootstrap(worker)
+
+ assert len(calls) == 3
+
+
+@pytest.mark.asyncio
+async def test_sender_listener_failure_propagates_to_caller(monkeypatch):
+ """A terminal registration failure must surface via the Future rather than
+ leaving register_kv_caches blocked on its ready event."""
+ monkeypatch.setenv("VLLM_MOONCAKE_CONNECTOR_TIMEOUT", "0.1")
+ monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 1)
+
+ loop = asyncio.new_event_loop()
+ thread = threading.Thread(target=loop.run_forever, daemon=True)
+ thread.start()
+
+ async def failing_listener(ready_event):
+ raise RuntimeError("simulated terminal registration failure")
+
+ try:
+ ready_event = threading.Event()
+ fut = asyncio.run_coroutine_threadsafe(failing_listener(ready_event), loop)
+
+ deadline = time.monotonic() + 10.0
+ raised = None
+ while not ready_event.wait(timeout=0.1):
+ if fut.done():
+ try:
+ fut.result()
+ except RuntimeError as e:
+ raised = e
+ break
+ assert time.monotonic() < deadline, "caller blocked past its deadline"
+
+ assert raised is not None
+ assert "simulated terminal registration failure" in str(raised)
+ finally:
+ loop.call_soon_threadsafe(loop.stop)
+ thread.join(timeout=5)
+ loop.close()
+
+
@pytest.mark.parametrize(
(
"tp_rank",
diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py
index 947e9f6a20..f08213444e 100644
--- a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py
+++ b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py
@@ -290,6 +290,8 @@ class _FakeKVTransferConfig:
class _FakeModelConfig:
model = "test-model"
use_mla = False
+ enable_sleep_mode = False
+ enable_cumem_allocator = False
def get_num_layers(self, parallel_config) -> int:
return 1
@@ -432,6 +434,64 @@ def _patch_worker_runtime(
monkeypatch.setattr(worker, "LookupKeyServer", MagicMock())
+def test_create_mem_pool_returns_none_when_unconfigured():
+ assert worker.MooncakeStoreWorker._create_mem_pool({}, _FakeModelConfig()) is None
+
+
+def test_create_mem_pool_rejects_unknown_pool():
+ with pytest.raises(
+ ValueError,
+ match="Unsupported custom_mem_pool='UNKNOWN'",
+ ):
+ worker.MooncakeStoreWorker._create_mem_pool(
+ {"custom_mem_pool": "unknown"}, _FakeModelConfig()
+ )
+
+
+@pytest.mark.parametrize(
+ "conflicting_option", ["enable_sleep_mode", "enable_cumem_allocator"]
+)
+def test_create_mem_pool_rejects_cumem_conflicts(conflicting_option):
+ model_config = _FakeModelConfig()
+ setattr(model_config, conflicting_option, True)
+
+ with pytest.raises(ValueError, match="custom_mem_pool is incompatible"):
+ worker.MooncakeStoreWorker._create_mem_pool(
+ {"custom_mem_pool": "NVLINK"}, model_config
+ )
+
+
+@pytest.mark.parametrize(
+ ("pool_type", "allocator_name"),
+ [("nvlink", "NVLinkAllocator"), ("barex", "BarexAllocator")],
+)
+def test_create_mem_pool_uses_mooncake_allocator(
+ monkeypatch, pool_type, allocator_name
+):
+ allocator_cls = MagicMock()
+ allocator = allocator_cls.get_allocator.return_value
+ allocator_module = types.ModuleType("mooncake.allocator")
+ setattr(allocator_module, allocator_name, allocator_cls)
+ mooncake_module = types.ModuleType("mooncake")
+ mooncake_module.allocator = allocator_module # type: ignore[attr-defined]
+ monkeypatch.setitem(sys.modules, "mooncake", mooncake_module)
+ monkeypatch.setitem(sys.modules, "mooncake.allocator", allocator_module)
+
+ monkeypatch.setattr(torch.accelerator, "current_device_index", lambda: 2)
+ mem_pool = MagicMock()
+ mem_pool_ctor = MagicMock(return_value=mem_pool)
+ monkeypatch.setattr(torch.cuda, "MemPool", mem_pool_ctor)
+
+ result = worker.MooncakeStoreWorker._create_mem_pool(
+ {"custom_mem_pool": pool_type}, _FakeModelConfig()
+ )
+
+ assert result is mem_pool
+ allocator_cls.get_allocator.assert_called_once_with(torch.device("cuda", 2))
+ allocator.allocator.assert_called_once_with()
+ mem_pool_ctor.assert_called_once_with(allocator.allocator.return_value)
+
+
def test_pool_key_to_string_without_prefix_is_unchanged():
"""Default (empty) cache_prefix keeps keys byte-identical to the
historical unprefixed format so existing deployments keep their hits."""
diff --git a/tests/v1/kv_connector/unit/test_multi_connector.py b/tests/v1/kv_connector/unit/test_multi_connector.py
index 1a85865560..9d04a2ea53 100644
--- a/tests/v1/kv_connector/unit/test_multi_connector.py
+++ b/tests/v1/kv_connector/unit/test_multi_connector.py
@@ -68,6 +68,7 @@ class MockConnector(KVConnectorBase_V1):
# Override just build_kv_connector_stats
mock.build_kv_connector_stats = cls.build_kv_connector_stats
mock.get_kv_connector_stats.return_value = None
+ mock.get_mem_pool_context.return_value = None
return mock
@classmethod
@@ -201,6 +202,36 @@ def mc() -> MultiConnector:
return mc
+def test_multi_connector_mem_pool_context_none(mc: MultiConnector):
+ assert mc.get_mem_pool_context() is None
+
+
+def test_multi_connector_forwards_mem_pool_context(mc: MultiConnector):
+ context = MagicMock()
+ provider = MagicMock(spec_set=KVConnectorBase_V1)
+ provider.get_mem_pool_context.return_value = context
+ mc._connectors = [mc._connectors[0], provider]
+
+ assert mc.get_mem_pool_context() is context
+ provider.get_mem_pool_context.assert_called_once_with()
+
+
+def test_multi_connector_rejects_multiple_mem_pool_contexts(mc: MultiConnector):
+ providers = [
+ MagicMock(spec_set=KVConnectorBase_V1),
+ MagicMock(spec_set=KVConnectorBase_V1),
+ ]
+ for provider in providers:
+ provider.get_mem_pool_context.return_value = MagicMock()
+ mc._connectors = providers
+
+ with pytest.raises(
+ ValueError,
+ match="Multiple connectors provide a KV cache memory pool",
+ ):
+ mc.get_mem_pool_context()
+
+
# Helper function to compare directories recursively
def _compare_directories(dir1: Path, dir2: Path) -> bool:
"""Compares two directories recursively for identical content."""
@@ -320,6 +351,7 @@ def test_multi_example_connector_consistency():
# First three events are from initialization. Layer hooks run before the
# deferred load starts after the forward pass.
expected_worker_prefix = [
+ "get_mem_pool_context",
"register_kv_caches",
"set_host_xfer_buffer_ops",
"get_handshake_metadata",
diff --git a/tests/v1/kv_connector/unit/test_nixl_connector.py b/tests/v1/kv_connector/unit/test_nixl_connector.py
index 39f8f88d5c..d8b6fcb5a0 100644
--- a/tests/v1/kv_connector/unit/test_nixl_connector.py
+++ b/tests/v1/kv_connector/unit/test_nixl_connector.py
@@ -7,6 +7,7 @@ import os
import queue
import tempfile
import textwrap
+import threading
import time
import uuid
from collections import defaultdict
@@ -49,6 +50,8 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl import (
NixlKVConnectorStats,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
+ RemoteMeta,
+ ReqMeta,
compute_nixl_compatibility_hash,
)
from vllm.distributed.kv_transfer.kv_transfer_state import (
@@ -2087,6 +2090,10 @@ def recv_worker():
worker._recving_transfers = defaultdict(list)
worker._failed_recv_reqs = queue.Queue()
worker._recv_failures = set()
+ worker._handshake_lock = threading.RLock()
+ worker._handshake_futures = {}
+ worker._remote_agents = {}
+ worker._engine_by_address = {}
worker._replicated_pcp_done_sending = set()
worker._invalid_block_ids = queue.Queue()
worker._pending_recv_notifs = {}
@@ -2681,6 +2688,174 @@ def test_engine_ttl_disabled(default_vllm_config, dist_init):
assert engine_id in worker.dst_xfer_side_handles
+@pytest.mark.cpu_test
+class TestPeerReplacement:
+ @pytest.fixture(autouse=True)
+ def setup(self, monkeypatch):
+ from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker as bw
+ from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
+
+ config = create_vllm_config(kv_connector_extra_config={"engine_ttl": 0})
+ config.kv_transfer_config.kv_buffer_device = "cpu"
+ platform = SimpleNamespace(
+ device_type="cpu",
+ discover_numa_topology=lambda: [],
+ get_nixl_memory_type=lambda: "DRAM",
+ is_rocm=lambda: False,
+ )
+ with (
+ patch.object(bw, "NixlWrapper", FakeNixlWrapper),
+ patch.object(bw, "current_platform", platform),
+ patch.object(bw, "get_tensor_model_parallel_rank", return_value=0),
+ patch.object(bw, "get_tensor_model_parallel_world_size", return_value=1),
+ patch.object(
+ bw, "get_current_attn_backends", return_value=[FlashAttentionBackend]
+ ),
+ ):
+ self.worker = FakeNixlConnectorWorker(config, "local", hand_shake_latency=0)
+ self.nixl = self.worker.nixl_wrapper
+ self.transport = MagicMock(wraps=self.nixl)
+ self.worker.nixl_wrapper = self.transport
+ monkeypatch.setattr(
+ self.worker._handshake_initiation_executor,
+ "submit",
+ lambda *args: Future(),
+ )
+ self._connect("old")
+ yield
+ self.worker.shutdown()
+
+ def _connect(self, engine_id="new", host="localhost", port=1234):
+ future = self.worker._ensure_handshake(engine_id, host, port, 1)
+ self.worker.REMOTE_ENGINE_ID = engine_id
+ self.nixl.REMOTE_AGENT_NAME = engine_id
+ future.set_result(self.worker._nixl_handshake(host, port, 1, engine_id))
+
+ def _request(self, engine_id="new", blocks=(1,), awaiting_kvs=True):
+ metadata = NixlConnectorMetadata()
+ metadata.reqs_to_recv[f"{engine_id}-req"] = ReqMeta(
+ local_block_ids=(blocks,),
+ local_physical_block_ids=(blocks,),
+ tp_size=1,
+ remote=RemoteMeta(
+ ([2],), "localhost", 1234, engine_id, f"{engine_id}-prefill"
+ ),
+ awaiting_kvs=awaiting_kvs,
+ )
+ return metadata
+
+ def test_cleans_old_peer_after_confirmed_handshake(self):
+ self._connect("healthy", host="other-host")
+ old_handle = self.worker.dst_xfer_side_handles["old"][0]
+ self.worker.start_load_kv(self._request())
+ self.worker.get_transfer_results()
+ self.transport.remove_remote_agent.assert_not_called()
+
+ self._connect()
+ self.worker.get_transfer_results()
+ self.transport.release_dlist_handle.assert_called_once_with(old_handle)
+ self.transport.remove_remote_agent.assert_called_once_with("old")
+ assert set(self.worker._remote_agents) == {"new", "healthy"}
+ assert self.worker._engine_by_address == {
+ ("localhost", 1234): "new",
+ ("other-host", 1234): "healthy",
+ }
+ with pytest.raises(KeyError):
+ self.worker.transfer_topo.get_engine_info("old")
+
+ self.worker.start_load_kv(NixlConnectorMetadata())
+ assert self.worker.get_transfer_results().finished_recving == {"new-req"}
+ self.transport.remove_remote_agent.assert_called_once_with("old")
+
+ @pytest.mark.parametrize("host,port", [("other-host", 1234), ("localhost", 5678)])
+ def test_preserves_other_addresses(self, host, port):
+ self._connect(host=host, port=port)
+ self.worker.get_transfer_results()
+ assert "old" in self.worker._remote_agents
+ self.transport.remove_remote_agent.assert_not_called()
+
+ def test_preserves_old_peer_when_handshake_fails(self):
+ self.worker.start_load_kv(self._request())
+ self.worker._handshake_futures["new"].set_exception(
+ RuntimeError("handshake failed")
+ )
+ assert self.worker.get_transfer_results().failed_recving == {"new-req"}
+ assert "old" in self.worker._remote_agents
+ assert self.worker._engine_by_address == {("localhost", 1234): "old"}
+ self.transport.remove_remote_agent.assert_not_called()
+
+ @pytest.mark.parametrize(
+ "states,failed",
+ [
+ (["PROC", "DONE"], False),
+ (["ERR", "PROC", "DONE"], True),
+ (["ERR", "PROC", "ERR"], True),
+ ],
+ ids=["healthy", "failed-sibling-completes", "failed-sibling-fails"],
+ )
+ def test_waits_for_reads_before_cleanup_and_failure_reporting(self, states, failed):
+ self.worker._recving_metadata.update(self._request("old").reqs_to_recv)
+ self.worker._recving_transfers["old-req"] = [1, 2] if failed else [1]
+ self.transport.check_xfer_state.side_effect = states
+ self._connect()
+
+ result = self.worker.get_transfer_results()
+ assert result.finished_recving == result.failed_recving == set()
+ assert self.worker.get_block_ids_with_load_errors() == set()
+ self.transport.remove_remote_agent.assert_not_called()
+
+ result = self.worker.get_transfer_results()
+ assert result.finished_recving == {"old-req"}
+ assert result.failed_recving == ({"old-req"} if failed else set())
+ assert self.worker.get_block_ids_with_load_errors() == (
+ {1} if failed else set()
+ )
+ assert self.transport.release_xfer_handle.call_count == (2 if failed else 1)
+ self.transport.remove_remote_agent.assert_called_once_with("old")
+
+ def test_waits_for_queued_notification(self):
+ metadata = self._request("old", blocks=())
+ self.worker._recving_metadata.update(metadata.reqs_to_recv)
+ self.worker._background_nixl_handshake(
+ "old-req", "old", metadata.reqs_to_recv["old-req"]
+ )
+ self._connect()
+ self.worker.get_transfer_results()
+ self.transport.remove_remote_agent.assert_not_called()
+
+ self.worker.start_load_kv(NixlConnectorMetadata())
+ self.transport.send_notif.assert_called_once_with(
+ "old", notif_msg=b"old-prefill:1"
+ )
+ result = self.worker.get_transfer_results()
+ assert result.finished_recving == {"old-req"}
+ assert result.failed_recving == set()
+ self.transport.remove_remote_agent.assert_called_once_with("old")
+
+ def test_waits_for_other_handshakes(self):
+ self.worker._ensure_handshake("other", "other-host", 1234, 1)
+ self._connect()
+ self.worker.get_transfer_results()
+ self.transport.remove_remote_agent.assert_not_called()
+
+ self._connect("other", host="other-host")
+ self.worker.get_transfer_results()
+ self.transport.remove_remote_agent.assert_called_once_with("old")
+
+ def test_rehandshakes_queued_request_for_released_engine(self):
+ # A request queued after its engine's handshake, but drained only after
+ # that engine was replaced and released, must not read from it.
+ metadata = self._request("old", blocks=(), awaiting_kvs=False)
+ self.worker._ready_requests.put(("old-req", metadata.reqs_to_recv["old-req"]))
+ self._connect()
+ self.worker.get_transfer_results()
+ self.transport.remove_remote_agent.assert_called_once_with("old")
+
+ self.worker.start_load_kv(NixlConnectorMetadata())
+ assert "old" in self.worker._handshake_futures
+ self.transport.send_notif.assert_not_called()
+
+
def test_transfer_topology_unregister():
"""TransferTopology.unregister_remote_engine removes the engine."""
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
diff --git a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py
index 3be1730ec3..125dfa6143 100644
--- a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py
+++ b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py
@@ -4,6 +4,7 @@
import gc
import queue
+import threading
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
@@ -54,6 +55,9 @@ def region_pull_worker():
worker.enable_permute_local_kv = False
worker.enable_heterogeneous_attn_post_process = False
worker._engine_last_active = {}
+ worker._handshake_lock = threading.RLock()
+ worker._handshake_futures = {}
+ worker._transfer_layer_group_ids = ()
worker._bidirectional_kv_xfer_enabled = False
worker._recving_transfers = {}
worker.use_mla, worker._has_mamba = True, False
@@ -94,6 +98,7 @@ def region_pull_worker():
worker.dst_num_blocks = {"P": 100, "D": 100}
worker.dst_region_num_blocks = {"P": [100, 100], "D": [100, 100]}
worker._remote_agents = {"P": {(0, 0): "P-rank0"}}
+ worker._engine_by_address = {("localhost", 1): "P"}
worker._read_blocks_mixed = MagicMock()
worker.nixl_wrapper = MagicMock()
return worker
diff --git a/tests/v1/kv_connector/unit/test_nixl_push_connector.py b/tests/v1/kv_connector/unit/test_nixl_push_connector.py
index 231343de37..4d8ead28f3 100644
--- a/tests/v1/kv_connector/unit/test_nixl_push_connector.py
+++ b/tests/v1/kv_connector/unit/test_nixl_push_connector.py
@@ -395,6 +395,7 @@ class _StubWriterWorker(NixlPushConnectorWorker):
w._group_spec_types = (FullAttentionSpec,)
w._engine_ttl = 0.0
w._engine_last_active = {}
+ w._engine_by_address = {}
# Track _do_start_push_kv invocations.
calls: list[tuple[str, Any, dict[str, Any]]] = []
diff --git a/tests/v1/kv_connector/unit/test_simple_cpu_offload_metrics.py b/tests/v1/kv_connector/unit/test_simple_cpu_offload_metrics.py
new file mode 100644
index 0000000000..16358dec7e
--- /dev/null
+++ b/tests/v1/kv_connector/unit/test_simple_cpu_offload_metrics.py
@@ -0,0 +1,466 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Unit tests for SimpleCPUOffloadConnector metrics."""
+
+from __future__ import annotations
+
+import dataclasses
+from types import SimpleNamespace
+from typing import Any
+
+import pytest
+from prometheus_client import Counter, Gauge, Histogram
+
+from tests.v1.kv_connector.unit.test_simple_cpu_offload_connector import (
+ _make_connector,
+)
+from tests.v1.simple_kv_offload.test_scheduler import (
+ make_request,
+ make_scheduler,
+ make_scheduler_output,
+)
+from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector import (
+ SimpleCPUOffloadConnector,
+)
+from vllm.v1.outputs import KVConnectorOutput
+from vllm.v1.simple_kv_offload.manager import (
+ BoundaryStoreStats,
+ LoadRequestState,
+ TransferMeta,
+)
+from vllm.v1.simple_kv_offload.metrics import (
+ INFO_LABELS,
+ OUTCOME_TO_FIELD,
+ MetricName,
+ SimpleCPUOffloadStats,
+ _MetricType,
+ _StatsKey,
+)
+
+SAVE_OUTCOMES = MetricName.SAVE_OUTCOMES
+LOAD_BLOCKS = MetricName.LOAD_BLOCKS
+USED_BLOCKS = MetricName.USED_BLOCKS
+PENDING_STORE_BLOCKS = MetricName.PENDING_STORE_BLOCKS
+INFO = MetricName.INFO
+
+
+class _FakeMetric:
+ def __init__(self, **kwargs: Any):
+ self.kwargs = kwargs
+ self.children: list[_FakeMetric] = []
+ self.observed: list[int | float] = []
+ self.increments: list[int | float] = []
+ self.set_values: list[int | float] = []
+ self.labelvalues: tuple[object, ...] = ()
+
+ def labels(self, *labelvalues: object) -> _FakeMetric:
+ child = _FakeMetric(**self.kwargs)
+ child.labelvalues = labelvalues
+ self.children.append(child)
+ return child
+
+ def observe(self, value: int | float) -> None:
+ self.observed.append(value)
+
+ def inc(self, value: int | float) -> None:
+ self.increments.append(value)
+
+ def set(self, value: int | float) -> None:
+ self.set_values.append(value)
+
+
+def _fake_metric_types() -> dict[type, type]:
+ return {Gauge: _FakeMetric, Counter: _FakeMetric, Histogram: _FakeMetric}
+
+
+def _make_prom_metrics(per_engine: dict[int, list[object]] | None = None) -> Any:
+ from vllm.v1.simple_kv_offload.metrics import SimpleCPUOffloadPromMetrics
+
+ return SimpleCPUOffloadPromMetrics(
+ vllm_config=SimpleNamespace(kv_transfer_config=None),
+ metric_types=_fake_metric_types(),
+ labelnames=["model_name", "engine"],
+ per_engine_labelvalues=per_engine or {0: ["model", "0"]},
+ )
+
+
+def _stats_payload(types: dict, data: dict) -> dict[str, Any]:
+ return {_StatsKey.TYPES: types, _StatsKey.DATA: data}
+
+
+def _values(stats: SimpleCPUOffloadStats, name: str) -> dict:
+ return stats.data[_StatsKey.DATA][name]
+
+
+# ---------------------------------------------------------------------------
+# Stats class
+# ---------------------------------------------------------------------------
+
+
+def test_build_kv_connector_stats_none_and_empty() -> None:
+ stats = SimpleCPUOffloadConnector.build_kv_connector_stats(data=None)
+ assert isinstance(stats, SimpleCPUOffloadStats)
+ assert stats.is_empty()
+
+ stats = SimpleCPUOffloadConnector.build_kv_connector_stats(data={})
+ assert isinstance(stats, SimpleCPUOffloadStats)
+ assert stats.is_empty()
+
+
+def test_stats_round_trip_through_serialized_dict() -> None:
+ stats = SimpleCPUOffloadStats()
+ stats.increase_counter(SAVE_OUTCOMES, 3, ("stored",))
+ stats.increase_counter(LOAD_BLOCKS, 4)
+ stats.set_gauge(USED_BLOCKS, 7)
+ stats.set_gauge(INFO, 1, ("cpu", "false", "false", "64"))
+
+ rebuilt = SimpleCPUOffloadConnector.build_kv_connector_stats(data=stats.to_dict())
+ assert isinstance(rebuilt, SimpleCPUOffloadStats)
+ assert _values(rebuilt, SAVE_OUTCOMES) == {("stored",): 3}
+ assert _values(rebuilt, LOAD_BLOCKS) == {(): 4}
+ assert _values(rebuilt, USED_BLOCKS) == {(): 7}
+ assert _values(rebuilt, INFO) == {("cpu", "false", "false", "64"): 1}
+
+
+def test_stats_aggregate_sums_counters_and_keeps_latest_gauge() -> None:
+ stats1 = SimpleCPUOffloadStats()
+ stats1.increase_counter(SAVE_OUTCOMES, 3, ("stored",))
+ stats1.increase_counter(SAVE_OUTCOMES, 1, ("dropped_cpu_full",))
+ stats1.set_gauge(USED_BLOCKS, 7)
+
+ stats2 = SimpleCPUOffloadStats()
+ stats2.increase_counter(SAVE_OUTCOMES, 4, ("stored",))
+ stats2.set_gauge(USED_BLOCKS, 2)
+
+ result = stats1.aggregate(stats2)
+ assert result is stats1
+ outcomes = _values(stats1, SAVE_OUTCOMES)
+ assert outcomes == {("stored",): 7, ("dropped_cpu_full",): 1}
+ assert _values(stats1, USED_BLOCKS) == {(): 2}
+
+
+def test_stats_aggregate_merges_types_from_other() -> None:
+ stats1 = SimpleCPUOffloadStats()
+ stats1.set_gauge(USED_BLOCKS, 1)
+
+ stats2 = SimpleCPUOffloadStats()
+ stats2.increase_counter(LOAD_BLOCKS, 2)
+
+ stats1.aggregate(stats2)
+ assert stats1.data[_StatsKey.TYPES][LOAD_BLOCKS] == _MetricType.COUNTER
+ assert _values(stats1, LOAD_BLOCKS) == {(): 2}
+
+
+def test_stats_reduce_and_reset() -> None:
+ stats = SimpleCPUOffloadStats()
+ stats.increase_counter(SAVE_OUTCOMES, 3, ("stored",))
+ stats.set_gauge(PENDING_STORE_BLOCKS, 4)
+ stats.increase_counter(LOAD_BLOCKS, 9)
+ stats.set_gauge(INFO, 1, ("cpu", "false", "false", "64"))
+
+ reduced = stats.reduce()
+ assert reduced[f"{SAVE_OUTCOMES}:{('stored',)}"] == 3
+ assert reduced[PENDING_STORE_BLOCKS] == 4
+ assert reduced[LOAD_BLOCKS] == 9
+ assert not any(key.startswith(INFO) for key in reduced)
+
+ stats.reset()
+ assert stats.is_empty()
+
+
+def test_outcome_labels_match_boundary_store_stats_fields() -> None:
+ """Every outcome label maps 1:1 to a BoundaryStoreStats counter field."""
+ boundary_fields = {
+ f.name for f in dataclasses.fields(BoundaryStoreStats) if f.name != "published"
+ }
+ assert set(OUTCOME_TO_FIELD.values()) == boundary_fields
+ assert len(set(OUTCOME_TO_FIELD.values())) == len(OUTCOME_TO_FIELD)
+
+
+# ---------------------------------------------------------------------------
+# Prom metrics class
+# ---------------------------------------------------------------------------
+
+
+def test_prom_metrics_registers_tier1_metrics() -> None:
+ prom = _make_prom_metrics()
+ defs = prom._defs
+ assert set(defs) == {
+ SAVE_OUTCOMES,
+ LOAD_BLOCKS,
+ USED_BLOCKS,
+ PENDING_STORE_BLOCKS,
+ INFO,
+ }
+ # The prom class registers literal names (required by the docs
+ # generator); they must stay in sync with the emission-side constants.
+ assert {m.kwargs["name"] for m in defs.values()} == {
+ SAVE_OUTCOMES,
+ LOAD_BLOCKS,
+ USED_BLOCKS,
+ PENDING_STORE_BLOCKS,
+ INFO,
+ }
+ for metric in defs.values():
+ assert metric.kwargs["documentation"]
+ assert metric.kwargs["labelnames"][:2] == ["model_name", "engine"]
+
+ assert defs[SAVE_OUTCOMES].kwargs["labelnames"] == [
+ "model_name",
+ "engine",
+ "outcome",
+ ]
+ assert defs[LOAD_BLOCKS].kwargs["labelnames"] == ["model_name", "engine"]
+ assert defs[INFO].kwargs["labelnames"] == [
+ "model_name",
+ "engine",
+ *INFO_LABELS,
+ ]
+ assert defs[USED_BLOCKS].kwargs["labelnames"] == ["model_name", "engine"]
+
+ # mostrecent: multiprocess scrapes expose one series per label set, not
+ # one pid-tagged series per writer process.
+ for gauge in (USED_BLOCKS, PENDING_STORE_BLOCKS, INFO):
+ assert defs[gauge].kwargs["multiprocess_mode"] == "mostrecent"
+
+
+def test_prom_metrics_observe_routes_each_series() -> None:
+ prom = _make_prom_metrics()
+ prom.observe(
+ _stats_payload(
+ types={
+ SAVE_OUTCOMES: _MetricType.COUNTER,
+ LOAD_BLOCKS: _MetricType.COUNTER,
+ USED_BLOCKS: _MetricType.GAUGE,
+ PENDING_STORE_BLOCKS: _MetricType.GAUGE,
+ INFO: _MetricType.GAUGE,
+ },
+ data={
+ SAVE_OUTCOMES: {("stored",): 3},
+ LOAD_BLOCKS: {(): 4},
+ USED_BLOCKS: {(): 5},
+ PENDING_STORE_BLOCKS: {(): 1},
+ INFO: {("cpu", "false", "false", "64"): 1},
+ },
+ )
+ )
+
+ assert prom._metrics[(0, SAVE_OUTCOMES, ("stored",))].increments == [3]
+ assert prom._metrics[(0, LOAD_BLOCKS, ())].increments == [4]
+ assert prom._metrics[(0, USED_BLOCKS, ())].set_values == [5]
+ assert prom._metrics[(0, PENDING_STORE_BLOCKS, ())].set_values == [1]
+ assert prom._metrics[(0, INFO, ("cpu", "false", "false", "64"))].set_values == [1]
+ assert prom._metrics[(0, SAVE_OUTCOMES, ("stored",))].labelvalues == (
+ "model",
+ "0",
+ "stored",
+ )
+ assert prom._metrics[(0, INFO, ("cpu", "false", "false", "64"))].labelvalues == (
+ "model",
+ "0",
+ "cpu",
+ "false",
+ "false",
+ "64",
+ )
+
+
+def test_prom_metrics_reuses_bound_children() -> None:
+ prom = _make_prom_metrics()
+ payload = _stats_payload(
+ types={SAVE_OUTCOMES: _MetricType.COUNTER},
+ data={SAVE_OUTCOMES: {("stored",): 1}},
+ )
+ prom.observe(payload)
+ prom.observe(payload)
+
+ child = prom._metrics[(0, SAVE_OUTCOMES, ("stored",))]
+ assert child.increments == [1, 1]
+ assert len(prom._defs[SAVE_OUTCOMES].children) == 1
+
+
+def test_prom_metrics_rejects_unknown_metric() -> None:
+ prom = _make_prom_metrics()
+ with pytest.raises(AssertionError, match="Unknown"):
+ prom.observe(
+ _stats_payload(
+ types={"vllm:not_a_metric": _MetricType.COUNTER},
+ data={"vllm:not_a_metric": {(): 1}},
+ )
+ )
+
+
+def test_prom_metrics_rejects_wrong_label_count() -> None:
+ prom = _make_prom_metrics()
+ with pytest.raises(AssertionError, match="labels"):
+ prom.observe(
+ _stats_payload(
+ types={SAVE_OUTCOMES: _MetricType.COUNTER},
+ data={SAVE_OUTCOMES: {("stored", "extra"): 1}},
+ )
+ )
+
+
+def test_prom_metrics_routes_per_engine() -> None:
+ prom = _make_prom_metrics(per_engine={0: ["model", "0"], 1: ["model", "1"]})
+ prom.observe(
+ _stats_payload(
+ types={USED_BLOCKS: _MetricType.GAUGE}, data={USED_BLOCKS: {(): 3}}
+ ),
+ engine_idx=1,
+ )
+ assert (0, USED_BLOCKS, ()) not in prom._metrics
+ assert prom._metrics[(1, USED_BLOCKS, ())].set_values == [3]
+
+
+def test_connector_build_prom_metrics() -> None:
+ from vllm.v1.simple_kv_offload.metrics import SimpleCPUOffloadPromMetrics
+
+ prom = SimpleCPUOffloadConnector.build_prom_metrics(
+ vllm_config=SimpleNamespace(kv_transfer_config=None),
+ metric_types=_fake_metric_types(),
+ labelnames=["model_name", "engine"],
+ per_engine_labelvalues={0: ["model", "0"]},
+ )
+ assert isinstance(prom, SimpleCPUOffloadPromMetrics)
+
+
+# ---------------------------------------------------------------------------
+# Manager: interval stats
+# ---------------------------------------------------------------------------
+
+
+def test_manager_get_stats_reports_outcome_deltas_per_interval() -> None:
+ fixture = make_scheduler()
+ sched = fixture.scheduler
+
+ sched.boundary_store_stats.stored += 3
+ sched.boundary_store_stats.dropped_cpu_full += 2
+ sched.boundary_store_stats.skipped_in_flight += 1
+
+ stats = sched.get_stats()
+ assert _values(stats, SAVE_OUTCOMES) == {
+ ("stored",): 3,
+ ("dropped_cpu_full",): 2,
+ ("skipped_in_flight",): 1,
+ }
+
+ # Deltas are per-interval: a second drain reports no new outcomes.
+ stats2 = sched.get_stats()
+ assert SAVE_OUTCOMES not in stats2.data[_StatsKey.DATA]
+
+
+def test_manager_reset_preserves_undrained_outcome_deltas() -> None:
+ fixture = make_scheduler()
+ sched = fixture.scheduler
+
+ sched.boundary_store_stats.stored += 3
+ sched.reset()
+
+ stats = sched.get_stats()
+ assert _values(stats, SAVE_OUTCOMES) == {("stored",): 3}
+
+
+def test_manager_get_stats_gauges_and_info_labels() -> None:
+ fixture = make_scheduler(num_cpu_blocks=8)
+ sched = fixture.scheduler
+
+ sched._store_event_to_blocks[0] = TransferMeta([10], [11])
+ sched._pending_finished_stores.append(TransferMeta([12], [13]))
+ sched._abandoned_store_event_to_blocks[1] = TransferMeta([14], [15])
+
+ stats = sched.get_stats()
+ assert _values(stats, PENDING_STORE_BLOCKS) == {(): 3}
+ used = _values(stats, USED_BLOCKS)[()]
+ assert used == sched.num_cpu_blocks - sched.cpu_block_pool.get_num_free_blocks()
+
+ info = _values(stats, INFO)
+ assert info == {("cpu", "false", "false", str(sched.num_cpu_blocks)): 1}
+
+ # Gauges are re-reported every drain.
+ stats2 = sched.get_stats()
+ assert _values(stats2, PENDING_STORE_BLOCKS) == {(): 3}
+
+
+def test_manager_info_labels_reflect_disk_mode() -> None:
+ connector = _make_connector(
+ extra_config={
+ "kv_offload_backend": "disk",
+ "disk_path": "/tmp/fake-disk-metrics",
+ "disk_capacity_bytes": 1024**3,
+ "use_page_cache": True,
+ }
+ )
+ sched = connector.scheduler_manager
+ assert sched is not None
+ assert sched._info_labelvalues[0] == "disk"
+ assert sched._info_labelvalues[1] == "true"
+ stats = sched.get_stats()
+ assert set(_values(stats, INFO)) == {sched._info_labelvalues}
+
+
+def test_manager_info_labels_page_cache_false_for_cpu_backend() -> None:
+ """CPU backend ignores use_page_cache; the info gauge must not claim it."""
+ connector = _make_connector(extra_config={"use_page_cache": True})
+ sched = connector.scheduler_manager
+ assert sched is not None
+ assert sched._info_labelvalues[:2] == ("cpu", "false")
+ stats = sched.get_stats()
+ assert set(_values(stats, INFO)) == {sched._info_labelvalues}
+
+
+def test_manager_counts_load_blocks_completed() -> None:
+ fixture = make_scheduler(num_cpu_blocks=8)
+ sched = fixture.scheduler
+
+ request = make_request(num_blocks=2)
+ gpu_blocks = fixture.gpu_block_pool.get_new_blocks(2)
+ cpu_blocks = sched.cpu_block_pool.get_new_blocks(2)
+ sched._reqs_to_load[request.request_id] = LoadRequestState(
+ request=request,
+ transfer_meta=TransferMeta(
+ [b.block_id for b in gpu_blocks], [b.block_id for b in cpu_blocks]
+ ),
+ )
+
+ sched.build_connector_meta(make_scheduler_output({}))
+ stats = sched.get_stats()
+ assert LOAD_BLOCKS not in stats.data[_StatsKey.DATA]
+
+ sched.update_connector_output(
+ KVConnectorOutput(finished_recving={request.request_id})
+ )
+ stats = sched.get_stats()
+ assert _values(stats, LOAD_BLOCKS) == {(): 2}
+
+
+def test_manager_abandoned_load_counts_as_completed_after_reset() -> None:
+ """Loads abandoned by reset() that still finish on the worker are
+ counted as loaded blocks.
+ """
+ fixture = make_scheduler(num_cpu_blocks=8)
+ sched = fixture.scheduler
+
+ request = make_request(num_blocks=2)
+ gpu_blocks = fixture.gpu_block_pool.get_new_blocks(2)
+ cpu_blocks = sched.cpu_block_pool.get_new_blocks(2)
+ sched._reqs_to_load[request.request_id] = LoadRequestState(
+ request=request,
+ transfer_meta=TransferMeta(
+ [b.block_id for b in gpu_blocks], [b.block_id for b in cpu_blocks]
+ ),
+ )
+
+ sched.build_connector_meta(make_scheduler_output({}))
+ stats = sched.get_stats()
+ assert LOAD_BLOCKS not in stats.data[_StatsKey.DATA]
+
+ assert sched.reset() is False
+ assert set(sched._abandoned_reqs_to_load) == {request.request_id}
+
+ sched.update_connector_output(
+ KVConnectorOutput(finished_recving={request.request_id})
+ )
+ assert sched._abandoned_reqs_to_load == {}
+ stats = sched.get_stats()
+ assert _values(stats, LOAD_BLOCKS) == {(): 2}
diff --git a/tests/v1/simple_kv_offload/test_scheduler.py b/tests/v1/simple_kv_offload/test_scheduler.py
index ed07718de6..3817a15e3e 100644
--- a/tests/v1/simple_kv_offload/test_scheduler.py
+++ b/tests/v1/simple_kv_offload/test_scheduler.py
@@ -2316,8 +2316,8 @@ def _make_hybrid_attention_mamba_scheduler(
def test_hybrid_store_uses_resolved_group_block_sizes() -> None:
"""Replicated groups must not be scaled by the DCP world size.
- ``dcp_world_size_for_kv_cache_spec`` gives full attention the process DCP
- size and every other spec 1. Reconstructing a group's geometry as
+ The spec's ``dcp_sharded`` flag controls its effective block size.
+ Reconstructing a group's geometry as
``spec.block_size * cp_world_size`` over-scales the replicated groups, so
the eager store scan believes far fewer of their blocks are ready and
silently offloads only a fraction of them.
diff --git a/tests/v1/spec_decode/test_dflash_prepare_inputs.py b/tests/v1/spec_decode/test_dflash_prepare_inputs.py
index 16d6d8e516..1d004c3667 100644
--- a/tests/v1/spec_decode/test_dflash_prepare_inputs.py
+++ b/tests/v1/spec_decode/test_dflash_prepare_inputs.py
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
+from unittest.mock import patch
import numpy as np
import pytest
@@ -9,7 +10,7 @@ import torch
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import (
- prepare_dflash_inputs,
+ DFlashSpeculator,
)
pytestmark = pytest.mark.skipif(
@@ -24,6 +25,7 @@ def _run_prepare(
cp_rank: int = 0,
cp_size: int = 1,
cp_interleave: int = 1,
+ draft_dcp_size: int | None = None,
):
device = torch.device("cuda")
max_num_reqs = 4
@@ -40,7 +42,9 @@ def _run_prepare(
)
input_batch = SimpleNamespace(
num_reqs=1,
+ num_tokens=4,
num_scheduled_tokens=np.array([4], dtype=np.int32),
+ seq_lens_cpu_upper_bound=torch.tensor([target_positions[-1] + 1]),
positions=torch.tensor(target_positions, dtype=torch.int64, device=device),
query_start_loc=torch.tensor([0, 4], dtype=torch.int32, device=device),
idx_mapping=torch.tensor([2], dtype=torch.int32, device=device),
@@ -74,36 +78,65 @@ def _run_prepare(
next_prefill_tokens = torch.zeros_like(last_sampled)
block_table = torch.tensor([block_table_values], dtype=torch.int32, device=device)
- prepare_dflash_inputs(
- input_buffers,
- query_slot_mapping,
- context_positions,
- context_slot_mapping,
- sample_indices,
- sample_pos,
- sample_idx_mapping,
- temperature,
- seeds,
- input_batch,
- torch.tensor([1], dtype=torch.int32, device=device),
- torch.tensor([2], dtype=torch.int32, device=device),
- last_sampled,
- next_prefill_tokens,
- input_temperature,
- input_seeds,
- block_table,
- 4,
- cp_rank,
- cp_size,
- cp_interleave,
- 123,
- num_speculative_steps,
- num_speculative_steps,
- max_num_reqs,
- max_num_tokens,
- 128,
+ class InputsPrepared(Exception):
+ pass
+
+ draft = SimpleNamespace(
+ input_buffers=input_buffers,
+ context_positions=context_positions,
+ _context_slot_mappings=context_slot_mapping.unsqueeze(0),
+ sample_indices=sample_indices,
+ sample_pos=sample_pos,
+ sample_idx_mapping=sample_idx_mapping,
+ temperature=temperature,
+ seeds=seeds,
+ hidden_states=torch.zeros(4, 1, device=device),
+ prepare_context_anchor=lambda *args: None,
+ query_cudagraph_manager=None,
+ dp_size=1,
+ dp_rank=0,
+ pcp_manager=None,
+ draft_kv_cache_group_id=0,
+ draft_kv_cache_group_ids=[0],
+ _layer_group_idx=None,
+ block_tables=SimpleNamespace(
+ slot_mappings=query_slot_mapping.unsqueeze(0),
+ input_block_tables=[block_table],
+ kernel_block_sizes=[4],
+ cp_rank=cp_rank,
+ cp_size=cp_size,
+ cp_interleave=cp_interleave,
+ ),
+ dcp_size=cp_size if draft_dcp_size is None else draft_dcp_size,
+ parallel_drafting_token_id=123,
+ num_query_per_req=num_speculative_steps,
+ num_speculative_steps=num_speculative_steps,
+ max_num_reqs=max_num_reqs,
+ max_num_tokens=max_num_tokens,
+ max_model_len=128,
sample_from_anchor=True,
)
+ with (
+ patch(
+ "vllm.v1.worker.gpu.spec_decode.dflash.speculator.dispatch_cg_and_sync_dp",
+ side_effect=InputsPrepared,
+ ),
+ pytest.raises(InputsPrepared),
+ ):
+ DFlashSpeculator.propose(
+ draft,
+ input_batch,
+ {},
+ {},
+ draft.hidden_states,
+ None,
+ torch.tensor([1], dtype=torch.int32, device=device),
+ torch.tensor([2], dtype=torch.int32, device=device),
+ last_sampled,
+ next_prefill_tokens,
+ input_temperature,
+ input_seeds,
+ )
torch.accelerator.synchronize()
return SimpleNamespace(
input_buffers=input_buffers,
@@ -118,12 +151,17 @@ def _run_prepare(
)
-def test_prepare_dflash_inputs_excludes_rejected_context_suffix():
+@pytest.mark.parametrize("cp_rank,cp_size", [(0, 1), (0, 2), (1, 2)])
+def test_prepare_dflash_inputs_excludes_rejected_context_suffix(cp_rank, cp_size):
# Positions 10/11 use physical block 7. Rejected positions 12/13 would use
# block 8, but must be PAD context rather than contaminating draft KV.
out = _run_prepare(
target_positions=[10, 11, 12, 13],
block_table_values=[0, 0, 7, 8, 9, 10, 11, 12],
+ cp_rank=cp_rank,
+ cp_size=cp_size,
+ cp_interleave=2,
+ draft_dcp_size=1,
)
assert out.context_positions[:4].tolist() == [10, 11, 0, 0]
diff --git a/tests/v1/spec_decode/test_eagle_draft_attn_metadata.py b/tests/v1/spec_decode/test_eagle_draft_attn_metadata.py
index b542388952..2c4b951be7 100644
--- a/tests/v1/spec_decode/test_eagle_draft_attn_metadata.py
+++ b/tests/v1/spec_decode/test_eagle_draft_attn_metadata.py
@@ -17,6 +17,7 @@ from types import MethodType, SimpleNamespace
from unittest.mock import patch
import numpy as np
+import pytest
import torch
from vllm.config.compilation import CUDAGraphMode
@@ -53,6 +54,7 @@ def _make_fake_speculator(
arange_np=np.arange(max_num_reqs + 1, dtype=np.int32),
draft_is_prefilling=torch.zeros(max_num_reqs, dtype=torch.bool),
block_tables=fake_block_tables,
+ dcp_size=1,
input_buffers=fake_input_buffers,
attn_groups=[],
kv_cache_config=SimpleNamespace(kv_cache_groups=[]),
@@ -152,14 +154,17 @@ def test_build_draft_attn_metadata_clamps_to_max_model_len():
assert torch.equal(bound, torch.tensor([1024, 503], dtype=torch.int32))
-def test_build_draft_attn_metadata_recomputes_dcp_local_seq_lens():
+@pytest.mark.parametrize("draft_dcp_size", [1, 2])
+def test_build_draft_attn_metadata_recomputes_dcp_local_seq_lens(draft_dcp_size):
fake = _make_fake_speculator()
+ fake.dcp_size = draft_dcp_size
fake.block_tables.cp_size = 2
fake.block_tables.cp_rank = 1
fake.block_tables.cp_interleave = 4
fake.input_buffers.seq_lens[:3] = torch.tensor([5, 9, 16])
def fake_prepare(out, seq_lens, num_reqs, dcp_size, dcp_rank, cp_interleave):
+ assert draft_dcp_size > 1
assert seq_lens is fake.input_buffers.seq_lens
assert (num_reqs, dcp_size, dcp_rank, cp_interleave) == (3, 2, 1, 4)
out[:num_reqs].copy_(torch.tensor([1, 4, 8], dtype=torch.int32))
@@ -176,6 +181,9 @@ def test_build_draft_attn_metadata_recomputes_dcp_local_seq_lens():
)
local = captured["dcp_local_seq_lens"]
+ if draft_dcp_size == 1:
+ assert local is None
+ return
assert isinstance(local, torch.Tensor)
assert local.data_ptr() == fake.input_buffers.dcp_local_seq_lens.data_ptr()
assert local.tolist() == [1, 4, 8, 0]
diff --git a/tests/v1/worker/test_cp_utils.py b/tests/v1/worker/test_cp_utils.py
index b38ae4d063..d477ace479 100644
--- a/tests/v1/worker/test_cp_utils.py
+++ b/tests/v1/worker/test_cp_utils.py
@@ -1,5 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from types import SimpleNamespace
+
import pytest
import torch
@@ -7,6 +9,38 @@ from vllm.v1.attention.backends.utils import get_dcp_local_seq_lens
from vllm.v1.worker.cp_utils import should_skip_dcp_context_attention
+def test_cp_checks_allow_a_replicated_draft_backend_without_dcp(monkeypatch):
+ from vllm.v1.worker import cp_utils
+
+ config = SimpleNamespace(
+ parallel_config=SimpleNamespace(
+ prefill_context_parallel_size=1,
+ decode_context_parallel_size=4,
+ cp_kv_cache_interleave_size=16,
+ ),
+ speculative_config=object(),
+ )
+ target = SimpleNamespace(
+ impl=SimpleNamespace(
+ dcp_world_size=4,
+ supports_mtp_with_cp_non_trivial_interleave_size=True,
+ need_to_return_lse_for_decode=True,
+ )
+ )
+ draft = SimpleNamespace(
+ impl=SimpleNamespace(
+ dcp_world_size=1,
+ supports_mtp_with_cp_non_trivial_interleave_size=False,
+ need_to_return_lse_for_decode=False,
+ )
+ )
+ layers = {"target": target, "draft": draft}
+ monkeypatch.setattr(cp_utils, "get_layers_from_vllm_config", lambda *args: layers)
+ cp_utils.check_attention_cp_compatibility(config, target_layer_names={"target"})
+ with pytest.raises(AssertionError, match="MTP with cp_kv_cache_interleave_size"):
+ cp_utils.check_attention_cp_compatibility(config)
+
+
def test_skip_gate_only_for_zero_context():
assert should_skip_dcp_context_attention(torch.zeros(3, dtype=torch.int32))
assert not should_skip_dcp_context_attention(
diff --git a/tests/v1/worker/test_dspark_utils.py b/tests/v1/worker/test_dspark_utils.py
index 94655f192c..1014051d64 100644
--- a/tests/v1/worker/test_dspark_utils.py
+++ b/tests/v1/worker/test_dspark_utils.py
@@ -2,8 +2,17 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass, field
+from types import SimpleNamespace
+import pytest
+import torch
+
+from vllm.config import ParallelConfig
+from vllm.v1.kv_cache_interface import FullAttentionSpec
+from vllm.v1.worker.gpu import model_runner
+from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator
from vllm.v1.worker.gpu.spec_decode.dspark.utils import _get_dspark_parallel_config
+from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
@dataclass
@@ -49,3 +58,86 @@ def test_dspark_parallel_config_disables_eplb_atomically():
assert draft_config.eplb_config.num_redundant_experts == 0
assert not draft_config.enable_elastic_ep
assert draft_config.eplb_config is not target_config.eplb_config
+
+
+@pytest.mark.parametrize("pcp_size", [1, 4])
+@pytest.mark.parametrize("dcp_size", [1, 4])
+@pytest.mark.parametrize("use_mla", [False, True])
+def test_draft_context_parallelism_without_changing_target(
+ monkeypatch, pcp_size, dcp_size, use_mla
+):
+ target_parallel = ParallelConfig(
+ tensor_parallel_size=4,
+ prefill_context_parallel_size=pcp_size,
+ decode_context_parallel_size=dcp_size,
+ cp_kv_cache_interleave_size=16,
+ distributed_executor_backend="mp",
+ )
+ target_config = SimpleNamespace(
+ parallel_config=target_parallel,
+ speculative_config=SimpleNamespace(
+ draft_model_config=SimpleNamespace(use_mla=use_mla)
+ ),
+ )
+
+ class CapturedConfig(Exception):
+ pass
+
+ def capture_init(self, config, device):
+ raise CapturedConfig(config)
+
+ monkeypatch.setattr(DraftModelSpeculator, "__init__", capture_init)
+ with pytest.raises(CapturedConfig) as captured:
+ DFlashSpeculator(target_config, device=None)
+ draft_parallel = captured.value.args[0].parallel_config
+ assert draft_parallel.tensor_parallel_size == 4
+ assert draft_parallel.prefill_context_parallel_size == 1
+ assert draft_parallel.decode_context_parallel_size == (dcp_size if use_mla else 1)
+ assert draft_parallel.cp_kv_cache_interleave_size == 16
+ assert target_parallel.prefill_context_parallel_size == pcp_size
+ assert target_parallel.decode_context_parallel_size == dcp_size
+ assert target_parallel.cp_kv_cache_interleave_size == 16
+
+
+@pytest.mark.parametrize("configured_dcp", [1, 4])
+def test_attention_uses_draft_dcp_setting_inside_target_process_group(
+ monkeypatch, configured_dcp
+):
+ import vllm.config as config_module
+ from vllm.distributed import parallel_state
+ from vllm.v1.attention.backend import AttentionImplBase
+
+ config = SimpleNamespace(
+ parallel_config=SimpleNamespace(decode_context_parallel_size=configured_dcp)
+ )
+ monkeypatch.setattr(
+ config_module, "get_current_vllm_config_or_none", lambda: config
+ )
+
+ monkeypatch.setattr(
+ parallel_state, "_DCP", SimpleNamespace(world_size=4, rank_in_group=2)
+ )
+ impl = AttentionImplBase()
+ assert impl.dcp_world_size == configured_dcp
+ assert impl.dcp_rank == (0 if configured_dcp == 1 else 2)
+
+
+@pytest.mark.parametrize("draft_dcp_size", [1, 4])
+def test_runner_marks_only_replicated_draft_caches(monkeypatch, draft_dcp_size):
+ spec = FullAttentionSpec(
+ block_size=16, num_kv_heads=1, head_size=64, dtype=torch.bfloat16
+ )
+ monkeypatch.setattr(
+ model_runner, "get_kv_cache_spec", lambda _: {"target": spec, "draft": spec}
+ )
+ runner = object.__new__(model_runner.GPUModelRunner)
+ runner.vllm_config = None
+ runner.dcp_size = 4
+ runner.speculator = object.__new__(DFlashSpeculator)
+ runner.speculator.dcp_size = draft_dcp_size
+ runner.speculator.draft_attn_layer_names = {"draft"}
+
+ specs = runner.get_kv_cache_spec()
+
+ assert specs["target"].dcp_sharded
+ assert specs["draft"].dcp_sharded == (draft_dcp_size == 4)
diff --git a/tests/v1/worker/test_gpu_autoregressive_speculator.py b/tests/v1/worker/test_gpu_autoregressive_speculator.py
index 4fc7d4d2e5..fde3605a20 100644
--- a/tests/v1/worker/test_gpu_autoregressive_speculator.py
+++ b/tests/v1/worker/test_gpu_autoregressive_speculator.py
@@ -88,6 +88,7 @@ def test_pcp_draft_metadata_keeps_graph_padding_in_decode(cg_mode):
query_start_loc=torch.tensor([0, 1, 2, 2, 2], dtype=torch.int32),
seq_lens=torch.tensor([11, 21, 0, 0], dtype=torch.int32),
)
+ speculator.dcp_size = 1
speculator.block_tables = SimpleNamespace(
cp_size=1,
input_block_tables=[torch.zeros(4, 1, dtype=torch.int32)],
@@ -97,7 +98,9 @@ def test_pcp_draft_metadata_keeps_graph_padding_in_decode(cg_mode):
speculator.attn_groups = [
[
SimpleNamespace(
- get_metadata_builder=lambda _: SimpleNamespace(build=build),
+ get_metadata_builder=lambda _: SimpleNamespace(
+ build=build, supports_update_block_table=False
+ ),
layer_names=["draft"],
)
]
@@ -183,6 +186,7 @@ def test_speculator_uses_draft_model_hidden_size(monkeypatch, hc_mult, expected)
parallel_config=SimpleNamespace(
data_parallel_size=1,
data_parallel_rank=0,
+ decode_context_parallel_size=1,
),
)
diff --git a/tests/v1/worker/test_gpu_block_table.py b/tests/v1/worker/test_gpu_block_table.py
index 5c245d1c4d..5c2231a1f2 100644
--- a/tests/v1/worker/test_gpu_block_table.py
+++ b/tests/v1/worker/test_gpu_block_table.py
@@ -171,22 +171,23 @@ def test_block_tables_skip_custom_slot_mapping_groups():
@pytest.mark.parametrize("cp_rank", range(4))
def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
- """DCP interleave is expressed in logical-block token coordinates."""
+ """Only sharded groups use DCP interleave in logical-block coordinates."""
device = torch.device("cuda")
block_tables = BlockTables(
- block_sizes=[128],
+ block_sizes=[128, 128],
max_num_reqs=1,
max_num_batched_tokens=1024,
- max_num_blocks_per_group=[2],
+ max_num_blocks_per_group=[2, 8],
device=device,
- kernel_block_sizes=[64],
+ kernel_block_sizes=[64, 64],
+ dcp_sharded=[True, False],
cp_size=4,
cp_rank=cp_rank,
cp_interleave=128,
)
block_tables.append_block_ids(
req_index=0,
- new_block_ids=([5, 9],),
+ new_block_ids=([5, 9], list(range(10, 18))),
overwrite=True,
)
block_tables.apply_staged_writes()
@@ -199,7 +200,7 @@ def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
query_start_loc,
positions,
num_tokens_padded=1024,
- )[0]
+ )
expected = torch.full((1024,), -1, dtype=torch.int64, device=device)
first_start = cp_rank * 128
@@ -210,7 +211,8 @@ def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
expected[second_start : second_start + 128] = torch.arange(
9 * 128, 10 * 128, dtype=torch.int64, device=device
)
- assert torch.equal(actual, expected)
+ assert torch.equal(actual[0], expected)
+ assert torch.equal(actual[1], positions + 10 * 128)
def test_v1_block_table_move_row_clears_vacated_row():
diff --git a/tests/v1/worker/test_gpu_input_batch.py b/tests/v1/worker/test_gpu_input_batch.py
index 7c6c9ba417..f37e824e6c 100644
--- a/tests/v1/worker/test_gpu_input_batch.py
+++ b/tests/v1/worker/test_gpu_input_batch.py
@@ -372,6 +372,62 @@ def test_swap_states_in_input_batch(device: str, batch_size: int, swap_list: lis
_compare_objs(input_batch, ref_input_batch)
+@pytest.mark.parametrize("device", DEVICES)
+def test_condense_clears_stale_allowed_token_ids_mask(device: str):
+ """condense() must clear the mask row a constrained request is moved out
+ of. Otherwise a later request that reuses that row without
+ allowed_token_ids inherits the stale whitelist.
+ See https://github.com/vllm-project/vllm/issues/43894.
+ """
+ batch_size = 4
+ allowed_token_id = 13
+ input_batch = InputBatch(
+ max_num_reqs=batch_size,
+ max_model_len=1024,
+ max_num_batched_tokens=1024,
+ device=torch.device(device),
+ vocab_size=VOCAB_SIZE,
+ block_sizes=[1],
+ kernel_block_sizes=[1],
+ max_num_blocks_per_req=[1024],
+ )
+
+ def _make_req(suffix: int, allowed_token_ids=None) -> CachedRequestState:
+ return CachedRequestState(
+ req_id=f"req_id_{suffix}",
+ prompt_token_ids=[1, 2, 3],
+ sampling_params=SamplingParams(allowed_token_ids=allowed_token_ids),
+ pooling_params=None,
+ mm_features=[],
+ block_ids=([],),
+ generator=None,
+ num_computed_tokens=0,
+ output_token_ids=[],
+ )
+
+ # Only req_id_2 is constrained, and it lands on the highest row.
+ assert input_batch.add_request(_make_req(0)) == 0
+ assert input_batch.add_request(_make_req(1)) == 1
+ assert input_batch.add_request(_make_req(2, [allowed_token_id])) == 2
+
+ mask = input_batch.allowed_token_ids_mask_cpu_tensor
+ assert mask is not None
+ # The constrained row masks every token except the single allowed id.
+ assert not mask[2][allowed_token_id].item()
+ assert int(mask[2].sum().item()) == VOCAB_SIZE - 1
+
+ # Free row 0, then condense: req_id_2 slides from row 2 down into row 0.
+ input_batch.remove_request("req_id_0")
+ input_batch.condense()
+ assert input_batch.req_id_to_index["req_id_2"] == 0
+ assert int(mask[2].sum().item()) == 0
+
+ # A new unrestricted request reuses row 2 and must stay unconstrained.
+ assert input_batch.add_request(_make_req(3)) == 2
+ assert "req_id_3" not in input_batch.has_allowed_token_ids
+ assert int(mask[2].sum().item()) == 0
+
+
@pytest.mark.parametrize("device", DEVICES)
def test_swap_states_preserves_allowed_token_ids_mask(device: str):
"""swap_states() must exchange the two allowed_token_ids_mask rows.
diff --git a/tests/v1/worker/test_gpu_pcp_manager.py b/tests/v1/worker/test_gpu_pcp_manager.py
index 0c4b5aff68..4612d49d92 100644
--- a/tests/v1/worker/test_gpu_pcp_manager.py
+++ b/tests/v1/worker/test_gpu_pcp_manager.py
@@ -75,6 +75,24 @@ def test_validate_config_rejects_full_graph_for_prefills():
)
+@pytest.mark.parametrize("draft_uses_mla", [False, True])
+@pytest.mark.parametrize("dcp_size", [1, 2, 4])
+def test_validate_config_dcp_for_dspark(draft_uses_mla, dcp_size):
+ config = _make_config(CUDAGraphMode.NONE)
+ config.parallel_config.prefill_context_parallel_size = 4
+ config.parallel_config.decode_context_parallel_size = dcp_size
+ config.model_config.hf_text_config.index_topk = 1
+ config.speculative_config = SimpleNamespace(
+ use_dspark=lambda: True,
+ draft_model_config=SimpleNamespace(use_mla=draft_uses_mla),
+ )
+ if draft_uses_mla and dcp_size == 2:
+ with pytest.raises(NotImplementedError, match="requires DCP=1 or DCP=PCP"):
+ PCPManager.validate_config(config, supports_mm_inputs=False)
+ return
+ PCPManager.validate_config(config, supports_mm_inputs=False)
+
+
def test_replicated_decode_piecewise_graph_padding(monkeypatch):
manager = PCPManager(
pcp_world_size=2,
diff --git a/tools/pre_commit/mypy.py b/tools/pre_commit/mypy.py
index f9d064a499..7262c1f2a9 100755
--- a/tools/pre_commit/mypy.py
+++ b/tools/pre_commit/mypy.py
@@ -113,7 +113,6 @@ SEPARATE_GROUPS = [
EXCLUDE = [
r"vllm/model_executor/models/[kK]",
- r"vllm/model_executor/models/[uU]",
r"vllm/model_executor/models/[vV]",
r"vllm/model_executor/models/[wW]",
r"vllm/model_executor/models/[zZ]",
diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py
index af0213361e..0e89b39818 100644
--- a/vllm/_aiter_ops.py
+++ b/vllm/_aiter_ops.py
@@ -1870,28 +1870,39 @@ def _mhc_delayed_pre_tail(
_OPS_REGISTERED = False
+_AITER_SITUV2_ACT_ENV = {
+ "a8w4": "AITER_SITUV2_A8W4",
+ "a4w4": "AITER_SITUV2_A4W4",
+}
+
+
+def _resolve_situv2_activation() -> str:
+ """VLLM_ROCM_USE_AITER_MOE_SITUV2 -> a4w4 | a8w4 | a16w4.
+
+ auto (default) and the legacy 1 mean a4w4; legacy 0 means a16w4.
+ """
+ value = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2.lower()
+ if value in ("auto", "1"):
+ return "a4w4"
+ if value == "0":
+ return "a16w4"
+ return value
+
+
def _sync_aiter_situv2_moe_env() -> None:
- """Mirror the SiTUv2 MoE toggle into AITER's a4w4 dispatch env.
+ """Set the AITER_SITUV2_* env matching VLLM_ROCM_USE_AITER_MOE_SITUV2.
- AITER selects afp8 vs afp4 activation kernels via AITER_SITUV2_A8W4 /
- AITER_SITUV2_A4W4 (see ROCm/aiter fused_moe.py, A8W4 checked first).
- When VLLM_ROCM_USE_AITER_MOE_SITUV2 is enabled we route to a4w4
- (afp4_wfp4_fp4 kernels) and clear any legacy AITER_SITUV2_A8W4 override.
-
- Requires AITER with ROCm/aiter#4463 (first tagged in v0.1.20): a4w4
- dispatch plus kimik3_a4w4_{un,}tuned_fmoe.csv. Older AITER still runs
- afp4 FlyDSL but falls back to heuristic configs, which is not the
- tuned a4w4 path this recipe is meant to use.
+ AITER reads AITER_SITUV2_A8W4 / AITER_SITUV2_A4W4 and defaults to a16w4,
+ so exactly one is set for a8w4/a4w4 and both are cleared for a16w4.
"""
import os
- import vllm.envs as envs
-
- if envs.VLLM_ROCM_USE_AITER_MOE_SITUV2:
- os.environ["AITER_SITUV2_A4W4"] = "1"
- os.environ.pop("AITER_SITUV2_A8W4", None)
- else:
- os.environ.pop("AITER_SITUV2_A4W4", None)
+ selected = _AITER_SITUV2_ACT_ENV.get(_resolve_situv2_activation())
+ for name in _AITER_SITUV2_ACT_ENV.values():
+ if name == selected:
+ os.environ[name] = "1"
+ else:
+ os.environ.pop(name, None)
class rocm_aiter_ops:
@@ -1917,7 +1928,8 @@ class rocm_aiter_ops:
VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: Controls FP4 assembly GEMM.
VLLM_ROCM_USE_AITER_TRITON_ROPE: Controls Triton rotary embeddings.
VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: Controls shared expert fusion.
- VLLM_ROCM_USE_AITER_MOE_SITUV2: Controls SiTUv2 FlyDSL MoE (a4w4).
+ VLLM_ROCM_USE_AITER_MOE_SITUV2: SiTUv2 FlyDSL MoE activation
+ dtype (a16w4 | a8w4 | a4w4).
VLLM_ROCM_USE_AITER_TRITON_GEMM: Controls Triton unquantized GEMM.
Note:
@@ -1987,7 +1999,7 @@ class rocm_aiter_ops:
# TODO: Consolidate under VLLM_ROCM_USE_AITER_ROPE
_TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE
_MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
- _MOE_SITUV2 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2
+ _MOE_SITUV2 = _resolve_situv2_activation()
# TODO: Consolidate under _LINEAR_ENABLED
_TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM
# Lazily probed: whether aiter.topk_softmax supports the
@@ -2016,7 +2028,7 @@ class rocm_aiter_ops:
cls._FP4_GEMM_DYNAMIC_QUANT_ASM = envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM
cls._TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE
cls._MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
- cls._MOE_SITUV2 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2
+ cls._MOE_SITUV2 = _resolve_situv2_activation()
_sync_aiter_situv2_moe_env()
cls._TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM
cls._MOE_DISPATCH_POLICY = envs.VLLM_ROCM_AITER_MOE_DISPATCH_POLICY
@@ -2132,9 +2144,25 @@ class rocm_aiter_ops:
@classmethod
@if_aiter_supported
def is_fused_moe_situv2_enabled(cls) -> bool:
- # _MOE_SITUV2 is a variant of aiter fused moe, so aiter
- # fused moe must be enabled as well.
- return cls.is_fused_moe_enabled() and cls._MOE_SITUV2
+ """True when a low-precision (a8w4/a4w4) SiTUv2 activation is selected."""
+ return cls.is_fused_moe_enabled() and cls._MOE_SITUV2 != "a16w4"
+
+ @classmethod
+ @if_aiter_supported
+ def get_fused_moe_situv2_activation(cls) -> str:
+ """SiTUv2 activation dtype name: a16w4, a8w4 or a4w4."""
+ return cls._MOE_SITUV2
+
+ @classmethod
+ @if_aiter_supported
+ def is_fused_moe_situv2_gate_up_interleaved(cls) -> bool:
+ """Whether the SiTUv2 kernels expect gate/up-interleaved w13.
+
+ The a8w4 FlyDSL stage-1 kernels (_gui_) are interleaved; a16w4 and
+ a4w4 are separated. Weight shuffle (oracle/mxfp4.py) and gate_mode
+ (experts/rocm_aiter_moe.py) must both use this.
+ """
+ return cls._MOE_SITUV2 == "a8w4"
@classmethod
@if_aiter_supported
diff --git a/vllm/benchmarks/datasets/datasets.py b/vllm/benchmarks/datasets/datasets.py
index 8f9e87cf83..76de75a66f 100644
--- a/vllm/benchmarks/datasets/datasets.py
+++ b/vllm/benchmarks/datasets/datasets.py
@@ -2148,7 +2148,9 @@ def get_samples(
dataset_path=args.dataset_path, disable_shuffle=args.disable_shuffle
)
# For the "sonnet" dataset, formatting depends on the backend.
- if args.backend == "openai-chat":
+ # Chat-style backends leave templating to the server; completions
+ # backends need the prompt rendered client side.
+ if args.backend in ("openai-chat", "openai-responses"):
input_requests = sonnet_dataset.sample(
num_requests=args.num_prompts,
input_len=args.sonnet_input_len,
diff --git a/vllm/benchmarks/lib/endpoint_request_func.py b/vllm/benchmarks/lib/endpoint_request_func.py
index 2b4027b502..14ff6c1ab1 100644
--- a/vllm/benchmarks/lib/endpoint_request_func.py
+++ b/vllm/benchmarks/lib/endpoint_request_func.py
@@ -447,6 +447,178 @@ async def async_request_openai_chat_completions(
return output
+def _extract_sse_data(message: str) -> str | None:
+ """Extract the ``data`` payload of a single SSE message.
+
+ The Responses API stream labels every message with an ``event:`` line
+ before its ``data:`` line, so the payload cannot be recovered by
+ stripping a leading ``data: `` the way the Completions and Chat
+ Completions streams allow.
+
+ Args:
+ message: One complete SSE message, without its trailing blank line.
+
+ Returns:
+ The payload, with multi-line payloads joined by newlines as required
+ by the SSE spec, or None if the message carries no ``data:`` line.
+
+ """
+ # NOTE: split on "\n" rather than using splitlines(), which also breaks on
+ # U+2028, U+2029 and U+0085. Those are legal inside a JSON string and the
+ # server emits them unescaped, so splitlines() would truncate the payload.
+ data_lines = [
+ line.removeprefix("data:").removeprefix(" ")
+ for line in message.split("\n")
+ if line.startswith("data:")
+ ]
+ if not data_lines:
+ return None
+
+ return "\n".join(data_lines)
+
+
+# Responses events that carry generated tokens. Reasoning deltas are included
+# because the server is already decoding when it emits them, so excluding them
+# would report the whole reasoning phase as time to first token.
+_RESPONSES_TOKEN_DELTA_EVENTS = frozenset(
+ {
+ "response.output_text.delta",
+ "response.reasoning_text.delta",
+ }
+)
+
+# Responses events that end a request. Truncation at the output length limit
+# is a normal benchmark outcome: vLLM reports it as `response.completed` with
+# `status: "incomplete"`, while the spec also defines a dedicated
+# `response.incomplete` terminal event.
+_RESPONSES_TERMINAL_EVENTS = frozenset(
+ {
+ "response.completed",
+ "response.incomplete",
+ }
+)
+
+# The spec's failure terminals. vLLM does not emit them today, but
+# `vllm bench serve` can be pointed at other Responses servers.
+_RESPONSES_ERROR_EVENTS = frozenset({"error", "response.failed"})
+
+
+async def async_request_openai_responses(
+ request_func_input: RequestFuncInput,
+ session: aiohttp.ClientSession,
+ pbar: tqdm | None = None,
+) -> RequestFuncOutput:
+ api_url = request_func_input.api_url
+ _validate_api_url(api_url, "OpenAI Responses API", "responses")
+
+ payload = {
+ "model": request_func_input.model_name
+ if request_func_input.model_name
+ else request_func_input.model,
+ "input": request_func_input.prompt,
+ "max_output_tokens": request_func_input.output_len,
+ "stream": True,
+ # `store` defaults to true. A server started with
+ # VLLM_ENABLE_RESPONSES_API_STORE=1 keeps every stored response for
+ # the lifetime of the process, which would grow its heap over a long
+ # benchmark run.
+ "store": False,
+ }
+ _update_payload_common(payload, request_func_input)
+
+ headers = _get_headers("application/json")
+ _update_headers_common(headers, request_func_input)
+
+ output = RequestFuncOutput()
+ output.prompt_len = request_func_input.prompt_len
+
+ # Only output text is accumulated. Reasoning text is timed but not
+ # returned, matching how `openai-chat` handles `DeltaMessage.reasoning`.
+ generated_text = ""
+ stream_error: str | None = None
+ saw_terminal_event = False
+ first_chunk_received = False
+ st = time.perf_counter()
+ output.start_time = st
+ most_recent_timestamp = st
+ try:
+ async with session.post(url=api_url, json=payload, headers=headers) as response:
+ if response.status == 200:
+ handler = StreamedResponseHandler()
+
+ async for chunk_bytes in response.content.iter_any():
+ for message in handler.add_chunk(chunk_bytes):
+ # NOTE: `_extract_sse_data` returns None for SSE
+ # comments, which servers may send as keep-alives.
+ chunk = _extract_sse_data(message)
+ if not chunk or chunk == "[DONE]":
+ continue
+
+ timestamp = time.perf_counter()
+ data = json.loads(chunk)
+ event_type = data.get("type")
+
+ if event_type in _RESPONSES_TOKEN_DELTA_EVENTS:
+ # First token
+ if not first_chunk_received:
+ first_chunk_received = True
+ output.ttft = timestamp - st
+
+ # Decoding phase
+ else:
+ output.itl.append(timestamp - most_recent_timestamp)
+
+ if event_type == "response.output_text.delta":
+ generated_text += data.get("delta") or ""
+
+ # Only token events advance the end of the
+ # request, so `latency - ttft` stays equal to
+ # `sum(itl)` and TPOT means what serve.py says.
+ most_recent_timestamp = timestamp
+ elif event_type in _RESPONSES_TERMINAL_EVENTS:
+ saw_terminal_event = True
+ usage = (data.get("response") or {}).get("usage") or {}
+ if (n_output := usage.get("output_tokens")) is not None:
+ output.output_tokens = n_output
+ if (n_input := usage.get("input_tokens")) is not None:
+ output.prompt_len = n_input
+ elif event_type in _RESPONSES_ERROR_EVENTS or data.get("error"):
+ stream_error = json.dumps(data.get("error") or data)
+
+ output.generated_text = generated_text
+ output.latency = most_recent_timestamp - st
+ if stream_error is not None:
+ output.success = False
+ output.error = stream_error
+ elif not first_chunk_received:
+ output.success = False
+ output.error = (
+ "Never received a token delta to calculate TTFT."
+ "This response will be marked as failed!"
+ )
+ elif not saw_terminal_event:
+ # Without the terminal event the response is truncated and
+ # its usage block is missing.
+ output.success = False
+ output.error = (
+ "Stream ended without a terminal event."
+ "This response will be marked as failed!"
+ )
+ else:
+ output.success = True
+ else:
+ output.error = response.reason or ""
+ output.success = False
+ except Exception:
+ output.success = False
+ exc_info = sys.exc_info()
+ output.error = "".join(traceback.format_exception(*exc_info))
+
+ if pbar:
+ pbar.update(1)
+ return output
+
+
async def async_request_openai_audio(
request_func_input: RequestFuncInput,
session: aiohttp.ClientSession,
@@ -920,6 +1092,7 @@ ASYNC_REQUEST_FUNCS: dict[str, RequestFunc] = {
"vllm": async_request_openai_completions,
"openai": async_request_openai_completions,
"openai-chat": async_request_openai_chat_completions,
+ "openai-responses": async_request_openai_responses,
"openai-audio": async_request_openai_audio,
"openai-embeddings": async_request_openai_embeddings,
"openai-embeddings-chat": async_request_openai_embeddings_chat,
@@ -947,5 +1120,10 @@ POOLING_BACKENDS = {
OPENAI_COMPATIBLE_BACKENDS = [
k
for k, v in ASYNC_REQUEST_FUNCS.items()
- if v in (async_request_openai_completions, async_request_openai_chat_completions)
+ if v
+ in (
+ async_request_openai_completions,
+ async_request_openai_chat_completions,
+ async_request_openai_responses,
+ )
]
diff --git a/vllm/benchmarks/serve.py b/vllm/benchmarks/serve.py
index d751f7f22d..19130716f4 100644
--- a/vllm/benchmarks/serve.py
+++ b/vllm/benchmarks/serve.py
@@ -2216,6 +2216,11 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
"Sampling parameters are only supported by openai-compatible backends."
)
+ # The Responses API accepts every sampling parameter above except
+ # min_p, which it would silently drop as an unknown field.
+ if args.backend == "openai-responses" and "min_p" in sampling_params:
+ raise ValueError("--min-p is not supported by the Responses API.")
+
if "temperature" not in sampling_params:
print(
"WARNING: vllm bench serve no longer sets temperature==0 (greedy) "
diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py
index c50edf4363..220da0c72b 100644
--- a/vllm/config/vllm.py
+++ b/vllm/config/vllm.py
@@ -1718,6 +1718,26 @@ class VllmConfig:
)
self.compilation_config.mode = CompilationMode.NONE
+ # TODO: This is a stopgap that turns sequence parallelism / async TP
+ # off under batch invariance. Make the sequence-parallel reduce-scatter
+ # path batch-invariant instead so they can stay enabled (#56370).
+ # Sequence parallelism / async TP rewrite all_reduce + rms_norm into a
+ # reduce-scatter whose reduction order depends on the batch, so they
+ # are not batch-invariant. Decide this before the breakable-CUDA-graph
+ # default below, which declines to auto-enable when they are on.
+ pass_config = self.compilation_config.pass_config
+ if envs.VLLM_BATCH_INVARIANT and (
+ pass_config.enable_sp or pass_config.fuse_gemm_comms
+ ):
+ logger.warning_once(
+ "Disabling sequence parallelism and async TP "
+ "(pass_config.enable_sp / fuse_gemm_comms) when "
+ "VLLM_BATCH_INVARIANT is enabled: the reduce-scatter path "
+ "is not batch-invariant (see vllm-project/vllm#56370)."
+ )
+ pass_config.enable_sp = False
+ pass_config.fuse_gemm_comms = False
+
breakable_cudagraph_enabled = self._maybe_enable_breakable_cudagraph()
if not breakable_cudagraph_enabled and (
diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/base.py b/vllm/distributed/kv_transfer/kv_connector/v1/base.py
index bc3c32ba4c..6df9d1cfbe 100644
--- a/vllm/distributed/kv_transfer/kv_connector/v1/base.py
+++ b/vllm/distributed/kv_transfer/kv_connector/v1/base.py
@@ -42,6 +42,7 @@ The class provides the following primitives:
import enum
from abc import ABC, abstractmethod
from collections.abc import Callable, Iterable
+from contextlib import AbstractContextManager
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, Literal
@@ -227,6 +228,13 @@ class KVConnectorBase_V1(ABC):
# Worker-side methods
# ==============================
+ def get_mem_pool_context(self) -> AbstractContextManager | None:
+ """Return a custom KV cache allocation context, if configured.
+
+ Returning None uses the engine's default memory pool.
+ """
+ return None
+
def bind_connector_metadata(self, connector_metadata: KVConnectorMetadata) -> None:
"""Set the connector metadata from the scheduler.
diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py
index a9255f7cda..cbb7339c91 100644
--- a/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py
+++ b/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py
@@ -59,7 +59,6 @@ from vllm.platforms import current_platform
from vllm.utils.math_utils import cdiv
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.kv_cache_utils import (
- dcp_world_size_for_kv_cache_spec,
resolve_dcp_kv_block_size,
)
from vllm.v1.kv_cache_interface import (
@@ -259,13 +258,7 @@ class DecodeBenchConnectorScheduler:
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
self.kv_cache_groups = kv_cache_config.kv_cache_groups
self.group_block_sizes = tuple(
- resolve_dcp_kv_block_size(
- group.kv_cache_spec,
- dcp_world_size_for_kv_cache_spec(
- group.kv_cache_spec,
- dcp_world_size,
- ),
- )
+ resolve_dcp_kv_block_size(group.kv_cache_spec, dcp_world_size)
for group in self.kv_cache_groups
)
diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py
index beabf157cc..5ef4f8f670 100644
--- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py
+++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py
@@ -10,7 +10,7 @@ from collections.abc import Collection
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from enum import IntEnum
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Final
import msgspec
import numpy as np
@@ -70,6 +70,8 @@ from vllm.v1.worker.utils import select_common_block_size
logger = init_logger(__name__)
+_BOOTSTRAP_MAX_ATTEMPTS: Final[int] = 3
+
try:
from mooncake.engine import TransferEngine
except ImportError:
@@ -1147,24 +1149,44 @@ class MooncakeConnectorWorker:
pp_rank=self.pp_rank,
addr=worker_addr,
)
- while True:
+ timeout = envs.VLLM_MOONCAKE_CONNECTOR_TIMEOUT
+ max_attempts = _BOOTSTRAP_MAX_ATTEMPTS
+ backoff = 1.0
+ for attempt in range(1, max_attempts + 1):
try:
- async with httpx.AsyncClient() as client:
+ async with httpx.AsyncClient(timeout=timeout) as client:
response = await client.post(url, json=payload.model_dump())
response.raise_for_status()
logger.debug("Successfully registered with bootstrap server at %s", url)
- break
- except httpx.ConnectError:
- # Bootstrap server not ready, wait for a while and retry.
- await asyncio.sleep(1)
+ return
+ except (httpx.ConnectError, httpx.TimeoutException) as e:
+ # Rank 0 may be busy registering a large memory segment.
+ if attempt == max_attempts:
+ break
+ logger.warning(
+ "Bootstrap registration attempt %d/%d for %s failed with %s: %s. "
+ "Retrying in %.1fs.",
+ attempt,
+ max_attempts,
+ payload,
+ type(e).__name__,
+ e,
+ backoff,
+ )
+ await asyncio.sleep(backoff)
+ backoff = min(backoff * 2, 10.0)
except Exception as e:
err_msg = (
e.response.text if isinstance(e, httpx.HTTPStatusError) else str(e)
)
- logger.error(
+ logger.exception(
"Error registering %s with bootstrap server: %s", payload, err_msg
)
- raise e
+ raise
+ raise RuntimeError(
+ f"Failed to register {payload} with the Mooncake bootstrap server at "
+ f"{url} after {max_attempts} attempts."
+ )
async def _mooncake_sender_listener(self, ready_event: threading.Event):
"""Background thread that listens for Mooncake requests, dispatches them
@@ -1800,11 +1822,29 @@ class MooncakeConnectorWorker:
if self.is_kv_consumer:
return
+ # httpx applies the timeout to each phase (connect, read, write, pool)
+ # rather than to the whole request, so budget for all four.
+ register_timeout = envs.VLLM_MOONCAKE_CONNECTOR_TIMEOUT
+ max_attempts = _BOOTSTRAP_MAX_ATTEMPTS
+ backoff_total = sum(min(2.0**i, 10.0) for i in range(max_attempts - 1))
+ ready_timeout = max_attempts * 4 * register_timeout + backoff_total + 30.0
ready_event = threading.Event()
- asyncio.run_coroutine_threadsafe(
+ fut = asyncio.run_coroutine_threadsafe(
self._mooncake_sender_listener(ready_event), self.sender_loop
)
- ready_event.wait() # Wait for listener ZMQ socket to be ready.
+ deadline = time.monotonic() + ready_timeout
+ while not ready_event.wait(timeout=1.0):
+ if fut.done():
+ fut.result()
+ raise RuntimeError(
+ "Mooncake sender listener exited before becoming ready."
+ )
+ if time.monotonic() > deadline:
+ fut.cancel()
+ raise RuntimeError(
+ "Mooncake sender listener did not become ready within "
+ f"{ready_timeout:.0f}s."
+ )
async def fetch_finished_recving_reqs(self) -> set[ReqId]:
finished_recving_reqs = self.finished_recving_reqs
diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_utils.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_utils.py
index df47f9fdb7..dfe0641b36 100644
--- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_utils.py
+++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_utils.py
@@ -97,14 +97,19 @@ class MooncakeBootstrapServer:
dp_entry.worker_addr[payload.tp_rank] = {}
tp_entry = dp_entry.worker_addr[payload.tp_rank]
- if payload.pp_rank in tp_entry:
+ existing = tp_entry.get(payload.pp_rank)
+ if existing is not None:
+ # A client timeout can fire after the server recorded the
+ # registration so an identical retry must not be an error.
+ if existing == payload.addr:
+ return {"status": "ok"}
raise HTTPException(
status_code=400,
detail=(
f"Worker with dp_rank={payload.dp_rank}, "
f"tp_rank={payload.tp_rank}, pp_rank={payload.pp_rank} "
f"is already registered at "
- f"{tp_entry[payload.pp_rank]}, "
+ f"{existing}, "
f"but still want to register at {payload.addr}"
),
)
diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py
index eb4a7fafe9..dd64dd671e 100644
--- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py
+++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py
@@ -13,6 +13,7 @@ enabling prefix caching via hash-based deduplication.
from collections import Counter
from collections.abc import Iterable, Sequence
+from contextlib import AbstractContextManager
from typing import Any
import torch
@@ -322,6 +323,17 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
# Worker-side methods
# ============================================================
+ def get_mem_pool_context(self) -> AbstractContextManager | None:
+ """Return a context manager for the custom MemPool, or None.
+
+ Called by the Worker before ``initialize_kv_cache`` so that KV
+ cache is allocated from the Mooncake-managed pool when
+ ``custom_mem_pool`` is set in ``kv_connector_extra_config``.
+ """
+ if self.connector_worker is None:
+ return None
+ return self.connector_worker.get_mem_pool_context()
+
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
assert self.connector_worker is not None
self.connector_worker.register_kv_caches(kv_caches)
diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py
index baf8967bc8..173aa324e7 100644
--- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py
+++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py
@@ -19,13 +19,14 @@ import threading
import time
from collections.abc import Callable, Sequence
from concurrent.futures import Future, ThreadPoolExecutor
+from contextlib import AbstractContextManager
from typing import Any, TypeVar
import torch
import zmq
import vllm.envs as envs
-from vllm.config import VllmConfig
+from vllm.config import ModelConfig, VllmConfig
from vllm.distributed import (
get_dcp_group,
get_pcp_group,
@@ -1414,6 +1415,41 @@ class KVCacheStoreRecvingThread(KVTransferThread):
class MooncakeStoreWorker:
"""Worker-side component for MooncakeStoreConnector."""
+ @staticmethod
+ def _create_mem_pool(
+ extra_config: dict[str, Any], model_config: ModelConfig
+ ) -> torch.cuda.MemPool | None:
+ pool_type = str(extra_config.get("custom_mem_pool") or "").upper()
+ if not pool_type:
+ return None
+ if pool_type not in ("NVLINK", "BAREX"):
+ raise ValueError(
+ f"Unsupported custom_mem_pool={pool_type!r}, "
+ "expected 'NVLINK' or 'BAREX'"
+ )
+ if model_config.enable_sleep_mode or model_config.enable_cumem_allocator:
+ raise ValueError(
+ "custom_mem_pool is incompatible with enable_sleep_mode "
+ "or enable_cumem_allocator; CuMemAllocator cannot manage "
+ "allocations from the custom pool."
+ )
+ try:
+ if pool_type == "NVLINK":
+ from mooncake.allocator import NVLinkAllocator as allocator_cls
+ else: # pool_type == "BAREX"
+ from mooncake.allocator import BarexAllocator as allocator_cls
+ except ImportError as e:
+ raise ImportError(
+ f"custom_mem_pool={pool_type!r} requires "
+ "mooncake-transfer-engine>=0.3.8. Please upgrade Mooncake."
+ ) from e
+
+ device = torch.device("cuda", torch.accelerator.current_device_index())
+ allocator = allocator_cls.get_allocator(device)
+ mem_pool = torch.cuda.MemPool(allocator.allocator())
+ logger.info("Using Mooncake custom memory pool: %s", pool_type)
+ return mem_pool
+
def __init__(
self,
vllm_config: VllmConfig,
@@ -1471,6 +1507,10 @@ class MooncakeStoreWorker:
# Initialize MooncakeDistributedStore with its own TransferEngine
store_config = MooncakeStoreConfig.load_from_config()
+ # Validate and create the custom memory pool before initializing the
+ # store so failures cannot leak a partially initialized store handle.
+ self._mem_pool = self._create_mem_pool(extra_config, model_config)
+
self.store = MooncakeDistributedStore()
local_ip = get_ip()
local_hostname = rdma_utils.get_requester_local_hostname(local_ip)
@@ -1741,6 +1781,13 @@ class MooncakeStoreWorker:
)
return token_dbs
+ def get_mem_pool_context(self) -> AbstractContextManager | None:
+ """Return a context manager for the custom MemPool, or None if
+ no custom pool is configured."""
+ if self._mem_pool is None:
+ return None
+ return torch.cuda.use_mem_pool(self._mem_pool)
+
def _spec_tp_replication_factor(self, spec: KVCacheSpec) -> int:
if self.dcp_size > 1:
return 1
diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py
index a6f0d4cafb..73b6b0ccf6 100644
--- a/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py
+++ b/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
from collections.abc import Callable, Iterable
+from contextlib import AbstractContextManager
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, cast
@@ -294,6 +295,26 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
# ==============================
# Worker-side methods
# ==============================
+ def get_mem_pool_context(self) -> AbstractContextManager | None:
+ """Forward a custom KV cache memory pool from a child connector.
+
+ KV cache is allocated once and can only live in one pool, so at
+ most one child may provide a context.
+ """
+ found: list[tuple[str, AbstractContextManager]] = []
+ for connector in self._connectors:
+ if (ctx := connector.get_mem_pool_context()) is not None:
+ found.append((type(connector).__name__, ctx))
+
+ if len(found) > 1:
+ names = [name for name, _ in found]
+ raise ValueError(
+ f"Multiple connectors provide a KV cache memory pool {names}; "
+ "KV cache is allocated once and can only live in one pool. "
+ "Configure custom_mem_pool on at most one connector."
+ )
+ return found[0][1] if found else None
+
def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None:
for c in self._connectors:
c.start_load_kv(forward_context, **kwargs)
diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py
index b630dedfc3..f4a643d2b7 100644
--- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py
+++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py
@@ -839,6 +839,8 @@ class NixlBaseConnectorWorker:
self._engine_ttl: float = vllm_config.kv_transfer_config.get_from_extra_config(
"engine_ttl", 3600.0
)
+ # Map of pull peer (host, port) -> engine_id currently serving it.
+ self._engine_by_address: dict[tuple[str, int], EngineId] = {}
self.model_config = vllm_config.model_config
@@ -1300,6 +1302,7 @@ class NixlBaseConnectorWorker:
self._remote_agents[eid] = remote_agents
self._engine_clock_offset[eid] = clock_offset
self._engine_last_active[eid] = time.perf_counter()
+ self._track_remote_engine_replacement(eid, host, port)
except Exception as e:
self._log_failure(
failure_type="handshake_setup_failed",
@@ -2953,6 +2956,7 @@ class NixlBaseConnectorWorker:
# Handle timeout to avoid stranding blocks on remote.
self._reap_expired_send_leases(done_sending)
+ self._cleanup_replaced_remote_engines()
return KVConnectorTransferResults(
finished_sending=done_sending,
finished_recving=done_recving,
@@ -3541,14 +3545,53 @@ class NixlBaseConnectorWorker:
and meta.remote is not None
}
+ def _track_remote_engine_replacement(
+ self, engine_id: EngineId, host: str, port: int
+ ) -> None:
+ """Record the engine currently serving a pull peer address."""
+ # TODO: Also handle push mode, which handshakes in both directions.
+ if self._TRANSFER_MODE != "pull":
+ return
+ previous = self._engine_by_address.get((host, port))
+ self._engine_by_address[(host, port)] = engine_id
+ if previous is None or previous == engine_id:
+ return
+ logger.info(
+ "Remote engine at %s:%d changed from %s to %s.",
+ host,
+ port,
+ previous,
+ engine_id,
+ )
+
+ def _cleanup_replaced_remote_engines(self) -> None:
+ """Release replaced pull peers once requests and handshakes have drained."""
+ if self._TRANSFER_MODE != "pull":
+ return
+ with self._handshake_lock:
+ if self._handshake_futures:
+ return
+ replaced = self._remote_agents.keys() - self._engine_by_address.values()
+ if not replaced:
+ return
+ # _recving_metadata is only accessed from this (main) thread.
+ busy = {
+ meta.remote.engine_id
+ for meta in self._recving_metadata.values()
+ if meta.remote is not None
+ }
+ for engine_id in replaced - busy:
+ self._cleanup_remote_engine(engine_id, log_eviction=False)
+ logger.info("Released NIXL state for replaced remote engine %s.", engine_id)
+
def _cleanup_remote_engine(
self, engine_id: EngineId, *, log_eviction: bool = True
) -> None:
"""Remove all state for a single remote engine.
Releases NIXL resources (dlist handles, remote agents) and clears
- all per-engine data structures. Used by both TTL eviction and
- shutdown.
+ all per-engine data structures. Used by TTL eviction, replaced peer
+ cleanup and shutdown.
"""
assert engine_id in self._remote_agents
@@ -3558,6 +3601,9 @@ class NixlBaseConnectorWorker:
# Pop under the handshake lock; NIXL teardown stays outside it.
with self._handshake_lock:
agents = self._remote_agents.pop(engine_id)
+ for address, eid in list(self._engine_by_address.items()):
+ if eid == engine_id:
+ del self._engine_by_address[address]
for agent_name in agents.values():
self.nixl_wrapper.remove_remote_agent(agent_name)
diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py
index 6539b7e57e..11b7215950 100644
--- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py
+++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py
@@ -84,7 +84,14 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
# Start transfers for requests whose handshakes have now finished.
while not self._ready_requests.empty():
- self._read_blocks_for_req(*self._ready_requests.get_nowait())
+ req_id, meta = self._ready_requests.get_nowait()
+ assert meta.remote is not None
+ if meta.remote.engine_id not in self._remote_agents:
+ # The engine was released after its handshake completed, so
+ # handshake again. This fails if the engine is gone.
+ self._background_nixl_handshake(req_id, meta.remote.engine_id, meta)
+ continue
+ self._read_blocks_for_req(req_id, meta)
if self.pcp_rank > 0 and not self.pcp_dcp_sharded:
# Replicated-KV PCP: only PCP rank 0 serves the KV, so this rank
diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py
index 64bf8146c9..9124d9eac1 100644
--- a/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py
+++ b/vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py
@@ -15,6 +15,12 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorRole,
SupportsHMA,
)
+from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
+ KVConnectorPromMetrics,
+ KVConnectorStats,
+ PromMetric,
+ PromMetricT,
+)
from vllm.logger import init_logger
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.outputs import KVConnectorOutput
@@ -25,6 +31,10 @@ from vllm.v1.simple_kv_offload.manager import (
from vllm.v1.simple_kv_offload.metadata import (
SimpleCPUOffloadMetadata,
)
+from vllm.v1.simple_kv_offload.metrics import (
+ SimpleCPUOffloadPromMetrics,
+ SimpleCPUOffloadStats,
+)
from vllm.v1.simple_kv_offload.worker import (
SimpleCPUOffloadWorker,
)
@@ -162,6 +172,7 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
hash_block_size=hash_block_size,
lazy_offload=lazy_offload,
disk_capacity_bytes=disk_capacity_bytes if disk_mode else 0,
+ use_page_cache=use_page_cache if disk_mode else False,
)
elif role == KVConnectorRole.WORKER:
self.worker_handler = SimpleCPUOffloadWorker(
@@ -307,6 +318,38 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
return self.scheduler_manager.get_boundary_store_stats()
return None
+ def get_kv_connector_stats(self) -> KVConnectorStats | None:
+ # Worker-side callers may invoke this hook on connectors constructed
+ # via __new__ (as in test_worker.py), which lack scheduler_manager.
+ scheduler_manager: SimpleCPUOffloadScheduler | None = getattr(
+ self, "scheduler_manager", None
+ )
+ if scheduler_manager is not None:
+ return scheduler_manager.get_stats()
+ return None
+
+ @classmethod
+ def build_kv_connector_stats(
+ cls, data: dict[str, Any] | None = None
+ ) -> KVConnectorStats | None:
+ return (
+ SimpleCPUOffloadStats(data=data)
+ if data is not None
+ else SimpleCPUOffloadStats()
+ )
+
+ @classmethod
+ def build_prom_metrics(
+ cls,
+ vllm_config: VllmConfig,
+ metric_types: dict[type[PromMetric], type[PromMetricT]],
+ labelnames: list[str],
+ per_engine_labelvalues: dict[int, list[object]],
+ ) -> KVConnectorPromMetrics:
+ return SimpleCPUOffloadPromMetrics(
+ vllm_config, metric_types, labelnames, per_engine_labelvalues
+ )
+
def take_events(self) -> Iterable[KVCacheEvent]:
if self.scheduler_manager is not None:
return self.scheduler_manager.take_events()
diff --git a/vllm/distributed/parallel_state.py b/vllm/distributed/parallel_state.py
index 20118649e1..88bb3d7c5a 100644
--- a/vllm/distributed/parallel_state.py
+++ b/vllm/distributed/parallel_state.py
@@ -1583,6 +1583,14 @@ def get_dcp_group() -> GroupCoordinator:
return _DCP
+def get_dcp_world_size_and_rank(enabled: bool = True) -> tuple[int, int]:
+ """Return ``(world_size, rank)`` in the DCP group, or ``(1, 0)`` when disabled
+ (e.g. a replicated draft cache) or the group is uninitialized (unit tests)."""
+ if not enabled or _DCP is None:
+ return 1, 0
+ return _DCP.world_size, _DCP.rank_in_group
+
+
_PP: GroupCoordinator | None = None
diff --git a/vllm/entrypoints/cli/preload.py b/vllm/entrypoints/cli/preload.py
index d312b6c36a..50734a052b 100644
--- a/vllm/entrypoints/cli/preload.py
+++ b/vllm/entrypoints/cli/preload.py
@@ -23,8 +23,14 @@ import multiprocessing
import queue
import signal
import sys
+import threading
import typing
+from collections.abc import Sequence
from itertools import product
+from multiprocessing.process import BaseProcess
+
+from fastapi import FastAPI
+from fastapi.responses import Response
from vllm.engine.arg_utils import EngineArgs
from vllm.entrypoints.cli.types import CLISubcommand
@@ -44,6 +50,55 @@ else:
logger = init_logger(__name__)
+class _HealthState:
+ def __init__(self, processes: Sequence[BaseProcess]) -> None:
+ self.processes = processes
+ self._ready = False
+ self._lock = threading.Lock()
+
+ def mark_ready(self) -> None:
+ with self._lock:
+ self._ready = True
+
+ @property
+ def is_ready(self) -> bool:
+ with self._lock:
+ ready = self._ready
+ return ready and not any(proc.exitcode is not None for proc in self.processes)
+
+
+def _build_health_app(state: _HealthState) -> FastAPI:
+ app = FastAPI(openapi_url=None, docs_url=None, redoc_url=None)
+
+ @app.get("/health", include_in_schema=False)
+ async def health() -> Response:
+ return Response(status_code=200 if state.is_ready else 503)
+
+ return app
+
+
+def _start_health_server(
+ host: str, port: int, state: _HealthState
+) -> tuple[typing.Any, threading.Thread]:
+ import uvicorn
+
+ config = uvicorn.Config(
+ _build_health_app(state),
+ host=host,
+ port=port,
+ log_level="error",
+ access_log=False,
+ )
+ server = uvicorn.Server(config)
+ thread = threading.Thread(
+ target=server.run,
+ name="vllm-weight-cache-health",
+ daemon=True,
+ )
+ thread.start()
+ return server, thread
+
+
class PreloadSubcommand(CLISubcommand):
"""The `preload` subcommand for the vLLM CLI."""
@@ -75,6 +130,18 @@ class PreloadSubcommand(CLISubcommand):
"--weight-cache-master-port + 1 for multi-node, or a free port "
"for single-node.",
)
+ parser.add_argument(
+ "--weight-cache-health-port",
+ type=int,
+ default=None,
+ help="Port for the parent /health endpoint; disabled by default.",
+ )
+ parser.add_argument(
+ "--weight-cache-health-host",
+ type=str,
+ default="0.0.0.0",
+ help="Host for the parent /health endpoint (default: 0.0.0.0).",
+ )
@staticmethod
def cmd(args: argparse.Namespace) -> None:
@@ -190,10 +257,21 @@ class PreloadSubcommand(CLISubcommand):
tp_rank,
) in product(groups, placements)
]
+ health_state = _HealthState(procs)
+ health_server = None
+ health_thread = None
+ if args.weight_cache_health_port is not None:
+ health_server, health_thread = _start_health_server(
+ args.weight_cache_health_host,
+ args.weight_cache_health_port,
+ health_state,
+ )
for proc in procs:
proc.start()
def _shutdown(signum, frame):
+ if health_server is not None:
+ health_server.should_exit = True
for proc in procs:
proc.terminate()
@@ -216,7 +294,12 @@ class PreloadSubcommand(CLISubcommand):
proc.terminate()
for proc in procs:
proc.join()
+ if health_server is not None:
+ health_server.should_exit = True
+ if health_thread is not None:
+ health_thread.join()
sys.exit(max((p.exitcode or 0) for p in procs))
+ health_state.mark_ready()
socket_dir_msg = args.weight_cache_socket_dir or "the default socket dir"
logger.info_once(
"===== Weight cache daemon READY: node %d/%d serving %d local "
@@ -230,6 +313,10 @@ class PreloadSubcommand(CLISubcommand):
for proc in procs:
proc.join()
+ if health_server is not None:
+ health_server.should_exit = True
+ if health_thread is not None:
+ health_thread.join()
sys.exit(max(proc.exitcode or 0 for proc in procs))
def subparser_init(
diff --git a/vllm/entrypoints/openai/chat_completion/batch_serving.py b/vllm/entrypoints/openai/chat_completion/batch_serving.py
index ea6601bff1..3fde958a3d 100644
--- a/vllm/entrypoints/openai/chat_completion/batch_serving.py
+++ b/vllm/entrypoints/openai/chat_completion/batch_serving.py
@@ -12,6 +12,7 @@ from vllm.entrypoints.chat_utils import ConversationMessage
from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata
from vllm.entrypoints.openai.chat_completion.protocol import (
BatchChatCompletionRequest,
+ ChatCompletionRequest,
ChatCompletionResponse,
ChatCompletionResponseChoice,
ChatMessage,
@@ -40,7 +41,14 @@ class OpenAIServingChatBatch(OpenAIServingChat):
async def render_batch_chat_request(
self,
request: BatchChatCompletionRequest,
- ) -> tuple[list[list[ConversationMessage]], list[EngineInput]] | ErrorResponse:
+ ) -> (
+ tuple[
+ list[ChatCompletionRequest],
+ list[list[ConversationMessage]],
+ list[EngineInput],
+ ]
+ | ErrorResponse
+ ):
"""Validate the model and preprocess a batched chat completion request.
Performs engine-aware checks then delegates per-conversation
@@ -48,8 +56,10 @@ class OpenAIServingChatBatch(OpenAIServingChat):
once for the whole batch.
Returns:
- A tuple of (all_conversations, engine_prompts) on success — one
- entry per conversation — or an ErrorResponse on failure.
+ A tuple of (single_requests, all_conversations, engine_prompts) on
+ success — one entry per conversation — or an ErrorResponse on
+ failure. The renderer adjusts ``single_requests`` in place, so
+ sampling must read them rather than rebuilding from ``request``.
"""
error_check_ret = await self._check_model(request)
@@ -74,12 +84,14 @@ class OpenAIServingChatBatch(OpenAIServingChat):
parser = renderer.parser
tool_dicts: list[dict] | None = None
+ single_requests: list[ChatCompletionRequest] = []
all_conversations: list[list[ConversationMessage]] = []
all_engine_prompts: list[EngineInput] = []
for messages in request.messages:
single_request = request.to_chat_completion_request(messages)
if renderer.use_harmony:
+ renderer.adjust_harmony_request(single_request)
conversation, engine_prompts = renderer._make_request_with_harmony(
single_request, should_include_tools=tool_dicts is not None
)
@@ -93,10 +105,11 @@ class OpenAIServingChatBatch(OpenAIServingChat):
tool_dicts=tool_dicts,
parser=parser,
)
+ single_requests.append(single_request)
all_conversations.append(conversation)
all_engine_prompts.append(engine_prompts[0])
- return all_conversations, all_engine_prompts
+ return single_requests, all_conversations, all_engine_prompts
async def create_batch_chat_completion(
self,
@@ -111,10 +124,10 @@ class OpenAIServingChatBatch(OpenAIServingChat):
"""
tokenizer = self.renderer.tokenizer
assert tokenizer is not None
- single_requests = [
- request.to_chat_completion_request(messages)
- for messages in request.messages
- ]
+ render_result = await self.render_batch_chat_request(request)
+ if isinstance(render_result, ErrorResponse):
+ return render_result
+ single_requests, all_conversations, engine_prompts = render_result
parser: Parser | None = None
if self.parser_cls is not None:
@@ -127,11 +140,6 @@ class OpenAIServingChatBatch(OpenAIServingChat):
chat_template_kwargs=chat_template_kwargs,
)
- render_result = await self.render_batch_chat_request(request)
- if isinstance(render_result, ErrorResponse):
- return render_result
- all_conversations, engine_prompts = render_result
-
request_id = (
f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
)
diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py
index be35415538..f22fed1e75 100644
--- a/vllm/entrypoints/openai/chat_completion/serving.py
+++ b/vllm/entrypoints/openai/chat_completion/serving.py
@@ -200,6 +200,20 @@ class OpenAIServingChat(GenerateBaseServing):
.chat_template_kwargs
)
+ def _make_parser(
+ self,
+ request: ChatCompletionRequest,
+ tokenizer: TokenizerLike,
+ chat_template_kwargs: dict[str, Any] | None,
+ ) -> Parser:
+ assert self.parser_cls is not None
+ return self.parser_cls(
+ tokenizer,
+ request.tools,
+ chat_template_kwargs=chat_template_kwargs,
+ model_config=self.model_config,
+ )
+
def _engine_chat_template_kwargs(
self, chat_template_kwargs: dict[str, Any]
) -> dict[str, Any]:
@@ -267,12 +281,7 @@ class OpenAIServingChat(GenerateBaseServing):
chat_template_kwargs = self._effective_chat_template_kwargs(request)
parser: Parser | None = None
if self.parser_cls is not None:
- parser = self.parser_cls(
- tokenizer,
- request.tools,
- chat_template_kwargs=chat_template_kwargs,
- model_config=self.model_config,
- )
+ parser = self._make_parser(request, tokenizer, chat_template_kwargs)
result = await self.render_chat_request(request)
if isinstance(result, ErrorResponse):
return result
@@ -490,12 +499,7 @@ class OpenAIServingChat(GenerateBaseServing):
"Tokenizer not available when `skip_tokenizer_init=True`"
)
parsers: list[Parser | None] = [
- self.parser_cls(
- tokenizer,
- request.tools,
- chat_template_kwargs=chat_template_kwargs,
- model_config=self.model_config,
- )
+ self._make_parser(request, tokenizer, chat_template_kwargs)
for _ in range(num_choices)
]
else:
@@ -961,10 +965,15 @@ class OpenAIServingChat(GenerateBaseServing):
tool_parser_cls = (
self.parser_cls.tool_parser_cls if self.parser_cls is not None else None
)
- for output in final_res.outputs:
+ for i, output in enumerate(final_res.outputs):
# check for error finish reason and raise GenerationError
# finish_reason='error' indicates a retryable request-level internal error
self._raise_if_error(output.finish_reason, request_id)
+ if i > 0 and parser is not None:
+ # Parsers are stateful: like streaming, use one per choice.
+ parser = self._make_parser(
+ request, tokenizer, self._effective_chat_template_kwargs(request)
+ )
token_ids = output.token_ids
out_logprobs = output.logprobs
diff --git a/vllm/entrypoints/openai/parser/harmony_utils.py b/vllm/entrypoints/openai/parser/harmony_utils.py
index 873b9e92bd..dc3aef4a95 100644
--- a/vllm/entrypoints/openai/parser/harmony_utils.py
+++ b/vllm/entrypoints/openai/parser/harmony_utils.py
@@ -449,12 +449,80 @@ def parse_chat_input_to_harmony_message(
def render_for_completion(messages: list[Message]) -> list[int]:
messages = auto_drop_analysis_messages(messages)
conversation = Conversation.from_messages(messages)
- token_ids = get_encoding().render_conversation_for_completion(
+ encoding = get_encoding()
+ token_ids = encoding.render_conversation_for_completion(
conversation,
Role.ASSISTANT,
config=RenderConversationConfig(auto_drop_analysis=False),
)
- return token_ids
+ return _use_legacy_tool_call_headers(token_ids)
+
+
+def _use_legacy_tool_call_headers(token_ids: list[int]) -> list[int]:
+ """Keep prior assistant tool calls in the format expected by GPT-OSS.
+
+ oss-harmony 0.0.10 changed a rendered tool-call header from::
+
+ assistant to=python<|channel|>commentary code
+
+ to::
+
+ assistant<|channel|>commentary to=python <|constrain|>code
+
+ GPT-OSS can echo the latter as a malformed header with the recipient in
+ both positions. Rewrite only that unambiguous token-level header shape,
+ while retaining oss-harmony's embedded vocabulary and parser.
+ """
+ encoding = get_encoding()
+ start = encoding.encode("<|start|>", allowed_special="all")[0]
+ channel_token = encoding.encode("<|channel|>", allowed_special="all")[0]
+ constrain = encoding.encode("<|constrain|>", allowed_special="all")[0]
+ message = encoding.encode("<|message|>", allowed_special="all")[0]
+
+ result = token_ids.copy()
+ index = 0
+ while index < len(result):
+ if result[index] != start:
+ index += 1
+ continue
+
+ try:
+ header_end = result.index(message, index + 1)
+ channel_index = result.index(channel_token, index + 1, header_end)
+ constrain_index = result.index(constrain, channel_index + 1, header_end)
+ except ValueError:
+ index += 1
+ continue
+
+ author = encoding.decode(result[index + 1 : channel_index])
+ channel_and_recipient = encoding.decode(
+ result[channel_index + 1 : constrain_index]
+ )
+ content_type = encoding.decode(result[constrain_index + 1 : header_end])
+ if (
+ author != "assistant"
+ or " to=" not in channel_and_recipient
+ or not channel_and_recipient.endswith(" ")
+ ):
+ index = header_end + 1
+ continue
+
+ channel, recipient = channel_and_recipient[:-1].split(" to=", 1)
+ if not channel or not recipient or not content_type:
+ index = header_end + 1
+ continue
+
+ legacy_header = (
+ [start]
+ + encoding.encode(f"assistant to={recipient}")
+ + [channel_token]
+ + encoding.encode(f"{channel} {content_type}")
+ + [message]
+ )
+ result[index : header_end + 1] = legacy_header
+ index += len(legacy_header)
+
+ return result
def get_streamable_parser_for_assistant() -> StreamableParser:
diff --git a/vllm/entrypoints/openai/responses/context.py b/vllm/entrypoints/openai/responses/context.py
index 59b42f96a3..fa233dcce1 100644
--- a/vllm/entrypoints/openai/responses/context.py
+++ b/vllm/entrypoints/openai/responses/context.py
@@ -359,6 +359,10 @@ class ParsableContext(ConversationContext):
enable_auto_tools=self.enable_auto_tools,
model_output_token_ids=completion.token_ids,
)
+ # Each round is a separate generation, so count it on its own ids.
+ self.num_reasoning_tokens += self.response_parser.count_reasoning_tokens(
+ completion.token_ids or []
+ )
if not self.request.include_reasoning:
reasoning = None
self.response_messages.extend(
diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py
index 9fba86b80c..f18c493259 100644
--- a/vllm/entrypoints/openai/responses/serving.py
+++ b/vllm/entrypoints/openai/responses/serving.py
@@ -832,18 +832,12 @@ class OpenAIServingResponses(GenerateBaseServing):
num_generated_tokens = context.num_output_tokens
num_cached_tokens = context.num_cached_tokens
num_reasoning_tokens = context.num_reasoning_tokens
- # For text-based reasoning parsers (e.g., ...),
- # HarmonyContext already counts reasoning tokens via channels.
- # For Simple/Parsable contexts, derive reasoning_tokens from
- # accumulated output token IDs using the parser if not already set.
- if (
- num_reasoning_tokens == 0
- and isinstance(context, (SimpleContext, ParsableContext))
- and context.response_parser is not None
- ):
- accumulated = getattr(context, "_accumulated_token_ids", []) or []
+ # HarmonyContext and ParsableContext count reasoning tokens as each
+ # round is appended. SimpleContext is single-round but accumulates
+ # streaming deltas, so count its full output once here.
+ if isinstance(context, SimpleContext) and context.response_parser is not None:
num_reasoning_tokens = context.response_parser.count_reasoning_tokens(
- accumulated
+ context._accumulated_token_ids
)
usage = ResponseUsage(
diff --git a/vllm/envs.py b/vllm/envs.py
index 78f99ea0ac..f663414a91 100755
--- a/vllm/envs.py
+++ b/vllm/envs.py
@@ -141,7 +141,7 @@ if TYPE_CHECKING:
VLLM_ROCM_USE_AITER_LINEAR_HIPBMM: bool = False
VLLM_ROCM_USE_AITER_MOE: bool = True
VLLM_ROCM_AITER_MOE_DISPATCH_POLICY: int = 0
- VLLM_ROCM_USE_AITER_MOE_SITUV2: bool = False
+ VLLM_ROCM_USE_AITER_MOE_SITUV2: Literal["auto", "a4w4", "a8w4", "a16w4"] = "auto"
VLLM_ROCM_USE_AITER_RMSNORM: bool = True
VLLM_ROCM_USE_AITER_MLA: bool = True
VLLM_ROCM_AITER_MLA_ASM_PADDING: Literal["auto", "gluon", "asm"] = "auto"
@@ -260,6 +260,7 @@ if TYPE_CHECKING:
VLLM_ROCM_QUICK_REDUCE_MIN_SIZE_BYTES_MB: int | None = None
VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB: int | None = None
VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT: int = 480
+ VLLM_MOONCAKE_CONNECTOR_TIMEOUT: float = 30.0
VLLM_ENABLE_CUDAGRAPH_GC: bool = False
VLLM_LOOPBACK_IP: str = ""
VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE: bool = True
@@ -1280,18 +1281,13 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_ROCM_USE_AITER_MOE": lambda: (
os.getenv("VLLM_ROCM_USE_AITER_MOE", "True").lower() in ("true", "1")
),
- # Route K3 SiTU MXFP4 MoE through the FlyDSL SiTUv2 path (a4w4 fp4
- # activations, separated gate/up layout) instead of default a16w4. vLLM
- # sets AITER_SITUV2_A4W4 at init when this flag is on and clears any
- # legacy AITER_SITUV2_A8W4 override (AITER checks A8W4 first).
- # VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4 is a deprecated alias for existing
- # recipes; it does not select a8w4 kernels.
- # Needs AITER >= v0.1.20 (ROCm/aiter#4463) for the a4w4 dispatch flag
- # and tuned kimik3_a4w4_*_fmoe.csv rows; otherwise FlyDSL uses heuristics.
- "VLLM_ROCM_USE_AITER_MOE_SITUV2": lambda: (
- os.getenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "0").lower() in ("true", "1")
- or os.getenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", "0").lower()
- in ("true", "1")
+ # Activation dtype for the Kimi-K3 SiTU MXFP4 MoE (AITER FlyDSL SiTUv2):
+ # auto (= a4w4), a4w4, a8w4 or a16w4. Legacy 1/0 mean a4w4/a16w4.
+ "VLLM_ROCM_USE_AITER_MOE_SITUV2": env_with_choices(
+ "VLLM_ROCM_USE_AITER_MOE_SITUV2",
+ "auto",
+ ["auto", "a4w4", "a8w4", "a16w4", "0", "1"],
+ case_sensitive=False,
),
# MoE sorting dispatch policy for AITER fused MoE kernels.
# 0 = auto (default): single-pass for small batches, multi-pass
@@ -1718,6 +1714,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_EC_SIDE_CHANNEL_PORT": lambda: int(
os.getenv("VLLM_EC_SIDE_CHANNEL_PORT", "5601")
),
+ # Per-request timeout (seconds) when a prefiller worker registers with
+ # the Mooncake bootstrap server.
+ "VLLM_MOONCAKE_CONNECTOR_TIMEOUT": lambda: float(
+ os.getenv("VLLM_MOONCAKE_CONNECTOR_TIMEOUT", "30.0")
+ ),
# Port used for Mooncake handshake between remote agents.
"VLLM_MOONCAKE_BOOTSTRAP_PORT": lambda: int(
os.getenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", "8998")
diff --git a/vllm/model_executor/layers/fused_moe/config.py b/vllm/model_executor/layers/fused_moe/config.py
index 9221e93824..11e2a57520 100644
--- a/vllm/model_executor/layers/fused_moe/config.py
+++ b/vllm/model_executor/layers/fused_moe/config.py
@@ -118,7 +118,7 @@ class RoutingMethodType(IntEnum):
# SigmoidRenorm: Sigmoid -> TopK -> Renormalize (divide by sum of top-K)
SigmoidRenorm = (6,)
# MiniMax2: Sigmoid + Bias -> TopK -> ScaledSumNormalize
- # (routeScale=1.0, epsilon=1e-20)
+ # (routeScale=routed_scaling_factor, epsilon=1e-20)
MiniMax2 = (7,)
# Sigmoid: Sigmoid -> TopK (no renormalization)
Sigmoid = (8,)
@@ -137,7 +137,6 @@ def get_routing_method_type(
renormalize: bool,
num_expert_group: int | None,
has_e_score_bias: bool,
- routed_scaling_factor: float | None = 1.0,
) -> RoutingMethodType:
if scoring_func == "sqrtsoftplus":
# DeepSeek V4 uses sqrtsoftplus routing with optional routing bias
@@ -153,9 +152,7 @@ def get_routing_method_type(
return RoutingMethodType.Unspecified
if (num_expert_group or 0) > 0:
return RoutingMethodType.DeepSeekV3
- if routed_scaling_factor in (None, 1.0):
- return RoutingMethodType.MiniMax2
- return RoutingMethodType.Unspecified
+ return RoutingMethodType.MiniMax2
else:
return RoutingMethodType.Unspecified
diff --git a/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a16_moe.py b/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a16_moe.py
index fb3256e1a4..13c34698c8 100644
--- a/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a16_moe.py
+++ b/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a16_moe.py
@@ -139,7 +139,7 @@ def aiter_triton_kernel_w4a16_moe_forward(
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
should_use_cdna4_mx_scale_swizzle,
)
- from vllm.platforms.rocm import on_gfx1250
+ from vllm.platforms.rocm import on_gfx950, on_gfx1250
try:
from aiter.ops.triton.moe.moe_op_gemm_a16w4 import moe_gemm_a16w4
@@ -236,6 +236,12 @@ def aiter_triton_kernel_w4a16_moe_forward(
# kernel indexes a swizzled buffer as if it were linear.
swz = "CDNA4_SCALE" if should_use_cdna4_mx_scale_swizzle() else None
+ # AITER's gfx950 Gluon a16w4 kernel (default backend="None"/auto-detect
+ # picks it) calls a Triton-Gluon AMD API that isn't available in vLLM's
+ # pinned Triton, so force the Triton backend on gfx950. gfx1250 already
+ # used Gluon before this and is unaffected, so leave it on auto-detect.
+ a16w4_backend = "triton" if on_gfx950() else None
+
intermediate = moe_gemm_a16w4(
hidden_states,
w1_data,
@@ -254,6 +260,7 @@ def aiter_triton_kernel_w4a16_moe_forward(
swiglu_add_residual=swiglu_add_residual,
unpadded_N=unpadded_N_w1,
unpadded_K=unpadded_K_w1,
+ backend=a16w4_backend,
)
out = moe_gemm_a16w4(
@@ -270,6 +277,7 @@ def aiter_triton_kernel_w4a16_moe_forward(
swizzle_mx_scale=swz,
unpadded_N=unpadded_N_w2,
unpadded_K=unpadded_K_w2,
+ backend=a16w4_backend,
)
return out
diff --git a/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py b/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py
index 62df395f69..4f2f760ad3 100644
--- a/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py
+++ b/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py
@@ -385,10 +385,11 @@ def rocm_aiter_fused_experts(
gate_mode = ""
if activation == MoEActivation.SITU:
- # SiTUv2 flydsl (VLLM_ROCM_USE_AITER_MOE_SITUV2=1) uses a4w4
- # fp4 activations with separated gate/up weights (AITER #4463);
- # default a16w4 SiTU also stays separated.
- gate_mode = GateMode.SEPARATED.value
+ gate_mode = (
+ GateMode.INTERLEAVE.value
+ if rocm_aiter_ops.is_fused_moe_situv2_gate_up_interleaved()
+ else GateMode.SEPARATED.value
+ )
elif quant_config.use_mxfp4_w4a16:
gate_mode = GateMode.INTERLEAVE.value
elif activation_interleave is not None:
diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py
index 5450f73f85..56ba3981b7 100644
--- a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py
+++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py
@@ -1596,8 +1596,7 @@ def convert_weight_to_mxfp4_moe_kernel_format(
fp4_dtype = torch.float4_e2m1fn_x2
e8m0_dtype = torch.float8_e8m0fnu
- # SiTUv2 flydsl uses separated gate/up layout (a4w4).
- guinterleave = False
+ guinterleave = rocm_aiter_ops.is_fused_moe_situv2_gate_up_interleaved()
w13 = rocm_aiter_ops.shuffle_weight_a16w4(
w13_weight.data.view(fp4_dtype), 16, guinterleave
)
diff --git a/vllm/model_executor/layers/fused_moe/router/cpu_router.py b/vllm/model_executor/layers/fused_moe/router/cpu_router.py
index 1f96e9107a..3d034efdf0 100644
--- a/vllm/model_executor/layers/fused_moe/router/cpu_router.py
+++ b/vllm/model_executor/layers/fused_moe/router/cpu_router.py
@@ -198,7 +198,6 @@ class CPURouter(BaseRouter):
renormalize=self.renormalize,
num_expert_group=self.num_expert_group,
has_e_score_bias=self.e_score_correction_bias is not None,
- routed_scaling_factor=self.routed_scaling_factor,
)
def _compute_routing(
diff --git a/vllm/model_executor/layers/fused_moe/router/fused_topk_bias_router.py b/vllm/model_executor/layers/fused_moe/router/fused_topk_bias_router.py
index 257ede27d5..2df66d063a 100644
--- a/vllm/model_executor/layers/fused_moe/router/fused_topk_bias_router.py
+++ b/vllm/model_executor/layers/fused_moe/router/fused_topk_bias_router.py
@@ -391,7 +391,6 @@ class FusedTopKBiasRouter(BaseRouter):
renormalize=self.renormalize,
num_expert_group=None,
has_e_score_bias=True,
- routed_scaling_factor=self.routed_scaling_factor,
)
def _compute_routing(
diff --git a/vllm/model_executor/layers/fused_moe/router/grouped_topk_router.py b/vllm/model_executor/layers/fused_moe/router/grouped_topk_router.py
index 9b134d385a..64a12e57cd 100644
--- a/vllm/model_executor/layers/fused_moe/router/grouped_topk_router.py
+++ b/vllm/model_executor/layers/fused_moe/router/grouped_topk_router.py
@@ -283,7 +283,6 @@ class GroupedTopKRouter(BaseRouter):
renormalize=self.renormalize,
num_expert_group=self.num_expert_group,
has_e_score_bias=self.e_score_correction_bias is not None,
- routed_scaling_factor=self.routed_scaling_factor,
)
def _compute_routing(
diff --git a/vllm/model_executor/layers/fused_moe/router/router_factory.py b/vllm/model_executor/layers/fused_moe/router/router_factory.py
index f03fb106f7..96ccc853b8 100644
--- a/vllm/model_executor/layers/fused_moe/router/router_factory.py
+++ b/vllm/model_executor/layers/fused_moe/router/router_factory.py
@@ -188,7 +188,6 @@ def create_fused_moe_router(
renormalize=renormalize,
num_expert_group=groups,
has_e_score_bias=e_score_correction_bias is not None,
- routed_scaling_factor=routed_scaling_factor,
)
routing_method_preserved = advertised_routing_method(
diff --git a/vllm/model_executor/layers/fused_moe/router/zero_expert_router.py b/vllm/model_executor/layers/fused_moe/router/zero_expert_router.py
index 0107f88156..4e4a562c2b 100644
--- a/vllm/model_executor/layers/fused_moe/router/zero_expert_router.py
+++ b/vllm/model_executor/layers/fused_moe/router/zero_expert_router.py
@@ -4,10 +4,7 @@
import torch
from vllm.distributed.eplb.eplb_state import EplbLayerState
-from vllm.model_executor.layers.fused_moe.config import (
- RoutingMethodType,
- get_routing_method_type,
-)
+from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
from vllm.model_executor.layers.fused_moe.fused_moe import (
zero_experts_compute_triton,
)
@@ -53,14 +50,9 @@ class ZeroExpertRouter(BaseRouter):
@property
def routing_method_type(self) -> RoutingMethodType:
- return get_routing_method_type(
- scoring_func=self.scoring_func,
- top_k=self.top_k,
- renormalize=self.renormalize,
- num_expert_group=None,
- has_e_score_bias=True,
- routed_scaling_factor=self.routed_scaling_factor,
- )
+ # Zero experts are resolved by this router, which kernels with
+ # built-in routing (e.g. FlashInfer TRT-LLM) cannot reproduce.
+ return RoutingMethodType.Unspecified
def _compute_routing(
self,
diff --git a/vllm/model_executor/layers/rotary_embedding/packed_qk_rope.py b/vllm/model_executor/layers/rotary_embedding/packed_qk_rope.py
new file mode 100644
index 0000000000..e7d916e48f
--- /dev/null
+++ b/vllm/model_executor/layers/rotary_embedding/packed_qk_rope.py
@@ -0,0 +1,81 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""In-place interleaved RoPE on the Q/K slices of a packed QKV buffer."""
+
+import torch
+
+from vllm.triton_utils import tl, triton
+
+
+# seqlen is not specialized: image sizes vary per request.
+@triton.jit(do_not_specialize=["seqlen"])
+def _packed_qk_rope_kernel(
+ qk_ptr, # (seqlen, 2 * nheads, headdim) strided view of the packed QKV
+ freqs_ptr, # (seqlen, rotary_dim) fp32 re/im view of the complex freqs
+ seqlen,
+ qk_row_stride,
+ qk_head_stride,
+ rotary_dim: tl.constexpr,
+ BLOCK_M: tl.constexpr,
+):
+ BLOCK_K: tl.constexpr = triton.next_power_of_2(rotary_dim)
+ pid_m = tl.program_id(0)
+ pid_h = tl.program_id(1)
+ rm = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
+ rk = tl.arange(0, BLOCK_K)
+ mask_m = rm < seqlen
+
+ base = qk_ptr + pid_h * qk_head_stride + rm[:, None] * qk_row_stride
+ mask_x = mask_m[:, None] & (rk < rotary_dim)[None, :]
+ x = tl.load(base + rk[None, :], mask=mask_x, other=0.0).to(tl.float32)
+ f = tl.load(freqs_ptr + rm[:, None] * rotary_dim + rk[None, :], mask=mask_x)
+ cos, sin = tl.split(tl.reshape(f, (BLOCK_M, BLOCK_K // 2, 2)))
+
+ x0, x1 = tl.split(tl.reshape(x, (BLOCK_M, BLOCK_K // 2, 2)))
+ o0 = x0 * cos - x1 * sin
+ o1 = x1 * cos + x0 * sin
+ tl.store(base + rk[None, :], tl.interleave(o0, o1), mask=mask_x)
+
+
+def packed_qk_rope_(xqkv: torch.Tensor, freqs_cis: torch.Tensor) -> None:
+ """Rotate Q and K in place inside a packed QKV buffer.
+
+ Args:
+ xqkv: contiguous (seqlen, 3, nheads, headdim); only the Q and K
+ slices are rotated.
+ freqs_cis: contiguous (seqlen, headdim // 2) complex64 rotary freqs,
+ read through its interleaved re/im fp32 view.
+
+ Bitwise identical to ``ApplyRotaryEmb(enable_fp32_compute=True)`` applied
+ to Q and K separately, but in one kernel with ~5x less memory traffic.
+ Requires triton: callers must check HAS_TRITON and use the unfused path
+ otherwise. Contract violations raise.
+
+ """
+ assert xqkv.ndim == 4 and xqkv.size(1) == 3, xqkv.shape
+ assert xqkv.is_contiguous(), xqkv.shape
+ assert freqs_cis.ndim == 2 and freqs_cis.is_contiguous(), freqs_cis.shape
+ assert freqs_cis.dtype == torch.complex64, freqs_cis.dtype
+
+ seq_length, _, num_heads, head_dim = xqkv.shape
+ rotary_dim = freqs_cis.size(-1) * 2
+ assert freqs_cis.size(0) == seq_length, (freqs_cis.shape, xqkv.shape)
+ assert rotary_dim == head_dim, (freqs_cis.shape, xqkv.shape) # full rotation
+ qk = xqkv.as_strided(
+ (seq_length, 2 * num_heads, head_dim),
+ (3 * num_heads * head_dim, head_dim, 1),
+ )
+
+ BLOCK_M = 4
+ # seqlen on grid.x: HIP caps gridDim.y at 65535.
+ grid = (triton.cdiv(seq_length, BLOCK_M), 2 * num_heads)
+ _packed_qk_rope_kernel[grid](
+ qk,
+ freqs_cis.view(torch.float32),
+ seq_length,
+ qk.stride(0),
+ qk.stride(1),
+ rotary_dim=rotary_dim,
+ BLOCK_M=BLOCK_M,
+ num_warps=2 if rotary_dim <= 64 else 4,
+ )
diff --git a/vllm/model_executor/layers/utils.py b/vllm/model_executor/layers/utils.py
index aec4ebdd44..af8b7d4d5c 100644
--- a/vllm/model_executor/layers/utils.py
+++ b/vllm/model_executor/layers/utils.py
@@ -377,6 +377,13 @@ def rocm_unquantized_gemm(
weight: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
+ # dispatch_unquantized_gemm() picks this backend for any ROCm build,
+ # regardless of the tensor's actual device; the custom op below is only
+ # registered for the CUDA/HIP dispatch key, so route CPU tensors (e.g.
+ # from unit tests that build layers without moving them to the GPU)
+ # through the device-agnostic fallback instead of hard-crashing.
+ if not x.is_cuda:
+ return default_unquantized_gemm(layer, x, weight, bias)
return torch.ops.vllm.rocm_unquantized_gemm(x, weight, bias)
diff --git a/vllm/model_executor/model_loader/reload/torchao_decorator.py b/vllm/model_executor/model_loader/reload/torchao_decorator.py
index dc8e0acc6e..10ced70d31 100644
--- a/vllm/model_executor/model_loader/reload/torchao_decorator.py
+++ b/vllm/model_executor/model_loader/reload/torchao_decorator.py
@@ -1,10 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-from collections.abc import Iterable
+from collections.abc import Callable
from functools import wraps
-from types import FunctionType
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Concatenate, ParamSpec, TypeVar
import torch
@@ -20,13 +19,18 @@ if TYPE_CHECKING:
__all__ = ["set_torchao_reload_attrs", "support_quantized_model_reload_from_hp_weights"]
+_P = ParamSpec("_P")
+_R = TypeVar("_R")
+
def set_torchao_reload_attrs(model: torch.nn.Module, model_config: ModelConfig):
model._do_torchao_reload = True
model._model_config = model_config
-def support_quantized_model_reload_from_hp_weights(original_load_weights: FunctionType):
+def support_quantized_model_reload_from_hp_weights(
+ original_load_weights: Callable[Concatenate["AutoWeightsLoader", _P], _R],
+) -> Callable[Concatenate["AutoWeightsLoader", _P], _R]:
"""Decorator for `load_weights` method for AutoWeightsLoader.load_weights to support
reloading high precision (bfloat16/float16/float32) weight for an already quantized
model, this involves restoring the weights to a high precision weights and
@@ -39,17 +43,16 @@ def support_quantized_model_reload_from_hp_weights(original_load_weights: Functi
@wraps(original_load_weights)
def patched_model_load_weights(
self: "AutoWeightsLoader",
- weights: Iterable[tuple[str, torch.Tensor]],
- *args,
- **kwargs,
- ):
+ *args: _P.args,
+ **kwargs: _P.kwargs,
+ ) -> _R:
model = self.module
if not getattr(model, "_do_torchao_reload", False):
- return original_load_weights(self, weights, *args, **kwargs)
+ return original_load_weights(self, *args, **kwargs)
initialize_layerwise_reload(model)
- loaded_weights = original_load_weights(self, weights, *args, **kwargs)
+ loaded_weights = original_load_weights(self, *args, **kwargs)
finalize_layerwise_reload(model, model._model_config)
return loaded_weights
diff --git a/vllm/model_executor/models/kimi_k25_vit.py b/vllm/model_executor/models/kimi_k25_vit.py
index 1b05f1f77b..72e45d82e6 100644
--- a/vllm/model_executor/models/kimi_k25_vit.py
+++ b/vllm/model_executor/models/kimi_k25_vit.py
@@ -30,6 +30,9 @@ from vllm.model_executor.layers.linear import (
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb
+from vllm.model_executor.layers.rotary_embedding.packed_qk_rope import (
+ packed_qk_rope_,
+)
from vllm.model_executor.models.utils import maybe_prefix
from vllm.model_executor.models.vision import (
is_vit_use_data_parallel,
@@ -37,6 +40,7 @@ from vllm.model_executor.models.vision import (
)
from vllm.platforms import current_platform
from vllm.transformers_utils.configs.kimi_k25 import KimiK25VisionConfig
+from vllm.triton_utils import HAS_TRITON
from vllm.utils.torch_utils import async_tensor_h2d
logger = init_logger(__name__)
@@ -451,14 +455,19 @@ class MoonViTEncoderLayer(nn.Module):
)
# xqkv: (seqlen, 3, nheads, headdim)
xqkv = xqkv.view(*qkv_shape)
- xq, xk, xv = torch.unbind(xqkv, dim=-3)
- _apply_rope_input_validation(xq, rope_freqs_cis)
- _apply_rope_input_validation(xk, rope_freqs_cis)
- rope_cos = rope_freqs_cis.real.contiguous()
- rope_sin = rope_freqs_cis.imag.contiguous()
- xq = self.apply_rotary_emb(xq, rope_cos, rope_sin)
- xk = self.apply_rotary_emb(xk, rope_cos, rope_sin)
+ # xq/xk alias xqkv, so the in-place rotation below is visible through
+ # them; only the fallback needs to rebind.
+ xq, xk, xv = torch.unbind(xqkv, dim=-3)
+ if HAS_TRITON:
+ packed_qk_rope_(xqkv, rope_freqs_cis)
+ else:
+ _apply_rope_input_validation(xq, rope_freqs_cis)
+ _apply_rope_input_validation(xk, rope_freqs_cis)
+ rope_cos = rope_freqs_cis.real.contiguous()
+ rope_sin = rope_freqs_cis.imag.contiguous()
+ xq = self.apply_rotary_emb(xq, rope_cos, rope_sin)
+ xk = self.apply_rotary_emb(xk, rope_cos, rope_sin)
if max_seqlen is None:
max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max()
diff --git a/vllm/model_executor/models/openpangu_vl.py b/vllm/model_executor/models/openpangu_vl.py
index b7d5c81032..8a62aa6ba6 100644
--- a/vllm/model_executor/models/openpangu_vl.py
+++ b/vllm/model_executor/models/openpangu_vl.py
@@ -1055,15 +1055,18 @@ class OpenPanguVLForConditionalGeneration(
input_ids: torch.Tensor,
multimodal_embeddings: MultiModalEmbeddings | None = None,
) -> torch.Tensor:
- inputs_embeds = self.language_model.embed_input_ids(input_ids)
- if multimodal_embeddings is not None:
- inputs_embeds = self.embed_input_ids( # type: ignore[call-overload]
- input_ids,
- inputs_embeds,
- multimodal_embeddings,
- [self.config.image_token_id, self.config.video_token_id],
- )
- return inputs_embeds
+ if multimodal_embeddings is None:
+ return self.language_model.embed_input_ids(input_ids)
+
+ mm_token_ids = input_ids.new_tensor(
+ [self.config.image_token_id, self.config.video_token_id]
+ )
+ is_multimodal = torch.isin(input_ids, mm_token_ids)
+ return self.embed_input_ids(
+ input_ids,
+ multimodal_embeddings,
+ is_multimodal=is_multimodal,
+ )
def _process_image_input(
self, image_input: OpenPanguVLImageInputs
diff --git a/vllm/model_executor/models/phi4mm.py b/vllm/model_executor/models/phi4mm.py
index 8524455975..7f223b1ccc 100644
--- a/vllm/model_executor/models/phi4mm.py
+++ b/vllm/model_executor/models/phi4mm.py
@@ -1297,7 +1297,7 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
- def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> None:
+ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
diff --git a/vllm/model_executor/models/roberta.py b/vllm/model_executor/models/roberta.py
index 6a65b3a7e9..8e141229f0 100644
--- a/vllm/model_executor/models/roberta.py
+++ b/vllm/model_executor/models/roberta.py
@@ -41,7 +41,7 @@ from vllm.model_executor.models.utils import (
from vllm.sequence import IntermediateTensors
from .bert_with_rope import BertWithRope, JinaRobertaModel
-from .interfaces import SupportsCrossEncoding
+from .interfaces import SupportsCrossEncoding, SupportsLoRA
from .interfaces_base import attn_type, default_pooling_type
@@ -266,7 +266,7 @@ class BgeM3EmbeddingModel(RobertaEmbeddingModel):
@default_pooling_type(seq_pooling_type="CLS")
-class RobertaForSequenceClassification(nn.Module, SupportsCrossEncoding):
+class RobertaForSequenceClassification(nn.Module, SupportsCrossEncoding, SupportsLoRA):
"""A model that uses Roberta to provide embedding functionalities.
This class encapsulates the BertModel and provides an interface for
@@ -279,6 +279,7 @@ class RobertaForSequenceClassification(nn.Module, SupportsCrossEncoding):
"""
is_pooling_model = True
+ packed_modules_mapping = {"qkv_proj": ["query", "key", "value"]}
jina_to_vllm_mapper = WeightsMapper(
orig_to_new_substr={
"emb_ln": "embeddings.LayerNorm",
@@ -295,7 +296,8 @@ class RobertaForSequenceClassification(nn.Module, SupportsCrossEncoding):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
- self.padding_idx: int = vllm_config.model_config.hf_config.pad_token_id
+ self.config = config
+ self.padding_idx: int = config.pad_token_id
self.num_labels = config.num_labels
self.roberta = BertModel(
diff --git a/vllm/model_executor/models/ultravox.py b/vllm/model_executor/models/ultravox.py
index a454ba5962..72ce26f21c 100644
--- a/vllm/model_executor/models/ultravox.py
+++ b/vllm/model_executor/models/ultravox.py
@@ -282,10 +282,14 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo
def get_replacement_ultravox(item_idx: int):
if "audio_num_tokens" in out_mm_data:
- return [replacement_id] * int(out_mm_data["audio_num_tokens"][item_idx])
+ audio_num_tokens = out_mm_data["audio_num_tokens"]
+ assert isinstance(audio_num_tokens, torch.Tensor)
+ return [replacement_id] * int(audio_num_tokens[item_idx])
start = chunks_start_idx[item_idx]
end = chunks_start_idx[item_idx + 1]
- audio_token_len = out_mm_data["audio_token_len"][start:end].sum()
+ audio_token_lens = out_mm_data["audio_token_len"]
+ assert isinstance(audio_token_lens, torch.Tensor)
+ audio_token_len = audio_token_lens[start:end].sum()
return [replacement_id] * int(audio_token_len) # type: ignore
return [
@@ -566,7 +570,8 @@ class UltravoxWhisperEncoder(WhisperEncoder):
) -> torch.Tensor:
return (input_lengths - 1) // 2 + 1
- def forward(
+ # Ultravox requires chunk lengths in addition to Whisper's input features.
+ def forward( # type: ignore[override]
self,
input_features: torch.Tensor,
audio_lens: torch.Tensor,
@@ -960,12 +965,12 @@ def pad_and_concat_to_dim3(
return features
- features = [pad_and_concat_to_dim3(f) for f in features]
+ tensors = [pad_and_concat_to_dim3(f) for f in features]
- max_len = max(f.shape[-1] for f in features)
+ max_len = max(f.shape[-1] for f in tensors)
# Ensure all features have dim=3
- features = [f.view(-1, *f.shape[-2:]) for f in features]
+ tensors = [f.view(-1, *f.shape[-2:]) for f in tensors]
# Pad and concatenate:
# [[B1, 80, M1], [B2, 80, M2]] -> [B1+B2, 80, max(M1, M2)]
- features = [F.pad(f, (0, max_len - f.shape[-1])) for f in features]
- return torch.cat(features)
+ tensors = [F.pad(f, (0, max_len - f.shape[-1])) for f in tensors]
+ return torch.cat(tensors)
diff --git a/vllm/model_executor/models/unlimited_ocr.py b/vllm/model_executor/models/unlimited_ocr.py
index d248e5c8e0..27e5503d2c 100644
--- a/vllm/model_executor/models/unlimited_ocr.py
+++ b/vllm/model_executor/models/unlimited_ocr.py
@@ -199,6 +199,7 @@ class UnlimitedOCRMultiModalProcessor(DeepseekOCRMultiModalProcessor):
if isinstance(images, ImageEmbeddingItems):
num_image_tokens = images.get_feature_size(item_idx)
else:
+ assert isinstance(images, ImageProcessorItems)
size = images.get_image_size(item_idx)
# Disable crop mode for multi-image input.
diff --git a/vllm/model_executor/models/utils.py b/vllm/model_executor/models/utils.py
index 6375e0572e..e762ad1c47 100644
--- a/vllm/model_executor/models/utils.py
+++ b/vllm/model_executor/models/utils.py
@@ -35,7 +35,6 @@ if TYPE_CHECKING:
from transformers import PreTrainedConfig
from transformers.conversion_mapping import WeightRenaming
- from vllm.config.model import ModelConfig
from vllm.model_executor.layers.quantization import QuantizationConfig
logger = init_logger(__name__)
@@ -337,7 +336,7 @@ class AutoWeightsLoader:
"""
# Add persistent registered buffers.
# Non-persistent buffers are excluded, matching PyTorch state_dict().
- non_persistent = getattr(module, "_non_persistent_buffers_set", set())
+ non_persistent: set[str] = getattr(module, "_non_persistent_buffers_set", set())
for buf_name, buf in module.named_buffers(recurse=False):
if buf_name not in child_params and buf_name not in non_persistent:
child_params[buf_name] = buf
@@ -558,7 +557,7 @@ def maybe_fuse_shared_experts(
def get_spec_layer_idx_from_weight_name(
- config: "ModelConfig", weight_name: str
+ config: "PreTrainedConfig", weight_name: str
) -> int | None:
"""Return the MTP layer index a weight belongs to, or None.
@@ -587,7 +586,7 @@ def get_spec_layer_idx_from_weight_name(
def skip_spec_layers(
- weights: Iterable[tuple[str, torch.Tensor]], config: "ModelConfig"
+ weights: Iterable[tuple[str, torch.Tensor]], config: "PreTrainedConfig"
) -> Iterable[tuple[str, torch.Tensor]]:
"""Drop MTP spec-layer weights (loaded by the MTP head, not the base model).
@@ -629,11 +628,11 @@ def init_vllm_registered_model(
@overload
-def flatten_bn(x: torch.Tensor) -> torch.Tensor: ...
+def flatten_bn(x: list[torch.Tensor]) -> list[torch.Tensor]: ...
@overload
-def flatten_bn(x: list[torch.Tensor]) -> list[torch.Tensor]: ...
+def flatten_bn(x: torch.Tensor) -> torch.Tensor: ...
@overload
@@ -973,7 +972,9 @@ def get_draft_quant_config(vllm_config: VllmConfig) -> "QuantizationConfig | Non
The draft model's config if available, None otherwise.
"""
- draft_model_config = vllm_config.speculative_config.draft_model_config
+ speculative_config = vllm_config.speculative_config
+ assert speculative_config is not None
+ draft_model_config = speculative_config.draft_model_config
draft_load_config = vllm_config.load_config
return (
diff --git a/vllm/models/deepseek_v41/amd/rocm.py b/vllm/models/deepseek_v41/amd/rocm.py
index e1588b55e7..9de09bad8c 100644
--- a/vllm/models/deepseek_v41/amd/rocm.py
+++ b/vllm/models/deepseek_v41/amd/rocm.py
@@ -29,6 +29,7 @@ from vllm.models.deepseek_v41.sparse_mla import (
from vllm.platforms import current_platform
from vllm.platforms.rocm import _ON_GFX950
from vllm.triton_utils import tl, triton
+from vllm.utils.multi_stream_utils import execute_in_parallel
from vllm.v1.attention.backend import (
CommonAttentionMetadata,
)
@@ -529,6 +530,9 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
"this rank; PP splits inside a v4.1 index-sharing group "
"are not supported."
)
+ if self.compressor is None and self.indexer is None:
+ # Dense layers have nothing to overlap; keep the base serial path.
+ self.aux_stream_list = None
@classmethod
def get_padded_num_q_heads(cls, num_heads: int) -> int:
@@ -600,8 +604,193 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
def _run_parallel_input_projections(
self, hidden_states: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None]:
+ if self.indexer is not None and self.compressor is None:
+ # Serial: qr must sit on the default stream before the q-side
+ # fork below can consume it (ROCm).
+ aux_streams = self.aux_stream_list
+ self.aux_stream_list = None
+ try:
+ return super()._run_parallel_input_projections(hidden_states)
+ finally:
+ self.aux_stream_list = aux_streams
return super()._run_parallel_input_projections(hidden_states)
+ def _enable_multi_stream_overlap(self) -> bool:
+ """ROCm multi-stream gates: streams and capture region.
+
+ Dict metadata marks piecewise cudagraph, whose eager breaks rebuild
+ the attention inputs on the owning stream. Forking side streams
+ there would rely on runtime HIP event sync, which is unreliable in
+ this overlap on ROCm (event waits can hang), so multi-stream only
+ runs where the fork/join becomes static graph edges: inside capture,
+ or with non-dict metadata (full cudagraph or the profile run), which
+ has no eager breaks.
+ """
+ attn_metadata = get_forward_context().attn_metadata
+ return self.aux_stream_list is not None and (
+ torch.cuda.is_current_stream_capturing()
+ or not isinstance(attn_metadata, dict)
+ )
+
+ def forward(
+ self,
+ positions: torch.Tensor,
+ hidden_states: torch.Tensor,
+ llama_4_scaling: torch.Tensor | None = None,
+ ) -> torch.Tensor:
+ if (
+ self.indexer is None
+ or self._prepare_and_attn_fn == self._prepare_and_attn_eager
+ or not self._enable_multi_stream_overlap()
+ ):
+ # Sequential fallback: no forks outside the capture region, where
+ # HIP event sync is unreliable; MRV1 keeps the input prep in its
+ # wide eager region (#51430).
+ aux_streams = self.aux_stream_list
+ self.aux_stream_list = None
+ try:
+ return super().forward(positions, hidden_states, llama_4_scaling)
+ finally:
+ self.aux_stream_list = aux_streams
+
+ attn_out = self._alloc_attn_out(hidden_states.shape[0], hidden_states)
+ if self.compressor is not None:
+ q, kv, index_q, index_q_scale, index_weights = self._forward_csa2_full(
+ hidden_states, positions
+ )
+ else:
+ q, kv, index_q, index_q_scale, index_weights = self._forward_csa2_reindex(
+ hidden_states, positions
+ )
+ self._sparse_indexer_and_attn(
+ hidden_states,
+ index_q,
+ index_q_scale,
+ index_weights,
+ q,
+ kv,
+ positions,
+ attn_out,
+ )
+ return self._o_proj(attn_out, positions)
+
+ def _forward_csa2_full(
+ self, hidden_states: torch.Tensor, positions: torch.Tensor
+ ) -> tuple[
+ torch.Tensor,
+ torch.Tensor,
+ torch.Tensor | None,
+ torch.Tensor | None,
+ torch.Tensor | None,
+ ]:
+ """CSA layers: one fork replaces the base pipeline's three.
+
+ Each fork/join is a HIP event pair per side stream on ROCm, so the
+ base three-fork pipeline is expensive there; the compressor chain and
+ the indexer weights projection run fully in parallel with the default
+ chain. The indexer K write and q-side stay serial after the join:
+ ``wk`` consumes the aux-0 latent and the q-side's wq_b consumes the
+ default-stream qr.
+ """
+ attn_metadata = get_forward_context().attn_metadata
+ compressor = self.compressor
+ indexer = self.indexer
+ assert compressor is not None and indexer is not None
+ aux_streams = self.aux_stream_list
+ assert aux_streams is not None and len(aux_streams) >= 2
+
+ def default_chain():
+ qr_kv = self._fused_wqa_wkv_gemm(hidden_states)
+ qr, qr_scale, kv = self._split_qkv_and_norm(qr_kv)
+ q = self._wq_b_proj(qr, qr_scale).view(
+ -1, self.n_local_heads, self.head_dim
+ )
+ return (
+ self._fused_qnorm_rope_kv_insert(q, kv, positions, attn_metadata),
+ qr,
+ qr_scale,
+ kv,
+ )
+
+ def compressor_chain():
+ kv_score = torch.mm(
+ hidden_states,
+ compressor.fused_wkv_wgate.weight.T,
+ out_dtype=torch.float32,
+ )
+ latent = compressor(kv_score, positions)
+ compressor.insert_cache(latent, positions, self.rotary_emb)
+ return latent
+
+ def indexer_weights_chain():
+ # ReplicatedLinear returns (output, bias); bias is None.
+ weights, _ = indexer.weights_proj(hidden_states)
+ return weights
+
+ (q, qr, qr_scale, kv), (latent, indexer_weights) = execute_in_parallel(
+ default_chain,
+ [compressor_chain, indexer_weights_chain],
+ self.ln_events[0],
+ self.ln_events[1:3],
+ aux_streams[:2],
+ enable=True,
+ default_first=True,
+ )
+
+ indexer._produce_k(latent, positions, self.indexer_rotary_emb)
+ index_q, index_q_scale, index_weights_out = indexer.forward_q(
+ qr, qr_scale, indexer_weights, positions, self.indexer_rotary_emb
+ )
+ return q, kv, index_q, index_q_scale, index_weights_out
+
+ def _forward_csa2_reindex(
+ self, hidden_states: torch.Tensor, positions: torch.Tensor
+ ) -> tuple[
+ torch.Tensor,
+ torch.Tensor,
+ torch.Tensor | None,
+ torch.Tensor | None,
+ torch.Tensor | None,
+ ]:
+ """Indexer-only layers: the q-side overlaps the SWA q path.
+
+ The input projections run serial first: the q-side consumes qr, so
+ it can only overlap the SWA q projection and KV insert.
+ """
+ indexer = self.indexer
+ assert indexer is not None
+ attn_metadata = get_forward_context().attn_metadata
+ # Serial on these layers via the _run_parallel_input_projections
+ # override: qr must sit on the default stream before the fork.
+ qr_kv, _, indexer_weights = self._run_parallel_input_projections(hidden_states)
+ assert indexer_weights is not None
+ qr, qr_scale, kv = self._split_qkv_and_norm(qr_kv)
+
+ def swa_q_chain():
+ q = self._wq_b_proj(qr, qr_scale).view(
+ -1, self.n_local_heads, self.head_dim
+ )
+ return self._fused_qnorm_rope_kv_insert(q, kv, positions, attn_metadata)
+
+ def indexer_q_chain():
+ return indexer.forward_q(
+ qr, qr_scale, indexer_weights, positions, self.indexer_rotary_emb
+ )
+
+ aux_streams = self.aux_stream_list
+ assert aux_streams is not None
+ q, aux_results = execute_in_parallel(
+ swa_q_chain,
+ [indexer_q_chain],
+ self.ln_events[0],
+ self.ln_events[1:2],
+ aux_streams[:1],
+ enable=True,
+ default_first=True,
+ )
+ index_q, index_q_scale, index_weights_out = aux_results[0]
+ return q, kv, index_q, index_q_scale, index_weights_out
+
@functools.cached_property
def _wq_b_uses_aiter_block_scaled(self) -> bool:
"""True when both wq_b GEMMs run the aiter block-scaled fp8 kernel.
diff --git a/vllm/models/deepseek_v41/attention.py b/vllm/models/deepseek_v41/attention.py
index d0e69fb358..c27a8d9baf 100644
--- a/vllm/models/deepseek_v41/attention.py
+++ b/vllm/models/deepseek_v41/attention.py
@@ -478,7 +478,6 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
# graph and MRV1 produces garbage (#51430).
self._prepare_and_attn_fn = self._prepare_and_attn_eager
- # Will be None on ROCm for now.
self.aux_stream_list = aux_stream_list
# [0]: GEMM start / post-GEMM event0. [1..3]: GEMM done events;
# [1] doubles as post-GEMM event1. Reuse is safe: GEMM fully joins
@@ -697,7 +696,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
else:
gemm_rs.warn_incompatible_projection()
- def _wo_b_proj(self, z: torch.Tensor) -> torch.Tensor:
+ def _wo_b_proj(self, z: torch.Tensor | QuantizedActivation) -> torch.Tensor:
"""Apply ``wo_b``; with GEMM-RS bound, also reduce-scatter the result.
Every ``_o_proj`` implementation projects through this so the decoder
@@ -707,7 +706,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
"""
if self.gemm_rs is None:
return self.wo_b(z)
- if self.gemm_rs.should_run(z):
+ if isinstance(z, torch.Tensor) and self.gemm_rs.should_run(z):
return self.gemm_rs.apply(z, self.wo_b)
# Small batches stay on the unfused path, which is faster there.
return sp_reduce_scatter(self.wo_b(z))
@@ -1381,6 +1380,37 @@ class DeepseekV4Indexer(nn.Module):
self.use_fp4_kv,
)
+ def forward_q(
+ self,
+ qr: torch.Tensor | QuantizedActivation,
+ qr_scale: torch.Tensor | None,
+ indexer_weights: torch.Tensor,
+ positions: torch.Tensor,
+ rotary_emb: nn.Module,
+ ) -> tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]:
+ """Build the indexer queries: wq_b over qr plus fused RoPE/quant.
+
+ Split out so the ROCm layer can schedule it apart from
+ ``_produce_k`` (e.g. on an aux stream once ``qr`` is ready).
+ """
+ q = self._wq_b_proj(qr, qr_scale)
+ q = q.view(-1, self.n_head, self.head_dim)
+ q_quant, weights = fused_indexer_q_rope_quant(
+ positions,
+ q,
+ rotary_emb.cos_sin_cache,
+ indexer_weights,
+ self.softmax_scale,
+ self.n_head**-0.5,
+ use_fp4=self.use_fp4_kv,
+ weights_out_dtype=self.indexer_weights_dtype,
+ )
+ if isinstance(q_quant, tuple):
+ q, q_scale = q_quant
+ else:
+ q, q_scale = q_quant, None
+ return q, q_scale, weights
+
def forward(
self,
qr: torch.Tensor | QuantizedActivation,
@@ -1422,23 +1452,7 @@ class DeepseekV4Indexer(nn.Module):
# (skip_k_cache_insert=True).
self._produce_k(latent, positions, rotary_emb)
- q = self._wq_b_proj(qr, qr_scale)
- q = q.view(-1, self.n_head, self.head_dim)
- q_quant, weights = fused_indexer_q_rope_quant(
- positions,
- q,
- rotary_emb.cos_sin_cache,
- indexer_weights,
- self.softmax_scale,
- self.n_head**-0.5,
- use_fp4=self.use_fp4_kv,
- weights_out_dtype=self.indexer_weights_dtype,
- )
- if isinstance(q_quant, tuple):
- q, q_scale = q_quant
- else:
- q, q_scale = q_quant, None
- return q, q_scale, weights
+ return self.forward_q(qr, qr_scale, indexer_weights, positions, rotary_emb)
def _wq_b_proj(
self,
diff --git a/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py b/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py
index 423849dcf1..0e02b486a5 100644
--- a/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py
+++ b/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py
@@ -9,15 +9,16 @@ import torch
from vllm.config import VllmConfig
from vllm.config.cache import CacheDType
from vllm.forward_context import get_forward_context
-from vllm.models.deepseek_v4.nvidia.ops.o_proj import (
- compute_fp8_einsum_recipe,
- deep_gemm_fp8_o_proj,
-)
+from vllm.models.deepseek_v4.nvidia.ops.o_proj import compute_fp8_einsum_recipe
from vllm.models.deepseek_v41.attention import DeepseekV4Attention
from vllm.models.deepseek_v41.common.ops import (
build_flashinfer_mixed_sparse_indices,
compute_global_topk_indices_and_lens,
)
+from vllm.models.deepseek_v41.nvidia.ops.o_proj import (
+ dsv41_o_proj,
+ register_dsv41_o_proj_warmup,
+)
from vllm.models.deepseek_v41.sparse_mla import (
DeepseekV4FlashMLAMetadata,
DeepseekV4SparseMLABackend,
@@ -242,27 +243,14 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention):
return _pad_to_supported_q_heads(num_heads)
def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
- o = attn_out[:, : self.n_local_heads, :]
- return deep_gemm_fp8_o_proj(
- o,
- positions,
- self.rotary_emb.cos_sin_cache,
- self.wo_a,
- self._wo_b_proj,
- n_groups=self.n_local_groups,
- heads_per_group=self.n_local_heads // self.n_local_groups,
- nope_dim=self.nope_head_dim,
- rope_dim=self.rope_head_dim,
- o_lora_rank=self.o_lora_rank,
- einsum_recipe=self._einsum_recipe,
- tma_aligned_scales=self._tma_aligned_scales,
- )
+ return dsv41_o_proj(self, attn_out, positions)
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
self._einsum_recipe, self._tma_aligned_scales = compute_fp8_einsum_recipe(
self._o_proj_block_size
)
+ register_dsv41_o_proj_warmup(self)
# Per-tensor FP8 scale buffers + precomputed scalar BMM scales. Only the
# per-tensor FP8 cache path consumes these; bf16 reads ``self.scale``.
if self.kv_cache_torch_dtype != torch.float8_e4m3fn:
@@ -623,21 +611,7 @@ class DeepseekV4FlashInferSM120Attention(DeepseekV4Attention):
return _pad_to_supported_q_heads(num_heads)
def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
- o = attn_out[:, : self.n_local_heads, :]
- return deep_gemm_fp8_o_proj(
- o,
- positions,
- self.rotary_emb.cos_sin_cache,
- self.wo_a,
- self._wo_b_proj,
- n_groups=self.n_local_groups,
- heads_per_group=self.n_local_heads // self.n_local_groups,
- nope_dim=self.nope_head_dim,
- rope_dim=self.rope_head_dim,
- o_lora_rank=self.o_lora_rank,
- einsum_recipe=self._einsum_recipe,
- tma_aligned_scales=self._tma_aligned_scales,
- )
+ return dsv41_o_proj(self, attn_out, positions)
def __init__(self, vllm_config: VllmConfig, *args, **kwargs) -> None:
super().__init__(vllm_config, *args, **kwargs)
diff --git a/vllm/models/deepseek_v41/nvidia/flashmla.py b/vllm/models/deepseek_v41/nvidia/flashmla.py
index 96dde36c61..ce3047ceca 100644
--- a/vllm/models/deepseek_v41/nvidia/flashmla.py
+++ b/vllm/models/deepseek_v41/nvidia/flashmla.py
@@ -6,16 +6,17 @@ from typing import TYPE_CHECKING, ClassVar, cast
import torch
from vllm.forward_context import get_forward_context
-from vllm.models.deepseek_v4.nvidia.ops.o_proj import (
- compute_fp8_einsum_recipe,
- deep_gemm_fp8_o_proj,
-)
+from vllm.models.deepseek_v4.nvidia.ops.o_proj import compute_fp8_einsum_recipe
from vllm.models.deepseek_v41.attention import DeepseekV4Attention
from vllm.models.deepseek_v41.common.ops import (
combine_topk_swa_indices,
compute_global_topk_indices_and_lens,
dequantize_and_gather_k_cache,
)
+from vllm.models.deepseek_v41.nvidia.ops.o_proj import (
+ dsv41_o_proj,
+ register_dsv41_o_proj_warmup,
+)
from vllm.models.deepseek_v41.sparse_mla import (
DeepseekV4FlashMLABackend,
DeepseekV4FlashMLAMetadata,
@@ -57,23 +58,10 @@ class DeepseekV4FlashMLAAttention(DeepseekV4Attention):
self._einsum_recipe, self._tma_aligned_scales = compute_fp8_einsum_recipe(
self._o_proj_block_size
)
+ register_dsv41_o_proj_warmup(self)
def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
- o = attn_out[:, : self.n_local_heads, :]
- return deep_gemm_fp8_o_proj(
- o,
- positions,
- self.rotary_emb.cos_sin_cache,
- self.wo_a,
- self._wo_b_proj,
- n_groups=self.n_local_groups,
- heads_per_group=self.n_local_heads // self.n_local_groups,
- nope_dim=self.nope_head_dim,
- rope_dim=self.rope_head_dim,
- o_lora_rank=self.o_lora_rank,
- einsum_recipe=self._einsum_recipe,
- tma_aligned_scales=self._tma_aligned_scales,
- )
+ return dsv41_o_proj(self, attn_out, positions)
@classmethod
def get_padded_num_q_heads(cls, num_heads: int) -> int:
diff --git a/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py b/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py
new file mode 100644
index 0000000000..b195d4b849
--- /dev/null
+++ b/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py
@@ -0,0 +1,599 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""DSV4.1 small-batch MXFP8 WO-A with inverse RoPE and quantization."""
+
+from dataclasses import dataclass
+from typing import Any
+
+import cutlass
+import cutlass.cute as cute
+import cutlass.utils as utils
+import torch
+from cuda.bindings.driver import CUstream
+from cutlass import (
+ BFloat16,
+ Float8E4M3FN,
+ Float32,
+ Int32,
+ Int64,
+ Uint8,
+ Uint32,
+ Uint64,
+)
+from cutlass._mlir.dialects import llvm
+from cutlass.cute.nvgpu import cpasync, tcgen05
+from cutlass.cute.runtime import make_fake_tensor
+from cutlass.cutlass_dsl import dsl_user_op
+
+from vllm.cute_utils import _tcgen05, mbarrier, recast_val, simple_tma_copy
+from vllm.model_executor.warmup.jit_warmup import WarmupIntRange, kernel_launcher
+from vllm.model_executor.warmup.jit_warmup_cutedsl_helper import (
+ CuTeDSLLaunchSpec,
+ VllmCuTeDSLJitKernel,
+)
+
+# Each head is [nope | rope] as in fused_inv_rope_fp8_quant: only the last
+# _ROPE_DIM columns carry RoPE.
+_HEAD_DIM = 512
+_ROPE_DIM = 64
+
+
+@dsl_user_op
+def _ue8m0_rp(value, *, loc=None, ip=None):
+ result = llvm.inline_asm(
+ Uint32.mlir_type,
+ [value.ir_value(loc=loc, ip=ip)],
+ "{ .reg .b16 sf; cvt.rp.satfinite.ue8m0x2.f32 sf, $1, $1; "
+ "cvt.u32.u16 $0, sf; and.b32 $0, $0, 255; }",
+ "=r,f",
+ has_side_effects=False,
+ loc=loc,
+ ip=ip,
+ )
+ return Uint32(result)
+
+
+@dsl_user_op
+def _rope_fma(value, cos, other, sin, *, subtract=False, loc=None, ip=None):
+ """Preserve multiply/FMA rounding when RoPE loads are hoisted."""
+ result = llvm.inline_asm(
+ Float32.mlir_type,
+ [v.ir_value(loc=loc, ip=ip) for v in (value, cos, other, sin)],
+ "{ .reg .f32 product; mul.rn.f32 product, $3, $4; "
+ + (
+ "neg.f32 product, product; fma.rn.f32 $0, $1, $2, product; }"
+ if subtract
+ else "fma.rn.f32 $0, $1, $2, product; }"
+ ),
+ "=f,f,f,f,f",
+ has_side_effects=False,
+ loc=loc,
+ ip=ip,
+ )
+ return Float32(result)
+
+
+@cute.jit
+def _scale(amax: Float32):
+ exponent = _ue8m0_rp(amax * Float32(1.0 / 448.0))
+ inv = recast_val((Uint32(254) - exponent) << 23, Float32)
+ if exponent == 0:
+ inv = Float32(0)
+ return exponent, inv
+
+
+class FusedWoAKernel(VllmCuTeDSLJitKernel["FusedWoAKernel.CompileKey"]):
+ @dataclass(frozen=True)
+ class CompileKey:
+ # T=1..96 uses at most 46 keys per (n_groups, heads_per_group, o_lora_rank).
+ tokens_per_tile: int
+ token_tiles: int
+ full_tiles: bool
+ n_groups: int
+ heads_per_group: int
+ o_lora_rank: int
+
+ @staticmethod
+ def kernel(compile_key: CompileKey) -> Any:
+ tokens = compile_key.tokens_per_tile
+ token_tiles = compile_key.token_tiles
+ full_tiles = compile_key.full_tiles
+ acc_cols = 1 << (tokens - 1).bit_length()
+ tile_m = max(8, acc_cols)
+ tile_n = 128
+ tile_k = 128
+ # tcgen05 MXFP8 MMA consumes K = 32, one MX block, per instruction.
+ mma_k = 32
+ # Each group's heads split K across one cluster, one head per CTA.
+ heads_per_group = compile_key.heads_per_group
+ tiles_per_group = compile_key.o_lora_rank // tile_n
+ n_tiles = compile_key.n_groups * tiles_per_group
+ # Each CTA keeps one whole head of K in SMEM, one TMA box per stage.
+ num_stages = _HEAD_DIM // tile_k
+ w_stage_bytes = tile_n * tile_k
+ nope_dim = _HEAD_DIM - _ROPE_DIM
+ threads_per_token = _HEAD_DIM // 4
+ sf_col = max(16, tile_m)
+ tmem_cols = sf_col * 2
+ threads = min(acc_cols * threads_per_token, 512)
+ tokens_per_iter = threads // threads_per_token
+
+ @cute.kernel
+ def device_kernel(
+ x: cute.Tensor,
+ positions: cute.Tensor,
+ rope: cute.Tensor,
+ w: cpasync.TmaInfo,
+ ws: cpasync.TmaInfo,
+ q: cute.Tensor,
+ qs: cute.Tensor,
+ num_tokens: Int32,
+ ):
+ total_tokens = num_tokens
+ if cutlass.const_expr(full_tiles):
+ total_tokens = Int32(tokens * token_tiles)
+ tid, _, _ = cute.arch.thread_idx()
+ bid, token_tile, _ = cute.arch.block_idx()
+ token_start = Int32(0)
+ if cutlass.const_expr(token_tiles > 1):
+ token_start = token_tile * tokens
+ warp = cute.arch.make_warp_uniform(tid // 32)
+ lane = tid % 32
+ split = bid % heads_per_group
+ tile = bid // heads_per_group
+ group = tile // tiles_per_group
+ smem = utils.SmemAllocator()
+ sW = smem.allocate_tensor(
+ Float8E4M3FN,
+ w.smem_layout.outer,
+ byte_alignment=128,
+ swizzle=w.smem_layout.inner,
+ )
+ sX = smem.allocate_tensor(
+ Float8E4M3FN,
+ cute.make_layout(
+ (tile_m, tile_k, num_stages), stride=(tile_k, 1, tile_m * tile_k)
+ ),
+ byte_alignment=1024,
+ swizzle=cute.make_swizzle(3, 4, 3),
+ )
+ # tcgen05.cp 32x128b.warpx4 order: row r at byte (r % 32) * 16 +
+ # (r // 32) * 4, one 4-byte cell of MX-block scales per row.
+ sW_SF = smem.allocate_tensor(
+ Int32,
+ cute.make_layout(((32, 4), num_stages), stride=((4, 1), tile_n)),
+ 128,
+ )
+ sW_SF_raw = smem.allocate_tensor(
+ Int32, cute.make_layout((tile_n, num_stages)), 128
+ )
+ sX_SF = smem.allocate_tensor(
+ Uint8,
+ cute.make_layout(
+ ((32, 4), tile_k // mma_k, num_stages),
+ stride=((16, 4), 1, tile_n * 4),
+ ),
+ 128,
+ )
+ copy_fp8x4 = cute.make_copy_atom(
+ cute.nvgpu.CopyUniversalOp(), Float8E4M3FN, num_bits_per_copy=32
+ )
+ partial = smem.allocate_tensor(
+ Float32,
+ cute.make_layout(
+ (tile_n, cute.ceil_div(tokens, heads_per_group), heads_per_group)
+ ),
+ 128,
+ )
+ mbar_loaded = smem.allocate_array(Int64, num_stages)
+ mbar_ws_loaded = smem.allocate_array(Int64, 1)
+ mbar_done = smem.allocate_array(Int64, 1)
+ mbar_reduced = smem.allocate_array(Int64, 1)
+ taddr = smem.allocate(Int32, 4)
+ if tid == 0:
+ for stage in cutlass.range_constexpr(num_stages):
+ # Wait for weight TMA and this stage's quantization threads.
+ cute.arch.mbarrier_init(mbar_loaded + stage, 1 + threads // 4)
+ cute.arch.mbarrier_init(mbar_ws_loaded, 1)
+ cute.arch.mbarrier_init(mbar_done, 1)
+ cute.arch.mbarrier_init(mbar_reduced, 1)
+ cute.arch.mbarrier_init_fence()
+ # Local users of the barriers (warp 0's TMA below) need the init too.
+ cute.arch.sync_threads()
+ # Paired with cluster_wait() below: peers must see mbar_reduced
+ # initialized before their st.async; the split hides the barrier.
+ cute.arch.cluster_arrive_relaxed()
+ if warp == 0:
+ # Raw DeepGEMM MN-major scales; the MMA warp transposes them to
+ # the UTCCP order, which TMA cannot scatter at 4-byte granularity.
+ ws_tiles = cute.zipped_divide(ws.tma_tensor, (tile_n, num_stages))
+ with cute.arch.elect_one():
+ mbarrier.arrive_expect_tx(mbar_ws_loaded, tile_n * num_stages * 4)
+ simple_tma_copy(
+ ws.atom,
+ ws_tiles[None, (tile % tiles_per_group, split, group)],
+ sW_SF_raw,
+ mbar_ws_loaded,
+ )
+ tiles = cute.zipped_divide(w.tma_tensor, (tile_n, tile_k))
+ for stage in cutlass.range_constexpr(num_stages):
+ with cute.arch.elect_one():
+ mbarrier.arrive_expect_tx(mbar_loaded + stage, w_stage_bytes)
+ simple_tma_copy(
+ w.atom,
+ tiles[None, (tile, split * num_stages + stage)],
+ sW[None, None, stage],
+ mbar_loaded + stage,
+ )
+ # x and positions come from the PDL predecessor; everything above
+ # only touches weights. Padded sX rows only feed unread MMA columns.
+ cute.arch.griddepcontrol_wait()
+ # TMEM is allocated at runtime, and a PDL predecessor on this SM may
+ # hold it until it exits; allocating earlier can deadlock.
+ if warp == 0:
+ cute.arch.alloc_tmem(tmem_cols, taddr)
+ cute.arch.relinquish_tmem_alloc_permit()
+ copy_x = cute.make_copy_atom(
+ cute.nvgpu.CopyUniversalOp(), BFloat16, num_bits_per_copy=64
+ )
+ n_iters = cute.ceil_div(tokens, tokens_per_iter)
+ qid = (tid + 32) % threads_per_token
+ k = qid * 4
+ # Tail rows repeat the last input; their output stores are masked.
+ # Issue every token's x load before any math so their latencies overlap.
+ x_bf16 = cute.make_rmem_tensor((4, n_iters), BFloat16)
+ for iteration in cutlass.range_constexpr(n_iters):
+ token = iteration * tokens_per_iter + tid // threads_per_token
+ if cutlass.const_expr(tokens % tokens_per_iter == 0) or token < tokens:
+ x_src = cute.local_tile(
+ x[
+ cute.min(token_start + token, total_tokens - 1),
+ group * heads_per_group + split,
+ None,
+ ],
+ (4,),
+ (qid,),
+ )
+ cute.copy(copy_x, x_src, x_bf16[None, iteration])
+ # Overlap the RoPE loads across tokens before consuming any of them.
+ cos_reg = cute.make_rmem_tensor((2, n_iters), Float32)
+ sin_reg = cute.make_rmem_tensor((2, n_iters), Float32)
+ if cutlass.const_expr(n_iters > 1):
+ copy_rope = cute.make_copy_atom(
+ cute.nvgpu.CopyUniversalOp(), Float32, num_bits_per_copy=32
+ )
+ if k >= nope_dim:
+ for iteration in cutlass.range_constexpr(n_iters):
+ token = iteration * tokens_per_iter + tid // threads_per_token
+ if (
+ cutlass.const_expr(tokens % tokens_per_iter == 0)
+ or token < tokens
+ ):
+ src_token = cute.min(token_start + token, total_tokens - 1)
+ pos = positions[src_token]
+ freq = (k - nope_dim) // 2
+ rope_row = rope[pos, None]
+ cos_src = cute.local_tile(rope_row, (2,), (freq // 2,))
+ cute.copy(copy_rope, cos_src, cos_reg[None, iteration])
+ sin_tile = (freq + _ROPE_DIM // 2) // 2
+ sin_src = cute.local_tile(rope_row, (2,), (sin_tile,))
+ cute.copy(copy_rope, sin_src, sin_reg[None, iteration])
+ for iteration in cutlass.range_constexpr(n_iters):
+ token = iteration * tokens_per_iter + tid // threads_per_token
+ if cutlass.const_expr(tokens % tokens_per_iter == 0) or token < tokens:
+ x_f32 = cute.make_rmem_tensor((4,), Float32)
+ x_f32.store(x_bf16[None, iteration].load().to(Float32))
+ if k >= nope_dim:
+ # Inverse RoPE on the interleaved pairs (k, k+1) and
+ # (k+2, k+3); each rope row is cos || sin.
+ for pair in cutlass.range_constexpr(2):
+ if cutlass.const_expr(n_iters > 1):
+ cos = cos_reg[pair, iteration]
+ sin = sin_reg[pair, iteration]
+ else:
+ pos = positions[
+ cute.min(token_start + token, total_tokens - 1)
+ ]
+ freq = (k - nope_dim) // 2 + pair
+ cos = Float32(rope[pos, freq])
+ sin = Float32(rope[pos, freq + _ROPE_DIM // 2])
+ even, odd = x_f32[2 * pair], x_f32[2 * pair + 1]
+ x_f32[2 * pair] = _rope_fma(even, cos, odd, sin)
+ x_f32[2 * pair + 1] = _rope_fma(
+ odd, cos, even, sin, subtract=True
+ )
+ amax = cute.arch.fmax(
+ cute.arch.fmax(cute.abs(x_f32[0]), cute.abs(x_f32[1])),
+ cute.arch.fmax(cute.abs(x_f32[2]), cute.abs(x_f32[3])),
+ )
+ amax = cute.arch.warp_reduction_max(amax, threads_in_group=8)
+ exponent, inv = _scale(cute.arch.fmax(amax, Float32(1e-10)))
+ x_fp8 = cute.make_rmem_tensor((4,), Float8E4M3FN)
+ x_fp8.store((x_f32.load() * inv).to(Float8E4M3FN))
+ cute.copy(
+ copy_fp8x4,
+ x_fp8,
+ cute.local_tile(
+ sX[token, None, k // tile_k], (4,), (qid % 32,)
+ ),
+ )
+ if lane % 8 == 0:
+ sX_SF[token, k % tile_k // mma_k, k // tile_k] = Uint8(exponent)
+ cute.arch.fence_proxy("async.shared", space="cta")
+ mbarrier.arrive(mbar_loaded + qid // 32, order="release")
+ if warp == 0:
+ base = cute.make_tensor(taddr, cute.make_layout(1))[0]
+ # sW / sX are 128B-swizzled K-major; one stage's K fills the
+ # swizzle atom, so the leading byte offset is unused.
+ sdesc = _tcgen05.make_sdesc_128B_swizzle(LBO=0)
+ # Unswizzled 32 x 16 B scale tiles: SBO = one 8-row core matrix
+ # (16 B units at bit 32); bit 46 is the SM100 descriptor version.
+ sf_sbo = 8 * 16
+ sfdesc = Uint64((sf_sbo >> 4 << 32) | (1 << 46))
+ # M = tile_n weight rows (A), N = tile_m token rows (B).
+ idesc = _tcgen05.make_mxfp8_idesc(tile_n, tile_m)
+ # sW_SF's layout is the UTCCP order, so a plain copy transposes;
+ # fence the generic-proxy writes before tcgen05.cp reads them.
+ cute.arch.mbarrier_wait(mbar_ws_loaded, 0)
+ for stage in cutlass.range_constexpr(num_stages):
+ for i in cutlass.range_constexpr(4):
+ sW_SF[i * 32 + lane, stage] = sW_SF_raw[i * 32 + lane, stage]
+ cute.arch.fence_proxy("async.shared", space="cta")
+ cute.arch.sync_warp()
+ for stage in cutlass.range_constexpr(num_stages):
+ cute.arch.mbarrier_wait(mbar_loaded + stage, 0)
+ _tcgen05.fence_after_thread_sync()
+ if cutlass.const_expr(stage == num_stages - 1):
+ with cute.arch.elect_one():
+ cute.arch.griddepcontrol_launch_dependents()
+ adesc = sdesc | (sW[None, None, stage].iterator.toint() >> 4)
+ bdesc = sdesc | (sX[None, None, stage].iterator.toint() >> 4)
+ _tcgen05.cp(
+ base + sf_col,
+ sfdesc | (sW_SF[None, stage].iterator.toint() >> 4),
+ "32x128b",
+ "warpx4",
+ )
+ _tcgen05.cp(
+ base + sf_col + 4,
+ sfdesc | (sX_SF[None, None, stage].iterator.toint() >> 4),
+ "32x128b",
+ "warpx4",
+ )
+ # Advance mma_k FP8 bytes (16 B units) and pick MX block kk of
+ # each 4-byte scale (B sf_id at bit 4, A sf_id at bit 29).
+ for kk in cutlass.range_constexpr(tile_k // mma_k):
+ _tcgen05.mma_mxfp8(
+ base,
+ adesc + kk * mma_k // 16,
+ bdesc + kk * mma_k // 16,
+ idesc + ((kk << 4) | (kk << 29)),
+ base + sf_col,
+ base + sf_col + 4,
+ stage > 0 or kk > 0,
+ )
+ _tcgen05.commit(mbar_done)
+ # Only the MMA warp waits; barrier 2 releases the TMEM readers.
+ cute.arch.mbarrier_wait(mbar_done, 0)
+ if tid < tile_n:
+ cute.arch.barrier(barrier_id=2, number_of_threads=tile_n)
+ # Tokens t with t % heads_per_group == split arrive from every peer
+ # (this CTA included), tile_n Float32 rows each.
+ if tid == 0:
+ owned = cute.ceil_div(tokens - split, heads_per_group)
+ mbarrier.arrive_expect_tx(
+ mbar_reduced, owned * tile_n * heads_per_group * 4
+ )
+ # Pairs with cluster_arrive_relaxed() above: every peer's
+ # mbar_reduced is initialized before the st.async below.
+ cute.arch.cluster_wait()
+ # quack's process-wide const_expr-if rewrite can leave these unbound
+ # after the dynamic path above; bind them so the regions join.
+ amax, exponent, inv = Float32(0), Uint32(0), Float32(0)
+ if tid < tile_n:
+ base = cute.make_tensor(taddr, cute.make_layout(1))[0]
+ _tcgen05.fence_after_thread_sync()
+ acc = cute.make_rmem_tensor(acc_cols, Float32)
+ if cutlass.const_expr(tokens == 1):
+ acc[0] = _tcgen05.ld(warp * 32, base, "32x32b", 1)
+ else:
+ acc.store(_tcgen05.ld(warp * 32, base, "32x32b", acc_cols))
+ _tcgen05.wait_ld()
+ for token in cutlass.range_constexpr(tokens):
+ ptr = cute.domain_offset(
+ (tid, token // heads_per_group, split), partial
+ ).iterator
+ cute.arch.store_async_dsmem(
+ ptr,
+ recast_val(acc[token], Int32),
+ mbar_reduced,
+ token % heads_per_group,
+ )
+ if split < tokens:
+ # One warp waits for the peers' bytes; named barrier 1 (0 is
+ # sync_threads) releases the other tile_n epilogue threads.
+ if warp == 0:
+ cute.arch.mbarrier_wait(mbar_reduced, 0)
+ cute.arch.barrier(barrier_id=1, number_of_threads=tile_n)
+ for i in cutlass.range_constexpr(
+ cute.ceil_div(tokens, heads_per_group)
+ ):
+ token = split + i * heads_per_group
+ if cutlass.const_expr(
+ full_tiles
+ and (
+ tokens <= heads_per_group
+ or tokens % heads_per_group == 0
+ )
+ ) or (token < tokens and token_start + token < total_tokens):
+ value = Float32(0)
+ for peer in cutlass.range_constexpr(heads_per_group):
+ value += partial[tid, i, peer]
+ value = Float32(BFloat16(value))
+ amax = cute.arch.warp_redux_sync(value, "max", abs=True)
+ exponent, inv = _scale(amax)
+ q[token_start + token, tile * tile_n + tid] = Float8E4M3FN(
+ value * inv
+ )
+ if lane == 0:
+ row = token_start + token
+ offset = row % 32 * 16 + row // 32 * 4 + warp
+ qs[tile * 512 + offset] = Uint8(exponent)
+ # Every scale tile owns its padding, with no second memset kernel.
+ for i in cutlass.range_constexpr(4):
+ offset = i * tile_n + tid
+ row = offset // 16 + (offset % 16 // 4) * 32
+ if token_tile == 0 and split == 0 and row >= total_tokens:
+ qs[tile * 512 + offset] = Uint8(0)
+ # Each receiver drained all writes into its own inbox. Only local
+ # TMEM readers must finish before deallocation; no peer reads SMEM.
+ if tid < tile_n:
+ cute.arch.barrier(barrier_id=2, number_of_threads=tile_n)
+ if warp == 0:
+ cute.arch.dealloc_tmem(
+ cute.make_ptr(
+ Float32,
+ cute.make_tensor(taddr, cute.make_layout(1))[0],
+ cute.AddressSpace.tmem,
+ ),
+ tmem_cols,
+ )
+
+ @cute.jit
+ def host_entrypoint(
+ x: cute.Tensor,
+ positions: cute.Tensor,
+ rope: cute.Tensor,
+ w: cute.Tensor,
+ ws: cute.Tensor,
+ q: cute.Tensor,
+ qs: cute.Tensor,
+ num_tokens: Int32,
+ stream: CUstream,
+ ):
+ layout = cute.make_composed_layout(
+ cute.make_swizzle(3, 4, 3),
+ 0,
+ cute.make_layout(
+ (tile_n, tile_k, num_stages), stride=(tile_k, 1, w_stage_bytes)
+ ),
+ )
+ tma = cpasync.make_tiled_tma_atom(
+ cpasync.CopyBulkTensorTileG2SOp(cta_group=tcgen05.CtaGroup.ONE),
+ w,
+ layout,
+ (tile_n, tile_k),
+ )
+ ws_tma = cpasync.make_tiled_tma_atom(
+ cpasync.CopyBulkTensorTileG2SOp(cta_group=tcgen05.CtaGroup.ONE),
+ cute.make_tensor(ws.iterator, cute.select(ws.layout, mode=[1, 2, 0])),
+ cute.make_layout((tile_n, num_stages)),
+ (tile_n, num_stages),
+ )
+ device_kernel(x, positions, rope, tma, ws_tma, q, qs, num_tokens).launch(
+ grid=(n_tiles * heads_per_group, token_tiles, 1),
+ block=(threads, 1, 1),
+ cluster=(heads_per_group, 1, 1),
+ stream=stream,
+ use_pdl=True,
+ )
+
+ return host_entrypoint
+
+ def dispatch( # type: ignore[override]
+ self, *, tokens: int, n_groups: int, heads_per_group: int, o_lora_rank: int
+ ) -> CompileKey:
+ token_tiles = (tokens + 31) // 32
+ # Host integer arithmetic: balance tiles in multiples of four tokens.
+ tile_tokens = (
+ tokens
+ if token_tiles == 1
+ else (tokens + 4 * token_tiles - 1) // (4 * token_tiles) * 4
+ )
+ return self.CompileKey(
+ tokens_per_tile=tile_tokens,
+ token_tiles=token_tiles,
+ full_tiles=tokens % tile_tokens == 0,
+ n_groups=n_groups,
+ heads_per_group=heads_per_group,
+ o_lora_rank=o_lora_rank,
+ )
+
+ def get_warmup_keys(
+ self, *, max_tokens: int, n_groups: int, heads_per_group: int, o_lora_rank: int
+ ) -> list[CompileKey]:
+ # Trace all accepted token counts and deduplicate shared tile shapes.
+ return self._trace_dispatch(self.dispatch)(
+ tokens=WarmupIntRange(1, max_tokens + 1),
+ n_groups=n_groups,
+ heads_per_group=heads_per_group,
+ o_lora_rank=o_lora_rank,
+ )
+
+ def warmup_inputs(self, compile_key: CompileKey) -> tuple[Any, ...]:
+ tokens = (
+ compile_key.tokens_per_tile * compile_key.token_tiles
+ if compile_key.full_tiles
+ else cute.sym_int()
+ )
+ groups = compile_key.n_groups
+ rank = compile_key.o_lora_rank
+ n = groups * rank
+ k = compile_key.heads_per_group * _HEAD_DIM
+ return (
+ make_fake_tensor(
+ BFloat16,
+ (tokens, groups * compile_key.heads_per_group, _HEAD_DIM),
+ (cute.sym_int(divisibility=8), _HEAD_DIM, 1),
+ assumed_align=16,
+ ),
+ make_fake_tensor(Int64, (tokens,), (1,)),
+ make_fake_tensor(Float32, (cute.sym_int(), _ROPE_DIM), (_ROPE_DIM, 1)),
+ make_fake_tensor(Float8E4M3FN, (n, k), (k, 1), assumed_align=16),
+ make_fake_tensor(
+ Int32,
+ (groups, rank, k // 128),
+ (rank * k // 128, 1, rank),
+ assumed_align=16,
+ ),
+ make_fake_tensor(Float8E4M3FN, (tokens, n), (n, 1)),
+ make_fake_tensor(Uint8, (128 * n // 32,), (1,)),
+ Int32(0),
+ )
+
+ @kernel_launcher
+ def __call__(
+ self,
+ *,
+ x: torch.Tensor,
+ positions: torch.Tensor,
+ rope: torch.Tensor,
+ weight: torch.Tensor,
+ weight_scale: torch.Tensor,
+ ) -> CuTeDSLLaunchSpec["FusedWoAKernel.CompileKey"]:
+ """Return FP8 WO-A output and FlashInfer F8_128x4 scale bytes."""
+ tokens = x.shape[0]
+ groups, rank = weight_scale.shape[:2]
+ n = groups * rank
+ q = torch.empty((tokens, n), device=x.device, dtype=torch.float8_e4m3fn)
+ scales = torch.empty(128 * n // 32, device=x.device, dtype=torch.uint8)
+ launch_args = (
+ x,
+ positions,
+ rope,
+ weight.view(n, -1),
+ weight_scale,
+ q,
+ scales,
+ tokens,
+ )
+ compile_key = self.dispatch(
+ tokens=tokens,
+ n_groups=groups,
+ heads_per_group=x.shape[1] // groups,
+ o_lora_rank=rank,
+ )
+ return compile_key, launch_args, (q, scales)
+
+
+_FUSED_WO_A_KERNEL = FusedWoAKernel()
diff --git a/vllm/models/deepseek_v41/nvidia/ops/o_proj.py b/vllm/models/deepseek_v41/nvidia/ops/o_proj.py
new file mode 100644
index 0000000000..4a315ddc9b
--- /dev/null
+++ b/vllm/models/deepseek_v41/nvidia/ops/o_proj.py
@@ -0,0 +1,95 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""DSV4.1 output projection with a small-batch SM100/SM103 fusion."""
+
+import torch
+from torch import nn
+
+from vllm.model_executor.kernels.linear.mxfp8.flashinfer import (
+ FlashInferCutedslMxfp8LinearKernel,
+)
+from vllm.model_executor.layers.fusion.quant_activation import (
+ QuantizedActivation,
+ get_input_quant_key,
+)
+from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Dynamic
+from vllm.models.deepseek_v4.nvidia.ops.o_proj import deep_gemm_fp8_o_proj
+from vllm.platforms import current_platform
+
+
+def _fused_wo_a_max_tokens(layer: nn.Module) -> int:
+ max_tokens = getattr(layer, "_fused_wo_a_token_limit", None)
+ if max_tokens is None:
+ # Additional token tiles only pay off for smaller grids.
+ ctas_per_tile = layer.n_local_heads * (layer.o_lora_rank // 128)
+ max_tokens = 96 if ctas_per_tile <= 128 else 32
+ layer._fused_wo_a_token_limit = max_tokens
+ return max_tokens
+
+
+def _can_fuse_wo_a(layer: nn.Module) -> bool:
+ """Whether the attention layer matches the fused WO-A kernel's layout."""
+ method = getattr(layer.wo_b, "scheme", layer.wo_b.quant_method)
+ return (
+ current_platform.is_device_capability_family(100)
+ # WO-A is FP8 with per-32 scales, and WO-B takes MXFP8 input directly.
+ and layer._einsum_recipe == (1, 1, 32)
+ and get_input_quant_key(layer.wo_b) == kMxfp8Dynamic
+ and isinstance(
+ getattr(method, "kernel", None), FlashInferCutedslMxfp8LinearKernel
+ )
+ # A group's heads form one portable (<= 8 CTA) cluster.
+ and layer.n_local_heads // layer.n_local_groups <= 8
+ and (layer.nope_head_dim, layer.rope_head_dim) == (448, 64)
+ and layer.o_lora_rank % 128 == 0
+ )
+
+
+def register_dsv41_o_proj_warmup(layer: nn.Module) -> None:
+ """Warm the fused WO-A token counts ``dsv41_o_proj`` can dispatch."""
+ # Draft models load outside the warmup registry; their layers share the
+ # target's attention shapes, so the target's registration covers them too.
+ if _can_fuse_wo_a(layer):
+ from .fused_wo_a import _FUSED_WO_A_KERNEL
+
+ _FUSED_WO_A_KERNEL.register_warmup(
+ max_tokens=_fused_wo_a_max_tokens(layer),
+ n_groups=layer.n_local_groups,
+ heads_per_group=layer.n_local_heads // layer.n_local_groups,
+ o_lora_rank=layer.o_lora_rank,
+ )
+
+
+def dsv41_o_proj(
+ layer: nn.Module, attn_out: torch.Tensor, positions: torch.Tensor
+) -> torch.Tensor:
+ """``deep_gemm_fp8_o_proj`` with WO-A fused for small SM100/SM103 batches."""
+ o = attn_out[:, : layer.n_local_heads, :]
+ cos_sin_cache = layer.rotary_emb.cos_sin_cache
+ if 1 <= o.shape[0] <= _fused_wo_a_max_tokens(layer) and _can_fuse_wo_a(layer):
+ from .fused_wo_a import _FUSED_WO_A_KERNEL
+
+ q, scales = _FUSED_WO_A_KERNEL(
+ x=o,
+ positions=positions,
+ rope=cos_sin_cache,
+ weight=layer.wo_a.weight,
+ weight_scale=layer.wo_a.weight_scale,
+ )
+ return layer._wo_b_proj(
+ QuantizedActivation(q, scales, o.dtype, q.shape, kMxfp8Dynamic)
+ )
+ return deep_gemm_fp8_o_proj(
+ o,
+ positions,
+ cos_sin_cache,
+ layer.wo_a,
+ layer._wo_b_proj,
+ n_groups=layer.n_local_groups,
+ heads_per_group=layer.n_local_heads // layer.n_local_groups,
+ nope_dim=layer.nope_head_dim,
+ rope_dim=layer.rope_head_dim,
+ o_lora_rank=layer.o_lora_rank,
+ einsum_recipe=layer._einsum_recipe,
+ tma_aligned_scales=layer._tma_aligned_scales,
+ )
diff --git a/vllm/models/minimax_m3/amd/model.py b/vllm/models/minimax_m3/amd/model.py
index 4871e66a9a..40daa4a6ee 100644
--- a/vllm/models/minimax_m3/amd/model.py
+++ b/vllm/models/minimax_m3/amd/model.py
@@ -106,6 +106,9 @@ from vllm.models.minimax_m3.amd.sparse_attention_msa import (
MiniMaxM3SparseAiterPAImpl,
MiniMaxM3SparseAiterPAPrefillMetadata,
)
+from vllm.models.minimax_m3.common.encoder_cudagraph import (
+ MiniMaxM3EncoderCudaGraphMixin,
+)
from vllm.models.minimax_m3.common.indexer import MiniMaxM3Indexer
from vllm.models.minimax_m3.common.mm_preprocess import (
MiniMaxM3VLDummyInputsBuilder,
@@ -1685,7 +1688,11 @@ class MiniMaxM3SparseForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
dummy_inputs=MiniMaxM3VLDummyInputsBuilder,
)
class MiniMaxM3SparseForConditionalGeneration(
- nn.Module, SupportsMultiModal, SupportsPP, SupportsEagle3
+ nn.Module,
+ SupportsMultiModal,
+ MiniMaxM3EncoderCudaGraphMixin,
+ SupportsPP,
+ SupportsEagle3,
):
"""Top-level (VL) entry point for MiniMax M3.
diff --git a/vllm/models/minimax_m3/common/encoder_cudagraph.py b/vllm/models/minimax_m3/common/encoder_cudagraph.py
new file mode 100644
index 0000000000..7c55cde946
--- /dev/null
+++ b/vllm/models/minimax_m3/common/encoder_cudagraph.py
@@ -0,0 +1,281 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Encoder CUDA graph support for the MiniMax M3 vision tower.
+
+Mixin implementing the SupportsEncoderCudaGraph protocol for any host class
+that holds the vision tower as ``self.vision_tower`` (a
+``MiniMaxVLVisionModel``). The manager routes image items through this mixin;
+video items use the eager encoder path.
+"""
+
+from collections.abc import Hashable
+from typing import TYPE_CHECKING, Any
+
+import torch
+
+from vllm.model_executor.models.interfaces import SupportsEncoderCudaGraph
+
+if TYPE_CHECKING:
+ from vllm.config import VllmConfig
+ from vllm.config.multimodal import MultiModalConfig
+ from vllm.models.minimax_m3.common.vision_tower import MiniMaxVLVisionModel
+ from vllm.v1.worker.encoder_cudagraph_defs import (
+ EncoderCudaGraphCaptureInputs,
+ EncoderCudaGraphConfig,
+ EncoderCudaGraphReplayBuffers,
+ EncoderItemSpec,
+ )
+
+
+class MiniMaxM3EncoderCudaGraphMixin(SupportsEncoderCudaGraph):
+ """SupportsEncoderCudaGraph for hosts with ``self.vision_tower``."""
+
+ vision_tower: "MiniMaxVLVisionModel"
+ multimodal_config: "MultiModalConfig | None"
+ _encoder_cg_pad_totals: dict[int, int] = {}
+
+ def get_encoder_cudagraph_config(self) -> "EncoderCudaGraphConfig":
+ from vllm.v1.attention.backends.registry import AttentionBackendEnum
+ from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphConfig
+
+ if (
+ self.multimodal_config is not None
+ and self.multimodal_config.get_limit_per_prompt("image") == 0
+ ):
+ raise ValueError(
+ "cudagraph_mm_encoder requires the MiniMax M3 vision tower, "
+ "but the image limit is 0. Disable cudagraph_mm_encoder."
+ )
+
+ mm_cfg = self.multimodal_config
+ if (
+ mm_cfg is not None
+ and mm_cfg.mm_encoder_attn_dtype == "fp8"
+ and mm_cfg.mm_encoder_fp8_scale_path is None
+ ):
+ raise ValueError(
+ "cudagraph_mm_encoder requires static FP8 scales for the "
+ "MiniMax M3 vision encoder. Set --mm-encoder-fp8-scale-path "
+ "or disable cudagraph_mm_encoder."
+ )
+
+ # FlashInfer reads max_seqlen on the host and TORCH_SDPA calls
+ # .tolist() on CUDA cu_seqlens; neither can be captured.
+ backend = self.vision_tower.vision_model.attn_backend
+ if backend not in (
+ AttentionBackendEnum.FLASH_ATTN,
+ AttentionBackendEnum.ROCM_AITER_FA,
+ ):
+ raise ValueError(
+ f"cudagraph_mm_encoder is not supported with the "
+ f"{backend.name} ViT attention backend for MiniMax M3. Set "
+ "--mm-encoder-attn-backend FLASH_ATTN or ROCM_AITER_FA "
+ "(ROCm), or disable cudagraph_mm_encoder."
+ )
+
+ pad_totals = self._encoder_cg_pad_totals
+
+ def pad_cu_seqlens(dst: torch.Tensor, src: torch.Tensor) -> None:
+ # Varlen attention requires cu_seqlens[-1] to equal the number of
+ # rows actually passed in. The captured buffers are sized for the
+ # full token budget, so a smaller real batch has to be completed
+ # with one trailing padding sequence; declaring fewer rows than the
+ # buffer holds is undefined behaviour and returns NaN on FlashAttn.
+ total = pad_totals.get(dst.data_ptr())
+ if total is None:
+ raise RuntimeError("cu_seqlens replay buffer was not registered")
+ n = src.shape[0]
+ dst[:n].copy_(src[:n])
+ dst[n:] = total
+
+ def pad_rope_planes(dst: torch.Tensor, src: torch.Tensor) -> None:
+ # cos/sin planes are (3, N, half_rot_dim): the row axis is dim 1,
+ # not dim 0 as the manager's default slice-copy assumes.
+ dst.zero_()
+ dst[:, : src.shape[1]].copy_(src)
+
+ return EncoderCudaGraphConfig(
+ modalities=["image"],
+ buffer_keys=[
+ "pixel_values",
+ "rotary_cos",
+ "rotary_sin",
+ "cu_seqlens",
+ "max_seqlen",
+ ],
+ out_hidden_size=self.vision_tower.out_hidden_size,
+ padding_logics={
+ "cu_seqlens": pad_cu_seqlens,
+ "rotary_cos": pad_rope_planes,
+ "rotary_sin": pad_rope_planes,
+ },
+ )
+
+ def get_encoder_cudagraph_budget_range(
+ self,
+ vllm_config: "VllmConfig",
+ ) -> tuple[int, int]:
+ min_budget = 64
+ max_budget = min(
+ vllm_config.scheduler_config.max_num_batched_tokens,
+ vllm_config.model_config.max_model_len,
+ )
+ return (min_budget, max_budget)
+
+ @staticmethod
+ def _get_grid_thw_list(mm_kwargs: dict[str, Any]) -> list[list[int]]:
+ grid_thw = mm_kwargs["image_grid_thw"]
+ if isinstance(grid_thw, torch.Tensor):
+ return [[int(x) for x in row] for row in grid_thw.tolist()]
+ return [[int(x) for x in row] for row in grid_thw]
+
+ @staticmethod
+ def _get_pixel_values(mm_kwargs: dict[str, Any]) -> torch.Tensor:
+ pixel_values = mm_kwargs["pixel_values"]
+ if isinstance(pixel_values, list):
+ pixel_values = torch.cat(pixel_values)
+ return pixel_values
+
+ def get_encoder_cudagraph_item_specs(
+ self, mm_kwargs: dict[str, Any]
+ ) -> list["EncoderItemSpec"]:
+ from vllm.v1.worker.encoder_cudagraph_defs import EncoderItemSpec
+
+ merge = self.vision_tower.spatial_merge_size
+ return [
+ EncoderItemSpec(
+ input_size=t * h * w,
+ output_tokens=t * h * w // (merge * merge),
+ )
+ for t, h, w in self._get_grid_thw_list(mm_kwargs)
+ ]
+
+ def select_encoder_cudagraph_items(
+ self, mm_kwargs: dict[str, Any], indices: list[int]
+ ) -> dict[str, Any]:
+ grid_thw_list = self._get_grid_thw_list(mm_kwargs)
+ pixel_values = self._get_pixel_values(mm_kwargs)
+ source_grid = mm_kwargs["image_grid_thw"]
+
+ if len(indices) == 0:
+ empty_grid = (
+ source_grid[:0]
+ if isinstance(source_grid, torch.Tensor)
+ else pixel_values.new_zeros((0, 3), dtype=torch.long)
+ )
+ return {"pixel_values": pixel_values[:0], "image_grid_thw": empty_grid}
+
+ patch_counts = [t * h * w for t, h, w in grid_thw_list]
+ cum = [0]
+ for pc in patch_counts:
+ cum.append(cum[-1] + pc)
+ selected_pv = torch.cat(
+ [pixel_values[cum[i] : cum[i + 1]] for i in indices], dim=0
+ )
+ grid_device = (
+ source_grid.device if isinstance(source_grid, torch.Tensor) else None
+ )
+ selected_grid = torch.tensor(
+ [grid_thw_list[i] for i in indices],
+ dtype=torch.long,
+ device=grid_device,
+ )
+ return {"pixel_values": selected_pv, "image_grid_thw": selected_grid}
+
+ def prepare_encoder_cudagraph_capture_inputs(
+ self,
+ token_budget: int,
+ max_batch_size: int,
+ max_frames_per_batch: int,
+ device: torch.device,
+ dtype: torch.dtype,
+ path: str = "default",
+ axis_keys: tuple[Hashable, ...] | None = None,
+ ) -> "EncoderCudaGraphCaptureInputs":
+ from vllm.v1.worker.encoder_cudagraph_defs import (
+ EncoderCudaGraphCaptureInputs,
+ )
+
+ merge = self.vision_tower.spatial_merge_size
+ # Output tokens per item in the dummy grid (ceiling so total >= budget).
+ per_item_out = (token_budget + max_batch_size - 1) // max_batch_size
+ grid_thw_list = [
+ [1, merge, per_item_out * merge] for _ in range(max_batch_size)
+ ]
+
+ embeddings = self.vision_tower.vision_model.embeddings
+ patch_dim = (
+ embeddings.num_channels
+ * embeddings.temporal_patch_size
+ * embeddings.patch_size**2
+ )
+ total_patches = sum(t * h * w for t, h, w in grid_thw_list)
+ # Match the tower dtype exactly: MiniMaxVLPatchEmbed.forward mutates
+ # the module on dtype mismatch, which must not happen during capture.
+ dummy_pixel_values = torch.zeros(
+ total_patches,
+ patch_dim,
+ device=device,
+ dtype=self.vision_tower.dtype,
+ )
+
+ # The padding sequence can contain nearly all captured rows when a
+ # small image replays this graph. Leave one spare cu_seqlens slot.
+ metadata = self.vision_tower.vision_model.prepare_encoder_metadata(
+ grid_thw_list,
+ device=device,
+ max_batch_size=max_batch_size + 1,
+ max_seqlen_override=total_patches,
+ )
+
+ values: dict[str, torch.Tensor] = {"pixel_values": dummy_pixel_values}
+ values.update({k: v for k, v in metadata.items() if v is not None})
+ self._encoder_cg_pad_totals[values["cu_seqlens"].data_ptr()] = total_patches
+ return EncoderCudaGraphCaptureInputs(values=values)
+
+ def prepare_encoder_cudagraph_replay_buffers(
+ self,
+ mm_kwargs: dict[str, Any],
+ max_batch_size: int,
+ max_frames_per_batch: int,
+ path: str = "default",
+ ) -> "EncoderCudaGraphReplayBuffers":
+ from vllm.v1.worker.encoder_cudagraph_defs import (
+ EncoderCudaGraphReplayBuffers,
+ )
+
+ pixel_values = self._get_pixel_values(mm_kwargs)
+ # Unpadded: pad_cu_seqlens completes the tail with the padding sequence
+ # so cu_seqlens[-1] matches the captured buffer's row count.
+ metadata = self.vision_tower.vision_model.prepare_encoder_metadata(
+ self._get_grid_thw_list(mm_kwargs),
+ device=pixel_values.device,
+ )
+ # FlashAttention reads this CPU scalar during capture, so the captured
+ # launch must retain the bound for the padded buffer.
+ metadata.pop("max_seqlen")
+
+ values: dict[str, torch.Tensor | None] = {"pixel_values": pixel_values}
+ values.update(metadata)
+ return EncoderCudaGraphReplayBuffers(values=values)
+
+ def encoder_cudagraph_forward(
+ self,
+ inputs: dict[str, torch.Tensor],
+ path: str = "default",
+ ) -> torch.Tensor:
+ pixel_values = inputs.pop("pixel_values")
+ # Remaining keys are consumed as encoder_metadata.
+ return self.vision_tower(pixel_values, grid_thw=None, encoder_metadata=inputs)
+
+ def encoder_eager_forward(
+ self,
+ mm_kwargs: dict[str, Any],
+ path: str = "default",
+ ) -> torch.Tensor:
+ return self.vision_tower(
+ pixel_values=self._get_pixel_values(mm_kwargs).type(
+ self.vision_tower.dtype
+ ),
+ grid_thw=self._get_grid_thw_list(mm_kwargs),
+ )
diff --git a/vllm/models/minimax_m3/common/vision_tower.py b/vllm/models/minimax_m3/common/vision_tower.py
index ce56a179d0..fd9cff2c6d 100644
--- a/vllm/models/minimax_m3/common/vision_tower.py
+++ b/vllm/models/minimax_m3/common/vision_tower.py
@@ -392,19 +392,33 @@ class MiniMaxVLVisionTransformer(nn.Module):
out.append([min(max_f, t - i), h, w])
return out
- # ── Forward ──────────────────────────────────────────────────────────
+ # ── Encoder metadata (shared by eager forward and CUDA graph replay) ──
- def forward(
+ def prepare_encoder_metadata(
self,
- pixel_values: torch.Tensor,
grid_thw: list[list[int]],
- ) -> torch.Tensor:
- # pixel_values: (total_N, C * temporal_patch_size * patch_size²)
- # Output: (total_N, hidden_size)
+ *,
+ device: torch.device,
+ max_batch_size: int | None = None,
+ max_seqlen_override: int | None = None,
+ ) -> dict[str, torch.Tensor | None]:
+ """Build all grid-dependent forward inputs as standalone tensors.
- hidden = self.embeddings(pixel_values) # (total_N, hidden_size)
- hidden = self.pre_layrnorm(hidden)
+ The eager path calls this per forward; encoder CUDA graphs call it at
+ capture (dummy grids, padded to fixed buffer sizes) and before each
+ replay (real grids, unpadded — the manager's padding logic completes
+ the captured buffers).
+ Args:
+ grid_thw: per-item [t, h, w] patch grids (host metadata).
+ device: device for the attention metadata tensors.
+ max_batch_size: pad cu_seqlens to this many segments with empty
+ (zero-length) trailing segments; None leaves it unpadded.
+ max_seqlen_override: bake this max_seqlen instead of deriving it
+ from the grids; CUDA graph capture passes a worst-case value
+ because max_seqlen is a CPU scalar baked into the graph.
+
+ """
limited = self._apply_max_frames_limit(grid_thw)
# Token-level cumulative sequence lengths (one segment per limited grid).
@@ -412,15 +426,23 @@ class MiniMaxVLVisionTransformer(nn.Module):
cu_seqlens_np = np.zeros(len(lens) + 1, dtype=np.int32)
np.cumsum(np.array(lens, dtype=np.int32), out=cu_seqlens_np[1:])
+ if max_batch_size is not None and len(lens) < max_batch_size:
+ pad = np.full(max_batch_size - len(lens), cu_seqlens_np[-1], dtype=np.int32)
+ cu_seqlens_np = np.concatenate([cu_seqlens_np, pad])
+
# Backend-specific encoder metadata. For FLASH_ATTN this returns the raw
# token cu_seqlens, the max segment length, and sequence_lengths=None;
# for FLASHINFER (cuDNN) it repacks cu_seqlens into element-offset
# indptrs, buckets max_seqlen, and builds padded per-sequence lengths.
sequence_lengths = MMEncoderAttention.maybe_compute_seq_lens(
- self.attn_backend, cu_seqlens_np, hidden.device
+ self.attn_backend, cu_seqlens_np, device
)
max_seqlen = torch.tensor(
- MMEncoderAttention.compute_max_seqlen(self.attn_backend, cu_seqlens_np),
+ max_seqlen_override
+ if max_seqlen_override is not None
+ else MMEncoderAttention.compute_max_seqlen(
+ self.attn_backend, cu_seqlens_np
+ ),
dtype=torch.int32,
)
cu_seqlens = MMEncoderAttention.maybe_recompute_cu_seqlens(
@@ -428,7 +450,7 @@ class MiniMaxVLVisionTransformer(nn.Module):
cu_seqlens_np,
self.hidden_size,
self.tp_size,
- hidden.device,
+ device,
)
# 3D RoPE cos/sin: (3, total_N, half_rot_dim) fp32 t/h/w planes for
@@ -442,6 +464,45 @@ class MiniMaxVLVisionTransformer(nn.Module):
self.spatial_merge_size,
)
+ return {
+ "rotary_cos": rotary_cos,
+ "rotary_sin": rotary_sin,
+ "cu_seqlens": cu_seqlens,
+ "max_seqlen": max_seqlen,
+ "sequence_lengths": sequence_lengths,
+ }
+
+ # ── Forward ──────────────────────────────────────────────────────────
+
+ def forward(
+ self,
+ pixel_values: torch.Tensor,
+ grid_thw: list[list[int]] | None = None,
+ *,
+ encoder_metadata: dict[str, torch.Tensor | None] | None = None,
+ ) -> torch.Tensor:
+ # pixel_values: (total_N, C * temporal_patch_size * patch_size²)
+ # Output: (total_N, hidden_size)
+
+ hidden = self.embeddings(pixel_values) # (total_N, hidden_size)
+ hidden = self.pre_layrnorm(hidden)
+
+ if encoder_metadata is None:
+ assert grid_thw is not None
+ encoder_metadata = self.prepare_encoder_metadata(
+ grid_thw, device=hidden.device
+ )
+
+ cu_seqlens = encoder_metadata["cu_seqlens"]
+ rotary_cos = encoder_metadata["rotary_cos"]
+ rotary_sin = encoder_metadata["rotary_sin"]
+ max_seqlen = encoder_metadata["max_seqlen"]
+ sequence_lengths = encoder_metadata.get("sequence_lengths")
+ assert cu_seqlens is not None
+ assert rotary_cos is not None
+ assert rotary_sin is not None
+ assert max_seqlen is not None
+
# Encoder expects (N, 1, hidden_size) — add batch dim
hidden = hidden.unsqueeze(1)
@@ -600,9 +661,15 @@ class MiniMaxVLVisionModel(nn.Module):
def forward(
self,
pixel_values: torch.Tensor,
- grid_thw: list[list[int]],
+ grid_thw: list[list[int]] | None = None,
+ *,
+ encoder_metadata: dict[str, torch.Tensor | None] | None = None,
) -> torch.Tensor:
- hidden = self.vision_model(pixel_values=pixel_values, grid_thw=grid_thw)
+ hidden = self.vision_model(
+ pixel_values=pixel_values,
+ grid_thw=grid_thw,
+ encoder_metadata=encoder_metadata,
+ )
if hidden.dim() == 3:
hidden = hidden.squeeze(0)
hidden = self.multi_modal_projector(hidden)
diff --git a/vllm/models/minimax_m3/nvidia/model.py b/vllm/models/minimax_m3/nvidia/model.py
index 4fb260330f..c16918bd37 100644
--- a/vllm/models/minimax_m3/nvidia/model.py
+++ b/vllm/models/minimax_m3/nvidia/model.py
@@ -70,6 +70,9 @@ from vllm.model_executor.models.utils import (
maybe_prefix,
)
from vllm.model_executor.models.vision import run_dp_sharded_mrope_vision_model
+from vllm.models.minimax_m3.common.encoder_cudagraph import (
+ MiniMaxM3EncoderCudaGraphMixin,
+)
from vllm.models.minimax_m3.common.indexer import MiniMaxM3Indexer
from vllm.models.minimax_m3.common.mm_preprocess import (
MiniMaxM3VLDummyInputsBuilder,
@@ -1076,7 +1079,11 @@ class MiniMaxM3SparseForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
dummy_inputs=MiniMaxM3VLDummyInputsBuilder,
)
class MiniMaxM3SparseForConditionalGeneration(
- nn.Module, SupportsMultiModal, SupportsPP, SupportsEagle3
+ nn.Module,
+ SupportsMultiModal,
+ MiniMaxM3EncoderCudaGraphMixin,
+ SupportsPP,
+ SupportsEagle3,
):
"""Top-level (VL) entry point for MiniMax M3.
diff --git a/vllm/models/qwen4_exp/nvidia/ops/qsa_indexer.py b/vllm/models/qwen4_exp/nvidia/ops/qsa_indexer.py
index 077dea91f0..9b28ed6dca 100644
--- a/vllm/models/qwen4_exp/nvidia/ops/qsa_indexer.py
+++ b/vllm/models/qwen4_exp/nvidia/ops/qsa_indexer.py
@@ -383,6 +383,7 @@ def _prefill_logits(
page_table: torch.Tensor,
query_start_loc: torch.Tensor,
visible_blocks: torch.Tensor,
+ logits_workspace: torch.Tensor,
max_query_len: int,
logits_width: int,
query_offset: int,
@@ -392,10 +393,14 @@ def _prefill_logits(
assert visible_blocks.shape == (q.shape[0],)
assert 0 <= query_offset <= query_offset + num_queries <= q.shape[0]
assert 0 < logits_width <= page_table.shape[1] * k_cache.shape[1]
+ assert logits_workspace.is_contiguous()
+ assert logits_workspace.numel() >= num_queries * logits_width
- logits = torch.empty(
- (num_queries, logits_width), dtype=torch.float32, device=q.device
+ # slice from pre-allocated workspace
+ logits = logits_workspace[: num_queries * logits_width].view(
+ num_queries, logits_width
)
+
# tuned on GB300
if k_cache.dtype == torch.float8_e4m3fn:
TILE_R, STAGES, num_warps = 32, 2, 8
@@ -610,8 +615,13 @@ def qsa_select_paged_prefill(
logits_width = min(max(64, logits_width), page_table.shape[1] * k_cache.shape[1])
# chunk the inputs to keep temp logits below VLLM_SPARSE_INDEXER_MAX_LOGITS_MB
+ # always allocate the worst case to avoid memory fragmentation.
max_logits_bytes = envs.VLLM_SPARSE_INDEXER_MAX_LOGITS_MB * 1024 * 1024
rows_per_chunk = max(1, max_logits_bytes // (logits_width * 4))
+
+ budget_bytes = max(max_logits_bytes, page_table.shape[1] * k_cache.shape[1] * 4)
+ logits_workspace = q.new_empty(budget_bytes // 4, dtype=torch.float32)
+
topk_workspace = torch.empty(
(_TOPK_WORKSPACE_BYTES,), dtype=torch.uint8, device=q.device
)
@@ -625,6 +635,7 @@ def qsa_select_paged_prefill(
page_table,
query_start_loc,
visible_blocks,
+ logits_workspace,
max_query_len,
logits_width,
query_offset=query_start,
diff --git a/vllm/parser/abstract_parser.py b/vllm/parser/abstract_parser.py
index 0e84e710ed..20f22bd0e3 100644
--- a/vllm/parser/abstract_parser.py
+++ b/vllm/parser/abstract_parser.py
@@ -5,10 +5,21 @@ import contextlib
import json
from abc import abstractmethod
from collections.abc import Sequence
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, replace
from openai.types.responses import ToolChoiceFunction
from pydantic import TypeAdapter, ValidationError
+from xgrammar import Grammar, StructuralTag
+from xgrammar.structural_tag import (
+ ConstStringFormat,
+ Format,
+ GrammarFormat,
+ JSONSchemaFormat,
+ OrFormat,
+ RegexFormat,
+ TagFormat,
+ TriggeredTagsFormat,
+)
from vllm.entrypoints.chat_utils import (
get_tool_call_id_type,
@@ -25,17 +36,19 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
+from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger
from vllm.parser.metrics import record_tool_parser_invocation
from vllm.parser.utils import count_history_tool_calls
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
-from vllm.sampling_params import StructuredOutputsParams
+from vllm.sampling_params import SamplingParams, StructuredOutputsParams
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser
from vllm.tool_parsers.streaming import (
extract_named_tool_call_streaming,
extract_required_tool_call_streaming,
)
+from vllm.tool_parsers.structural_tag_registry import resolve_tool_strictness
from vllm.tool_parsers.tool_strict_level import ToolStrictLevel
logger = init_logger(__name__)
@@ -236,6 +249,67 @@ class Parser:
return 0
+def structured_outputs_to_format(params: StructuredOutputsParams) -> Format | None:
+ """Map StructuredOutputsParams in a XGrammar Format."""
+ if params.json_object:
+ return JSONSchemaFormat(json_schema={"type": "object"})
+ if params.json is not None:
+ schema = params.json
+ if isinstance(schema, str):
+ schema = json.loads(schema)
+ return JSONSchemaFormat(json_schema=schema)
+ if params.regex is not None:
+ return RegexFormat(pattern=params.regex)
+ if params.choice is not None:
+ return OrFormat(
+ elements=[ConstStringFormat(value=choice) for choice in params.choice]
+ )
+ if params.grammar is not None:
+ from vllm.v1.structured_output.utils import grammar_is_likely_lark
+
+ grammar = params.grammar
+ # IMPORTANT(arpera):
+ # GrammarFormat only accepts EBNF.
+ if grammar_is_likely_lark(grammar):
+ try:
+ grammar = str(Grammar.from_lark(grammar))
+ except Exception as e:
+ raise VLLMValidationError("Invalid grammar specification.") from e
+ return GrammarFormat(grammar=grammar)
+ if params.structural_tag is not None:
+ s_tag = json.loads(params.structural_tag)
+ if "structures" in s_tag:
+ # LegacyStructuralTagResponseFormat
+ return TriggeredTagsFormat(
+ triggers=s_tag["triggers"],
+ tags=[
+ TagFormat(
+ begin=structure["begin"],
+ content=JSONSchemaFormat(json_schema=structure["schema"]),
+ end=structure["end"],
+ )
+ for structure in s_tag["structures"]
+ ],
+ )
+ # StructuralTagResponseFormat
+ return StructuralTag.model_validate(s_tag).format
+ return None
+
+
+def _xgrammar_supports(structured_outputs: StructuredOutputsParams) -> bool:
+ """Whether the constraint passes xgrammar validation on its own."""
+ from vllm.v1.structured_output.backend_xgrammar import validate_xgrammar_grammar
+
+ try:
+ # validate_xgrammar_grammar rewrites `choice` into `grammar` in place.
+ validate_xgrammar_grammar(
+ SamplingParams(structured_outputs=replace(structured_outputs))
+ )
+ except (TypeError, ValueError, VLLMValidationError):
+ return False
+ return True
+
+
class DelegatingParser(Parser):
"""A Parser implementation that delegates to separate ReasoningParser and
ToolParser instances.
@@ -390,18 +464,14 @@ class DelegatingParser(Parser):
request = self._reasoning_parser.adjust_request(request)
if self._tool_parser is not None:
request = self._apply_structural_tag(request)
- if self._tool_parser is not None:
request = self._tool_parser.adjust_request(request)
return request
def _apply_structural_tag(
self, request: ChatCompletionRequest | ResponsesRequest
) -> ChatCompletionRequest | ResponsesRequest:
- if (
- self._tool_parser is None
- or self._tool_parser.structural_tag_model is None
- or not request.tools
- ):
+ tool_parser = self._tool_parser
+ if tool_parser is None or not request.tools:
return request
need_tool_calling = (
@@ -415,17 +485,69 @@ class DelegatingParser(Parser):
if not need_tool_calling:
return request
- structure_tag = self._tool_parser.get_structural_tag(
- request,
- reasoning=False,
- strict_level=self.tool_strict_level,
- )
- if structure_tag is None:
+ structured_outputs = request.extract_structured_outputs()
+ is_auto = request.tool_choice == "auto"
+
+ resolved_tools = None
+ if tool_parser.structural_tag_model is not None:
+ resolved_tools = resolve_tool_strictness(
+ request.tools,
+ request.tool_choice,
+ self.tool_strict_level,
+ )
+
+ output_format = None
+ if resolved_tools and is_auto and structured_outputs:
+ if not _xgrammar_supports(structured_outputs):
+ logger.warning_once(
+ "Tool calls are not constrained for tool_choice=auto with "
+ "structured outputs because xgrammar does not support the "
+ "structured output constraint; the structured output "
+ "constraint applies.",
+ scope="local",
+ )
+ return request
+ output_format = structured_outputs_to_format(structured_outputs)
+
+ if resolved_tools is not None:
+ tag_request = request
+ if output_format is not None:
+ # IMPORTANT(arpera):
+ # The "auto" tag allows plain text in response.
+ # Use tag "required" here to ensure structured-output branch
+ # remains meaningful.
+ tag_request = request.model_copy(update={"tool_choice": "required"})
+ tools_structural_tag = tool_parser.get_structural_tag(
+ tag_request,
+ reasoning=False,
+ strict_level=self.tool_strict_level,
+ )
+ else:
+ tools_structural_tag = None
+
+ if tools_structural_tag is None:
+ if is_auto and structured_outputs is not None:
+ logger.warning_once(
+ "Tool calls are not constrained for tool_choice=auto with "
+ "structured outputs because structural tags are unavailable; "
+ "the structured output constraint applies.",
+ scope="local",
+ )
return request
- structural_tag = json.dumps(structure_tag.model_dump())
+ structural_tag = tools_structural_tag
+ if output_format is not None:
+ structural_tag = StructuralTag(
+ format=OrFormat(elements=[tools_structural_tag.format, output_format])
+ )
+ elif structured_outputs is not None and not is_auto:
+ logger.warning_once(
+ "structured outputs are ignored because tool_choice forces tool call.",
+ scope="local",
+ )
+
request.structured_outputs = StructuredOutputsParams(
- structural_tag=structural_tag,
+ structural_tag=json.dumps(structural_tag.model_dump()),
)
if isinstance(request, ResponsesRequest):
request.text = None
diff --git a/vllm/parser/cohere_command.py b/vllm/parser/cohere_command.py
index e5ed7aed33..c37cc569ac 100644
--- a/vllm/parser/cohere_command.py
+++ b/vllm/parser/cohere_command.py
@@ -563,6 +563,15 @@ class CohereCommandParser(DelegatingParser):
request.response_format = None
return request
+ def _apply_structural_tag(
+ self, request: ChatCompletionRequest | ResponsesRequest
+ ) -> ChatCompletionRequest | ResponsesRequest:
+ # Note(arpera):
+ # No need to go through _apply_structural_tag in abstract_parser.py
+ # since we have already go through _apply_structural_tags here
+ # So, override this method to return request as is
+ return request
+
def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
count = 0
depth = 0
diff --git a/vllm/parser/harmony.py b/vllm/parser/harmony.py
index 5748bd6653..26f311c7e9 100644
--- a/vllm/parser/harmony.py
+++ b/vllm/parser/harmony.py
@@ -16,14 +16,11 @@ from xgrammar.structural_tag import (
AnyTextFormat,
ConstStringFormat,
Format,
- GrammarFormat,
JSONSchemaFormat,
OptionalFormat,
OrFormat,
- RegexFormat,
SequenceFormat,
TagFormat,
- TriggeredTagsFormat,
)
from vllm.entrypoints.chat_utils import make_tool_call_id
@@ -44,9 +41,8 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.logger import init_logger
-from vllm.parser.abstract_parser import DelegatingParser
+from vllm.parser.abstract_parser import DelegatingParser, structured_outputs_to_format
from vllm.reasoning.gptoss_reasoning_parser import GptOssReasoningParser
-from vllm.sampling_params import StructuredOutputsParams
from vllm.tool_parsers.gptoss_tool_parser import GptOssToolParser
from vllm.tool_parsers.structural_tag_registry import (
SimplifiedToolChoice,
@@ -428,7 +424,6 @@ _FUNCTION_CALL_BEGINS = [
" to=functions.{name}{channel}{constrain}<|message|>",
"{channel} to=functions.{name}{constrain}<|message|>",
]
-_JSON_CONTENT = JSONSchemaFormat(json_schema={"type": "object"})
_ANY_CONTENT = AnyTextFormat()
@@ -531,43 +526,6 @@ def get_harmony_structural_tag(
)
-def _params_to_final_content(params: StructuredOutputsParams) -> Format | None:
- """Map StructuredOutputsParams in a XGrammar Format."""
- if params.json_object:
- return _JSON_CONTENT
- if params.json is not None:
- schema = params.json
- if isinstance(schema, str):
- schema = json.loads(schema)
- return JSONSchemaFormat(json_schema=schema)
- if params.regex is not None:
- return RegexFormat(pattern=params.regex)
- if params.choice is not None:
- return OrFormat(
- elements=[ConstStringFormat(value=choice) for choice in params.choice]
- )
- if params.grammar is not None:
- return GrammarFormat(grammar=params.grammar)
- if params.structural_tag is not None:
- s_tag = json.loads(params.structural_tag)
- if "structures" in s_tag:
- # LegacyStructuralTagResponseFormat
- return TriggeredTagsFormat(
- triggers=s_tag["triggers"],
- tags=[
- TagFormat(
- begin=structure["begin"],
- content=JSONSchemaFormat(json_schema=structure["schema"]),
- end=structure["end"],
- )
- for structure in s_tag["structures"]
- ],
- )
- # StructuralTagResponseFormat
- return StructuralTag.model_validate(s_tag).format
- return None
-
-
def _adjust_output_format(
request: ChatCompletionRequest | ResponsesRequest,
) -> ChatCompletionRequest | ResponsesRequest:
@@ -576,7 +534,7 @@ def _adjust_output_format(
if params is None:
return request
- final_content = _params_to_final_content(params)
+ final_content = structured_outputs_to_format(params)
if final_content is None:
return request
diff --git a/vllm/renderers/online_renderer.py b/vllm/renderers/online_renderer.py
index 139b783aaa..be0056a350 100644
--- a/vllm/renderers/online_renderer.py
+++ b/vllm/renderers/online_renderer.py
@@ -259,15 +259,7 @@ class OnlineRenderer:
)
else:
# For GPT-OSS.
- if self.parser is not None:
- # HarmonyParser doesn't need chat_template_kwargs
- # TODO: Unify adjust_request() call with non-harmony branch
- self.parser(
- self.renderer.get_tokenizer(),
- request.tools,
- model_config=self.model_config,
- ).adjust_request(request=request)
-
+ self.adjust_harmony_request(request)
should_include_tools = tool_dicts is not None
conversation, engine_inputs = self._make_request_with_harmony(
request, should_include_tools
@@ -355,14 +347,7 @@ class OnlineRenderer:
previous_response_outputs: list[ResponseOutputItem] | None,
tool_server: "ToolServer | None",
) -> ResponsesRenderResult | ErrorResponse:
- if self.parser is not None:
- # HarmonyParser doesn't need chat_template_kwargs
- # TODO: Unify adjust_request() call with non-harmony branch
- self.parser(
- self.renderer.get_tokenizer(),
- request.tools,
- model_config=self.model_config,
- ).adjust_request(request=request)
+ self.adjust_harmony_request(request)
if previous_messages is not None and any(
not isinstance(message, OpenAIMessage) for message in previous_messages
@@ -488,6 +473,24 @@ class OnlineRenderer:
engine_input=engine_inputs[0],
)
+ def adjust_harmony_request(
+ self, request: ChatCompletionRequest | ResponsesRequest
+ ) -> None:
+ """Apply the Harmony parser's ``adjust_request`` to ``request`` in place.
+
+ Every Harmony render path must call this before sampling params are
+ built from ``request``.
+ """
+ if self.parser is None:
+ return
+ # HarmonyParser doesn't need chat_template_kwargs
+ # TODO: Unify adjust_request() call with non-harmony branch
+ self.parser(
+ self.renderer.get_tokenizer(),
+ request.tools,
+ model_config=self.model_config,
+ ).adjust_request(request=request)
+
def _make_request_with_harmony(
self,
request: ChatCompletionRequest,
diff --git a/vllm/tool_parsers/structural_tag_registry.py b/vllm/tool_parsers/structural_tag_registry.py
index ba86395b7a..f3e5c9ec98 100644
--- a/vllm/tool_parsers/structural_tag_registry.py
+++ b/vllm/tool_parsers/structural_tag_registry.py
@@ -121,7 +121,7 @@ def _with_tool_strict(
return tool
-def _resolve_tool_strictness(
+def resolve_tool_strictness(
tools: Sequence[ChatCompletionToolsParam | ResponsesTool],
tool_choice: ToolChoice,
strict_level: ToolStrictLevel,
@@ -158,7 +158,7 @@ def get_model_structural_tag(
if not tools or tool_choice == "none":
return None
- tools = _resolve_tool_strictness(tools, tool_choice, strict_level)
+ tools = resolve_tool_strictness(tools, tool_choice, strict_level)
if tools is None:
return None
diff --git a/vllm/utils/multi_stream_utils.py b/vllm/utils/multi_stream_utils.py
index 6190b76991..5cb7f00f76 100644
--- a/vllm/utils/multi_stream_utils.py
+++ b/vllm/utils/multi_stream_utils.py
@@ -70,6 +70,7 @@ def execute_in_parallel(
done_events: list[torch.cuda.Event],
aux_streams: list[torch.cuda.Stream] | None = None,
enable: bool = False,
+ default_first: bool = False,
) -> tuple[Any, list[Any]]:
"""Run default_fn on the current stream and aux_fns concurrently on
aux_streams.
@@ -97,6 +98,10 @@ def execute_in_parallel(
so callers that pass aux_streams must also pass enable=True
(typically gated by an env var) to actually overlap. When False,
execution falls back to sequential on the current stream.
+ default_first: When True, enqueue the default chain before the aux
+ chains. The default chain is typically the longest, so it starts
+ first; the aux streams still only wait on start_event, so they
+ overlap the default chain. Defaults to False.
Returns:
Tuple of (default_result, aux_results) where aux_results[i] is the
@@ -117,6 +122,8 @@ def execute_in_parallel(
pending: list[torch.cuda.Event] = []
start_event.record()
+ if default_first:
+ default_result = default_fn()
for i, fn in enumerate(aux_fns):
if fn is None:
continue
@@ -125,8 +132,8 @@ def execute_in_parallel(
aux_results[i] = fn()
done_events[i].record()
pending.append(done_events[i])
-
- default_result = default_fn()
+ if not default_first:
+ default_result = default_fn()
for ev in pending:
ev.wait()
diff --git a/vllm/v1/attention/backend.py b/vllm/v1/attention/backend.py
index b5cd6e4b32..5d661634d7 100644
--- a/vllm/v1/attention/backend.py
+++ b/vllm/v1/attention/backend.py
@@ -870,15 +870,14 @@ class AttentionImplBase(ABC, Generic[T]):
def __new__(cls, *args, **kwargs):
# use __new__ so that all subclasses will call this
self = super().__new__(cls)
- try:
- from vllm.distributed.parallel_state import get_dcp_group
+ from vllm.config import get_current_vllm_config_or_none
+ from vllm.distributed.parallel_state import get_dcp_world_size_and_rank
- self.dcp_world_size = get_dcp_group().world_size
- self.dcp_rank = get_dcp_group().rank_in_group
- except AssertionError:
- # DCP might not be initialized in testing
- self.dcp_world_size = 1
- self.dcp_rank = 0
+ # Replicated drafts run at DCP=1 inside a DCP target's process group.
+ config = get_current_vllm_config_or_none()
+ self.dcp_world_size, self.dcp_rank = get_dcp_world_size_and_rank(
+ config is None or config.parallel_config.decode_context_parallel_size > 1
+ )
try:
from vllm.distributed.parallel_state import get_pcp_group
diff --git a/vllm/v1/attention/backends/flash_attn.py b/vllm/v1/attention/backends/flash_attn.py
index 129b3e39c3..6035678394 100755
--- a/vllm/v1/attention/backends/flash_attn.py
+++ b/vllm/v1/attention/backends/flash_attn.py
@@ -60,7 +60,10 @@ from vllm.config import (
get_layers_from_vllm_config,
)
from vllm.config.cache import CacheDType
-from vllm.distributed.parallel_state import get_dcp_group
+from vllm.distributed.parallel_state import (
+ get_dcp_group,
+ get_dcp_world_size_and_rank,
+)
from vllm.logger import init_logger
from vllm.platforms.interface import DeviceCapability
from vllm.utils.math_utils import cdiv, round_up
@@ -726,15 +729,9 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad
num_qo_heads=self.num_heads_q,
)
- try:
- from vllm.distributed.parallel_state import get_dcp_group
-
- self.dcp_world_size = get_dcp_group().world_size
- self.dcp_rank = get_dcp_group().rank_in_group
- except AssertionError:
- # DCP might not be initialized in testing
- self.dcp_world_size = 1
- self.dcp_rank = 0
+ self.dcp_world_size, self.dcp_rank = get_dcp_world_size_and_rank(
+ kv_cache_spec.dcp_sharded
+ )
# Fused draft decode reuses the captured metadata object across draft
# steps. For DCP, build-time host-side decisions such as
diff --git a/vllm/v1/attention/backends/flashinfer.py b/vllm/v1/attention/backends/flashinfer.py
index f40c75b474..ecc66a75ca 100755
--- a/vllm/v1/attention/backends/flashinfer.py
+++ b/vllm/v1/attention/backends/flashinfer.py
@@ -30,7 +30,10 @@ from vllm.config import (
get_current_vllm_config_or_none,
)
from vllm.config.cache import CacheDType
-from vllm.distributed.parallel_state import get_dcp_group
+from vllm.distributed.parallel_state import (
+ get_dcp_group,
+ get_dcp_world_size_and_rank,
+)
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
@@ -641,6 +644,7 @@ class FlashInferTrtllmAPIDecode:
@dataclass
class FlashInferMetadata:
+ kv_cache_layout: KVCacheLayout
num_actual_tokens: int
"""Total number of tokens in the batch (excluding padding)."""
@@ -696,6 +700,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
self.cache_config = vllm_config.cache_config
self.model_config = vllm_config.model_config
self.attention_config = vllm_config.attention_config
+ self._num_speculative_tokens = vllm_config.num_speculative_tokens
self._workspace_buffer = None
self._prefill_wrapper: (
BatchPrefillWithPagedKVCacheWrapper | BatchDCPPrefillWrapper | None
@@ -747,17 +752,12 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
self._decode_cudagraph_max_bs,
self.compilation_config.max_cudagraph_capture_size,
)
- try:
- self.dcp_world_size = get_dcp_group().world_size
- self.dcp_rank = get_dcp_group().rank_in_group
- self.dcp_kv_cache_interleave_size = (
- vllm_config.parallel_config.cp_kv_cache_interleave_size
- )
- except AssertionError:
- # DCP might not be initialized in testing
- self.dcp_world_size = 1
- self.dcp_rank = 0
- self.dcp_kv_cache_interleave_size = 1
+ self.dcp_world_size, self.dcp_rank = get_dcp_world_size_and_rank(
+ kv_cache_spec.dcp_sharded
+ )
+ self.dcp_kv_cache_interleave_size = (
+ vllm_config.parallel_config.cp_kv_cache_interleave_size
+ )
self.use_dcp = self.dcp_world_size > 1
self.dcp_a2a = (
self.use_dcp and vllm_config.parallel_config.dcp_comm_backend == "a2a"
@@ -1424,6 +1424,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
# Leave prefill/decode/cascade_wrapper empty, to be populated
# case by case depending on the batch contents and backend selection.
attn_metadata = FlashInferMetadata(
+ kv_cache_layout=self.kv_cache_layout,
num_actual_tokens=num_actual_tokens,
slot_mapping=common_attn_metadata.slot_mapping,
q_data_type_prefill=self.q_data_type_prefill,
@@ -1446,8 +1447,15 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
# (block_tables, seq_lens) directly.
needs_seq_lens_cpu = self.use_dcp or use_cascade or not all_uses_trtllm
if needs_seq_lens_cpu:
- with gpu_sync_allowed():
- seq_lens_cpu = common_attn_metadata.seq_lens.cpu()
+ if (
+ self._num_speculative_tokens == 0
+ and common_attn_metadata.seq_lens_cpu_upper_bound is not None
+ ):
+ # No speculative token accounting, so the upper bound is exact.
+ seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
+ else:
+ with gpu_sync_allowed():
+ seq_lens_cpu = common_attn_metadata.seq_lens.cpu()
else:
seq_lens_cpu = None
@@ -1904,8 +1912,6 @@ class FlashInferImpl(AttentionImpl):
num_heads, num_kv_heads, is_prefill=False
)
vllm_config = get_current_vllm_config_or_none()
- # The layout is resolved after model construction, so read it lazily.
- self.cache_config = vllm_config.cache_config if vllm_config else None
# Query pre-quantization needs a single dtype for the whole query tensor.
# SM90 XQA needs BF16/FP16-Q for decode and FP8 for prefill,
# so only enable this for SM100 trtllm-gen where both use FP8-Q.
@@ -1941,11 +1947,6 @@ class FlashInferImpl(AttentionImpl):
else:
self.dcp_combine = partial(cp_lse_ag_out_rs, is_lse_base_on_e=False)
- @property
- def kv_cache_layout(self) -> KVCacheLayout:
- assert self.cache_config is not None
- return self.cache_config.get_resolved_kv_cache_layout()
-
def fused_output_quant_supported(self, quant_key: QuantKey):
if quant_key == kNvfp4Dynamic and self.is_kvcache_nvfp4:
logger.warning_once(
@@ -2111,6 +2112,7 @@ class FlashInferImpl(AttentionImpl):
# performance to make sure it does not introduce any overhead.
num_actual_tokens = attn_metadata.num_actual_tokens
+ kv_cache_layout = attn_metadata.kv_cache_layout
# FlashInfer treats uint8 KV cache as NVFP4. vLLM stores FP8 KV cache
# as uint8 bytes, so pass FP8 caches with their logical dtype.
@@ -2134,7 +2136,7 @@ class FlashInferImpl(AttentionImpl):
if attn_metadata.use_cascade:
# Cascade attention (rare case).
assert attn_metadata.cascade_wrapper is not None
- stride_order = self.kv_cache_layout.layer_view_order
+ stride_order = kv_cache_layout.layer_view_order
if self.is_kvcache_nvfp4:
kv_cache_views = tuple(
cache.permute(*stride_order)
@@ -2154,7 +2156,7 @@ class FlashInferImpl(AttentionImpl):
num_decode_tokens = attn_metadata.num_decode_tokens
num_prefill_tokens = attn_metadata.num_prefill_tokens
- stride_order = self.kv_cache_layout.layer_view_order
+ stride_order = kv_cache_layout.layer_view_order
kv_cache_permute = kv_cache.permute(*stride_order) # HND and contiguous
# Fix degenerate strides on any size-1 dimension (e.g. num_kv_heads=1
# with TP=8). PyTorch permits non-canonical strides on size-1 dims;
@@ -2311,7 +2313,7 @@ class FlashInferImpl(AttentionImpl):
seq_lens_prefill = attn_metadata.prefill.seq_lens
# This path needs to be enabled with VLLM_KV_CACHE_LAYOUT = HND
- assert get_flashinfer_layout_string(self.kv_cache_layout) == "HND"
+ assert get_flashinfer_layout_string(kv_cache_layout) == "HND"
assert is_strictly_contiguous(prefill_query)
assert is_strictly_contiguous(workspace_buffer)
assert is_strictly_contiguous(block_tables_prefill)
@@ -2501,7 +2503,7 @@ class FlashInferImpl(AttentionImpl):
# trtllm-gen needs HND layout on SM100. XQA is selected
# separately on SM90 and does not use this SM100 layout gate.
if decode_with_trtllm_gen:
- assert get_flashinfer_layout_string(self.kv_cache_layout) == "HND"
+ assert get_flashinfer_layout_string(kv_cache_layout) == "HND"
else:
assert decode_with_xqa
assert is_strictly_contiguous(decode_query)
@@ -2547,7 +2549,7 @@ class FlashInferImpl(AttentionImpl):
window_left=self.window_left,
out=output_padded[:decode_query_tokens],
sinks=self.sinks,
- kv_layout=get_flashinfer_layout_string(self.kv_cache_layout),
+ kv_layout=get_flashinfer_layout_string(kv_cache_layout),
q_len_per_req=q_len_per_req,
mask=attn_metadata.decode.mask,
q_cu_seq_lens=attn_metadata.decode.q_cu_seq_lens,
@@ -2614,7 +2616,7 @@ class FlashInferImpl(AttentionImpl):
sinks=self.sinks,
o_sf_scale=self.o_sf_scale,
out=out,
- kv_layout=get_flashinfer_layout_string(self.kv_cache_layout),
+ kv_layout=get_flashinfer_layout_string(kv_cache_layout),
backend=attn_metadata.decode.kernel.value,
q_len_per_req=q_len_per_req,
max_q_len=max_q_len,
diff --git a/vllm/v1/attention/backends/gdn_attn.py b/vllm/v1/attention/backends/gdn_attn.py
index 85173dc2e4..ae8a9437d5 100644
--- a/vllm/v1/attention/backends/gdn_attn.py
+++ b/vllm/v1/attention/backends/gdn_attn.py
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Backend for GatedDeltaNet attention."""
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
from typing import Literal
import torch
@@ -60,6 +60,7 @@ class GDNAttentionMetadata:
None # shape: [batch - num_spec_decodes,]
)
spec_sequence_masks: torch.Tensor | None = None # shape: [batch,]
+ spec_sequence_masks_cpu: torch.Tensor | None = None # shape: [batch,]
spec_token_indx: torch.Tensor | None = None
non_spec_token_indx: torch.Tensor | None = None
@@ -114,6 +115,18 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
self.use_full_cuda_graph: bool = (
self.compilation_config.cudagraph_mode.has_full_cudagraphs()
)
+ # update_block_table() keeps the source group's batch-level FULL graph
+ # buffers, so only MRV2, which also reuses metadata at capture, may use
+ # it.
+ self.supports_update_block_table = (
+ vllm_config.use_v2_model_runner
+ and device.type == "cuda"
+ # Not isinstance: KDA's RecoverSSM/checkpoint metadata is per group.
+ and type(self) is GDNAttentionMetadataBuilder
+ )
+ if self.supports_update_block_table:
+ # Opts into MRV2's CUDA-only aligned-index precompute.
+ self.mamba_aligned_state_indices: torch.Tensor | None = None
self.decode_cudagraph_max_bs: int = (
self.vllm_config.scheduler_config.max_num_seqs * (self.num_spec + 1)
@@ -220,12 +233,16 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
query_start_loc = m.query_start_loc
query_start_loc_cpu = m.query_start_loc_cpu
nums_dict, batch_ptr, token_chunk_offset_ptr = None, None, None
- block_table_tensor = mamba_get_block_table_tensor(
- m.block_table_tensor,
- m.seq_lens,
- self.kv_cache_spec,
- self.vllm_config.cache_config.mamba_cache_mode,
- )
+ aligned_state_indices = getattr(self, "mamba_aligned_state_indices", None)
+ if aligned_state_indices is not None:
+ block_table_tensor = aligned_state_indices[: m.num_reqs]
+ else:
+ block_table_tensor = mamba_get_block_table_tensor(
+ m.block_table_tensor,
+ m.seq_lens,
+ self.kv_cache_spec,
+ self.vllm_config.cache_config.mamba_cache_mode,
+ )
uniform_spec_sequence_length = None
spec_sequence_masks_cpu: torch.Tensor | None = None
@@ -460,12 +477,8 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
# metadata below is indexed by request.
batch_size = m.num_reqs
- if (
- self.use_full_cuda_graph
- and num_prefills == 0
- and num_decodes == 0
- and num_spec_decodes <= self.decode_cudagraph_max_bs
- and num_spec_decode_tokens <= self.decode_cudagraph_max_bs
+ if self._stage_spec_decode(
+ num_prefills, num_decodes, num_spec_decodes, num_spec_decode_tokens
):
assert spec_sequence_masks is not None
self.spec_state_indices_tensor[:num_spec_decodes].copy_(
@@ -506,12 +519,7 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
num_accepted_tokens = self.num_accepted_tokens[:batch_size]
num_accepted_tokens[num_spec_decodes:].fill_(1)
- if (
- self.use_full_cuda_graph
- and num_prefills == 0
- and num_spec_decodes == 0
- and num_decodes <= self.decode_cudagraph_max_bs
- ):
+ if self._stage_decode(num_prefills, num_decodes, num_spec_decodes):
self.non_spec_state_indices_tensor[:num_decodes].copy_(
non_spec_state_indices_tensor, non_blocking=True
)
@@ -546,6 +554,7 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
spec_state_indices_tensor=spec_state_indices_tensor,
non_spec_state_indices_tensor=non_spec_state_indices_tensor,
spec_sequence_masks=spec_sequence_masks,
+ spec_sequence_masks_cpu=spec_sequence_masks_cpu,
spec_token_indx=spec_token_indx,
non_spec_token_indx=non_spec_token_indx,
num_accepted_tokens=num_accepted_tokens,
@@ -556,6 +565,83 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
)
return attn_metadata
+ def _stage_spec_decode(
+ self,
+ num_prefills: int,
+ num_decodes: int,
+ num_spec_decodes: int,
+ num_spec_decode_tokens: int,
+ ) -> bool:
+ """Whether spec-decode metadata goes into the FULL cudagraph buffers."""
+ return (
+ self.use_full_cuda_graph
+ and num_prefills == 0
+ and num_decodes == 0
+ and num_spec_decodes <= self.decode_cudagraph_max_bs
+ and num_spec_decode_tokens <= self.decode_cudagraph_max_bs
+ )
+
+ def _stage_decode(
+ self, num_prefills: int, num_decodes: int, num_spec_decodes: int
+ ) -> bool:
+ """Whether decode metadata goes into the FULL cudagraph buffers."""
+ return (
+ self.use_full_cuda_graph
+ and num_prefills == 0
+ and num_spec_decodes == 0
+ and num_decodes <= self.decode_cudagraph_max_bs
+ )
+
+ def update_block_table(
+ self,
+ metadata: GDNAttentionMetadata,
+ blk_table: torch.Tensor,
+ slot_mapping: torch.Tensor,
+ ) -> GDNAttentionMetadata:
+ """Re-gather this group's state indices. The other fields are
+ batch-level and stay shared with ``metadata``."""
+ m = metadata
+ if self.vllm_config.cache_config.mamba_cache_mode == "align":
+ assert self.mamba_aligned_state_indices is not None
+ blk_table = self.mamba_aligned_state_indices
+ masks = m.spec_sequence_masks_cpu
+ spec_indices = non_spec_indices = prefill_indices = None
+ if masks is None:
+ non_spec_indices = blk_table[:, 0]
+ if m.num_prefills > 0:
+ prefill_indices = non_spec_indices[m.num_decodes :]
+ else:
+ spec_indices = blk_table[masks, : self.num_spec + 1]
+ if m.num_prefills > 0:
+ non_spec_indices = prefill_indices = blk_table[~masks, 0]
+
+ if self._stage_spec_decode(
+ m.num_prefills, m.num_decodes, m.num_spec_decodes, m.num_spec_decode_tokens
+ ):
+ assert m.spec_state_indices_tensor is not None
+ batch_size = m.spec_state_indices_tensor.shape[0]
+ self.spec_state_indices_tensor[: m.num_spec_decodes].copy_(
+ spec_indices, non_blocking=True
+ )
+ spec_indices = self.spec_state_indices_tensor[:batch_size]
+ spec_indices[m.num_spec_decodes :].fill_(NULL_BLOCK_ID)
+
+ if self._stage_decode(m.num_prefills, m.num_decodes, m.num_spec_decodes):
+ assert m.non_spec_state_indices_tensor is not None
+ batch_size = m.non_spec_state_indices_tensor.shape[0]
+ self.non_spec_state_indices_tensor[: m.num_decodes].copy_(
+ non_spec_indices, non_blocking=True
+ )
+ non_spec_indices = self.non_spec_state_indices_tensor[:batch_size]
+ non_spec_indices[m.num_decodes :].fill_(NULL_BLOCK_ID)
+
+ return replace(
+ m,
+ spec_state_indices_tensor=spec_indices,
+ non_spec_state_indices_tensor=non_spec_indices,
+ prefill_state_indices=prefill_indices,
+ )
+
def build_for_cudagraph_capture(
self, common_attn_metadata: CommonAttentionMetadata
):
diff --git a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse_sm90.py b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse_sm90.py
index 9dc899c903..a5f7757d38 100644
--- a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse_sm90.py
+++ b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse_sm90.py
@@ -17,14 +17,14 @@ encoded by the indexer's selection, so ``causal=False``.
CUDA-graph handling: ``plan()`` copies its inputs to host unconditionally,
so it must stay outside graph capture. Each metadata builder owns a wrapper,
reserved capture-stable device buffers, and the plan parameters. The wrapper
-bakes the per-row ``kv_len`` into its int schedule
-at plan() time — ``run()`` never reads the device-side buffer — so the
-metadata builder replans every step (outside capture) with exact host-side
-lengths derived from the batch's sequence lengths; a full-width schedule
-would send the kernel past each row's valid count into the -1 tail of the
-converted index buffer (illegal address). Per-step content (top-k slots)
-is written into the reserved buffers by kernels inside the captured
-forward, and captured runs read the refreshed plan buffers on replay.
+bakes the per-row ``kv_len`` into its int schedule at plan() time — ``run()``
+never reads the device-side buffer. The builder plans (outside capture) from
+sync-free host upper bounds and clamps each scheduled work item's ``kv_end``
+on device to the exact valid count, so the kernel never reads past a row's
+valid prefix (the -1 tail of the converted index buffer) and async scheduling
+needs no D2H sync. Per-step content (top-k slots) is written into the
+reserved buffers by kernels inside the captured forward, and captured runs
+read the refreshed plan buffers on replay.
KV cache format: plain contiguous E4M3 ``[num_blocks, block_size, 512]``
(uint8 storage) with a per-tensor ``k_scale``; BF16 caches also work. The
@@ -63,6 +63,26 @@ from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheLayout
_FP8_KV_DTYPES = ("fp8", "fp8_e4m3")
_WORKSPACE_BYTES = 128 * 1024 * 1024
+# TODO: FlashInfer bakes per-row kv_len into the plan on the host, which
+# forces a D2H sync under async scheduling. The plan-info layout constants
+# below and the int-workspace clamp in _SM90State work around this by
+# planning from sync-free upper bounds; delete them once FlashInfer accepts
+# an upper-bound plan() that clamps kv_end internally, or exposes a
+# GPU-side plan API (upstream ask:
+# https://github.com/flashinfer-ai/flashinfer/issues/5559, see
+# vllm-project/vllm#58684).
+# FlashInfer MLAPlanInfo layout (scheduler.cuh) field indices.
+_PLAN_INFO_LEN = 18
+_PI_NUM_BLKS_Y = 1
+_PI_Q_INDPTR = 2
+_PI_KV_INDPTR = 3
+_PI_KV_START = 13
+_PI_KV_END = 14
+_PI_WORK_INDPTR = 15
+# Planned lengths exceed the host upper bound by up to this many tokens so a
+# plan stays reusable across draft steps and consecutive decode steps; the
+# device-side clamp restores the exact lengths.
+_PLAN_SLACK = 32
@triton.jit
@@ -82,6 +102,40 @@ def _pack_topk_indices(
tl.store(indices + start + cols, tl.maximum(values, 0), mask=mask)
+@triton.jit
+def _clamp_work_kv_end(
+ int_ws,
+ saved_kv_end,
+ seq_lens,
+ query_start_loc,
+ req_id_per_token,
+ num_works_idx,
+ q_indptr_off,
+ kv_start_off,
+ kv_end_off,
+ TOPK: tl.constexpr,
+ KPOOL: tl.constexpr,
+ BLOCK_SIZE: tl.constexpr,
+):
+ # Clamp each planned work item to its row's exact valid count, derived
+ # from the device batch layout exactly as the convert kernel counts it.
+ works = tl.program_id(0) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
+ mask = works < tl.load(int_ws + num_works_idx)
+ row = tl.load(int_ws + q_indptr_off + works, mask=mask, other=0)
+ req = tl.load(req_id_per_token + row, mask=mask, other=0)
+ seq_len = tl.load(seq_lens + req, mask=mask, other=0)
+ query_end = tl.load(query_start_loc + req + 1, mask=mask, other=0)
+ ctx = seq_len - query_end + row + 1
+ valid = tl.where(ctx <= TOPK, ctx, TOPK + ctx % KPOOL)
+ start = tl.load(int_ws + kv_start_off + works, mask=mask, other=0)
+ end = tl.load(saved_kv_end + works, mask=mask, other=0)
+ tl.store(
+ int_ws + kv_end_off + works,
+ tl.maximum(tl.minimum(end, valid), start),
+ mask=mask,
+ )
+
+
class FlashInferMLASparseSM90Backend(AttentionBackend):
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
@@ -190,6 +244,8 @@ class _SM90State:
kv_lora_rank: int,
qk_rope_head_dim: int,
sm_scale: float,
+ index_topk: int,
+ index_kpool: int,
) -> None:
from flashinfer.mla import BatchMLAPagedAttentionWrapper
@@ -202,6 +258,10 @@ class _SM90State:
self.kv_lora_rank = kv_lora_rank
self.qk_rope_head_dim = qk_rope_head_dim
self.sm_scale = sm_scale
+ self.index_topk = index_topk
+ self.index_kpool = max(index_kpool, 1)
+ # Largest valid count the convert kernel can produce for any context.
+ self.max_valid = min(self.index_topk + self.index_kpool - 1, topk_width)
# User-reserved buffers: with use_cuda_graph=True plan() refreshes
# these in place, so run()'s captured kernels always read them.
self.kv_indices = torch.zeros(
@@ -226,24 +286,54 @@ class _SM90State:
self._lens_cpu = torch.full(
(self.max_tokens,), self.topk_width, dtype=torch.int32
)
+ # Planned per-work kv_end of the current plan, restored by each clamp;
+ # sized on first plan to the planner's work capacity.
+ self._saved_kv_end = torch.empty(0, dtype=torch.int32, device=device)
+ self._planned_num_tokens = -1
+ self._int_ws: torch.Tensor | None = None
+ self._plan_offsets: tuple[int, int, int, int] | None = None
- def plan(self, num_tokens: int, kv_lens: torch.Tensor) -> None:
- """Replan with exact per-row KV lengths (CPU int32, ``[num_tokens]``).
+ def plan(
+ self,
+ num_tokens: int,
+ kv_lens: torch.Tensor,
+ cam: CommonAttentionMetadata | None,
+ req_id_per_token: torch.Tensor | None,
+ ) -> None:
+ """Plan per-row KV lengths (CPU int32, ``[num_tokens]``).
- The wrapper bakes kv_len into its int schedule from host values;
- run() never reads the device kv_len_arr buffer. Scheduling with the
- full buffer width while kv_indices rows carry a -1 tail past each
- row's valid count makes the kernel compute ``-1 * ckv_stride_page``
- (illegal address), so the lengths must be exact at plan time and
- replanned every step as contexts grow. Must run outside CUDA graph
- capture: the in-place refreshed plan_info/indptr buffers are what
- captured runs read.
+ The wrapper bakes kv_len into its int schedule from host values, so
+ rows past their valid count would send the kernel into the -1 tail of
+ the converted index buffer. With ``cam`` and ``req_id_per_token``
+ (device batch layout), ``kv_lens`` only needs to upper-bound the
+ valid counts: the plan is padded by ``_PLAN_SLACK`` and reused while
+ the bound still fits, and every call clamps each work item's kv_end
+ on device to the exact count, so no D2H sync is needed. Without them
+ the lengths must be exact. Must run outside CUDA graph capture: the
+ in-place refreshed plan_info/indptr buffers are what captured runs
+ read.
"""
if torch.cuda.is_current_stream_capturing():
raise RuntimeError(
"FlashInferMLASparseSM90 plan() called inside CUDA graph "
"capture; lengths must be planned host-side before capture."
)
+ kv_lens = kv_lens.to(torch.int32)
+ if cam is None or req_id_per_token is None:
+ self._plan(num_tokens, kv_lens)
+ self._planned_num_tokens = -1
+ return
+ planned = self._lens_cpu[:num_tokens]
+ if num_tokens != self._planned_num_tokens or not bool(
+ ((planned >= kv_lens) & (planned <= kv_lens + _PLAN_SLACK)).all()
+ ):
+ self._plan(
+ num_tokens, torch.clamp(kv_lens + _PLAN_SLACK, max=self.max_valid)
+ )
+ self._planned_num_tokens = num_tokens
+ self._clamp_kv_end(cam, req_id_per_token)
+
+ def _plan(self, num_tokens: int, kv_lens: torch.Tensor) -> None:
# CPU staging buffers are filled in place: plan() runs per step
# (once per draft/verify metadata build), so per-call allocations and
# device round trips are on the hot path. Passing CPU tensors lets
@@ -257,13 +347,16 @@ class _SM90State:
# matching the exact lengths, including zero-length padded rows.
torch.clamp(self._arange_cpu, max=num_tokens, out=self._qo_cpu)
self._lens_cpu.zero_()
- self._lens_cpu[:num_tokens] = kv_lens.to(torch.int32)
+ self._lens_cpu[:num_tokens] = kv_lens
self._kv_cpu[0] = 0
torch.cumsum(self._lens_cpu, dim=0, out=self._kv_cpu[1:])
+ # Only the live prefix of kv_indices: the wrapper snapshots and
+ # copies whatever is passed.
+ num_indices = max(int(self._kv_cpu[-1]), 1)
self.wrapper.plan(
self._qo_cpu,
self._kv_cpu,
- self.kv_indices,
+ self.kv_indices[:num_indices],
self._lens_cpu,
self.num_heads,
self.kv_lora_rank, # head_dim_ckv
@@ -274,6 +367,42 @@ class _SM90State:
q_data_type=torch.bfloat16,
kv_data_type=self.kv_dtype,
)
+ plan_info = [int(x) for x in self.wrapper._plan_info]
+ assert len(plan_info) == _PLAN_INFO_LEN, plan_info
+ # Per-work arrays are laid out back to back at the planner's capacity.
+ max_works = (plan_info[_PI_KV_INDPTR] - plan_info[_PI_Q_INDPTR]) // 4
+ if self._saved_kv_end.numel() != max_works:
+ self._saved_kv_end = torch.empty(
+ max_works, dtype=torch.int32, device=self.device
+ )
+ self._int_ws = self.wrapper._planned_backend._int_workspace_buffer.view(
+ torch.int32
+ )
+ self._plan_offsets = (
+ plan_info[_PI_WORK_INDPTR] // 4 + plan_info[_PI_NUM_BLKS_Y],
+ plan_info[_PI_Q_INDPTR] // 4,
+ plan_info[_PI_KV_START] // 4,
+ plan_info[_PI_KV_END] // 4,
+ )
+ kv_end_off = self._plan_offsets[3]
+ self._saved_kv_end.copy_(self._int_ws[kv_end_off : kv_end_off + max_works])
+
+ def _clamp_kv_end(
+ self, cam: CommonAttentionMetadata, req_id_per_token: torch.Tensor
+ ) -> None:
+ assert self._int_ws is not None and self._plan_offsets is not None
+ block = 1024
+ _clamp_work_kv_end[(triton.cdiv(self._saved_kv_end.numel(), block),)](
+ self._int_ws,
+ self._saved_kv_end,
+ cam.seq_lens,
+ cam.query_start_loc,
+ req_id_per_token,
+ *self._plan_offsets,
+ TOPK=self.index_topk,
+ KPOOL=self.index_kpool,
+ BLOCK_SIZE=block,
+ )
def pack_indices(self, topk_slots: torch.Tensor) -> None:
num_tokens, width = topk_slots.shape
@@ -312,6 +441,8 @@ class FlashInferMLASparseSM90Builder(FlashInferMLASparseMetadataBuilder):
)
topk_indices_buffer = impl.topk_indices_buffer
assert topk_indices_buffer is not None
+ hf_config = vllm_config.model_config.hf_text_config
+ assert hf_config.index_topk is not None
self.state = _SM90State(
device,
impl.num_heads,
@@ -321,20 +452,18 @@ class FlashInferMLASparseSM90Builder(FlashInferMLASparseMetadataBuilder):
kv_lora_rank=impl.kv_lora_rank,
qk_rope_head_dim=impl.qk_rope_head_dim,
sm_scale=impl.scale,
+ index_topk=int(hf_config.index_topk),
+ index_kpool=int(getattr(hf_config, "index_kpool", 1) or 1),
+ )
+ spec_config = vllm_config.speculative_config
+ self._adaptive_verification = bool(
+ spec_config is not None and spec_config.enable_adaptive_verification
)
- # seq_lens_cpu_upper_bound is optimistic on decode rows under async
- # spec decode, so the fast sync-free path is only safe without it;
- # under async scheduling the exact (device) positions are used at
- # the cost of one D2H sync per metadata build.
- self._async_scheduling = bool(vllm_config.scheduler_config.async_scheduling)
- hf_config = vllm_config.model_config.hf_text_config
- assert hf_config.index_topk is not None
- self._index_topk = int(hf_config.index_topk)
- self._index_kpool = int(getattr(hf_config, "index_kpool", 1) or 1)
- def _kv_lens_host(self, cam: CommonAttentionMetadata) -> tuple[int, torch.Tensor]:
- """Exact per-row KV lengths, host-side (the flashinfer wrapper bakes
- them into its schedule at plan time; there is no device-side path).
+ def _kv_lens_host(
+ self, cam: CommonAttentionMetadata
+ ) -> tuple[int, torch.Tensor, bool]:
+ """Host per-row KV lengths for the plan and whether they are exact.
A row for the j-th query token of request i attends
``seq_lens[i] - q_len[i] + j + 1`` tokens. The indexer's selection
@@ -343,53 +472,41 @@ class FlashInferMLASparseSM90Builder(FlashInferMLASparseMetadataBuilder):
``index_topk`` pool-expanded tokens plus the trailing incomplete
pool (valid == ``index_topk + context % index_kpool``). Both match
the count of non -1 entries the convert kernel produces.
+
+ With ``seq_lens_cpu_upper_bound`` (optimistic under async spec
+ decode) returns sync-free upper bounds, which the plan clamps on
+ device. Otherwise returns exact lengths via a D2H copy.
"""
num_reqs = cam.num_reqs
qsl = cam.query_start_loc_cpu[: num_reqs + 1]
num_rows = int(qsl[-1])
if num_rows == 0:
- return 0, torch.zeros(0, dtype=torch.int32)
- # Row context == position + 1. Without async scheduling the
- # maintained host upper bound is exact, giving a sync-free path;
- # under async scheduling it is optimistic on decode rows, so fall
- # back to the exact device positions (one D2H sync per build). The
- # seq_lens derivation equals position + 1 because positions are
- # contiguous per request.
+ return 0, torch.zeros(0, dtype=torch.int32), True
+ state = self.state
sl_host = cam.seq_lens_cpu_upper_bound
- positions = cam.positions
- if not self._async_scheduling and sl_host is not None:
- seq_lens = sl_host[:num_reqs].to(torch.int32)
- q_lens = qsl[1:] - qsl[:-1]
- first_pos = seq_lens - q_lens
- req_of_row = torch.repeat_interleave(
- torch.arange(num_reqs, dtype=torch.int64), q_lens.to(torch.int64)
- )
- rows = torch.arange(num_rows, dtype=torch.int32)
- ctx = (
- first_pos.to(torch.int64)[req_of_row]
- + rows.to(torch.int64)
- - qsl.to(torch.int64)[req_of_row]
- + 1
- )
- elif positions is not None and num_rows <= positions.shape[0]:
- ctx = positions[:num_rows].cpu().to(torch.int64) + 1
+ if sl_host is None:
+ ctx = self._row_contexts(qsl, cam.seq_lens[:num_reqs].cpu())
+ topk, kpool = state.index_topk, state.index_kpool
+ lens = torch.where(ctx <= topk, ctx, topk + ctx % kpool)
+ return num_rows, lens.to(torch.int32), True
+ if self._adaptive_verification:
+ # The host query split may not match the device one, so bound
+ # every row by the longest request.
+ ctx = torch.full((num_rows,), int(sl_host[:num_reqs].max()))
else:
- seq_lens = cam.seq_lens[:num_reqs].cpu().to(torch.int32)
- q_lens = qsl[1:] - qsl[:-1]
- first_pos = seq_lens - q_lens
- req_of_row = torch.repeat_interleave(
- torch.arange(num_reqs, dtype=torch.int64), q_lens.to(torch.int64)
- )
- rows = torch.arange(num_rows, dtype=torch.int32)
- ctx = (
- first_pos.to(torch.int64)[req_of_row]
- + rows.to(torch.int64)
- - qsl.to(torch.int64)[req_of_row]
- + 1
- )
- topk = self._index_topk
- lens = torch.where(ctx <= topk, ctx, topk + ctx % self._index_kpool)
- return num_rows, lens.to(torch.int32)
+ ctx = self._row_contexts(qsl, sl_host[:num_reqs])
+ return num_rows, torch.clamp(ctx, max=state.max_valid), False
+
+ @staticmethod
+ def _row_contexts(qsl: torch.Tensor, seq_lens: torch.Tensor) -> torch.Tensor:
+ num_reqs = qsl.shape[0] - 1
+ q_lens = (qsl[1:] - qsl[:-1]).to(torch.int64)
+ first_ctx = seq_lens.to(torch.int64) - q_lens + 1
+ req_of_row = torch.repeat_interleave(
+ torch.arange(num_reqs, dtype=torch.int64), q_lens
+ )
+ rows = torch.arange(int(qsl[-1]), dtype=torch.int64)
+ return first_ctx[req_of_row] + rows - qsl.to(torch.int64)[req_of_row]
def build(
self,
@@ -399,9 +516,9 @@ class FlashInferMLASparseSM90Builder(FlashInferMLASparseMetadataBuilder):
) -> FlashInferMLASparseSM90Metadata:
metadata = super().build(common_prefix_len, common_attn_metadata, fast_build)
assert isinstance(metadata, FlashInferMLASparseSM90Metadata)
- # Replan every step outside any CUDA graph capture with this step's
- # exact per-row lengths; captured runs read the refreshed buffers.
- num_rows, kv_lens = self._kv_lens_host(common_attn_metadata)
+ # Plan outside any CUDA graph capture; captured runs read the
+ # refreshed buffers.
+ num_rows, kv_lens, exact = self._kv_lens_host(common_attn_metadata)
# MHA prefills are removed from q before forward_mqa. Plan only the
# decode prefix in that case, using the same routing decision as MLA.
if metadata.num_prefills > 0 and self._attention_layer._use_sparse_mha(
@@ -409,7 +526,12 @@ class FlashInferMLASparseSM90Builder(FlashInferMLASparseMetadataBuilder):
):
num_rows = metadata.num_decode_tokens
kv_lens = kv_lens[:num_rows]
- self.state.plan(num_rows, kv_lens)
+ if exact:
+ self.state.plan(num_rows, kv_lens, None, None)
+ else:
+ self.state.plan(
+ num_rows, kv_lens, common_attn_metadata, metadata.req_id_per_token
+ )
metadata.state = self.state
return metadata
diff --git a/vllm/v1/attention/backends/mla/prefill/registry.py b/vllm/v1/attention/backends/mla/prefill/registry.py
index 1cf1761a77..f7ec31e153 100644
--- a/vllm/v1/attention/backends/mla/prefill/registry.py
+++ b/vllm/v1/attention/backends/mla/prefill/registry.py
@@ -53,6 +53,10 @@ class MLAPrefillBackendEnum(Enum, metaclass=_MLAPrefillBackendEnumMeta):
"AiterFlashAttnPrefillBackend"
)
CPU = "vllm.v1.attention.backends.mla.prefill.cpu_sdpa.CPUSDPAMLAPrefillBackend"
+ ZEN_CPU = (
+ "vllm.v1.attention.backends.mla.prefill.zen_cpu_sdpa."
+ "ZenCPUSDPAMLAPrefillBackend"
+ )
# Placeholder for third-party/custom backends - must be registered before use
# set to None to avoid alias with other backend, whose value is an empty string
CUSTOM = None
diff --git a/vllm/v1/attention/backends/mla/prefill/selector.py b/vllm/v1/attention/backends/mla/prefill/selector.py
index a2b74e4327..0cb29f51b4 100644
--- a/vllm/v1/attention/backends/mla/prefill/selector.py
+++ b/vllm/v1/attention/backends/mla/prefill/selector.py
@@ -110,8 +110,18 @@ def get_mla_prefill_backend(
from vllm.platforms import current_platform
if current_platform.is_cpu():
- logger.info_once("Using CPU SDPA MLA prefill backend.")
- return MLAPrefillBackendEnum.CPU.get_class()
+ # CPUs have no compute capability, so the capability-driven priority
+ # path below does not apply. Prefer an accelerator-specific CPU backend
+ # when its kernels are present, else the generic SDPA one.
+ for backend_enum in (MLAPrefillBackendEnum.ZEN_CPU, MLAPrefillBackendEnum.CPU):
+ try:
+ cpu_backend_cls = backend_enum.get_class()
+ except ImportError:
+ continue
+ if cpu_backend_cls.is_available():
+ logger.info_once("Using %s MLA prefill backend.", backend_enum.name)
+ return cpu_backend_cls
+ raise ValueError("No valid CPU MLA prefill backend found.")
device_capability = current_platform.get_device_capability()
if device_capability is None:
diff --git a/vllm/v1/attention/backends/mla/prefill/zen_cpu_sdpa.py b/vllm/v1/attention/backends/mla/prefill/zen_cpu_sdpa.py
new file mode 100644
index 0000000000..88c6ce98a0
--- /dev/null
+++ b/vllm/v1/attention/backends/mla/prefill/zen_cpu_sdpa.py
@@ -0,0 +1,106 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""CPU MLA prefill backend using zentorch SDPA on AMD Zen CPUs.
+
+Subclasses the generic CPU SDPA backend and replaces the per-request attention
+with ``zentorch_sdpa``, which fuses the scale, causal mask, softmax and both
+matmuls of the base implementation into a single call.
+
+``zentorch_sdpa`` does not return a usable log-sum-exp, so requests that need
+one (context chunks, prefix/suffix merge) are delegated to the base backend.
+
+Availability is reported through :meth:`is_available`, so the prefill selector
+gates this backend once at selection time rather than per attention call.
+"""
+
+import torch
+
+from vllm.model_executor.kernels.linear.zentorch_utils import has_zentorch_op
+from vllm.v1.attention.backends.mla.prefill.cpu_sdpa import CPUSDPAMLAPrefillBackend
+
+
+class ZenCPUSDPAMLAPrefillBackend(CPUSDPAMLAPrefillBackend):
+ """MLA prefill backend for AMD Zen CPUs backed by ``zentorch_sdpa``."""
+
+ @staticmethod
+ def get_name() -> str:
+ return "ZEN_CPU_SDPA_MLA"
+
+ @classmethod
+ def is_available(cls) -> bool:
+ return has_zentorch_op(["zentorch_sdpa"])
+
+ @staticmethod
+ def _sdpa_layout(
+ q_r: torch.Tensor,
+ k_r: torch.Tensor,
+ v_r: torch.Tensor,
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, int]:
+ """Convert one request's [S, NH, D] tensors to the [1, NH, S, D] layout.
+
+ MLA has ``v_head_dim < qk_head_dim``, which the fused kernel does not
+ accept, so V is padded up to the query head dim. The original V head
+ dim is returned so the caller can slice the padding off afterwards.
+ """
+ h_q = q_r.shape[-1]
+ h_v = v_r.shape[-1]
+ pad = h_q - h_v
+
+ qs = q_r.transpose(0, 1).unsqueeze(0).contiguous()
+ ks = k_r.transpose(0, 1).unsqueeze(0).contiguous()
+ vs = v_r.transpose(0, 1).unsqueeze(0).contiguous()
+ if pad > 0:
+ vs = torch.nn.functional.pad(vs, (0, pad))
+ return qs, ks, vs, h_v
+
+ def _ragged_sdpa(
+ self,
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ q_cu_seq_lens: torch.Tensor,
+ kv_cu_seq_lens: torch.Tensor,
+ causal: bool,
+ return_softmax_lse: bool,
+ ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
+ if return_softmax_lse:
+ # zentorch_sdpa's second return is not a usable LSE, so fall back
+ # to the base fp32 implementation whenever one is required.
+ return super()._ragged_sdpa(
+ q,
+ k,
+ v,
+ q_cu_seq_lens,
+ kv_cu_seq_lens,
+ causal,
+ return_softmax_lse,
+ )
+
+ num_reqs = q_cu_seq_lens.numel() - 1
+ assert num_reqs == kv_cu_seq_lens.numel() - 1
+
+ out_chunks: list[torch.Tensor] = []
+ for r in range(num_reqs):
+ q_start = int(q_cu_seq_lens[r])
+ q_end = int(q_cu_seq_lens[r + 1])
+ kv_start = int(kv_cu_seq_lens[r])
+ kv_end = int(kv_cu_seq_lens[r + 1])
+
+ qs, ks, vs, h_v = self._sdpa_layout(
+ q[q_start:q_end],
+ k[kv_start:kv_end],
+ v[kv_start:kv_end],
+ )
+
+ attn, _ = torch.ops.zentorch.zentorch_sdpa(
+ qs,
+ ks,
+ vs,
+ 0.0,
+ causal,
+ attn_mask=None,
+ scale=self.scale,
+ )
+ out_chunks.append(attn[0].transpose(0, 1)[..., :h_v].to(v.dtype))
+
+ return torch.cat(out_chunks, dim=0)
diff --git a/vllm/v1/attention/backends/utils.py b/vllm/v1/attention/backends/utils.py
index e918774088..c97483912d 100644
--- a/vllm/v1/attention/backends/utils.py
+++ b/vllm/v1/attention/backends/utils.py
@@ -21,7 +21,12 @@ from vllm.config.cache import _layout_from_name
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d, np_to_pinned_tensor
-from vllm.v1.kv_cache_interface import KVCacheLayout, KVCacheSpec, MambaSpec
+from vllm.v1.kv_cache_interface import (
+ AttentionSpec,
+ KVCacheLayout,
+ KVCacheSpec,
+ MambaSpec,
+)
if TYPE_CHECKING:
from vllm.v1.core.sched.output import SchedulerOutput
@@ -286,9 +291,10 @@ def resolve_kv_cache_layout(
# specs can re-interpret HNC with different sizes as long as the total number of
# bytes is the same. If not block-compact, each spec must agree on HNC to alias
# the same page (this aliasing is done by the Hybrid Memory Allocator, HMA).
+ kv_cache_specs = tuple(kv_cache_specs or ())
hnc_shapes = {
(spec.num_heads, spec.num_states, spec.page_size_bytes)
- for spec in kv_cache_specs or ()
+ for spec in kv_cache_specs
}
if len(hnc_shapes) > 1:
candidates = [m for m in candidates if m.is_block_compact]
@@ -298,6 +304,20 @@ def resolve_kv_cache_layout(
f"none is in every supported set: {supported_layouts}."
)
+ dcp_sharding = {
+ spec.dcp_sharded for spec in kv_cache_specs if isinstance(spec, AttentionSpec)
+ }
+ page_sizes = {spec.page_size_bytes for spec in kv_cache_specs}
+ if len(dcp_sharding) > 1 and len(page_sizes) > 1:
+ # Sharded target and replicated draft caches need independent groups.
+ # Block-outer layouts can pack those groups without equalizing pages.
+ candidates = [layout for layout in candidates if layout.is_block_outermost]
+ if not candidates:
+ raise ValueError(
+ "DCP with a replicated draft and mixed KV page sizes requires "
+ f"a block-outer KV cache layout; supported sets: {supported_layouts}."
+ )
+
if (requested := envs.VLLM_KV_CACHE_LAYOUT) is not None:
layout = _layout_from_name(requested)
if layout not in candidates:
diff --git a/vllm/v1/core/kv_cache_coordinator.py b/vllm/v1/core/kv_cache_coordinator.py
index 8061999409..d69b2f5fb3 100644
--- a/vllm/v1/core/kv_cache_coordinator.py
+++ b/vllm/v1/core/kv_cache_coordinator.py
@@ -11,7 +11,6 @@ from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector
from vllm.v1.core.kv_cache_utils import (
BlockHash,
KVCacheBlock,
- dcp_world_size_for_kv_cache_spec,
)
from vllm.v1.core.single_type_kv_cache_manager import (
CrossAttentionManager,
@@ -20,6 +19,7 @@ from vllm.v1.core.single_type_kv_cache_manager import (
get_manager_for_kv_cache_spec,
)
from vllm.v1.kv_cache_interface import (
+ AttentionSpec,
FullAttentionSpec,
KVCacheConfig,
KVCacheSpec,
@@ -140,9 +140,7 @@ class KVCacheCoordinator(ABC):
role=kv_cache_group.role,
enable_caching=enable_caching,
kv_cache_group_id=i,
- dcp_world_size=dcp_world_size_for_kv_cache_spec(
- kv_cache_group.kv_cache_spec, dcp_world_size
- ),
+ dcp_world_size=dcp_world_size,
pcp_world_size=pcp_world_size,
scheduler_block_size=self.scheduler_block_size,
needs_kv_cache_zeroing=self.kv_cache_config.needs_kv_cache_zeroing,
@@ -664,16 +662,18 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
f"hash_block_size. block_sizes={cacheable_block_sizes}, "
f"hash_block_size={hash_block_size}"
)
- assert pcp_world_size == 1, "PCP not support hybrid attn now."
- if dcp_world_size > 1:
- # DCP shards full-attention KV across ranks and replicates Mamba
- # state; other spec types (e.g. sliding window) have no DCP-aware
- # handling yet, so reject them explicitly.
- for g in kv_cache_config.kv_cache_groups:
- assert isinstance(g.kv_cache_spec, (FullAttentionSpec, MambaSpec)), (
+ for g in kv_cache_config.kv_cache_groups:
+ spec = g.kv_cache_spec
+ replicated = not spec.dcp_sharded
+ if pcp_world_size > 1:
+ assert isinstance(spec, FullAttentionSpec) or (
+ isinstance(spec, AttentionSpec) and replicated
+ ), "PCP only supports full attention and replicated draft groups."
+ if dcp_world_size > 1:
+ assert isinstance(spec, FullAttentionSpec) or replicated, (
"DCP with hybrid KV cache layouts only supports "
- "full-attention and Mamba groups, got: "
- f"{type(g.kv_cache_spec).__name__}."
+ "full-attention, Mamba, and replicated draft groups, got: "
+ f"{type(spec).__name__}."
)
# Fine-grained hash hits require Mamba "align" and compatible cache
# managers in every group. TP needs hashing finer than the Mamba block;
diff --git a/vllm/v1/core/kv_cache_utils.py b/vllm/v1/core/kv_cache_utils.py
index ee0957839e..3f4f4e7bb1 100644
--- a/vllm/v1/core/kv_cache_utils.py
+++ b/vllm/v1/core/kv_cache_utils.py
@@ -682,12 +682,7 @@ def hash_block_tokens(
def resolve_dcp_kv_block_size(spec: KVCacheSpec, dcp_world_size: int) -> int:
"""Return the token span of a cache block under DCP."""
- layer_specs = iter_layer_specs(spec)
- if len(layer_specs) > 0 and all(
- isinstance(layer_spec, AttentionSpec) for layer_spec in layer_specs
- ):
- return spec.block_size * dcp_world_size
- return spec.block_size
+ return spec.block_size * (dcp_world_size if spec.dcp_sharded else 1)
def resolve_dcp_kv_cache_spec(spec: KVCacheSpec, dcp_world_size: int) -> KVCacheSpec:
@@ -707,28 +702,6 @@ def resolve_dcp_kv_cache_spec(spec: KVCacheSpec, dcp_world_size: int) -> KVCache
return replace(spec, block_size=block_size)
-def dcp_world_size_for_kv_cache_spec(spec: KVCacheSpec, dcp_world_size: int) -> int:
- """Return the DCP size that owns this group's block geometry.
-
- Full-attention KV (including MLA) is sharded across DCP ranks, so prefix
- hashing and manager ``block_size`` use the process DCP size. Other specs
- keep replicated per-rank state (Mamba, sliding window, chunked-local) and
- must keep ``dcp_world_size=1`` even when the process runs with DCP > 1.
-
- Draft MLA groups on the sharded DSpark path are ``FullAttentionSpec`` /
- ``MLAAttentionSpec`` and therefore keep the process DCP size. A replicated
- draft group would need a different spec, not this helper.
- """
- if dcp_world_size <= 1:
- return 1
- inner = spec
- if isinstance(spec, UniformTypeKVCacheSpecs):
- inner = next(iter(spec.kv_cache_specs.values()))
- if isinstance(inner, FullAttentionSpec):
- return dcp_world_size
- return 1
-
-
def resolve_kv_cache_block_sizes(
kv_cache_config: KVCacheConfig,
vllm_config: VllmConfig,
@@ -752,6 +725,8 @@ def resolve_kv_cache_block_sizes(
groups = kv_cache_config.kv_cache_groups
if len(groups) <= 1:
+ if groups and not groups[0].kv_cache_spec.dcp_sharded:
+ dcp = 1
bs = cache_config.block_size * dcp
return bs, bs
@@ -1935,18 +1910,21 @@ def unify_hybrid_kv_cache_specs(kv_cache_spec: dict[str, KVCacheSpec]):
kv_cache_spec: The kv cache spec of each attention layer in the model
"""
- if is_kv_cache_spec_uniform(
- kv_cache_spec
- ) or UniformTypeKVCacheSpecs.is_uniform_type(kv_cache_spec):
- return
-
- logger.warning(
- "Hybrid KV cache manager is disabled for this hybrid model, "
- "This means we do not enable any optimizations for saving KV cache "
- "memory (e.g., dropping the KV cache outside the sliding window). "
- "The compute of layers like sliding window is still saved."
- )
- kv_cache_spec.update(_promote_local_kv_cache_specs(kv_cache_spec))
+ groups: defaultdict[bool, dict[str, KVCacheSpec]] = defaultdict(dict)
+ for name, spec in kv_cache_spec.items():
+ replicated = isinstance(spec, AttentionSpec) and not spec.dcp_sharded
+ groups[replicated][name] = spec
+ for specs in groups.values():
+ promoted_specs = _promote_local_kv_cache_specs(specs)
+ if promoted_specs == specs:
+ continue
+ logger.warning(
+ "Hybrid KV cache manager is disabled for this hybrid model, "
+ "This means we do not enable any optimizations for saving KV cache "
+ "memory (e.g., dropping the KV cache outside the sliding window). "
+ "The compute of layers like sliding window is still saved."
+ )
+ kv_cache_spec.update(promoted_specs)
def _approximate_gcd(values: Sequence[int], *, lower_bound: int | None = None) -> int:
@@ -2276,7 +2254,7 @@ def _ensure_min_page_size(
scaled: list[KVCacheGroupSpec] = []
for g in groups:
s = g.kv_cache_spec
- kw: dict[str, int] = {"block_size": s.block_size * scale}
+ kw: dict[str, Any] = {"block_size": s.block_size * scale}
if isinstance(s, (AttentionSpec, MambaSpec)) and s.page_size_padded is not None:
kw["page_size_padded"] = s.page_size_padded * scale
scaled.append(KVCacheGroupSpec(g.layer_names, replace(s, **kw)))
diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py
index 4c56ba8d81..7c0c7e7fff 100644
--- a/vllm/v1/core/single_type_kv_cache_manager.py
+++ b/vllm/v1/core/single_type_kv_cache_manager.py
@@ -106,11 +106,9 @@ class SingleTypeKVCacheManager(ABC):
# managers have been validated by the coordinator.
self.cache_hit_alignment_tokens = scheduler_block_size
# The block size for this manager; used for actual block allocation.
- self.block_size = kv_cache_spec.block_size
- self.dcp_world_size = dcp_world_size
- self.pcp_world_size = pcp_world_size
- if dcp_world_size > 1:
- self.block_size *= dcp_world_size
+ self.dcp_world_size = dcp_world_size if kv_cache_spec.dcp_sharded else 1
+ self.pcp_world_size = pcp_world_size if kv_cache_spec.dcp_sharded else 1
+ self.block_size = kv_cache_spec.block_size * self.dcp_world_size
self.kv_cache_spec = kv_cache_spec
self.block_pool = block_pool
self.enable_caching = enable_caching
@@ -1455,10 +1453,6 @@ class MambaManager(SingleTypeKVCacheManager):
self, kv_cache_spec: MambaSpec, block_pool: BlockPool, **kwargs
) -> None:
super().__init__(kv_cache_spec, block_pool, **kwargs)
- # Mamba layers use TP instead of DCP, so each rank holds the full
- # recurrent state. Undo the DCP/PCP block_size scaling that the base
- # class applies for attention groups whose KV cache is partitioned.
- self.block_size = kv_cache_spec.block_size
self.mamba_cache_mode = kv_cache_spec.mamba_cache_mode
self.num_speculative_blocks: int = kv_cache_spec.num_speculative_blocks
self.has_prefill_checkpoint_blocks = (
diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py
index b36caf092c..5c09a0059e 100644
--- a/vllm/v1/engine/core.py
+++ b/vllm/v1/engine/core.py
@@ -219,10 +219,7 @@ class EngineCore:
logger.debug("Batch queue is enabled with size %d", self.batch_queue_size)
self.batch_queue = deque(maxlen=self.batch_queue_size)
- self.is_ec_consumer = (
- vllm_config.ec_transfer_config is None
- or vllm_config.ec_transfer_config.is_ec_consumer
- )
+ self.is_mm_encoder_only = vllm_config.is_mm_encoder_only
self.is_pooling_model = vllm_config.model_config.runner_type == "pooling"
self.request_block_hasher: Callable[[Request], list[BlockHash]] | None = None
@@ -702,7 +699,7 @@ class EngineCore:
exec_future = self.model_executor.execute_model(
scheduler_output, non_block=True
)
- if self.is_ec_consumer:
+ if not self.is_mm_encoder_only:
model_executed = scheduler_output.total_num_scheduled_tokens > 0
if self.is_pooling_model or not model_executed:
diff --git a/vllm/v1/kv_cache_interface.py b/vllm/v1/kv_cache_interface.py
index 499b50a3a2..5791704b2b 100644
--- a/vllm/v1/kv_cache_interface.py
+++ b/vllm/v1/kv_cache_interface.py
@@ -158,6 +158,8 @@ class KVCacheSpec:
# number of tokens in a block
block_size: int
+ dcp_sharded: bool = field(default=False, kw_only=True)
+ """Whether DCP shards this cache's token positions across ranks."""
block_stride_alignment: int | None = field(default=None, kw_only=True)
"""Required byte alignment between physical blocks, including packed layers."""
@@ -481,6 +483,7 @@ class HiSparseResidentSpec(KVCacheSpec):
@dataclass(frozen=True, kw_only=True)
class AttentionSpec(KVCacheSpec):
+ dcp_sharded: bool = True
num_kv_heads: int
head_size: int
dtype: torch.dtype
@@ -536,7 +539,9 @@ class AttentionSpec(KVCacheSpec):
def max_num_blocks_per_req(self, vllm_config: VllmConfig, max_len: int) -> int:
parallel_config = vllm_config.parallel_config
- kv_shard_count = parallel_config.decode_context_parallel_size
+ kv_shard_count = (
+ parallel_config.decode_context_parallel_size if self.dcp_sharded else 1
+ )
return cdiv(max_len, self.block_size * kv_shard_count)
@@ -568,7 +573,7 @@ class FullAttentionSpec(AttentionSpec):
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
max_model_len = vllm_config.model_config.max_model_len
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
- if dcp_world_size > 1:
+ if self.dcp_sharded and dcp_world_size > 1:
max_model_len = cdiv(max_model_len, dcp_world_size)
return cdiv(max_model_len, self.block_size) * self.page_size_bytes
@@ -612,6 +617,7 @@ class FullAttentionSpec(AttentionSpec):
head_size_v=specs[0].head_size_v,
dtype=specs[0].dtype,
kv_quant_mode=specs[0].kv_quant_mode,
+ dcp_sharded=specs[0].dcp_sharded,
page_size_padded=specs[0].page_size_padded,
num_head_slots=specs[0].num_head_slots,
state_content_bytes=specs[0].state_content_bytes,
@@ -698,6 +704,7 @@ class MLAAttentionSpec(FullAttentionSpec):
head_size=specs[0].head_size,
dtype=specs[0].dtype,
kv_quant_mode=specs[0].kv_quant_mode,
+ dcp_sharded=specs[0].dcp_sharded,
page_size_padded=specs[0].page_size_padded,
num_head_slots=specs[0].num_head_slots,
state_content_bytes=specs[0].state_content_bytes,
@@ -760,6 +767,7 @@ class RSWASpec(FullAttentionSpec):
head_size_v=base.head_size_v,
dtype=base.dtype,
kv_quant_mode=base.kv_quant_mode,
+ dcp_sharded=base.dcp_sharded,
page_size_padded=base.page_size_padded,
num_head_slots=base.num_head_slots,
state_content_bytes=base.state_content_bytes,
@@ -850,9 +858,10 @@ class SlidingWindowSpec(AttentionSpec):
return cdiv(num_tokens, self.block_size) + 1
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
- assert vllm_config.parallel_config.decode_context_parallel_size == 1, (
- "DCP not support sliding window."
- )
+ assert (
+ not self.dcp_sharded
+ or vllm_config.parallel_config.decode_context_parallel_size == 1
+ ), "DCP not support sliding window."
max_blocks = self.max_admission_blocks_per_request(
max_in_flight_tokens=vllm_config.max_in_flight_tokens,
max_model_len=vllm_config.model_config.max_model_len,
@@ -950,6 +959,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
extra_retained_set = set(spec.extra_retained_tokens for spec in specs)
bounded_replay_set = set(spec.bounded_replay for spec in specs)
block_stride_alignment_set = {spec.block_stride_alignment for spec in specs}
+ assert len({spec.dcp_sharded for spec in specs}) == 1
assert (
len(cache_dtype_str_set) == 1
and len(tokens_per_state_set) == 1
@@ -969,6 +979,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
num_kv_heads=specs[0].num_kv_heads,
head_size=specs[0].head_size,
dtype=specs[0].dtype,
+ dcp_sharded=specs[0].dcp_sharded,
page_size_padded=specs[0].page_size_padded,
num_head_slots=specs[0].num_head_slots,
state_content_bytes=specs[0].state_content_bytes,
@@ -1185,6 +1196,7 @@ class SinkFullAttentionSpec(FullAttentionSpec):
block_stride_alignment=specs[0].block_stride_alignment,
dtype=specs[0].dtype,
kv_quant_mode=specs[0].kv_quant_mode,
+ dcp_sharded=specs[0].dcp_sharded,
page_size_padded=specs[0].page_size_padded,
num_head_slots=specs[0].num_head_slots,
state_content_bytes=specs[0].state_content_bytes,
@@ -1217,6 +1229,11 @@ class UniformTypeKVCacheSpecs(KVCacheSpec):
kv_cache_specs: dict[str, KVCacheSpec]
+ def __post_init__(self):
+ super().__post_init__()
+ if self.kv_cache_specs:
+ object.__setattr__(self, "dcp_sharded", self.first_spec.dcp_sharded)
+
@property
def prefix_cacheable(self) -> bool:
return all(spec.prefix_cacheable for spec in self.kv_cache_specs.values())
@@ -1270,6 +1287,8 @@ class UniformTypeKVCacheSpecs(KVCacheSpec):
if len(block_sizes) > 1:
# Different block sizes, not uniform.
return False
+ if len({spec.dcp_sharded for spec in kv_cache_specs.values()}) > 1:
+ return False
first_spec = next(iter(kv_cache_specs.values()))
return first_spec.is_uniform_with_collection(kv_cache_specs)
diff --git a/vllm/v1/simple_kv_offload/manager.py b/vllm/v1/simple_kv_offload/manager.py
index 96bfb35a67..f70afaa343 100644
--- a/vllm/v1/simple_kv_offload/manager.py
+++ b/vllm/v1/simple_kv_offload/manager.py
@@ -4,7 +4,7 @@
import contextlib
from collections.abc import Iterable
-from dataclasses import dataclass, field, replace
+from dataclasses import dataclass, field, fields, replace
from enum import Enum, auto
from typing import TYPE_CHECKING, Any
@@ -27,13 +27,13 @@ from vllm.v1.core.kv_cache_coordinator import (
from vllm.v1.core.kv_cache_utils import (
BlockHashWithGroupId,
ExternalBlockHash,
- dcp_world_size_for_kv_cache_spec,
generate_block_hash_extra_keys,
get_block_hash,
get_group_id,
make_block_hash_with_group_id,
maybe_convert_block_hash,
resolve_block_hashes,
+ resolve_dcp_kv_block_size,
)
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import (
@@ -48,6 +48,11 @@ from vllm.v1.simple_kv_offload.metadata import (
SimpleCPUOffloadMetadata,
SimpleCPUOffloadWorkerMetadata,
)
+from vllm.v1.simple_kv_offload.metrics import (
+ OUTCOME_TO_FIELD,
+ MetricName,
+ SimpleCPUOffloadStats,
+)
if TYPE_CHECKING:
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
@@ -118,7 +123,8 @@ class BoundaryStoreStats:
symptom is a lower cache hit rate with nothing in the logs, so each decline
reason is counted and exposed through
``SimpleCPUOffloadConnector.get_boundary_store_stats()``. Reset by
- ``reset()``; not otherwise cleared.
+ ``reset()``, which carries any not-yet-drained deltas; not otherwise
+ cleared.
"""
published: int = 0
@@ -143,6 +149,7 @@ class SimpleCPUOffloadScheduler:
hash_block_size: int,
lazy_offload: bool = False,
disk_capacity_bytes: int = 0,
+ use_page_cache: bool = False,
):
self.vllm_config = vllm_config
self.kv_cache_config = kv_cache_config
@@ -169,6 +176,13 @@ class SimpleCPUOffloadScheduler:
self.cpu_kv_cache_config.prefix_cacheable_group_ids
)
self.kv_event_medium = MEDIUM_STORAGE if disk_capacity_bytes > 0 else MEDIUM_CPU
+ # Must track kv_event_medium.
+ self._info_labelvalues: tuple[str, ...] = (
+ "disk" if disk_capacity_bytes > 0 else "cpu",
+ str(use_page_cache).lower(),
+ str(lazy_offload).lower(),
+ str(self.num_cpu_blocks),
+ )
# Find the full attention kv group for prefix cache matching.
self.fa_gidx = -1
for g_idx, g in enumerate(self.cpu_kv_cache_config.kv_cache_groups):
@@ -251,6 +265,9 @@ class SimpleCPUOffloadScheduler:
self._load_event_counter: int = 0
self._store_event_counter: int = 0
self.boundary_store_stats = BoundaryStoreStats()
+ # Interval stats state drained by get_stats()
+ self._boundary_stats_snapshot = BoundaryStoreStats()
+ self._interval_load_blocks_completed = 0
# For TP/PP: track partial store completions across steps.
# Events must be reported by all world_size workers before considered complete.
@@ -302,11 +319,7 @@ class SimpleCPUOffloadScheduler:
target = 0
for g in kv_cache_config.prefix_cacheable_groups:
spec = g.kv_cache_spec
- # Only full attention is sharded across DCP ranks; replicated specs
- # (mamba, sliding window, chunked-local) keep their own block size.
- block_size = spec.block_size * dcp_world_size_for_kv_cache_spec(
- spec, cp_world_size
- )
+ block_size = resolve_dcp_kv_block_size(spec, cp_world_size)
if isinstance(spec, MambaSpec):
target += 2
elif isinstance(spec, SlidingWindowSpec):
@@ -955,6 +968,42 @@ class SimpleCPUOffloadScheduler:
def get_boundary_store_stats(self) -> BoundaryStoreStats:
return replace(self.boundary_store_stats)
+ def get_stats(self) -> SimpleCPUOffloadStats:
+ """Drain per-step stats for the connector's stats hooks."""
+ stats = SimpleCPUOffloadStats()
+
+ current = self.boundary_store_stats
+ for outcome, field_name in OUTCOME_TO_FIELD.items():
+ delta = getattr(current, field_name) - getattr(
+ self._boundary_stats_snapshot, field_name
+ )
+ if delta > 0:
+ stats.increase_counter(MetricName.SAVE_OUTCOMES, delta, (outcome,))
+ self._boundary_stats_snapshot = replace(current)
+
+ if self._interval_load_blocks_completed:
+ stats.increase_counter(
+ MetricName.LOAD_BLOCKS,
+ self._interval_load_blocks_completed,
+ )
+ self._interval_load_blocks_completed = 0
+
+ stats.set_gauge(
+ MetricName.USED_BLOCKS,
+ self.num_cpu_blocks - self.cpu_block_pool.get_num_free_blocks(),
+ )
+ pending = sum(
+ len(t.cpu_block_ids)
+ for t in (
+ *self._store_event_to_blocks.values(),
+ *self._pending_finished_stores,
+ *self._abandoned_store_event_to_blocks.values(),
+ )
+ )
+ stats.set_gauge(MetricName.PENDING_STORE_BLOCKS, pending)
+ stats.set_gauge(MetricName.INFO, 1, self._info_labelvalues)
+ return stats
+
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
"""Handle async transfer completions from worker.
@@ -965,7 +1014,9 @@ class SimpleCPUOffloadScheduler:
"""
# --- Load completions ---
for req_id in list(connector_output.finished_recving or []):
- self._cleanup_load_request(req_id)
+ completed_blocks = self._cleanup_load_request(req_id)
+ if completed_blocks:
+ self._interval_load_blocks_completed += completed_blocks
# --- Store completions ---
meta = connector_output.kv_connector_worker_meta
@@ -1270,18 +1321,22 @@ class SimpleCPUOffloadScheduler:
if blocks_to_free:
self.cpu_block_pool.free_blocks(blocks_to_free)
- def _cleanup_load_request(self, req_id: str) -> None:
+ def _cleanup_load_request(self, req_id: str) -> int:
"""Release all load resources for a request.
Shared between request_finished() and update_connector_output() paths.
Removes the request from _reqs_to_load, cleans up event mappings,
and frees CPU/GPU touch refs.
+
+ Returns the number of blocks in the load if it had been issued to
+ the worker, else 0 (never-issued loads are not counted as completed
+ by the caller).
"""
state = self._reqs_to_load.pop(req_id, None)
if state is None:
state = self._abandoned_reqs_to_load.pop(req_id, None)
if state is None:
- return
+ return 0
# Remove from load event mapping (only this req, not whole event)
if state.load_event is not None:
reqs = self._load_event_to_reqs.get(state.load_event)
@@ -1304,6 +1359,10 @@ class SimpleCPUOffloadScheduler:
for bid in state.transfer_meta.gpu_block_ids
)
+ if state.load_event is None:
+ return 0
+ return len(state.transfer_meta.gpu_block_ids)
+
def _cleanup_store_request(self, req_id: str) -> None:
"""Release store metadata for a request.
@@ -1363,7 +1422,20 @@ class SimpleCPUOffloadScheduler:
if event_idx in self._abandoned_store_event_to_blocks
}
self._cursor = None
- self.boundary_store_stats = BoundaryStoreStats()
+ # Seed the fresh counters with outcomes since the last drain so a
+ # mid-interval reset() does not drop them from the next report.
+ undrained = BoundaryStoreStats(
+ **{
+ f.name: max(
+ 0,
+ getattr(self.boundary_store_stats, f.name)
+ - getattr(self._boundary_stats_snapshot, f.name),
+ )
+ for f in fields(BoundaryStoreStats)
+ }
+ )
+ self.boundary_store_stats = undrained
+ self._boundary_stats_snapshot = BoundaryStoreStats()
# NOTE: _load_event_counter / _store_event_counter are not
# reset as they are monotonic and must stay ahead of the workers
# high-water marks to avoid event index collisions
diff --git a/vllm/v1/simple_kv_offload/metrics.py b/vllm/v1/simple_kv_offload/metrics.py
new file mode 100644
index 0000000000..9017d40cfd
--- /dev/null
+++ b/vllm/v1/simple_kv_offload/metrics.py
@@ -0,0 +1,284 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Prometheus metrics for SimpleCPUOffloadConnector.
+
+The stats payload uses the same self-describing wire format as the
+offloading connector, so it survives IPC serialization without the
+metric classes on the receiving side.
+"""
+
+from dataclasses import dataclass
+from typing import Any
+
+from vllm.config import VllmConfig
+from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
+ KVConnectorPromMetrics,
+ KVConnectorStats,
+ PromMetric,
+ PromMetricT,
+)
+
+
+class MetricName:
+ """Metric names for SimpleCPUOffloadConnector."""
+
+ SAVE_OUTCOMES = "vllm:simple_kv_offload_save_outcomes_total"
+ LOAD_BLOCKS = "vllm:simple_kv_offload_load_blocks_total"
+ USED_BLOCKS = "vllm:simple_kv_offload_used_blocks"
+ PENDING_STORE_BLOCKS = "vllm:simple_kv_offload_pending_store_blocks"
+ INFO = "vllm:simple_kv_offload_info"
+
+
+# outcome label value -> BoundaryStoreStats field name.
+# ``published`` is the sum of all outcomes and is not labeled.
+OUTCOME_TO_FIELD = {
+ "stored": "stored",
+ "dropped_cpu_full": "dropped_cpu_full",
+ "dropped_request_gone": "dropped_request_gone",
+ "dropped_null_block": "dropped_null_block",
+ "dropped_not_hashed": "dropped_not_hashed",
+ "skipped_already_cached": "skipped_already_cached",
+ "skipped_in_flight": "skipped_in_flight",
+}
+
+INFO_LABELS = ["backend", "page_cache", "lazy_offload", "capacity_blocks"]
+
+
+class _MetricType:
+ """Type tags embedded in the serialized stats payload."""
+
+ COUNTER = "counter"
+ GAUGE = "gauge"
+
+
+class _StatsKey:
+ """Top-level keys in the serialized stats dict."""
+
+ # Maps metric name -> _MetricType value
+ TYPES = "types"
+ # Maps metric name -> {label values tuple -> observed value}
+ DATA = "data"
+
+
+@dataclass
+class SimpleCPUOffloadStats(KVConnectorStats):
+ """Stats payload for SimpleCPUOffloadConnector.
+
+ The ``data`` dict is structured as::
+
+ {
+ _StatsKey.TYPES: {name: _MetricType.*, ...},
+ _StatsKey.DATA: {name: {labelvalues: value, ...}, ...},
+ }
+
+ Counter values are summed when aggregating, gauge values keep the
+ latest snapshot, and unlabeled metrics use ``()`` as their
+ labelvalues tuple.
+ """
+
+ def __post_init__(self):
+ if _StatsKey.DATA not in self.data:
+ self.reset()
+
+ def reset(self):
+ self.data: dict[str, Any] = {
+ _StatsKey.TYPES: {},
+ _StatsKey.DATA: {},
+ }
+
+ @property
+ def _types(self) -> dict[str, str]:
+ return self.data[_StatsKey.TYPES]
+
+ @property
+ def _values(self) -> dict[str, Any]:
+ return self.data[_StatsKey.DATA]
+
+ def aggregate(self, other: "KVConnectorStats") -> "KVConnectorStats":
+ if other.is_empty():
+ return self
+ assert isinstance(other, SimpleCPUOffloadStats)
+ for key, other_label_values in other._values.items():
+ type_str = other._types.get(key)
+ if type_str is None:
+ raise AssertionError(f"Unknown simple KV offload stats key: {key}")
+ self._types.setdefault(key, type_str)
+ current_label_values = self._values.setdefault(key, {})
+ for labelvalues, value in other_label_values.items():
+ if type_str == _MetricType.COUNTER:
+ assert isinstance(value, int | float)
+ current_label_values[labelvalues] = (
+ current_label_values.get(labelvalues, 0) + value
+ )
+ elif type_str == _MetricType.GAUGE:
+ assert isinstance(value, int | float)
+ current_label_values[labelvalues] = value
+ else:
+ raise AssertionError(
+ f"Unknown metric type '{type_str}' for key: {key}"
+ )
+ return self
+
+ def reduce(self) -> dict[str, int | float]:
+ """Reduce the interval observations to a flat summary for logging."""
+ return_dict: dict[str, int | float] = {}
+ for key, label_value_map in self._values.items():
+ if key == MetricName.INFO:
+ # Constant config fingerprint; exclude from the log summary.
+ continue
+ type_str = self._types.get(key)
+ if type_str is None:
+ raise AssertionError(f"Unknown simple KV offload stats key: {key}")
+ for labelvalues, value in label_value_map.items():
+ key_with_labels = f"{key}:{labelvalues}" if labelvalues else key
+ if type_str in (_MetricType.COUNTER, _MetricType.GAUGE):
+ assert isinstance(value, int | float)
+ return_dict[key_with_labels] = value
+ else:
+ raise AssertionError(
+ f"Unknown metric type '{type_str}' for key: {key}"
+ )
+ return return_dict
+
+ def is_empty(self) -> bool:
+ return not self.data.get(_StatsKey.DATA)
+
+ def increase_counter(
+ self,
+ counter_name: str,
+ counter_increase_value: int | float = 1,
+ labelvalues: tuple[str, ...] = (),
+ ) -> None:
+ """Increase a counter on the stats payload."""
+ self._types.setdefault(counter_name, _MetricType.COUNTER)
+ counter_values = self._values.setdefault(counter_name, {})
+ counter_values[labelvalues] = (
+ counter_values.get(labelvalues, 0) + counter_increase_value
+ )
+
+ def set_gauge(
+ self,
+ gauge_name: str,
+ gauge_value: int | float,
+ labelvalues: tuple[str, ...] = (),
+ ) -> None:
+ """Set a gauge snapshot on the stats payload."""
+ self._types.setdefault(gauge_name, _MetricType.GAUGE)
+ gauge_values = self._values.setdefault(gauge_name, {})
+ gauge_values[labelvalues] = gauge_value
+
+
+class SimpleCPUOffloadPromMetrics(KVConnectorPromMetrics):
+ """Prometheus registration and recording for SimpleCPUOffloadConnector."""
+
+ def __init__(
+ self,
+ vllm_config: VllmConfig,
+ metric_types: dict[type[PromMetric], type[PromMetricT]],
+ labelnames: list[str],
+ per_engine_labelvalues: dict[int, list[object]],
+ ):
+ super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues)
+ # (engine_idx, metric_name, labelvalues) -> bound child metric
+ self._metrics: dict[tuple[int, str, tuple[str, ...]], PromMetricT] = {}
+
+ self._counter_save_outcomes = self._counter_cls(
+ name="vllm:simple_kv_offload_save_outcomes_total",
+ documentation=(
+ "Store admission decisions in SimpleCPUOffloadConnector, by "
+ "outcome. Outcomes classify eager-mode boundary hand-off "
+ "stores; lazy-mode stores are not classified."
+ ),
+ labelnames=self._labelnames + ["outcome"],
+ )
+ self._counter_load_blocks = self._counter_cls(
+ name="vllm:simple_kv_offload_load_blocks_total",
+ documentation=(
+ "KV blocks restored to GPU from the offload pool, counted "
+ "when a load finishes. This is prefill work avoided "
+ "through offload cache hits."
+ ),
+ labelnames=self._labelnames,
+ )
+ self._gauge_used_blocks = self._gauge_cls(
+ name="vllm:simple_kv_offload_used_blocks",
+ documentation=(
+ "Offload-pool blocks currently pinned by in-flight "
+ "transfers or cache hits (capacity minus free). Evictable "
+ "cached blocks are not counted; capacity_blocks is on "
+ "vllm:simple_kv_offload_info."
+ ),
+ labelnames=self._labelnames,
+ multiprocess_mode="mostrecent",
+ )
+ self._gauge_pending_store_blocks = self._gauge_cls(
+ name="vllm:simple_kv_offload_pending_store_blocks",
+ documentation=(
+ "Offload blocks in pending stores: queued for dispatch, "
+ "issued to workers, or awaiting release after a cache "
+ "reset. A persistently growing value indicates a stuck "
+ "transfer."
+ ),
+ labelnames=self._labelnames,
+ multiprocess_mode="mostrecent",
+ )
+ self._gauge_info = self._gauge_cls(
+ name="vllm:simple_kv_offload_info",
+ documentation=(
+ "SimpleCPUOffloadConnector deployment facts. Value is "
+ "always 1. backend is cpu or disk; page_cache and "
+ "lazy_offload are true/false; capacity_blocks is the "
+ "offload pool size."
+ ),
+ labelnames=self._labelnames + INFO_LABELS,
+ multiprocess_mode="mostrecent",
+ )
+
+ self._defs: dict[str, PromMetricT] = {
+ MetricName.SAVE_OUTCOMES: self._counter_save_outcomes,
+ MetricName.LOAD_BLOCKS: self._counter_load_blocks,
+ MetricName.USED_BLOCKS: self._gauge_used_blocks,
+ MetricName.PENDING_STORE_BLOCKS: self._gauge_pending_store_blocks,
+ MetricName.INFO: self._gauge_info,
+ }
+ self._num_label_values: dict[str, int] = {
+ MetricName.SAVE_OUTCOMES: 1,
+ MetricName.LOAD_BLOCKS: 0,
+ MetricName.USED_BLOCKS: 0,
+ MetricName.PENDING_STORE_BLOCKS: 0,
+ MetricName.INFO: len(INFO_LABELS),
+ }
+
+ def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
+ """Observe transfer statistics."""
+ metric_types = transfer_stats_data.get(_StatsKey.TYPES, {})
+ metric_data = transfer_stats_data.get(_StatsKey.DATA, {})
+ for key, label_value_map in metric_data.items():
+ prom_metric = self._defs.get(key)
+ if prom_metric is None:
+ raise AssertionError(f"Unknown simple KV offload stats key: {key}")
+ type_str = metric_types.get(key)
+ for labelvalues, value in label_value_map.items():
+ if len(labelvalues) != self._num_label_values[key]:
+ raise AssertionError(
+ f"Metric {key} expects "
+ f"{self._num_label_values[key]} labels, "
+ f"got {len(labelvalues)}"
+ )
+ child = self._metrics.get((engine_idx, key, labelvalues))
+ if child is None:
+ engine_labelvalues = self.per_engine_labelvalues[engine_idx]
+ child = prom_metric.labels(
+ *(engine_labelvalues + list(labelvalues))
+ )
+ self._metrics[(engine_idx, key, labelvalues)] = child
+ if type_str == _MetricType.COUNTER:
+ assert isinstance(value, int | float)
+ child.inc(value)
+ elif type_str == _MetricType.GAUGE:
+ assert isinstance(value, int | float)
+ child.set(value)
+ else:
+ raise AssertionError(
+ f"Unknown metric type '{type_str}' for key: {key}"
+ )
diff --git a/vllm/v1/spec_decode/draft_model.py b/vllm/v1/spec_decode/draft_model.py
index 08542f03a1..17be2732e6 100644
--- a/vllm/v1/spec_decode/draft_model.py
+++ b/vllm/v1/spec_decode/draft_model.py
@@ -81,13 +81,20 @@ class DraftModelProposer(SpecDecodeBaseProposer):
def _create_draft_vllm_config(self) -> VllmConfig:
base = super()._create_draft_vllm_config()
spec = self.speculative_config
+ target_parallel = base.parallel_config
return replace(
base,
quant_config=None,
parallel_config=replace(
spec.draft_parallel_config,
- rank=self.vllm_config.parallel_config.rank,
+ rank=target_parallel.rank,
+ decode_context_parallel_size=(
+ target_parallel.decode_context_parallel_size
+ ),
+ cp_kv_cache_interleave_size=target_parallel.cp_kv_cache_interleave_size,
+ dcp_comm_backend=target_parallel.dcp_comm_backend,
+ dcp_q_replicate=target_parallel.dcp_q_replicate,
),
model_config=spec.draft_model_config,
)
diff --git a/vllm/v1/worker/cp_utils.py b/vllm/v1/worker/cp_utils.py
index 79651759aa..bcf56dc0bf 100644
--- a/vllm/v1/worker/cp_utils.py
+++ b/vllm/v1/worker/cp_utils.py
@@ -40,6 +40,8 @@ def check_attention_cp_compatibility(
layer_impl = getattr(layer, "impl", None)
if layer_impl is None:
continue
+ if not check_pcp and layer_impl.dcp_world_size == 1:
+ continue
if vllm_config.speculative_config is not None and interleave_size > 1:
assert layer_impl.supports_mtp_with_cp_non_trivial_interleave_size, (
"MTP with cp_kv_cache_interleave_size > 1 is not "
diff --git a/vllm/v1/worker/gpu/attn_utils.py b/vllm/v1/worker/gpu/attn_utils.py
index 423bcebb2d..19de125a9b 100644
--- a/vllm/v1/worker/gpu/attn_utils.py
+++ b/vllm/v1/worker/gpu/attn_utils.py
@@ -29,6 +29,7 @@ from vllm.v1.kv_cache_interface import (
AttentionSpec,
KVCacheConfig,
KVCacheSpec,
+ MambaSpec,
UniformTypeKVCacheSpecs,
)
from vllm.v1.worker.gpu.model_states.interface import ModelSpecificAttnMetadata
@@ -462,6 +463,10 @@ def build_attn_metadata(
attn_metadata: dict[str, Any] = {}
token_to_req_indices: torch.Tensor | None = None
+ # Mamba groups with the same spec and builder differ only in their state
+ # indices, so later groups re-gather those from the first group's metadata.
+ # Also at capture, so FULL graphs share the batch-level buffers.
+ cached_metadata: dict[tuple[KVCacheSpec, type], Any] = {}
num_kv_cache_groups = len(kv_cache_config.kv_cache_groups)
for i in range(num_kv_cache_groups):
if not attn_groups[i]:
@@ -514,7 +519,16 @@ def build_attn_metadata(
for attn_group in attn_groups[i]:
attn_metadata_builder = attn_group.get_metadata_builder(ubatch_idx)
- if for_cudagraph_capture:
+ reuse_key = None
+ if attn_metadata_builder.supports_update_block_table and isinstance(
+ attn_group.kv_cache_spec, MambaSpec
+ ):
+ reuse_key = (attn_group.kv_cache_spec, type(attn_metadata_builder))
+ if reuse_key in cached_metadata:
+ metadata = attn_metadata_builder.update_block_table(
+ cached_metadata[reuse_key], block_table, slot_mapping
+ )
+ elif for_cudagraph_capture:
metadata = attn_metadata_builder.build_for_cudagraph_capture(
common_attn_metadata
)
@@ -532,6 +546,8 @@ def build_attn_metadata(
common_attn_metadata=common_attn_metadata,
**attn_metadata_extra_kwargs,
)
+ if reuse_key is not None and reuse_key not in cached_metadata:
+ cached_metadata[reuse_key] = metadata
for layer_name in attn_group.layer_names:
attn_metadata[layer_name] = metadata
token_to_req_indices = common_attn_metadata._token_to_req_indices_cache
diff --git a/vllm/v1/worker/gpu/block_table.py b/vllm/v1/worker/gpu/block_table.py
index 096d9b99aa..8cf3ba98dd 100644
--- a/vllm/v1/worker/gpu/block_table.py
+++ b/vllm/v1/worker/gpu/block_table.py
@@ -27,6 +27,7 @@ class BlockTables:
cp_rank: int = 0,
cp_interleave: int = 1,
slot_mapping_enabled: list[bool] | None = None,
+ dcp_sharded: list[bool] | None = None,
):
self.block_sizes = block_sizes
self.kernel_block_sizes = kernel_block_sizes
@@ -44,6 +45,10 @@ class BlockTables:
slot_mapping_enabled = [True] * self.num_kv_cache_groups
assert len(slot_mapping_enabled) == self.num_kv_cache_groups
self._slot_mapping_enabled = slot_mapping_enabled
+ if dcp_sharded is None:
+ dcp_sharded = [True] * self.num_kv_cache_groups
+ assert len(dcp_sharded) == self.num_kv_cache_groups
+ self.dcp_sharded = torch.tensor(dcp_sharded, dtype=torch.bool, device=device)
self.blocks_per_kv_block = [
bs // kbs for bs, kbs in zip(block_sizes, kernel_block_sizes)
@@ -210,6 +215,7 @@ class BlockTables:
self.block_sizes_tensor,
self.kernel_block_sizes_tensor,
self.slot_mapping_enabled,
+ self.dcp_sharded,
slot_mappings,
slot_mappings.stride(0),
self.cp_rank,
@@ -283,6 +289,7 @@ def _compute_slot_mappings_kernel(
block_sizes, # [num_kv_cache_groups]
kernel_block_sizes, # [num_kv_cache_groups]
slot_mapping_enabled, # [num_kv_cache_groups]
+ dcp_sharded, # [num_kv_cache_groups]
slot_mappings_ptr, # [num_kv_cache_groups, max_num_tokens]
slot_mappings_stride,
cp_rank,
@@ -312,6 +319,8 @@ def _compute_slot_mappings_kernel(
kv_block_size = tl.load(block_sizes + group_id)
kernel_block_size = tl.load(kernel_block_sizes + group_id)
mapping_enabled = tl.load(slot_mapping_enabled + group_id)
+ if CP_SIZE != 1:
+ sharded = tl.load(dcp_sharded + group_id)
req_state_idx = tl.load(idx_mapping + batch_idx)
# idx_mapping == -1 marks a dummy (or CUDA-graph padding) request that owns
@@ -337,6 +346,8 @@ def _compute_slot_mappings_kernel(
remainder = virtual_block_offsets % CP_INTERLEAVE
local_offsets = rounds * CP_INTERLEAVE + remainder
local_positions = virtual_block_indices * kv_block_size + local_offsets
+ local_positions = tl.where(sharded, local_positions, positions)
+ is_local = ~sharded | is_local
block_indices = tl.where(
mapping_enabled, local_positions // kernel_block_size, 0
diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py
index 61ceaeef2e..9f25425ab6 100644
--- a/vllm/v1/worker/gpu/model_runner.py
+++ b/vllm/v1/worker/gpu/model_runner.py
@@ -21,6 +21,7 @@ import gc
import time
from contextlib import AbstractContextManager
from copy import deepcopy
+from dataclasses import replace
from typing import Any, NamedTuple
import numpy as np
@@ -561,7 +562,16 @@ class GPUModelRunner(LoRAModelRunnerMixin):
return encoder_runner.get_encoder_timing_stats()
def get_kv_cache_spec(self):
- return get_kv_cache_spec(self.vllm_config)
+ kv_cache_spec = get_kv_cache_spec(self.vllm_config)
+ # A draft running at DCP=1 under a DCP target keeps a replicated cache.
+ if (
+ isinstance(self.speculator, DraftModelSpeculator)
+ and self.speculator.dcp_size == 1
+ and self.dcp_size > 1
+ ):
+ for name in self.speculator.draft_attn_layer_names & kv_cache_spec.keys():
+ kv_cache_spec[name] = replace(kv_cache_spec[name], dcp_sharded=False)
+ return kv_cache_spec
def initialize_kv_cache(
self,
@@ -587,6 +597,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
block_sizes = []
max_num_blocks_per_group = []
slot_mapping_enabled = []
+ dcp_sharded = []
for kv_cache_group in kv_cache_config.kv_cache_groups:
spec = kv_cache_group.kv_cache_spec
block_sizes.append(spec.block_size)
@@ -594,6 +605,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
spec.first_spec if isinstance(spec, UniformTypeKVCacheSpecs) else spec
)
slot_mapping_enabled.append(layer_spec.uses_slot_mapping)
+ dcp_sharded.append(spec.dcp_sharded)
# Let each cache type account for CP. Attention KV is DCP-sharded,
# while Mamba/GDN recurrent state is replicated across DCP ranks.
max_num_blocks = spec.max_num_blocks_per_req(
@@ -657,6 +669,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
device=self.device,
kernel_block_sizes=self.kernel_block_sizes,
slot_mapping_enabled=slot_mapping_enabled,
+ dcp_sharded=dcp_sharded,
cp_size=self.dcp_size,
cp_rank=self.dcp_rank,
cp_interleave=self.cp_interleave,
diff --git a/vllm/v1/worker/gpu/pcp_manager.py b/vllm/v1/worker/gpu/pcp_manager.py
index aff8f04cf3..de49fcffb9 100644
--- a/vllm/v1/worker/gpu/pcp_manager.py
+++ b/vllm/v1/worker/gpu/pcp_manager.py
@@ -145,7 +145,10 @@ class PCPManager:
if speculative_config is not None:
if speculative_config.use_dspark():
dcp_size = parallel_config.decode_context_parallel_size
- if dcp_size not in (1, pcp_size):
+ if (
+ dcp_size not in (1, pcp_size)
+ and speculative_config.draft_model_config.use_mla
+ ):
raise NotImplementedError(
"MRV2 PCP DSpark requires DCP=1 or DCP=PCP; got "
f"DCP={dcp_size}, PCP={pcp_size}."
diff --git a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py
index cdcdd5fcba..57110e0b05 100644
--- a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py
+++ b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py
@@ -37,12 +37,18 @@ class DFlashSpeculator(DraftModelSpeculator):
def __init__(self, vllm_config: VllmConfig, device: torch.device):
parallel_config = vllm_config.parallel_config
- if parallel_config.prefill_context_parallel_size > 1:
- vllm_config = copy.copy(vllm_config)
- vllm_config.parallel_config = replace(
- parallel_config,
- prefill_context_parallel_size=1,
- )
+ speculative_config = vllm_config.speculative_config
+ assert speculative_config is not None
+ vllm_config = copy.copy(vllm_config)
+ vllm_config.parallel_config = replace(
+ parallel_config,
+ prefill_context_parallel_size=1,
+ decode_context_parallel_size=(
+ parallel_config.decode_context_parallel_size
+ if speculative_config.draft_model_config.use_mla
+ else 1
+ ),
+ )
super().__init__(vllm_config, device)
self.hidden_states = torch.zeros(
@@ -427,7 +433,7 @@ class DFlashSpeculator(DraftModelSpeculator):
self.block_tables.input_block_tables[gid],
self.block_tables.kernel_block_sizes[gid],
self.block_tables.cp_rank,
- self.block_tables.cp_size,
+ self.dcp_size,
self.block_tables.cp_interleave,
self.parallel_drafting_token_id,
self.num_query_per_req,
diff --git a/vllm/v1/worker/gpu/spec_decode/speculator.py b/vllm/v1/worker/gpu/spec_decode/speculator.py
index d1d6d888aa..6f88febb66 100644
--- a/vllm/v1/worker/gpu/spec_decode/speculator.py
+++ b/vllm/v1/worker/gpu/spec_decode/speculator.py
@@ -138,6 +138,7 @@ class DraftModelSpeculator(BaseSpeculator):
# DP configuration
self.dp_size = vllm_config.parallel_config.data_parallel_size
self.dp_rank = vllm_config.parallel_config.data_parallel_rank
+ self.dcp_size = vllm_config.parallel_config.decode_context_parallel_size
self.eplb_state: EplbState | None = None
@@ -332,14 +333,14 @@ class DraftModelSpeculator(BaseSpeculator):
out=draft_seq_lens_cpu_upper_bound[:num_reqs],
)
draft_seq_lens_cpu_upper_bound[:num_reqs].clamp_(max=self.max_model_len)
- if dcp_local_seq_lens is None and self.block_tables.cp_size > 1:
+ if dcp_local_seq_lens is None and self.dcp_size > 1:
# Draft steps advance and rewind their own global sequence lengths,
# so the target model's DCP-local lengths may already be stale.
dcp_local_seq_lens = prepare_dcp_local_seq_lens(
self.input_buffers.dcp_local_seq_lens,
self.input_buffers.seq_lens,
num_reqs,
- self.block_tables.cp_size,
+ self.dcp_size,
self.block_tables.cp_rank,
self.block_tables.cp_interleave,
)
diff --git a/vllm/v1/worker/gpu_input_batch.py b/vllm/v1/worker/gpu_input_batch.py
index 2a1a3534c9..4707e4f2ca 100644
--- a/vllm/v1/worker/gpu_input_batch.py
+++ b/vllm/v1/worker/gpu_input_batch.py
@@ -817,6 +817,7 @@ class InputBatch:
self.allowed_token_ids_mask_cpu_tensor[empty_index] = (
self.allowed_token_ids_mask_cpu_tensor[last_req_index]
)
+ self.allowed_token_ids_mask_cpu_tensor[last_req_index].fill_(False)
bad_words_token_ids = self.bad_words_token_ids.pop(last_req_index, None)
if bad_words_token_ids is not None:
diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py
index def92f601d..778d71bdbc 100644
--- a/vllm/v1/worker/gpu_worker.py
+++ b/vllm/v1/worker/gpu_worker.py
@@ -801,11 +801,20 @@ class Worker(WorkerBase):
# related to kv cache connector (e.g. kv cache sharing layers).
ensure_kv_transfer_initialized(self.vllm_config, kv_cache_config)
+ # If the connector provides a custom memory pool (e.g. Mooncake
+ # NVLink/BAREX), use it for KV cache allocation; otherwise fall
+ # back to the standard CuMem pool.
+ mem_pool_context = (
+ get_kv_transfer_group().get_mem_pool_context()
+ if has_kv_transfer_group()
+ else None
+ )
+ if mem_pool_context is None:
+ mem_pool_context = self._maybe_get_memory_pool_context(tag="kv_cache")
+
self.model_runner.initialize_kv_cache(
kv_cache_config,
- kv_cache_allocation_context=self._maybe_get_memory_pool_context(
- tag="kv_cache"
- ),
+ kv_cache_allocation_context=mem_pool_context,
)
# Build KV-zero metadata outside the CuMem pool so the bookkeeping