Merge remote-tracking branch 'upstream/main' into patches

This commit is contained in:
2026-09-28 12:06:52 +07:00
170 changed files with 7605 additions and 1142 deletions
+1 -2
View File
@@ -23,8 +23,7 @@ steps:
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'uv pip install "gpt-oss[eval]==0.0.5" &&
cd tests &&
'cd tests &&
pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-xpu.txt'
- label: LM Eval Large Models(2 GPUs)
key: lm-eval-large-models-2-gpus
-2
View File
@@ -1287,7 +1287,6 @@ steps:
- vllm/platforms/rocm.py
- tests/evals/gpt_oss/
commands:
- uv pip install --system 'gpt-oss[eval]==0.0.5'
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx942.txt
- label: ":amd: (MI300) LM Eval Small Models Distributed"
@@ -3584,7 +3583,6 @@ steps:
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- uv pip install --system 'gpt-oss[eval]==0.0.5'
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx950.txt
- label: ":amd: (MI355 DPX) LM Eval Watermarking"
-4
View File
@@ -829,7 +829,6 @@ steps:
- vllm/model_executor/layers/quantization
- tests/evals/gpt_oss/
commands:
- uv pip install --system 'gpt-oss[eval]==0.0.5'
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-h100.txt
- label: ":nvidia: (B200) GPQA Eval (GPT-OSS)"
@@ -843,7 +842,6 @@ steps:
- vllm/model_executor/layers/quantization
- tests/evals/gpt_oss/
commands:
- uv pip install --system 'gpt-oss[eval]==0.0.5'
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-b200.txt
mirror:
amd:
@@ -865,7 +863,6 @@ steps:
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- uv pip install --system 'gpt-oss[eval]==0.0.5'
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx950.txt
- label: ":nvidia: (B200) Qwen3.8-Flash-Next-FP8 Accuracy Eval"
@@ -940,7 +937,6 @@ steps:
- vllm/model_executor/layers/quantization
- tests/evals/gpt_oss/
commands:
- uv pip install --system 'gpt-oss[eval]==0.0.5'
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-spark.txt
- label: ":nvidia: (H200 MIG 35GB) LM Eval KV-Offload"
+98 -6
View File
@@ -98,9 +98,11 @@ steps:
commands:
- pytest -v -s v1/e2e/spec_decode/eagle/
- label: ":nvidia: (H200 MIG 35GB) Spec Decode Speculators + MTP"
key: spec-decode-speculators-mtp
timeout_in_minutes: 50
# MTP correctness is one parametrized test split into per-model folders;
# each job below runs whole directories.
- label: ":nvidia: (H200 MIG 35GB) Spec Decode Speculators"
key: spec-decode-speculators
timeout_in_minutes: 30
device: h200_35gb
source_file_dependencies:
- vllm/v1/spec_decode/
@@ -110,13 +112,103 @@ steps:
- tests/v1/e2e/spec_decode/
commands:
- pytest -v -s v1/e2e/spec_decode/speculators/
- pytest -v -s v1/e2e/spec_decode/mtp/
mirror:
amd:
label: ":amd: (MI300) Spec Decode Speculators + MTP"
label: ":amd: (MI300) Spec Decode Speculators"
dind: false
device: mi300_1
timeout_in_minutes: 75
timeout_in_minutes: 40
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/v1/spec_decode/
- vllm/v1/worker/gpu/spec_decode/
- vllm/model_executor/model_loader/
- vllm/v1/sample/
- vllm/model_executor/layers/
- vllm/transformers_utils/configs/speculators/
- tests/v1/e2e/spec_decode/
- vllm/platforms/rocm.py
- label: ":nvidia: (H200 MIG 35GB) Spec Decode MTP Gemma4"
key: spec-decode-mtp-gemma4
timeout_in_minutes: 30
device: h200_35gb
source_file_dependencies:
- vllm/v1/spec_decode/
- vllm/v1/worker/gpu/spec_decode/
- vllm/v1/attention/backends/
- vllm/transformers_utils/configs/speculators/
- tests/v1/e2e/spec_decode/
commands:
- pytest -v -s v1/e2e/spec_decode/mtp/gemma4/
mirror:
amd:
label: ":amd: (MI300) Spec Decode MTP Gemma4"
dind: false
device: mi300_1
timeout_in_minutes: 40
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/v1/spec_decode/
- vllm/v1/worker/gpu/spec_decode/
- vllm/model_executor/model_loader/
- vllm/v1/sample/
- vllm/model_executor/layers/
- vllm/transformers_utils/configs/speculators/
- tests/v1/e2e/spec_decode/
- vllm/platforms/rocm.py
- label: ":nvidia: (H200 MIG 35GB) Spec Decode MTP DeepSeek + MiMo"
key: spec-decode-mtp-deepseek-mimo
timeout_in_minutes: 30
device: h200_35gb
source_file_dependencies:
- vllm/v1/spec_decode/
- vllm/v1/worker/gpu/spec_decode/
- vllm/v1/attention/backends/
- vllm/transformers_utils/configs/speculators/
- tests/v1/e2e/spec_decode/
commands:
- pytest -v -s v1/e2e/spec_decode/mtp/deepseek/
- pytest -v -s v1/e2e/spec_decode/mtp/mimo/
mirror:
amd:
label: ":amd: (MI300) Spec Decode MTP DeepSeek + MiMo"
dind: false
device: mi300_1
timeout_in_minutes: 40
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/v1/spec_decode/
- vllm/v1/worker/gpu/spec_decode/
- vllm/model_executor/model_loader/
- vllm/v1/sample/
- vllm/model_executor/layers/
- vllm/transformers_utils/configs/speculators/
- tests/v1/e2e/spec_decode/
- vllm/platforms/rocm.py
- label: ":nvidia: (H200 MIG 35GB) Spec Decode MTP Qwen3.5"
key: spec-decode-mtp-qwen3-5
timeout_in_minutes: 30
device: h200_35gb
source_file_dependencies:
- vllm/v1/spec_decode/
- vllm/v1/worker/gpu/spec_decode/
- vllm/v1/attention/backends/
- vllm/transformers_utils/configs/speculators/
- tests/v1/e2e/spec_decode/
commands:
- pytest -v -s v1/e2e/spec_decode/mtp/qwen3_5/
mirror:
amd:
label: ":amd: (MI300) Spec Decode MTP Qwen3.5"
dind: false
device: mi300_1
timeout_in_minutes: 40
depends_on:
- image-build-amd
source_file_dependencies:
+1 -1
View File
@@ -13,7 +13,7 @@ else()
FetchContent_Declare(
flashkda
GIT_REPOSITORY https://github.com/vllm-project/FlashKDA.git
GIT_TAG b59532f1f464fbd536272780e30df5bf6a2ccc02
GIT_TAG 17a037d98da546deb4591e967cf961a43c034d8b
GIT_PROGRESS TRUE
GIT_SUBMODULES cutlass
)
+4
View File
@@ -729,6 +729,8 @@ COPY requirements/lint.txt requirements/lint.txt
COPY requirements/test/cuda.in requirements/test/cuda.in
COPY requirements/test/cuda.txt requirements/test/cuda.txt
COPY requirements/dev.txt requirements/dev.txt
# uv reads `[tool.uv] exclude-dependencies` from here when recompiling below.
COPY pyproject.toml pyproject.toml
COPY tools/use_existing_torch.py tools/use_existing_torch.py
COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt
RUN --mount=type=cache,target=/opt/uv/cache \
@@ -1143,6 +1145,8 @@ COPY requirements/lint.txt requirements/lint.txt
COPY requirements/test/cuda.in requirements/test/cuda.in
COPY requirements/test/cuda.txt requirements/test/cuda.txt
COPY requirements/dev.txt requirements/dev.txt
# uv reads `[tool.uv] exclude-dependencies` from here when recompiling below.
COPY pyproject.toml pyproject.toml
COPY tools/use_existing_torch.py tools/use_existing_torch.py
COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt
RUN --mount=type=cache,target=/opt/uv/cache \
+31 -3
View File
@@ -27,7 +27,7 @@
# cache mounts (ccache, cargo registry) aren't exported by `--cache-to
# type=registry`, so they don't survive a build landing on a different/fresh
# builder. sccache's cache lives in S3 instead, so it does.
FROM ubuntu:25.04 AS base
FROM ubuntu:22.04 AS base
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
--mount=type=cache,target=/var/lib/apt,sharing=locked \
@@ -52,11 +52,38 @@ RUN if [ "$USE_SCCACHE" = "1" ]; then \
&& rm -rf sccache.tar.gz sccache-v0.8.1-${SCCACHE_ARCH}-unknown-linux-musl; \
fi
######################### BINUTILS BUILD IMAGE #########################
# Build a newer GNU assembler for the AMX-FP8 instruction set.
FROM base AS binutils-build
ARG BINUTILS_VERSION=2.47
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
--mount=type=cache,target=/var/lib/apt,sharing=locked \
apt-get update -y \
&& apt-get install -y --no-install-recommends \
bison \
build-essential \
flex \
texinfo \
xz-utils \
&& curl -fsSLO "https://sourceware.org/pub/binutils/releases/binutils-${BINUTILS_VERSION}.tar.xz" \
&& tar -xf "binutils-${BINUTILS_VERSION}.tar.xz" \
&& mkdir binutils-build \
&& cd binutils-build \
&& "../binutils-${BINUTILS_VERSION}/configure" --prefix=/opt/binutils --disable-nls --disable-werror \
&& make -j"$(nproc)" \
&& make install \
&& cd .. \
&& rm -rf binutils-build "/binutils-${BINUTILS_VERSION}" "binutils-${BINUTILS_VERSION}.tar.xz"
######################### COMMON BASE IMAGE #########################
FROM base AS base-common
WORKDIR /workspace
COPY --from=binutils-build /opt/binutils /opt/binutils
ARG PYTHON_VERSION=3.12
ARG max_jobs=32
@@ -93,7 +120,8 @@ ENV CC=/usr/bin/gcc-15 CXX=/usr/bin/g++-15
ENV CCACHE_DIR=/root/.cache/ccache
ENV CMAKE_CXX_COMPILER_LAUNCHER=ccache
ENV PATH="/root/.local/bin:$PATH"
ENV PATH="/opt/binutils/bin:/root/.local/bin:$PATH"
ENV COMPILER_PATH="/opt/binutils/bin"
ENV VIRTUAL_ENV="/opt/venv"
ENV UV_PYTHON_INSTALL_DIR=/opt/uv/python
RUN uv venv --python ${PYTHON_VERSION} --seed ${VIRTUAL_ENV}
@@ -168,7 +196,7 @@ RUN apt-get update -y \
WORKDIR /workspace
COPY requirements/build/rust.txt requirements/build/rust.txt
RUN python3 -m pip install --break-system-packages --no-cache-dir -r requirements/build/rust.txt
RUN python3 -m pip install --no-cache-dir -r requirements/build/rust.txt
ARG USE_SCCACHE
ARG SCCACHE_ENDPOINT
+1 -1
View File
@@ -8,7 +8,7 @@ ARG TRITON_VERSION=3.8.0+gitc0a142ff
ARG TRITON_REPO="https://github.com/ROCm/triton.git"
ARG FA_BRANCH="0e60e394"
ARG FA_REPO="https://github.com/Dao-AILab/flash-attention.git"
ARG AITER_BRANCH="v0.1.22.post1"
ARG AITER_BRANCH="v0.1.23"
ARG AITER_REPO="https://github.com/ROCm/aiter.git"
ARG MORI_BRANCH="f95cd5e0ca7b730f92f9ffdfb358484d6965d9f7" # TheRock compatible v1.1.0
ARG MORI_REPO="https://github.com/ROCm/mori.git"
+1 -1
View File
@@ -9,7 +9,7 @@ ARG PYTORCH_AUDIO_BRANCH="v2.11.0"
ARG PYTORCH_AUDIO_REPO="https://github.com/pytorch/audio.git"
ARG FA_BRANCH="0e60e394"
ARG FA_REPO="https://github.com/Dao-AILab/flash-attention.git"
ARG AITER_BRANCH="v0.1.22.post1"
ARG AITER_BRANCH="v0.1.23"
ARG AITER_REPO="https://github.com/ROCm/aiter.git"
ARG MORI_BRANCH="v1.1.0"
ARG MORI_REPO="https://github.com/ROCm/mori.git"
+40
View File
@@ -596,6 +596,46 @@ vllm bench serve \
--max-concurrency 512
```
#### Responses API Benchmark
The `openai-responses` backend benchmarks vLLM's
[Responses API](../serving/online_serving/openai_compatible_server.md#responses-api)
directly, instead of routing the same workload through `/v1/chat/completions`.
```bash
# Server
vllm serve openai/gpt-oss-20b
# Client
vllm bench serve \
--backend openai-responses \
--endpoint /v1/responses \
--model openai/gpt-oss-20b \
--dataset-name random \
--random-input-len 1024 \
--random-output-len 1024 \
--num-prompts 200
```
Reasoning deltas (`response.reasoning_text.delta`) count towards TTFT and ITL,
because the server is already decoding tokens when it emits them. Only output
text (`response.output_text.delta`) is collected as the generated text, which
matches how the `openai-chat` backend treats `DeltaMessage.reasoning`. End-to-end
latency stops at the last token event, so `latency - ttft` equals `sum(itl)` and
TPOT is comparable with the other endpoints. The input and output token counts
come from the usage block on `response.completed`, which is still required: a
stream that ends without a terminal event is reported as a failed request.
The backend measures one streamed text-generation request per prompt. Built-in
tools, MCP, and multi-turn state via `previous_response_id` are out of scope.
All sampling parameter flags are supported except `--min-p`, which the
Responses API does not accept.
!!! warning
Do not pass `--extra-body '{"include_reasoning": false}'` when benchmarking
a reasoning model. The server still generates reasoning tokens but emits no
events for them, so the whole reasoning phase is absorbed into TTFT.
#### Running With Sampling Parameters
When using OpenAI-compatible backends such as `vllm`, optional sampling
+1 -1
View File
@@ -144,7 +144,7 @@ When batch invariance is enabled, vLLM:
1. Uses deterministic kernel implementations for attention and other operations
2. Ensures consistent numerical behavior across different batch sizes
3. Disables certain optimizations that may introduce non-determinism (such as custom all-reduce operations in tensor parallel mode)
3. Disables certain optimizations that may introduce non-determinism (such as custom all-reduce operations in tensor parallel mode, and sequence parallelism / async TP, whose reduce-scatter path is not batch-invariant)
4. On CUDA devices with tuned matmul table entries for the model's bf16 unquantized forward linear layers (Ada, Hopper, Blackwell),
runs without `torch.compile` using breakable CUDA graphs so tile configs follow the runtime batch size; set `VLLM_USE_BREAKABLE_CUDAGRAPH=0` to opt out
(not applied when sequence parallelism / async TP are enabled, since those are `torch.compile` passes).
@@ -46,6 +46,13 @@ Now you can send requests to the proxy server through port 8000.
- For headless instances, must be the same as the master instance
- Each instance needs a unique port on its host; using the same port number across different hosts is fine
- `VLLM_MOONCAKE_CONNECTOR_TIMEOUT`: Per-request timeout (in seconds) for MooncakeConnector HTTP calls to the bootstrap server, including prefiller worker registration at startup. (Optional)
- Default: 30.0
- Global rank 0 hosts the bootstrap server in the same process that mounts the Mooncake transfer engine's host segment, so a large `MOONCAKE_GLOBAL_SEGMENT_SIZE` can delay responses for several seconds during startup
- Raise this if bootstrap server timeout warnings appear on hosts with very large host-memory segments
- Bootstrap registration uses up to 3 attempts internally, with exponential backoff capped at 10 seconds; connection errors and timeouts are both retried
- Once the attempts are exhausted the caller raises instead of blocking, so failures surface rather than hanging
- `WITH_NVIDIA_PEERMEM`: Selects how mooncake registers GPU memory for RDMA. Read by mooncake, not vLLM.
- Default: 1, which uses `ibv_reg_mr()` and requires the `nvidia-peermem` kernel module to be loaded
- Set to 0 to use the DMA-BUF path, which does not need that module. Required on hosts where `nvidia-peermem` is not loaded, such as GB200
@@ -23,6 +23,10 @@ METRIC_SOURCE_FILES = [
"path": "vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py",
"key": "metrics-nixl",
},
{
"path": "vllm/v1/simple_kv_offload/metrics.py",
"key": "metrics-simple-kv-offload",
},
{"path": "vllm/v1/metrics/perf.py", "key": "metrics-mfu"},
]
+2 -2
View File
@@ -50,8 +50,8 @@ The three supported scoring functions are as illustrated in the figure below.
| `ModernBertForSequenceClassification` | ModernBERT-based | `Alibaba-NLP/gte-reranker-modernbert-base`, etc. | N/A | | |
| `Qwen2ForSequenceClassification`<sup>C</sup> | Qwen2-based | `mixedbread-ai/mxbai-rerank-base-v2`(see note), etc. | [mxbai_rerank_v2.jinja](../../../examples/pooling/score/template/mxbai_rerank_v2.jinja) | ✅︎ | ✅︎ |
| `Qwen3ForSequenceClassification`<sup>C</sup> | Qwen3-based | `tomaarsen/Qwen3-Reranker-0.6B-seq-cls`, `Qwen/Qwen3-Reranker-0.6B`(see note), etc. | [qwen3_reranker.jinja](../../../examples/pooling/score/template/qwen3_reranker.jinja) | ✅︎ | ✅︎ |
| `RobertaForSequenceClassification` | RoBERTa-based | `cross-encoder/quora-roberta-base`, etc. | N/A | | |
| `XLMRobertaForSequenceClassification` | XLM-RoBERTa-based | `BAAI/bge-reranker-v2-m3`, etc. | N/A | | |
| `RobertaForSequenceClassification` | RoBERTa-based | `cross-encoder/quora-roberta-base`, etc. | N/A | ✅︎ | |
| `XLMRobertaForSequenceClassification` | XLM-RoBERTa-based | `BAAI/bge-reranker-v2-m3`, etc. | N/A | ✅︎ | |
| `*Model`<sup>C</sup>, `*ForCausalLM`<sup>C</sup>, etc. | Generative models | N/A | N/A | \* | \* |
<sup>C</sup> Automatically converted into a classification model via `--convert classify`. ([details](./README.md#model-conversion))
+24
View File
@@ -45,6 +45,30 @@ The following metrics are exposed:
--8<-- "gen:metrics-nixl"
## Simple CPU Offload Connector Metrics
These metrics are exposed when the `SimpleCPUOffloadConnector` KV connector
is configured (e.g. `--kv-transfer-config='{"kv_connector":
"SimpleCPUOffloadConnector", "kv_role": "kv_both", "kv_connector_extra_config":
{"kv_offload_backend": "disk", "disk_path": "/mnt/nvme/kv"}}'`). They are
updated once per engine step.
Caveats to keep in mind when interpreting them:
- A "completed" store means the write syscalls returned; it is **not**
fsync-durable, and the disk backend's file is process-lifetime scratch,
unlinked at startup and shutdown.
- `save_outcomes_total` classifies eager-mode boundary hand-off stores;
lazy-mode stores are not classified.
- `used_blocks` counts blocks pinned by in-flight transfers or cache hits;
warm cached blocks that are evictable are not counted. Use the
`capacity_blocks` label of `simple_kv_offload_info` as the denominator.
- Counters and gauges are quantized to engine steps. They are reported by the
scheduler process and reflect engine-wide logical block counts, not values
pooled from individual tensor-parallel workers.
--8<-- "gen:metrics-simple-kv-offload"
## Model Flops Utilization (MFU) Performance Metrics
These metrics are available via `--enable-mfu-metrics`:
+5
View File
@@ -258,3 +258,8 @@ CANN = "CANN"
[tool.uv]
no-build-isolation-package = ["torch"]
# vLLM depends on oss-harmony, a fork that ships the same `openai_harmony`
# module without downloading tiktoken vocabularies at runtime. Installing
# openai-harmony alongside it silently overwrites those files, so keep it out
# of every resolution; gpt-oss pulls it in transitively.
exclude-dependencies = ["openai-harmony"]
+1 -1
View File
@@ -49,7 +49,7 @@ pybase64 # fast base64 implementation
cbor2 # Required for cross-language serialization of hashable objects
ijson # Required for mistral streaming tool parser
setproctitle # Used to set process names for better debugging and monitoring
openai-harmony >= 0.0.3 # Required for gpt-oss
oss-harmony >= 0.0.11 # Required for gpt-oss; includes embedded vocabularies
anthropic >= 0.71.0
model-hosting-container-standards >= 0.1.14, < 1.0.0
mcp >= 2.0.0, < 3.0.0
+31 -10
View File
@@ -76,7 +76,9 @@ black==24.10.0
blake3==1.0.9
# via -r requirements/test/../common.txt
blobfile==3.0.0
# via -r requirements/test/cuda.in
# via
# -r requirements/test/cuda.in
# gpt-oss
bm25s==0.2.13
# via mteb
boto3==1.35.57
@@ -111,7 +113,7 @@ chardet==5.2.0
# via mbstrdecoder
charset-normalizer==3.4.0
# via requests
chz==0.3.0
chz==0.4.0
# via gpt-oss
click==8.4.2
# via
@@ -190,7 +192,7 @@ distro==1.9.0
# openai
dnspython==2.7.0
# via email-validator
docker==7.1.0
docker==7.2.0
# via gpt-oss
docopt==0.6.2
# via num2words
@@ -291,7 +293,7 @@ googleapis-common-protos==1.70.0
# google-api-core
# opentelemetry-exporter-otlp-proto-grpc
# opentelemetry-exporter-otlp-proto-http
gpt-oss==0.0.8
gpt-oss==0.0.9
# via -r requirements/test/cuda.in
graphql-core==3.2.6
# via hypothesis-graphql
@@ -402,6 +404,7 @@ jinja2==3.1.6
# datamodel-code-generator
# fastapi
# genai-perf
# gpt-oss
# lm-eval
# torch
jiter==0.15.0
@@ -433,6 +436,10 @@ jsonschema-specifications==2024.10.1
# via jsonschema
junit-xml==1.9
# via schemathesis
jupyter-client==8.10.0
# via gpt-oss
jupyter-core==5.9.1
# via jupyter-client
kaldi-native-fbank==1.22.3
# via -r requirements/test/cuda.in
kaleido==0.2.1
@@ -549,6 +556,7 @@ numpy==2.2.6
# evaluate
# fastparquet
# genai-perf
# gpt-oss
# imagehash
# imageio
# librosa
@@ -583,8 +591,6 @@ numpy==2.2.6
open-clip-torch==2.32.0
# via -r requirements/test/cuda.in
openai==2.44.0
# via -r requirements/test/../common.txt
openai-harmony==0.0.4
# via
# -r requirements/test/../common.txt
# gpt-oss
@@ -641,6 +647,8 @@ optuna==3.6.1
# via genai-perf
orjson==3.11.5
# via genai-perf
oss-harmony==0.0.11
# via -r requirements/test/../common.txt
outlines-core==0.2.14
# via -r requirements/test/../common.txt
packaging==24.2
@@ -672,6 +680,7 @@ pandas==2.2.3
# evaluate
# fastparquet
# genai-perf
# gpt-oss
# statsmodels
partial-json-parser==0.2.1.1.post7
# via -r requirements/test/../common.txt
@@ -702,6 +711,7 @@ pillow==10.4.0
platformdirs==4.3.6
# via
# black
# jupyter-core
# pooch
# virtualenv
plotly==5.24.1
@@ -794,7 +804,7 @@ pydantic==2.12.0
# model-hosting-container-standards
# mteb
# openai
# openai-harmony
# oss-harmony
# pydantic-extra-types
# pydantic-settings
# ray
@@ -855,6 +865,7 @@ pytest-timeout==2.3.1
python-dateutil==2.9.0.post0
# via
# botocore
# jupyter-client
# matplotlib
# pandas
# typepy
@@ -898,7 +909,9 @@ pyyaml==6.0.2
# uvicorn
# vocos
pyzmq==27.1.0
# via -r requirements/test/../common.txt
# via
# -r requirements/test/../common.txt
# jupyter-client
rapidfuzz==3.12.1
# via jiwer
ray==2.56.1
@@ -1077,7 +1090,7 @@ starlette-testclient==0.4.1
# via schemathesis
statsmodels==0.14.4
# via genai-perf
structlog==25.4.0
structlog==26.1.0
# via gpt-oss
supervisor==4.3.0
# via model-hosting-container-standards
@@ -1101,7 +1114,7 @@ tenacity==9.1.2
# schemathesis
tensorizer==2.10.1
# via -r requirements/test/cuda.in
termcolor==3.1.0
termcolor==3.3.0
# via gpt-oss
threadpoolctl==3.5.0
# via scikit-learn
@@ -1157,11 +1170,14 @@ torchvision==0.28.0+cpu
# open-clip-torch
# segmentation-models-pytorch
# timm
tornado==6.5.10
# via jupyter-client
tqdm==4.67.3
# via
# -r requirements/test/../common.txt
# datasets
# evaluate
# gpt-oss
# huggingface-hub
# lm-eval
# mteb
@@ -1174,6 +1190,10 @@ tqdm==4.67.3
# segmentation-models-pytorch
# sentence-transformers
# transformers
traitlets==5.16.1
# via
# jupyter-client
# jupyter-core
transformers==5.17.0
# via
# -r requirements/test/../common.txt
@@ -1226,6 +1246,7 @@ typing-extensions==4.15.0
# grpcio
# httpx2
# huggingface-hub
# jupyter-client
# librosa
# lm-eval
# mcp
+1 -1
View File
@@ -66,7 +66,7 @@ decord==0.6.0; platform_machine == "x86_64"
imagehash # Required for Prithvi tests
segmentation-models-pytorch > 0.4.0 # Required for Prithvi tests
gpt-oss >= 0.0.7; python_version > '3.11'
gpt-oss[eval] >= 0.0.7; python_version > '3.11'
perceptron # required for isaac test
kaldi-native-fbank >= 1.18.7 # required for fireredasr2 test
+31 -11
View File
@@ -81,7 +81,9 @@ black==24.10.0
blake3==1.0.9
# via -r requirements/test/../common.txt
blobfile==3.0.0
# via -r requirements/test/cuda.in
# via
# -r requirements/test/cuda.in
# gpt-oss
bm25s==0.2.13
# via mteb
boto3==1.35.57
@@ -116,7 +118,7 @@ chardet==5.2.0
# via mbstrdecoder
charset-normalizer==3.4.0
# via requests
chz==0.3.0
chz==0.4.0
# via gpt-oss
click==8.4.2
# via
@@ -205,7 +207,7 @@ distro==1.9.0
# openai
dnspython==2.7.0
# via email-validator
docker==7.1.0
docker==7.2.0
# via gpt-oss
docopt==0.6.2
# via num2words
@@ -310,7 +312,7 @@ googleapis-common-protos==1.70.0
# google-api-core
# opentelemetry-exporter-otlp-proto-grpc
# opentelemetry-exporter-otlp-proto-http
gpt-oss==0.0.8
gpt-oss==0.0.9
# via -r requirements/test/cuda.in
graphql-core==3.2.6
# via hypothesis-graphql
@@ -424,6 +426,7 @@ jinja2==3.1.6
# datamodel-code-generator
# fastapi
# genai-perf
# gpt-oss
# lm-eval
# torch
jiter==0.15.0
@@ -456,6 +459,10 @@ jsonschema-specifications==2024.10.1
# via jsonschema
junit-xml==1.9
# via schemathesis
jupyter-client==8.10.0
# via gpt-oss
jupyter-core==5.9.1
# via jupyter-client
kaldi-native-fbank==1.22.3
# via -r requirements/test/cuda.in
kaleido==0.2.1
@@ -584,6 +591,7 @@ numpy==2.2.6
# evaluate
# fastparquet
# genai-perf
# gpt-oss
# imagehash
# imageio
# librosa
@@ -662,10 +670,6 @@ openai==2.44.0
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
openai-harmony==0.0.4
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
# gpt-oss
opencensus==0.11.4
# via ray
@@ -727,6 +731,10 @@ optuna==3.6.1
# via genai-perf
orjson==3.11.5
# via genai-perf
oss-harmony==0.0.11
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
outlines-core==0.2.14
# via
# -c requirements/common.txt
@@ -760,6 +768,7 @@ pandas==2.2.3
# evaluate
# fastparquet
# genai-perf
# gpt-oss
# statsmodels
partial-json-parser==0.2.1.1.post7
# via -r requirements/test/../common.txt
@@ -790,6 +799,7 @@ pillow==10.4.0
platformdirs==4.3.6
# via
# black
# jupyter-core
# pooch
# virtualenv
plotly==5.24.1
@@ -887,7 +897,7 @@ pydantic==2.12.0
# model-hosting-container-standards
# mteb
# openai
# openai-harmony
# oss-harmony
# pydantic-extra-types
# pydantic-settings
# ray
@@ -948,6 +958,7 @@ pytest-timeout==2.3.1
python-dateutil==2.9.0.post0
# via
# botocore
# jupyter-client
# matplotlib
# pandas
# typepy
@@ -994,6 +1005,7 @@ pyzmq==27.1.0
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
# jupyter-client
rapidfuzz==3.12.1
# via jiwer
ray==2.56.1
@@ -1177,7 +1189,7 @@ starlette-testclient==0.4.1
# via schemathesis
statsmodels==0.14.4
# via genai-perf
structlog==25.4.0
structlog==26.1.0
# via gpt-oss
supervisor==4.3.0
# via model-hosting-container-standards
@@ -1201,7 +1213,7 @@ tenacity==9.1.2
# schemathesis
tensorizer==2.10.1
# via -r requirements/test/cuda.in
termcolor==3.1.0
termcolor==3.3.0
# via gpt-oss
threadpoolctl==3.5.0
# via scikit-learn
@@ -1264,11 +1276,14 @@ torchvision==0.28.0+cu130
# open-clip-torch
# segmentation-models-pytorch
# timm
tornado==6.5.10
# via jupyter-client
tqdm==4.67.3
# via
# -r requirements/test/../common.txt
# datasets
# evaluate
# gpt-oss
# huggingface-hub
# lm-eval
# mteb
@@ -1281,6 +1296,10 @@ tqdm==4.67.3
# segmentation-models-pytorch
# sentence-transformers
# transformers
traitlets==5.16.1
# via
# jupyter-client
# jupyter-core
transformers==5.17.0
# via
# -c requirements/common.txt
@@ -1337,6 +1356,7 @@ typing-extensions==4.15.0
# grpcio
# httpx2
# huggingface-hub
# jupyter-client
# librosa
# lm-eval
# mcp
+1 -1
View File
@@ -65,7 +65,7 @@ decord==0.6.0
imagehash # Required for Prithvi tests
segmentation-models-pytorch>0.4.0 # Required for Prithvi tests
gpt-oss>=0.0.7; python_version > '3.11'
gpt-oss[eval]>=0.0.7; python_version > '3.11'
perceptron # required for isaac test
kaldi-native-fbank>=1.18.7 # required for fireredasr2 test
+29 -9
View File
@@ -80,7 +80,9 @@ black==26.3.1
blake3==1.0.8
# via -r requirements/test/../common.txt
blobfile==3.0.0
# via -r requirements/test/rocm.in
# via
# -r requirements/test/rocm.in
# gpt-oss
bm25s==0.2.13
# via mteb
boto3==1.42.74
@@ -198,7 +200,7 @@ distro==1.9.0
# openai
dnspython==2.8.0
# via email-validator
docker==7.1.0
docker==7.2.0
# via gpt-oss
docopt==0.6.2
# via num2words
@@ -300,7 +302,7 @@ googleapis-common-protos==1.73.0
# google-api-core
# opentelemetry-exporter-otlp-proto-grpc
# opentelemetry-exporter-otlp-proto-http
gpt-oss==0.0.8
gpt-oss==0.0.9
# via -r requirements/test/rocm.in
graphql-core==3.2.8
# via hypothesis-graphql
@@ -415,6 +417,7 @@ jinja2==3.1.6
# datamodel-code-generator
# fastapi
# genai-perf
# gpt-oss
# lm-eval
# torch
jiter==0.14.0
@@ -447,6 +450,10 @@ jsonschema-specifications==2025.9.1
# via jsonschema
junit-xml==1.9
# via schemathesis
jupyter-client==8.10.0
# via gpt-oss
jupyter-core==5.9.1
# via jupyter-client
kaldi-native-fbank==1.22.3
# via -r requirements/test/rocm.in
kaleido==1.0.0
@@ -587,6 +594,7 @@ numpy==2.2.6
# evaluate
# fastparquet
# genai-perf
# gpt-oss
# imagehash
# imageio
# librosa
@@ -629,10 +637,6 @@ openai==2.31.0
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
openai-harmony==0.0.8
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
# gpt-oss
opencensus==0.11.4
# via ray
@@ -699,6 +703,10 @@ orjson==3.11.7
# via
# genai-perf
# kaleido
oss-harmony==0.0.11
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
outlines-core==0.2.14
# via
# -c requirements/common.txt
@@ -734,6 +742,7 @@ pandas==3.0.1
# evaluate
# fastparquet
# genai-perf
# gpt-oss
# statsmodels
partial-json-parser==0.2.1.1.post7
# via -r requirements/test/../common.txt
@@ -764,6 +773,7 @@ pillow==12.1.1
platformdirs==4.3.6
# via
# black
# jupyter-core
# pooch
# python-discovery
# virtualenv
@@ -861,7 +871,7 @@ pydantic==2.12.5
# model-hosting-container-standards
# mteb
# openai
# openai-harmony
# oss-harmony
# pydantic-extra-types
# pydantic-settings
# ray
@@ -922,6 +932,7 @@ pytest-timeout==2.3.1
python-dateutil==2.9.0.post0
# via
# botocore
# jupyter-client
# matplotlib
# pandas
# typepy
@@ -970,6 +981,7 @@ pyzmq==27.1.0
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
# jupyter-client
rapidfuzz==3.12.1
# via
# -r requirements/test/rocm.in
@@ -1159,7 +1171,7 @@ starlette-testclient==0.4.1
# via schemathesis
statsmodels==0.14.6
# via genai-perf
structlog==25.5.0
structlog==26.1.0
# via gpt-oss
supervisor==4.3.0
# via model-hosting-container-standards
@@ -1215,11 +1227,14 @@ tokenizers==0.23.1
# transformers
torch-c-dlpack-ext==0.1.5
# via tilelang
tornado==6.5.10
# via jupyter-client
tqdm==4.67.3
# via
# -r requirements/test/../common.txt
# datasets
# evaluate
# gpt-oss
# huggingface-hub
# lm-eval
# mteb
@@ -1233,6 +1248,10 @@ tqdm==4.67.3
# sentence-transformers
# tilelang
# transformers
traitlets==5.16.1
# via
# jupyter-client
# jupyter-core
transformers==5.17.0
# via
# -c requirements/common.txt
@@ -1287,6 +1306,7 @@ typing-extensions==4.15.0
# grpcio
# httpx2
# huggingface-hub
# jupyter-client
# librosa
# lm-eval
# mcp
+1 -1
View File
@@ -32,7 +32,7 @@ soundfile
# --- Tool Parsing & Evaluation ---
blobfile
rapidfuzz
gpt-oss >= 0.0.7; python_version > '3.11'
gpt-oss[eval] >= 0.0.7; python_version > '3.11'
schemathesis>=4.0.0 # Required for openai schema test.
jiwer
bm25s
+29 -8
View File
@@ -67,7 +67,9 @@ azure-storage-blob==12.30.1
blake3==1.0.9
# via -r requirements/test/../common.txt
blobfile==3.2.0
# via -r requirements/test/xpu.in
# via
# -r requirements/test/xpu.in
# gpt-oss
bm25s==0.3.10
# via
# -r requirements/test/xpu.in
@@ -257,7 +259,7 @@ googleapis-common-protos==1.75.1
# google-api-core
# opentelemetry-exporter-otlp-proto-grpc
# opentelemetry-exporter-otlp-proto-http
gpt-oss==0.0.8
gpt-oss==0.0.9
# via -r requirements/test/xpu.in
graphql-core==3.2.11
# via hypothesis-graphql
@@ -377,6 +379,7 @@ jinja2==3.1.6
# via
# -c requirements/xpu.txt
# fastapi
# gpt-oss
# lm-eval
# torch
jiter==0.16.0
@@ -406,6 +409,10 @@ jsonschema-rs==0.49.9
# via schemathesis
jsonschema-specifications==2025.9.1
# via jsonschema
jupyter-client==8.10.0
# via gpt-oss
jupyter-core==5.9.1
# via jupyter-client
kiwisolver==1.5.1
# via matplotlib
lark==1.2.2
@@ -526,6 +533,7 @@ numpy==2.2.6
# datasets
# decord
# evaluate
# gpt-oss
# librosa
# lm-eval
# matplotlib
@@ -575,10 +583,6 @@ openai==3.2.0
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
openai-harmony==0.0.8
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
# gpt-oss
opencv-python-headless==5.0.0.93
# via
@@ -632,6 +636,10 @@ opentelemetry-semantic-conventions-ai==0.5.1
# -c requirements/common.txt
# -r requirements/test/../common.txt
# -r requirements/test/xpu.in
oss-harmony==0.0.11
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
outlines-core==0.2.14
# via
# -c requirements/common.txt
@@ -656,6 +664,7 @@ pandas==3.0.5
# via
# datasets
# evaluate
# gpt-oss
partial-json-parser==0.2.1.1.post7
# via -r requirements/test/../common.txt
pathvalidate==3.3.1
@@ -667,7 +676,9 @@ pillow==12.3.0
# mistral-common
# torchvision
platformdirs==4.11.3
# via pooch
# via
# jupyter-core
# pooch
pluggy==1.6.0
# via
# pytest
@@ -747,7 +758,7 @@ pydantic==2.13.4
# model-hosting-container-standards
# mteb
# openai
# openai-harmony
# oss-harmony
# pydantic-extra-types
# pydantic-settings
# xgrammar
@@ -806,6 +817,7 @@ pytest-timeout==2.4.0
python-dateutil==2.9.0.post0
# via
# botocore
# jupyter-client
# matplotlib
# pandas
# typepy
@@ -841,6 +853,7 @@ pyzmq==27.1.0
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
# jupyter-client
rapidfuzz==3.14.5
# via
# -r requirements/test/xpu.in
@@ -1066,11 +1079,14 @@ torchvision==0.29.0+xpu
# via
# open-clip-torch
# timm
tornado==6.5.10
# via jupyter-client
tqdm==4.70.0
# via
# -r requirements/test/../common.txt
# datasets
# evaluate
# gpt-oss
# huggingface-hub
# lm-eval
# modelscope
@@ -1081,6 +1097,10 @@ tqdm==4.70.0
# pqdm
# sentence-transformers
# transformers
traitlets==5.16.1
# via
# jupyter-client
# jupyter-core
transformers==5.17.0
# via
# -c requirements/common.txt
@@ -1132,6 +1152,7 @@ typing-extensions==4.16.0
# grpcio
# httpx2
# huggingface-hub
# jupyter-client
# lm-eval
# mcp
# mcp-types
@@ -0,0 +1,455 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the ``openai-responses`` backend of ``vllm bench serve``.
The Responses stream differs from Chat Completions in ways the benchmark client
has to get right: every SSE message is labelled with an ``event:`` line,
reasoning and output text arrive as separate token-bearing events, and a
request ends with ``response.completed`` rather than with its last token. These
tests drive the real request function against a local SSE server so the
resulting latency and usage numbers are covered end to end.
"""
import asyncio
import json
from collections.abc import Sequence
import aiohttp
import pytest
from aiohttp import web
from aiohttp.test_utils import TestServer
from vllm.benchmarks.lib.endpoint_request_func import (
ASYNC_REQUEST_FUNCS,
OPENAI_COMPATIBLE_BACKENDS,
RequestFuncInput,
RequestFuncOutput,
async_request_openai_responses,
)
# These tests never touch torch, so skip the GPU cleanup fixture.
pytestmark = pytest.mark.skip_global_cleanup
MODEL = "openai/gpt-oss-20b"
# Gaps between scripted events. GAP_TOKEN is much larger than GAP_PREFILL so
# that a time to first token measured at the wrong event is unambiguous.
GAP_PREFILL = 0.15
GAP_TOKEN = 0.5
GAP_FINALIZE = 0.3
def sse(event_type: str, payload: dict) -> bytes:
"""Frame one event the way ``/v1/responses`` does, with an ``event:`` line.
``ensure_ascii=False`` matches the server, whose pydantic
``model_dump_json`` emits non-ASCII characters unescaped.
"""
data = json.dumps({"type": event_type, **payload}, ensure_ascii=False)
return f"event: {event_type}\ndata: {data}\n\n".encode()
def usage_event(
input_tokens: int,
output_tokens: int,
event_type: str = "response.completed",
status: str = "completed",
):
return sse(
event_type,
{
"response": {
"status": status,
"usage": {
"input_tokens": input_tokens,
"output_tokens": output_tokens,
"total_tokens": input_tokens + output_tokens,
},
}
},
)
def text_delta(delta: str) -> bytes:
return sse("response.output_text.delta", {"delta": delta})
def reasoning_delta(delta: str) -> bytes:
return sse("response.reasoning_text.delta", {"delta": delta})
# The metadata events the server sends before any token. None of these may
# start the TTFT clock or contribute an ITL sample.
PREAMBLE = (
sse("response.created", {"response": {"status": "in_progress"}})
+ sse("response.in_progress", {"response": {"status": "in_progress"}})
+ sse("response.output_item.added", {"output_index": 0})
+ sse("response.reasoning_part.added", {"output_index": 0})
+ sse("response.content_part.added", {"output_index": 0})
)
# The `.done` family the server sends after the last delta. These repeat the
# accumulated text, so timing them would add bogus ITL samples and counting
# their payload would duplicate the output.
DONE_EVENTS = (
sse("response.reasoning_text.done", {"text": "The user asks "})
+ sse("response.reasoning_part.done", {"output_index": 0})
+ sse("response.output_text.done", {"text": "Rayleigh scattering."})
+ sse("response.content_part.done", {"output_index": 0})
+ sse("response.output_item.done", {"output_index": 0})
)
async def _serve(
script: Sequence[bytes | float],
*,
status: int = 200,
chunk_size: int | None = None,
seen: dict | None = None,
**request_kwargs,
) -> RequestFuncOutput:
"""Run the request function against a server replaying ``script``.
Args:
script: Byte strings to write, interleaved with floats to sleep for.
status: HTTP status to respond with. Non-200 skips the script.
chunk_size: If set, split every write into chunks of this many bytes to
emulate fragmented transport.
seen: If given, populated with the request payload.
request_kwargs: Extra fields for the ``RequestFuncInput``.
"""
async def handler(request: web.Request) -> web.StreamResponse:
if seen is not None:
seen["payload"] = await request.json()
seen["headers"] = dict(request.headers)
if status != 200:
return web.Response(status=status, reason="Bad Request")
response = web.StreamResponse(headers={"Content-Type": "text/event-stream"})
await response.prepare(request)
for item in script:
if isinstance(item, float):
await asyncio.sleep(item)
elif chunk_size is None:
await response.write(item)
else:
for i in range(0, len(item), chunk_size):
await response.write(item[i : i + chunk_size])
await response.write_eof()
return response
app = web.Application()
app.router.add_post("/v1/responses", handler)
server = TestServer(app)
await server.start_server()
try:
request_input = RequestFuncInput(
prompt="Why is the sky blue?",
api_url=str(server.make_url("/v1/responses")),
prompt_len=5,
output_len=16,
model=MODEL,
**request_kwargs,
)
async with aiohttp.ClientSession() as session:
return await async_request_openai_responses(request_input, session)
finally:
await server.close()
def run(script: Sequence[bytes | float], **kwargs) -> RequestFuncOutput:
return asyncio.run(_serve(script, **kwargs))
def test_backend_is_registered_as_openai_compatible():
assert ASYNC_REQUEST_FUNCS["openai-responses"] is async_request_openai_responses
# Membership enables --ignore-eos on random datasets and the sampling
# parameter flags, all of which ResponsesRequest accepts.
assert "openai-responses" in OPENAI_COMPATIBLE_BACKENDS
def test_request_payload_is_accepted_by_the_server_schema():
"""Every field sent must be a real ResponsesRequest field.
This is what makes the backend safe to list in OPENAI_COMPATIBLE_BACKENDS:
the flags that membership enables have to reach the model, and the server
drops unknown fields without complaining.
"""
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
sampling = {
"temperature": 0.7,
"top_p": 0.9,
"top_k": 20,
"frequency_penalty": 0.2,
"presence_penalty": 0.1,
"repetition_penalty": 1.05,
}
seen: dict = {}
output = run(
[PREAMBLE, text_delta("hi"), usage_event(5, 1)],
seen=seen,
extra_body=sampling,
ignore_eos=True,
model_name="served-alias",
request_id="bench-7",
extra_headers={"x-team": "perf"},
)
assert output.success
# --served-model-name wins over --model, as on the other backends.
assert seen["payload"]["model"] == "served-alias"
# The route is behind validate_json_request, which 415s a missing type.
assert seen["headers"]["Content-Type"] == "application/json"
assert seen["headers"]["x-request-id"] == "bench-7"
assert seen["headers"]["x-team"] == "perf"
assert seen["payload"] == {
"model": "served-alias",
"input": "Why is the sky blue?",
"max_output_tokens": 16,
"stream": True,
"store": False,
"ignore_eos": True,
**sampling,
}
assert set(seen["payload"]) <= set(ResponsesRequest.model_fields)
ResponsesRequest.model_validate(seen["payload"])
# `vllm bench serve` rejects --min-p for this backend for the same reason.
assert "min_p" not in ResponsesRequest.model_fields
def test_latency_accounting_matches_the_other_endpoints():
"""`latency - ttft` must equal `sum(itl)`.
`serve.py` derives TPOT as `(latency - ttft) / (output_len - 1)`, which only
means average inter-token latency when that identity holds. #55508 fixed the
same accounting on the chat and audio backends; the trailing events after
the last token must not extend the request.
"""
output = run(
[
PREAMBLE,
GAP_PREFILL / 2,
# A keep-alive comment sent during prefill is not a token.
b": keep-alive\n\n",
GAP_PREFILL / 2,
reasoning_delta("The user asks "),
GAP_TOKEN,
text_delta("Rayleigh"),
text_delta(" scattering."),
DONE_EVENTS,
GAP_FINALIZE,
usage_event(5, 8),
]
)
assert output.success
# Reasoning is timed but not returned, matching how `openai-chat` treats
# `DeltaMessage.reasoning`.
assert output.generated_text == "Rayleigh scattering."
# Neither the response.created/in_progress/output_item.added preamble nor
# the keep-alive comment may start the clock, and the reasoning delta must,
# so TTFT lands on the prefill gap rather than near zero or out at the
# first output text delta.
assert GAP_PREFILL * 0.8 <= output.ttft < GAP_PREFILL + GAP_TOKEN * 0.5
# One ITL per token event after the first: the gap before "Rayleigh" and
# the back-to-back delta that follows it.
assert len(output.itl) == 2
assert output.itl[0] >= GAP_TOKEN * 0.5
assert output.itl[1] < GAP_TOKEN * 0.5
# The identity is exact, so no wall-clock margin is involved. The .done
# family and the GAP_FINALIZE wait before response.completed sit outside
# the request, exactly as the trailing usage chunk does on chat.
assert output.latency - output.ttft == pytest.approx(sum(output.itl), abs=1e-9)
assert output.start_time > 0.0
def test_usage_is_taken_from_the_terminal_event():
output = run([PREAMBLE, text_delta("blue"), usage_event(17, 4)])
assert output.success
assert output.prompt_len == 17
assert output.output_tokens == 4
@pytest.mark.parametrize("event_type", ["response.completed", "response.incomplete"])
def test_incomplete_response_is_a_successful_request(event_type: str):
"""Hitting max_output_tokens is the Responses form of finish_reason=length.
vLLM reports it as `response.completed` carrying `status: "incomplete"`.
The spec's dedicated `response.incomplete` event is accepted as well.
"""
output = run(
[
PREAMBLE,
text_delta("truncated"),
usage_event(17, 16, event_type=event_type, status="incomplete"),
]
)
assert output.success
assert output.generated_text == "truncated"
assert output.output_tokens == 16
@pytest.mark.parametrize("chunk_size", [1, 3, 7, 64])
def test_output_survives_transport_chunk_boundaries(chunk_size: int):
"""Splits inside the event line, the JSON, or a multibyte character."""
text = " leading 中 trailing "
output = run(
[PREAMBLE, text_delta(text), usage_event(5, 6)],
chunk_size=chunk_size,
)
assert output.success
assert output.generated_text == text
assert output.output_tokens == 6
@pytest.mark.parametrize(
"char", ["\u2028", "\u2029", "\u0085"], ids=["u2028", "u2029", "u0085"]
)
def test_unicode_line_separators_do_not_truncate_the_payload(char: str):
"""The server emits these unescaped and they are legal inside a JSON string.
`str.splitlines()` treats them as line breaks, which would cut the `data:`
line short and fail the request on valid model output.
"""
text = f"before{char}after"
output = run([PREAMBLE, text_delta(text), usage_event(5, 3)])
assert output.success, output.error
assert output.generated_text == text
def test_empty_data_line_is_skipped():
"""A bare `data:` line carries no JSON and must not fail the request."""
output = run([PREAMBLE, b"data:\n\n", text_delta("ok"), usage_event(5, 1)])
assert output.success, output.error
assert output.generated_text == "ok"
def test_prompt_len_falls_back_when_usage_is_absent():
"""A terminal event without usage keeps the locally computed prompt length."""
output = run(
[
PREAMBLE,
text_delta("hi"),
sse("response.completed", {"response": {"status": "completed"}}),
]
)
assert output.success, output.error
assert output.prompt_len == 5
def test_multi_line_data_payload_is_joined_with_newlines():
"""The SSE spec allows one payload to span several `data:` lines."""
script: list[bytes | float] = [
PREAMBLE,
b"event: response.output_text.delta\n"
b'data: {"type": "response.output_text.delta",\n'
b'data: "delta": "ok"}\n\n',
usage_event(5, 1),
]
output = run(script)
assert output.success, output.error
assert output.generated_text == "ok"
def test_done_sentinel_is_skipped():
"""Other Responses servers send `[DONE]`, but vLLM does not."""
output = run([PREAMBLE, text_delta("ok"), usage_event(5, 1), b"data: [DONE]\n\n"])
assert output.success, output.error
assert output.generated_text == "ok"
def test_error_before_any_token_reports_the_server_error():
"""The server's message must win over the generic no-TTFT message."""
output = run(
[PREAMBLE, b'event: error\ndata: {"type": "error", "message": "boom"}\n\n']
)
assert not output.success
assert "boom" in output.error
def test_http_error_is_not_reported_as_success():
output = run([], status=400)
assert not output.success
assert output.error
def test_stream_without_tokens_is_not_reported_as_success():
"""A stream that finishes without generating anything has no TTFT."""
output = run([PREAMBLE, usage_event(5, 0)])
assert not output.success
assert "TTFT" in output.error
def test_truncated_stream_is_not_reported_as_success():
"""A stream cut short after a token has no usage and an understated E2EL."""
output = run([PREAMBLE, text_delta("partial")])
assert not output.success
assert "terminal event" in output.error
def test_mid_stream_error_is_not_reported_as_success():
"""An error payload mid-stream must not be counted as a success.
vLLM cannot emit this today: its streaming error path fails to validate
against the event union, so a generation failure truncates the stream
instead. Proxies and other Responses servers do send it.
"""
output = run(
[
PREAMBLE,
text_delta("partial"),
b'event: error\ndata: {"error": {"message": "boom", "code": 500}}\n\n',
]
)
assert not output.success
assert "boom" in output.error
def test_data_only_framing_is_still_parsed():
"""Some proxies drop the ``event:`` line and forward only ``data:``."""
script: list[bytes | float] = [
b'data: {"type": "response.output_text.delta", "delta": "ok"}\n\n',
usage_event(5, 1),
]
output = run(script)
assert output.success
assert output.generated_text == "ok"
def test_endpoint_path_is_validated():
request_input = RequestFuncInput(
prompt="hi",
api_url="http://localhost:8000/v1/chat/completions",
prompt_len=1,
output_len=1,
model=MODEL,
)
async def call() -> None:
async with aiohttp.ClientSession() as session:
await async_request_openai_responses(request_input, session)
with pytest.raises(ValueError, match="responses"):
asyncio.run(call())
@@ -3,10 +3,20 @@
import json
from collections.abc import AsyncGenerator
from unittest.mock import MagicMock
import httpx
import pytest
from tests.entrypoints.openai.chat_completion.test_serving_chat import (
BASE_MODEL_PATHS,
MockHFConfig,
MockModelConfig,
_build_renderer,
)
from tests.entrypoints.openai.chat_completion.test_serving_chat import (
MODEL_NAME as MOCK_MODEL_NAME,
)
from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata
from vllm.entrypoints.openai.chat_completion.batch_serving import (
@@ -15,7 +25,10 @@ from vllm.entrypoints.openai.chat_completion.batch_serving import (
from vllm.entrypoints.openai.chat_completion.protocol import (
BatchChatCompletionRequest,
)
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.v1.engine.async_llm import AsyncLLM
# any model with a chat template defined in tokenizer_config should work here
MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
@@ -322,3 +335,53 @@ async def test_batched_echo_prepends_matching_assistant_prefix() -> None:
)
assert response.choices[0].message.content == "PREFIX ASSISTANT ANSWER"
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
async def test_batched_harmony_response_format_uses_structural_tag() -> None:
"""Harmony must constrain only the final channel of every conversation."""
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.model_config = MockModelConfig()
mock_engine.errored = False
mock_engine.input_processor = MagicMock()
mock_engine.model_config.hf_config = MockHFConfig(model_type="gpt_oss")
mock_engine.renderer = _build_renderer(mock_engine.model_config)
mock_engine.generate.side_effect = lambda *args, **kwargs: _generator(0, "ok")
models = OpenAIServingModels(mock_engine, BASE_MODEL_PATHS)
online_renderer = OnlineRenderer(
model_config=mock_engine.model_config,
renderer=mock_engine.renderer,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
reasoning_parser="openai_gptoss",
)
serving = OpenAIServingChatBatch(
mock_engine,
models,
response_role="assistant",
online_renderer=online_renderer,
chat_template=None,
chat_template_content_format="auto",
request_logger=None,
)
request = BatchChatCompletionRequest(
model=MOCK_MODEL_NAME,
messages=[
[{"role": "user", "content": "France?"}],
[{"role": "user", "content": "Japan?"}],
],
response_format={
"type": "json_schema",
"json_schema": {"name": "city", "schema": {"type": "object"}},
},
)
await serving.create_batch_chat_completion(request)
calls = mock_engine.generate.call_args_list
assert len(calls) == 2
for call in calls:
assert call.args[1].structured_outputs.structural_tag is not None
@@ -20,6 +20,7 @@ Each test:
"""
from openai.types.responses import ResponseFunctionToolCall
from openai_harmony import Message, Role
from tests.entrypoints.openai.utils import verify_harmony_messages
from vllm.entrypoints.openai.parser.harmony_utils import (
@@ -33,6 +34,23 @@ from vllm.entrypoints.openai.responses.harmony import (
response_previous_input_to_harmony,
)
def test_render_uses_legacy_tool_call_header():
tool_call = (
Message.from_role_and_content(Role.ASSISTANT, "print('hello')")
.with_channel("commentary")
.with_recipient("python")
.with_content_type("code")
)
rendered = get_encoding().decode(render_for_completion([tool_call]))
assert (
"<|start|>assistant to=python<|channel|>commentary code<|message|>" in rendered
)
assert "commentary to=python <|constrain|>code" not in rendered
# Use a fixed date so the system message is deterministic across both paths.
_DATE = "2025-01-01"
@@ -84,6 +84,18 @@ class _ReasoningOnlyParser(DelegatingParser):
return None
class _CountingParser(_NoOpParser):
"""Parser that records the ids it counts; token id 7 is reasoning."""
def __init__(self, tokenizer, *args, **kwargs):
super().__init__(tokenizer)
self.counted_ids: list[list[int]] = []
def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
self.counted_ids.append(list(token_ids))
return sum(token_id == 7 for token_id in token_ids)
class _StubToolParser:
"""Minimal tool parser stub that always returns a hardcoded tool call."""
@@ -339,6 +351,25 @@ def test_multi_turn_accumulation():
assert texts == ["First turn", "Second turn"]
def test_reasoning_tokens_counted_per_round():
"""Each round is counted on its own ids, not the concatenated rounds."""
ctx = _make_context(_CountingParser)
ctx.append_output(_make_request_output(token_ids=[7, 7, 1]))
ctx.append_output(_make_request_output(token_ids=[7, 2]))
assert ctx.response_parser.counted_ids == [[7, 7, 1], [7, 2]]
assert ctx.num_reasoning_tokens == 3
def test_reasoning_tokens_zero_without_parser():
ctx = _make_context(None)
ctx.append_output(_make_request_output(token_ids=[7, 7, 1]))
assert ctx.num_reasoning_tokens == 0
def test_num_init_messages_offset():
"""Initial messages are preserved and offset works correctly."""
init_messages = [MagicMock(type="message")]
@@ -960,24 +960,19 @@ class TestValidateGeneratorInput:
assert isinstance(result, ErrorResponse)
@pytest.mark.asyncio
async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
"""Ensure reasoning_tokens usage is derived from thinking token spans."""
class _Qwen3FakeTokenizer:
def __init__(self):
self._vocab = {"<think>": 1, "</think>": 2, "reason": 3, "final": 4}
class FakeTokenizer:
def __init__(self):
self._vocab = {"<think>": 1, "</think>": 2, "reason": 3, "final": 4}
def get_vocab(self):
return self._vocab
def get_vocab(self):
return self._vocab
def decode(self, token_ids):
id_to_token = {v: k for k, v in self._vocab.items()}
return "".join(id_to_token.get(token_id, "x") for token_id in token_ids)
def decode(self, token_ids):
id_to_token = {v: k for k, v in self._vocab.items()}
return "".join(id_to_token.get(token_id, "x") for token_id in token_ids)
# Force non-harmony, SimpleContext path
monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
def _make_qwen3_serving() -> tuple[OpenAIServingResponses, _Qwen3FakeTokenizer]:
engine_client = MagicMock()
model_config = MagicMock()
model_config.hf_config.model_type = "test"
@@ -987,21 +982,66 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
engine_client.input_processor = MagicMock()
engine_client.renderer = MagicMock()
tokenizer = FakeTokenizer()
tokenizer = _Qwen3FakeTokenizer()
engine_client.renderer.get_tokenizer.return_value = tokenizer
models = MagicMock()
serving = OpenAIServingResponses(
engine_client=engine_client,
models=models,
models=MagicMock(),
online_renderer=MagicMock(),
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
reasoning_parser="qwen3",
)
return serving, tokenizer
def _make_text_request_output(text: str, token_ids: list[int]) -> RequestOutput:
return RequestOutput(
request_id="req",
prompt="hi",
prompt_token_ids=[7, 8],
prompt_logprobs=None,
outputs=[
CompletionOutput(
index=0,
text=text,
token_ids=token_ids,
cumulative_logprob=0.0,
logprobs=None,
finish_reason="stop",
stop_reason=None,
)
],
finished=True,
num_cached_tokens=0,
num_cache_creation_tokens=3,
)
async def _run_full_generator(serving, request, context, tokenizer):
async def dummy_result_generator():
yield None
return await serving.responses_full_generator(
request=request,
sampling_params=SamplingParams(max_tokens=16),
result_generator=dummy_result_generator(),
context=context,
model_name="test-model",
tokenizer=tokenizer,
request_metadata=RequestResponseMetadata(request_id="req"),
)
@pytest.mark.asyncio
async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
"""Ensure reasoning_tokens usage is derived from thinking token spans."""
# Force non-harmony, SimpleContext path
monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
serving, tokenizer = _make_qwen3_serving()
request = ResponsesRequest(input="hi", tools=[], stream=False)
response_parser = serving._make_response_parser(
request,
@@ -1011,43 +1051,12 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
# Build a SimpleContext with thinking tokens in the output.
context = SimpleContext(response_parser=response_parser)
token_ids = [1, 10, 2, 20] # <think> 10 </think> 20 -> reasoning token count = 1
completion = CompletionOutput(
index=0,
text="<think>reason</think>final",
token_ids=token_ids,
cumulative_logprob=0.0,
logprobs=None,
finish_reason="stop",
stop_reason=None,
# <think> 10 </think> 20 -> reasoning token count = 1
context.append_output(
_make_text_request_output("<think>reason</think>final", [1, 10, 2, 20])
)
req_output = RequestOutput(
request_id="req",
prompt="hi",
prompt_token_ids=[7, 8],
prompt_logprobs=None,
outputs=[completion],
finished=True,
num_cached_tokens=0,
num_cache_creation_tokens=3,
)
context.append_output(req_output)
async def dummy_result_generator():
yield None
sampling_params = SamplingParams(max_tokens=16)
metadata = RequestResponseMetadata(request_id="req")
response = await serving.responses_full_generator(
request=request,
sampling_params=sampling_params,
result_generator=dummy_result_generator(),
context=context,
model_name="test-model",
tokenizer=tokenizer,
request_metadata=metadata,
)
response = await _run_full_generator(serving, request, context, tokenizer)
assert response.usage.output_tokens_details.reasoning_tokens == 1
assert response.usage.input_tokens_details.cache_write_tokens == 3
+34
View File
@@ -0,0 +1,34 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Run the gpt-oss GPQA evaluator with an explicit output-token budget."""
import argparse
from gpt_oss.evals.gpqa_eval import GPQAEval
from gpt_oss.evals.responses_sampler import ResponsesSampler
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--model", required=True)
parser.add_argument("--reasoning-effort", required=True)
parser.add_argument("--base-url", required=True)
parser.add_argument("--n-threads", required=True, type=int)
parser.add_argument("--max-output-tokens", required=True, type=int)
args = parser.parse_args()
sampler = ResponsesSampler(
model=args.model,
reasoning_model=True,
reasoning_effort=args.reasoning_effort,
base_url=args.base_url,
max_tokens=args.max_output_tokens,
)
result = GPQAEval(n_repeats=8, n_threads=args.n_threads)(sampler)
# gpt-oss returns a NumPy scalar. Convert it so the parent process gets a
# stable, dependency-independent representation that its regex can parse.
print({"metric": float(result.score)})
if __name__ == "__main__":
main()
+9 -4
View File
@@ -21,6 +21,12 @@ from tests.utils import RemoteOpenAIServer
TOL = 0.05 # Absolute tolerance for accuracy comparison
# gpt-oss >= 0.0.7 explicitly sends its 131072-token default as the output
# budget, which leaves no room for the prompt in GPT-OSS's 131072-token context.
# This test originally ran with a 32768-token model context, so retain that
# proven generation budget while using the current evaluator package.
MAX_OUTPUT_TOKENS = 32_768
# Path to tiktoken encoding files
TIKTOKEN_DATA_DIR = Path(__file__).parent / "data"
@@ -50,10 +56,7 @@ def run_gpqa_eval(model_name: str, base_url: str, reasoning_effort: str) -> floa
# Build the command to run the evaluation
cmd = [
sys.executable,
"-m",
"gpt_oss.evals",
"--eval",
"gpqa",
str(Path(__file__).with_name("run_gpqa_eval.py")),
"--model",
model_name,
"--reasoning-effort",
@@ -62,6 +65,8 @@ def run_gpqa_eval(model_name: str, base_url: str, reasoning_effort: str) -> floa
base_url,
"--n-threads",
"200",
"--max-output-tokens",
str(MAX_OUTPUT_TOKENS),
]
try:
@@ -225,3 +225,46 @@ def test_apply_rotary_emb_xpu_matches_native(
expected = op.forward_native(x, cos, sin)
actual = op.forward_xpu(x, cos, sin)
torch.testing.assert_close(actual, expected, atol=0, rtol=0)
@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="CUDA/ROCm only test.")
@pytest.mark.parametrize("num_tokens", [257, 2304])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
def test_packed_qk_rope_correctness(
num_tokens: int, dtype: torch.dtype, default_vllm_config
):
"""packed_qk_rope_ must be bitwise identical to the per-tensor
ApplyRotaryEmb path with enable_fp32_compute=True, and must leave the V
slice untouched."""
from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb
from vllm.model_executor.layers.rotary_embedding.packed_qk_rope import (
packed_qk_rope_,
)
num_heads, head_dim = 12, 128
rng = torch.Generator(device="cuda").manual_seed(0)
angles = (
torch.rand(num_tokens, head_dim // 2, device="cuda", generator=rng) * torch.pi
)
freqs_cis = torch.polar(torch.ones_like(angles), angles)
cos = freqs_cis.real.contiguous()
sin = freqs_cis.imag.contiguous()
xqkv = torch.randn(
num_tokens, 3, num_heads, head_dim, dtype=dtype, device="cuda", generator=rng
)
xqkv_ref = xqkv.clone()
xq, xk, xv = torch.unbind(xqkv_ref, dim=-3)
op = ApplyRotaryEmb(
enforce_enable=True, is_neox_style=False, enable_fp32_compute=True
)
# op() dispatches to forward_cuda / forward_hip per platform.
xq_ref = op(xq, cos, sin)
xk_ref = op(xk, cos, sin)
packed_qk_rope_(xqkv, freqs_cis)
torch.testing.assert_close(xqkv[:, 0], xq_ref, atol=0, rtol=0)
torch.testing.assert_close(xqkv[:, 1], xk_ref, atol=0, rtol=0)
torch.testing.assert_close(xqkv[:, 2], xv, atol=0, rtol=0)
+97
View File
@@ -42,6 +42,9 @@ from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
convert_to_fp8_moe_kernel_format,
make_fp8_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import (
fused_topk_bias,
)
from vllm.model_executor.layers.fused_moe.router.fused_topk_router import fused_topk
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
rotate_weights_for_fi_trtllm_fp8_per_tensor_moe,
@@ -887,3 +890,97 @@ def test_trtllm_fp8_block_moe_deferred_finalize(
),
router_weights=topk_weights,
)
@pytest.mark.parametrize("m", [1, 16])
@pytest.mark.skipif(
not current_platform.is_device_capability_family(100),
reason="Requires TRTLLM-Gen FP8 MoE (SM100)",
)
def test_trtllm_mxfp8_minimax2_routing_applies_routed_scale(m: int, workspace_init):
"""FlashInfer's fused MiniMax2 routing must apply routed_scaling_factor,
matching vLLM's own sigmoid+bias top-k routing (MiniMax-M3 uses 2.0)."""
e, topk, n, k, scale = 32, 4, 1024, 1024, 2.0
block_shape = [1, 32]
set_random_seed(7)
with set_current_vllm_config(vllm_config):
w1, w2, w1_scale, w2_scale = convert_to_fp8_moe_kernel_format(
Fp8MoeBackend.FLASHINFER_TRTLLM,
SimpleNamespace(
weight_block_size=block_shape,
moe_config=SimpleNamespace(
is_act_and_mul=True, intermediate_size_per_partition=n
),
activation=MoEActivation.SILU,
),
*_make_mxfp8_moe_weights(e, n, k),
w13_input_scale=None,
w2_input_scale=None,
)
quant_config = make_fp8_moe_quant_config(
Fp8MoeBackend.FLASHINFER_TRTLLM,
w1_scale,
w2_scale,
a1_scale=None,
a2_scale=None,
block_shape=block_shape,
)
moe_config = FusedMoEConfig(
num_experts=e,
experts_per_token=topk,
hidden_dim=k,
intermediate_size=n,
num_local_experts=e,
num_logical_experts=e,
activation=MoEActivation.SILU,
device="cuda",
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
in_dtype=torch.bfloat16,
routing_method=RoutingMethodType.MiniMax2,
max_num_tokens=next_power_of_2(m),
)
def make_kernel(experts_cls) -> mk.FusedMoEKernel:
return mk.FusedMoEKernel(
maybe_make_prepare_finalize(
moe=moe_config,
quant_config=quant_config,
allow_new_interface=True,
use_monolithic=experts_cls is TrtLlmFp8ExpertsMonolithic,
),
experts_cls(moe_config=moe_config, quant_config=quant_config),
)
a = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) / 10
logits = torch.randn((m, e), device="cuda", dtype=torch.float32)
bias = torch.randn(e, device="cuda", dtype=torch.float32) / 10
common = dict(
w1=w1,
w2=w2,
activation=MoEActivation.SILU,
global_num_experts=e,
expert_map=None,
apply_router_weight_on_input=False,
)
fused = make_kernel(TrtLlmFp8ExpertsMonolithic).apply_monolithic(
hidden_states=a,
router_logits=logits,
e_score_correction_bias=bias,
routed_scaling_factor=scale,
**common,
)
topk_weights, topk_ids = fused_topk_bias(
a,
logits,
scoring_func="sigmoid",
e_score_correction_bias=bias,
topk=topk,
renormalize=True,
routed_scaling_factor=scale,
)
reference = make_kernel(TrtLlmFp8ExpertsModular).apply(
hidden_states=a, topk_weights=topk_weights, topk_ids=topk_ids, **common
)
torch.testing.assert_close(fused, reference, atol=2e-2, rtol=2e-2)
+3 -12
View File
@@ -47,7 +47,6 @@ if ROCM_AVAILABLE:
ROCM_GFX950 = on_gfx950()
if ROCM_AITER_AVAILABLE:
from aiter.ops.triton.moe.quant_moe import upcast_from_mxfp
from aiter.ops.triton.quant import dynamic_mxfp4_quant
if TRTLLM_GEN_MXFP4_AVAILABLE:
@@ -191,11 +190,7 @@ def mxfp8_dequantize(x, scale):
def mxfp4_quant_dequant(x: torch.Tensor) -> torch.Tensor:
shape = x.shape
quantized, scale = dynamic_mxfp4_quant(x.to(torch.bfloat16).flatten(0, -2))
return (
upcast_from_mxfp(quantized.view(torch.uint8), scale, torch.bfloat16, axis=-1)
.reshape(shape)
.float()
)
return mxfp4_dequantize(quantized.view(torch.uint8), scale).reshape(shape)
def reference_moe(
@@ -1809,12 +1804,8 @@ def test_rocm_mxfp4_moe_oracle(
assert out.abs().max() > 0.01, "Output is effectively zero"
# Dequantize weights for reference computation
w13_dq = upcast_from_mxfp(
w13_quant_ref.view(torch.uint8), w13_scale_ref, torch.bfloat16, axis=-1
)
w2_dq = upcast_from_mxfp(
w2_quant_ref.view(torch.uint8), w2_scale_ref, torch.bfloat16, axis=-1
)
w13_dq = mxfp4_dequantize(w13_quant_ref.view(torch.uint8), w13_scale_ref)
w2_dq = mxfp4_dequantize(w2_quant_ref.view(torch.uint8), w2_scale_ref)
# Determine activation type and layout
# SWIGLUOAI uses interleaved layout (gate/up alternating)
+44 -52
View File
@@ -566,72 +566,64 @@ def test_aiter_moe_shared_experts_enablement_follows_env(
assert rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() is expected
def test_aiter_moe_situv2_syncs_aiter_a4w4_env(
monkeypatch: pytest.MonkeyPatch,
):
"""SiTUv2 MoE should route to AITER a4w4 and clear legacy a8w4 overrides."""
from vllm._aiter_ops import rocm_aiter_ops
_assert_aiter_supported()
with monkeypatch.context() as mp:
mp.delenv("AITER_SITUV2_A8W4", raising=False)
mp.delenv("AITER_SITUV2_A4W4", raising=False)
mp.setenv("VLLM_ROCM_USE_AITER", "1")
mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "1")
_reload_envs()
rocm_aiter_ops.refresh_env_variables()
import os
assert os.environ.get("AITER_SITUV2_A4W4") == "1"
assert "AITER_SITUV2_A8W4" not in os.environ
def test_aiter_moe_situv2_legacy_a8w4_alias_enables_a4w4(
monkeypatch: pytest.MonkeyPatch,
):
"""Deprecated VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4 still enables SiTUv2 a4w4."""
from vllm._aiter_ops import rocm_aiter_ops
_assert_aiter_supported()
with monkeypatch.context() as mp:
mp.delenv("AITER_SITUV2_A8W4", raising=False)
mp.delenv("AITER_SITUV2_A4W4", raising=False)
mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", raising=False)
mp.setenv("VLLM_ROCM_USE_AITER", "1")
mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", "1")
_reload_envs()
rocm_aiter_ops.refresh_env_variables()
import os
assert os.environ.get("AITER_SITUV2_A4W4") == "1"
assert "AITER_SITUV2_A8W4" not in os.environ
def test_aiter_moe_situv2_clears_aiter_env_when_disabled(
@pytest.mark.parametrize(
"value,expected_act,expected_env",
[
# Unset / auto / legacy "1" all mean the a4w4 default.
(None, "a4w4", {"AITER_SITUV2_A4W4": "1"}),
("auto", "a4w4", {"AITER_SITUV2_A4W4": "1"}),
("1", "a4w4", {"AITER_SITUV2_A4W4": "1"}),
("a4w4", "a4w4", {"AITER_SITUV2_A4W4": "1"}),
("0", "a16w4", {}),
("a16w4", "a16w4", {}),
("A8W4", "a8w4", {"AITER_SITUV2_A8W4": "1"}),
],
)
def test_aiter_moe_situv2_activation_syncs_aiter_env(
value: str | None,
expected_act: str,
expected_env: dict[str, str],
monkeypatch: pytest.MonkeyPatch,
):
"""Each SiTUv2 activation choice sets exactly its AITER dispatch env."""
import os
from vllm._aiter_ops import rocm_aiter_ops
_assert_aiter_supported()
with monkeypatch.context() as mp:
# Stale AITER overrides must be replaced by the vLLM selection.
mp.setenv("AITER_SITUV2_A8W4", "1")
mp.setenv("AITER_SITUV2_A4W4", "1")
mp.setenv("VLLM_ROCM_USE_AITER", "1")
mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", raising=False)
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "0")
if value is None:
mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", raising=False)
else:
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", value)
_reload_envs()
rocm_aiter_ops.refresh_env_variables()
import os
assert rocm_aiter_ops.get_fused_moe_situv2_activation() == expected_act
assert rocm_aiter_ops.is_fused_moe_situv2_gate_up_interleaved() is (
expected_act == "a8w4"
)
assert rocm_aiter_ops.is_fused_moe_situv2_enabled() is (expected_act != "a16w4")
for name in ("AITER_SITUV2_A8W4", "AITER_SITUV2_A4W4"):
assert os.environ.get(name) == expected_env.get(name)
assert "AITER_SITUV2_A4W4" not in os.environ
def test_aiter_moe_situv2_rejects_unknown_activation(
monkeypatch: pytest.MonkeyPatch,
):
with monkeypatch.context() as mp:
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "a8w8")
_reload_envs()
import vllm.envs as envs
with pytest.raises(ValueError, match="VLLM_ROCM_USE_AITER_MOE_SITUV2"):
_ = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2
@pytest.mark.parametrize("moe_padding", [True, False])
+32
View File
@@ -167,6 +167,38 @@ def test_single_expert_group_with_non_unit_scale_uses_grouped_topk() -> None:
assert isinstance(router, GroupedTopKRouter)
def test_sigmoid_bias_routing_with_routed_scale_is_minimax2() -> None:
"""FlashInfer's MiniMax2 routing applies routed_scaling_factor, so a
non-unit scale (MiniMax-M3 uses 2.0) must not block fused routing."""
router = create_fused_moe_router(
top_k=4,
global_num_experts=128,
scoring_func="sigmoid",
renormalize=True,
routed_scaling_factor=2.0,
e_score_correction_bias=torch.empty(128),
)
assert isinstance(router, FusedTopKBiasRouter)
assert router.routing_method_type == RoutingMethodType.MiniMax2
def test_zero_expert_routing_is_unspecified() -> None:
"""Zero experts are resolved in the router, so kernels with built-in
routing must never be selected for them."""
router = create_fused_moe_router(
top_k=4,
global_num_experts=128,
scoring_func="sigmoid",
renormalize=True,
e_score_correction_bias=torch.empty(160),
zero_expert_type="identity",
num_logical_experts=160,
)
assert router.routing_method_type == RoutingMethodType.Unspecified
def setup_eplb_state(
enable_eplb: bool, global_num_experts: int
) -> EplbLayerState | None:
+266
View File
@@ -2075,3 +2075,269 @@ def test_fused_kv_insert_split(num_tokens: int, kv_block_size: int):
# RoPE (last 64): stored as bf16. The kernel recomputes the rotation, so it
# is bf16-close to the reference rather than bit-exact (cf. test_cutedsl).
torch.testing.assert_close(recovered[:, NOPE_DIM:], ref[:, NOPE_DIM:])
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm stream coverage")
def test_v41_rocm_csa2_full_pipeline():
"""_forward_csa2_full: one fork, then K write and q-side after the join.
The default chain must run first on the current stream, the compressor
chain on aux 0 and the indexer weights projection on aux 1; the join must
publish both cache writes before _produce_k / forward_q run.
"""
from vllm.forward_context import ForwardContext, override_forward_context
from vllm.models.deepseek_v41.amd.rocm import DeepseekV41ROCMAiterMLAAttention
from vllm.models.deepseek_v41.compressor import DeepseekCompressor
torch.manual_seed(43)
num_tokens = 19
hidden_states = torch.randn(num_tokens, 1024, device="cuda")
positions = torch.arange(7, 26, device="cuda")
qr_kv = torch.zeros(num_tokens, 1536, device="cuda")
qr = torch.zeros(num_tokens, 4, device="cuda")
kv = torch.zeros(num_tokens, 512, device="cuda")
q = torch.zeros(num_tokens, 512, device="cuda")
latent = torch.ones(num_tokens, 512, dtype=torch.bfloat16, device="cuda")
weights = torch.randn(num_tokens, 32, device="cuda")
index_q = torch.full((num_tokens, 32, 128), 2.0, device="cuda")
index_q_scale = torch.full((num_tokens, 32), 3.0, device="cuda")
index_weights_out = torch.full((num_tokens, 32), 4.0, device="cuda")
main_cache = torch.full((1, 128, 584), 165, dtype=torch.uint8, device="cuda")
index_cache = torch.full((1, 128, 132), 165, dtype=torch.uint8, device="cuda")
order: list[str] = []
streams: dict[str, torch.cuda.Stream] = {}
compressor = DeepseekCompressor.__new__(DeepseekCompressor)
torch.nn.Module.__init__(compressor)
compressor.fused_wkv_wgate = SimpleNamespace(
weight=torch.randn(1024, 1024, device="cuda")
)
def compressor_forward(kv_score, positions_):
order.append("compressor")
streams["compressor"] = torch.cuda.current_stream()
return latent
def compressor_insert(latent_, positions_, rotary_emb):
order.append("insert_cache")
streams["insert_cache"] = torch.cuda.current_stream()
main_cache.fill_(5)
compressor.forward = compressor_forward
compressor.insert_cache = compressor_insert
def weights_proj(hidden_states_):
order.append("weights_proj")
streams["weights_proj"] = torch.cuda.current_stream()
return weights, None
def produce_k(latent_, positions_, rotary_emb):
order.append("produce_k")
streams["produce_k"] = torch.cuda.current_stream()
index_cache.fill_(7)
def forward_q(qr_, qr_scale, indexer_weights, positions_, rotary_emb):
order.append("forward_q")
streams["forward_q"] = torch.cuda.current_stream()
return index_q, index_q_scale, index_weights_out
indexer = SimpleNamespace(
weights_proj=weights_proj,
_produce_k=produce_k,
forward_q=forward_q,
)
rotary = SimpleNamespace(cos_sin_cache=torch.randn(32, 64, device="cuda"))
aux_streams = [torch.cuda.Stream(), torch.cuda.Stream()]
def wq_b(qr_, qr_scale):
order.append("wq_b")
streams["wq_b"] = torch.cuda.current_stream()
return q
def qnorm_insert(q_, kv_, positions_, attn_metadata):
order.append("qnorm_insert")
streams["qnorm_insert"] = torch.cuda.current_stream()
return q_
attention = SimpleNamespace(
compressor=compressor,
indexer=indexer,
aux_stream_list=aux_streams,
ln_events=[torch.cuda.Event() for _ in range(4)],
rotary_emb=rotary,
indexer_rotary_emb=rotary,
n_local_heads=1,
head_dim=512,
_fused_wqa_wkv_gemm=lambda hidden_states_: qr_kv,
_split_qkv_and_norm=lambda qr_kv_: (qr, None, kv),
_wq_b_proj=wq_b,
_fused_qnorm_rope_kv_insert=qnorm_insert,
)
context = ForwardContext({}, {}, {})
with override_forward_context(context):
result = DeepseekV41ROCMAiterMLAAttention._forward_csa2_full(
attention, hidden_states, positions
)
q_out, kv_out, index_q_out, index_q_scale_out, index_weights_out_out = result
assert torch.equal(q_out, q)
assert torch.equal(kv_out, kv)
assert torch.equal(index_q_out, index_q)
assert torch.equal(index_q_scale_out, index_q_scale)
assert torch.equal(index_weights_out_out, index_weights_out)
# Join publishes the aux-0 cache insert and orders the post-join K write.
assert torch.equal(main_cache, torch.full_like(main_cache, 5))
assert torch.equal(index_cache, torch.full_like(index_cache, 7))
assert order == [
"wq_b",
"qnorm_insert",
"compressor",
"insert_cache",
"weights_proj",
"produce_k",
"forward_q",
]
current = torch.cuda.current_stream()
for name in ("wq_b", "qnorm_insert", "produce_k", "forward_q"):
assert streams[name] == current, name
for name in ("compressor", "insert_cache"):
assert streams[name] == aux_streams[0], name
assert streams["weights_proj"] == aux_streams[1]
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm stream coverage")
def test_v41_rocm_csa2_reindex_pipeline():
"""_forward_csa2_reindex: serial projections, then SWA q path vs q-side."""
from vllm.forward_context import ForwardContext, override_forward_context
from vllm.models.deepseek_v41.amd.rocm import DeepseekV41ROCMAiterMLAAttention
torch.manual_seed(45)
num_tokens = 19
hidden_states = torch.randn(num_tokens, 1024, device="cuda")
positions = torch.arange(7, 26, device="cuda")
qr_kv = torch.zeros(num_tokens, 1536, device="cuda")
qr = torch.zeros(num_tokens, 4, device="cuda")
kv = torch.zeros(num_tokens, 512, device="cuda")
q = torch.zeros(num_tokens, 512, device="cuda")
indexer_weights = torch.randn(num_tokens, 32, device="cuda")
index_q = torch.full((num_tokens, 32, 128), 2.0, device="cuda")
index_q_scale = torch.full((num_tokens, 32), 3.0, device="cuda")
index_weights_out = torch.full((num_tokens, 32), 4.0, device="cuda")
order: list[str] = []
streams: dict[str, torch.cuda.Stream] = {}
def forward_q(qr_, qr_scale, indexer_weights_, positions_, rotary_emb):
order.append("forward_q")
streams["forward_q"] = torch.cuda.current_stream()
return index_q, index_q_scale, index_weights_out
indexer = SimpleNamespace(forward_q=forward_q)
rotary = SimpleNamespace(cos_sin_cache=torch.randn(32, 64, device="cuda"))
aux0 = torch.cuda.Stream()
def wq_b(qr_, qr_scale):
order.append("wq_b")
streams["wq_b"] = torch.cuda.current_stream()
return q
def qnorm_insert(q_, kv_, positions_, attn_metadata):
order.append("qnorm_insert")
streams["qnorm_insert"] = torch.cuda.current_stream()
return q_
attention = SimpleNamespace(
indexer=indexer,
aux_stream_list=[aux0],
ln_events=[torch.cuda.Event() for _ in range(2)],
indexer_rotary_emb=rotary,
n_local_heads=1,
head_dim=512,
_run_parallel_input_projections=lambda hidden_states_: (
qr_kv,
None,
indexer_weights,
),
_split_qkv_and_norm=lambda qr_kv_: (qr, None, kv),
_wq_b_proj=wq_b,
_fused_qnorm_rope_kv_insert=qnorm_insert,
)
context = ForwardContext({}, {}, {})
with override_forward_context(context):
result = DeepseekV41ROCMAiterMLAAttention._forward_csa2_reindex(
attention, hidden_states, positions
)
q_out, kv_out, index_q_out, index_q_scale_out, index_weights_out_out = result
assert torch.equal(q_out, q)
assert torch.equal(kv_out, kv)
assert torch.equal(index_q_out, index_q)
assert torch.equal(index_q_scale_out, index_q_scale)
assert torch.equal(index_weights_out_out, index_weights_out)
assert order == ["wq_b", "qnorm_insert", "forward_q"]
current = torch.cuda.current_stream()
for name in ("wq_b", "qnorm_insert"):
assert streams[name] == current, name
assert streams["forward_q"] == aux0
@pytest.mark.parametrize("quant_tuple", [False, True])
def test_v41_indexer_forward_q(quant_tuple):
"""forward_q runs wq_b then the fused RoPE/quant and unpacks both shapes."""
from unittest.mock import patch
from vllm.models.deepseek_v41.attention import DeepseekV4Indexer
num_tokens, n_head, head_dim = 5, 32, 128
qr = torch.randn(num_tokens, 1280)
q = torch.randn(num_tokens, n_head * head_dim)
positions = torch.arange(num_tokens)
indexer_weights = torch.randn(num_tokens, n_head)
rotary = SimpleNamespace(cos_sin_cache=torch.randn(32, 64))
fake = SimpleNamespace(
n_head=n_head,
head_dim=head_dim,
softmax_scale=head_dim**-0.5,
use_fp4_kv=False,
indexer_weights_dtype=torch.float32,
_wq_b_proj=lambda qr_, qr_scale: q,
)
q_quant = torch.randn(num_tokens, n_head, head_dim)
q_scale = torch.randn(num_tokens, n_head)
weights_out = torch.randn(num_tokens, n_head)
if quant_tuple:
fused_return = ((q_quant, q_scale), weights_out)
expected = (q_quant, q_scale, weights_out)
else:
fused_return = (q_quant, weights_out)
expected = (q_quant, None, weights_out)
with patch(
"vllm.models.deepseek_v41.attention.fused_indexer_q_rope_quant",
return_value=fused_return,
) as mock_fused:
result = DeepseekV4Indexer.forward_q(
fake, qr, None, indexer_weights, positions, rotary
)
assert torch.equal(result[0], expected[0])
assert (result[1] is None) == (expected[1] is None)
if expected[1] is not None:
assert torch.equal(result[1], expected[1])
assert torch.equal(result[2], expected[2])
args = mock_fused.call_args.args
assert args[0] is positions
torch.testing.assert_close(args[1], q.view(-1, n_head, head_dim))
assert args[2] is rotary.cos_sin_cache
assert args[3] is indexer_weights
assert args[4] == head_dim**-0.5
assert args[5] == n_head**-0.5
assert mock_fused.call_args.kwargs == {
"use_fp4": False,
"weights_out_dtype": torch.float32,
}
@@ -21,7 +21,12 @@ Usage:
import pytest
import torch
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
deepgemm_post_process_fp8_weight_block,
)
from vllm.models.deepseek_v4.common.ops import fused_inv_rope_fp8_quant
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import fp8_einsum
# -- Default dimensions matching DeepSeek V3/V4 --------------------------
HEAD_DIM = 512
@@ -33,6 +38,49 @@ FP8_DTYPE = torch.float8_e4m3fn
EPS = 1e-10
@pytest.mark.skipif(
not current_platform.is_device_capability_family(100), reason="Requires SM100/SM103"
)
@pytest.mark.parametrize("groups", [2, 4])
@pytest.mark.parametrize("tokens", [1, 6, 18, 32])
def test_dsv41_fused_wo_a_matches_deepgemm(groups, tokens):
"""Fused WO-A matches inverse RoPE + fp8_einsum + MXFP8 quantize."""
from flashinfer import mxfp8_quantize
from vllm.models.deepseek_v41.nvidia.ops.fused_wo_a import _FUSED_WO_A_KERNEL
torch.manual_seed(0)
n = groups * 1024
x = torch.randn(tokens, 64, 512, device="cuda", dtype=torch.bfloat16)[:, : n // 128]
positions = torch.randint(0, 128, (tokens,), device="cuda")
rope = make_cos_sin_cache(128, device="cuda")
wq = torch.randn(n, 4096, device="cuda").to(FP8_DTYPE)
scale = torch.exp2(torch.randint(-8, 0, (n, 128), device="cuda").float())
wq, ws = deepgemm_post_process_fp8_weight_block(
wq, scale, (1, 32), False, True, groups
)
q, sf = _FUSED_WO_A_KERNEL(
x=x, positions=positions, rope=rope, weight=wq, weight_scale=ws
)
a, sa = fused_inv_rope_fp8_quant(
x, positions, rope, groups, 8, quant_group_size=32, tma_aligned_scales=True
)
y = torch.empty(tokens, groups, 1024, device="cuda", dtype=torch.bfloat16)
fp8_einsum("bhr,hdr->bhd", (a, sa), (wq, ws), y, recipe=(1, 1, 32))
ref, ref_sf = mxfp8_quantize(y.flatten(1), backend="cute-dsl")
def rows(scales): # F8_128x4 bytes -> [128 rows, n / 32 blocks]
return scales.view(n // 128, 32, 4, 4).permute(2, 1, 0, 3).reshape(128, -1)
actual, expected = (
d.float().view(tokens, -1, 32) * torch.exp2(rows(s)[:tokens, :, None] - 127.0)
for d, s in ((q, sf), (ref, ref_sf))
)
assert (actual - expected).norm() / expected.norm() < 3e-3
assert not rows(sf)[tokens:].any(), "padded scale rows must be zero"
# =========================================================================
# Helpers
# =========================================================================
+1
View File
@@ -139,6 +139,7 @@ def test_olmoe_lora_mixed(olmoe_lora_files):
max_loras=4,
enforce_eager=True,
trust_remote_code=True,
hf_overrides={"head_dtype": "float32"},
enable_chunked_prefill=True,
)
@@ -12,6 +12,8 @@ import subprocess
import sys
import tempfile
import threading
import urllib.error
import urllib.request
from dataclasses import dataclass, field
from typing import Any
@@ -20,6 +22,7 @@ import pytest
from vllm import SamplingParams
from vllm.assets.image import ImageAsset
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_port
class WeightCacheDaemon:
@@ -33,6 +36,7 @@ class WeightCacheDaemon:
):
# Short base path: Unix socket paths are limited to ~107 characters.
self.socket_dir = tempfile.mkdtemp(prefix="vllm_ipc_")
self.health_port = get_open_port()
self._cmd = [
sys.executable,
"-m",
@@ -45,6 +49,10 @@ class WeightCacheDaemon:
"--weight-cache-socket-dir",
self.socket_dir,
"--enforce-eager",
"--weight-cache-health-host",
"127.0.0.1",
"--weight-cache-health-port",
str(self.health_port),
*(extra_args or []),
]
self._proc: subprocess.Popen | None = None
@@ -63,6 +71,16 @@ class WeightCacheDaemon:
assert self._proc is not None and self._proc.stderr is not None
self._proc.stderr.read()
def check_health(self) -> None:
url = f"http://127.0.0.1:{self.health_port}/health"
try:
with urllib.request.urlopen(url, timeout=5) as response:
assert response.status == 200
except urllib.error.HTTPError as error:
raise AssertionError(
f"Weight cache daemon is not ready: HTTP {error.code}"
) from error
def _stop(self) -> None:
assert self._proc is not None
self._proc.terminate()
@@ -200,6 +218,7 @@ def test_ipc_cache_cold_start_and_warm_restart(vllm_runner, case: ModelCase):
extra_args=case.daemon_args,
) as d:
warm_outputs = generate(vllm_runner, case, d.socket_dir, fallback=False)
d.check_health()
# Warm restart: a second engine lifetime against the same daemon.
restart_outputs = generate(vllm_runner, case, d.socket_dir, fallback=False)
@@ -49,6 +49,7 @@ PRUNED_METADATA_FIELDS = {
"prefill_state_indices",
"prefill_has_initial_state",
"spec_sequence_masks",
"spec_sequence_masks_cpu",
"uniform_spec_sequence_length",
"flashinfer_prefill_query_start_loc",
"flashinfer_prefill_seq_order",
@@ -175,6 +175,24 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
},
marks=[pytest.mark.core_model],
),
"minimax_m3": VitCudagraphTestConfig(
model="MiniMaxAI/MiniMax-M3",
modalities=["image"],
image_prompt=("]~b]user\n]<]image[>[What is in this image?[e~[\n]~b]ai\n"),
compilation_config_overrides={
"encoder_cudagraph_token_budgets": [512],
},
vllm_runner_kwargs={
"trust_remote_code": True,
"load_format": "dummy",
"hf_overrides": partial(
dummy_hf_overrides,
model_arch="MiniMaxM3SparseForConditionalGeneration",
),
"mm_processor_kwargs": {"max_long_side_pixel": 336},
},
marks=[pytest.mark.core_model],
),
"qwen3_vl": VitCudagraphTestConfig(
model="Qwen/Qwen3-VL-2B-Instruct",
image_prompt=qwen_vl_chat_template(
+3 -18
View File
@@ -5,13 +5,7 @@
import pytest
import torch
from tests.utils import ensure_current_vllm_config
from vllm.distributed.parallel_state import (
init_distributed_environment,
initialize_model_parallel,
)
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_port
EMBED_DIM = 256
NUM_HEADS = 4
@@ -21,20 +15,11 @@ WINDOW = 8
SEQ_LENS = [5, 37]
@pytest.fixture(scope="module")
def vision_attn_env():
init_distributed_environment(
world_size=1,
rank=0,
local_rank=0,
distributed_init_method=f"tcp://127.0.0.1:{get_open_port()}",
backend="nccl",
)
@pytest.fixture
def vision_attn_env(dist_init):
default_dtype = torch.get_default_dtype()
torch.set_default_dtype(torch.bfloat16)
with ensure_current_vllm_config():
initialize_model_parallel(tensor_model_parallel_size=1)
yield
yield
torch.set_default_dtype(default_dtype)
+305
View File
@@ -0,0 +1,305 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from typing import Any
from unittest.mock import MagicMock, patch
import pytest
from xgrammar import Grammar
from xgrammar.testing import _is_grammar_accept_string
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionToolsParam,
)
from vllm.parser.abstract_parser import DelegatingParser
from vllm.sampling_params import StructuredOutputsParams
from vllm.tool_parsers.abstract_tool_parser import ToolParser
from vllm.tool_parsers.qwen3_engine_tool_parser import Qwen3EngineToolParser
from vllm.tool_parsers.structural_tag_registry import ToolChoice
class TestToolChoice_Plus_ResponseFormat:
"""Note(arpera):
Test cases for tool_choice={auto,required} + response_format
To keep it short:
DelegatingParser.adjust_request behavior in some corner cases is checked there
Initial bug report:
https://github.com/vllm-project/vllm/issues/39929
And PR that fixed this:
https://github.com/vllm-project/vllm/pull/56086
"""
# ================================
# Helper methods
# ================================
@staticmethod
def _tools(strict: bool) -> list[ChatCompletionToolsParam]:
"""Single get_weather tool, optionally marked as strict"""
function: dict[str, Any] = {
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
},
}
if strict:
function["strict"] = True
return [ChatCompletionToolsParam(type="function", function=function)]
@staticmethod
def _qwen_tool_call() -> str:
"""Tool call for Qwen model that do support structural tag"""
return (
"<tool_call>\n<function=get_weather>\n<parameter=city>\nParis\n"
"</parameter>\n</function>\n</tool_call>"
)
@staticmethod
def _json_schema_response_format() -> dict:
return {
"type": "json_schema",
"json_schema": {
"name": "answer",
"schema": {
"type": "object",
"properties": {"text": {"type": "string"}},
"required": ["text"],
},
},
}
@staticmethod
def _setup_request(
tools: list[ChatCompletionToolsParam],
tool_choice: ToolChoice,
response_format: dict,
) -> ChatCompletionRequest:
request = ChatCompletionRequest(
messages=[], # for our test cases it's always empty
model="m", # Just a placeholder, don't pay much attention
tools=tools,
tool_choice=tool_choice,
response_format=response_format,
)
return request
@staticmethod
def _setup_abstract_parser(
tools: list[ChatCompletionToolsParam],
) -> DelegatingParser:
"""Construct parser that does NOT support structural tag"""
class TestParser(DelegatingParser):
tool_parser_cls = ToolParser
return TestParser(MagicMock(), tools=tools)
@staticmethod
def _setup_qwen_parser(
tools: list[ChatCompletionToolsParam],
) -> DelegatingParser:
"""Construct parser that supports structural tag"""
class TestParser(DelegatingParser):
tool_parser_cls = Qwen3EngineToolParser
return TestParser(MagicMock(), tools=tools)
# ================================
# Test cases
# tool_choice=auto + response_format
# ================================
@pytest.mark.parametrize(
# In this test we check that for response_format
# resulting grammar accepts @compliant_output and rejects @non_compliant_output
# You can add more examples here if you see some corner cases not covered
("response_format", "compliant_output", "non_compliant_output"),
[
(_json_schema_response_format(), '{"text": "hi"}', '{"foo": 1}'),
({"type": "json_object"}, '{"any": 1}', "[1, 2]"),
],
# We test here two different response_format types:
ids=["json_schema", "json_object"],
)
def test_auto_with_strict_tools(
self,
response_format: dict,
compliant_output: str,
non_compliant_output: str,
):
tools = self._tools(strict=True)
request = self._setup_request(
tools=tools,
tool_choice="auto",
response_format=response_format,
)
parser = self._setup_qwen_parser(tools)
out = parser.adjust_request(request)
# Now check that request does not have response_format anymore
# but instead has structured_outputs set as structural tag
# And this structural tag is OR operation
assert out.tool_choice == "auto"
assert out.response_format is None
assert out.structured_outputs is not None
tag = json.loads(out.structured_outputs.structural_tag)
assert tag["format"]["type"] == "or"
grammar = Grammar.from_structural_tag(out.structured_outputs.structural_tag)
assert _is_grammar_accept_string(grammar, compliant_output)
assert not _is_grammar_accept_string(grammar, non_compliant_output)
# Also tool call must be accepted by grammar
assert _is_grammar_accept_string(grammar, self._qwen_tool_call())
# IMPORTANT(arpera): Regression test
# If we in adjust_request implementation by mistake
# construct structural tag using tool_choice=auto
# then such a structural tag would allow plain text as well
# We need to be sure that plain text is NOT accepted in our case
assert not _is_grammar_accept_string(grammar, "Hello")
def test_auto_without_strict_tools(self):
tools = self._tools(strict=False)
request = self._setup_request(
tools=tools,
tool_choice="auto",
response_format=self._json_schema_response_format(),
)
parser = self._setup_qwen_parser(tools)
# There must be a warning that tool calls are disabled
# Consume that warning
with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn:
out = parser.adjust_request(request)
assert out.response_format is not None
assert out.structured_outputs is None
mock_warn.assert_called_once()
def test_when_model_does_not_have_structural_tag(self):
"""Note(arpera):
When model does NOT have structural tag support
we apply constraint only for response_format
"""
tools = self._tools(strict=True)
request = self._setup_request(
tools=tools,
tool_choice="auto",
response_format=self._json_schema_response_format(),
)
# SIC! use parser whose model does NOT support structural tag
parser = self._setup_abstract_parser(tools)
with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn:
out = parser.adjust_request(request)
assert out.response_format is not None
assert out.structured_outputs is None
mock_warn.assert_called_once()
def test_auto_with_xgrammar_unsupported_schema(self):
"""Conrer case based on Vadim's feedback in PR #56086
https://github.com/vllm-project/vllm/pull/56086#pullrequestreview-5329203459
"""
tools = self._tools(strict=True)
request = self._setup_request(
tools=tools,
tool_choice="auto",
response_format={
"type": "json_schema",
"json_schema": {
"name": "answer",
"schema": {"type": "integer", "multipleOf": 2},
},
},
)
parser = self._setup_qwen_parser(tools)
with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn:
out = parser.adjust_request(request)
assert out.response_format is not None
assert out.structured_outputs is None
mock_warn.assert_called_once()
def test_auto_with_xgrammar_unsupported_nested_schema(self):
"""Conrer case based on Vadim's feedback in PR #56086
https://github.com/vllm-project/vllm/pull/56086#pullrequestreview-5329203459
"""
tools = self._tools(strict=True)
request = self._setup_request(
tools=tools,
tool_choice="auto",
response_format={
"type": "json_schema",
"json_schema": {
"name": "answer",
"schema": {
"type": "object",
"properties": {"value": {"type": "number", "multipleOf": 0.5}},
"required": ["value"],
},
},
},
)
parser = self._setup_qwen_parser(tools)
with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn:
out = parser.adjust_request(request)
assert out.response_format is not None
assert out.structured_outputs is None
mock_warn.assert_called_once()
def test_auto_with_lark_grammar(self):
"""Conrer case based on Vadim's feedback in PR #56086
https://github.com/vllm-project/vllm/pull/56086#pullrequestreview-5329203459
Lark grammars are converted to EBNF before being merged into the tag.
"""
tools = self._tools(strict=True)
request = ChatCompletionRequest(
messages=[],
model="m",
tools=tools,
tool_choice="auto",
structured_outputs=StructuredOutputsParams(grammar='start: "ok"'),
)
parser = self._setup_qwen_parser(tools)
out = parser.adjust_request(request)
assert out.structured_outputs is not None
grammar = Grammar.from_structural_tag(out.structured_outputs.structural_tag)
assert _is_grammar_accept_string(grammar, "ok")
assert not _is_grammar_accept_string(grammar, "bad")
assert _is_grammar_accept_string(grammar, self._qwen_tool_call())
# ================================
# Test cases
# tool_choice=required + response_format
# ================================
def test_required(self):
tools = self._tools(strict=True)
request = self._setup_request(
tools=tools,
tool_choice="required",
response_format=self._json_schema_response_format(),
)
parser = self._setup_qwen_parser(tools)
with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn:
out = parser.adjust_request(request)
assert out.response_format is None
assert out.structured_outputs is not None
grammar = Grammar.from_structural_tag(out.structured_outputs.structural_tag)
assert _is_grammar_accept_string(grammar, self._qwen_tool_call())
assert not _is_grammar_accept_string(grammar, '{"text": "hi"}')
mock_warn.assert_called_once()
+16
View File
@@ -1196,6 +1196,21 @@ class TestAdjustRequest:
"TOOL_CALL_2_FUNCTION_FIRST",
],
),
(
{
"tool_choice": "auto",
"strict_tools": True,
"response_format_type": "json_schema",
},
[
"COMMENTARY",
"TOOL_CALL_1_CHANNEL_FIRST",
"TOOL_CALL_1_FUNCTION_FIRST",
"TOOL_CALL_2_CHANNEL_FIRST",
"TOOL_CALL_2_FUNCTION_FIRST",
"FINAL_JSON_SCHEMA",
],
),
(
{"response_format_type": "json_schema"},
["FINAL_JSON_SCHEMA"],
@@ -1281,6 +1296,7 @@ class TestAdjustRequest:
"tool_auto_strict",
"tool_required",
"tool_named",
"pr56086_tool_auto_strict_response_format_json_schema",
"response_format_json_schema",
"response_format_json_object",
"structured_outputs_json",
+88 -3
View File
@@ -19,7 +19,12 @@ from tests.v1.attention.utils import (
try_backend_includes_kv_cache_update,
try_get_attention_backend,
)
from vllm.config import ModelConfig, set_current_vllm_config
from vllm.config import (
DiffusionConfig,
ModelConfig,
SpeculativeConfig,
set_current_vllm_config,
)
from vllm.platforms import current_platform
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import (
@@ -942,8 +947,6 @@ def test_flashinfer_xqa_single_token_decode_preserves_cudagraph_padding(monkeypa
impl.o_sf_scale = None
impl.window_left = -1
impl.sinks = None
impl.cache_config = unittest.mock.Mock()
impl.cache_config.get_resolved_kv_cache_layout.return_value = KVCacheLayout.LBHNC
layer = unittest.mock.Mock(
_q_scale=torch.tensor(1.0),
@@ -959,6 +962,7 @@ def test_flashinfer_xqa_single_token_decode_preserves_cudagraph_padding(monkeypa
q_len_per_req=1,
)
attn_metadata = flashinfer_backend.FlashInferMetadata(
kv_cache_layout=KVCacheLayout.LBHNC,
num_actual_tokens=2,
slot_mapping=torch.empty(0, dtype=torch.int64),
q_data_type_prefill=torch.bfloat16,
@@ -1070,6 +1074,87 @@ def test_flashinfer_trtllm_gen_padded_decode_uses_varlen_offsets(
assert q_cu_seq_lens.tolist() == qo_indptr_values
@pytest.mark.skipif(
not current_platform.is_cuda()
or AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
reason="FlashInfer with CUDA is required.",
)
@pytest.mark.parametrize(
"runner_type,spec_kind,has_upper_bound,expected_copies",
[
("generate", None, True, 0),
("pooling", None, True, 0),
("generate", None, False, 1),
("generate", "spec_decode", True, 1),
("generate", "diffusion", True, 1),
],
)
def test_flashinfer_avoids_seq_lens_copy_without_spec_tokens(
monkeypatch, runner_type, spec_kind, has_upper_bound, expected_copies
):
"""Planning reuses the host seq_lens bound only when it is exact.
Without speculative token accounting (no spec-decode drafts, no dLLM
canvas tokens) scheduled tokens always land in KV, so the CPU upper bound
is exact regardless of runner type. Any speculative accounting keeps the
bound optimistic (rejected tokens are in-flight) and forces the copy.
"""
import unittest.mock
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
from vllm.v1.attention.backends.utils import PerLayerParameters
monkeypatch.setattr(
flashinfer_backend,
"get_per_layer_parameters",
lambda *args: {
"layer.0": PerLayerParameters(
window_left=-1, logits_soft_cap=0.0, sm_scale=0.125
)
},
)
config = create_vllm_config(
model_name="Qwen/Qwen3-0.6B", max_model_len=64, max_num_seqs=3
)
config.model_config.runner_type = runner_type
config.attention_config.use_trtllm_attention = False
if spec_kind == "spec_decode":
config.speculative_config = SpeculativeConfig(
method="ngram", num_speculative_tokens=3
)
elif spec_kind == "diffusion":
config.diffusion_config = DiffusionConfig(canvas_length=8)
device = torch.device(f"{DEVICE_TYPE}:0")
# Include a one-token final chunk and both partial and full KV pages.
common_attn_metadata = create_common_attn_metadata(
BatchSpec(seq_lens=[17, 20, 32], query_lens=[1, 4, 16]), 16, device
)
if not has_upper_bound:
common_attn_metadata.seq_lens_cpu_upper_bound = None
elif spec_kind is not None:
# Simulate rejected speculative tokens: CPU says 18, GPU says 17.
assert common_attn_metadata.seq_lens_cpu_upper_bound is not None
common_attn_metadata.seq_lens_cpu_upper_bound[0] += 1
with set_current_vllm_config(config):
builder = flashinfer_backend.FlashInferMetadataBuilder(
create_standard_kv_cache_spec(config), ["layer.0"], config, device
)
with unittest.mock.patch.object(
common_attn_metadata.seq_lens,
"cpu",
wraps=common_attn_metadata.seq_lens.cpu,
) as copy_to_cpu:
metadata = builder.build(0, common_attn_metadata)
assert copy_to_cpu.call_count == expected_copies
assert metadata.num_decodes == 1
assert metadata.num_prefills == 2
assert builder.paged_kv_indptr.cpu[:4].tolist() == [0, 2, 4, 6]
assert builder.paged_kv_last_page_len.cpu[:3].tolist() == [1, 4, 16]
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
reason="FlashInfer is not available.",
+67 -1
View File
@@ -25,6 +25,9 @@ from vllm.v1.attention.backends.mla.prefill.cpu_sdpa import (
from vllm.v1.attention.backends.mla.prefill.selector import (
get_mla_prefill_backend,
)
from vllm.v1.attention.backends.mla.prefill.zen_cpu_sdpa import (
ZenCPUSDPAMLAPrefillBackend,
)
def _reference_ragged_attention(
@@ -157,8 +160,10 @@ def test_cpu_mla_backend_smoke(tmp_path) -> None:
def test_cpu_mla_prefill_backend_selected() -> None:
# On CPU (no device capability) the MLA prefill backend must be the
# SDPA-based CPU backend, not flash-attn which is unavailable on CPU.
# Accelerator-specific subclasses (e.g. the zentorch-backed Zen CPU
# backend) are also valid selections.
backend_cls = get_mla_prefill_backend(None)
assert backend_cls is CPUSDPAMLAPrefillBackend
assert issubclass(backend_cls, CPUSDPAMLAPrefillBackend)
@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only")
@@ -248,3 +253,64 @@ def test_cpu_mla_prefill_context_chunk() -> None:
assert lse.float().allclose(ref_lse.float(), atol=2e-2)
assert out.shape == (5, 4, 16)
assert lse.shape == (4, 5)
@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only")
@pytest.mark.skipif(
not ZenCPUSDPAMLAPrefillBackend.is_available(),
reason="requires a Zen CPU with the zentorch_sdpa op registered",
)
def test_zen_cpu_mla_prefill_new_tokens() -> None:
# Runs the zentorch kernel itself. The head dims are deliberately
# asymmetric (v_head_dim < qk_head_dim, as in real MLA) so that the V
# padding and output slicing are exercised; the generic-CPU cases above
# use equal dims and never reach that code.
qk_head_dim = 24
v_head_dim = 16
backend = ZenCPUSDPAMLAPrefillBackend(
num_heads=4,
scale=0.25,
kv_lora_rank=16,
qk_nope_head_dim=16,
qk_rope_head_dim=8,
v_head_dim=v_head_dim,
vllm_config=None,
)
class _PrefillMeta:
query_start_loc = torch.tensor([0, 3, 6], dtype=torch.int32)
backend.prepare_metadata(_PrefillMeta())
q = torch.randn(6, 4, qk_head_dim, dtype=torch.bfloat16)
k = torch.randn(6, 4, qk_head_dim, dtype=torch.bfloat16)
v = torch.randn(6, 4, v_head_dim, dtype=torch.bfloat16)
out = backend.run_prefill_new_tokens(q, k, v, return_softmax_lse=False)
ref, _ = _reference_ragged_attention(
q,
k,
v,
_PrefillMeta.query_start_loc,
_PrefillMeta.query_start_loc,
scale=0.25,
causal=True,
)
assert out.shape == (6, 4, v_head_dim)
assert out.float().allclose(ref.float(), atol=2e-2)
# zentorch_sdpa does not return a usable LSE, so that path must fall back
# to the fp32 reference and match the generic backend exactly.
generic = CPUSDPAMLAPrefillBackend(
num_heads=4,
scale=0.25,
kv_lora_rank=16,
qk_nope_head_dim=16,
qk_rope_head_dim=8,
v_head_dim=v_head_dim,
vllm_config=None,
)
generic.prepare_metadata(_PrefillMeta())
zen_out, zen_lse = backend.run_prefill_new_tokens(q, k, v, return_softmax_lse=True)
gen_out, gen_lse = generic.run_prefill_new_tokens(q, k, v, return_softmax_lse=True)
assert torch.equal(zen_out, gen_out)
assert torch.equal(zen_lse, gen_lse)
@@ -9,6 +9,7 @@ causality), ckv/kpe cache splitting, and the backend's model-shape gates.
"""
from types import SimpleNamespace
from typing import Any
import pytest
import torch
@@ -50,6 +51,8 @@ class FakeWrapper:
def __init__(self):
self.plan_args = None
self.run_args = None
self._plan_info: Any = None
self._planned_backend: Any = None
def plan(self, *args, **kwargs):
self.plan_args = (args, kwargs)
@@ -69,8 +72,11 @@ class FakeState:
self.kv_indptr = torch.zeros(max_tokens + 1, dtype=torch.int32)
self.wrapper = FakeWrapper()
self.plan_calls = []
self.index_topk = 2048
self.index_kpool = 4
self.max_valid = 2051
def plan(self, num_tokens, kv_lens):
def plan(self, num_tokens, kv_lens, cam, req_id_per_token):
self.plan_calls.append((num_tokens, kv_lens))
def pack_indices(self, slots):
@@ -168,13 +174,12 @@ def test_forward_wiring(monkeypatch, qk_rope, kv_dtype):
def test_builder_plans_only_rows_dispatched_to_mqa(monkeypatch, use_mha, num_decodes):
"""MHA prefill rows must not make the MQA kernel read beyond its query."""
builder = object.__new__(FlashInferMLASparseSM90Builder)
builder._index_topk = 2048
builder._index_kpool = 4
builder._async_scheduling = False
builder._adaptive_verification = False
builder.state = FakeState(TOPK)
builder._attention_layer = SimpleNamespace(_use_sparse_mha=lambda _: use_mha)
metadata = object.__new__(sm90_mod.FlashInferMLASparseSM90Metadata)
metadata.state = None
metadata.req_id_per_token = None
metadata.num_prefills = 1
metadata.num_decode_tokens = num_decodes
monkeypatch.setattr(
@@ -206,12 +211,17 @@ def test_builder_plans_only_rows_dispatched_to_mqa(monkeypatch, use_mha, num_dec
def test_plan_uses_state_params(monkeypatch):
"""The NoPE/rope dims and scale live on the builder state, not the layer.
plan() takes exact per-row KV lengths; the schedule is rebuilt on every
call (contexts grow between steps) and the indptrs are always full-size
Without a device batch layout plan() takes exact per-row KV lengths and
rebuilds the schedule on every call; the indptrs are always full-size
with zero-query padding rows past num_tokens.
"""
impl, rows = make_impl(64, "auto")
wrapper = FakeWrapper()
plan_info = [0] * sm90_mod._PLAN_INFO_LEN
plan_info[sm90_mod._PI_KV_INDPTR] = 64 * 4
wrapper._plan_info = torch.tensor(plan_info)
wrapper._planned_backend = SimpleNamespace(
_int_workspace_buffer=torch.zeros(64, dtype=torch.int32)
)
state = sm90_mod._SM90State.__new__(sm90_mod._SM90State)
state.device = torch.device("cpu")
state.wrapper = wrapper
@@ -227,8 +237,9 @@ def test_plan_uses_state_params(monkeypatch):
state._qo_cpu = torch.empty(5, dtype=torch.int32)
state._kv_cpu = torch.empty(5, dtype=torch.int32)
state._lens_cpu = torch.full((4,), TOPK, dtype=torch.int32)
state._saved_kv_end = torch.empty(0, dtype=torch.int32)
state.plan(3, torch.tensor([2, 5, 7], dtype=torch.int32))
state.plan(3, torch.tensor([2, 5, 7], dtype=torch.int32), None, None)
assert wrapper.plan_args is not None
args, kwargs = wrapper.plan_args
(qo, kv, indices, kv_len, heads, ckv, kpe, page, causal, scale) = args
@@ -241,10 +252,10 @@ def test_plan_uses_state_params(monkeypatch):
assert kwargs["kv_data_type"] == torch.bfloat16
# Replanning a smaller batch must clear the previous rows' lengths.
state.plan(1, torch.tensor([TOPK], dtype=torch.int32))
state.plan(1, torch.tensor([TOPK], dtype=torch.int32), None, None)
assert state._kv_cpu.tolist() == [0, TOPK, TOPK, TOPK, TOPK]
assert state._lens_cpu.tolist() == [TOPK, 0, 0, 0]
state.plan(0, torch.empty(0, dtype=torch.int32))
state.plan(0, torch.empty(0, dtype=torch.int32), None, None)
assert state._kv_cpu.tolist() == [0, 0, 0, 0, 0]
assert state._lens_cpu.tolist() == [0, 0, 0, 0]
@@ -279,38 +290,49 @@ def test_pack_indices_replays_with_updated_offsets(width):
assert (state.kv_indices[expected.numel() :] == -99).all()
def test_kv_lens_host_formula():
@pytest.mark.parametrize("adaptive", [False, True])
def test_kv_lens_host_formula(adaptive):
"""Per-row host lengths: context == position + 1; capped at
index_topk + trailing-pool remainder past the sparse threshold."""
index_topk + trailing-pool remainder past the sparse threshold.
Exact (synced) without a host upper bound; with one, a sync-free bound
capped at the largest valid count, which the plan clamps on device.
"""
builder = object.__new__(FlashInferMLASparseSM90Builder)
builder._index_topk = 2048
builder._index_kpool = 4
builder._async_scheduling = False
builder.state = FakeState(TOPK)
builder._adaptive_verification = adaptive
seq_lens = torch.tensor([100, 9, 3000], dtype=torch.int32)
cam = SimpleNamespace(
num_reqs=3,
query_start_loc_cpu=torch.tensor([0, 5, 7, 10], dtype=torch.int32),
seq_lens=torch.tensor([100, 9, 3000], dtype=torch.int32),
seq_lens_cpu_upper_bound=torch.tensor([100, 9, 3000], dtype=torch.int32),
positions=None,
seq_lens=seq_lens,
seq_lens_cpu_upper_bound=None,
)
num_rows, lens = builder._kv_lens_host(cam)
assert num_rows == 10
num_rows, lens, exact = builder._kv_lens_host(cam)
assert num_rows == 10 and exact
# req0: positions 95..99 -> ctx 96..100 (all <= 2048: full context)
# req1: positions 7,8 -> ctx 8,9
# req2: positions 2997..2999 -> ctx 2998..3000 (> 2048: topk + ctx%4)
assert lens.tolist() == [96, 97, 98, 99, 100, 8, 9, 2050, 2051, 2048]
cam.seq_lens_cpu_upper_bound = seq_lens
num_rows, lens, exact = builder._kv_lens_host(cam)
assert num_rows == 10 and not exact
if adaptive:
# Host query split may differ from the device one: batch-wide bound.
assert lens.tolist() == [2051] * 10
else:
assert lens.tolist() == [96, 97, 98, 99, 100, 8, 9, 2051, 2051, 2051]
def test_kv_lens_host_empty():
builder = object.__new__(FlashInferMLASparseSM90Builder)
builder._index_topk = 2048
builder._index_kpool = 4
cam = SimpleNamespace(
num_reqs=0,
query_start_loc_cpu=torch.tensor([0], dtype=torch.int32),
seq_lens=torch.zeros(0, dtype=torch.int32),
)
num_rows, lens = builder._kv_lens_host(cam)
num_rows, lens, _ = builder._kv_lens_host(cam)
assert num_rows == 0 and lens.numel() == 0
@@ -334,7 +356,13 @@ def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens)
"__init__",
lambda *_args, **_kwargs: None,
)
monkeypatch.setattr(sm90_mod, "_SM90State", lambda *_args, **_kwargs: None)
class _RecordingState:
def __init__(self, *_args, **kwargs):
self.index_topk = kwargs["index_topk"]
self.index_kpool = kwargs["index_kpool"]
monkeypatch.setattr(sm90_mod, "_SM90State", _RecordingState)
impl, _ = make_impl(64)
spec = MLAAttentionSpec(
block_size=BLOCK_SIZE, num_kv_heads=1, head_size=576, dtype=torch.bfloat16
@@ -351,6 +379,7 @@ def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens)
max_num_batched_tokens=64, async_scheduling=False
),
model_config=SimpleNamespace(hf_text_config=hf_config),
speculative_config=None,
)
builder = FlashInferMLASparseSM90Builder(
spec, ["attn"], vllm_config, torch.device("cpu")
@@ -359,14 +388,138 @@ def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens)
cam = SimpleNamespace(
num_reqs=2,
query_start_loc_cpu=torch.tensor([0, 4, 6], dtype=torch.int32),
seq_lens_cpu_upper_bound=torch.tensor([42295, 100], dtype=torch.int32),
positions=None,
seq_lens=torch.tensor([42295, 100], dtype=torch.int32),
seq_lens_cpu_upper_bound=None,
)
num_rows, lens = builder._kv_lens_host(cam)
assert num_rows == 6
num_rows, lens, exact = builder._kv_lens_host(cam)
assert num_rows == 6 and exact
assert lens.tolist() == prefill_lens + [99, 100]
def _sm90_flashinfer_available():
return (
torch.cuda.is_available()
and torch.cuda.get_device_capability()[0] == 9
and sm90_mod.has_flashinfer_sm90_nope_mla()
)
SM90_HEADS, SM90_TOPK, SM90_KPOOL, SM90_WIDTH = 16, 2048, 4, 2176
def _real_state(rows):
return sm90_mod._SM90State(
torch.device("cuda"),
SM90_HEADS,
torch.bfloat16,
rows,
SM90_WIDTH,
HEAD,
0,
HEAD**-0.5,
SM90_TOPK,
SM90_KPOOL,
)
def _clamp_batch(num_reqs, q_per_req, lo, hi, topk, kpool, width):
"""Random batch: per-row contexts, device layout, and top-k slots."""
last = torch.randint(lo + q_per_req, hi, (num_reqs,))
ctx = (last[:, None] + torch.arange(1 - q_per_req, 1)).reshape(-1)
valid = torch.where(ctx <= topk, ctx, topk + ctx % kpool)
cam = SimpleNamespace(
seq_lens=last.to(torch.int32).cuda(),
query_start_loc=(torch.arange(num_reqs + 1) * q_per_req).to(torch.int32).cuda(),
)
req_id = torch.arange(num_reqs, dtype=torch.int32).repeat_interleave(q_per_req)
slots = torch.randint(0, 1 << 16, (ctx.numel(), width), dtype=torch.int32)
slots[torch.arange(width) >= valid[:, None]] = -1
return ctx, valid.to(torch.int32), cam, req_id.cuda(), slots.cuda()
@pytest.mark.skipif(
not _sm90_flashinfer_available(), reason="Requires SM90 FlashInfer MLA"
)
@pytest.mark.parametrize(
"num_reqs,q_per_req,lo,hi",
[
(64, 1, 1, 3000),
(16, 6, 2030, 2070), # MTP-style rows straddling index_topk
(4096, 1, 1, 3000), # clamp spans several program blocks
],
)
def test_upper_bound_plan_matches_exact(num_reqs, q_per_req, lo, hi):
"""Sync-free planning from optimistic host bounds (async spec decode)
must match exact planning: the device clamp trims every work item back
to its row's valid count, so no -1 slot is read."""
torch.manual_seed(0)
heads, topk, kpool, width = SM90_HEADS, SM90_TOPK, SM90_KPOOL, SM90_WIDTH
ctx, valid, cam, req_id, slots = _clamp_batch(
num_reqs, q_per_req, lo, hi, topk, kpool, width
)
rows = ctx.numel()
kv = torch.randn(1 << 16, 1, HEAD, device="cuda", dtype=torch.bfloat16)
q = torch.randn(rows, heads, HEAD, device="cuda", dtype=torch.bfloat16)
q_pe = q.new_empty(rows, heads, 0)
def run(state):
state.pack_indices(slots)
return state.wrapper.run(q, q_pe, kv, kv[..., :0])
exact = _real_state(rows)
exact.plan(rows, valid, None, None)
ub = _real_state(rows)
host_bound = torch.clamp(ctx + torch.randint(0, 7, (rows,)), max=ub.max_valid)
ub.plan(rows, host_bound, cam, req_id)
out = run(ub)
assert not out.isnan().any()
torch.testing.assert_close(out, run(exact), atol=1e-2, rtol=1e-2)
@pytest.mark.skipif(
not _sm90_flashinfer_available(), reason="Requires SM90 FlashInfer MLA"
)
def test_upper_bound_plan_reused_across_steps(monkeypatch):
"""Growing contexts (draft/decode steps) reuse the padded plan and only
re-clamp on device, matching an exact replan every step."""
torch.manual_seed(0)
heads, topk, kpool, width = SM90_HEADS, SM90_TOPK, SM90_KPOOL, SM90_WIDTH
rows = 48
ctx, _, _, req_id, _ = _clamp_batch(rows, 1, 1, 3000, topk, kpool, width)
exact, ub = _real_state(rows), _real_state(rows)
replans: list[int] = []
plan_fn = ub._plan
def counting_plan(num_tokens, kv_lens):
replans.append(num_tokens)
plan_fn(num_tokens, kv_lens)
monkeypatch.setattr(ub, "_plan", counting_plan)
kv = torch.randn(1 << 16, 1, HEAD, device="cuda", dtype=torch.bfloat16)
for step in range(5):
step_ctx = ctx + step
valid = torch.where(step_ctx <= topk, step_ctx, topk + step_ctx % kpool)
slots = torch.randint(0, 1 << 16, (rows, width), dtype=torch.int32)
slots[torch.arange(width) >= valid[:, None]] = -1
slots = slots.cuda()
cam = SimpleNamespace(
seq_lens=step_ctx.to(torch.int32).cuda(),
query_start_loc=torch.arange(rows + 1, dtype=torch.int32).cuda(),
)
num_plans = len(replans)
ub.plan(rows, torch.clamp(step_ctx + 5, max=ub.max_valid), cam, req_id)
assert len(replans) - num_plans == (1 if step == 0 else 0)
exact.plan(rows, valid.to(torch.int32), None, None)
q = torch.randn(rows, heads, HEAD, device="cuda", dtype=torch.bfloat16)
outs = []
for state in (ub, exact):
state.pack_indices(slots)
outs.append(state.wrapper.run(q, q[..., :0], kv, kv[..., :0]))
torch.testing.assert_close(outs[0], outs[1], atol=1e-2, rtol=1e-2)
def test_supports_combination_gates(monkeypatch, default_vllm_config):
monkeypatch.setattr(sm90_mod, "has_flashinfer_sm90_nope_mla", lambda: True)
call = lambda **kw: FlashInferMLASparseSM90Backend.supports_combination(
@@ -21,6 +21,7 @@ from vllm.v1.attention.backends.gdn_attn import (
GDNAttentionMetadata,
GDNAttentionMetadataBuilder,
)
from vllm.v1.attention.backends.utils import mamba_get_block_table_tensor
from vllm.v1.kv_cache_interface import MambaSpec
BLOCK_SIZE = 16
@@ -75,6 +76,17 @@ GDN_BUILD_TEST_CASES = {
expected_num_prefill_tokens=0,
expected_num_spec_decodes=0,
),
# No speculative config, decode alongside prefill
"regular_decode_with_prefill": GDNBuildTestCase(
seq_lens=[40, 100],
query_lens=[1, 50],
num_decode_draft_tokens=None,
num_speculative_tokens=0,
expected_num_decodes=1,
expected_num_prefills=1,
expected_num_prefill_tokens=50,
expected_num_spec_decodes=0,
),
# Multi-token prefill alongside spec decode — no decode to reclassify
"spec_decode_with_real_prefill": GDNBuildTestCase(
seq_lens=[100, 20],
@@ -155,9 +167,12 @@ def _build(
builder: GDNAttentionMetadataBuilder,
batch_spec: BatchSpec,
num_decode_draft_tokens: list[int] | None = None,
block_table: torch.Tensor | None = None,
) -> GDNAttentionMetadata:
"""Build GDN attention metadata, optionally with spec-decode kwargs."""
common = create_common_attn_metadata(batch_spec, BLOCK_SIZE, DEVICE)
if block_table is not None:
common = common.replace(block_table_tensor=block_table)
kwargs: dict = {}
if num_decode_draft_tokens is not None:
kwargs["num_decode_draft_tokens_cpu"] = torch.tensor(
@@ -184,6 +199,54 @@ def test_gdn_build_classification(test_case: GDNBuildTestCase):
assert meta.num_spec_decodes == test_case.expected_num_spec_decodes
@pytest.mark.parametrize("mamba_cache_mode", ["none", "align"])
@pytest.mark.parametrize("full_cuda_graph", [False, True])
@pytest.mark.parametrize(
"test_case", GDN_BUILD_TEST_CASES.values(), ids=GDN_BUILD_TEST_CASES.keys()
)
def test_update_block_table_matches_build(
test_case: GDNBuildTestCase, full_cuda_graph: bool, mamba_cache_mode: str
):
"""update_block_table() on another group's metadata matches build()."""
batch = BatchSpec(seq_lens=test_case.seq_lens, query_lens=test_case.query_lens)
src, dst, ref = (
_create_gdn_builder(test_case.num_speculative_tokens, full_cuda_graph)
for _ in range(3)
)
for builder in (src, dst, ref):
builder.vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
common = create_common_attn_metadata(batch, BLOCK_SIZE, DEVICE)
if mamba_cache_mode == "align":
# MRV2 precomputes these for every Mamba group each step.
dst.mamba_aligned_state_indices = ref.mamba_aligned_state_indices = (
mamba_get_block_table_tensor(
common.block_table_tensor, common.seq_lens, ref.kv_cache_spec, "align"
)
)
draft_tokens = test_case.num_decode_draft_tokens
expected = _build(ref, batch, draft_tokens, common.block_table_tensor)
source = _build(src, batch, draft_tokens)
fields = (
"spec_state_indices_tensor",
"non_spec_state_indices_tensor",
"prefill_state_indices",
)
source_indices = [getattr(source, f) for f in fields]
source_indices = [t if t is None else t.clone() for t in source_indices]
meta = dst.update_block_table(
source, common.block_table_tensor, common.slot_mapping
)
for field, source_index in zip(fields, source_indices):
actual = getattr(meta, field)
torch.testing.assert_close(actual, getattr(expected, field))
# The source group's indices are untouched.
torch.testing.assert_close(getattr(source, field), source_index)
# FULL graph state indices land in this group's own buffers.
if full_cuda_graph and meta.num_prefills == 0 and actual is not None:
assert actual.data_ptr() == getattr(dst, field).data_ptr()
def test_has_initial_state_after_reclassification():
"""After reclassification, num_prefills > 0 so the prefill kernel path
should compute has_initial_state. For the reclassified request with
@@ -101,6 +101,7 @@ def test_flash_attention_geometry_comes_from_the_group():
num_kv_heads=2,
head_size=64,
dtype=torch.bfloat16,
dcp_sharded=True,
)
with (
@@ -60,10 +60,43 @@ class TestGetMLAPrefillBackend:
def test_cpu_uses_sdpa_prefill(self):
vllm_config = _make_vllm_config()
zen_cls = MLAPrefillBackendEnum.ZEN_CPU.get_class()
# A Zen CPU with zentorch selects the Zen backend, so expect whichever
# one this host offers.
expected = (
zen_cls if zen_cls.is_available() else MLAPrefillBackendEnum.CPU.get_class()
)
with patch("vllm.platforms.current_platform") as mock_platform:
mock_platform.is_cpu.return_value = True
backend = get_mla_prefill_backend(vllm_config)
assert backend is expected
def test_zen_cpu_prefers_zentorch_prefill(self):
vllm_config = _make_vllm_config()
zen_cls = MLAPrefillBackendEnum.ZEN_CPU.get_class()
with (
patch("vllm.platforms.current_platform") as mock_platform,
patch.object(zen_cls, "is_available", return_value=True),
):
mock_platform.is_cpu.return_value = True
backend = get_mla_prefill_backend(vllm_config)
assert backend is zen_cls
def test_zen_cpu_falls_back_when_zentorch_unavailable(self):
vllm_config = _make_vllm_config()
zen_cls = MLAPrefillBackendEnum.ZEN_CPU.get_class()
with (
patch("vllm.platforms.current_platform") as mock_platform,
patch.object(zen_cls, "is_available", return_value=False),
):
mock_platform.is_cpu.return_value = True
backend = get_mla_prefill_backend(vllm_config)
assert backend is MLAPrefillBackendEnum.CPU.get_class()
+143 -12
View File
@@ -1599,19 +1599,142 @@ def test_project_kv_cache_groups_to_worker():
assert set(proj_spec.kv_cache_specs.keys()) == {"layer1", "layer3"}
def test_dcp_world_size_for_kv_cache_spec_shards_full_attention_only():
dcp = 8
full = FullAttentionSpec(
block_size=16, num_kv_heads=1, head_size=1, dtype=torch.float32
@pytest.mark.parametrize("sliding_window", [None, 256])
@pytest.mark.parametrize("disable_hybrid", [False, True])
@pytest.mark.parametrize("pcp_size", [1, 4])
def test_dcp_target_allocates_replicated_draft_independently(
monkeypatch, sliding_window, disable_hybrid, pcp_size
):
"""A draft must retain all positions even when the target shards them."""
from vllm.v1.attention.backends.utils import resolve_kv_cache_layout
monkeypatch.delenv("VLLM_KV_CACHE_LAYOUT", raising=False)
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
config.parallel_config.decode_context_parallel_size = 4
config.parallel_config.prefill_context_parallel_size = pcp_size
config.scheduler_config.disable_hybrid_kv_cache_manager = disable_hybrid
config.cache_config.block_size = 16
config.cache_config.kv_cache_layout = None
draft_args = dict(
block_size=16,
num_kv_heads=8,
head_size=64,
dtype=torch.bfloat16,
dcp_sharded=False,
)
mla = new_mla_spec()
mamba = new_mamba_spec()
uniform_mla = UniformTypeKVCacheSpecs(block_size=16, kv_cache_specs={"layer": mla})
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(full, dcp) == dcp
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(mla, dcp) == dcp
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(uniform_mla, dcp) == dcp
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(mamba, dcp) == 1
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(full, 1) == 1
draft = (
FullAttentionSpec(**draft_args)
if sliding_window is None
else SlidingWindowSpec(**draft_args, sliding_window=sliding_window)
)
specs = {"target": new_mla_spec(), "draft": draft}
layout = resolve_kv_cache_layout(config, [["LBHNC", "BLHNC"]], specs.values())
assert layout == KVCacheLayout.BLHNC
groups = get_kv_cache_groups(config, specs)
assert [group.layer_names for group in groups] == [["target"], ["draft"]]
widths = [g.kv_cache_spec.max_num_blocks_per_req(config, 1024) for g in groups]
assert widths == [16, 64]
cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
config, groups, available_memory=16 * 1024 * 1024
)
scheduler_config = generate_scheduler_kv_cache_config([cache_config])
assert get_max_concurrency_for_kv_cache_config(config, cache_config) > 0
assert get_max_concurrency_for_kv_cache_config(config, cache_config) == (
get_max_concurrency_for_kv_cache_config(config, scheduler_config)
)
manager = KVCacheManager(
scheduler_config,
max_model_len=1024,
hash_block_size=16,
scheduler_block_size=64,
dcp_world_size=4,
pcp_world_size=pcp_size,
enable_caching=True,
)
assert manager.coordinator.group_block_sizes == (64, 16)
request = make_request("replicated-draft", [1] * 65, block_size=16, hash_fn=sha256)
blocks = manager.allocate_slots(request, 65)
assert blocks is not None
assert [len(group) for group in blocks.blocks] == [2, 5]
manager.cache_blocks(request, 64)
cached_request = make_request(
"cached-draft", [1] * 65, block_size=16, hash_fn=sha256
)
cached_blocks, num_cached, _ = manager.get_computed_blocks(cached_request)
assert num_cached == 64
assert [len(group) for group in cached_blocks.blocks] == [1, 4]
@pytest.mark.parametrize("use_mla", [False, True])
def test_full_attention_merge_preserves_replicated_cache_geometry(use_mla):
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
config.parallel_config.decode_context_parallel_size = 4
draft = replace(
new_mla_spec() if use_mla else new_kv_cache_spec(),
block_size=16,
dcp_sharded=False,
)
merged = type(draft).merge([draft, draft])
wrapped = UniformTypeKVCacheSpecs.from_specs({"draft": merged})
assert wrapped is not None and not wrapped.dcp_sharded
assert merged.max_num_blocks_per_req(config, 1024) == 64
assert merged.max_memory_usage_bytes(config) == 64 * draft.page_size_bytes
assert kv_cache_utils.resolve_dcp_kv_block_size(merged, 4) == 16
with pytest.raises(AssertionError):
type(draft).merge([draft, replace(draft, dcp_sharded=True)])
@pytest.mark.parametrize("with_draft", [False, True])
@pytest.mark.parametrize("indexer_alignment", [None, 512])
def test_sparse_mla_preserves_physical_row_addressing(with_draft, indexer_alignment):
from vllm.v1.attention.backends.mla.sparse_utils import flat_kv_row_view
from vllm.v1.worker.utils import allocate_kv_cache
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
config.cache_config.kv_cache_layout = "BLHNC"
config.parallel_config.decode_context_parallel_size = 4
common = dict(block_size=64, num_kv_heads=1, dtype=torch.uint8)
specs = {
"target": MLAAttentionSpec(
**common,
head_size=576,
state_content_bytes=656,
block_stride_alignment=656,
cache_dtype_str="fp8_ds_mla",
is_index_group_leader=True,
),
"indexer": MLAAttentionSpec(
**common,
head_size=132,
cache_role=SparseCacheRole.INDEXER,
block_stride_alignment=indexer_alignment,
),
"draft": SlidingWindowSpec(
block_size=64,
num_kv_heads=64,
head_size=64,
dtype=torch.bfloat16,
sliding_window=2048,
dcp_sharded=False,
),
}
if not with_draft:
del specs["draft"]
groups = get_kv_cache_groups(config, specs)
cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
config, groups, available_memory=8 * 1024 * 1024
)
caches = allocate_kv_cache(
cache_config, torch.device("cpu"), KVCacheLayout.BLHNC, [64] * len(groups)
)
cache = caches["target"].squeeze(1)
if indexer_alignment is not None:
assert caches["indexer"].stride(0) % indexer_alignment == 0
rows, stride_rows = flat_kv_row_view(cache, 64)
cache[1, 0].fill_(7)
torch.testing.assert_close(rows[stride_rows], cache[1, 0])
assert torch.all(rows[stride_rows] == 7)
@pytest.mark.parametrize(
@@ -3815,6 +3938,8 @@ def test_unify_hybrid_kv_cache_specs():
"layer_1": before_spec_1,
"layer_2": before_spec_2,
}
kv_cache_spec["draft_layer_1"] = replace(before_spec_1, dcp_sharded=False)
kv_cache_spec["draft_layer_2"] = replace(before_spec_2, dcp_sharded=False)
kv_cache_utils.unify_hybrid_kv_cache_specs(kv_cache_spec)
expected_spec_1 = new_kv_cache_spec(block_size=64)
expected_spec_2 = new_kv_cache_spec(
@@ -3823,6 +3948,7 @@ def test_unify_hybrid_kv_cache_specs():
assert kv_cache_spec["layer_1"] == expected_spec_1
assert kv_cache_spec["layer_2"] == expected_spec_2
assert kv_cache_spec["layer_2"].page_size_bytes == 64 * 1024
assert kv_cache_spec["draft_layer_2"] == replace(expected_spec_2, dcp_sharded=False)
# 2. has_full_attention and has_chunked_local_attention
before_spec_1 = new_kv_cache_spec()
@@ -3874,6 +4000,11 @@ def test_unify_hybrid_kv_cache_specs():
with pytest.raises(ValueError):
kv_cache_utils.unify_hybrid_kv_cache_specs(kv_cache_spec)
# Replicated Mamba state still requires the hybrid cache manager.
kv_cache_spec = {"attention": new_kv_cache_spec(), "mamba": new_mamba_spec()}
with pytest.raises(ValueError):
kv_cache_utils.unify_hybrid_kv_cache_specs(kv_cache_spec)
def test_unify_kv_cache_spec_page_size_mamba():
"""Regression test for https://github.com/vllm-project/vllm/issues/43626.
+12 -7
View File
@@ -1336,12 +1336,12 @@ def make_kv_cache_config_three_types(
)
def test_prefix_cache_hit_uses_per_group_dcp_geometry():
@pytest.mark.parametrize("draft_sharded", [True, False])
def test_prefix_cache_hit_uses_per_group_dcp_geometry(draft_sharded):
"""Prefix lookup must use each group's DCP size, not the process-wide one.
Target and draft MLA are both sharded (DCP=8); Mamba stays replicated
(DCP=1). Hits then align to the sharded full-attention block, not the
unsharded page size.
Target MLA is sharded; draft MLA can opt out and Mamba stays replicated.
Hits must align to the target block in either case.
"""
block_size = 16
dcp = 8
@@ -1362,6 +1362,7 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry():
KVCacheGroupSpec(
["draft_mla"],
MLAAttentionSpec(
dcp_sharded=draft_sharded,
block_size=block_size,
num_kv_heads=1,
head_size=1,
@@ -1388,10 +1389,10 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry():
)
target_mgr, draft_mgr, mamba_mgr = manager.coordinator.single_type_managers
assert target_mgr.dcp_world_size == dcp
assert draft_mgr.dcp_world_size == dcp
assert draft_mgr.dcp_world_size == (dcp if draft_sharded else 1)
assert mamba_mgr.dcp_world_size == 1
assert target_mgr.block_size == sharded_block
assert draft_mgr.block_size == sharded_block
assert draft_mgr.block_size == (sharded_block if draft_sharded else block_size)
assert mamba_mgr.block_size == block_size
hash_fn = sha256
@@ -1407,7 +1408,11 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry():
req1 = make_request("1", common_token_ids + [100] * 5, block_size, hash_fn)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
assert num_computed_tokens == 2 * sharded_block
assert [len(group) for group in computed_blocks.blocks] == [2, 2, 16]
assert [len(group) for group in computed_blocks.blocks] == [
2,
2 if draft_sharded else 16,
16,
]
manager.free(req0)
manager.free(req1)
@@ -11,6 +11,7 @@ from vllm.config import CompilationConfig, VllmConfig, replace
from vllm.config.kernel import MoEBackend
from vllm.engine.arg_utils import EngineArgs
from vllm.platforms import current_platform
from vllm.v1.spec_decode.draft_model import DraftModelProposer
from ..utils import (
Messages,
@@ -196,6 +197,38 @@ def test_draft_model_engine_args_tensor_parallelism():
assert draft_config.quant_config is None
@pytest.mark.cpu_test
@pytest.mark.parametrize("dcp_size", [1, 2])
def test_draft_model_inherits_target_dcp_config(monkeypatch, dcp_size):
monkeypatch.setattr(current_platform, "device_count", lambda: 16)
target_config = EngineArgs(
model="Qwen/Qwen3-1.7B",
tensor_parallel_size=16,
decode_context_parallel_size=dcp_size,
cp_kv_cache_interleave_size=16,
dcp_comm_backend="ag_rs",
distributed_executor_backend="mp",
speculative_config={
"model": "Qwen/Qwen3-0.6B",
"method": "draft_model",
"num_speculative_tokens": 3,
},
).create_engine_config()
proposer = object.__new__(DraftModelProposer)
proposer.vllm_config = target_config
proposer.speculative_config = target_config.speculative_config
draft_config = proposer._create_draft_vllm_config()
draft_parallel = draft_config.parallel_config
assert draft_parallel.decode_context_parallel_size == dcp_size
assert draft_parallel.cp_kv_cache_interleave_size == 16
assert draft_parallel.dcp_comm_backend == "ag_rs"
assert (
draft_parallel.dcp_q_replicate == target_config.parallel_config.dcp_q_replicate
)
assert draft_parallel is not target_config.parallel_config
assert target_config.parallel_config.decode_context_parallel_size == dcp_size
def _apply_draft_moe_backend(vllm_config: VllmConfig) -> VllmConfig:
"""Replicate SpecDecodeBaseProposer._create_draft_vllm_config logic
so we can test it without instantiating a full proposer."""
@@ -0,0 +1,120 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import pytest
from vllm import SamplingParams
from vllm.config import CompilationConfig
from vllm.platforms import current_platform
from ..utils import (
_skip_if_insufficient_gpus_for_tp,
assert_request_outputs_match,
evaluate_llm_for_gsm8k,
get_test_prompts,
)
def check_mtp_correctness(
monkeypatch: pytest.MonkeyPatch,
sampling_config: SamplingParams,
model_setup: tuple[str, str, int, str | None],
mm_enabled: bool,
expected_accuracy_threshold: float,
vllm_runner,
):
"""Compare the outputs of a original LLM and a speculative LLM
which should be the same when using MTP speculative decoding. Due to some variance
in the engine, it is possible for some outputs to differ, so we expect that at least
6/10 output tokens match exactly, and that the GSM8k accuracy is above a precomputed
reference threshold for each model.
"""
method, model_name, tp_size, draft_model = model_setup
_skip_if_insufficient_gpus_for_tp(tp_size)
# Generate test prompts inside the function instead of using fixture
test_prompts = get_test_prompts(mm_enabled)
with monkeypatch.context() as m:
m.setenv("VLLM_MLA_DISABLE", "1")
attn_backend = "TRITON_ATTN" if current_platform.is_rocm() else "auto"
# Skip multimodal profiling for models that don't need it in this test.
extra_kwargs: dict[str, Any] = {}
if "Qwen3.5" in model_name:
extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "video": 0}
elif "gemma-4" in model_name:
extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "audio": 0}
if draft_model is not None and "gemma-4" in draft_model:
import transformers
from packaging.version import Version
if Version(transformers.__version__) < Version("5.8.0"):
pytest.skip(
"Gemma4 MTP assistant requires transformers>=5.8.0, "
f"got {transformers.__version__}"
)
with vllm_runner(
model_name,
block_size=None,
max_model_len=2048,
tensor_parallel_size=tp_size,
trust_remote_code=True,
attention_backend=attn_backend,
enable_chunked_prefill=None,
compilation_config=CompilationConfig(),
**extra_kwargs,
) as ref_runner:
ref_outputs = ref_runner.llm.chat(test_prompts, sampling_config)
evaluate_llm_for_gsm8k(
ref_runner.llm,
expected_accuracy_threshold=expected_accuracy_threshold,
)
speculative_config: dict[str, Any] = {
"method": method,
"num_speculative_tokens": 1,
"max_model_len": 2048,
}
if draft_model is not None:
speculative_config["model"] = draft_model
speculative_config["num_speculative_tokens"] = 2
with vllm_runner(
model_name,
block_size=None,
trust_remote_code=True,
tensor_parallel_size=tp_size,
speculative_config=speculative_config,
max_model_len=2048,
attention_backend=attn_backend,
enable_chunked_prefill=None,
compilation_config=CompilationConfig(),
**extra_kwargs,
) as spec_runner:
# MTP supports async scheduling by default.
has_async = (
spec_runner.llm.llm_engine.vllm_config.scheduler_config.async_scheduling
)
assert has_async, (
f"Expected async scheduling for {method}: target={model_name}, "
f"draft={draft_model}; got {has_async}"
)
evaluate_llm_for_gsm8k(
spec_runner.llm,
expected_accuracy_threshold=expected_accuracy_threshold,
)
spec_outputs = spec_runner.llm.chat(test_prompts, sampling_config)
# Heuristic: expect at least 80% of the prompts to match exactly
# Upon failure, inspect the outputs to check for inaccuracy.
assert_request_outputs_match(
ref_outputs,
spec_outputs,
required_matches=int(0.8 * len(ref_outputs)) + 1,
context=f"{method} target={model_name}, draft={draft_model}",
)
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,44 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from tests.utils import single_gpu_only
from vllm import SamplingParams
from vllm.platforms import current_platform
from .._correctness import check_mtp_correctness
@pytest.mark.parametrize(
["model_setup", "mm_enabled", "expected_accuracy_threshold"],
[
pytest.param(
("mtp", "ZixiQi/DeepSeek-V3-4layers-MTP-FP8", 1, None),
False,
0.0,
marks=pytest.mark.skipif(
current_platform.is_device_capability_family(100),
reason="DeepSeek MTP: TRTLLM MoE top_k check fails on Blackwell",
),
), # dummy model
],
ids=["deepseek"],
)
@single_gpu_only
def test_mtp_correctness(
monkeypatch: pytest.MonkeyPatch,
sampling_config: SamplingParams,
model_setup: tuple[str, str, int, str | None],
mm_enabled: bool,
expected_accuracy_threshold: float,
vllm_runner,
):
check_mtp_correctness(
monkeypatch,
sampling_config,
model_setup,
mm_enabled,
expected_accuracy_threshold,
vllm_runner,
)
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,39 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from tests.utils import single_gpu_only
from vllm import SamplingParams
from .._correctness import check_mtp_correctness
@pytest.mark.parametrize(
["model_setup", "mm_enabled", "expected_accuracy_threshold"],
[
(
("mtp", "google/gemma-4-E4B-it", 1, "google/gemma-4-E4B-it-assistant"),
False,
0.50,
), # gemma4 MTP with assistant model, ref: ~62%
],
ids=["gemma4-e4b"],
)
@single_gpu_only
def test_mtp_correctness(
monkeypatch: pytest.MonkeyPatch,
sampling_config: SamplingParams,
model_setup: tuple[str, str, int, str | None],
mm_enabled: bool,
expected_accuracy_threshold: float,
vllm_runner,
):
check_mtp_correctness(
monkeypatch,
sampling_config,
model_setup,
mm_enabled,
expected_accuracy_threshold,
vllm_runner,
)
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,36 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from tests.utils import single_gpu_only
from vllm import SamplingParams
from .._correctness import check_mtp_correctness
@pytest.mark.parametrize(
["model_setup", "mm_enabled", "expected_accuracy_threshold"],
[
# Reference accuracy: 65%-70%.
(("mtp", "XiaomiMiMo/MiMo-7B-Base", 1, None), False, 0.5),
],
ids=["mimo"],
)
@single_gpu_only
def test_mtp_correctness(
monkeypatch: pytest.MonkeyPatch,
sampling_config: SamplingParams,
model_setup: tuple[str, str, int, str | None],
mm_enabled: bool,
expected_accuracy_threshold: float,
vllm_runner,
):
check_mtp_correctness(
monkeypatch,
sampling_config,
model_setup,
mm_enabled,
expected_accuracy_threshold,
vllm_runner,
)
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,39 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from tests.utils import single_gpu_only
from vllm import SamplingParams
from .._correctness import check_mtp_correctness
@pytest.mark.parametrize(
["model_setup", "mm_enabled", "expected_accuracy_threshold"],
[
(
("mtp", "Qwen/Qwen3.5-0.8B-Base", 1, None),
False,
0.20,
), # hybrid + MTP, ref: ~34%-35%
],
ids=["qwen3_5-hybrid"],
)
@single_gpu_only
def test_mtp_correctness(
monkeypatch: pytest.MonkeyPatch,
sampling_config: SamplingParams,
model_setup: tuple[str, str, int, str | None],
mm_enabled: bool,
expected_accuracy_threshold: float,
vllm_runner,
):
check_mtp_correctness(
monkeypatch,
sampling_config,
model_setup,
mm_enabled,
expected_accuracy_threshold,
vllm_runner,
)
@@ -0,0 +1,115 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import pytest
from tests.utils import single_gpu_only
from vllm import SamplingParams
from vllm.config import CompilationConfig, CUDAGraphMode
from vllm.platforms import current_platform
from ...utils import get_spec_decode_metric_value
PLACEHOLDER_REGRESSION_MODEL = "Qwen/Qwen3.5-0.8B-Base"
PLACEHOLDER_REGRESSION_PROMPT = "The capital of France is"
PLACEHOLDER_REGRESSION_MAX_TOKENS = 16
PLACEHOLDER_REGRESSION_ALLOWED_TOKEN_ID = 42
PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS = 3
@pytest.mark.parametrize(
"cudagraph_mode",
[CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL_AND_PIECEWISE],
ids=["piecewise", "full-and-piecewise"],
)
@single_gpu_only
def test_mtp_rejected_placeholders_match_greedy_token_ids(
cudagraph_mode: CUDAGraphMode,
vllm_runner,
):
"""Rejected MTP padding must not alter the non-speculative token stream."""
common_kwargs: dict[str, Any] = {
"block_size": None,
"max_model_len": 256,
"max_num_seqs": 1,
"trust_remote_code": True,
"enable_chunked_prefill": None,
"limit_mm_per_prompt": {"image": 0, "video": 0},
"attention_backend": ("TRITON_ATTN" if current_platform.is_rocm() else "auto"),
}
sampling_params = SamplingParams(
temperature=0.0,
max_tokens=PLACEHOLDER_REGRESSION_MAX_TOKENS,
ignore_eos=True,
allowed_token_ids=[PLACEHOLDER_REGRESSION_ALLOWED_TOKEN_ID],
logprobs=1,
)
def generate_token_ids(runner) -> tuple[int, ...]:
outputs = runner.llm.generate([PLACEHOLDER_REGRESSION_PROMPT], sampling_params)
assert outputs and outputs[0].outputs, "Expected one completion"
token_ids = tuple(outputs[0].outputs[0].token_ids)
assert len(token_ids) == PLACEHOLDER_REGRESSION_MAX_TOKENS
return token_ids
def compilation_config() -> CompilationConfig:
return CompilationConfig(
cudagraph_mode=cudagraph_mode,
cudagraph_capture_sizes=[1, PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS + 1],
)
with vllm_runner(
PLACEHOLDER_REGRESSION_MODEL,
compilation_config=compilation_config(),
**common_kwargs,
) as ref_runner:
ref_mode = (
ref_runner.llm.llm_engine.vllm_config.compilation_config.cudagraph_mode
)
assert ref_mode == cudagraph_mode
ref_token_ids = generate_token_ids(ref_runner)
expected_token_ids = (PLACEHOLDER_REGRESSION_ALLOWED_TOKEN_ID,) * (
PLACEHOLDER_REGRESSION_MAX_TOKENS
)
assert ref_token_ids == expected_token_ids
with vllm_runner(
PLACEHOLDER_REGRESSION_MODEL,
speculative_config={
"method": "mtp",
"num_speculative_tokens": PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS,
"max_model_len": 256,
"rejection_sample_method": "synthetic",
"synthetic_acceptance_rates": [0.0]
* PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS,
},
disable_log_stats=False,
compilation_config=compilation_config(),
**common_kwargs,
) as spec_runner:
spec_mode = (
spec_runner.llm.llm_engine.vllm_config.compilation_config.cudagraph_mode
)
assert spec_mode == cudagraph_mode
spec_token_ids = generate_token_ids(spec_runner)
metrics = spec_runner.llm.get_metrics()
num_drafts = get_spec_decode_metric_value(
metrics, "vllm:spec_decode_num_drafts"
)
num_draft_tokens = get_spec_decode_metric_value(
metrics, "vllm:spec_decode_num_draft_tokens"
)
num_accepted_tokens = get_spec_decode_metric_value(
metrics, "vllm:spec_decode_num_accepted_tokens"
)
assert num_drafts > 0, "MTP drafter did not run"
assert num_draft_tokens > 0, "MTP drafter did not produce draft tokens"
assert num_accepted_tokens == 0, "Synthetic sampler accepted a draft token"
assert spec_token_ids == ref_token_ids, (
f"MTP changed greedy token ids under {cudagraph_mode.name}:\n"
f" reference={ref_token_ids}\n"
f" speculative={spec_token_ids}"
)
-252
View File
@@ -1,252 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import pytest
from tests.utils import single_gpu_only
from vllm import SamplingParams
from vllm.config import CompilationConfig, CUDAGraphMode
from vllm.platforms import current_platform
from ..utils import (
_skip_if_insufficient_gpus_for_tp,
assert_request_outputs_match,
evaluate_llm_for_gsm8k,
get_spec_decode_metric_value,
get_test_prompts,
)
PLACEHOLDER_REGRESSION_MODEL = "Qwen/Qwen3.5-0.8B-Base"
PLACEHOLDER_REGRESSION_PROMPT = "The capital of France is"
PLACEHOLDER_REGRESSION_MAX_TOKENS = 16
PLACEHOLDER_REGRESSION_ALLOWED_TOKEN_ID = 42
PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS = 3
@pytest.mark.parametrize(
"cudagraph_mode",
[CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL_AND_PIECEWISE],
ids=["piecewise", "full-and-piecewise"],
)
@single_gpu_only
def test_mtp_rejected_placeholders_match_greedy_token_ids(
cudagraph_mode: CUDAGraphMode,
vllm_runner,
):
"""Rejected MTP padding must not alter the non-speculative token stream."""
common_kwargs: dict[str, Any] = {
"block_size": None,
"max_model_len": 256,
"max_num_seqs": 1,
"trust_remote_code": True,
"enable_chunked_prefill": None,
"limit_mm_per_prompt": {"image": 0, "video": 0},
"attention_backend": ("TRITON_ATTN" if current_platform.is_rocm() else "auto"),
}
sampling_params = SamplingParams(
temperature=0.0,
max_tokens=PLACEHOLDER_REGRESSION_MAX_TOKENS,
ignore_eos=True,
allowed_token_ids=[PLACEHOLDER_REGRESSION_ALLOWED_TOKEN_ID],
logprobs=1,
)
def generate_token_ids(runner) -> tuple[int, ...]:
outputs = runner.llm.generate([PLACEHOLDER_REGRESSION_PROMPT], sampling_params)
assert outputs and outputs[0].outputs, "Expected one completion"
token_ids = tuple(outputs[0].outputs[0].token_ids)
assert len(token_ids) == PLACEHOLDER_REGRESSION_MAX_TOKENS
return token_ids
def compilation_config() -> CompilationConfig:
return CompilationConfig(
cudagraph_mode=cudagraph_mode,
cudagraph_capture_sizes=[1, PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS + 1],
)
with vllm_runner(
PLACEHOLDER_REGRESSION_MODEL,
compilation_config=compilation_config(),
**common_kwargs,
) as ref_runner:
ref_mode = (
ref_runner.llm.llm_engine.vllm_config.compilation_config.cudagraph_mode
)
assert ref_mode == cudagraph_mode
ref_token_ids = generate_token_ids(ref_runner)
expected_token_ids = (PLACEHOLDER_REGRESSION_ALLOWED_TOKEN_ID,) * (
PLACEHOLDER_REGRESSION_MAX_TOKENS
)
assert ref_token_ids == expected_token_ids
with vllm_runner(
PLACEHOLDER_REGRESSION_MODEL,
speculative_config={
"method": "mtp",
"num_speculative_tokens": PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS,
"max_model_len": 256,
"rejection_sample_method": "synthetic",
"synthetic_acceptance_rates": [0.0]
* PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS,
},
disable_log_stats=False,
compilation_config=compilation_config(),
**common_kwargs,
) as spec_runner:
spec_mode = (
spec_runner.llm.llm_engine.vllm_config.compilation_config.cudagraph_mode
)
assert spec_mode == cudagraph_mode
spec_token_ids = generate_token_ids(spec_runner)
metrics = spec_runner.llm.get_metrics()
num_drafts = get_spec_decode_metric_value(
metrics, "vllm:spec_decode_num_drafts"
)
num_draft_tokens = get_spec_decode_metric_value(
metrics, "vllm:spec_decode_num_draft_tokens"
)
num_accepted_tokens = get_spec_decode_metric_value(
metrics, "vllm:spec_decode_num_accepted_tokens"
)
assert num_drafts > 0, "MTP drafter did not run"
assert num_draft_tokens > 0, "MTP drafter did not produce draft tokens"
assert num_accepted_tokens == 0, "Synthetic sampler accepted a draft token"
assert spec_token_ids == ref_token_ids, (
f"MTP changed greedy token ids under {cudagraph_mode.name}:\n"
f" reference={ref_token_ids}\n"
f" speculative={spec_token_ids}"
)
@pytest.mark.parametrize(
["model_setup", "mm_enabled", "expected_accuracy_threshold"],
[
# Reference accuracy: 65%-70%.
(("mtp", "XiaomiMiMo/MiMo-7B-Base", 1, None), False, 0.5),
pytest.param(
("mtp", "ZixiQi/DeepSeek-V3-4layers-MTP-FP8", 1, None),
False,
0.0,
marks=pytest.mark.skipif(
current_platform.is_device_capability_family(100),
reason="DeepSeek MTP: TRTLLM MoE top_k check fails on Blackwell",
),
), # dummy model
(
("mtp", "Qwen/Qwen3.5-0.8B-Base", 1, None),
False,
0.20,
), # hybrid + MTP, ref: ~34%-35%
(
("mtp", "google/gemma-4-E4B-it", 1, "google/gemma-4-E4B-it-assistant"),
False,
0.50,
), # gemma4 MTP with assistant model, ref: ~62%
],
ids=["mimo", "deepseek", "qwen3_5-hybrid", "gemma4-e4b"],
)
@single_gpu_only
def test_mtp_correctness(
monkeypatch: pytest.MonkeyPatch,
sampling_config: SamplingParams,
model_setup: tuple[str, str, int, str | None],
mm_enabled: bool,
expected_accuracy_threshold: float,
vllm_runner,
):
"""Compare the outputs of a original LLM and a speculative LLM
which should be the same when using MTP speculative decoding. Due to some variance
in the engine, it is possible for some outputs to differ, so we expect that at least
6/10 output tokens match exactly, and that the GSM8k accuracy is above a precomputed
reference threshold for each model.
"""
method, model_name, tp_size, draft_model = model_setup
_skip_if_insufficient_gpus_for_tp(tp_size)
# Generate test prompts inside the function instead of using fixture
test_prompts = get_test_prompts(mm_enabled)
with monkeypatch.context() as m:
m.setenv("VLLM_MLA_DISABLE", "1")
attn_backend = "TRITON_ATTN" if current_platform.is_rocm() else "auto"
# Skip multimodal profiling for models that don't need it in this test.
extra_kwargs: dict[str, Any] = {}
if "Qwen3.5" in model_name:
extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "video": 0}
elif "gemma-4" in model_name:
extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "audio": 0}
if draft_model is not None and "gemma-4" in draft_model:
import transformers
from packaging.version import Version
if Version(transformers.__version__) < Version("5.8.0"):
pytest.skip(
"Gemma4 MTP assistant requires transformers>=5.8.0, "
f"got {transformers.__version__}"
)
with vllm_runner(
model_name,
block_size=None,
max_model_len=2048,
tensor_parallel_size=tp_size,
trust_remote_code=True,
attention_backend=attn_backend,
enable_chunked_prefill=None,
compilation_config=CompilationConfig(),
**extra_kwargs,
) as ref_runner:
ref_outputs = ref_runner.llm.chat(test_prompts, sampling_config)
evaluate_llm_for_gsm8k(
ref_runner.llm,
expected_accuracy_threshold=expected_accuracy_threshold,
)
speculative_config: dict[str, Any] = {
"method": method,
"num_speculative_tokens": 1,
"max_model_len": 2048,
}
if draft_model is not None:
speculative_config["model"] = draft_model
speculative_config["num_speculative_tokens"] = 2
with vllm_runner(
model_name,
block_size=None,
trust_remote_code=True,
tensor_parallel_size=tp_size,
speculative_config=speculative_config,
max_model_len=2048,
attention_backend=attn_backend,
enable_chunked_prefill=None,
compilation_config=CompilationConfig(),
**extra_kwargs,
) as spec_runner:
# MTP supports async scheduling by default.
has_async = (
spec_runner.llm.llm_engine.vllm_config.scheduler_config.async_scheduling
)
assert has_async, (
f"Expected async scheduling for {method}: target={model_name}, "
f"draft={draft_model}; got {has_async}"
)
evaluate_llm_for_gsm8k(
spec_runner.llm,
expected_accuracy_threshold=expected_accuracy_threshold,
)
spec_outputs = spec_runner.llm.chat(test_prompts, sampling_config)
# Heuristic: expect at least 80% of the prompts to match exactly
# Upon failure, inspect the outputs to check for inaccuracy.
assert_request_outputs_match(
ref_outputs,
spec_outputs,
required_matches=int(0.8 * len(ref_outputs)) + 1,
context=f"{method} target={model_name}, draft={draft_model}",
)
+33
View File
@@ -4,6 +4,7 @@
import copy
import time
import uuid
from collections import deque
from concurrent.futures import Future, ThreadPoolExecutor
from unittest.mock import MagicMock, PropertyMock, patch
@@ -395,6 +396,38 @@ def test_engine_core_concurrent_batches():
req_id = (req_id + 1) % 2
@pytest.mark.parametrize("encoder_only", [True, False])
def test_async_step_skips_sampling_for_mm_encoder_only(encoder_only):
"""Encoder outputs are complete; only language-model steps need sampling."""
core = MagicMock()
core.batch_queue = deque()
core.batch_queue_size = 2
core.is_mm_encoder_only = encoder_only
core.is_pooling_model = False
core.scheduler.has_requests.side_effect = [True, False, False]
scheduled = core.scheduler.schedule.return_value
scheduled.total_num_scheduled_tokens = 16
scheduled.pending_structured_output_tokens = False
output = MagicMock()
executed: Future[ModelRunnerOutput | None] = Future()
executed.set_result(output if encoder_only else None)
sampled: Future[ModelRunnerOutput | None] = Future()
sampled.set_result(None if encoder_only else output)
core.model_executor.execute_model.return_value = executed
core.model_executor.sample_tokens.return_value = sampled
EngineCore.step_with_batch_queue(core)
if core.batch_queue:
EngineCore.step_with_batch_queue(core)
core.scheduler.update_from_output.assert_called_once_with(scheduled, output)
if encoder_only:
core.model_executor.sample_tokens.assert_not_called()
else:
core.model_executor.sample_tokens.assert_called_once()
@multi_gpu_test(num_gpus=2)
def test_engine_core_tp():
"""Test engine can initialize worker in tp properly."""
@@ -500,7 +500,7 @@ def test_decode_bench_connector_skips_null_padding():
def test_decode_bench_connector_uses_per_group_dcp_block_sizes():
"""DCP scales full-attention blocks but not sliding-window blocks."""
"""DCP scales target blocks but not the replicated draft's window."""
block_size = 16
dcp_world_size = 2
vllm_config = create_vllm_config(
@@ -515,6 +515,7 @@ def test_decode_bench_connector_uses_per_group_dcp_block_sizes():
dtype=torch.float32,
)
sliding_window = SlidingWindowSpec(
dcp_sharded=False,
block_size=block_size,
num_kv_heads=1,
head_size=1,
@@ -99,6 +99,7 @@ def _run_engine_core_handshake(
),
speculative_config=None,
ec_transfer_config=None,
is_mm_encoder_only=False,
max_concurrent_batches=1,
model_config=SimpleNamespace(
runner_type="generate",
@@ -3,10 +3,12 @@
import asyncio
import contextlib
import threading
import time
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
import torch
import zmq.asyncio
@@ -14,6 +16,7 @@ import zmq.asyncio
from tests.v1.attention.utils import dense_kv_cache_views
from vllm import envs
from vllm.config import set_current_vllm_config
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import mooncake_connector
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector import (
KVConnectorRole,
MooncakeConnector,
@@ -33,6 +36,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector im
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_utils import (
MooncakeBootstrapServer,
RegisterWorkerPayload,
)
from vllm.utils.network_utils import get_open_port
from vllm.v1.kv_cache_interface import (
@@ -538,6 +542,46 @@ def bootstrap_server():
server.shutdown()
@pytest.mark.asyncio
async def test_register_worker_recovers_from_slow_bootstrap_server(monkeypatch):
"""End-to-end: a real HTTP server that responds slower than the configured
timeout must be retried on the wire, not treated as fatal."""
original = MooncakeBootstrapServer.register_worker
state = {"done": False, "calls": 0}
async def slow_register(self, payload: RegisterWorkerPayload):
state["calls"] += 1
if not state["done"]:
state["done"] = True
# Record the worker first, then stall past the client timeout, so
# the retry exercises the duplicate-registration path.
response = await original(self, payload)
await asyncio.sleep(1.5)
return response
return await original(self, payload)
monkeypatch.setattr(MooncakeBootstrapServer, "register_worker", slow_register)
port = get_open_port()
monkeypatch.setenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", str(port))
monkeypatch.setenv("VLLM_MOONCAKE_CONNECTOR_TIMEOUT", "0.5")
monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 5)
server = MooncakeBootstrapServer("127.0.0.1", port)
server.start()
try:
worker = _make_local_register_worker_stub()
await MooncakeConnectorWorker.register_worker_with_bootstrap(worker)
assert state["calls"] == 2
async with httpx.AsyncClient() as client:
response = await client.get(f"http://127.0.0.1:{port}/query")
assert response.status_code == 200
assert response.json()["0"]["engine_id"] == "eng-1"
finally:
server.shutdown()
@pytest.mark.asyncio
async def test_bootstrap_server(bootstrap_server: MooncakeBootstrapServer):
"""Tests the bootstrap server's api for worker registration and querying.
@@ -589,9 +633,16 @@ async def test_bootstrap_server(bootstrap_server: MooncakeBootstrapServer):
assert data["0"]["worker_addr"]["0"]["0"] == "tcp://1.1.1.1:1111"
assert data["0"]["worker_addr"]["0"]["1"] == "tcp://2.2.2.2:2222"
# Test failure: re-registering the same worker
# Re-registering the identical payload is idempotent.
async with httpx.AsyncClient() as client:
response = await client.post(f"{base_url}/register", json=payload1)
assert response.status_code == 200
assert response.json() == {"status": "ok"}
# Test failure: same ranks, conflicting address
payload1_conflict = dict(payload1, addr="tcp://9.9.9.9:9999")
async with httpx.AsyncClient() as client:
response = await client.post(f"{base_url}/register", json=payload1_conflict)
assert response.status_code == 400
assert "is already registered" in response.text
@@ -628,6 +679,137 @@ def _make_bootstrap_vllm_config(
)
def _make_local_register_worker_stub():
return SimpleNamespace(
vllm_config=SimpleNamespace(
parallel_config=SimpleNamespace(
local_engines_only=False,
data_parallel_rank_local=0,
data_parallel_index=0,
nnodes_within_dp=1,
master_addr="127.0.0.1",
data_parallel_master_ip="127.0.0.1",
)
),
hostname="127.0.0.1",
side_channel_port=1234,
engine_id="eng-1",
dp_rank=0,
tp_rank=0,
pp_rank=0,
)
def _make_register_worker_stub():
return SimpleNamespace(
vllm_config=_make_bootstrap_vllm_config(),
hostname="127.0.0.1",
side_channel_port=1234,
engine_id="eng-1",
dp_rank=0,
tp_rank=0,
pp_rank=0,
)
class _FlakyAsyncClient:
"""httpx.AsyncClient stub that read-times-out `failures` times first."""
def __init__(self, failures: int, calls: list[int], **kwargs):
self.failures = failures
self.calls = calls
async def __aenter__(self):
return self
async def __aexit__(self, *exc_info):
return False
async def post(self, url, json=None):
self.calls.append(1)
if len(self.calls) <= self.failures:
raise httpx.ReadTimeout("simulated slow bootstrap server")
response = MagicMock()
response.raise_for_status = MagicMock()
return response
@pytest.mark.asyncio
async def test_register_worker_retries_on_read_timeout(monkeypatch):
"""A slow rank-0 bootstrap response must be retried, not treated as fatal."""
monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 5)
monkeypatch.setattr(asyncio, "sleep", AsyncMock())
calls: list[int] = []
monkeypatch.setattr(
httpx,
"AsyncClient",
lambda **kwargs: _FlakyAsyncClient(2, calls),
)
worker = _make_register_worker_stub()
await MooncakeConnectorWorker.register_worker_with_bootstrap(worker)
assert len(calls) == 3
@pytest.mark.asyncio
async def test_register_worker_raises_after_max_attempts(monkeypatch):
"""Terminal registration failure must raise, not loop forever."""
monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 3)
monkeypatch.setattr(asyncio, "sleep", AsyncMock())
calls: list[int] = []
monkeypatch.setattr(
httpx,
"AsyncClient",
lambda **kwargs: _FlakyAsyncClient(99, calls),
)
worker = _make_register_worker_stub()
with pytest.raises(RuntimeError, match="after 3 attempts"):
await MooncakeConnectorWorker.register_worker_with_bootstrap(worker)
assert len(calls) == 3
@pytest.mark.asyncio
async def test_sender_listener_failure_propagates_to_caller(monkeypatch):
"""A terminal registration failure must surface via the Future rather than
leaving register_kv_caches blocked on its ready event."""
monkeypatch.setenv("VLLM_MOONCAKE_CONNECTOR_TIMEOUT", "0.1")
monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 1)
loop = asyncio.new_event_loop()
thread = threading.Thread(target=loop.run_forever, daemon=True)
thread.start()
async def failing_listener(ready_event):
raise RuntimeError("simulated terminal registration failure")
try:
ready_event = threading.Event()
fut = asyncio.run_coroutine_threadsafe(failing_listener(ready_event), loop)
deadline = time.monotonic() + 10.0
raised = None
while not ready_event.wait(timeout=0.1):
if fut.done():
try:
fut.result()
except RuntimeError as e:
raised = e
break
assert time.monotonic() < deadline, "caller blocked past its deadline"
assert raised is not None
assert "simulated terminal registration failure" in str(raised)
finally:
loop.call_soon_threadsafe(loop.stop)
thread.join(timeout=5)
loop.close()
@pytest.mark.parametrize(
(
"tp_rank",
@@ -290,6 +290,8 @@ class _FakeKVTransferConfig:
class _FakeModelConfig:
model = "test-model"
use_mla = False
enable_sleep_mode = False
enable_cumem_allocator = False
def get_num_layers(self, parallel_config) -> int:
return 1
@@ -432,6 +434,64 @@ def _patch_worker_runtime(
monkeypatch.setattr(worker, "LookupKeyServer", MagicMock())
def test_create_mem_pool_returns_none_when_unconfigured():
assert worker.MooncakeStoreWorker._create_mem_pool({}, _FakeModelConfig()) is None
def test_create_mem_pool_rejects_unknown_pool():
with pytest.raises(
ValueError,
match="Unsupported custom_mem_pool='UNKNOWN'",
):
worker.MooncakeStoreWorker._create_mem_pool(
{"custom_mem_pool": "unknown"}, _FakeModelConfig()
)
@pytest.mark.parametrize(
"conflicting_option", ["enable_sleep_mode", "enable_cumem_allocator"]
)
def test_create_mem_pool_rejects_cumem_conflicts(conflicting_option):
model_config = _FakeModelConfig()
setattr(model_config, conflicting_option, True)
with pytest.raises(ValueError, match="custom_mem_pool is incompatible"):
worker.MooncakeStoreWorker._create_mem_pool(
{"custom_mem_pool": "NVLINK"}, model_config
)
@pytest.mark.parametrize(
("pool_type", "allocator_name"),
[("nvlink", "NVLinkAllocator"), ("barex", "BarexAllocator")],
)
def test_create_mem_pool_uses_mooncake_allocator(
monkeypatch, pool_type, allocator_name
):
allocator_cls = MagicMock()
allocator = allocator_cls.get_allocator.return_value
allocator_module = types.ModuleType("mooncake.allocator")
setattr(allocator_module, allocator_name, allocator_cls)
mooncake_module = types.ModuleType("mooncake")
mooncake_module.allocator = allocator_module # type: ignore[attr-defined]
monkeypatch.setitem(sys.modules, "mooncake", mooncake_module)
monkeypatch.setitem(sys.modules, "mooncake.allocator", allocator_module)
monkeypatch.setattr(torch.accelerator, "current_device_index", lambda: 2)
mem_pool = MagicMock()
mem_pool_ctor = MagicMock(return_value=mem_pool)
monkeypatch.setattr(torch.cuda, "MemPool", mem_pool_ctor)
result = worker.MooncakeStoreWorker._create_mem_pool(
{"custom_mem_pool": pool_type}, _FakeModelConfig()
)
assert result is mem_pool
allocator_cls.get_allocator.assert_called_once_with(torch.device("cuda", 2))
allocator.allocator.assert_called_once_with()
mem_pool_ctor.assert_called_once_with(allocator.allocator.return_value)
def test_pool_key_to_string_without_prefix_is_unchanged():
"""Default (empty) cache_prefix keeps keys byte-identical to the
historical unprefixed format so existing deployments keep their hits."""
@@ -68,6 +68,7 @@ class MockConnector(KVConnectorBase_V1):
# Override just build_kv_connector_stats
mock.build_kv_connector_stats = cls.build_kv_connector_stats
mock.get_kv_connector_stats.return_value = None
mock.get_mem_pool_context.return_value = None
return mock
@classmethod
@@ -201,6 +202,36 @@ def mc() -> MultiConnector:
return mc
def test_multi_connector_mem_pool_context_none(mc: MultiConnector):
assert mc.get_mem_pool_context() is None
def test_multi_connector_forwards_mem_pool_context(mc: MultiConnector):
context = MagicMock()
provider = MagicMock(spec_set=KVConnectorBase_V1)
provider.get_mem_pool_context.return_value = context
mc._connectors = [mc._connectors[0], provider]
assert mc.get_mem_pool_context() is context
provider.get_mem_pool_context.assert_called_once_with()
def test_multi_connector_rejects_multiple_mem_pool_contexts(mc: MultiConnector):
providers = [
MagicMock(spec_set=KVConnectorBase_V1),
MagicMock(spec_set=KVConnectorBase_V1),
]
for provider in providers:
provider.get_mem_pool_context.return_value = MagicMock()
mc._connectors = providers
with pytest.raises(
ValueError,
match="Multiple connectors provide a KV cache memory pool",
):
mc.get_mem_pool_context()
# Helper function to compare directories recursively
def _compare_directories(dir1: Path, dir2: Path) -> bool:
"""Compares two directories recursively for identical content."""
@@ -320,6 +351,7 @@ def test_multi_example_connector_consistency():
# First three events are from initialization. Layer hooks run before the
# deferred load starts after the forward pass.
expected_worker_prefix = [
"get_mem_pool_context",
"register_kv_caches",
"set_host_xfer_buffer_ops",
"get_handshake_metadata",
@@ -7,6 +7,7 @@ import os
import queue
import tempfile
import textwrap
import threading
import time
import uuid
from collections import defaultdict
@@ -49,6 +50,8 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl import (
NixlKVConnectorStats,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
RemoteMeta,
ReqMeta,
compute_nixl_compatibility_hash,
)
from vllm.distributed.kv_transfer.kv_transfer_state import (
@@ -2087,6 +2090,10 @@ def recv_worker():
worker._recving_transfers = defaultdict(list)
worker._failed_recv_reqs = queue.Queue()
worker._recv_failures = set()
worker._handshake_lock = threading.RLock()
worker._handshake_futures = {}
worker._remote_agents = {}
worker._engine_by_address = {}
worker._replicated_pcp_done_sending = set()
worker._invalid_block_ids = queue.Queue()
worker._pending_recv_notifs = {}
@@ -2681,6 +2688,174 @@ def test_engine_ttl_disabled(default_vllm_config, dist_init):
assert engine_id in worker.dst_xfer_side_handles
@pytest.mark.cpu_test
class TestPeerReplacement:
@pytest.fixture(autouse=True)
def setup(self, monkeypatch):
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker as bw
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
config = create_vllm_config(kv_connector_extra_config={"engine_ttl": 0})
config.kv_transfer_config.kv_buffer_device = "cpu"
platform = SimpleNamespace(
device_type="cpu",
discover_numa_topology=lambda: [],
get_nixl_memory_type=lambda: "DRAM",
is_rocm=lambda: False,
)
with (
patch.object(bw, "NixlWrapper", FakeNixlWrapper),
patch.object(bw, "current_platform", platform),
patch.object(bw, "get_tensor_model_parallel_rank", return_value=0),
patch.object(bw, "get_tensor_model_parallel_world_size", return_value=1),
patch.object(
bw, "get_current_attn_backends", return_value=[FlashAttentionBackend]
),
):
self.worker = FakeNixlConnectorWorker(config, "local", hand_shake_latency=0)
self.nixl = self.worker.nixl_wrapper
self.transport = MagicMock(wraps=self.nixl)
self.worker.nixl_wrapper = self.transport
monkeypatch.setattr(
self.worker._handshake_initiation_executor,
"submit",
lambda *args: Future(),
)
self._connect("old")
yield
self.worker.shutdown()
def _connect(self, engine_id="new", host="localhost", port=1234):
future = self.worker._ensure_handshake(engine_id, host, port, 1)
self.worker.REMOTE_ENGINE_ID = engine_id
self.nixl.REMOTE_AGENT_NAME = engine_id
future.set_result(self.worker._nixl_handshake(host, port, 1, engine_id))
def _request(self, engine_id="new", blocks=(1,), awaiting_kvs=True):
metadata = NixlConnectorMetadata()
metadata.reqs_to_recv[f"{engine_id}-req"] = ReqMeta(
local_block_ids=(blocks,),
local_physical_block_ids=(blocks,),
tp_size=1,
remote=RemoteMeta(
([2],), "localhost", 1234, engine_id, f"{engine_id}-prefill"
),
awaiting_kvs=awaiting_kvs,
)
return metadata
def test_cleans_old_peer_after_confirmed_handshake(self):
self._connect("healthy", host="other-host")
old_handle = self.worker.dst_xfer_side_handles["old"][0]
self.worker.start_load_kv(self._request())
self.worker.get_transfer_results()
self.transport.remove_remote_agent.assert_not_called()
self._connect()
self.worker.get_transfer_results()
self.transport.release_dlist_handle.assert_called_once_with(old_handle)
self.transport.remove_remote_agent.assert_called_once_with("old")
assert set(self.worker._remote_agents) == {"new", "healthy"}
assert self.worker._engine_by_address == {
("localhost", 1234): "new",
("other-host", 1234): "healthy",
}
with pytest.raises(KeyError):
self.worker.transfer_topo.get_engine_info("old")
self.worker.start_load_kv(NixlConnectorMetadata())
assert self.worker.get_transfer_results().finished_recving == {"new-req"}
self.transport.remove_remote_agent.assert_called_once_with("old")
@pytest.mark.parametrize("host,port", [("other-host", 1234), ("localhost", 5678)])
def test_preserves_other_addresses(self, host, port):
self._connect(host=host, port=port)
self.worker.get_transfer_results()
assert "old" in self.worker._remote_agents
self.transport.remove_remote_agent.assert_not_called()
def test_preserves_old_peer_when_handshake_fails(self):
self.worker.start_load_kv(self._request())
self.worker._handshake_futures["new"].set_exception(
RuntimeError("handshake failed")
)
assert self.worker.get_transfer_results().failed_recving == {"new-req"}
assert "old" in self.worker._remote_agents
assert self.worker._engine_by_address == {("localhost", 1234): "old"}
self.transport.remove_remote_agent.assert_not_called()
@pytest.mark.parametrize(
"states,failed",
[
(["PROC", "DONE"], False),
(["ERR", "PROC", "DONE"], True),
(["ERR", "PROC", "ERR"], True),
],
ids=["healthy", "failed-sibling-completes", "failed-sibling-fails"],
)
def test_waits_for_reads_before_cleanup_and_failure_reporting(self, states, failed):
self.worker._recving_metadata.update(self._request("old").reqs_to_recv)
self.worker._recving_transfers["old-req"] = [1, 2] if failed else [1]
self.transport.check_xfer_state.side_effect = states
self._connect()
result = self.worker.get_transfer_results()
assert result.finished_recving == result.failed_recving == set()
assert self.worker.get_block_ids_with_load_errors() == set()
self.transport.remove_remote_agent.assert_not_called()
result = self.worker.get_transfer_results()
assert result.finished_recving == {"old-req"}
assert result.failed_recving == ({"old-req"} if failed else set())
assert self.worker.get_block_ids_with_load_errors() == (
{1} if failed else set()
)
assert self.transport.release_xfer_handle.call_count == (2 if failed else 1)
self.transport.remove_remote_agent.assert_called_once_with("old")
def test_waits_for_queued_notification(self):
metadata = self._request("old", blocks=())
self.worker._recving_metadata.update(metadata.reqs_to_recv)
self.worker._background_nixl_handshake(
"old-req", "old", metadata.reqs_to_recv["old-req"]
)
self._connect()
self.worker.get_transfer_results()
self.transport.remove_remote_agent.assert_not_called()
self.worker.start_load_kv(NixlConnectorMetadata())
self.transport.send_notif.assert_called_once_with(
"old", notif_msg=b"old-prefill:1"
)
result = self.worker.get_transfer_results()
assert result.finished_recving == {"old-req"}
assert result.failed_recving == set()
self.transport.remove_remote_agent.assert_called_once_with("old")
def test_waits_for_other_handshakes(self):
self.worker._ensure_handshake("other", "other-host", 1234, 1)
self._connect()
self.worker.get_transfer_results()
self.transport.remove_remote_agent.assert_not_called()
self._connect("other", host="other-host")
self.worker.get_transfer_results()
self.transport.remove_remote_agent.assert_called_once_with("old")
def test_rehandshakes_queued_request_for_released_engine(self):
# A request queued after its engine's handshake, but drained only after
# that engine was replaced and released, must not read from it.
metadata = self._request("old", blocks=(), awaiting_kvs=False)
self.worker._ready_requests.put(("old-req", metadata.reqs_to_recv["old-req"]))
self._connect()
self.worker.get_transfer_results()
self.transport.remove_remote_agent.assert_called_once_with("old")
self.worker.start_load_kv(NixlConnectorMetadata())
assert "old" in self.worker._handshake_futures
self.transport.send_notif.assert_not_called()
def test_transfer_topology_unregister():
"""TransferTopology.unregister_remote_engine removes the engine."""
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
@@ -4,6 +4,7 @@
import gc
import queue
import threading
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
@@ -54,6 +55,9 @@ def region_pull_worker():
worker.enable_permute_local_kv = False
worker.enable_heterogeneous_attn_post_process = False
worker._engine_last_active = {}
worker._handshake_lock = threading.RLock()
worker._handshake_futures = {}
worker._transfer_layer_group_ids = ()
worker._bidirectional_kv_xfer_enabled = False
worker._recving_transfers = {}
worker.use_mla, worker._has_mamba = True, False
@@ -94,6 +98,7 @@ def region_pull_worker():
worker.dst_num_blocks = {"P": 100, "D": 100}
worker.dst_region_num_blocks = {"P": [100, 100], "D": [100, 100]}
worker._remote_agents = {"P": {(0, 0): "P-rank0"}}
worker._engine_by_address = {("localhost", 1): "P"}
worker._read_blocks_mixed = MagicMock()
worker.nixl_wrapper = MagicMock()
return worker
@@ -395,6 +395,7 @@ class _StubWriterWorker(NixlPushConnectorWorker):
w._group_spec_types = (FullAttentionSpec,)
w._engine_ttl = 0.0
w._engine_last_active = {}
w._engine_by_address = {}
# Track _do_start_push_kv invocations.
calls: list[tuple[str, Any, dict[str, Any]]] = []
@@ -0,0 +1,466 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for SimpleCPUOffloadConnector metrics."""
from __future__ import annotations
import dataclasses
from types import SimpleNamespace
from typing import Any
import pytest
from prometheus_client import Counter, Gauge, Histogram
from tests.v1.kv_connector.unit.test_simple_cpu_offload_connector import (
_make_connector,
)
from tests.v1.simple_kv_offload.test_scheduler import (
make_request,
make_scheduler,
make_scheduler_output,
)
from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector import (
SimpleCPUOffloadConnector,
)
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.simple_kv_offload.manager import (
BoundaryStoreStats,
LoadRequestState,
TransferMeta,
)
from vllm.v1.simple_kv_offload.metrics import (
INFO_LABELS,
OUTCOME_TO_FIELD,
MetricName,
SimpleCPUOffloadStats,
_MetricType,
_StatsKey,
)
SAVE_OUTCOMES = MetricName.SAVE_OUTCOMES
LOAD_BLOCKS = MetricName.LOAD_BLOCKS
USED_BLOCKS = MetricName.USED_BLOCKS
PENDING_STORE_BLOCKS = MetricName.PENDING_STORE_BLOCKS
INFO = MetricName.INFO
class _FakeMetric:
def __init__(self, **kwargs: Any):
self.kwargs = kwargs
self.children: list[_FakeMetric] = []
self.observed: list[int | float] = []
self.increments: list[int | float] = []
self.set_values: list[int | float] = []
self.labelvalues: tuple[object, ...] = ()
def labels(self, *labelvalues: object) -> _FakeMetric:
child = _FakeMetric(**self.kwargs)
child.labelvalues = labelvalues
self.children.append(child)
return child
def observe(self, value: int | float) -> None:
self.observed.append(value)
def inc(self, value: int | float) -> None:
self.increments.append(value)
def set(self, value: int | float) -> None:
self.set_values.append(value)
def _fake_metric_types() -> dict[type, type]:
return {Gauge: _FakeMetric, Counter: _FakeMetric, Histogram: _FakeMetric}
def _make_prom_metrics(per_engine: dict[int, list[object]] | None = None) -> Any:
from vllm.v1.simple_kv_offload.metrics import SimpleCPUOffloadPromMetrics
return SimpleCPUOffloadPromMetrics(
vllm_config=SimpleNamespace(kv_transfer_config=None),
metric_types=_fake_metric_types(),
labelnames=["model_name", "engine"],
per_engine_labelvalues=per_engine or {0: ["model", "0"]},
)
def _stats_payload(types: dict, data: dict) -> dict[str, Any]:
return {_StatsKey.TYPES: types, _StatsKey.DATA: data}
def _values(stats: SimpleCPUOffloadStats, name: str) -> dict:
return stats.data[_StatsKey.DATA][name]
# ---------------------------------------------------------------------------
# Stats class
# ---------------------------------------------------------------------------
def test_build_kv_connector_stats_none_and_empty() -> None:
stats = SimpleCPUOffloadConnector.build_kv_connector_stats(data=None)
assert isinstance(stats, SimpleCPUOffloadStats)
assert stats.is_empty()
stats = SimpleCPUOffloadConnector.build_kv_connector_stats(data={})
assert isinstance(stats, SimpleCPUOffloadStats)
assert stats.is_empty()
def test_stats_round_trip_through_serialized_dict() -> None:
stats = SimpleCPUOffloadStats()
stats.increase_counter(SAVE_OUTCOMES, 3, ("stored",))
stats.increase_counter(LOAD_BLOCKS, 4)
stats.set_gauge(USED_BLOCKS, 7)
stats.set_gauge(INFO, 1, ("cpu", "false", "false", "64"))
rebuilt = SimpleCPUOffloadConnector.build_kv_connector_stats(data=stats.to_dict())
assert isinstance(rebuilt, SimpleCPUOffloadStats)
assert _values(rebuilt, SAVE_OUTCOMES) == {("stored",): 3}
assert _values(rebuilt, LOAD_BLOCKS) == {(): 4}
assert _values(rebuilt, USED_BLOCKS) == {(): 7}
assert _values(rebuilt, INFO) == {("cpu", "false", "false", "64"): 1}
def test_stats_aggregate_sums_counters_and_keeps_latest_gauge() -> None:
stats1 = SimpleCPUOffloadStats()
stats1.increase_counter(SAVE_OUTCOMES, 3, ("stored",))
stats1.increase_counter(SAVE_OUTCOMES, 1, ("dropped_cpu_full",))
stats1.set_gauge(USED_BLOCKS, 7)
stats2 = SimpleCPUOffloadStats()
stats2.increase_counter(SAVE_OUTCOMES, 4, ("stored",))
stats2.set_gauge(USED_BLOCKS, 2)
result = stats1.aggregate(stats2)
assert result is stats1
outcomes = _values(stats1, SAVE_OUTCOMES)
assert outcomes == {("stored",): 7, ("dropped_cpu_full",): 1}
assert _values(stats1, USED_BLOCKS) == {(): 2}
def test_stats_aggregate_merges_types_from_other() -> None:
stats1 = SimpleCPUOffloadStats()
stats1.set_gauge(USED_BLOCKS, 1)
stats2 = SimpleCPUOffloadStats()
stats2.increase_counter(LOAD_BLOCKS, 2)
stats1.aggregate(stats2)
assert stats1.data[_StatsKey.TYPES][LOAD_BLOCKS] == _MetricType.COUNTER
assert _values(stats1, LOAD_BLOCKS) == {(): 2}
def test_stats_reduce_and_reset() -> None:
stats = SimpleCPUOffloadStats()
stats.increase_counter(SAVE_OUTCOMES, 3, ("stored",))
stats.set_gauge(PENDING_STORE_BLOCKS, 4)
stats.increase_counter(LOAD_BLOCKS, 9)
stats.set_gauge(INFO, 1, ("cpu", "false", "false", "64"))
reduced = stats.reduce()
assert reduced[f"{SAVE_OUTCOMES}:{('stored',)}"] == 3
assert reduced[PENDING_STORE_BLOCKS] == 4
assert reduced[LOAD_BLOCKS] == 9
assert not any(key.startswith(INFO) for key in reduced)
stats.reset()
assert stats.is_empty()
def test_outcome_labels_match_boundary_store_stats_fields() -> None:
"""Every outcome label maps 1:1 to a BoundaryStoreStats counter field."""
boundary_fields = {
f.name for f in dataclasses.fields(BoundaryStoreStats) if f.name != "published"
}
assert set(OUTCOME_TO_FIELD.values()) == boundary_fields
assert len(set(OUTCOME_TO_FIELD.values())) == len(OUTCOME_TO_FIELD)
# ---------------------------------------------------------------------------
# Prom metrics class
# ---------------------------------------------------------------------------
def test_prom_metrics_registers_tier1_metrics() -> None:
prom = _make_prom_metrics()
defs = prom._defs
assert set(defs) == {
SAVE_OUTCOMES,
LOAD_BLOCKS,
USED_BLOCKS,
PENDING_STORE_BLOCKS,
INFO,
}
# The prom class registers literal names (required by the docs
# generator); they must stay in sync with the emission-side constants.
assert {m.kwargs["name"] for m in defs.values()} == {
SAVE_OUTCOMES,
LOAD_BLOCKS,
USED_BLOCKS,
PENDING_STORE_BLOCKS,
INFO,
}
for metric in defs.values():
assert metric.kwargs["documentation"]
assert metric.kwargs["labelnames"][:2] == ["model_name", "engine"]
assert defs[SAVE_OUTCOMES].kwargs["labelnames"] == [
"model_name",
"engine",
"outcome",
]
assert defs[LOAD_BLOCKS].kwargs["labelnames"] == ["model_name", "engine"]
assert defs[INFO].kwargs["labelnames"] == [
"model_name",
"engine",
*INFO_LABELS,
]
assert defs[USED_BLOCKS].kwargs["labelnames"] == ["model_name", "engine"]
# mostrecent: multiprocess scrapes expose one series per label set, not
# one pid-tagged series per writer process.
for gauge in (USED_BLOCKS, PENDING_STORE_BLOCKS, INFO):
assert defs[gauge].kwargs["multiprocess_mode"] == "mostrecent"
def test_prom_metrics_observe_routes_each_series() -> None:
prom = _make_prom_metrics()
prom.observe(
_stats_payload(
types={
SAVE_OUTCOMES: _MetricType.COUNTER,
LOAD_BLOCKS: _MetricType.COUNTER,
USED_BLOCKS: _MetricType.GAUGE,
PENDING_STORE_BLOCKS: _MetricType.GAUGE,
INFO: _MetricType.GAUGE,
},
data={
SAVE_OUTCOMES: {("stored",): 3},
LOAD_BLOCKS: {(): 4},
USED_BLOCKS: {(): 5},
PENDING_STORE_BLOCKS: {(): 1},
INFO: {("cpu", "false", "false", "64"): 1},
},
)
)
assert prom._metrics[(0, SAVE_OUTCOMES, ("stored",))].increments == [3]
assert prom._metrics[(0, LOAD_BLOCKS, ())].increments == [4]
assert prom._metrics[(0, USED_BLOCKS, ())].set_values == [5]
assert prom._metrics[(0, PENDING_STORE_BLOCKS, ())].set_values == [1]
assert prom._metrics[(0, INFO, ("cpu", "false", "false", "64"))].set_values == [1]
assert prom._metrics[(0, SAVE_OUTCOMES, ("stored",))].labelvalues == (
"model",
"0",
"stored",
)
assert prom._metrics[(0, INFO, ("cpu", "false", "false", "64"))].labelvalues == (
"model",
"0",
"cpu",
"false",
"false",
"64",
)
def test_prom_metrics_reuses_bound_children() -> None:
prom = _make_prom_metrics()
payload = _stats_payload(
types={SAVE_OUTCOMES: _MetricType.COUNTER},
data={SAVE_OUTCOMES: {("stored",): 1}},
)
prom.observe(payload)
prom.observe(payload)
child = prom._metrics[(0, SAVE_OUTCOMES, ("stored",))]
assert child.increments == [1, 1]
assert len(prom._defs[SAVE_OUTCOMES].children) == 1
def test_prom_metrics_rejects_unknown_metric() -> None:
prom = _make_prom_metrics()
with pytest.raises(AssertionError, match="Unknown"):
prom.observe(
_stats_payload(
types={"vllm:not_a_metric": _MetricType.COUNTER},
data={"vllm:not_a_metric": {(): 1}},
)
)
def test_prom_metrics_rejects_wrong_label_count() -> None:
prom = _make_prom_metrics()
with pytest.raises(AssertionError, match="labels"):
prom.observe(
_stats_payload(
types={SAVE_OUTCOMES: _MetricType.COUNTER},
data={SAVE_OUTCOMES: {("stored", "extra"): 1}},
)
)
def test_prom_metrics_routes_per_engine() -> None:
prom = _make_prom_metrics(per_engine={0: ["model", "0"], 1: ["model", "1"]})
prom.observe(
_stats_payload(
types={USED_BLOCKS: _MetricType.GAUGE}, data={USED_BLOCKS: {(): 3}}
),
engine_idx=1,
)
assert (0, USED_BLOCKS, ()) not in prom._metrics
assert prom._metrics[(1, USED_BLOCKS, ())].set_values == [3]
def test_connector_build_prom_metrics() -> None:
from vllm.v1.simple_kv_offload.metrics import SimpleCPUOffloadPromMetrics
prom = SimpleCPUOffloadConnector.build_prom_metrics(
vllm_config=SimpleNamespace(kv_transfer_config=None),
metric_types=_fake_metric_types(),
labelnames=["model_name", "engine"],
per_engine_labelvalues={0: ["model", "0"]},
)
assert isinstance(prom, SimpleCPUOffloadPromMetrics)
# ---------------------------------------------------------------------------
# Manager: interval stats
# ---------------------------------------------------------------------------
def test_manager_get_stats_reports_outcome_deltas_per_interval() -> None:
fixture = make_scheduler()
sched = fixture.scheduler
sched.boundary_store_stats.stored += 3
sched.boundary_store_stats.dropped_cpu_full += 2
sched.boundary_store_stats.skipped_in_flight += 1
stats = sched.get_stats()
assert _values(stats, SAVE_OUTCOMES) == {
("stored",): 3,
("dropped_cpu_full",): 2,
("skipped_in_flight",): 1,
}
# Deltas are per-interval: a second drain reports no new outcomes.
stats2 = sched.get_stats()
assert SAVE_OUTCOMES not in stats2.data[_StatsKey.DATA]
def test_manager_reset_preserves_undrained_outcome_deltas() -> None:
fixture = make_scheduler()
sched = fixture.scheduler
sched.boundary_store_stats.stored += 3
sched.reset()
stats = sched.get_stats()
assert _values(stats, SAVE_OUTCOMES) == {("stored",): 3}
def test_manager_get_stats_gauges_and_info_labels() -> None:
fixture = make_scheduler(num_cpu_blocks=8)
sched = fixture.scheduler
sched._store_event_to_blocks[0] = TransferMeta([10], [11])
sched._pending_finished_stores.append(TransferMeta([12], [13]))
sched._abandoned_store_event_to_blocks[1] = TransferMeta([14], [15])
stats = sched.get_stats()
assert _values(stats, PENDING_STORE_BLOCKS) == {(): 3}
used = _values(stats, USED_BLOCKS)[()]
assert used == sched.num_cpu_blocks - sched.cpu_block_pool.get_num_free_blocks()
info = _values(stats, INFO)
assert info == {("cpu", "false", "false", str(sched.num_cpu_blocks)): 1}
# Gauges are re-reported every drain.
stats2 = sched.get_stats()
assert _values(stats2, PENDING_STORE_BLOCKS) == {(): 3}
def test_manager_info_labels_reflect_disk_mode() -> None:
connector = _make_connector(
extra_config={
"kv_offload_backend": "disk",
"disk_path": "/tmp/fake-disk-metrics",
"disk_capacity_bytes": 1024**3,
"use_page_cache": True,
}
)
sched = connector.scheduler_manager
assert sched is not None
assert sched._info_labelvalues[0] == "disk"
assert sched._info_labelvalues[1] == "true"
stats = sched.get_stats()
assert set(_values(stats, INFO)) == {sched._info_labelvalues}
def test_manager_info_labels_page_cache_false_for_cpu_backend() -> None:
"""CPU backend ignores use_page_cache; the info gauge must not claim it."""
connector = _make_connector(extra_config={"use_page_cache": True})
sched = connector.scheduler_manager
assert sched is not None
assert sched._info_labelvalues[:2] == ("cpu", "false")
stats = sched.get_stats()
assert set(_values(stats, INFO)) == {sched._info_labelvalues}
def test_manager_counts_load_blocks_completed() -> None:
fixture = make_scheduler(num_cpu_blocks=8)
sched = fixture.scheduler
request = make_request(num_blocks=2)
gpu_blocks = fixture.gpu_block_pool.get_new_blocks(2)
cpu_blocks = sched.cpu_block_pool.get_new_blocks(2)
sched._reqs_to_load[request.request_id] = LoadRequestState(
request=request,
transfer_meta=TransferMeta(
[b.block_id for b in gpu_blocks], [b.block_id for b in cpu_blocks]
),
)
sched.build_connector_meta(make_scheduler_output({}))
stats = sched.get_stats()
assert LOAD_BLOCKS not in stats.data[_StatsKey.DATA]
sched.update_connector_output(
KVConnectorOutput(finished_recving={request.request_id})
)
stats = sched.get_stats()
assert _values(stats, LOAD_BLOCKS) == {(): 2}
def test_manager_abandoned_load_counts_as_completed_after_reset() -> None:
"""Loads abandoned by reset() that still finish on the worker are
counted as loaded blocks.
"""
fixture = make_scheduler(num_cpu_blocks=8)
sched = fixture.scheduler
request = make_request(num_blocks=2)
gpu_blocks = fixture.gpu_block_pool.get_new_blocks(2)
cpu_blocks = sched.cpu_block_pool.get_new_blocks(2)
sched._reqs_to_load[request.request_id] = LoadRequestState(
request=request,
transfer_meta=TransferMeta(
[b.block_id for b in gpu_blocks], [b.block_id for b in cpu_blocks]
),
)
sched.build_connector_meta(make_scheduler_output({}))
stats = sched.get_stats()
assert LOAD_BLOCKS not in stats.data[_StatsKey.DATA]
assert sched.reset() is False
assert set(sched._abandoned_reqs_to_load) == {request.request_id}
sched.update_connector_output(
KVConnectorOutput(finished_recving={request.request_id})
)
assert sched._abandoned_reqs_to_load == {}
stats = sched.get_stats()
assert _values(stats, LOAD_BLOCKS) == {(): 2}
+2 -2
View File
@@ -2316,8 +2316,8 @@ def _make_hybrid_attention_mamba_scheduler(
def test_hybrid_store_uses_resolved_group_block_sizes() -> None:
"""Replicated groups must not be scaled by the DCP world size.
``dcp_world_size_for_kv_cache_spec`` gives full attention the process DCP
size and every other spec 1. Reconstructing a group's geometry as
The spec's ``dcp_sharded`` flag controls its effective block size.
Reconstructing a group's geometry as
``spec.block_size * cp_world_size`` over-scales the replicated groups, so
the eager store scan believes far fewer of their blocks are ready and
silently offloads only a fraction of them.
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from unittest.mock import patch
import numpy as np
import pytest
@@ -9,7 +10,7 @@ import torch
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import (
prepare_dflash_inputs,
DFlashSpeculator,
)
pytestmark = pytest.mark.skipif(
@@ -24,6 +25,7 @@ def _run_prepare(
cp_rank: int = 0,
cp_size: int = 1,
cp_interleave: int = 1,
draft_dcp_size: int | None = None,
):
device = torch.device("cuda")
max_num_reqs = 4
@@ -40,7 +42,9 @@ def _run_prepare(
)
input_batch = SimpleNamespace(
num_reqs=1,
num_tokens=4,
num_scheduled_tokens=np.array([4], dtype=np.int32),
seq_lens_cpu_upper_bound=torch.tensor([target_positions[-1] + 1]),
positions=torch.tensor(target_positions, dtype=torch.int64, device=device),
query_start_loc=torch.tensor([0, 4], dtype=torch.int32, device=device),
idx_mapping=torch.tensor([2], dtype=torch.int32, device=device),
@@ -74,36 +78,65 @@ def _run_prepare(
next_prefill_tokens = torch.zeros_like(last_sampled)
block_table = torch.tensor([block_table_values], dtype=torch.int32, device=device)
prepare_dflash_inputs(
input_buffers,
query_slot_mapping,
context_positions,
context_slot_mapping,
sample_indices,
sample_pos,
sample_idx_mapping,
temperature,
seeds,
input_batch,
torch.tensor([1], dtype=torch.int32, device=device),
torch.tensor([2], dtype=torch.int32, device=device),
last_sampled,
next_prefill_tokens,
input_temperature,
input_seeds,
block_table,
4,
cp_rank,
cp_size,
cp_interleave,
123,
num_speculative_steps,
num_speculative_steps,
max_num_reqs,
max_num_tokens,
128,
class InputsPrepared(Exception):
pass
draft = SimpleNamespace(
input_buffers=input_buffers,
context_positions=context_positions,
_context_slot_mappings=context_slot_mapping.unsqueeze(0),
sample_indices=sample_indices,
sample_pos=sample_pos,
sample_idx_mapping=sample_idx_mapping,
temperature=temperature,
seeds=seeds,
hidden_states=torch.zeros(4, 1, device=device),
prepare_context_anchor=lambda *args: None,
query_cudagraph_manager=None,
dp_size=1,
dp_rank=0,
pcp_manager=None,
draft_kv_cache_group_id=0,
draft_kv_cache_group_ids=[0],
_layer_group_idx=None,
block_tables=SimpleNamespace(
slot_mappings=query_slot_mapping.unsqueeze(0),
input_block_tables=[block_table],
kernel_block_sizes=[4],
cp_rank=cp_rank,
cp_size=cp_size,
cp_interleave=cp_interleave,
),
dcp_size=cp_size if draft_dcp_size is None else draft_dcp_size,
parallel_drafting_token_id=123,
num_query_per_req=num_speculative_steps,
num_speculative_steps=num_speculative_steps,
max_num_reqs=max_num_reqs,
max_num_tokens=max_num_tokens,
max_model_len=128,
sample_from_anchor=True,
)
with (
patch(
"vllm.v1.worker.gpu.spec_decode.dflash.speculator.dispatch_cg_and_sync_dp",
side_effect=InputsPrepared,
),
pytest.raises(InputsPrepared),
):
DFlashSpeculator.propose(
draft,
input_batch,
{},
{},
draft.hidden_states,
None,
torch.tensor([1], dtype=torch.int32, device=device),
torch.tensor([2], dtype=torch.int32, device=device),
last_sampled,
next_prefill_tokens,
input_temperature,
input_seeds,
)
torch.accelerator.synchronize()
return SimpleNamespace(
input_buffers=input_buffers,
@@ -118,12 +151,17 @@ def _run_prepare(
)
def test_prepare_dflash_inputs_excludes_rejected_context_suffix():
@pytest.mark.parametrize("cp_rank,cp_size", [(0, 1), (0, 2), (1, 2)])
def test_prepare_dflash_inputs_excludes_rejected_context_suffix(cp_rank, cp_size):
# Positions 10/11 use physical block 7. Rejected positions 12/13 would use
# block 8, but must be PAD context rather than contaminating draft KV.
out = _run_prepare(
target_positions=[10, 11, 12, 13],
block_table_values=[0, 0, 7, 8, 9, 10, 11, 12],
cp_rank=cp_rank,
cp_size=cp_size,
cp_interleave=2,
draft_dcp_size=1,
)
assert out.context_positions[:4].tolist() == [10, 11, 0, 0]
@@ -17,6 +17,7 @@ from types import MethodType, SimpleNamespace
from unittest.mock import patch
import numpy as np
import pytest
import torch
from vllm.config.compilation import CUDAGraphMode
@@ -53,6 +54,7 @@ def _make_fake_speculator(
arange_np=np.arange(max_num_reqs + 1, dtype=np.int32),
draft_is_prefilling=torch.zeros(max_num_reqs, dtype=torch.bool),
block_tables=fake_block_tables,
dcp_size=1,
input_buffers=fake_input_buffers,
attn_groups=[],
kv_cache_config=SimpleNamespace(kv_cache_groups=[]),
@@ -152,14 +154,17 @@ def test_build_draft_attn_metadata_clamps_to_max_model_len():
assert torch.equal(bound, torch.tensor([1024, 503], dtype=torch.int32))
def test_build_draft_attn_metadata_recomputes_dcp_local_seq_lens():
@pytest.mark.parametrize("draft_dcp_size", [1, 2])
def test_build_draft_attn_metadata_recomputes_dcp_local_seq_lens(draft_dcp_size):
fake = _make_fake_speculator()
fake.dcp_size = draft_dcp_size
fake.block_tables.cp_size = 2
fake.block_tables.cp_rank = 1
fake.block_tables.cp_interleave = 4
fake.input_buffers.seq_lens[:3] = torch.tensor([5, 9, 16])
def fake_prepare(out, seq_lens, num_reqs, dcp_size, dcp_rank, cp_interleave):
assert draft_dcp_size > 1
assert seq_lens is fake.input_buffers.seq_lens
assert (num_reqs, dcp_size, dcp_rank, cp_interleave) == (3, 2, 1, 4)
out[:num_reqs].copy_(torch.tensor([1, 4, 8], dtype=torch.int32))
@@ -176,6 +181,9 @@ def test_build_draft_attn_metadata_recomputes_dcp_local_seq_lens():
)
local = captured["dcp_local_seq_lens"]
if draft_dcp_size == 1:
assert local is None
return
assert isinstance(local, torch.Tensor)
assert local.data_ptr() == fake.input_buffers.dcp_local_seq_lens.data_ptr()
assert local.tolist() == [1, 4, 8, 0]
+34
View File
@@ -1,5 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
@@ -7,6 +9,38 @@ from vllm.v1.attention.backends.utils import get_dcp_local_seq_lens
from vllm.v1.worker.cp_utils import should_skip_dcp_context_attention
def test_cp_checks_allow_a_replicated_draft_backend_without_dcp(monkeypatch):
from vllm.v1.worker import cp_utils
config = SimpleNamespace(
parallel_config=SimpleNamespace(
prefill_context_parallel_size=1,
decode_context_parallel_size=4,
cp_kv_cache_interleave_size=16,
),
speculative_config=object(),
)
target = SimpleNamespace(
impl=SimpleNamespace(
dcp_world_size=4,
supports_mtp_with_cp_non_trivial_interleave_size=True,
need_to_return_lse_for_decode=True,
)
)
draft = SimpleNamespace(
impl=SimpleNamespace(
dcp_world_size=1,
supports_mtp_with_cp_non_trivial_interleave_size=False,
need_to_return_lse_for_decode=False,
)
)
layers = {"target": target, "draft": draft}
monkeypatch.setattr(cp_utils, "get_layers_from_vllm_config", lambda *args: layers)
cp_utils.check_attention_cp_compatibility(config, target_layer_names={"target"})
with pytest.raises(AssertionError, match="MTP with cp_kv_cache_interleave_size"):
cp_utils.check_attention_cp_compatibility(config)
def test_skip_gate_only_for_zero_context():
assert should_skip_dcp_context_attention(torch.zeros(3, dtype=torch.int32))
assert not should_skip_dcp_context_attention(
+92
View File
@@ -2,8 +2,17 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass, field
from types import SimpleNamespace
import pytest
import torch
from vllm.config import ParallelConfig
from vllm.v1.kv_cache_interface import FullAttentionSpec
from vllm.v1.worker.gpu import model_runner
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator
from vllm.v1.worker.gpu.spec_decode.dspark.utils import _get_dspark_parallel_config
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
@dataclass
@@ -49,3 +58,86 @@ def test_dspark_parallel_config_disables_eplb_atomically():
assert draft_config.eplb_config.num_redundant_experts == 0
assert not draft_config.enable_elastic_ep
assert draft_config.eplb_config is not target_config.eplb_config
@pytest.mark.parametrize("pcp_size", [1, 4])
@pytest.mark.parametrize("dcp_size", [1, 4])
@pytest.mark.parametrize("use_mla", [False, True])
def test_draft_context_parallelism_without_changing_target(
monkeypatch, pcp_size, dcp_size, use_mla
):
target_parallel = ParallelConfig(
tensor_parallel_size=4,
prefill_context_parallel_size=pcp_size,
decode_context_parallel_size=dcp_size,
cp_kv_cache_interleave_size=16,
distributed_executor_backend="mp",
)
target_config = SimpleNamespace(
parallel_config=target_parallel,
speculative_config=SimpleNamespace(
draft_model_config=SimpleNamespace(use_mla=use_mla)
),
)
class CapturedConfig(Exception):
pass
def capture_init(self, config, device):
raise CapturedConfig(config)
monkeypatch.setattr(DraftModelSpeculator, "__init__", capture_init)
with pytest.raises(CapturedConfig) as captured:
DFlashSpeculator(target_config, device=None)
draft_parallel = captured.value.args[0].parallel_config
assert draft_parallel.tensor_parallel_size == 4
assert draft_parallel.prefill_context_parallel_size == 1
assert draft_parallel.decode_context_parallel_size == (dcp_size if use_mla else 1)
assert draft_parallel.cp_kv_cache_interleave_size == 16
assert target_parallel.prefill_context_parallel_size == pcp_size
assert target_parallel.decode_context_parallel_size == dcp_size
assert target_parallel.cp_kv_cache_interleave_size == 16
@pytest.mark.parametrize("configured_dcp", [1, 4])
def test_attention_uses_draft_dcp_setting_inside_target_process_group(
monkeypatch, configured_dcp
):
import vllm.config as config_module
from vllm.distributed import parallel_state
from vllm.v1.attention.backend import AttentionImplBase
config = SimpleNamespace(
parallel_config=SimpleNamespace(decode_context_parallel_size=configured_dcp)
)
monkeypatch.setattr(
config_module, "get_current_vllm_config_or_none", lambda: config
)
monkeypatch.setattr(
parallel_state, "_DCP", SimpleNamespace(world_size=4, rank_in_group=2)
)
impl = AttentionImplBase()
assert impl.dcp_world_size == configured_dcp
assert impl.dcp_rank == (0 if configured_dcp == 1 else 2)
@pytest.mark.parametrize("draft_dcp_size", [1, 4])
def test_runner_marks_only_replicated_draft_caches(monkeypatch, draft_dcp_size):
spec = FullAttentionSpec(
block_size=16, num_kv_heads=1, head_size=64, dtype=torch.bfloat16
)
monkeypatch.setattr(
model_runner, "get_kv_cache_spec", lambda _: {"target": spec, "draft": spec}
)
runner = object.__new__(model_runner.GPUModelRunner)
runner.vllm_config = None
runner.dcp_size = 4
runner.speculator = object.__new__(DFlashSpeculator)
runner.speculator.dcp_size = draft_dcp_size
runner.speculator.draft_attn_layer_names = {"draft"}
specs = runner.get_kv_cache_spec()
assert specs["target"].dcp_sharded
assert specs["draft"].dcp_sharded == (draft_dcp_size == 4)
@@ -88,6 +88,7 @@ def test_pcp_draft_metadata_keeps_graph_padding_in_decode(cg_mode):
query_start_loc=torch.tensor([0, 1, 2, 2, 2], dtype=torch.int32),
seq_lens=torch.tensor([11, 21, 0, 0], dtype=torch.int32),
)
speculator.dcp_size = 1
speculator.block_tables = SimpleNamespace(
cp_size=1,
input_block_tables=[torch.zeros(4, 1, dtype=torch.int32)],
@@ -97,7 +98,9 @@ def test_pcp_draft_metadata_keeps_graph_padding_in_decode(cg_mode):
speculator.attn_groups = [
[
SimpleNamespace(
get_metadata_builder=lambda _: SimpleNamespace(build=build),
get_metadata_builder=lambda _: SimpleNamespace(
build=build, supports_update_block_table=False
),
layer_names=["draft"],
)
]
@@ -183,6 +186,7 @@ def test_speculator_uses_draft_model_hidden_size(monkeypatch, hc_mult, expected)
parallel_config=SimpleNamespace(
data_parallel_size=1,
data_parallel_rank=0,
decode_context_parallel_size=1,
),
)
+9 -7
View File
@@ -171,22 +171,23 @@ def test_block_tables_skip_custom_slot_mapping_groups():
@pytest.mark.parametrize("cp_rank", range(4))
def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
"""DCP interleave is expressed in logical-block token coordinates."""
"""Only sharded groups use DCP interleave in logical-block coordinates."""
device = torch.device("cuda")
block_tables = BlockTables(
block_sizes=[128],
block_sizes=[128, 128],
max_num_reqs=1,
max_num_batched_tokens=1024,
max_num_blocks_per_group=[2],
max_num_blocks_per_group=[2, 8],
device=device,
kernel_block_sizes=[64],
kernel_block_sizes=[64, 64],
dcp_sharded=[True, False],
cp_size=4,
cp_rank=cp_rank,
cp_interleave=128,
)
block_tables.append_block_ids(
req_index=0,
new_block_ids=([5, 9],),
new_block_ids=([5, 9], list(range(10, 18))),
overwrite=True,
)
block_tables.apply_staged_writes()
@@ -199,7 +200,7 @@ def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
query_start_loc,
positions,
num_tokens_padded=1024,
)[0]
)
expected = torch.full((1024,), -1, dtype=torch.int64, device=device)
first_start = cp_rank * 128
@@ -210,7 +211,8 @@ def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
expected[second_start : second_start + 128] = torch.arange(
9 * 128, 10 * 128, dtype=torch.int64, device=device
)
assert torch.equal(actual, expected)
assert torch.equal(actual[0], expected)
assert torch.equal(actual[1], positions + 10 * 128)
def test_v1_block_table_move_row_clears_vacated_row():
+56
View File
@@ -372,6 +372,62 @@ def test_swap_states_in_input_batch(device: str, batch_size: int, swap_list: lis
_compare_objs(input_batch, ref_input_batch)
@pytest.mark.parametrize("device", DEVICES)
def test_condense_clears_stale_allowed_token_ids_mask(device: str):
"""condense() must clear the mask row a constrained request is moved out
of. Otherwise a later request that reuses that row without
allowed_token_ids inherits the stale whitelist.
See https://github.com/vllm-project/vllm/issues/43894.
"""
batch_size = 4
allowed_token_id = 13
input_batch = InputBatch(
max_num_reqs=batch_size,
max_model_len=1024,
max_num_batched_tokens=1024,
device=torch.device(device),
vocab_size=VOCAB_SIZE,
block_sizes=[1],
kernel_block_sizes=[1],
max_num_blocks_per_req=[1024],
)
def _make_req(suffix: int, allowed_token_ids=None) -> CachedRequestState:
return CachedRequestState(
req_id=f"req_id_{suffix}",
prompt_token_ids=[1, 2, 3],
sampling_params=SamplingParams(allowed_token_ids=allowed_token_ids),
pooling_params=None,
mm_features=[],
block_ids=([],),
generator=None,
num_computed_tokens=0,
output_token_ids=[],
)
# Only req_id_2 is constrained, and it lands on the highest row.
assert input_batch.add_request(_make_req(0)) == 0
assert input_batch.add_request(_make_req(1)) == 1
assert input_batch.add_request(_make_req(2, [allowed_token_id])) == 2
mask = input_batch.allowed_token_ids_mask_cpu_tensor
assert mask is not None
# The constrained row masks every token except the single allowed id.
assert not mask[2][allowed_token_id].item()
assert int(mask[2].sum().item()) == VOCAB_SIZE - 1
# Free row 0, then condense: req_id_2 slides from row 2 down into row 0.
input_batch.remove_request("req_id_0")
input_batch.condense()
assert input_batch.req_id_to_index["req_id_2"] == 0
assert int(mask[2].sum().item()) == 0
# A new unrestricted request reuses row 2 and must stay unconstrained.
assert input_batch.add_request(_make_req(3)) == 2
assert "req_id_3" not in input_batch.has_allowed_token_ids
assert int(mask[2].sum().item()) == 0
@pytest.mark.parametrize("device", DEVICES)
def test_swap_states_preserves_allowed_token_ids_mask(device: str):
"""swap_states() must exchange the two allowed_token_ids_mask rows.
+18
View File
@@ -75,6 +75,24 @@ def test_validate_config_rejects_full_graph_for_prefills():
)
@pytest.mark.parametrize("draft_uses_mla", [False, True])
@pytest.mark.parametrize("dcp_size", [1, 2, 4])
def test_validate_config_dcp_for_dspark(draft_uses_mla, dcp_size):
config = _make_config(CUDAGraphMode.NONE)
config.parallel_config.prefill_context_parallel_size = 4
config.parallel_config.decode_context_parallel_size = dcp_size
config.model_config.hf_text_config.index_topk = 1
config.speculative_config = SimpleNamespace(
use_dspark=lambda: True,
draft_model_config=SimpleNamespace(use_mla=draft_uses_mla),
)
if draft_uses_mla and dcp_size == 2:
with pytest.raises(NotImplementedError, match="requires DCP=1 or DCP=PCP"):
PCPManager.validate_config(config, supports_mm_inputs=False)
return
PCPManager.validate_config(config, supports_mm_inputs=False)
def test_replicated_decode_piecewise_graph_padding(monkeypatch):
manager = PCPManager(
pcp_world_size=2,
-1
View File
@@ -113,7 +113,6 @@ SEPARATE_GROUPS = [
EXCLUDE = [
r"vllm/model_executor/models/[kK]",
r"vllm/model_executor/models/[uU]",
r"vllm/model_executor/models/[vV]",
r"vllm/model_executor/models/[wW]",
r"vllm/model_executor/models/[zZ]",
+51 -23
View File
@@ -1870,28 +1870,39 @@ def _mhc_delayed_pre_tail(
_OPS_REGISTERED = False
_AITER_SITUV2_ACT_ENV = {
"a8w4": "AITER_SITUV2_A8W4",
"a4w4": "AITER_SITUV2_A4W4",
}
def _resolve_situv2_activation() -> str:
"""VLLM_ROCM_USE_AITER_MOE_SITUV2 -> a4w4 | a8w4 | a16w4.
auto (default) and the legacy 1 mean a4w4; legacy 0 means a16w4.
"""
value = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2.lower()
if value in ("auto", "1"):
return "a4w4"
if value == "0":
return "a16w4"
return value
def _sync_aiter_situv2_moe_env() -> None:
"""Mirror the SiTUv2 MoE toggle into AITER's a4w4 dispatch env.
"""Set the AITER_SITUV2_* env matching VLLM_ROCM_USE_AITER_MOE_SITUV2.
AITER selects afp8 vs afp4 activation kernels via AITER_SITUV2_A8W4 /
AITER_SITUV2_A4W4 (see ROCm/aiter fused_moe.py, A8W4 checked first).
When VLLM_ROCM_USE_AITER_MOE_SITUV2 is enabled we route to a4w4
(afp4_wfp4_fp4 kernels) and clear any legacy AITER_SITUV2_A8W4 override.
Requires AITER with ROCm/aiter#4463 (first tagged in v0.1.20): a4w4
dispatch plus kimik3_a4w4_{un,}tuned_fmoe.csv. Older AITER still runs
afp4 FlyDSL but falls back to heuristic configs, which is not the
tuned a4w4 path this recipe is meant to use.
AITER reads AITER_SITUV2_A8W4 / AITER_SITUV2_A4W4 and defaults to a16w4,
so exactly one is set for a8w4/a4w4 and both are cleared for a16w4.
"""
import os
import vllm.envs as envs
if envs.VLLM_ROCM_USE_AITER_MOE_SITUV2:
os.environ["AITER_SITUV2_A4W4"] = "1"
os.environ.pop("AITER_SITUV2_A8W4", None)
else:
os.environ.pop("AITER_SITUV2_A4W4", None)
selected = _AITER_SITUV2_ACT_ENV.get(_resolve_situv2_activation())
for name in _AITER_SITUV2_ACT_ENV.values():
if name == selected:
os.environ[name] = "1"
else:
os.environ.pop(name, None)
class rocm_aiter_ops:
@@ -1917,7 +1928,8 @@ class rocm_aiter_ops:
VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: Controls FP4 assembly GEMM.
VLLM_ROCM_USE_AITER_TRITON_ROPE: Controls Triton rotary embeddings.
VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: Controls shared expert fusion.
VLLM_ROCM_USE_AITER_MOE_SITUV2: Controls SiTUv2 FlyDSL MoE (a4w4).
VLLM_ROCM_USE_AITER_MOE_SITUV2: SiTUv2 FlyDSL MoE activation
dtype (a16w4 | a8w4 | a4w4).
VLLM_ROCM_USE_AITER_TRITON_GEMM: Controls Triton unquantized GEMM.
Note:
@@ -1987,7 +1999,7 @@ class rocm_aiter_ops:
# TODO: Consolidate under VLLM_ROCM_USE_AITER_ROPE
_TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE
_MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
_MOE_SITUV2 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2
_MOE_SITUV2 = _resolve_situv2_activation()
# TODO: Consolidate under _LINEAR_ENABLED
_TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM
# Lazily probed: whether aiter.topk_softmax supports the
@@ -2016,7 +2028,7 @@ class rocm_aiter_ops:
cls._FP4_GEMM_DYNAMIC_QUANT_ASM = envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM
cls._TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE
cls._MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
cls._MOE_SITUV2 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2
cls._MOE_SITUV2 = _resolve_situv2_activation()
_sync_aiter_situv2_moe_env()
cls._TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM
cls._MOE_DISPATCH_POLICY = envs.VLLM_ROCM_AITER_MOE_DISPATCH_POLICY
@@ -2132,9 +2144,25 @@ class rocm_aiter_ops:
@classmethod
@if_aiter_supported
def is_fused_moe_situv2_enabled(cls) -> bool:
# _MOE_SITUV2 is a variant of aiter fused moe, so aiter
# fused moe must be enabled as well.
return cls.is_fused_moe_enabled() and cls._MOE_SITUV2
"""True when a low-precision (a8w4/a4w4) SiTUv2 activation is selected."""
return cls.is_fused_moe_enabled() and cls._MOE_SITUV2 != "a16w4"
@classmethod
@if_aiter_supported
def get_fused_moe_situv2_activation(cls) -> str:
"""SiTUv2 activation dtype name: a16w4, a8w4 or a4w4."""
return cls._MOE_SITUV2
@classmethod
@if_aiter_supported
def is_fused_moe_situv2_gate_up_interleaved(cls) -> bool:
"""Whether the SiTUv2 kernels expect gate/up-interleaved w13.
The a8w4 FlyDSL stage-1 kernels (_gui_) are interleaved; a16w4 and
a4w4 are separated. Weight shuffle (oracle/mxfp4.py) and gate_mode
(experts/rocm_aiter_moe.py) must both use this.
"""
return cls._MOE_SITUV2 == "a8w4"
@classmethod
@if_aiter_supported
+3 -1
View File
@@ -2148,7 +2148,9 @@ def get_samples(
dataset_path=args.dataset_path, disable_shuffle=args.disable_shuffle
)
# For the "sonnet" dataset, formatting depends on the backend.
if args.backend == "openai-chat":
# Chat-style backends leave templating to the server; completions
# backends need the prompt rendered client side.
if args.backend in ("openai-chat", "openai-responses"):
input_requests = sonnet_dataset.sample(
num_requests=args.num_prompts,
input_len=args.sonnet_input_len,
+179 -1
View File
@@ -447,6 +447,178 @@ async def async_request_openai_chat_completions(
return output
def _extract_sse_data(message: str) -> str | None:
"""Extract the ``data`` payload of a single SSE message.
The Responses API stream labels every message with an ``event:`` line
before its ``data:`` line, so the payload cannot be recovered by
stripping a leading ``data: `` the way the Completions and Chat
Completions streams allow.
Args:
message: One complete SSE message, without its trailing blank line.
Returns:
The payload, with multi-line payloads joined by newlines as required
by the SSE spec, or None if the message carries no ``data:`` line.
"""
# NOTE: split on "\n" rather than using splitlines(), which also breaks on
# U+2028, U+2029 and U+0085. Those are legal inside a JSON string and the
# server emits them unescaped, so splitlines() would truncate the payload.
data_lines = [
line.removeprefix("data:").removeprefix(" ")
for line in message.split("\n")
if line.startswith("data:")
]
if not data_lines:
return None
return "\n".join(data_lines)
# Responses events that carry generated tokens. Reasoning deltas are included
# because the server is already decoding when it emits them, so excluding them
# would report the whole reasoning phase as time to first token.
_RESPONSES_TOKEN_DELTA_EVENTS = frozenset(
{
"response.output_text.delta",
"response.reasoning_text.delta",
}
)
# Responses events that end a request. Truncation at the output length limit
# is a normal benchmark outcome: vLLM reports it as `response.completed` with
# `status: "incomplete"`, while the spec also defines a dedicated
# `response.incomplete` terminal event.
_RESPONSES_TERMINAL_EVENTS = frozenset(
{
"response.completed",
"response.incomplete",
}
)
# The spec's failure terminals. vLLM does not emit them today, but
# `vllm bench serve` can be pointed at other Responses servers.
_RESPONSES_ERROR_EVENTS = frozenset({"error", "response.failed"})
async def async_request_openai_responses(
request_func_input: RequestFuncInput,
session: aiohttp.ClientSession,
pbar: tqdm | None = None,
) -> RequestFuncOutput:
api_url = request_func_input.api_url
_validate_api_url(api_url, "OpenAI Responses API", "responses")
payload = {
"model": request_func_input.model_name
if request_func_input.model_name
else request_func_input.model,
"input": request_func_input.prompt,
"max_output_tokens": request_func_input.output_len,
"stream": True,
# `store` defaults to true. A server started with
# VLLM_ENABLE_RESPONSES_API_STORE=1 keeps every stored response for
# the lifetime of the process, which would grow its heap over a long
# benchmark run.
"store": False,
}
_update_payload_common(payload, request_func_input)
headers = _get_headers("application/json")
_update_headers_common(headers, request_func_input)
output = RequestFuncOutput()
output.prompt_len = request_func_input.prompt_len
# Only output text is accumulated. Reasoning text is timed but not
# returned, matching how `openai-chat` handles `DeltaMessage.reasoning`.
generated_text = ""
stream_error: str | None = None
saw_terminal_event = False
first_chunk_received = False
st = time.perf_counter()
output.start_time = st
most_recent_timestamp = st
try:
async with session.post(url=api_url, json=payload, headers=headers) as response:
if response.status == 200:
handler = StreamedResponseHandler()
async for chunk_bytes in response.content.iter_any():
for message in handler.add_chunk(chunk_bytes):
# NOTE: `_extract_sse_data` returns None for SSE
# comments, which servers may send as keep-alives.
chunk = _extract_sse_data(message)
if not chunk or chunk == "[DONE]":
continue
timestamp = time.perf_counter()
data = json.loads(chunk)
event_type = data.get("type")
if event_type in _RESPONSES_TOKEN_DELTA_EVENTS:
# First token
if not first_chunk_received:
first_chunk_received = True
output.ttft = timestamp - st
# Decoding phase
else:
output.itl.append(timestamp - most_recent_timestamp)
if event_type == "response.output_text.delta":
generated_text += data.get("delta") or ""
# Only token events advance the end of the
# request, so `latency - ttft` stays equal to
# `sum(itl)` and TPOT means what serve.py says.
most_recent_timestamp = timestamp
elif event_type in _RESPONSES_TERMINAL_EVENTS:
saw_terminal_event = True
usage = (data.get("response") or {}).get("usage") or {}
if (n_output := usage.get("output_tokens")) is not None:
output.output_tokens = n_output
if (n_input := usage.get("input_tokens")) is not None:
output.prompt_len = n_input
elif event_type in _RESPONSES_ERROR_EVENTS or data.get("error"):
stream_error = json.dumps(data.get("error") or data)
output.generated_text = generated_text
output.latency = most_recent_timestamp - st
if stream_error is not None:
output.success = False
output.error = stream_error
elif not first_chunk_received:
output.success = False
output.error = (
"Never received a token delta to calculate TTFT."
"This response will be marked as failed!"
)
elif not saw_terminal_event:
# Without the terminal event the response is truncated and
# its usage block is missing.
output.success = False
output.error = (
"Stream ended without a terminal event."
"This response will be marked as failed!"
)
else:
output.success = True
else:
output.error = response.reason or ""
output.success = False
except Exception:
output.success = False
exc_info = sys.exc_info()
output.error = "".join(traceback.format_exception(*exc_info))
if pbar:
pbar.update(1)
return output
async def async_request_openai_audio(
request_func_input: RequestFuncInput,
session: aiohttp.ClientSession,
@@ -920,6 +1092,7 @@ ASYNC_REQUEST_FUNCS: dict[str, RequestFunc] = {
"vllm": async_request_openai_completions,
"openai": async_request_openai_completions,
"openai-chat": async_request_openai_chat_completions,
"openai-responses": async_request_openai_responses,
"openai-audio": async_request_openai_audio,
"openai-embeddings": async_request_openai_embeddings,
"openai-embeddings-chat": async_request_openai_embeddings_chat,
@@ -947,5 +1120,10 @@ POOLING_BACKENDS = {
OPENAI_COMPATIBLE_BACKENDS = [
k
for k, v in ASYNC_REQUEST_FUNCS.items()
if v in (async_request_openai_completions, async_request_openai_chat_completions)
if v
in (
async_request_openai_completions,
async_request_openai_chat_completions,
async_request_openai_responses,
)
]
+5
View File
@@ -2216,6 +2216,11 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
"Sampling parameters are only supported by openai-compatible backends."
)
# The Responses API accepts every sampling parameter above except
# min_p, which it would silently drop as an unknown field.
if args.backend == "openai-responses" and "min_p" in sampling_params:
raise ValueError("--min-p is not supported by the Responses API.")
if "temperature" not in sampling_params:
print(
"WARNING: vllm bench serve no longer sets temperature==0 (greedy) "
+20
View File
@@ -1718,6 +1718,26 @@ class VllmConfig:
)
self.compilation_config.mode = CompilationMode.NONE
# TODO: This is a stopgap that turns sequence parallelism / async TP
# off under batch invariance. Make the sequence-parallel reduce-scatter
# path batch-invariant instead so they can stay enabled (#56370).
# Sequence parallelism / async TP rewrite all_reduce + rms_norm into a
# reduce-scatter whose reduction order depends on the batch, so they
# are not batch-invariant. Decide this before the breakable-CUDA-graph
# default below, which declines to auto-enable when they are on.
pass_config = self.compilation_config.pass_config
if envs.VLLM_BATCH_INVARIANT and (
pass_config.enable_sp or pass_config.fuse_gemm_comms
):
logger.warning_once(
"Disabling sequence parallelism and async TP "
"(pass_config.enable_sp / fuse_gemm_comms) when "
"VLLM_BATCH_INVARIANT is enabled: the reduce-scatter path "
"is not batch-invariant (see vllm-project/vllm#56370)."
)
pass_config.enable_sp = False
pass_config.fuse_gemm_comms = False
breakable_cudagraph_enabled = self._maybe_enable_breakable_cudagraph()
if not breakable_cudagraph_enabled and (
@@ -42,6 +42,7 @@ The class provides the following primitives:
import enum
from abc import ABC, abstractmethod
from collections.abc import Callable, Iterable
from contextlib import AbstractContextManager
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, Literal
@@ -227,6 +228,13 @@ class KVConnectorBase_V1(ABC):
# Worker-side methods
# ==============================
def get_mem_pool_context(self) -> AbstractContextManager | None:
"""Return a custom KV cache allocation context, if configured.
Returning None uses the engine's default memory pool.
"""
return None
def bind_connector_metadata(self, connector_metadata: KVConnectorMetadata) -> None:
"""Set the connector metadata from the scheduler.
@@ -59,7 +59,6 @@ from vllm.platforms import current_platform
from vllm.utils.math_utils import cdiv
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.kv_cache_utils import (
dcp_world_size_for_kv_cache_spec,
resolve_dcp_kv_block_size,
)
from vllm.v1.kv_cache_interface import (
@@ -259,13 +258,7 @@ class DecodeBenchConnectorScheduler:
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
self.kv_cache_groups = kv_cache_config.kv_cache_groups
self.group_block_sizes = tuple(
resolve_dcp_kv_block_size(
group.kv_cache_spec,
dcp_world_size_for_kv_cache_spec(
group.kv_cache_spec,
dcp_world_size,
),
)
resolve_dcp_kv_block_size(group.kv_cache_spec, dcp_world_size)
for group in self.kv_cache_groups
)
@@ -10,7 +10,7 @@ from collections.abc import Collection
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from enum import IntEnum
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, Final
import msgspec
import numpy as np
@@ -70,6 +70,8 @@ from vllm.v1.worker.utils import select_common_block_size
logger = init_logger(__name__)
_BOOTSTRAP_MAX_ATTEMPTS: Final[int] = 3
try:
from mooncake.engine import TransferEngine
except ImportError:
@@ -1147,24 +1149,44 @@ class MooncakeConnectorWorker:
pp_rank=self.pp_rank,
addr=worker_addr,
)
while True:
timeout = envs.VLLM_MOONCAKE_CONNECTOR_TIMEOUT
max_attempts = _BOOTSTRAP_MAX_ATTEMPTS
backoff = 1.0
for attempt in range(1, max_attempts + 1):
try:
async with httpx.AsyncClient() as client:
async with httpx.AsyncClient(timeout=timeout) as client:
response = await client.post(url, json=payload.model_dump())
response.raise_for_status()
logger.debug("Successfully registered with bootstrap server at %s", url)
break
except httpx.ConnectError:
# Bootstrap server not ready, wait for a while and retry.
await asyncio.sleep(1)
return
except (httpx.ConnectError, httpx.TimeoutException) as e:
# Rank 0 may be busy registering a large memory segment.
if attempt == max_attempts:
break
logger.warning(
"Bootstrap registration attempt %d/%d for %s failed with %s: %s. "
"Retrying in %.1fs.",
attempt,
max_attempts,
payload,
type(e).__name__,
e,
backoff,
)
await asyncio.sleep(backoff)
backoff = min(backoff * 2, 10.0)
except Exception as e:
err_msg = (
e.response.text if isinstance(e, httpx.HTTPStatusError) else str(e)
)
logger.error(
logger.exception(
"Error registering %s with bootstrap server: %s", payload, err_msg
)
raise e
raise
raise RuntimeError(
f"Failed to register {payload} with the Mooncake bootstrap server at "
f"{url} after {max_attempts} attempts."
)
async def _mooncake_sender_listener(self, ready_event: threading.Event):
"""Background thread that listens for Mooncake requests, dispatches them
@@ -1800,11 +1822,29 @@ class MooncakeConnectorWorker:
if self.is_kv_consumer:
return
# httpx applies the timeout to each phase (connect, read, write, pool)
# rather than to the whole request, so budget for all four.
register_timeout = envs.VLLM_MOONCAKE_CONNECTOR_TIMEOUT
max_attempts = _BOOTSTRAP_MAX_ATTEMPTS
backoff_total = sum(min(2.0**i, 10.0) for i in range(max_attempts - 1))
ready_timeout = max_attempts * 4 * register_timeout + backoff_total + 30.0
ready_event = threading.Event()
asyncio.run_coroutine_threadsafe(
fut = asyncio.run_coroutine_threadsafe(
self._mooncake_sender_listener(ready_event), self.sender_loop
)
ready_event.wait() # Wait for listener ZMQ socket to be ready.
deadline = time.monotonic() + ready_timeout
while not ready_event.wait(timeout=1.0):
if fut.done():
fut.result()
raise RuntimeError(
"Mooncake sender listener exited before becoming ready."
)
if time.monotonic() > deadline:
fut.cancel()
raise RuntimeError(
"Mooncake sender listener did not become ready within "
f"{ready_timeout:.0f}s."
)
async def fetch_finished_recving_reqs(self) -> set[ReqId]:
finished_recving_reqs = self.finished_recving_reqs
@@ -97,14 +97,19 @@ class MooncakeBootstrapServer:
dp_entry.worker_addr[payload.tp_rank] = {}
tp_entry = dp_entry.worker_addr[payload.tp_rank]
if payload.pp_rank in tp_entry:
existing = tp_entry.get(payload.pp_rank)
if existing is not None:
# A client timeout can fire after the server recorded the
# registration so an identical retry must not be an error.
if existing == payload.addr:
return {"status": "ok"}
raise HTTPException(
status_code=400,
detail=(
f"Worker with dp_rank={payload.dp_rank}, "
f"tp_rank={payload.tp_rank}, pp_rank={payload.pp_rank} "
f"is already registered at "
f"{tp_entry[payload.pp_rank]}, "
f"{existing}, "
f"but still want to register at {payload.addr}"
),
)
@@ -13,6 +13,7 @@ enabling prefix caching via hash-based deduplication.
from collections import Counter
from collections.abc import Iterable, Sequence
from contextlib import AbstractContextManager
from typing import Any
import torch
@@ -322,6 +323,17 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
# Worker-side methods
# ============================================================
def get_mem_pool_context(self) -> AbstractContextManager | None:
"""Return a context manager for the custom MemPool, or None.
Called by the Worker before ``initialize_kv_cache`` so that KV
cache is allocated from the Mooncake-managed pool when
``custom_mem_pool`` is set in ``kv_connector_extra_config``.
"""
if self.connector_worker is None:
return None
return self.connector_worker.get_mem_pool_context()
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
assert self.connector_worker is not None
self.connector_worker.register_kv_caches(kv_caches)
@@ -19,13 +19,14 @@ import threading
import time
from collections.abc import Callable, Sequence
from concurrent.futures import Future, ThreadPoolExecutor
from contextlib import AbstractContextManager
from typing import Any, TypeVar
import torch
import zmq
import vllm.envs as envs
from vllm.config import VllmConfig
from vllm.config import ModelConfig, VllmConfig
from vllm.distributed import (
get_dcp_group,
get_pcp_group,
@@ -1414,6 +1415,41 @@ class KVCacheStoreRecvingThread(KVTransferThread):
class MooncakeStoreWorker:
"""Worker-side component for MooncakeStoreConnector."""
@staticmethod
def _create_mem_pool(
extra_config: dict[str, Any], model_config: ModelConfig
) -> torch.cuda.MemPool | None:
pool_type = str(extra_config.get("custom_mem_pool") or "").upper()
if not pool_type:
return None
if pool_type not in ("NVLINK", "BAREX"):
raise ValueError(
f"Unsupported custom_mem_pool={pool_type!r}, "
"expected 'NVLINK' or 'BAREX'"
)
if model_config.enable_sleep_mode or model_config.enable_cumem_allocator:
raise ValueError(
"custom_mem_pool is incompatible with enable_sleep_mode "
"or enable_cumem_allocator; CuMemAllocator cannot manage "
"allocations from the custom pool."
)
try:
if pool_type == "NVLINK":
from mooncake.allocator import NVLinkAllocator as allocator_cls
else: # pool_type == "BAREX"
from mooncake.allocator import BarexAllocator as allocator_cls
except ImportError as e:
raise ImportError(
f"custom_mem_pool={pool_type!r} requires "
"mooncake-transfer-engine>=0.3.8. Please upgrade Mooncake."
) from e
device = torch.device("cuda", torch.accelerator.current_device_index())
allocator = allocator_cls.get_allocator(device)
mem_pool = torch.cuda.MemPool(allocator.allocator())
logger.info("Using Mooncake custom memory pool: %s", pool_type)
return mem_pool
def __init__(
self,
vllm_config: VllmConfig,
@@ -1471,6 +1507,10 @@ class MooncakeStoreWorker:
# Initialize MooncakeDistributedStore with its own TransferEngine
store_config = MooncakeStoreConfig.load_from_config()
# Validate and create the custom memory pool before initializing the
# store so failures cannot leak a partially initialized store handle.
self._mem_pool = self._create_mem_pool(extra_config, model_config)
self.store = MooncakeDistributedStore()
local_ip = get_ip()
local_hostname = rdma_utils.get_requester_local_hostname(local_ip)
@@ -1741,6 +1781,13 @@ class MooncakeStoreWorker:
)
return token_dbs
def get_mem_pool_context(self) -> AbstractContextManager | None:
"""Return a context manager for the custom MemPool, or None if
no custom pool is configured."""
if self._mem_pool is None:
return None
return torch.cuda.use_mem_pool(self._mem_pool)
def _spec_tp_replication_factor(self, spec: KVCacheSpec) -> int:
if self.dcp_size > 1:
return 1
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
from collections.abc import Callable, Iterable
from contextlib import AbstractContextManager
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, cast
@@ -294,6 +295,26 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
# ==============================
# Worker-side methods
# ==============================
def get_mem_pool_context(self) -> AbstractContextManager | None:
"""Forward a custom KV cache memory pool from a child connector.
KV cache is allocated once and can only live in one pool, so at
most one child may provide a context.
"""
found: list[tuple[str, AbstractContextManager]] = []
for connector in self._connectors:
if (ctx := connector.get_mem_pool_context()) is not None:
found.append((type(connector).__name__, ctx))
if len(found) > 1:
names = [name for name, _ in found]
raise ValueError(
f"Multiple connectors provide a KV cache memory pool {names}; "
"KV cache is allocated once and can only live in one pool. "
"Configure custom_mem_pool on at most one connector."
)
return found[0][1] if found else None
def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None:
for c in self._connectors:
c.start_load_kv(forward_context, **kwargs)
@@ -839,6 +839,8 @@ class NixlBaseConnectorWorker:
self._engine_ttl: float = vllm_config.kv_transfer_config.get_from_extra_config(
"engine_ttl", 3600.0
)
# Map of pull peer (host, port) -> engine_id currently serving it.
self._engine_by_address: dict[tuple[str, int], EngineId] = {}
self.model_config = vllm_config.model_config
@@ -1300,6 +1302,7 @@ class NixlBaseConnectorWorker:
self._remote_agents[eid] = remote_agents
self._engine_clock_offset[eid] = clock_offset
self._engine_last_active[eid] = time.perf_counter()
self._track_remote_engine_replacement(eid, host, port)
except Exception as e:
self._log_failure(
failure_type="handshake_setup_failed",
@@ -2953,6 +2956,7 @@ class NixlBaseConnectorWorker:
# Handle timeout to avoid stranding blocks on remote.
self._reap_expired_send_leases(done_sending)
self._cleanup_replaced_remote_engines()
return KVConnectorTransferResults(
finished_sending=done_sending,
finished_recving=done_recving,
@@ -3541,14 +3545,53 @@ class NixlBaseConnectorWorker:
and meta.remote is not None
}
def _track_remote_engine_replacement(
self, engine_id: EngineId, host: str, port: int
) -> None:
"""Record the engine currently serving a pull peer address."""
# TODO: Also handle push mode, which handshakes in both directions.
if self._TRANSFER_MODE != "pull":
return
previous = self._engine_by_address.get((host, port))
self._engine_by_address[(host, port)] = engine_id
if previous is None or previous == engine_id:
return
logger.info(
"Remote engine at %s:%d changed from %s to %s.",
host,
port,
previous,
engine_id,
)
def _cleanup_replaced_remote_engines(self) -> None:
"""Release replaced pull peers once requests and handshakes have drained."""
if self._TRANSFER_MODE != "pull":
return
with self._handshake_lock:
if self._handshake_futures:
return
replaced = self._remote_agents.keys() - self._engine_by_address.values()
if not replaced:
return
# _recving_metadata is only accessed from this (main) thread.
busy = {
meta.remote.engine_id
for meta in self._recving_metadata.values()
if meta.remote is not None
}
for engine_id in replaced - busy:
self._cleanup_remote_engine(engine_id, log_eviction=False)
logger.info("Released NIXL state for replaced remote engine %s.", engine_id)
def _cleanup_remote_engine(
self, engine_id: EngineId, *, log_eviction: bool = True
) -> None:
"""Remove all state for a single remote engine.
Releases NIXL resources (dlist handles, remote agents) and clears
all per-engine data structures. Used by both TTL eviction and
shutdown.
all per-engine data structures. Used by TTL eviction, replaced peer
cleanup and shutdown.
"""
assert engine_id in self._remote_agents
@@ -3558,6 +3601,9 @@ class NixlBaseConnectorWorker:
# Pop under the handshake lock; NIXL teardown stays outside it.
with self._handshake_lock:
agents = self._remote_agents.pop(engine_id)
for address, eid in list(self._engine_by_address.items()):
if eid == engine_id:
del self._engine_by_address[address]
for agent_name in agents.values():
self.nixl_wrapper.remove_remote_agent(agent_name)
@@ -84,7 +84,14 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
# Start transfers for requests whose handshakes have now finished.
while not self._ready_requests.empty():
self._read_blocks_for_req(*self._ready_requests.get_nowait())
req_id, meta = self._ready_requests.get_nowait()
assert meta.remote is not None
if meta.remote.engine_id not in self._remote_agents:
# The engine was released after its handshake completed, so
# handshake again. This fails if the engine is gone.
self._background_nixl_handshake(req_id, meta.remote.engine_id, meta)
continue
self._read_blocks_for_req(req_id, meta)
if self.pcp_rank > 0 and not self.pcp_dcp_sharded:
# Replicated-KV PCP: only PCP rank 0 serves the KV, so this rank
@@ -15,6 +15,12 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorRole,
SupportsHMA,
)
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorPromMetrics,
KVConnectorStats,
PromMetric,
PromMetricT,
)
from vllm.logger import init_logger
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.outputs import KVConnectorOutput
@@ -25,6 +31,10 @@ from vllm.v1.simple_kv_offload.manager import (
from vllm.v1.simple_kv_offload.metadata import (
SimpleCPUOffloadMetadata,
)
from vllm.v1.simple_kv_offload.metrics import (
SimpleCPUOffloadPromMetrics,
SimpleCPUOffloadStats,
)
from vllm.v1.simple_kv_offload.worker import (
SimpleCPUOffloadWorker,
)
@@ -162,6 +172,7 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
hash_block_size=hash_block_size,
lazy_offload=lazy_offload,
disk_capacity_bytes=disk_capacity_bytes if disk_mode else 0,
use_page_cache=use_page_cache if disk_mode else False,
)
elif role == KVConnectorRole.WORKER:
self.worker_handler = SimpleCPUOffloadWorker(
@@ -307,6 +318,38 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
return self.scheduler_manager.get_boundary_store_stats()
return None
def get_kv_connector_stats(self) -> KVConnectorStats | None:
# Worker-side callers may invoke this hook on connectors constructed
# via __new__ (as in test_worker.py), which lack scheduler_manager.
scheduler_manager: SimpleCPUOffloadScheduler | None = getattr(
self, "scheduler_manager", None
)
if scheduler_manager is not None:
return scheduler_manager.get_stats()
return None
@classmethod
def build_kv_connector_stats(
cls, data: dict[str, Any] | None = None
) -> KVConnectorStats | None:
return (
SimpleCPUOffloadStats(data=data)
if data is not None
else SimpleCPUOffloadStats()
)
@classmethod
def build_prom_metrics(
cls,
vllm_config: VllmConfig,
metric_types: dict[type[PromMetric], type[PromMetricT]],
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
) -> KVConnectorPromMetrics:
return SimpleCPUOffloadPromMetrics(
vllm_config, metric_types, labelnames, per_engine_labelvalues
)
def take_events(self) -> Iterable[KVCacheEvent]:
if self.scheduler_manager is not None:
return self.scheduler_manager.take_events()

Some files were not shown because too many files have changed in this diff Show More