Merge remote-tracking branch 'upstream/main' into patches
This commit is contained in:
@@ -23,8 +23,7 @@ steps:
|
||||
commands:
|
||||
- >-
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'uv pip install "gpt-oss[eval]==0.0.5" &&
|
||||
cd tests &&
|
||||
'cd tests &&
|
||||
pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-xpu.txt'
|
||||
- label: LM Eval Large Models(2 GPUs)
|
||||
key: lm-eval-large-models-2-gpus
|
||||
|
||||
@@ -1287,7 +1287,6 @@ steps:
|
||||
- vllm/platforms/rocm.py
|
||||
- tests/evals/gpt_oss/
|
||||
commands:
|
||||
- uv pip install --system 'gpt-oss[eval]==0.0.5'
|
||||
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx942.txt
|
||||
|
||||
- label: ":amd: (MI300) LM Eval Small Models Distributed"
|
||||
@@ -3584,7 +3583,6 @@ steps:
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
- uv pip install --system 'gpt-oss[eval]==0.0.5'
|
||||
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx950.txt
|
||||
|
||||
- label: ":amd: (MI355 DPX) LM Eval Watermarking"
|
||||
|
||||
@@ -829,7 +829,6 @@ steps:
|
||||
- vllm/model_executor/layers/quantization
|
||||
- tests/evals/gpt_oss/
|
||||
commands:
|
||||
- uv pip install --system 'gpt-oss[eval]==0.0.5'
|
||||
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-h100.txt
|
||||
|
||||
- label: ":nvidia: (B200) GPQA Eval (GPT-OSS)"
|
||||
@@ -843,7 +842,6 @@ steps:
|
||||
- vllm/model_executor/layers/quantization
|
||||
- tests/evals/gpt_oss/
|
||||
commands:
|
||||
- uv pip install --system 'gpt-oss[eval]==0.0.5'
|
||||
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-b200.txt
|
||||
mirror:
|
||||
amd:
|
||||
@@ -865,7 +863,6 @@ steps:
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
- uv pip install --system 'gpt-oss[eval]==0.0.5'
|
||||
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx950.txt
|
||||
|
||||
- label: ":nvidia: (B200) Qwen3.8-Flash-Next-FP8 Accuracy Eval"
|
||||
@@ -940,7 +937,6 @@ steps:
|
||||
- vllm/model_executor/layers/quantization
|
||||
- tests/evals/gpt_oss/
|
||||
commands:
|
||||
- uv pip install --system 'gpt-oss[eval]==0.0.5'
|
||||
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-spark.txt
|
||||
|
||||
- label: ":nvidia: (H200 MIG 35GB) LM Eval KV-Offload"
|
||||
|
||||
@@ -98,9 +98,11 @@ steps:
|
||||
commands:
|
||||
- pytest -v -s v1/e2e/spec_decode/eagle/
|
||||
|
||||
- label: ":nvidia: (H200 MIG 35GB) Spec Decode Speculators + MTP"
|
||||
key: spec-decode-speculators-mtp
|
||||
timeout_in_minutes: 50
|
||||
# MTP correctness is one parametrized test split into per-model folders;
|
||||
# each job below runs whole directories.
|
||||
- label: ":nvidia: (H200 MIG 35GB) Spec Decode Speculators"
|
||||
key: spec-decode-speculators
|
||||
timeout_in_minutes: 30
|
||||
device: h200_35gb
|
||||
source_file_dependencies:
|
||||
- vllm/v1/spec_decode/
|
||||
@@ -110,13 +112,103 @@ steps:
|
||||
- tests/v1/e2e/spec_decode/
|
||||
commands:
|
||||
- pytest -v -s v1/e2e/spec_decode/speculators/
|
||||
- pytest -v -s v1/e2e/spec_decode/mtp/
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI300) Spec Decode Speculators + MTP"
|
||||
label: ":amd: (MI300) Spec Decode Speculators"
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 75
|
||||
timeout_in_minutes: 40
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- vllm/v1/spec_decode/
|
||||
- vllm/v1/worker/gpu/spec_decode/
|
||||
- vllm/model_executor/model_loader/
|
||||
- vllm/v1/sample/
|
||||
- vllm/model_executor/layers/
|
||||
- vllm/transformers_utils/configs/speculators/
|
||||
- tests/v1/e2e/spec_decode/
|
||||
- vllm/platforms/rocm.py
|
||||
|
||||
- label: ":nvidia: (H200 MIG 35GB) Spec Decode MTP Gemma4"
|
||||
key: spec-decode-mtp-gemma4
|
||||
timeout_in_minutes: 30
|
||||
device: h200_35gb
|
||||
source_file_dependencies:
|
||||
- vllm/v1/spec_decode/
|
||||
- vllm/v1/worker/gpu/spec_decode/
|
||||
- vllm/v1/attention/backends/
|
||||
- vllm/transformers_utils/configs/speculators/
|
||||
- tests/v1/e2e/spec_decode/
|
||||
commands:
|
||||
- pytest -v -s v1/e2e/spec_decode/mtp/gemma4/
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI300) Spec Decode MTP Gemma4"
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 40
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- vllm/v1/spec_decode/
|
||||
- vllm/v1/worker/gpu/spec_decode/
|
||||
- vllm/model_executor/model_loader/
|
||||
- vllm/v1/sample/
|
||||
- vllm/model_executor/layers/
|
||||
- vllm/transformers_utils/configs/speculators/
|
||||
- tests/v1/e2e/spec_decode/
|
||||
- vllm/platforms/rocm.py
|
||||
|
||||
- label: ":nvidia: (H200 MIG 35GB) Spec Decode MTP DeepSeek + MiMo"
|
||||
key: spec-decode-mtp-deepseek-mimo
|
||||
timeout_in_minutes: 30
|
||||
device: h200_35gb
|
||||
source_file_dependencies:
|
||||
- vllm/v1/spec_decode/
|
||||
- vllm/v1/worker/gpu/spec_decode/
|
||||
- vllm/v1/attention/backends/
|
||||
- vllm/transformers_utils/configs/speculators/
|
||||
- tests/v1/e2e/spec_decode/
|
||||
commands:
|
||||
- pytest -v -s v1/e2e/spec_decode/mtp/deepseek/
|
||||
- pytest -v -s v1/e2e/spec_decode/mtp/mimo/
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI300) Spec Decode MTP DeepSeek + MiMo"
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 40
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- vllm/v1/spec_decode/
|
||||
- vllm/v1/worker/gpu/spec_decode/
|
||||
- vllm/model_executor/model_loader/
|
||||
- vllm/v1/sample/
|
||||
- vllm/model_executor/layers/
|
||||
- vllm/transformers_utils/configs/speculators/
|
||||
- tests/v1/e2e/spec_decode/
|
||||
- vllm/platforms/rocm.py
|
||||
|
||||
- label: ":nvidia: (H200 MIG 35GB) Spec Decode MTP Qwen3.5"
|
||||
key: spec-decode-mtp-qwen3-5
|
||||
timeout_in_minutes: 30
|
||||
device: h200_35gb
|
||||
source_file_dependencies:
|
||||
- vllm/v1/spec_decode/
|
||||
- vllm/v1/worker/gpu/spec_decode/
|
||||
- vllm/v1/attention/backends/
|
||||
- vllm/transformers_utils/configs/speculators/
|
||||
- tests/v1/e2e/spec_decode/
|
||||
commands:
|
||||
- pytest -v -s v1/e2e/spec_decode/mtp/qwen3_5/
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI300) Spec Decode MTP Qwen3.5"
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 40
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
|
||||
@@ -13,7 +13,7 @@ else()
|
||||
FetchContent_Declare(
|
||||
flashkda
|
||||
GIT_REPOSITORY https://github.com/vllm-project/FlashKDA.git
|
||||
GIT_TAG b59532f1f464fbd536272780e30df5bf6a2ccc02
|
||||
GIT_TAG 17a037d98da546deb4591e967cf961a43c034d8b
|
||||
GIT_PROGRESS TRUE
|
||||
GIT_SUBMODULES cutlass
|
||||
)
|
||||
|
||||
@@ -729,6 +729,8 @@ COPY requirements/lint.txt requirements/lint.txt
|
||||
COPY requirements/test/cuda.in requirements/test/cuda.in
|
||||
COPY requirements/test/cuda.txt requirements/test/cuda.txt
|
||||
COPY requirements/dev.txt requirements/dev.txt
|
||||
# uv reads `[tool.uv] exclude-dependencies` from here when recompiling below.
|
||||
COPY pyproject.toml pyproject.toml
|
||||
COPY tools/use_existing_torch.py tools/use_existing_torch.py
|
||||
COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt
|
||||
RUN --mount=type=cache,target=/opt/uv/cache \
|
||||
@@ -1143,6 +1145,8 @@ COPY requirements/lint.txt requirements/lint.txt
|
||||
COPY requirements/test/cuda.in requirements/test/cuda.in
|
||||
COPY requirements/test/cuda.txt requirements/test/cuda.txt
|
||||
COPY requirements/dev.txt requirements/dev.txt
|
||||
# uv reads `[tool.uv] exclude-dependencies` from here when recompiling below.
|
||||
COPY pyproject.toml pyproject.toml
|
||||
COPY tools/use_existing_torch.py tools/use_existing_torch.py
|
||||
COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt
|
||||
RUN --mount=type=cache,target=/opt/uv/cache \
|
||||
|
||||
+31
-3
@@ -27,7 +27,7 @@
|
||||
# cache mounts (ccache, cargo registry) aren't exported by `--cache-to
|
||||
# type=registry`, so they don't survive a build landing on a different/fresh
|
||||
# builder. sccache's cache lives in S3 instead, so it does.
|
||||
FROM ubuntu:25.04 AS base
|
||||
FROM ubuntu:22.04 AS base
|
||||
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
|
||||
--mount=type=cache,target=/var/lib/apt,sharing=locked \
|
||||
@@ -52,11 +52,38 @@ RUN if [ "$USE_SCCACHE" = "1" ]; then \
|
||||
&& rm -rf sccache.tar.gz sccache-v0.8.1-${SCCACHE_ARCH}-unknown-linux-musl; \
|
||||
fi
|
||||
|
||||
######################### BINUTILS BUILD IMAGE #########################
|
||||
# Build a newer GNU assembler for the AMX-FP8 instruction set.
|
||||
FROM base AS binutils-build
|
||||
|
||||
ARG BINUTILS_VERSION=2.47
|
||||
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
|
||||
--mount=type=cache,target=/var/lib/apt,sharing=locked \
|
||||
apt-get update -y \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
bison \
|
||||
build-essential \
|
||||
flex \
|
||||
texinfo \
|
||||
xz-utils \
|
||||
&& curl -fsSLO "https://sourceware.org/pub/binutils/releases/binutils-${BINUTILS_VERSION}.tar.xz" \
|
||||
&& tar -xf "binutils-${BINUTILS_VERSION}.tar.xz" \
|
||||
&& mkdir binutils-build \
|
||||
&& cd binutils-build \
|
||||
&& "../binutils-${BINUTILS_VERSION}/configure" --prefix=/opt/binutils --disable-nls --disable-werror \
|
||||
&& make -j"$(nproc)" \
|
||||
&& make install \
|
||||
&& cd .. \
|
||||
&& rm -rf binutils-build "/binutils-${BINUTILS_VERSION}" "binutils-${BINUTILS_VERSION}.tar.xz"
|
||||
|
||||
######################### COMMON BASE IMAGE #########################
|
||||
FROM base AS base-common
|
||||
|
||||
WORKDIR /workspace
|
||||
|
||||
COPY --from=binutils-build /opt/binutils /opt/binutils
|
||||
|
||||
ARG PYTHON_VERSION=3.12
|
||||
|
||||
ARG max_jobs=32
|
||||
@@ -93,7 +120,8 @@ ENV CC=/usr/bin/gcc-15 CXX=/usr/bin/g++-15
|
||||
ENV CCACHE_DIR=/root/.cache/ccache
|
||||
ENV CMAKE_CXX_COMPILER_LAUNCHER=ccache
|
||||
|
||||
ENV PATH="/root/.local/bin:$PATH"
|
||||
ENV PATH="/opt/binutils/bin:/root/.local/bin:$PATH"
|
||||
ENV COMPILER_PATH="/opt/binutils/bin"
|
||||
ENV VIRTUAL_ENV="/opt/venv"
|
||||
ENV UV_PYTHON_INSTALL_DIR=/opt/uv/python
|
||||
RUN uv venv --python ${PYTHON_VERSION} --seed ${VIRTUAL_ENV}
|
||||
@@ -168,7 +196,7 @@ RUN apt-get update -y \
|
||||
WORKDIR /workspace
|
||||
|
||||
COPY requirements/build/rust.txt requirements/build/rust.txt
|
||||
RUN python3 -m pip install --break-system-packages --no-cache-dir -r requirements/build/rust.txt
|
||||
RUN python3 -m pip install --no-cache-dir -r requirements/build/rust.txt
|
||||
|
||||
ARG USE_SCCACHE
|
||||
ARG SCCACHE_ENDPOINT
|
||||
|
||||
@@ -8,7 +8,7 @@ ARG TRITON_VERSION=3.8.0+gitc0a142ff
|
||||
ARG TRITON_REPO="https://github.com/ROCm/triton.git"
|
||||
ARG FA_BRANCH="0e60e394"
|
||||
ARG FA_REPO="https://github.com/Dao-AILab/flash-attention.git"
|
||||
ARG AITER_BRANCH="v0.1.22.post1"
|
||||
ARG AITER_BRANCH="v0.1.23"
|
||||
ARG AITER_REPO="https://github.com/ROCm/aiter.git"
|
||||
ARG MORI_BRANCH="f95cd5e0ca7b730f92f9ffdfb358484d6965d9f7" # TheRock compatible v1.1.0
|
||||
ARG MORI_REPO="https://github.com/ROCm/mori.git"
|
||||
|
||||
@@ -9,7 +9,7 @@ ARG PYTORCH_AUDIO_BRANCH="v2.11.0"
|
||||
ARG PYTORCH_AUDIO_REPO="https://github.com/pytorch/audio.git"
|
||||
ARG FA_BRANCH="0e60e394"
|
||||
ARG FA_REPO="https://github.com/Dao-AILab/flash-attention.git"
|
||||
ARG AITER_BRANCH="v0.1.22.post1"
|
||||
ARG AITER_BRANCH="v0.1.23"
|
||||
ARG AITER_REPO="https://github.com/ROCm/aiter.git"
|
||||
ARG MORI_BRANCH="v1.1.0"
|
||||
ARG MORI_REPO="https://github.com/ROCm/mori.git"
|
||||
|
||||
@@ -596,6 +596,46 @@ vllm bench serve \
|
||||
--max-concurrency 512
|
||||
```
|
||||
|
||||
#### Responses API Benchmark
|
||||
|
||||
The `openai-responses` backend benchmarks vLLM's
|
||||
[Responses API](../serving/online_serving/openai_compatible_server.md#responses-api)
|
||||
directly, instead of routing the same workload through `/v1/chat/completions`.
|
||||
|
||||
```bash
|
||||
# Server
|
||||
vllm serve openai/gpt-oss-20b
|
||||
|
||||
# Client
|
||||
vllm bench serve \
|
||||
--backend openai-responses \
|
||||
--endpoint /v1/responses \
|
||||
--model openai/gpt-oss-20b \
|
||||
--dataset-name random \
|
||||
--random-input-len 1024 \
|
||||
--random-output-len 1024 \
|
||||
--num-prompts 200
|
||||
```
|
||||
|
||||
Reasoning deltas (`response.reasoning_text.delta`) count towards TTFT and ITL,
|
||||
because the server is already decoding tokens when it emits them. Only output
|
||||
text (`response.output_text.delta`) is collected as the generated text, which
|
||||
matches how the `openai-chat` backend treats `DeltaMessage.reasoning`. End-to-end
|
||||
latency stops at the last token event, so `latency - ttft` equals `sum(itl)` and
|
||||
TPOT is comparable with the other endpoints. The input and output token counts
|
||||
come from the usage block on `response.completed`, which is still required: a
|
||||
stream that ends without a terminal event is reported as a failed request.
|
||||
|
||||
The backend measures one streamed text-generation request per prompt. Built-in
|
||||
tools, MCP, and multi-turn state via `previous_response_id` are out of scope.
|
||||
All sampling parameter flags are supported except `--min-p`, which the
|
||||
Responses API does not accept.
|
||||
|
||||
!!! warning
|
||||
Do not pass `--extra-body '{"include_reasoning": false}'` when benchmarking
|
||||
a reasoning model. The server still generates reasoning tokens but emits no
|
||||
events for them, so the whole reasoning phase is absorbed into TTFT.
|
||||
|
||||
#### Running With Sampling Parameters
|
||||
|
||||
When using OpenAI-compatible backends such as `vllm`, optional sampling
|
||||
|
||||
@@ -144,7 +144,7 @@ When batch invariance is enabled, vLLM:
|
||||
|
||||
1. Uses deterministic kernel implementations for attention and other operations
|
||||
2. Ensures consistent numerical behavior across different batch sizes
|
||||
3. Disables certain optimizations that may introduce non-determinism (such as custom all-reduce operations in tensor parallel mode)
|
||||
3. Disables certain optimizations that may introduce non-determinism (such as custom all-reduce operations in tensor parallel mode, and sequence parallelism / async TP, whose reduce-scatter path is not batch-invariant)
|
||||
4. On CUDA devices with tuned matmul table entries for the model's bf16 unquantized forward linear layers (Ada, Hopper, Blackwell),
|
||||
runs without `torch.compile` using breakable CUDA graphs so tile configs follow the runtime batch size; set `VLLM_USE_BREAKABLE_CUDAGRAPH=0` to opt out
|
||||
(not applied when sequence parallelism / async TP are enabled, since those are `torch.compile` passes).
|
||||
|
||||
@@ -46,6 +46,13 @@ Now you can send requests to the proxy server through port 8000.
|
||||
- For headless instances, must be the same as the master instance
|
||||
- Each instance needs a unique port on its host; using the same port number across different hosts is fine
|
||||
|
||||
- `VLLM_MOONCAKE_CONNECTOR_TIMEOUT`: Per-request timeout (in seconds) for MooncakeConnector HTTP calls to the bootstrap server, including prefiller worker registration at startup. (Optional)
|
||||
- Default: 30.0
|
||||
- Global rank 0 hosts the bootstrap server in the same process that mounts the Mooncake transfer engine's host segment, so a large `MOONCAKE_GLOBAL_SEGMENT_SIZE` can delay responses for several seconds during startup
|
||||
- Raise this if bootstrap server timeout warnings appear on hosts with very large host-memory segments
|
||||
- Bootstrap registration uses up to 3 attempts internally, with exponential backoff capped at 10 seconds; connection errors and timeouts are both retried
|
||||
- Once the attempts are exhausted the caller raises instead of blocking, so failures surface rather than hanging
|
||||
|
||||
- `WITH_NVIDIA_PEERMEM`: Selects how mooncake registers GPU memory for RDMA. Read by mooncake, not vLLM.
|
||||
- Default: 1, which uses `ibv_reg_mr()` and requires the `nvidia-peermem` kernel module to be loaded
|
||||
- Set to 0 to use the DMA-BUF path, which does not need that module. Required on hosts where `nvidia-peermem` is not loaded, such as GB200
|
||||
|
||||
@@ -23,6 +23,10 @@ METRIC_SOURCE_FILES = [
|
||||
"path": "vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py",
|
||||
"key": "metrics-nixl",
|
||||
},
|
||||
{
|
||||
"path": "vllm/v1/simple_kv_offload/metrics.py",
|
||||
"key": "metrics-simple-kv-offload",
|
||||
},
|
||||
{"path": "vllm/v1/metrics/perf.py", "key": "metrics-mfu"},
|
||||
]
|
||||
|
||||
|
||||
@@ -50,8 +50,8 @@ The three supported scoring functions are as illustrated in the figure below.
|
||||
| `ModernBertForSequenceClassification` | ModernBERT-based | `Alibaba-NLP/gte-reranker-modernbert-base`, etc. | N/A | | |
|
||||
| `Qwen2ForSequenceClassification`<sup>C</sup> | Qwen2-based | `mixedbread-ai/mxbai-rerank-base-v2`(see note), etc. | [mxbai_rerank_v2.jinja](../../../examples/pooling/score/template/mxbai_rerank_v2.jinja) | ✅︎ | ✅︎ |
|
||||
| `Qwen3ForSequenceClassification`<sup>C</sup> | Qwen3-based | `tomaarsen/Qwen3-Reranker-0.6B-seq-cls`, `Qwen/Qwen3-Reranker-0.6B`(see note), etc. | [qwen3_reranker.jinja](../../../examples/pooling/score/template/qwen3_reranker.jinja) | ✅︎ | ✅︎ |
|
||||
| `RobertaForSequenceClassification` | RoBERTa-based | `cross-encoder/quora-roberta-base`, etc. | N/A | | |
|
||||
| `XLMRobertaForSequenceClassification` | XLM-RoBERTa-based | `BAAI/bge-reranker-v2-m3`, etc. | N/A | | |
|
||||
| `RobertaForSequenceClassification` | RoBERTa-based | `cross-encoder/quora-roberta-base`, etc. | N/A | ✅︎ | |
|
||||
| `XLMRobertaForSequenceClassification` | XLM-RoBERTa-based | `BAAI/bge-reranker-v2-m3`, etc. | N/A | ✅︎ | |
|
||||
| `*Model`<sup>C</sup>, `*ForCausalLM`<sup>C</sup>, etc. | Generative models | N/A | N/A | \* | \* |
|
||||
|
||||
<sup>C</sup> Automatically converted into a classification model via `--convert classify`. ([details](./README.md#model-conversion))
|
||||
|
||||
@@ -45,6 +45,30 @@ The following metrics are exposed:
|
||||
|
||||
--8<-- "gen:metrics-nixl"
|
||||
|
||||
## Simple CPU Offload Connector Metrics
|
||||
|
||||
These metrics are exposed when the `SimpleCPUOffloadConnector` KV connector
|
||||
is configured (e.g. `--kv-transfer-config='{"kv_connector":
|
||||
"SimpleCPUOffloadConnector", "kv_role": "kv_both", "kv_connector_extra_config":
|
||||
{"kv_offload_backend": "disk", "disk_path": "/mnt/nvme/kv"}}'`). They are
|
||||
updated once per engine step.
|
||||
|
||||
Caveats to keep in mind when interpreting them:
|
||||
|
||||
- A "completed" store means the write syscalls returned; it is **not**
|
||||
fsync-durable, and the disk backend's file is process-lifetime scratch,
|
||||
unlinked at startup and shutdown.
|
||||
- `save_outcomes_total` classifies eager-mode boundary hand-off stores;
|
||||
lazy-mode stores are not classified.
|
||||
- `used_blocks` counts blocks pinned by in-flight transfers or cache hits;
|
||||
warm cached blocks that are evictable are not counted. Use the
|
||||
`capacity_blocks` label of `simple_kv_offload_info` as the denominator.
|
||||
- Counters and gauges are quantized to engine steps. They are reported by the
|
||||
scheduler process and reflect engine-wide logical block counts, not values
|
||||
pooled from individual tensor-parallel workers.
|
||||
|
||||
--8<-- "gen:metrics-simple-kv-offload"
|
||||
|
||||
## Model Flops Utilization (MFU) Performance Metrics
|
||||
|
||||
These metrics are available via `--enable-mfu-metrics`:
|
||||
|
||||
@@ -258,3 +258,8 @@ CANN = "CANN"
|
||||
|
||||
[tool.uv]
|
||||
no-build-isolation-package = ["torch"]
|
||||
# vLLM depends on oss-harmony, a fork that ships the same `openai_harmony`
|
||||
# module without downloading tiktoken vocabularies at runtime. Installing
|
||||
# openai-harmony alongside it silently overwrites those files, so keep it out
|
||||
# of every resolution; gpt-oss pulls it in transitively.
|
||||
exclude-dependencies = ["openai-harmony"]
|
||||
|
||||
@@ -49,7 +49,7 @@ pybase64 # fast base64 implementation
|
||||
cbor2 # Required for cross-language serialization of hashable objects
|
||||
ijson # Required for mistral streaming tool parser
|
||||
setproctitle # Used to set process names for better debugging and monitoring
|
||||
openai-harmony >= 0.0.3 # Required for gpt-oss
|
||||
oss-harmony >= 0.0.11 # Required for gpt-oss; includes embedded vocabularies
|
||||
anthropic >= 0.71.0
|
||||
model-hosting-container-standards >= 0.1.14, < 1.0.0
|
||||
mcp >= 2.0.0, < 3.0.0
|
||||
|
||||
+31
-10
@@ -76,7 +76,9 @@ black==24.10.0
|
||||
blake3==1.0.9
|
||||
# via -r requirements/test/../common.txt
|
||||
blobfile==3.0.0
|
||||
# via -r requirements/test/cuda.in
|
||||
# via
|
||||
# -r requirements/test/cuda.in
|
||||
# gpt-oss
|
||||
bm25s==0.2.13
|
||||
# via mteb
|
||||
boto3==1.35.57
|
||||
@@ -111,7 +113,7 @@ chardet==5.2.0
|
||||
# via mbstrdecoder
|
||||
charset-normalizer==3.4.0
|
||||
# via requests
|
||||
chz==0.3.0
|
||||
chz==0.4.0
|
||||
# via gpt-oss
|
||||
click==8.4.2
|
||||
# via
|
||||
@@ -190,7 +192,7 @@ distro==1.9.0
|
||||
# openai
|
||||
dnspython==2.7.0
|
||||
# via email-validator
|
||||
docker==7.1.0
|
||||
docker==7.2.0
|
||||
# via gpt-oss
|
||||
docopt==0.6.2
|
||||
# via num2words
|
||||
@@ -291,7 +293,7 @@ googleapis-common-protos==1.70.0
|
||||
# google-api-core
|
||||
# opentelemetry-exporter-otlp-proto-grpc
|
||||
# opentelemetry-exporter-otlp-proto-http
|
||||
gpt-oss==0.0.8
|
||||
gpt-oss==0.0.9
|
||||
# via -r requirements/test/cuda.in
|
||||
graphql-core==3.2.6
|
||||
# via hypothesis-graphql
|
||||
@@ -402,6 +404,7 @@ jinja2==3.1.6
|
||||
# datamodel-code-generator
|
||||
# fastapi
|
||||
# genai-perf
|
||||
# gpt-oss
|
||||
# lm-eval
|
||||
# torch
|
||||
jiter==0.15.0
|
||||
@@ -433,6 +436,10 @@ jsonschema-specifications==2024.10.1
|
||||
# via jsonschema
|
||||
junit-xml==1.9
|
||||
# via schemathesis
|
||||
jupyter-client==8.10.0
|
||||
# via gpt-oss
|
||||
jupyter-core==5.9.1
|
||||
# via jupyter-client
|
||||
kaldi-native-fbank==1.22.3
|
||||
# via -r requirements/test/cuda.in
|
||||
kaleido==0.2.1
|
||||
@@ -549,6 +556,7 @@ numpy==2.2.6
|
||||
# evaluate
|
||||
# fastparquet
|
||||
# genai-perf
|
||||
# gpt-oss
|
||||
# imagehash
|
||||
# imageio
|
||||
# librosa
|
||||
@@ -583,8 +591,6 @@ numpy==2.2.6
|
||||
open-clip-torch==2.32.0
|
||||
# via -r requirements/test/cuda.in
|
||||
openai==2.44.0
|
||||
# via -r requirements/test/../common.txt
|
||||
openai-harmony==0.0.4
|
||||
# via
|
||||
# -r requirements/test/../common.txt
|
||||
# gpt-oss
|
||||
@@ -641,6 +647,8 @@ optuna==3.6.1
|
||||
# via genai-perf
|
||||
orjson==3.11.5
|
||||
# via genai-perf
|
||||
oss-harmony==0.0.11
|
||||
# via -r requirements/test/../common.txt
|
||||
outlines-core==0.2.14
|
||||
# via -r requirements/test/../common.txt
|
||||
packaging==24.2
|
||||
@@ -672,6 +680,7 @@ pandas==2.2.3
|
||||
# evaluate
|
||||
# fastparquet
|
||||
# genai-perf
|
||||
# gpt-oss
|
||||
# statsmodels
|
||||
partial-json-parser==0.2.1.1.post7
|
||||
# via -r requirements/test/../common.txt
|
||||
@@ -702,6 +711,7 @@ pillow==10.4.0
|
||||
platformdirs==4.3.6
|
||||
# via
|
||||
# black
|
||||
# jupyter-core
|
||||
# pooch
|
||||
# virtualenv
|
||||
plotly==5.24.1
|
||||
@@ -794,7 +804,7 @@ pydantic==2.12.0
|
||||
# model-hosting-container-standards
|
||||
# mteb
|
||||
# openai
|
||||
# openai-harmony
|
||||
# oss-harmony
|
||||
# pydantic-extra-types
|
||||
# pydantic-settings
|
||||
# ray
|
||||
@@ -855,6 +865,7 @@ pytest-timeout==2.3.1
|
||||
python-dateutil==2.9.0.post0
|
||||
# via
|
||||
# botocore
|
||||
# jupyter-client
|
||||
# matplotlib
|
||||
# pandas
|
||||
# typepy
|
||||
@@ -898,7 +909,9 @@ pyyaml==6.0.2
|
||||
# uvicorn
|
||||
# vocos
|
||||
pyzmq==27.1.0
|
||||
# via -r requirements/test/../common.txt
|
||||
# via
|
||||
# -r requirements/test/../common.txt
|
||||
# jupyter-client
|
||||
rapidfuzz==3.12.1
|
||||
# via jiwer
|
||||
ray==2.56.1
|
||||
@@ -1077,7 +1090,7 @@ starlette-testclient==0.4.1
|
||||
# via schemathesis
|
||||
statsmodels==0.14.4
|
||||
# via genai-perf
|
||||
structlog==25.4.0
|
||||
structlog==26.1.0
|
||||
# via gpt-oss
|
||||
supervisor==4.3.0
|
||||
# via model-hosting-container-standards
|
||||
@@ -1101,7 +1114,7 @@ tenacity==9.1.2
|
||||
# schemathesis
|
||||
tensorizer==2.10.1
|
||||
# via -r requirements/test/cuda.in
|
||||
termcolor==3.1.0
|
||||
termcolor==3.3.0
|
||||
# via gpt-oss
|
||||
threadpoolctl==3.5.0
|
||||
# via scikit-learn
|
||||
@@ -1157,11 +1170,14 @@ torchvision==0.28.0+cpu
|
||||
# open-clip-torch
|
||||
# segmentation-models-pytorch
|
||||
# timm
|
||||
tornado==6.5.10
|
||||
# via jupyter-client
|
||||
tqdm==4.67.3
|
||||
# via
|
||||
# -r requirements/test/../common.txt
|
||||
# datasets
|
||||
# evaluate
|
||||
# gpt-oss
|
||||
# huggingface-hub
|
||||
# lm-eval
|
||||
# mteb
|
||||
@@ -1174,6 +1190,10 @@ tqdm==4.67.3
|
||||
# segmentation-models-pytorch
|
||||
# sentence-transformers
|
||||
# transformers
|
||||
traitlets==5.16.1
|
||||
# via
|
||||
# jupyter-client
|
||||
# jupyter-core
|
||||
transformers==5.17.0
|
||||
# via
|
||||
# -r requirements/test/../common.txt
|
||||
@@ -1226,6 +1246,7 @@ typing-extensions==4.15.0
|
||||
# grpcio
|
||||
# httpx2
|
||||
# huggingface-hub
|
||||
# jupyter-client
|
||||
# librosa
|
||||
# lm-eval
|
||||
# mcp
|
||||
|
||||
@@ -66,7 +66,7 @@ decord==0.6.0; platform_machine == "x86_64"
|
||||
imagehash # Required for Prithvi tests
|
||||
segmentation-models-pytorch > 0.4.0 # Required for Prithvi tests
|
||||
|
||||
gpt-oss >= 0.0.7; python_version > '3.11'
|
||||
gpt-oss[eval] >= 0.0.7; python_version > '3.11'
|
||||
|
||||
perceptron # required for isaac test
|
||||
kaldi-native-fbank >= 1.18.7 # required for fireredasr2 test
|
||||
|
||||
+31
-11
@@ -81,7 +81,9 @@ black==24.10.0
|
||||
blake3==1.0.9
|
||||
# via -r requirements/test/../common.txt
|
||||
blobfile==3.0.0
|
||||
# via -r requirements/test/cuda.in
|
||||
# via
|
||||
# -r requirements/test/cuda.in
|
||||
# gpt-oss
|
||||
bm25s==0.2.13
|
||||
# via mteb
|
||||
boto3==1.35.57
|
||||
@@ -116,7 +118,7 @@ chardet==5.2.0
|
||||
# via mbstrdecoder
|
||||
charset-normalizer==3.4.0
|
||||
# via requests
|
||||
chz==0.3.0
|
||||
chz==0.4.0
|
||||
# via gpt-oss
|
||||
click==8.4.2
|
||||
# via
|
||||
@@ -205,7 +207,7 @@ distro==1.9.0
|
||||
# openai
|
||||
dnspython==2.7.0
|
||||
# via email-validator
|
||||
docker==7.1.0
|
||||
docker==7.2.0
|
||||
# via gpt-oss
|
||||
docopt==0.6.2
|
||||
# via num2words
|
||||
@@ -310,7 +312,7 @@ googleapis-common-protos==1.70.0
|
||||
# google-api-core
|
||||
# opentelemetry-exporter-otlp-proto-grpc
|
||||
# opentelemetry-exporter-otlp-proto-http
|
||||
gpt-oss==0.0.8
|
||||
gpt-oss==0.0.9
|
||||
# via -r requirements/test/cuda.in
|
||||
graphql-core==3.2.6
|
||||
# via hypothesis-graphql
|
||||
@@ -424,6 +426,7 @@ jinja2==3.1.6
|
||||
# datamodel-code-generator
|
||||
# fastapi
|
||||
# genai-perf
|
||||
# gpt-oss
|
||||
# lm-eval
|
||||
# torch
|
||||
jiter==0.15.0
|
||||
@@ -456,6 +459,10 @@ jsonschema-specifications==2024.10.1
|
||||
# via jsonschema
|
||||
junit-xml==1.9
|
||||
# via schemathesis
|
||||
jupyter-client==8.10.0
|
||||
# via gpt-oss
|
||||
jupyter-core==5.9.1
|
||||
# via jupyter-client
|
||||
kaldi-native-fbank==1.22.3
|
||||
# via -r requirements/test/cuda.in
|
||||
kaleido==0.2.1
|
||||
@@ -584,6 +591,7 @@ numpy==2.2.6
|
||||
# evaluate
|
||||
# fastparquet
|
||||
# genai-perf
|
||||
# gpt-oss
|
||||
# imagehash
|
||||
# imageio
|
||||
# librosa
|
||||
@@ -662,10 +670,6 @@ openai==2.44.0
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
openai-harmony==0.0.4
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
# gpt-oss
|
||||
opencensus==0.11.4
|
||||
# via ray
|
||||
@@ -727,6 +731,10 @@ optuna==3.6.1
|
||||
# via genai-perf
|
||||
orjson==3.11.5
|
||||
# via genai-perf
|
||||
oss-harmony==0.0.11
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
outlines-core==0.2.14
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
@@ -760,6 +768,7 @@ pandas==2.2.3
|
||||
# evaluate
|
||||
# fastparquet
|
||||
# genai-perf
|
||||
# gpt-oss
|
||||
# statsmodels
|
||||
partial-json-parser==0.2.1.1.post7
|
||||
# via -r requirements/test/../common.txt
|
||||
@@ -790,6 +799,7 @@ pillow==10.4.0
|
||||
platformdirs==4.3.6
|
||||
# via
|
||||
# black
|
||||
# jupyter-core
|
||||
# pooch
|
||||
# virtualenv
|
||||
plotly==5.24.1
|
||||
@@ -887,7 +897,7 @@ pydantic==2.12.0
|
||||
# model-hosting-container-standards
|
||||
# mteb
|
||||
# openai
|
||||
# openai-harmony
|
||||
# oss-harmony
|
||||
# pydantic-extra-types
|
||||
# pydantic-settings
|
||||
# ray
|
||||
@@ -948,6 +958,7 @@ pytest-timeout==2.3.1
|
||||
python-dateutil==2.9.0.post0
|
||||
# via
|
||||
# botocore
|
||||
# jupyter-client
|
||||
# matplotlib
|
||||
# pandas
|
||||
# typepy
|
||||
@@ -994,6 +1005,7 @@ pyzmq==27.1.0
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
# jupyter-client
|
||||
rapidfuzz==3.12.1
|
||||
# via jiwer
|
||||
ray==2.56.1
|
||||
@@ -1177,7 +1189,7 @@ starlette-testclient==0.4.1
|
||||
# via schemathesis
|
||||
statsmodels==0.14.4
|
||||
# via genai-perf
|
||||
structlog==25.4.0
|
||||
structlog==26.1.0
|
||||
# via gpt-oss
|
||||
supervisor==4.3.0
|
||||
# via model-hosting-container-standards
|
||||
@@ -1201,7 +1213,7 @@ tenacity==9.1.2
|
||||
# schemathesis
|
||||
tensorizer==2.10.1
|
||||
# via -r requirements/test/cuda.in
|
||||
termcolor==3.1.0
|
||||
termcolor==3.3.0
|
||||
# via gpt-oss
|
||||
threadpoolctl==3.5.0
|
||||
# via scikit-learn
|
||||
@@ -1264,11 +1276,14 @@ torchvision==0.28.0+cu130
|
||||
# open-clip-torch
|
||||
# segmentation-models-pytorch
|
||||
# timm
|
||||
tornado==6.5.10
|
||||
# via jupyter-client
|
||||
tqdm==4.67.3
|
||||
# via
|
||||
# -r requirements/test/../common.txt
|
||||
# datasets
|
||||
# evaluate
|
||||
# gpt-oss
|
||||
# huggingface-hub
|
||||
# lm-eval
|
||||
# mteb
|
||||
@@ -1281,6 +1296,10 @@ tqdm==4.67.3
|
||||
# segmentation-models-pytorch
|
||||
# sentence-transformers
|
||||
# transformers
|
||||
traitlets==5.16.1
|
||||
# via
|
||||
# jupyter-client
|
||||
# jupyter-core
|
||||
transformers==5.17.0
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
@@ -1337,6 +1356,7 @@ typing-extensions==4.15.0
|
||||
# grpcio
|
||||
# httpx2
|
||||
# huggingface-hub
|
||||
# jupyter-client
|
||||
# librosa
|
||||
# lm-eval
|
||||
# mcp
|
||||
|
||||
@@ -65,7 +65,7 @@ decord==0.6.0
|
||||
imagehash # Required for Prithvi tests
|
||||
segmentation-models-pytorch>0.4.0 # Required for Prithvi tests
|
||||
|
||||
gpt-oss>=0.0.7; python_version > '3.11'
|
||||
gpt-oss[eval]>=0.0.7; python_version > '3.11'
|
||||
|
||||
perceptron # required for isaac test
|
||||
kaldi-native-fbank>=1.18.7 # required for fireredasr2 test
|
||||
|
||||
@@ -80,7 +80,9 @@ black==26.3.1
|
||||
blake3==1.0.8
|
||||
# via -r requirements/test/../common.txt
|
||||
blobfile==3.0.0
|
||||
# via -r requirements/test/rocm.in
|
||||
# via
|
||||
# -r requirements/test/rocm.in
|
||||
# gpt-oss
|
||||
bm25s==0.2.13
|
||||
# via mteb
|
||||
boto3==1.42.74
|
||||
@@ -198,7 +200,7 @@ distro==1.9.0
|
||||
# openai
|
||||
dnspython==2.8.0
|
||||
# via email-validator
|
||||
docker==7.1.0
|
||||
docker==7.2.0
|
||||
# via gpt-oss
|
||||
docopt==0.6.2
|
||||
# via num2words
|
||||
@@ -300,7 +302,7 @@ googleapis-common-protos==1.73.0
|
||||
# google-api-core
|
||||
# opentelemetry-exporter-otlp-proto-grpc
|
||||
# opentelemetry-exporter-otlp-proto-http
|
||||
gpt-oss==0.0.8
|
||||
gpt-oss==0.0.9
|
||||
# via -r requirements/test/rocm.in
|
||||
graphql-core==3.2.8
|
||||
# via hypothesis-graphql
|
||||
@@ -415,6 +417,7 @@ jinja2==3.1.6
|
||||
# datamodel-code-generator
|
||||
# fastapi
|
||||
# genai-perf
|
||||
# gpt-oss
|
||||
# lm-eval
|
||||
# torch
|
||||
jiter==0.14.0
|
||||
@@ -447,6 +450,10 @@ jsonschema-specifications==2025.9.1
|
||||
# via jsonschema
|
||||
junit-xml==1.9
|
||||
# via schemathesis
|
||||
jupyter-client==8.10.0
|
||||
# via gpt-oss
|
||||
jupyter-core==5.9.1
|
||||
# via jupyter-client
|
||||
kaldi-native-fbank==1.22.3
|
||||
# via -r requirements/test/rocm.in
|
||||
kaleido==1.0.0
|
||||
@@ -587,6 +594,7 @@ numpy==2.2.6
|
||||
# evaluate
|
||||
# fastparquet
|
||||
# genai-perf
|
||||
# gpt-oss
|
||||
# imagehash
|
||||
# imageio
|
||||
# librosa
|
||||
@@ -629,10 +637,6 @@ openai==2.31.0
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
openai-harmony==0.0.8
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
# gpt-oss
|
||||
opencensus==0.11.4
|
||||
# via ray
|
||||
@@ -699,6 +703,10 @@ orjson==3.11.7
|
||||
# via
|
||||
# genai-perf
|
||||
# kaleido
|
||||
oss-harmony==0.0.11
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
outlines-core==0.2.14
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
@@ -734,6 +742,7 @@ pandas==3.0.1
|
||||
# evaluate
|
||||
# fastparquet
|
||||
# genai-perf
|
||||
# gpt-oss
|
||||
# statsmodels
|
||||
partial-json-parser==0.2.1.1.post7
|
||||
# via -r requirements/test/../common.txt
|
||||
@@ -764,6 +773,7 @@ pillow==12.1.1
|
||||
platformdirs==4.3.6
|
||||
# via
|
||||
# black
|
||||
# jupyter-core
|
||||
# pooch
|
||||
# python-discovery
|
||||
# virtualenv
|
||||
@@ -861,7 +871,7 @@ pydantic==2.12.5
|
||||
# model-hosting-container-standards
|
||||
# mteb
|
||||
# openai
|
||||
# openai-harmony
|
||||
# oss-harmony
|
||||
# pydantic-extra-types
|
||||
# pydantic-settings
|
||||
# ray
|
||||
@@ -922,6 +932,7 @@ pytest-timeout==2.3.1
|
||||
python-dateutil==2.9.0.post0
|
||||
# via
|
||||
# botocore
|
||||
# jupyter-client
|
||||
# matplotlib
|
||||
# pandas
|
||||
# typepy
|
||||
@@ -970,6 +981,7 @@ pyzmq==27.1.0
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
# jupyter-client
|
||||
rapidfuzz==3.12.1
|
||||
# via
|
||||
# -r requirements/test/rocm.in
|
||||
@@ -1159,7 +1171,7 @@ starlette-testclient==0.4.1
|
||||
# via schemathesis
|
||||
statsmodels==0.14.6
|
||||
# via genai-perf
|
||||
structlog==25.5.0
|
||||
structlog==26.1.0
|
||||
# via gpt-oss
|
||||
supervisor==4.3.0
|
||||
# via model-hosting-container-standards
|
||||
@@ -1215,11 +1227,14 @@ tokenizers==0.23.1
|
||||
# transformers
|
||||
torch-c-dlpack-ext==0.1.5
|
||||
# via tilelang
|
||||
tornado==6.5.10
|
||||
# via jupyter-client
|
||||
tqdm==4.67.3
|
||||
# via
|
||||
# -r requirements/test/../common.txt
|
||||
# datasets
|
||||
# evaluate
|
||||
# gpt-oss
|
||||
# huggingface-hub
|
||||
# lm-eval
|
||||
# mteb
|
||||
@@ -1233,6 +1248,10 @@ tqdm==4.67.3
|
||||
# sentence-transformers
|
||||
# tilelang
|
||||
# transformers
|
||||
traitlets==5.16.1
|
||||
# via
|
||||
# jupyter-client
|
||||
# jupyter-core
|
||||
transformers==5.17.0
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
@@ -1287,6 +1306,7 @@ typing-extensions==4.15.0
|
||||
# grpcio
|
||||
# httpx2
|
||||
# huggingface-hub
|
||||
# jupyter-client
|
||||
# librosa
|
||||
# lm-eval
|
||||
# mcp
|
||||
|
||||
@@ -32,7 +32,7 @@ soundfile
|
||||
# --- Tool Parsing & Evaluation ---
|
||||
blobfile
|
||||
rapidfuzz
|
||||
gpt-oss >= 0.0.7; python_version > '3.11'
|
||||
gpt-oss[eval] >= 0.0.7; python_version > '3.11'
|
||||
schemathesis>=4.0.0 # Required for openai schema test.
|
||||
jiwer
|
||||
bm25s
|
||||
|
||||
@@ -67,7 +67,9 @@ azure-storage-blob==12.30.1
|
||||
blake3==1.0.9
|
||||
# via -r requirements/test/../common.txt
|
||||
blobfile==3.2.0
|
||||
# via -r requirements/test/xpu.in
|
||||
# via
|
||||
# -r requirements/test/xpu.in
|
||||
# gpt-oss
|
||||
bm25s==0.3.10
|
||||
# via
|
||||
# -r requirements/test/xpu.in
|
||||
@@ -257,7 +259,7 @@ googleapis-common-protos==1.75.1
|
||||
# google-api-core
|
||||
# opentelemetry-exporter-otlp-proto-grpc
|
||||
# opentelemetry-exporter-otlp-proto-http
|
||||
gpt-oss==0.0.8
|
||||
gpt-oss==0.0.9
|
||||
# via -r requirements/test/xpu.in
|
||||
graphql-core==3.2.11
|
||||
# via hypothesis-graphql
|
||||
@@ -377,6 +379,7 @@ jinja2==3.1.6
|
||||
# via
|
||||
# -c requirements/xpu.txt
|
||||
# fastapi
|
||||
# gpt-oss
|
||||
# lm-eval
|
||||
# torch
|
||||
jiter==0.16.0
|
||||
@@ -406,6 +409,10 @@ jsonschema-rs==0.49.9
|
||||
# via schemathesis
|
||||
jsonschema-specifications==2025.9.1
|
||||
# via jsonschema
|
||||
jupyter-client==8.10.0
|
||||
# via gpt-oss
|
||||
jupyter-core==5.9.1
|
||||
# via jupyter-client
|
||||
kiwisolver==1.5.1
|
||||
# via matplotlib
|
||||
lark==1.2.2
|
||||
@@ -526,6 +533,7 @@ numpy==2.2.6
|
||||
# datasets
|
||||
# decord
|
||||
# evaluate
|
||||
# gpt-oss
|
||||
# librosa
|
||||
# lm-eval
|
||||
# matplotlib
|
||||
@@ -575,10 +583,6 @@ openai==3.2.0
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
openai-harmony==0.0.8
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
# gpt-oss
|
||||
opencv-python-headless==5.0.0.93
|
||||
# via
|
||||
@@ -632,6 +636,10 @@ opentelemetry-semantic-conventions-ai==0.5.1
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
# -r requirements/test/xpu.in
|
||||
oss-harmony==0.0.11
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
outlines-core==0.2.14
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
@@ -656,6 +664,7 @@ pandas==3.0.5
|
||||
# via
|
||||
# datasets
|
||||
# evaluate
|
||||
# gpt-oss
|
||||
partial-json-parser==0.2.1.1.post7
|
||||
# via -r requirements/test/../common.txt
|
||||
pathvalidate==3.3.1
|
||||
@@ -667,7 +676,9 @@ pillow==12.3.0
|
||||
# mistral-common
|
||||
# torchvision
|
||||
platformdirs==4.11.3
|
||||
# via pooch
|
||||
# via
|
||||
# jupyter-core
|
||||
# pooch
|
||||
pluggy==1.6.0
|
||||
# via
|
||||
# pytest
|
||||
@@ -747,7 +758,7 @@ pydantic==2.13.4
|
||||
# model-hosting-container-standards
|
||||
# mteb
|
||||
# openai
|
||||
# openai-harmony
|
||||
# oss-harmony
|
||||
# pydantic-extra-types
|
||||
# pydantic-settings
|
||||
# xgrammar
|
||||
@@ -806,6 +817,7 @@ pytest-timeout==2.4.0
|
||||
python-dateutil==2.9.0.post0
|
||||
# via
|
||||
# botocore
|
||||
# jupyter-client
|
||||
# matplotlib
|
||||
# pandas
|
||||
# typepy
|
||||
@@ -841,6 +853,7 @@ pyzmq==27.1.0
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
# jupyter-client
|
||||
rapidfuzz==3.14.5
|
||||
# via
|
||||
# -r requirements/test/xpu.in
|
||||
@@ -1066,11 +1079,14 @@ torchvision==0.29.0+xpu
|
||||
# via
|
||||
# open-clip-torch
|
||||
# timm
|
||||
tornado==6.5.10
|
||||
# via jupyter-client
|
||||
tqdm==4.70.0
|
||||
# via
|
||||
# -r requirements/test/../common.txt
|
||||
# datasets
|
||||
# evaluate
|
||||
# gpt-oss
|
||||
# huggingface-hub
|
||||
# lm-eval
|
||||
# modelscope
|
||||
@@ -1081,6 +1097,10 @@ tqdm==4.70.0
|
||||
# pqdm
|
||||
# sentence-transformers
|
||||
# transformers
|
||||
traitlets==5.16.1
|
||||
# via
|
||||
# jupyter-client
|
||||
# jupyter-core
|
||||
transformers==5.17.0
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
@@ -1132,6 +1152,7 @@ typing-extensions==4.16.0
|
||||
# grpcio
|
||||
# httpx2
|
||||
# huggingface-hub
|
||||
# jupyter-client
|
||||
# lm-eval
|
||||
# mcp
|
||||
# mcp-types
|
||||
|
||||
@@ -0,0 +1,455 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the ``openai-responses`` backend of ``vllm bench serve``.
|
||||
|
||||
The Responses stream differs from Chat Completions in ways the benchmark client
|
||||
has to get right: every SSE message is labelled with an ``event:`` line,
|
||||
reasoning and output text arrive as separate token-bearing events, and a
|
||||
request ends with ``response.completed`` rather than with its last token. These
|
||||
tests drive the real request function against a local SSE server so the
|
||||
resulting latency and usage numbers are covered end to end.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
|
||||
import aiohttp
|
||||
import pytest
|
||||
from aiohttp import web
|
||||
from aiohttp.test_utils import TestServer
|
||||
|
||||
from vllm.benchmarks.lib.endpoint_request_func import (
|
||||
ASYNC_REQUEST_FUNCS,
|
||||
OPENAI_COMPATIBLE_BACKENDS,
|
||||
RequestFuncInput,
|
||||
RequestFuncOutput,
|
||||
async_request_openai_responses,
|
||||
)
|
||||
|
||||
# These tests never touch torch, so skip the GPU cleanup fixture.
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
MODEL = "openai/gpt-oss-20b"
|
||||
|
||||
# Gaps between scripted events. GAP_TOKEN is much larger than GAP_PREFILL so
|
||||
# that a time to first token measured at the wrong event is unambiguous.
|
||||
GAP_PREFILL = 0.15
|
||||
GAP_TOKEN = 0.5
|
||||
GAP_FINALIZE = 0.3
|
||||
|
||||
|
||||
def sse(event_type: str, payload: dict) -> bytes:
|
||||
"""Frame one event the way ``/v1/responses`` does, with an ``event:`` line.
|
||||
|
||||
``ensure_ascii=False`` matches the server, whose pydantic
|
||||
``model_dump_json`` emits non-ASCII characters unescaped.
|
||||
"""
|
||||
data = json.dumps({"type": event_type, **payload}, ensure_ascii=False)
|
||||
return f"event: {event_type}\ndata: {data}\n\n".encode()
|
||||
|
||||
|
||||
def usage_event(
|
||||
input_tokens: int,
|
||||
output_tokens: int,
|
||||
event_type: str = "response.completed",
|
||||
status: str = "completed",
|
||||
):
|
||||
return sse(
|
||||
event_type,
|
||||
{
|
||||
"response": {
|
||||
"status": status,
|
||||
"usage": {
|
||||
"input_tokens": input_tokens,
|
||||
"output_tokens": output_tokens,
|
||||
"total_tokens": input_tokens + output_tokens,
|
||||
},
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def text_delta(delta: str) -> bytes:
|
||||
return sse("response.output_text.delta", {"delta": delta})
|
||||
|
||||
|
||||
def reasoning_delta(delta: str) -> bytes:
|
||||
return sse("response.reasoning_text.delta", {"delta": delta})
|
||||
|
||||
|
||||
# The metadata events the server sends before any token. None of these may
|
||||
# start the TTFT clock or contribute an ITL sample.
|
||||
PREAMBLE = (
|
||||
sse("response.created", {"response": {"status": "in_progress"}})
|
||||
+ sse("response.in_progress", {"response": {"status": "in_progress"}})
|
||||
+ sse("response.output_item.added", {"output_index": 0})
|
||||
+ sse("response.reasoning_part.added", {"output_index": 0})
|
||||
+ sse("response.content_part.added", {"output_index": 0})
|
||||
)
|
||||
|
||||
# The `.done` family the server sends after the last delta. These repeat the
|
||||
# accumulated text, so timing them would add bogus ITL samples and counting
|
||||
# their payload would duplicate the output.
|
||||
DONE_EVENTS = (
|
||||
sse("response.reasoning_text.done", {"text": "The user asks "})
|
||||
+ sse("response.reasoning_part.done", {"output_index": 0})
|
||||
+ sse("response.output_text.done", {"text": "Rayleigh scattering."})
|
||||
+ sse("response.content_part.done", {"output_index": 0})
|
||||
+ sse("response.output_item.done", {"output_index": 0})
|
||||
)
|
||||
|
||||
|
||||
async def _serve(
|
||||
script: Sequence[bytes | float],
|
||||
*,
|
||||
status: int = 200,
|
||||
chunk_size: int | None = None,
|
||||
seen: dict | None = None,
|
||||
**request_kwargs,
|
||||
) -> RequestFuncOutput:
|
||||
"""Run the request function against a server replaying ``script``.
|
||||
|
||||
Args:
|
||||
script: Byte strings to write, interleaved with floats to sleep for.
|
||||
status: HTTP status to respond with. Non-200 skips the script.
|
||||
chunk_size: If set, split every write into chunks of this many bytes to
|
||||
emulate fragmented transport.
|
||||
seen: If given, populated with the request payload.
|
||||
request_kwargs: Extra fields for the ``RequestFuncInput``.
|
||||
|
||||
"""
|
||||
|
||||
async def handler(request: web.Request) -> web.StreamResponse:
|
||||
if seen is not None:
|
||||
seen["payload"] = await request.json()
|
||||
seen["headers"] = dict(request.headers)
|
||||
if status != 200:
|
||||
return web.Response(status=status, reason="Bad Request")
|
||||
|
||||
response = web.StreamResponse(headers={"Content-Type": "text/event-stream"})
|
||||
await response.prepare(request)
|
||||
for item in script:
|
||||
if isinstance(item, float):
|
||||
await asyncio.sleep(item)
|
||||
elif chunk_size is None:
|
||||
await response.write(item)
|
||||
else:
|
||||
for i in range(0, len(item), chunk_size):
|
||||
await response.write(item[i : i + chunk_size])
|
||||
await response.write_eof()
|
||||
return response
|
||||
|
||||
app = web.Application()
|
||||
app.router.add_post("/v1/responses", handler)
|
||||
server = TestServer(app)
|
||||
await server.start_server()
|
||||
try:
|
||||
request_input = RequestFuncInput(
|
||||
prompt="Why is the sky blue?",
|
||||
api_url=str(server.make_url("/v1/responses")),
|
||||
prompt_len=5,
|
||||
output_len=16,
|
||||
model=MODEL,
|
||||
**request_kwargs,
|
||||
)
|
||||
async with aiohttp.ClientSession() as session:
|
||||
return await async_request_openai_responses(request_input, session)
|
||||
finally:
|
||||
await server.close()
|
||||
|
||||
|
||||
def run(script: Sequence[bytes | float], **kwargs) -> RequestFuncOutput:
|
||||
return asyncio.run(_serve(script, **kwargs))
|
||||
|
||||
|
||||
def test_backend_is_registered_as_openai_compatible():
|
||||
assert ASYNC_REQUEST_FUNCS["openai-responses"] is async_request_openai_responses
|
||||
# Membership enables --ignore-eos on random datasets and the sampling
|
||||
# parameter flags, all of which ResponsesRequest accepts.
|
||||
assert "openai-responses" in OPENAI_COMPATIBLE_BACKENDS
|
||||
|
||||
|
||||
def test_request_payload_is_accepted_by_the_server_schema():
|
||||
"""Every field sent must be a real ResponsesRequest field.
|
||||
|
||||
This is what makes the backend safe to list in OPENAI_COMPATIBLE_BACKENDS:
|
||||
the flags that membership enables have to reach the model, and the server
|
||||
drops unknown fields without complaining.
|
||||
"""
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
|
||||
sampling = {
|
||||
"temperature": 0.7,
|
||||
"top_p": 0.9,
|
||||
"top_k": 20,
|
||||
"frequency_penalty": 0.2,
|
||||
"presence_penalty": 0.1,
|
||||
"repetition_penalty": 1.05,
|
||||
}
|
||||
seen: dict = {}
|
||||
output = run(
|
||||
[PREAMBLE, text_delta("hi"), usage_event(5, 1)],
|
||||
seen=seen,
|
||||
extra_body=sampling,
|
||||
ignore_eos=True,
|
||||
model_name="served-alias",
|
||||
request_id="bench-7",
|
||||
extra_headers={"x-team": "perf"},
|
||||
)
|
||||
|
||||
assert output.success
|
||||
# --served-model-name wins over --model, as on the other backends.
|
||||
assert seen["payload"]["model"] == "served-alias"
|
||||
# The route is behind validate_json_request, which 415s a missing type.
|
||||
assert seen["headers"]["Content-Type"] == "application/json"
|
||||
assert seen["headers"]["x-request-id"] == "bench-7"
|
||||
assert seen["headers"]["x-team"] == "perf"
|
||||
assert seen["payload"] == {
|
||||
"model": "served-alias",
|
||||
"input": "Why is the sky blue?",
|
||||
"max_output_tokens": 16,
|
||||
"stream": True,
|
||||
"store": False,
|
||||
"ignore_eos": True,
|
||||
**sampling,
|
||||
}
|
||||
assert set(seen["payload"]) <= set(ResponsesRequest.model_fields)
|
||||
ResponsesRequest.model_validate(seen["payload"])
|
||||
|
||||
# `vllm bench serve` rejects --min-p for this backend for the same reason.
|
||||
assert "min_p" not in ResponsesRequest.model_fields
|
||||
|
||||
|
||||
def test_latency_accounting_matches_the_other_endpoints():
|
||||
"""`latency - ttft` must equal `sum(itl)`.
|
||||
|
||||
`serve.py` derives TPOT as `(latency - ttft) / (output_len - 1)`, which only
|
||||
means average inter-token latency when that identity holds. #55508 fixed the
|
||||
same accounting on the chat and audio backends; the trailing events after
|
||||
the last token must not extend the request.
|
||||
"""
|
||||
output = run(
|
||||
[
|
||||
PREAMBLE,
|
||||
GAP_PREFILL / 2,
|
||||
# A keep-alive comment sent during prefill is not a token.
|
||||
b": keep-alive\n\n",
|
||||
GAP_PREFILL / 2,
|
||||
reasoning_delta("The user asks "),
|
||||
GAP_TOKEN,
|
||||
text_delta("Rayleigh"),
|
||||
text_delta(" scattering."),
|
||||
DONE_EVENTS,
|
||||
GAP_FINALIZE,
|
||||
usage_event(5, 8),
|
||||
]
|
||||
)
|
||||
|
||||
assert output.success
|
||||
# Reasoning is timed but not returned, matching how `openai-chat` treats
|
||||
# `DeltaMessage.reasoning`.
|
||||
assert output.generated_text == "Rayleigh scattering."
|
||||
|
||||
# Neither the response.created/in_progress/output_item.added preamble nor
|
||||
# the keep-alive comment may start the clock, and the reasoning delta must,
|
||||
# so TTFT lands on the prefill gap rather than near zero or out at the
|
||||
# first output text delta.
|
||||
assert GAP_PREFILL * 0.8 <= output.ttft < GAP_PREFILL + GAP_TOKEN * 0.5
|
||||
|
||||
# One ITL per token event after the first: the gap before "Rayleigh" and
|
||||
# the back-to-back delta that follows it.
|
||||
assert len(output.itl) == 2
|
||||
assert output.itl[0] >= GAP_TOKEN * 0.5
|
||||
assert output.itl[1] < GAP_TOKEN * 0.5
|
||||
|
||||
# The identity is exact, so no wall-clock margin is involved. The .done
|
||||
# family and the GAP_FINALIZE wait before response.completed sit outside
|
||||
# the request, exactly as the trailing usage chunk does on chat.
|
||||
assert output.latency - output.ttft == pytest.approx(sum(output.itl), abs=1e-9)
|
||||
|
||||
assert output.start_time > 0.0
|
||||
|
||||
|
||||
def test_usage_is_taken_from_the_terminal_event():
|
||||
output = run([PREAMBLE, text_delta("blue"), usage_event(17, 4)])
|
||||
|
||||
assert output.success
|
||||
assert output.prompt_len == 17
|
||||
assert output.output_tokens == 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize("event_type", ["response.completed", "response.incomplete"])
|
||||
def test_incomplete_response_is_a_successful_request(event_type: str):
|
||||
"""Hitting max_output_tokens is the Responses form of finish_reason=length.
|
||||
|
||||
vLLM reports it as `response.completed` carrying `status: "incomplete"`.
|
||||
The spec's dedicated `response.incomplete` event is accepted as well.
|
||||
"""
|
||||
output = run(
|
||||
[
|
||||
PREAMBLE,
|
||||
text_delta("truncated"),
|
||||
usage_event(17, 16, event_type=event_type, status="incomplete"),
|
||||
]
|
||||
)
|
||||
|
||||
assert output.success
|
||||
assert output.generated_text == "truncated"
|
||||
assert output.output_tokens == 16
|
||||
|
||||
|
||||
@pytest.mark.parametrize("chunk_size", [1, 3, 7, 64])
|
||||
def test_output_survives_transport_chunk_boundaries(chunk_size: int):
|
||||
"""Splits inside the event line, the JSON, or a multibyte character."""
|
||||
text = " leading 中 trailing "
|
||||
output = run(
|
||||
[PREAMBLE, text_delta(text), usage_event(5, 6)],
|
||||
chunk_size=chunk_size,
|
||||
)
|
||||
|
||||
assert output.success
|
||||
assert output.generated_text == text
|
||||
assert output.output_tokens == 6
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"char", ["\u2028", "\u2029", "\u0085"], ids=["u2028", "u2029", "u0085"]
|
||||
)
|
||||
def test_unicode_line_separators_do_not_truncate_the_payload(char: str):
|
||||
"""The server emits these unescaped and they are legal inside a JSON string.
|
||||
|
||||
`str.splitlines()` treats them as line breaks, which would cut the `data:`
|
||||
line short and fail the request on valid model output.
|
||||
"""
|
||||
text = f"before{char}after"
|
||||
output = run([PREAMBLE, text_delta(text), usage_event(5, 3)])
|
||||
|
||||
assert output.success, output.error
|
||||
assert output.generated_text == text
|
||||
|
||||
|
||||
def test_empty_data_line_is_skipped():
|
||||
"""A bare `data:` line carries no JSON and must not fail the request."""
|
||||
output = run([PREAMBLE, b"data:\n\n", text_delta("ok"), usage_event(5, 1)])
|
||||
|
||||
assert output.success, output.error
|
||||
assert output.generated_text == "ok"
|
||||
|
||||
|
||||
def test_prompt_len_falls_back_when_usage_is_absent():
|
||||
"""A terminal event without usage keeps the locally computed prompt length."""
|
||||
output = run(
|
||||
[
|
||||
PREAMBLE,
|
||||
text_delta("hi"),
|
||||
sse("response.completed", {"response": {"status": "completed"}}),
|
||||
]
|
||||
)
|
||||
|
||||
assert output.success, output.error
|
||||
assert output.prompt_len == 5
|
||||
|
||||
|
||||
def test_multi_line_data_payload_is_joined_with_newlines():
|
||||
"""The SSE spec allows one payload to span several `data:` lines."""
|
||||
script: list[bytes | float] = [
|
||||
PREAMBLE,
|
||||
b"event: response.output_text.delta\n"
|
||||
b'data: {"type": "response.output_text.delta",\n'
|
||||
b'data: "delta": "ok"}\n\n',
|
||||
usage_event(5, 1),
|
||||
]
|
||||
output = run(script)
|
||||
|
||||
assert output.success, output.error
|
||||
assert output.generated_text == "ok"
|
||||
|
||||
|
||||
def test_done_sentinel_is_skipped():
|
||||
"""Other Responses servers send `[DONE]`, but vLLM does not."""
|
||||
output = run([PREAMBLE, text_delta("ok"), usage_event(5, 1), b"data: [DONE]\n\n"])
|
||||
|
||||
assert output.success, output.error
|
||||
assert output.generated_text == "ok"
|
||||
|
||||
|
||||
def test_error_before_any_token_reports_the_server_error():
|
||||
"""The server's message must win over the generic no-TTFT message."""
|
||||
output = run(
|
||||
[PREAMBLE, b'event: error\ndata: {"type": "error", "message": "boom"}\n\n']
|
||||
)
|
||||
|
||||
assert not output.success
|
||||
assert "boom" in output.error
|
||||
|
||||
|
||||
def test_http_error_is_not_reported_as_success():
|
||||
output = run([], status=400)
|
||||
|
||||
assert not output.success
|
||||
assert output.error
|
||||
|
||||
|
||||
def test_stream_without_tokens_is_not_reported_as_success():
|
||||
"""A stream that finishes without generating anything has no TTFT."""
|
||||
output = run([PREAMBLE, usage_event(5, 0)])
|
||||
|
||||
assert not output.success
|
||||
assert "TTFT" in output.error
|
||||
|
||||
|
||||
def test_truncated_stream_is_not_reported_as_success():
|
||||
"""A stream cut short after a token has no usage and an understated E2EL."""
|
||||
output = run([PREAMBLE, text_delta("partial")])
|
||||
|
||||
assert not output.success
|
||||
assert "terminal event" in output.error
|
||||
|
||||
|
||||
def test_mid_stream_error_is_not_reported_as_success():
|
||||
"""An error payload mid-stream must not be counted as a success.
|
||||
|
||||
vLLM cannot emit this today: its streaming error path fails to validate
|
||||
against the event union, so a generation failure truncates the stream
|
||||
instead. Proxies and other Responses servers do send it.
|
||||
"""
|
||||
output = run(
|
||||
[
|
||||
PREAMBLE,
|
||||
text_delta("partial"),
|
||||
b'event: error\ndata: {"error": {"message": "boom", "code": 500}}\n\n',
|
||||
]
|
||||
)
|
||||
|
||||
assert not output.success
|
||||
assert "boom" in output.error
|
||||
|
||||
|
||||
def test_data_only_framing_is_still_parsed():
|
||||
"""Some proxies drop the ``event:`` line and forward only ``data:``."""
|
||||
script: list[bytes | float] = [
|
||||
b'data: {"type": "response.output_text.delta", "delta": "ok"}\n\n',
|
||||
usage_event(5, 1),
|
||||
]
|
||||
output = run(script)
|
||||
|
||||
assert output.success
|
||||
assert output.generated_text == "ok"
|
||||
|
||||
|
||||
def test_endpoint_path_is_validated():
|
||||
request_input = RequestFuncInput(
|
||||
prompt="hi",
|
||||
api_url="http://localhost:8000/v1/chat/completions",
|
||||
prompt_len=1,
|
||||
output_len=1,
|
||||
model=MODEL,
|
||||
)
|
||||
|
||||
async def call() -> None:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
await async_request_openai_responses(request_input, session)
|
||||
|
||||
with pytest.raises(ValueError, match="responses"):
|
||||
asyncio.run(call())
|
||||
@@ -3,10 +3,20 @@
|
||||
|
||||
import json
|
||||
from collections.abc import AsyncGenerator
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from tests.entrypoints.openai.chat_completion.test_serving_chat import (
|
||||
BASE_MODEL_PATHS,
|
||||
MockHFConfig,
|
||||
MockModelConfig,
|
||||
_build_renderer,
|
||||
)
|
||||
from tests.entrypoints.openai.chat_completion.test_serving_chat import (
|
||||
MODEL_NAME as MOCK_MODEL_NAME,
|
||||
)
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata
|
||||
from vllm.entrypoints.openai.chat_completion.batch_serving import (
|
||||
@@ -15,7 +25,10 @@ from vllm.entrypoints.openai.chat_completion.batch_serving import (
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
BatchChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
# any model with a chat template defined in tokenizer_config should work here
|
||||
MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
|
||||
@@ -322,3 +335,53 @@ async def test_batched_echo_prepends_matching_assistant_prefix() -> None:
|
||||
)
|
||||
|
||||
assert response.choices[0].message.content == "PREFIX ASSISTANT ANSWER"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip_global_cleanup
|
||||
async def test_batched_harmony_response_format_uses_structural_tag() -> None:
|
||||
"""Harmony must constrain only the final channel of every conversation."""
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
mock_engine.errored = False
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.model_config.hf_config = MockHFConfig(model_type="gpt_oss")
|
||||
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
||||
mock_engine.generate.side_effect = lambda *args, **kwargs: _generator(0, "ok")
|
||||
|
||||
models = OpenAIServingModels(mock_engine, BASE_MODEL_PATHS)
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=mock_engine.model_config,
|
||||
renderer=mock_engine.renderer,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
reasoning_parser="openai_gptoss",
|
||||
)
|
||||
serving = OpenAIServingChatBatch(
|
||||
mock_engine,
|
||||
models,
|
||||
response_role="assistant",
|
||||
online_renderer=online_renderer,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
request_logger=None,
|
||||
)
|
||||
|
||||
request = BatchChatCompletionRequest(
|
||||
model=MOCK_MODEL_NAME,
|
||||
messages=[
|
||||
[{"role": "user", "content": "France?"}],
|
||||
[{"role": "user", "content": "Japan?"}],
|
||||
],
|
||||
response_format={
|
||||
"type": "json_schema",
|
||||
"json_schema": {"name": "city", "schema": {"type": "object"}},
|
||||
},
|
||||
)
|
||||
await serving.create_batch_chat_completion(request)
|
||||
|
||||
calls = mock_engine.generate.call_args_list
|
||||
assert len(calls) == 2
|
||||
for call in calls:
|
||||
assert call.args[1].structured_outputs.structural_tag is not None
|
||||
|
||||
@@ -20,6 +20,7 @@ Each test:
|
||||
"""
|
||||
|
||||
from openai.types.responses import ResponseFunctionToolCall
|
||||
from openai_harmony import Message, Role
|
||||
|
||||
from tests.entrypoints.openai.utils import verify_harmony_messages
|
||||
from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
@@ -33,6 +34,23 @@ from vllm.entrypoints.openai.responses.harmony import (
|
||||
response_previous_input_to_harmony,
|
||||
)
|
||||
|
||||
|
||||
def test_render_uses_legacy_tool_call_header():
|
||||
tool_call = (
|
||||
Message.from_role_and_content(Role.ASSISTANT, "print('hello')")
|
||||
.with_channel("commentary")
|
||||
.with_recipient("python")
|
||||
.with_content_type("code")
|
||||
)
|
||||
|
||||
rendered = get_encoding().decode(render_for_completion([tool_call]))
|
||||
|
||||
assert (
|
||||
"<|start|>assistant to=python<|channel|>commentary code<|message|>" in rendered
|
||||
)
|
||||
assert "commentary to=python <|constrain|>code" not in rendered
|
||||
|
||||
|
||||
# Use a fixed date so the system message is deterministic across both paths.
|
||||
_DATE = "2025-01-01"
|
||||
|
||||
|
||||
@@ -84,6 +84,18 @@ class _ReasoningOnlyParser(DelegatingParser):
|
||||
return None
|
||||
|
||||
|
||||
class _CountingParser(_NoOpParser):
|
||||
"""Parser that records the ids it counts; token id 7 is reasoning."""
|
||||
|
||||
def __init__(self, tokenizer, *args, **kwargs):
|
||||
super().__init__(tokenizer)
|
||||
self.counted_ids: list[list[int]] = []
|
||||
|
||||
def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
|
||||
self.counted_ids.append(list(token_ids))
|
||||
return sum(token_id == 7 for token_id in token_ids)
|
||||
|
||||
|
||||
class _StubToolParser:
|
||||
"""Minimal tool parser stub that always returns a hardcoded tool call."""
|
||||
|
||||
@@ -339,6 +351,25 @@ def test_multi_turn_accumulation():
|
||||
assert texts == ["First turn", "Second turn"]
|
||||
|
||||
|
||||
def test_reasoning_tokens_counted_per_round():
|
||||
"""Each round is counted on its own ids, not the concatenated rounds."""
|
||||
ctx = _make_context(_CountingParser)
|
||||
|
||||
ctx.append_output(_make_request_output(token_ids=[7, 7, 1]))
|
||||
ctx.append_output(_make_request_output(token_ids=[7, 2]))
|
||||
|
||||
assert ctx.response_parser.counted_ids == [[7, 7, 1], [7, 2]]
|
||||
assert ctx.num_reasoning_tokens == 3
|
||||
|
||||
|
||||
def test_reasoning_tokens_zero_without_parser():
|
||||
ctx = _make_context(None)
|
||||
|
||||
ctx.append_output(_make_request_output(token_ids=[7, 7, 1]))
|
||||
|
||||
assert ctx.num_reasoning_tokens == 0
|
||||
|
||||
|
||||
def test_num_init_messages_offset():
|
||||
"""Initial messages are preserved and offset works correctly."""
|
||||
init_messages = [MagicMock(type="message")]
|
||||
|
||||
@@ -960,24 +960,19 @@ class TestValidateGeneratorInput:
|
||||
assert isinstance(result, ErrorResponse)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
|
||||
"""Ensure reasoning_tokens usage is derived from thinking token spans."""
|
||||
class _Qwen3FakeTokenizer:
|
||||
def __init__(self):
|
||||
self._vocab = {"<think>": 1, "</think>": 2, "reason": 3, "final": 4}
|
||||
|
||||
class FakeTokenizer:
|
||||
def __init__(self):
|
||||
self._vocab = {"<think>": 1, "</think>": 2, "reason": 3, "final": 4}
|
||||
def get_vocab(self):
|
||||
return self._vocab
|
||||
|
||||
def get_vocab(self):
|
||||
return self._vocab
|
||||
def decode(self, token_ids):
|
||||
id_to_token = {v: k for k, v in self._vocab.items()}
|
||||
return "".join(id_to_token.get(token_id, "x") for token_id in token_ids)
|
||||
|
||||
def decode(self, token_ids):
|
||||
id_to_token = {v: k for k, v in self._vocab.items()}
|
||||
return "".join(id_to_token.get(token_id, "x") for token_id in token_ids)
|
||||
|
||||
# Force non-harmony, SimpleContext path
|
||||
monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
|
||||
|
||||
def _make_qwen3_serving() -> tuple[OpenAIServingResponses, _Qwen3FakeTokenizer]:
|
||||
engine_client = MagicMock()
|
||||
model_config = MagicMock()
|
||||
model_config.hf_config.model_type = "test"
|
||||
@@ -987,21 +982,66 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
|
||||
engine_client.input_processor = MagicMock()
|
||||
engine_client.renderer = MagicMock()
|
||||
|
||||
tokenizer = FakeTokenizer()
|
||||
tokenizer = _Qwen3FakeTokenizer()
|
||||
engine_client.renderer.get_tokenizer.return_value = tokenizer
|
||||
|
||||
models = MagicMock()
|
||||
|
||||
serving = OpenAIServingResponses(
|
||||
engine_client=engine_client,
|
||||
models=models,
|
||||
models=MagicMock(),
|
||||
online_renderer=MagicMock(),
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
reasoning_parser="qwen3",
|
||||
)
|
||||
return serving, tokenizer
|
||||
|
||||
|
||||
def _make_text_request_output(text: str, token_ids: list[int]) -> RequestOutput:
|
||||
return RequestOutput(
|
||||
request_id="req",
|
||||
prompt="hi",
|
||||
prompt_token_ids=[7, 8],
|
||||
prompt_logprobs=None,
|
||||
outputs=[
|
||||
CompletionOutput(
|
||||
index=0,
|
||||
text=text,
|
||||
token_ids=token_ids,
|
||||
cumulative_logprob=0.0,
|
||||
logprobs=None,
|
||||
finish_reason="stop",
|
||||
stop_reason=None,
|
||||
)
|
||||
],
|
||||
finished=True,
|
||||
num_cached_tokens=0,
|
||||
num_cache_creation_tokens=3,
|
||||
)
|
||||
|
||||
|
||||
async def _run_full_generator(serving, request, context, tokenizer):
|
||||
async def dummy_result_generator():
|
||||
yield None
|
||||
|
||||
return await serving.responses_full_generator(
|
||||
request=request,
|
||||
sampling_params=SamplingParams(max_tokens=16),
|
||||
result_generator=dummy_result_generator(),
|
||||
context=context,
|
||||
model_name="test-model",
|
||||
tokenizer=tokenizer,
|
||||
request_metadata=RequestResponseMetadata(request_id="req"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
|
||||
"""Ensure reasoning_tokens usage is derived from thinking token spans."""
|
||||
# Force non-harmony, SimpleContext path
|
||||
monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
|
||||
|
||||
serving, tokenizer = _make_qwen3_serving()
|
||||
request = ResponsesRequest(input="hi", tools=[], stream=False)
|
||||
response_parser = serving._make_response_parser(
|
||||
request,
|
||||
@@ -1011,43 +1051,12 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
|
||||
|
||||
# Build a SimpleContext with thinking tokens in the output.
|
||||
context = SimpleContext(response_parser=response_parser)
|
||||
token_ids = [1, 10, 2, 20] # <think> 10 </think> 20 -> reasoning token count = 1
|
||||
completion = CompletionOutput(
|
||||
index=0,
|
||||
text="<think>reason</think>final",
|
||||
token_ids=token_ids,
|
||||
cumulative_logprob=0.0,
|
||||
logprobs=None,
|
||||
finish_reason="stop",
|
||||
stop_reason=None,
|
||||
# <think> 10 </think> 20 -> reasoning token count = 1
|
||||
context.append_output(
|
||||
_make_text_request_output("<think>reason</think>final", [1, 10, 2, 20])
|
||||
)
|
||||
req_output = RequestOutput(
|
||||
request_id="req",
|
||||
prompt="hi",
|
||||
prompt_token_ids=[7, 8],
|
||||
prompt_logprobs=None,
|
||||
outputs=[completion],
|
||||
finished=True,
|
||||
num_cached_tokens=0,
|
||||
num_cache_creation_tokens=3,
|
||||
)
|
||||
context.append_output(req_output)
|
||||
|
||||
async def dummy_result_generator():
|
||||
yield None
|
||||
|
||||
sampling_params = SamplingParams(max_tokens=16)
|
||||
metadata = RequestResponseMetadata(request_id="req")
|
||||
|
||||
response = await serving.responses_full_generator(
|
||||
request=request,
|
||||
sampling_params=sampling_params,
|
||||
result_generator=dummy_result_generator(),
|
||||
context=context,
|
||||
model_name="test-model",
|
||||
tokenizer=tokenizer,
|
||||
request_metadata=metadata,
|
||||
)
|
||||
response = await _run_full_generator(serving, request, context, tokenizer)
|
||||
|
||||
assert response.usage.output_tokens_details.reasoning_tokens == 1
|
||||
assert response.usage.input_tokens_details.cache_write_tokens == 3
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Run the gpt-oss GPQA evaluator with an explicit output-token budget."""
|
||||
|
||||
import argparse
|
||||
|
||||
from gpt_oss.evals.gpqa_eval import GPQAEval
|
||||
from gpt_oss.evals.responses_sampler import ResponsesSampler
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--model", required=True)
|
||||
parser.add_argument("--reasoning-effort", required=True)
|
||||
parser.add_argument("--base-url", required=True)
|
||||
parser.add_argument("--n-threads", required=True, type=int)
|
||||
parser.add_argument("--max-output-tokens", required=True, type=int)
|
||||
args = parser.parse_args()
|
||||
|
||||
sampler = ResponsesSampler(
|
||||
model=args.model,
|
||||
reasoning_model=True,
|
||||
reasoning_effort=args.reasoning_effort,
|
||||
base_url=args.base_url,
|
||||
max_tokens=args.max_output_tokens,
|
||||
)
|
||||
result = GPQAEval(n_repeats=8, n_threads=args.n_threads)(sampler)
|
||||
# gpt-oss returns a NumPy scalar. Convert it so the parent process gets a
|
||||
# stable, dependency-independent representation that its regex can parse.
|
||||
print({"metric": float(result.score)})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -21,6 +21,12 @@ from tests.utils import RemoteOpenAIServer
|
||||
|
||||
TOL = 0.05 # Absolute tolerance for accuracy comparison
|
||||
|
||||
# gpt-oss >= 0.0.7 explicitly sends its 131072-token default as the output
|
||||
# budget, which leaves no room for the prompt in GPT-OSS's 131072-token context.
|
||||
# This test originally ran with a 32768-token model context, so retain that
|
||||
# proven generation budget while using the current evaluator package.
|
||||
MAX_OUTPUT_TOKENS = 32_768
|
||||
|
||||
# Path to tiktoken encoding files
|
||||
TIKTOKEN_DATA_DIR = Path(__file__).parent / "data"
|
||||
|
||||
@@ -50,10 +56,7 @@ def run_gpqa_eval(model_name: str, base_url: str, reasoning_effort: str) -> floa
|
||||
# Build the command to run the evaluation
|
||||
cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"gpt_oss.evals",
|
||||
"--eval",
|
||||
"gpqa",
|
||||
str(Path(__file__).with_name("run_gpqa_eval.py")),
|
||||
"--model",
|
||||
model_name,
|
||||
"--reasoning-effort",
|
||||
@@ -62,6 +65,8 @@ def run_gpqa_eval(model_name: str, base_url: str, reasoning_effort: str) -> floa
|
||||
base_url,
|
||||
"--n-threads",
|
||||
"200",
|
||||
"--max-output-tokens",
|
||||
str(MAX_OUTPUT_TOKENS),
|
||||
]
|
||||
|
||||
try:
|
||||
|
||||
@@ -225,3 +225,46 @@ def test_apply_rotary_emb_xpu_matches_native(
|
||||
expected = op.forward_native(x, cos, sin)
|
||||
actual = op.forward_xpu(x, cos, sin)
|
||||
torch.testing.assert_close(actual, expected, atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="CUDA/ROCm only test.")
|
||||
@pytest.mark.parametrize("num_tokens", [257, 2304])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
def test_packed_qk_rope_correctness(
|
||||
num_tokens: int, dtype: torch.dtype, default_vllm_config
|
||||
):
|
||||
"""packed_qk_rope_ must be bitwise identical to the per-tensor
|
||||
ApplyRotaryEmb path with enable_fp32_compute=True, and must leave the V
|
||||
slice untouched."""
|
||||
from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb
|
||||
from vllm.model_executor.layers.rotary_embedding.packed_qk_rope import (
|
||||
packed_qk_rope_,
|
||||
)
|
||||
|
||||
num_heads, head_dim = 12, 128
|
||||
rng = torch.Generator(device="cuda").manual_seed(0)
|
||||
|
||||
angles = (
|
||||
torch.rand(num_tokens, head_dim // 2, device="cuda", generator=rng) * torch.pi
|
||||
)
|
||||
freqs_cis = torch.polar(torch.ones_like(angles), angles)
|
||||
cos = freqs_cis.real.contiguous()
|
||||
sin = freqs_cis.imag.contiguous()
|
||||
|
||||
xqkv = torch.randn(
|
||||
num_tokens, 3, num_heads, head_dim, dtype=dtype, device="cuda", generator=rng
|
||||
)
|
||||
xqkv_ref = xqkv.clone()
|
||||
xq, xk, xv = torch.unbind(xqkv_ref, dim=-3)
|
||||
|
||||
op = ApplyRotaryEmb(
|
||||
enforce_enable=True, is_neox_style=False, enable_fp32_compute=True
|
||||
)
|
||||
# op() dispatches to forward_cuda / forward_hip per platform.
|
||||
xq_ref = op(xq, cos, sin)
|
||||
xk_ref = op(xk, cos, sin)
|
||||
|
||||
packed_qk_rope_(xqkv, freqs_cis)
|
||||
torch.testing.assert_close(xqkv[:, 0], xq_ref, atol=0, rtol=0)
|
||||
torch.testing.assert_close(xqkv[:, 1], xk_ref, atol=0, rtol=0)
|
||||
torch.testing.assert_close(xqkv[:, 2], xv, atol=0, rtol=0)
|
||||
|
||||
@@ -42,6 +42,9 @@ from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
convert_to_fp8_moe_kernel_format,
|
||||
make_fp8_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import (
|
||||
fused_topk_bias,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.fused_topk_router import fused_topk
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
rotate_weights_for_fi_trtllm_fp8_per_tensor_moe,
|
||||
@@ -887,3 +890,97 @@ def test_trtllm_fp8_block_moe_deferred_finalize(
|
||||
),
|
||||
router_weights=topk_weights,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m", [1, 16])
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(100),
|
||||
reason="Requires TRTLLM-Gen FP8 MoE (SM100)",
|
||||
)
|
||||
def test_trtllm_mxfp8_minimax2_routing_applies_routed_scale(m: int, workspace_init):
|
||||
"""FlashInfer's fused MiniMax2 routing must apply routed_scaling_factor,
|
||||
matching vLLM's own sigmoid+bias top-k routing (MiniMax-M3 uses 2.0)."""
|
||||
e, topk, n, k, scale = 32, 4, 1024, 1024, 2.0
|
||||
block_shape = [1, 32]
|
||||
set_random_seed(7)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
w1, w2, w1_scale, w2_scale = convert_to_fp8_moe_kernel_format(
|
||||
Fp8MoeBackend.FLASHINFER_TRTLLM,
|
||||
SimpleNamespace(
|
||||
weight_block_size=block_shape,
|
||||
moe_config=SimpleNamespace(
|
||||
is_act_and_mul=True, intermediate_size_per_partition=n
|
||||
),
|
||||
activation=MoEActivation.SILU,
|
||||
),
|
||||
*_make_mxfp8_moe_weights(e, n, k),
|
||||
w13_input_scale=None,
|
||||
w2_input_scale=None,
|
||||
)
|
||||
quant_config = make_fp8_moe_quant_config(
|
||||
Fp8MoeBackend.FLASHINFER_TRTLLM,
|
||||
w1_scale,
|
||||
w2_scale,
|
||||
a1_scale=None,
|
||||
a2_scale=None,
|
||||
block_shape=block_shape,
|
||||
)
|
||||
moe_config = FusedMoEConfig(
|
||||
num_experts=e,
|
||||
experts_per_token=topk,
|
||||
hidden_dim=k,
|
||||
intermediate_size=n,
|
||||
num_local_experts=e,
|
||||
num_logical_experts=e,
|
||||
activation=MoEActivation.SILU,
|
||||
device="cuda",
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
in_dtype=torch.bfloat16,
|
||||
routing_method=RoutingMethodType.MiniMax2,
|
||||
max_num_tokens=next_power_of_2(m),
|
||||
)
|
||||
|
||||
def make_kernel(experts_cls) -> mk.FusedMoEKernel:
|
||||
return mk.FusedMoEKernel(
|
||||
maybe_make_prepare_finalize(
|
||||
moe=moe_config,
|
||||
quant_config=quant_config,
|
||||
allow_new_interface=True,
|
||||
use_monolithic=experts_cls is TrtLlmFp8ExpertsMonolithic,
|
||||
),
|
||||
experts_cls(moe_config=moe_config, quant_config=quant_config),
|
||||
)
|
||||
|
||||
a = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) / 10
|
||||
logits = torch.randn((m, e), device="cuda", dtype=torch.float32)
|
||||
bias = torch.randn(e, device="cuda", dtype=torch.float32) / 10
|
||||
common = dict(
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
activation=MoEActivation.SILU,
|
||||
global_num_experts=e,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
fused = make_kernel(TrtLlmFp8ExpertsMonolithic).apply_monolithic(
|
||||
hidden_states=a,
|
||||
router_logits=logits,
|
||||
e_score_correction_bias=bias,
|
||||
routed_scaling_factor=scale,
|
||||
**common,
|
||||
)
|
||||
topk_weights, topk_ids = fused_topk_bias(
|
||||
a,
|
||||
logits,
|
||||
scoring_func="sigmoid",
|
||||
e_score_correction_bias=bias,
|
||||
topk=topk,
|
||||
renormalize=True,
|
||||
routed_scaling_factor=scale,
|
||||
)
|
||||
reference = make_kernel(TrtLlmFp8ExpertsModular).apply(
|
||||
hidden_states=a, topk_weights=topk_weights, topk_ids=topk_ids, **common
|
||||
)
|
||||
|
||||
torch.testing.assert_close(fused, reference, atol=2e-2, rtol=2e-2)
|
||||
|
||||
@@ -47,7 +47,6 @@ if ROCM_AVAILABLE:
|
||||
ROCM_GFX950 = on_gfx950()
|
||||
|
||||
if ROCM_AITER_AVAILABLE:
|
||||
from aiter.ops.triton.moe.quant_moe import upcast_from_mxfp
|
||||
from aiter.ops.triton.quant import dynamic_mxfp4_quant
|
||||
|
||||
if TRTLLM_GEN_MXFP4_AVAILABLE:
|
||||
@@ -191,11 +190,7 @@ def mxfp8_dequantize(x, scale):
|
||||
def mxfp4_quant_dequant(x: torch.Tensor) -> torch.Tensor:
|
||||
shape = x.shape
|
||||
quantized, scale = dynamic_mxfp4_quant(x.to(torch.bfloat16).flatten(0, -2))
|
||||
return (
|
||||
upcast_from_mxfp(quantized.view(torch.uint8), scale, torch.bfloat16, axis=-1)
|
||||
.reshape(shape)
|
||||
.float()
|
||||
)
|
||||
return mxfp4_dequantize(quantized.view(torch.uint8), scale).reshape(shape)
|
||||
|
||||
|
||||
def reference_moe(
|
||||
@@ -1809,12 +1804,8 @@ def test_rocm_mxfp4_moe_oracle(
|
||||
assert out.abs().max() > 0.01, "Output is effectively zero"
|
||||
|
||||
# Dequantize weights for reference computation
|
||||
w13_dq = upcast_from_mxfp(
|
||||
w13_quant_ref.view(torch.uint8), w13_scale_ref, torch.bfloat16, axis=-1
|
||||
)
|
||||
w2_dq = upcast_from_mxfp(
|
||||
w2_quant_ref.view(torch.uint8), w2_scale_ref, torch.bfloat16, axis=-1
|
||||
)
|
||||
w13_dq = mxfp4_dequantize(w13_quant_ref.view(torch.uint8), w13_scale_ref)
|
||||
w2_dq = mxfp4_dequantize(w2_quant_ref.view(torch.uint8), w2_scale_ref)
|
||||
|
||||
# Determine activation type and layout
|
||||
# SWIGLUOAI uses interleaved layout (gate/up alternating)
|
||||
|
||||
@@ -566,72 +566,64 @@ def test_aiter_moe_shared_experts_enablement_follows_env(
|
||||
assert rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() is expected
|
||||
|
||||
|
||||
def test_aiter_moe_situv2_syncs_aiter_a4w4_env(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""SiTUv2 MoE should route to AITER a4w4 and clear legacy a8w4 overrides."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
_assert_aiter_supported()
|
||||
|
||||
with monkeypatch.context() as mp:
|
||||
mp.delenv("AITER_SITUV2_A8W4", raising=False)
|
||||
mp.delenv("AITER_SITUV2_A4W4", raising=False)
|
||||
mp.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "1")
|
||||
_reload_envs()
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
import os
|
||||
|
||||
assert os.environ.get("AITER_SITUV2_A4W4") == "1"
|
||||
assert "AITER_SITUV2_A8W4" not in os.environ
|
||||
|
||||
|
||||
def test_aiter_moe_situv2_legacy_a8w4_alias_enables_a4w4(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Deprecated VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4 still enables SiTUv2 a4w4."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
_assert_aiter_supported()
|
||||
|
||||
with monkeypatch.context() as mp:
|
||||
mp.delenv("AITER_SITUV2_A8W4", raising=False)
|
||||
mp.delenv("AITER_SITUV2_A4W4", raising=False)
|
||||
mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", raising=False)
|
||||
mp.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", "1")
|
||||
_reload_envs()
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
import os
|
||||
|
||||
assert os.environ.get("AITER_SITUV2_A4W4") == "1"
|
||||
assert "AITER_SITUV2_A8W4" not in os.environ
|
||||
|
||||
|
||||
def test_aiter_moe_situv2_clears_aiter_env_when_disabled(
|
||||
@pytest.mark.parametrize(
|
||||
"value,expected_act,expected_env",
|
||||
[
|
||||
# Unset / auto / legacy "1" all mean the a4w4 default.
|
||||
(None, "a4w4", {"AITER_SITUV2_A4W4": "1"}),
|
||||
("auto", "a4w4", {"AITER_SITUV2_A4W4": "1"}),
|
||||
("1", "a4w4", {"AITER_SITUV2_A4W4": "1"}),
|
||||
("a4w4", "a4w4", {"AITER_SITUV2_A4W4": "1"}),
|
||||
("0", "a16w4", {}),
|
||||
("a16w4", "a16w4", {}),
|
||||
("A8W4", "a8w4", {"AITER_SITUV2_A8W4": "1"}),
|
||||
],
|
||||
)
|
||||
def test_aiter_moe_situv2_activation_syncs_aiter_env(
|
||||
value: str | None,
|
||||
expected_act: str,
|
||||
expected_env: dict[str, str],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Each SiTUv2 activation choice sets exactly its AITER dispatch env."""
|
||||
import os
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
_assert_aiter_supported()
|
||||
|
||||
with monkeypatch.context() as mp:
|
||||
# Stale AITER overrides must be replaced by the vLLM selection.
|
||||
mp.setenv("AITER_SITUV2_A8W4", "1")
|
||||
mp.setenv("AITER_SITUV2_A4W4", "1")
|
||||
mp.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
|
||||
mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", raising=False)
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "0")
|
||||
if value is None:
|
||||
mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", raising=False)
|
||||
else:
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", value)
|
||||
_reload_envs()
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
import os
|
||||
assert rocm_aiter_ops.get_fused_moe_situv2_activation() == expected_act
|
||||
assert rocm_aiter_ops.is_fused_moe_situv2_gate_up_interleaved() is (
|
||||
expected_act == "a8w4"
|
||||
)
|
||||
assert rocm_aiter_ops.is_fused_moe_situv2_enabled() is (expected_act != "a16w4")
|
||||
for name in ("AITER_SITUV2_A8W4", "AITER_SITUV2_A4W4"):
|
||||
assert os.environ.get(name) == expected_env.get(name)
|
||||
|
||||
assert "AITER_SITUV2_A4W4" not in os.environ
|
||||
|
||||
def test_aiter_moe_situv2_rejects_unknown_activation(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
with monkeypatch.context() as mp:
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "a8w8")
|
||||
_reload_envs()
|
||||
import vllm.envs as envs
|
||||
|
||||
with pytest.raises(ValueError, match="VLLM_ROCM_USE_AITER_MOE_SITUV2"):
|
||||
_ = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("moe_padding", [True, False])
|
||||
|
||||
@@ -167,6 +167,38 @@ def test_single_expert_group_with_non_unit_scale_uses_grouped_topk() -> None:
|
||||
assert isinstance(router, GroupedTopKRouter)
|
||||
|
||||
|
||||
def test_sigmoid_bias_routing_with_routed_scale_is_minimax2() -> None:
|
||||
"""FlashInfer's MiniMax2 routing applies routed_scaling_factor, so a
|
||||
non-unit scale (MiniMax-M3 uses 2.0) must not block fused routing."""
|
||||
router = create_fused_moe_router(
|
||||
top_k=4,
|
||||
global_num_experts=128,
|
||||
scoring_func="sigmoid",
|
||||
renormalize=True,
|
||||
routed_scaling_factor=2.0,
|
||||
e_score_correction_bias=torch.empty(128),
|
||||
)
|
||||
|
||||
assert isinstance(router, FusedTopKBiasRouter)
|
||||
assert router.routing_method_type == RoutingMethodType.MiniMax2
|
||||
|
||||
|
||||
def test_zero_expert_routing_is_unspecified() -> None:
|
||||
"""Zero experts are resolved in the router, so kernels with built-in
|
||||
routing must never be selected for them."""
|
||||
router = create_fused_moe_router(
|
||||
top_k=4,
|
||||
global_num_experts=128,
|
||||
scoring_func="sigmoid",
|
||||
renormalize=True,
|
||||
e_score_correction_bias=torch.empty(160),
|
||||
zero_expert_type="identity",
|
||||
num_logical_experts=160,
|
||||
)
|
||||
|
||||
assert router.routing_method_type == RoutingMethodType.Unspecified
|
||||
|
||||
|
||||
def setup_eplb_state(
|
||||
enable_eplb: bool, global_num_experts: int
|
||||
) -> EplbLayerState | None:
|
||||
|
||||
@@ -2075,3 +2075,269 @@ def test_fused_kv_insert_split(num_tokens: int, kv_block_size: int):
|
||||
# RoPE (last 64): stored as bf16. The kernel recomputes the rotation, so it
|
||||
# is bf16-close to the reference rather than bit-exact (cf. test_cutedsl).
|
||||
torch.testing.assert_close(recovered[:, NOPE_DIM:], ref[:, NOPE_DIM:])
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm stream coverage")
|
||||
def test_v41_rocm_csa2_full_pipeline():
|
||||
"""_forward_csa2_full: one fork, then K write and q-side after the join.
|
||||
|
||||
The default chain must run first on the current stream, the compressor
|
||||
chain on aux 0 and the indexer weights projection on aux 1; the join must
|
||||
publish both cache writes before _produce_k / forward_q run.
|
||||
"""
|
||||
from vllm.forward_context import ForwardContext, override_forward_context
|
||||
from vllm.models.deepseek_v41.amd.rocm import DeepseekV41ROCMAiterMLAAttention
|
||||
from vllm.models.deepseek_v41.compressor import DeepseekCompressor
|
||||
|
||||
torch.manual_seed(43)
|
||||
num_tokens = 19
|
||||
hidden_states = torch.randn(num_tokens, 1024, device="cuda")
|
||||
positions = torch.arange(7, 26, device="cuda")
|
||||
qr_kv = torch.zeros(num_tokens, 1536, device="cuda")
|
||||
qr = torch.zeros(num_tokens, 4, device="cuda")
|
||||
kv = torch.zeros(num_tokens, 512, device="cuda")
|
||||
q = torch.zeros(num_tokens, 512, device="cuda")
|
||||
latent = torch.ones(num_tokens, 512, dtype=torch.bfloat16, device="cuda")
|
||||
weights = torch.randn(num_tokens, 32, device="cuda")
|
||||
index_q = torch.full((num_tokens, 32, 128), 2.0, device="cuda")
|
||||
index_q_scale = torch.full((num_tokens, 32), 3.0, device="cuda")
|
||||
index_weights_out = torch.full((num_tokens, 32), 4.0, device="cuda")
|
||||
main_cache = torch.full((1, 128, 584), 165, dtype=torch.uint8, device="cuda")
|
||||
index_cache = torch.full((1, 128, 132), 165, dtype=torch.uint8, device="cuda")
|
||||
|
||||
order: list[str] = []
|
||||
streams: dict[str, torch.cuda.Stream] = {}
|
||||
|
||||
compressor = DeepseekCompressor.__new__(DeepseekCompressor)
|
||||
torch.nn.Module.__init__(compressor)
|
||||
compressor.fused_wkv_wgate = SimpleNamespace(
|
||||
weight=torch.randn(1024, 1024, device="cuda")
|
||||
)
|
||||
|
||||
def compressor_forward(kv_score, positions_):
|
||||
order.append("compressor")
|
||||
streams["compressor"] = torch.cuda.current_stream()
|
||||
return latent
|
||||
|
||||
def compressor_insert(latent_, positions_, rotary_emb):
|
||||
order.append("insert_cache")
|
||||
streams["insert_cache"] = torch.cuda.current_stream()
|
||||
main_cache.fill_(5)
|
||||
|
||||
compressor.forward = compressor_forward
|
||||
compressor.insert_cache = compressor_insert
|
||||
|
||||
def weights_proj(hidden_states_):
|
||||
order.append("weights_proj")
|
||||
streams["weights_proj"] = torch.cuda.current_stream()
|
||||
return weights, None
|
||||
|
||||
def produce_k(latent_, positions_, rotary_emb):
|
||||
order.append("produce_k")
|
||||
streams["produce_k"] = torch.cuda.current_stream()
|
||||
index_cache.fill_(7)
|
||||
|
||||
def forward_q(qr_, qr_scale, indexer_weights, positions_, rotary_emb):
|
||||
order.append("forward_q")
|
||||
streams["forward_q"] = torch.cuda.current_stream()
|
||||
return index_q, index_q_scale, index_weights_out
|
||||
|
||||
indexer = SimpleNamespace(
|
||||
weights_proj=weights_proj,
|
||||
_produce_k=produce_k,
|
||||
forward_q=forward_q,
|
||||
)
|
||||
rotary = SimpleNamespace(cos_sin_cache=torch.randn(32, 64, device="cuda"))
|
||||
aux_streams = [torch.cuda.Stream(), torch.cuda.Stream()]
|
||||
|
||||
def wq_b(qr_, qr_scale):
|
||||
order.append("wq_b")
|
||||
streams["wq_b"] = torch.cuda.current_stream()
|
||||
return q
|
||||
|
||||
def qnorm_insert(q_, kv_, positions_, attn_metadata):
|
||||
order.append("qnorm_insert")
|
||||
streams["qnorm_insert"] = torch.cuda.current_stream()
|
||||
return q_
|
||||
|
||||
attention = SimpleNamespace(
|
||||
compressor=compressor,
|
||||
indexer=indexer,
|
||||
aux_stream_list=aux_streams,
|
||||
ln_events=[torch.cuda.Event() for _ in range(4)],
|
||||
rotary_emb=rotary,
|
||||
indexer_rotary_emb=rotary,
|
||||
n_local_heads=1,
|
||||
head_dim=512,
|
||||
_fused_wqa_wkv_gemm=lambda hidden_states_: qr_kv,
|
||||
_split_qkv_and_norm=lambda qr_kv_: (qr, None, kv),
|
||||
_wq_b_proj=wq_b,
|
||||
_fused_qnorm_rope_kv_insert=qnorm_insert,
|
||||
)
|
||||
|
||||
context = ForwardContext({}, {}, {})
|
||||
with override_forward_context(context):
|
||||
result = DeepseekV41ROCMAiterMLAAttention._forward_csa2_full(
|
||||
attention, hidden_states, positions
|
||||
)
|
||||
|
||||
q_out, kv_out, index_q_out, index_q_scale_out, index_weights_out_out = result
|
||||
assert torch.equal(q_out, q)
|
||||
assert torch.equal(kv_out, kv)
|
||||
assert torch.equal(index_q_out, index_q)
|
||||
assert torch.equal(index_q_scale_out, index_q_scale)
|
||||
assert torch.equal(index_weights_out_out, index_weights_out)
|
||||
# Join publishes the aux-0 cache insert and orders the post-join K write.
|
||||
assert torch.equal(main_cache, torch.full_like(main_cache, 5))
|
||||
assert torch.equal(index_cache, torch.full_like(index_cache, 7))
|
||||
|
||||
assert order == [
|
||||
"wq_b",
|
||||
"qnorm_insert",
|
||||
"compressor",
|
||||
"insert_cache",
|
||||
"weights_proj",
|
||||
"produce_k",
|
||||
"forward_q",
|
||||
]
|
||||
current = torch.cuda.current_stream()
|
||||
for name in ("wq_b", "qnorm_insert", "produce_k", "forward_q"):
|
||||
assert streams[name] == current, name
|
||||
for name in ("compressor", "insert_cache"):
|
||||
assert streams[name] == aux_streams[0], name
|
||||
assert streams["weights_proj"] == aux_streams[1]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm stream coverage")
|
||||
def test_v41_rocm_csa2_reindex_pipeline():
|
||||
"""_forward_csa2_reindex: serial projections, then SWA q path vs q-side."""
|
||||
from vllm.forward_context import ForwardContext, override_forward_context
|
||||
from vllm.models.deepseek_v41.amd.rocm import DeepseekV41ROCMAiterMLAAttention
|
||||
|
||||
torch.manual_seed(45)
|
||||
num_tokens = 19
|
||||
hidden_states = torch.randn(num_tokens, 1024, device="cuda")
|
||||
positions = torch.arange(7, 26, device="cuda")
|
||||
qr_kv = torch.zeros(num_tokens, 1536, device="cuda")
|
||||
qr = torch.zeros(num_tokens, 4, device="cuda")
|
||||
kv = torch.zeros(num_tokens, 512, device="cuda")
|
||||
q = torch.zeros(num_tokens, 512, device="cuda")
|
||||
indexer_weights = torch.randn(num_tokens, 32, device="cuda")
|
||||
index_q = torch.full((num_tokens, 32, 128), 2.0, device="cuda")
|
||||
index_q_scale = torch.full((num_tokens, 32), 3.0, device="cuda")
|
||||
index_weights_out = torch.full((num_tokens, 32), 4.0, device="cuda")
|
||||
|
||||
order: list[str] = []
|
||||
streams: dict[str, torch.cuda.Stream] = {}
|
||||
|
||||
def forward_q(qr_, qr_scale, indexer_weights_, positions_, rotary_emb):
|
||||
order.append("forward_q")
|
||||
streams["forward_q"] = torch.cuda.current_stream()
|
||||
return index_q, index_q_scale, index_weights_out
|
||||
|
||||
indexer = SimpleNamespace(forward_q=forward_q)
|
||||
rotary = SimpleNamespace(cos_sin_cache=torch.randn(32, 64, device="cuda"))
|
||||
aux0 = torch.cuda.Stream()
|
||||
|
||||
def wq_b(qr_, qr_scale):
|
||||
order.append("wq_b")
|
||||
streams["wq_b"] = torch.cuda.current_stream()
|
||||
return q
|
||||
|
||||
def qnorm_insert(q_, kv_, positions_, attn_metadata):
|
||||
order.append("qnorm_insert")
|
||||
streams["qnorm_insert"] = torch.cuda.current_stream()
|
||||
return q_
|
||||
|
||||
attention = SimpleNamespace(
|
||||
indexer=indexer,
|
||||
aux_stream_list=[aux0],
|
||||
ln_events=[torch.cuda.Event() for _ in range(2)],
|
||||
indexer_rotary_emb=rotary,
|
||||
n_local_heads=1,
|
||||
head_dim=512,
|
||||
_run_parallel_input_projections=lambda hidden_states_: (
|
||||
qr_kv,
|
||||
None,
|
||||
indexer_weights,
|
||||
),
|
||||
_split_qkv_and_norm=lambda qr_kv_: (qr, None, kv),
|
||||
_wq_b_proj=wq_b,
|
||||
_fused_qnorm_rope_kv_insert=qnorm_insert,
|
||||
)
|
||||
|
||||
context = ForwardContext({}, {}, {})
|
||||
with override_forward_context(context):
|
||||
result = DeepseekV41ROCMAiterMLAAttention._forward_csa2_reindex(
|
||||
attention, hidden_states, positions
|
||||
)
|
||||
|
||||
q_out, kv_out, index_q_out, index_q_scale_out, index_weights_out_out = result
|
||||
assert torch.equal(q_out, q)
|
||||
assert torch.equal(kv_out, kv)
|
||||
assert torch.equal(index_q_out, index_q)
|
||||
assert torch.equal(index_q_scale_out, index_q_scale)
|
||||
assert torch.equal(index_weights_out_out, index_weights_out)
|
||||
|
||||
assert order == ["wq_b", "qnorm_insert", "forward_q"]
|
||||
current = torch.cuda.current_stream()
|
||||
for name in ("wq_b", "qnorm_insert"):
|
||||
assert streams[name] == current, name
|
||||
assert streams["forward_q"] == aux0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("quant_tuple", [False, True])
|
||||
def test_v41_indexer_forward_q(quant_tuple):
|
||||
"""forward_q runs wq_b then the fused RoPE/quant and unpacks both shapes."""
|
||||
from unittest.mock import patch
|
||||
|
||||
from vllm.models.deepseek_v41.attention import DeepseekV4Indexer
|
||||
|
||||
num_tokens, n_head, head_dim = 5, 32, 128
|
||||
qr = torch.randn(num_tokens, 1280)
|
||||
q = torch.randn(num_tokens, n_head * head_dim)
|
||||
positions = torch.arange(num_tokens)
|
||||
indexer_weights = torch.randn(num_tokens, n_head)
|
||||
rotary = SimpleNamespace(cos_sin_cache=torch.randn(32, 64))
|
||||
fake = SimpleNamespace(
|
||||
n_head=n_head,
|
||||
head_dim=head_dim,
|
||||
softmax_scale=head_dim**-0.5,
|
||||
use_fp4_kv=False,
|
||||
indexer_weights_dtype=torch.float32,
|
||||
_wq_b_proj=lambda qr_, qr_scale: q,
|
||||
)
|
||||
q_quant = torch.randn(num_tokens, n_head, head_dim)
|
||||
q_scale = torch.randn(num_tokens, n_head)
|
||||
weights_out = torch.randn(num_tokens, n_head)
|
||||
if quant_tuple:
|
||||
fused_return = ((q_quant, q_scale), weights_out)
|
||||
expected = (q_quant, q_scale, weights_out)
|
||||
else:
|
||||
fused_return = (q_quant, weights_out)
|
||||
expected = (q_quant, None, weights_out)
|
||||
|
||||
with patch(
|
||||
"vllm.models.deepseek_v41.attention.fused_indexer_q_rope_quant",
|
||||
return_value=fused_return,
|
||||
) as mock_fused:
|
||||
result = DeepseekV4Indexer.forward_q(
|
||||
fake, qr, None, indexer_weights, positions, rotary
|
||||
)
|
||||
|
||||
assert torch.equal(result[0], expected[0])
|
||||
assert (result[1] is None) == (expected[1] is None)
|
||||
if expected[1] is not None:
|
||||
assert torch.equal(result[1], expected[1])
|
||||
assert torch.equal(result[2], expected[2])
|
||||
args = mock_fused.call_args.args
|
||||
assert args[0] is positions
|
||||
torch.testing.assert_close(args[1], q.view(-1, n_head, head_dim))
|
||||
assert args[2] is rotary.cos_sin_cache
|
||||
assert args[3] is indexer_weights
|
||||
assert args[4] == head_dim**-0.5
|
||||
assert args[5] == n_head**-0.5
|
||||
assert mock_fused.call_args.kwargs == {
|
||||
"use_fp4": False,
|
||||
"weights_out_dtype": torch.float32,
|
||||
}
|
||||
|
||||
@@ -21,7 +21,12 @@ Usage:
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
deepgemm_post_process_fp8_weight_block,
|
||||
)
|
||||
from vllm.models.deepseek_v4.common.ops import fused_inv_rope_fp8_quant
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import fp8_einsum
|
||||
|
||||
# -- Default dimensions matching DeepSeek V3/V4 --------------------------
|
||||
HEAD_DIM = 512
|
||||
@@ -33,6 +38,49 @@ FP8_DTYPE = torch.float8_e4m3fn
|
||||
EPS = 1e-10
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(100), reason="Requires SM100/SM103"
|
||||
)
|
||||
@pytest.mark.parametrize("groups", [2, 4])
|
||||
@pytest.mark.parametrize("tokens", [1, 6, 18, 32])
|
||||
def test_dsv41_fused_wo_a_matches_deepgemm(groups, tokens):
|
||||
"""Fused WO-A matches inverse RoPE + fp8_einsum + MXFP8 quantize."""
|
||||
from flashinfer import mxfp8_quantize
|
||||
|
||||
from vllm.models.deepseek_v41.nvidia.ops.fused_wo_a import _FUSED_WO_A_KERNEL
|
||||
|
||||
torch.manual_seed(0)
|
||||
n = groups * 1024
|
||||
x = torch.randn(tokens, 64, 512, device="cuda", dtype=torch.bfloat16)[:, : n // 128]
|
||||
positions = torch.randint(0, 128, (tokens,), device="cuda")
|
||||
rope = make_cos_sin_cache(128, device="cuda")
|
||||
wq = torch.randn(n, 4096, device="cuda").to(FP8_DTYPE)
|
||||
scale = torch.exp2(torch.randint(-8, 0, (n, 128), device="cuda").float())
|
||||
wq, ws = deepgemm_post_process_fp8_weight_block(
|
||||
wq, scale, (1, 32), False, True, groups
|
||||
)
|
||||
q, sf = _FUSED_WO_A_KERNEL(
|
||||
x=x, positions=positions, rope=rope, weight=wq, weight_scale=ws
|
||||
)
|
||||
|
||||
a, sa = fused_inv_rope_fp8_quant(
|
||||
x, positions, rope, groups, 8, quant_group_size=32, tma_aligned_scales=True
|
||||
)
|
||||
y = torch.empty(tokens, groups, 1024, device="cuda", dtype=torch.bfloat16)
|
||||
fp8_einsum("bhr,hdr->bhd", (a, sa), (wq, ws), y, recipe=(1, 1, 32))
|
||||
ref, ref_sf = mxfp8_quantize(y.flatten(1), backend="cute-dsl")
|
||||
|
||||
def rows(scales): # F8_128x4 bytes -> [128 rows, n / 32 blocks]
|
||||
return scales.view(n // 128, 32, 4, 4).permute(2, 1, 0, 3).reshape(128, -1)
|
||||
|
||||
actual, expected = (
|
||||
d.float().view(tokens, -1, 32) * torch.exp2(rows(s)[:tokens, :, None] - 127.0)
|
||||
for d, s in ((q, sf), (ref, ref_sf))
|
||||
)
|
||||
assert (actual - expected).norm() / expected.norm() < 3e-3
|
||||
assert not rows(sf)[tokens:].any(), "padded scale rows must be zero"
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Helpers
|
||||
# =========================================================================
|
||||
|
||||
@@ -139,6 +139,7 @@ def test_olmoe_lora_mixed(olmoe_lora_files):
|
||||
max_loras=4,
|
||||
enforce_eager=True,
|
||||
trust_remote_code=True,
|
||||
hf_overrides={"head_dtype": "float32"},
|
||||
enable_chunked_prefill=True,
|
||||
)
|
||||
|
||||
|
||||
@@ -12,6 +12,8 @@ import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
@@ -20,6 +22,7 @@ import pytest
|
||||
from vllm import SamplingParams
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
|
||||
|
||||
class WeightCacheDaemon:
|
||||
@@ -33,6 +36,7 @@ class WeightCacheDaemon:
|
||||
):
|
||||
# Short base path: Unix socket paths are limited to ~107 characters.
|
||||
self.socket_dir = tempfile.mkdtemp(prefix="vllm_ipc_")
|
||||
self.health_port = get_open_port()
|
||||
self._cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
@@ -45,6 +49,10 @@ class WeightCacheDaemon:
|
||||
"--weight-cache-socket-dir",
|
||||
self.socket_dir,
|
||||
"--enforce-eager",
|
||||
"--weight-cache-health-host",
|
||||
"127.0.0.1",
|
||||
"--weight-cache-health-port",
|
||||
str(self.health_port),
|
||||
*(extra_args or []),
|
||||
]
|
||||
self._proc: subprocess.Popen | None = None
|
||||
@@ -63,6 +71,16 @@ class WeightCacheDaemon:
|
||||
assert self._proc is not None and self._proc.stderr is not None
|
||||
self._proc.stderr.read()
|
||||
|
||||
def check_health(self) -> None:
|
||||
url = f"http://127.0.0.1:{self.health_port}/health"
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=5) as response:
|
||||
assert response.status == 200
|
||||
except urllib.error.HTTPError as error:
|
||||
raise AssertionError(
|
||||
f"Weight cache daemon is not ready: HTTP {error.code}"
|
||||
) from error
|
||||
|
||||
def _stop(self) -> None:
|
||||
assert self._proc is not None
|
||||
self._proc.terminate()
|
||||
@@ -200,6 +218,7 @@ def test_ipc_cache_cold_start_and_warm_restart(vllm_runner, case: ModelCase):
|
||||
extra_args=case.daemon_args,
|
||||
) as d:
|
||||
warm_outputs = generate(vllm_runner, case, d.socket_dir, fallback=False)
|
||||
d.check_health()
|
||||
# Warm restart: a second engine lifetime against the same daemon.
|
||||
restart_outputs = generate(vllm_runner, case, d.socket_dir, fallback=False)
|
||||
|
||||
|
||||
@@ -49,6 +49,7 @@ PRUNED_METADATA_FIELDS = {
|
||||
"prefill_state_indices",
|
||||
"prefill_has_initial_state",
|
||||
"spec_sequence_masks",
|
||||
"spec_sequence_masks_cpu",
|
||||
"uniform_spec_sequence_length",
|
||||
"flashinfer_prefill_query_start_loc",
|
||||
"flashinfer_prefill_seq_order",
|
||||
|
||||
@@ -175,6 +175,24 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
|
||||
},
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
"minimax_m3": VitCudagraphTestConfig(
|
||||
model="MiniMaxAI/MiniMax-M3",
|
||||
modalities=["image"],
|
||||
image_prompt=("]~b]user\n]<]image[>[What is in this image?[e~[\n]~b]ai\n"),
|
||||
compilation_config_overrides={
|
||||
"encoder_cudagraph_token_budgets": [512],
|
||||
},
|
||||
vllm_runner_kwargs={
|
||||
"trust_remote_code": True,
|
||||
"load_format": "dummy",
|
||||
"hf_overrides": partial(
|
||||
dummy_hf_overrides,
|
||||
model_arch="MiniMaxM3SparseForConditionalGeneration",
|
||||
),
|
||||
"mm_processor_kwargs": {"max_long_side_pixel": 336},
|
||||
},
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
"qwen3_vl": VitCudagraphTestConfig(
|
||||
model="Qwen/Qwen3-VL-2B-Instruct",
|
||||
image_prompt=qwen_vl_chat_template(
|
||||
|
||||
@@ -5,13 +5,7 @@
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.utils import ensure_current_vllm_config
|
||||
from vllm.distributed.parallel_state import (
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
|
||||
EMBED_DIM = 256
|
||||
NUM_HEADS = 4
|
||||
@@ -21,20 +15,11 @@ WINDOW = 8
|
||||
SEQ_LENS = [5, 37]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def vision_attn_env():
|
||||
init_distributed_environment(
|
||||
world_size=1,
|
||||
rank=0,
|
||||
local_rank=0,
|
||||
distributed_init_method=f"tcp://127.0.0.1:{get_open_port()}",
|
||||
backend="nccl",
|
||||
)
|
||||
@pytest.fixture
|
||||
def vision_attn_env(dist_init):
|
||||
default_dtype = torch.get_default_dtype()
|
||||
torch.set_default_dtype(torch.bfloat16)
|
||||
with ensure_current_vllm_config():
|
||||
initialize_model_parallel(tensor_model_parallel_size=1)
|
||||
yield
|
||||
yield
|
||||
torch.set_default_dtype(default_dtype)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,305 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from xgrammar import Grammar
|
||||
from xgrammar.testing import _is_grammar_accept_string
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionToolsParam,
|
||||
)
|
||||
from vllm.parser.abstract_parser import DelegatingParser
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
from vllm.tool_parsers.abstract_tool_parser import ToolParser
|
||||
from vllm.tool_parsers.qwen3_engine_tool_parser import Qwen3EngineToolParser
|
||||
from vllm.tool_parsers.structural_tag_registry import ToolChoice
|
||||
|
||||
|
||||
class TestToolChoice_Plus_ResponseFormat:
|
||||
"""Note(arpera):
|
||||
Test cases for tool_choice={auto,required} + response_format
|
||||
To keep it short:
|
||||
DelegatingParser.adjust_request behavior in some corner cases is checked there
|
||||
|
||||
Initial bug report:
|
||||
https://github.com/vllm-project/vllm/issues/39929
|
||||
And PR that fixed this:
|
||||
https://github.com/vllm-project/vllm/pull/56086
|
||||
"""
|
||||
|
||||
# ================================
|
||||
# Helper methods
|
||||
# ================================
|
||||
|
||||
@staticmethod
|
||||
def _tools(strict: bool) -> list[ChatCompletionToolsParam]:
|
||||
"""Single get_weather tool, optionally marked as strict"""
|
||||
function: dict[str, Any] = {
|
||||
"name": "get_weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
},
|
||||
}
|
||||
if strict:
|
||||
function["strict"] = True
|
||||
return [ChatCompletionToolsParam(type="function", function=function)]
|
||||
|
||||
@staticmethod
|
||||
def _qwen_tool_call() -> str:
|
||||
"""Tool call for Qwen model that do support structural tag"""
|
||||
return (
|
||||
"<tool_call>\n<function=get_weather>\n<parameter=city>\nParis\n"
|
||||
"</parameter>\n</function>\n</tool_call>"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _json_schema_response_format() -> dict:
|
||||
return {
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": "answer",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {"text": {"type": "string"}},
|
||||
"required": ["text"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _setup_request(
|
||||
tools: list[ChatCompletionToolsParam],
|
||||
tool_choice: ToolChoice,
|
||||
response_format: dict,
|
||||
) -> ChatCompletionRequest:
|
||||
request = ChatCompletionRequest(
|
||||
messages=[], # for our test cases it's always empty
|
||||
model="m", # Just a placeholder, don't pay much attention
|
||||
tools=tools,
|
||||
tool_choice=tool_choice,
|
||||
response_format=response_format,
|
||||
)
|
||||
return request
|
||||
|
||||
@staticmethod
|
||||
def _setup_abstract_parser(
|
||||
tools: list[ChatCompletionToolsParam],
|
||||
) -> DelegatingParser:
|
||||
"""Construct parser that does NOT support structural tag"""
|
||||
|
||||
class TestParser(DelegatingParser):
|
||||
tool_parser_cls = ToolParser
|
||||
|
||||
return TestParser(MagicMock(), tools=tools)
|
||||
|
||||
@staticmethod
|
||||
def _setup_qwen_parser(
|
||||
tools: list[ChatCompletionToolsParam],
|
||||
) -> DelegatingParser:
|
||||
"""Construct parser that supports structural tag"""
|
||||
|
||||
class TestParser(DelegatingParser):
|
||||
tool_parser_cls = Qwen3EngineToolParser
|
||||
|
||||
return TestParser(MagicMock(), tools=tools)
|
||||
|
||||
# ================================
|
||||
# Test cases
|
||||
# tool_choice=auto + response_format
|
||||
# ================================
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
# In this test we check that for response_format
|
||||
# resulting grammar accepts @compliant_output and rejects @non_compliant_output
|
||||
# You can add more examples here if you see some corner cases not covered
|
||||
("response_format", "compliant_output", "non_compliant_output"),
|
||||
[
|
||||
(_json_schema_response_format(), '{"text": "hi"}', '{"foo": 1}'),
|
||||
({"type": "json_object"}, '{"any": 1}', "[1, 2]"),
|
||||
],
|
||||
# We test here two different response_format types:
|
||||
ids=["json_schema", "json_object"],
|
||||
)
|
||||
def test_auto_with_strict_tools(
|
||||
self,
|
||||
response_format: dict,
|
||||
compliant_output: str,
|
||||
non_compliant_output: str,
|
||||
):
|
||||
tools = self._tools(strict=True)
|
||||
request = self._setup_request(
|
||||
tools=tools,
|
||||
tool_choice="auto",
|
||||
response_format=response_format,
|
||||
)
|
||||
parser = self._setup_qwen_parser(tools)
|
||||
out = parser.adjust_request(request)
|
||||
|
||||
# Now check that request does not have response_format anymore
|
||||
# but instead has structured_outputs set as structural tag
|
||||
# And this structural tag is OR operation
|
||||
assert out.tool_choice == "auto"
|
||||
assert out.response_format is None
|
||||
assert out.structured_outputs is not None
|
||||
tag = json.loads(out.structured_outputs.structural_tag)
|
||||
assert tag["format"]["type"] == "or"
|
||||
grammar = Grammar.from_structural_tag(out.structured_outputs.structural_tag)
|
||||
|
||||
assert _is_grammar_accept_string(grammar, compliant_output)
|
||||
assert not _is_grammar_accept_string(grammar, non_compliant_output)
|
||||
|
||||
# Also tool call must be accepted by grammar
|
||||
assert _is_grammar_accept_string(grammar, self._qwen_tool_call())
|
||||
|
||||
# IMPORTANT(arpera): Regression test
|
||||
# If we in adjust_request implementation by mistake
|
||||
# construct structural tag using tool_choice=auto
|
||||
# then such a structural tag would allow plain text as well
|
||||
# We need to be sure that plain text is NOT accepted in our case
|
||||
assert not _is_grammar_accept_string(grammar, "Hello")
|
||||
|
||||
def test_auto_without_strict_tools(self):
|
||||
tools = self._tools(strict=False)
|
||||
request = self._setup_request(
|
||||
tools=tools,
|
||||
tool_choice="auto",
|
||||
response_format=self._json_schema_response_format(),
|
||||
)
|
||||
parser = self._setup_qwen_parser(tools)
|
||||
|
||||
# There must be a warning that tool calls are disabled
|
||||
# Consume that warning
|
||||
with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn:
|
||||
out = parser.adjust_request(request)
|
||||
|
||||
assert out.response_format is not None
|
||||
assert out.structured_outputs is None
|
||||
mock_warn.assert_called_once()
|
||||
|
||||
def test_when_model_does_not_have_structural_tag(self):
|
||||
"""Note(arpera):
|
||||
When model does NOT have structural tag support
|
||||
we apply constraint only for response_format
|
||||
"""
|
||||
tools = self._tools(strict=True)
|
||||
request = self._setup_request(
|
||||
tools=tools,
|
||||
tool_choice="auto",
|
||||
response_format=self._json_schema_response_format(),
|
||||
)
|
||||
# SIC! use parser whose model does NOT support structural tag
|
||||
parser = self._setup_abstract_parser(tools)
|
||||
|
||||
with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn:
|
||||
out = parser.adjust_request(request)
|
||||
|
||||
assert out.response_format is not None
|
||||
assert out.structured_outputs is None
|
||||
mock_warn.assert_called_once()
|
||||
|
||||
def test_auto_with_xgrammar_unsupported_schema(self):
|
||||
"""Conrer case based on Vadim's feedback in PR #56086
|
||||
https://github.com/vllm-project/vllm/pull/56086#pullrequestreview-5329203459
|
||||
"""
|
||||
tools = self._tools(strict=True)
|
||||
request = self._setup_request(
|
||||
tools=tools,
|
||||
tool_choice="auto",
|
||||
response_format={
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": "answer",
|
||||
"schema": {"type": "integer", "multipleOf": 2},
|
||||
},
|
||||
},
|
||||
)
|
||||
parser = self._setup_qwen_parser(tools)
|
||||
|
||||
with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn:
|
||||
out = parser.adjust_request(request)
|
||||
|
||||
assert out.response_format is not None
|
||||
assert out.structured_outputs is None
|
||||
mock_warn.assert_called_once()
|
||||
|
||||
def test_auto_with_xgrammar_unsupported_nested_schema(self):
|
||||
"""Conrer case based on Vadim's feedback in PR #56086
|
||||
https://github.com/vllm-project/vllm/pull/56086#pullrequestreview-5329203459
|
||||
"""
|
||||
tools = self._tools(strict=True)
|
||||
request = self._setup_request(
|
||||
tools=tools,
|
||||
tool_choice="auto",
|
||||
response_format={
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": "answer",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {"value": {"type": "number", "multipleOf": 0.5}},
|
||||
"required": ["value"],
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
parser = self._setup_qwen_parser(tools)
|
||||
|
||||
with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn:
|
||||
out = parser.adjust_request(request)
|
||||
|
||||
assert out.response_format is not None
|
||||
assert out.structured_outputs is None
|
||||
mock_warn.assert_called_once()
|
||||
|
||||
def test_auto_with_lark_grammar(self):
|
||||
"""Conrer case based on Vadim's feedback in PR #56086
|
||||
https://github.com/vllm-project/vllm/pull/56086#pullrequestreview-5329203459
|
||||
Lark grammars are converted to EBNF before being merged into the tag.
|
||||
"""
|
||||
tools = self._tools(strict=True)
|
||||
request = ChatCompletionRequest(
|
||||
messages=[],
|
||||
model="m",
|
||||
tools=tools,
|
||||
tool_choice="auto",
|
||||
structured_outputs=StructuredOutputsParams(grammar='start: "ok"'),
|
||||
)
|
||||
parser = self._setup_qwen_parser(tools)
|
||||
out = parser.adjust_request(request)
|
||||
|
||||
assert out.structured_outputs is not None
|
||||
grammar = Grammar.from_structural_tag(out.structured_outputs.structural_tag)
|
||||
assert _is_grammar_accept_string(grammar, "ok")
|
||||
assert not _is_grammar_accept_string(grammar, "bad")
|
||||
assert _is_grammar_accept_string(grammar, self._qwen_tool_call())
|
||||
|
||||
# ================================
|
||||
# Test cases
|
||||
# tool_choice=required + response_format
|
||||
# ================================
|
||||
|
||||
def test_required(self):
|
||||
tools = self._tools(strict=True)
|
||||
request = self._setup_request(
|
||||
tools=tools,
|
||||
tool_choice="required",
|
||||
response_format=self._json_schema_response_format(),
|
||||
)
|
||||
parser = self._setup_qwen_parser(tools)
|
||||
|
||||
with patch("vllm.parser.abstract_parser.logger.warning_once") as mock_warn:
|
||||
out = parser.adjust_request(request)
|
||||
|
||||
assert out.response_format is None
|
||||
assert out.structured_outputs is not None
|
||||
grammar = Grammar.from_structural_tag(out.structured_outputs.structural_tag)
|
||||
assert _is_grammar_accept_string(grammar, self._qwen_tool_call())
|
||||
assert not _is_grammar_accept_string(grammar, '{"text": "hi"}')
|
||||
mock_warn.assert_called_once()
|
||||
@@ -1196,6 +1196,21 @@ class TestAdjustRequest:
|
||||
"TOOL_CALL_2_FUNCTION_FIRST",
|
||||
],
|
||||
),
|
||||
(
|
||||
{
|
||||
"tool_choice": "auto",
|
||||
"strict_tools": True,
|
||||
"response_format_type": "json_schema",
|
||||
},
|
||||
[
|
||||
"COMMENTARY",
|
||||
"TOOL_CALL_1_CHANNEL_FIRST",
|
||||
"TOOL_CALL_1_FUNCTION_FIRST",
|
||||
"TOOL_CALL_2_CHANNEL_FIRST",
|
||||
"TOOL_CALL_2_FUNCTION_FIRST",
|
||||
"FINAL_JSON_SCHEMA",
|
||||
],
|
||||
),
|
||||
(
|
||||
{"response_format_type": "json_schema"},
|
||||
["FINAL_JSON_SCHEMA"],
|
||||
@@ -1281,6 +1296,7 @@ class TestAdjustRequest:
|
||||
"tool_auto_strict",
|
||||
"tool_required",
|
||||
"tool_named",
|
||||
"pr56086_tool_auto_strict_response_format_json_schema",
|
||||
"response_format_json_schema",
|
||||
"response_format_json_object",
|
||||
"structured_outputs_json",
|
||||
|
||||
@@ -19,7 +19,12 @@ from tests.v1.attention.utils import (
|
||||
try_backend_includes_kv_cache_update,
|
||||
try_get_attention_backend,
|
||||
)
|
||||
from vllm.config import ModelConfig, set_current_vllm_config
|
||||
from vllm.config import (
|
||||
DiffusionConfig,
|
||||
ModelConfig,
|
||||
SpeculativeConfig,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import (
|
||||
@@ -942,8 +947,6 @@ def test_flashinfer_xqa_single_token_decode_preserves_cudagraph_padding(monkeypa
|
||||
impl.o_sf_scale = None
|
||||
impl.window_left = -1
|
||||
impl.sinks = None
|
||||
impl.cache_config = unittest.mock.Mock()
|
||||
impl.cache_config.get_resolved_kv_cache_layout.return_value = KVCacheLayout.LBHNC
|
||||
|
||||
layer = unittest.mock.Mock(
|
||||
_q_scale=torch.tensor(1.0),
|
||||
@@ -959,6 +962,7 @@ def test_flashinfer_xqa_single_token_decode_preserves_cudagraph_padding(monkeypa
|
||||
q_len_per_req=1,
|
||||
)
|
||||
attn_metadata = flashinfer_backend.FlashInferMetadata(
|
||||
kv_cache_layout=KVCacheLayout.LBHNC,
|
||||
num_actual_tokens=2,
|
||||
slot_mapping=torch.empty(0, dtype=torch.int64),
|
||||
q_data_type_prefill=torch.bfloat16,
|
||||
@@ -1070,6 +1074,87 @@ def test_flashinfer_trtllm_gen_padded_decode_uses_varlen_offsets(
|
||||
assert q_cu_seq_lens.tolist() == qo_indptr_values
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda()
|
||||
or AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
||||
reason="FlashInfer with CUDA is required.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"runner_type,spec_kind,has_upper_bound,expected_copies",
|
||||
[
|
||||
("generate", None, True, 0),
|
||||
("pooling", None, True, 0),
|
||||
("generate", None, False, 1),
|
||||
("generate", "spec_decode", True, 1),
|
||||
("generate", "diffusion", True, 1),
|
||||
],
|
||||
)
|
||||
def test_flashinfer_avoids_seq_lens_copy_without_spec_tokens(
|
||||
monkeypatch, runner_type, spec_kind, has_upper_bound, expected_copies
|
||||
):
|
||||
"""Planning reuses the host seq_lens bound only when it is exact.
|
||||
|
||||
Without speculative token accounting (no spec-decode drafts, no dLLM
|
||||
canvas tokens) scheduled tokens always land in KV, so the CPU upper bound
|
||||
is exact regardless of runner type. Any speculative accounting keeps the
|
||||
bound optimistic (rejected tokens are in-flight) and forces the copy.
|
||||
"""
|
||||
import unittest.mock
|
||||
|
||||
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
|
||||
from vllm.v1.attention.backends.utils import PerLayerParameters
|
||||
|
||||
monkeypatch.setattr(
|
||||
flashinfer_backend,
|
||||
"get_per_layer_parameters",
|
||||
lambda *args: {
|
||||
"layer.0": PerLayerParameters(
|
||||
window_left=-1, logits_soft_cap=0.0, sm_scale=0.125
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
config = create_vllm_config(
|
||||
model_name="Qwen/Qwen3-0.6B", max_model_len=64, max_num_seqs=3
|
||||
)
|
||||
config.model_config.runner_type = runner_type
|
||||
config.attention_config.use_trtllm_attention = False
|
||||
if spec_kind == "spec_decode":
|
||||
config.speculative_config = SpeculativeConfig(
|
||||
method="ngram", num_speculative_tokens=3
|
||||
)
|
||||
elif spec_kind == "diffusion":
|
||||
config.diffusion_config = DiffusionConfig(canvas_length=8)
|
||||
device = torch.device(f"{DEVICE_TYPE}:0")
|
||||
# Include a one-token final chunk and both partial and full KV pages.
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
BatchSpec(seq_lens=[17, 20, 32], query_lens=[1, 4, 16]), 16, device
|
||||
)
|
||||
if not has_upper_bound:
|
||||
common_attn_metadata.seq_lens_cpu_upper_bound = None
|
||||
elif spec_kind is not None:
|
||||
# Simulate rejected speculative tokens: CPU says 18, GPU says 17.
|
||||
assert common_attn_metadata.seq_lens_cpu_upper_bound is not None
|
||||
common_attn_metadata.seq_lens_cpu_upper_bound[0] += 1
|
||||
|
||||
with set_current_vllm_config(config):
|
||||
builder = flashinfer_backend.FlashInferMetadataBuilder(
|
||||
create_standard_kv_cache_spec(config), ["layer.0"], config, device
|
||||
)
|
||||
with unittest.mock.patch.object(
|
||||
common_attn_metadata.seq_lens,
|
||||
"cpu",
|
||||
wraps=common_attn_metadata.seq_lens.cpu,
|
||||
) as copy_to_cpu:
|
||||
metadata = builder.build(0, common_attn_metadata)
|
||||
|
||||
assert copy_to_cpu.call_count == expected_copies
|
||||
assert metadata.num_decodes == 1
|
||||
assert metadata.num_prefills == 2
|
||||
assert builder.paged_kv_indptr.cpu[:4].tolist() == [0, 2, 4, 6]
|
||||
assert builder.paged_kv_last_page_len.cpu[:3].tolist() == [1, 4, 16]
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
||||
reason="FlashInfer is not available.",
|
||||
|
||||
@@ -25,6 +25,9 @@ from vllm.v1.attention.backends.mla.prefill.cpu_sdpa import (
|
||||
from vllm.v1.attention.backends.mla.prefill.selector import (
|
||||
get_mla_prefill_backend,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.prefill.zen_cpu_sdpa import (
|
||||
ZenCPUSDPAMLAPrefillBackend,
|
||||
)
|
||||
|
||||
|
||||
def _reference_ragged_attention(
|
||||
@@ -157,8 +160,10 @@ def test_cpu_mla_backend_smoke(tmp_path) -> None:
|
||||
def test_cpu_mla_prefill_backend_selected() -> None:
|
||||
# On CPU (no device capability) the MLA prefill backend must be the
|
||||
# SDPA-based CPU backend, not flash-attn which is unavailable on CPU.
|
||||
# Accelerator-specific subclasses (e.g. the zentorch-backed Zen CPU
|
||||
# backend) are also valid selections.
|
||||
backend_cls = get_mla_prefill_backend(None)
|
||||
assert backend_cls is CPUSDPAMLAPrefillBackend
|
||||
assert issubclass(backend_cls, CPUSDPAMLAPrefillBackend)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only")
|
||||
@@ -248,3 +253,64 @@ def test_cpu_mla_prefill_context_chunk() -> None:
|
||||
assert lse.float().allclose(ref_lse.float(), atol=2e-2)
|
||||
assert out.shape == (5, 4, 16)
|
||||
assert lse.shape == (4, 5)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only")
|
||||
@pytest.mark.skipif(
|
||||
not ZenCPUSDPAMLAPrefillBackend.is_available(),
|
||||
reason="requires a Zen CPU with the zentorch_sdpa op registered",
|
||||
)
|
||||
def test_zen_cpu_mla_prefill_new_tokens() -> None:
|
||||
# Runs the zentorch kernel itself. The head dims are deliberately
|
||||
# asymmetric (v_head_dim < qk_head_dim, as in real MLA) so that the V
|
||||
# padding and output slicing are exercised; the generic-CPU cases above
|
||||
# use equal dims and never reach that code.
|
||||
qk_head_dim = 24
|
||||
v_head_dim = 16
|
||||
backend = ZenCPUSDPAMLAPrefillBackend(
|
||||
num_heads=4,
|
||||
scale=0.25,
|
||||
kv_lora_rank=16,
|
||||
qk_nope_head_dim=16,
|
||||
qk_rope_head_dim=8,
|
||||
v_head_dim=v_head_dim,
|
||||
vllm_config=None,
|
||||
)
|
||||
|
||||
class _PrefillMeta:
|
||||
query_start_loc = torch.tensor([0, 3, 6], dtype=torch.int32)
|
||||
|
||||
backend.prepare_metadata(_PrefillMeta())
|
||||
q = torch.randn(6, 4, qk_head_dim, dtype=torch.bfloat16)
|
||||
k = torch.randn(6, 4, qk_head_dim, dtype=torch.bfloat16)
|
||||
v = torch.randn(6, 4, v_head_dim, dtype=torch.bfloat16)
|
||||
|
||||
out = backend.run_prefill_new_tokens(q, k, v, return_softmax_lse=False)
|
||||
ref, _ = _reference_ragged_attention(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
_PrefillMeta.query_start_loc,
|
||||
_PrefillMeta.query_start_loc,
|
||||
scale=0.25,
|
||||
causal=True,
|
||||
)
|
||||
assert out.shape == (6, 4, v_head_dim)
|
||||
assert out.float().allclose(ref.float(), atol=2e-2)
|
||||
|
||||
# zentorch_sdpa does not return a usable LSE, so that path must fall back
|
||||
# to the fp32 reference and match the generic backend exactly.
|
||||
generic = CPUSDPAMLAPrefillBackend(
|
||||
num_heads=4,
|
||||
scale=0.25,
|
||||
kv_lora_rank=16,
|
||||
qk_nope_head_dim=16,
|
||||
qk_rope_head_dim=8,
|
||||
v_head_dim=v_head_dim,
|
||||
vllm_config=None,
|
||||
)
|
||||
generic.prepare_metadata(_PrefillMeta())
|
||||
zen_out, zen_lse = backend.run_prefill_new_tokens(q, k, v, return_softmax_lse=True)
|
||||
gen_out, gen_lse = generic.run_prefill_new_tokens(q, k, v, return_softmax_lse=True)
|
||||
assert torch.equal(zen_out, gen_out)
|
||||
assert torch.equal(zen_lse, gen_lse)
|
||||
|
||||
@@ -9,6 +9,7 @@ causality), ckv/kpe cache splitting, and the backend's model-shape gates.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -50,6 +51,8 @@ class FakeWrapper:
|
||||
def __init__(self):
|
||||
self.plan_args = None
|
||||
self.run_args = None
|
||||
self._plan_info: Any = None
|
||||
self._planned_backend: Any = None
|
||||
|
||||
def plan(self, *args, **kwargs):
|
||||
self.plan_args = (args, kwargs)
|
||||
@@ -69,8 +72,11 @@ class FakeState:
|
||||
self.kv_indptr = torch.zeros(max_tokens + 1, dtype=torch.int32)
|
||||
self.wrapper = FakeWrapper()
|
||||
self.plan_calls = []
|
||||
self.index_topk = 2048
|
||||
self.index_kpool = 4
|
||||
self.max_valid = 2051
|
||||
|
||||
def plan(self, num_tokens, kv_lens):
|
||||
def plan(self, num_tokens, kv_lens, cam, req_id_per_token):
|
||||
self.plan_calls.append((num_tokens, kv_lens))
|
||||
|
||||
def pack_indices(self, slots):
|
||||
@@ -168,13 +174,12 @@ def test_forward_wiring(monkeypatch, qk_rope, kv_dtype):
|
||||
def test_builder_plans_only_rows_dispatched_to_mqa(monkeypatch, use_mha, num_decodes):
|
||||
"""MHA prefill rows must not make the MQA kernel read beyond its query."""
|
||||
builder = object.__new__(FlashInferMLASparseSM90Builder)
|
||||
builder._index_topk = 2048
|
||||
builder._index_kpool = 4
|
||||
builder._async_scheduling = False
|
||||
builder._adaptive_verification = False
|
||||
builder.state = FakeState(TOPK)
|
||||
builder._attention_layer = SimpleNamespace(_use_sparse_mha=lambda _: use_mha)
|
||||
metadata = object.__new__(sm90_mod.FlashInferMLASparseSM90Metadata)
|
||||
metadata.state = None
|
||||
metadata.req_id_per_token = None
|
||||
metadata.num_prefills = 1
|
||||
metadata.num_decode_tokens = num_decodes
|
||||
monkeypatch.setattr(
|
||||
@@ -206,12 +211,17 @@ def test_builder_plans_only_rows_dispatched_to_mqa(monkeypatch, use_mha, num_dec
|
||||
def test_plan_uses_state_params(monkeypatch):
|
||||
"""The NoPE/rope dims and scale live on the builder state, not the layer.
|
||||
|
||||
plan() takes exact per-row KV lengths; the schedule is rebuilt on every
|
||||
call (contexts grow between steps) and the indptrs are always full-size
|
||||
Without a device batch layout plan() takes exact per-row KV lengths and
|
||||
rebuilds the schedule on every call; the indptrs are always full-size
|
||||
with zero-query padding rows past num_tokens.
|
||||
"""
|
||||
impl, rows = make_impl(64, "auto")
|
||||
wrapper = FakeWrapper()
|
||||
plan_info = [0] * sm90_mod._PLAN_INFO_LEN
|
||||
plan_info[sm90_mod._PI_KV_INDPTR] = 64 * 4
|
||||
wrapper._plan_info = torch.tensor(plan_info)
|
||||
wrapper._planned_backend = SimpleNamespace(
|
||||
_int_workspace_buffer=torch.zeros(64, dtype=torch.int32)
|
||||
)
|
||||
state = sm90_mod._SM90State.__new__(sm90_mod._SM90State)
|
||||
state.device = torch.device("cpu")
|
||||
state.wrapper = wrapper
|
||||
@@ -227,8 +237,9 @@ def test_plan_uses_state_params(monkeypatch):
|
||||
state._qo_cpu = torch.empty(5, dtype=torch.int32)
|
||||
state._kv_cpu = torch.empty(5, dtype=torch.int32)
|
||||
state._lens_cpu = torch.full((4,), TOPK, dtype=torch.int32)
|
||||
state._saved_kv_end = torch.empty(0, dtype=torch.int32)
|
||||
|
||||
state.plan(3, torch.tensor([2, 5, 7], dtype=torch.int32))
|
||||
state.plan(3, torch.tensor([2, 5, 7], dtype=torch.int32), None, None)
|
||||
assert wrapper.plan_args is not None
|
||||
args, kwargs = wrapper.plan_args
|
||||
(qo, kv, indices, kv_len, heads, ckv, kpe, page, causal, scale) = args
|
||||
@@ -241,10 +252,10 @@ def test_plan_uses_state_params(monkeypatch):
|
||||
assert kwargs["kv_data_type"] == torch.bfloat16
|
||||
|
||||
# Replanning a smaller batch must clear the previous rows' lengths.
|
||||
state.plan(1, torch.tensor([TOPK], dtype=torch.int32))
|
||||
state.plan(1, torch.tensor([TOPK], dtype=torch.int32), None, None)
|
||||
assert state._kv_cpu.tolist() == [0, TOPK, TOPK, TOPK, TOPK]
|
||||
assert state._lens_cpu.tolist() == [TOPK, 0, 0, 0]
|
||||
state.plan(0, torch.empty(0, dtype=torch.int32))
|
||||
state.plan(0, torch.empty(0, dtype=torch.int32), None, None)
|
||||
assert state._kv_cpu.tolist() == [0, 0, 0, 0, 0]
|
||||
assert state._lens_cpu.tolist() == [0, 0, 0, 0]
|
||||
|
||||
@@ -279,38 +290,49 @@ def test_pack_indices_replays_with_updated_offsets(width):
|
||||
assert (state.kv_indices[expected.numel() :] == -99).all()
|
||||
|
||||
|
||||
def test_kv_lens_host_formula():
|
||||
@pytest.mark.parametrize("adaptive", [False, True])
|
||||
def test_kv_lens_host_formula(adaptive):
|
||||
"""Per-row host lengths: context == position + 1; capped at
|
||||
index_topk + trailing-pool remainder past the sparse threshold."""
|
||||
index_topk + trailing-pool remainder past the sparse threshold.
|
||||
|
||||
Exact (synced) without a host upper bound; with one, a sync-free bound
|
||||
capped at the largest valid count, which the plan clamps on device.
|
||||
"""
|
||||
builder = object.__new__(FlashInferMLASparseSM90Builder)
|
||||
builder._index_topk = 2048
|
||||
builder._index_kpool = 4
|
||||
builder._async_scheduling = False
|
||||
builder.state = FakeState(TOPK)
|
||||
builder._adaptive_verification = adaptive
|
||||
seq_lens = torch.tensor([100, 9, 3000], dtype=torch.int32)
|
||||
cam = SimpleNamespace(
|
||||
num_reqs=3,
|
||||
query_start_loc_cpu=torch.tensor([0, 5, 7, 10], dtype=torch.int32),
|
||||
seq_lens=torch.tensor([100, 9, 3000], dtype=torch.int32),
|
||||
seq_lens_cpu_upper_bound=torch.tensor([100, 9, 3000], dtype=torch.int32),
|
||||
positions=None,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu_upper_bound=None,
|
||||
)
|
||||
num_rows, lens = builder._kv_lens_host(cam)
|
||||
assert num_rows == 10
|
||||
num_rows, lens, exact = builder._kv_lens_host(cam)
|
||||
assert num_rows == 10 and exact
|
||||
# req0: positions 95..99 -> ctx 96..100 (all <= 2048: full context)
|
||||
# req1: positions 7,8 -> ctx 8,9
|
||||
# req2: positions 2997..2999 -> ctx 2998..3000 (> 2048: topk + ctx%4)
|
||||
assert lens.tolist() == [96, 97, 98, 99, 100, 8, 9, 2050, 2051, 2048]
|
||||
|
||||
cam.seq_lens_cpu_upper_bound = seq_lens
|
||||
num_rows, lens, exact = builder._kv_lens_host(cam)
|
||||
assert num_rows == 10 and not exact
|
||||
if adaptive:
|
||||
# Host query split may differ from the device one: batch-wide bound.
|
||||
assert lens.tolist() == [2051] * 10
|
||||
else:
|
||||
assert lens.tolist() == [96, 97, 98, 99, 100, 8, 9, 2051, 2051, 2051]
|
||||
|
||||
|
||||
def test_kv_lens_host_empty():
|
||||
builder = object.__new__(FlashInferMLASparseSM90Builder)
|
||||
builder._index_topk = 2048
|
||||
builder._index_kpool = 4
|
||||
cam = SimpleNamespace(
|
||||
num_reqs=0,
|
||||
query_start_loc_cpu=torch.tensor([0], dtype=torch.int32),
|
||||
seq_lens=torch.zeros(0, dtype=torch.int32),
|
||||
)
|
||||
num_rows, lens = builder._kv_lens_host(cam)
|
||||
num_rows, lens, _ = builder._kv_lens_host(cam)
|
||||
assert num_rows == 0 and lens.numel() == 0
|
||||
|
||||
|
||||
@@ -334,7 +356,13 @@ def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens)
|
||||
"__init__",
|
||||
lambda *_args, **_kwargs: None,
|
||||
)
|
||||
monkeypatch.setattr(sm90_mod, "_SM90State", lambda *_args, **_kwargs: None)
|
||||
|
||||
class _RecordingState:
|
||||
def __init__(self, *_args, **kwargs):
|
||||
self.index_topk = kwargs["index_topk"]
|
||||
self.index_kpool = kwargs["index_kpool"]
|
||||
|
||||
monkeypatch.setattr(sm90_mod, "_SM90State", _RecordingState)
|
||||
impl, _ = make_impl(64)
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=BLOCK_SIZE, num_kv_heads=1, head_size=576, dtype=torch.bfloat16
|
||||
@@ -351,6 +379,7 @@ def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens)
|
||||
max_num_batched_tokens=64, async_scheduling=False
|
||||
),
|
||||
model_config=SimpleNamespace(hf_text_config=hf_config),
|
||||
speculative_config=None,
|
||||
)
|
||||
builder = FlashInferMLASparseSM90Builder(
|
||||
spec, ["attn"], vllm_config, torch.device("cpu")
|
||||
@@ -359,14 +388,138 @@ def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens)
|
||||
cam = SimpleNamespace(
|
||||
num_reqs=2,
|
||||
query_start_loc_cpu=torch.tensor([0, 4, 6], dtype=torch.int32),
|
||||
seq_lens_cpu_upper_bound=torch.tensor([42295, 100], dtype=torch.int32),
|
||||
positions=None,
|
||||
seq_lens=torch.tensor([42295, 100], dtype=torch.int32),
|
||||
seq_lens_cpu_upper_bound=None,
|
||||
)
|
||||
num_rows, lens = builder._kv_lens_host(cam)
|
||||
assert num_rows == 6
|
||||
num_rows, lens, exact = builder._kv_lens_host(cam)
|
||||
assert num_rows == 6 and exact
|
||||
assert lens.tolist() == prefill_lens + [99, 100]
|
||||
|
||||
|
||||
def _sm90_flashinfer_available():
|
||||
return (
|
||||
torch.cuda.is_available()
|
||||
and torch.cuda.get_device_capability()[0] == 9
|
||||
and sm90_mod.has_flashinfer_sm90_nope_mla()
|
||||
)
|
||||
|
||||
|
||||
SM90_HEADS, SM90_TOPK, SM90_KPOOL, SM90_WIDTH = 16, 2048, 4, 2176
|
||||
|
||||
|
||||
def _real_state(rows):
|
||||
return sm90_mod._SM90State(
|
||||
torch.device("cuda"),
|
||||
SM90_HEADS,
|
||||
torch.bfloat16,
|
||||
rows,
|
||||
SM90_WIDTH,
|
||||
HEAD,
|
||||
0,
|
||||
HEAD**-0.5,
|
||||
SM90_TOPK,
|
||||
SM90_KPOOL,
|
||||
)
|
||||
|
||||
|
||||
def _clamp_batch(num_reqs, q_per_req, lo, hi, topk, kpool, width):
|
||||
"""Random batch: per-row contexts, device layout, and top-k slots."""
|
||||
last = torch.randint(lo + q_per_req, hi, (num_reqs,))
|
||||
ctx = (last[:, None] + torch.arange(1 - q_per_req, 1)).reshape(-1)
|
||||
valid = torch.where(ctx <= topk, ctx, topk + ctx % kpool)
|
||||
cam = SimpleNamespace(
|
||||
seq_lens=last.to(torch.int32).cuda(),
|
||||
query_start_loc=(torch.arange(num_reqs + 1) * q_per_req).to(torch.int32).cuda(),
|
||||
)
|
||||
req_id = torch.arange(num_reqs, dtype=torch.int32).repeat_interleave(q_per_req)
|
||||
slots = torch.randint(0, 1 << 16, (ctx.numel(), width), dtype=torch.int32)
|
||||
slots[torch.arange(width) >= valid[:, None]] = -1
|
||||
return ctx, valid.to(torch.int32), cam, req_id.cuda(), slots.cuda()
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _sm90_flashinfer_available(), reason="Requires SM90 FlashInfer MLA"
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"num_reqs,q_per_req,lo,hi",
|
||||
[
|
||||
(64, 1, 1, 3000),
|
||||
(16, 6, 2030, 2070), # MTP-style rows straddling index_topk
|
||||
(4096, 1, 1, 3000), # clamp spans several program blocks
|
||||
],
|
||||
)
|
||||
def test_upper_bound_plan_matches_exact(num_reqs, q_per_req, lo, hi):
|
||||
"""Sync-free planning from optimistic host bounds (async spec decode)
|
||||
must match exact planning: the device clamp trims every work item back
|
||||
to its row's valid count, so no -1 slot is read."""
|
||||
torch.manual_seed(0)
|
||||
heads, topk, kpool, width = SM90_HEADS, SM90_TOPK, SM90_KPOOL, SM90_WIDTH
|
||||
ctx, valid, cam, req_id, slots = _clamp_batch(
|
||||
num_reqs, q_per_req, lo, hi, topk, kpool, width
|
||||
)
|
||||
rows = ctx.numel()
|
||||
|
||||
kv = torch.randn(1 << 16, 1, HEAD, device="cuda", dtype=torch.bfloat16)
|
||||
q = torch.randn(rows, heads, HEAD, device="cuda", dtype=torch.bfloat16)
|
||||
q_pe = q.new_empty(rows, heads, 0)
|
||||
|
||||
def run(state):
|
||||
state.pack_indices(slots)
|
||||
return state.wrapper.run(q, q_pe, kv, kv[..., :0])
|
||||
|
||||
exact = _real_state(rows)
|
||||
exact.plan(rows, valid, None, None)
|
||||
ub = _real_state(rows)
|
||||
host_bound = torch.clamp(ctx + torch.randint(0, 7, (rows,)), max=ub.max_valid)
|
||||
ub.plan(rows, host_bound, cam, req_id)
|
||||
out = run(ub)
|
||||
assert not out.isnan().any()
|
||||
torch.testing.assert_close(out, run(exact), atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _sm90_flashinfer_available(), reason="Requires SM90 FlashInfer MLA"
|
||||
)
|
||||
def test_upper_bound_plan_reused_across_steps(monkeypatch):
|
||||
"""Growing contexts (draft/decode steps) reuse the padded plan and only
|
||||
re-clamp on device, matching an exact replan every step."""
|
||||
torch.manual_seed(0)
|
||||
heads, topk, kpool, width = SM90_HEADS, SM90_TOPK, SM90_KPOOL, SM90_WIDTH
|
||||
rows = 48
|
||||
ctx, _, _, req_id, _ = _clamp_batch(rows, 1, 1, 3000, topk, kpool, width)
|
||||
|
||||
exact, ub = _real_state(rows), _real_state(rows)
|
||||
replans: list[int] = []
|
||||
plan_fn = ub._plan
|
||||
|
||||
def counting_plan(num_tokens, kv_lens):
|
||||
replans.append(num_tokens)
|
||||
plan_fn(num_tokens, kv_lens)
|
||||
|
||||
monkeypatch.setattr(ub, "_plan", counting_plan)
|
||||
kv = torch.randn(1 << 16, 1, HEAD, device="cuda", dtype=torch.bfloat16)
|
||||
for step in range(5):
|
||||
step_ctx = ctx + step
|
||||
valid = torch.where(step_ctx <= topk, step_ctx, topk + step_ctx % kpool)
|
||||
slots = torch.randint(0, 1 << 16, (rows, width), dtype=torch.int32)
|
||||
slots[torch.arange(width) >= valid[:, None]] = -1
|
||||
slots = slots.cuda()
|
||||
cam = SimpleNamespace(
|
||||
seq_lens=step_ctx.to(torch.int32).cuda(),
|
||||
query_start_loc=torch.arange(rows + 1, dtype=torch.int32).cuda(),
|
||||
)
|
||||
num_plans = len(replans)
|
||||
ub.plan(rows, torch.clamp(step_ctx + 5, max=ub.max_valid), cam, req_id)
|
||||
assert len(replans) - num_plans == (1 if step == 0 else 0)
|
||||
exact.plan(rows, valid.to(torch.int32), None, None)
|
||||
q = torch.randn(rows, heads, HEAD, device="cuda", dtype=torch.bfloat16)
|
||||
outs = []
|
||||
for state in (ub, exact):
|
||||
state.pack_indices(slots)
|
||||
outs.append(state.wrapper.run(q, q[..., :0], kv, kv[..., :0]))
|
||||
torch.testing.assert_close(outs[0], outs[1], atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
def test_supports_combination_gates(monkeypatch, default_vllm_config):
|
||||
monkeypatch.setattr(sm90_mod, "has_flashinfer_sm90_nope_mla", lambda: True)
|
||||
call = lambda **kw: FlashInferMLASparseSM90Backend.supports_combination(
|
||||
|
||||
@@ -21,6 +21,7 @@ from vllm.v1.attention.backends.gdn_attn import (
|
||||
GDNAttentionMetadata,
|
||||
GDNAttentionMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import mamba_get_block_table_tensor
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
BLOCK_SIZE = 16
|
||||
@@ -75,6 +76,17 @@ GDN_BUILD_TEST_CASES = {
|
||||
expected_num_prefill_tokens=0,
|
||||
expected_num_spec_decodes=0,
|
||||
),
|
||||
# No speculative config, decode alongside prefill
|
||||
"regular_decode_with_prefill": GDNBuildTestCase(
|
||||
seq_lens=[40, 100],
|
||||
query_lens=[1, 50],
|
||||
num_decode_draft_tokens=None,
|
||||
num_speculative_tokens=0,
|
||||
expected_num_decodes=1,
|
||||
expected_num_prefills=1,
|
||||
expected_num_prefill_tokens=50,
|
||||
expected_num_spec_decodes=0,
|
||||
),
|
||||
# Multi-token prefill alongside spec decode — no decode to reclassify
|
||||
"spec_decode_with_real_prefill": GDNBuildTestCase(
|
||||
seq_lens=[100, 20],
|
||||
@@ -155,9 +167,12 @@ def _build(
|
||||
builder: GDNAttentionMetadataBuilder,
|
||||
batch_spec: BatchSpec,
|
||||
num_decode_draft_tokens: list[int] | None = None,
|
||||
block_table: torch.Tensor | None = None,
|
||||
) -> GDNAttentionMetadata:
|
||||
"""Build GDN attention metadata, optionally with spec-decode kwargs."""
|
||||
common = create_common_attn_metadata(batch_spec, BLOCK_SIZE, DEVICE)
|
||||
if block_table is not None:
|
||||
common = common.replace(block_table_tensor=block_table)
|
||||
kwargs: dict = {}
|
||||
if num_decode_draft_tokens is not None:
|
||||
kwargs["num_decode_draft_tokens_cpu"] = torch.tensor(
|
||||
@@ -184,6 +199,54 @@ def test_gdn_build_classification(test_case: GDNBuildTestCase):
|
||||
assert meta.num_spec_decodes == test_case.expected_num_spec_decodes
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mamba_cache_mode", ["none", "align"])
|
||||
@pytest.mark.parametrize("full_cuda_graph", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"test_case", GDN_BUILD_TEST_CASES.values(), ids=GDN_BUILD_TEST_CASES.keys()
|
||||
)
|
||||
def test_update_block_table_matches_build(
|
||||
test_case: GDNBuildTestCase, full_cuda_graph: bool, mamba_cache_mode: str
|
||||
):
|
||||
"""update_block_table() on another group's metadata matches build()."""
|
||||
batch = BatchSpec(seq_lens=test_case.seq_lens, query_lens=test_case.query_lens)
|
||||
src, dst, ref = (
|
||||
_create_gdn_builder(test_case.num_speculative_tokens, full_cuda_graph)
|
||||
for _ in range(3)
|
||||
)
|
||||
for builder in (src, dst, ref):
|
||||
builder.vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
|
||||
common = create_common_attn_metadata(batch, BLOCK_SIZE, DEVICE)
|
||||
if mamba_cache_mode == "align":
|
||||
# MRV2 precomputes these for every Mamba group each step.
|
||||
dst.mamba_aligned_state_indices = ref.mamba_aligned_state_indices = (
|
||||
mamba_get_block_table_tensor(
|
||||
common.block_table_tensor, common.seq_lens, ref.kv_cache_spec, "align"
|
||||
)
|
||||
)
|
||||
draft_tokens = test_case.num_decode_draft_tokens
|
||||
expected = _build(ref, batch, draft_tokens, common.block_table_tensor)
|
||||
source = _build(src, batch, draft_tokens)
|
||||
fields = (
|
||||
"spec_state_indices_tensor",
|
||||
"non_spec_state_indices_tensor",
|
||||
"prefill_state_indices",
|
||||
)
|
||||
source_indices = [getattr(source, f) for f in fields]
|
||||
source_indices = [t if t is None else t.clone() for t in source_indices]
|
||||
meta = dst.update_block_table(
|
||||
source, common.block_table_tensor, common.slot_mapping
|
||||
)
|
||||
|
||||
for field, source_index in zip(fields, source_indices):
|
||||
actual = getattr(meta, field)
|
||||
torch.testing.assert_close(actual, getattr(expected, field))
|
||||
# The source group's indices are untouched.
|
||||
torch.testing.assert_close(getattr(source, field), source_index)
|
||||
# FULL graph state indices land in this group's own buffers.
|
||||
if full_cuda_graph and meta.num_prefills == 0 and actual is not None:
|
||||
assert actual.data_ptr() == getattr(dst, field).data_ptr()
|
||||
|
||||
|
||||
def test_has_initial_state_after_reclassification():
|
||||
"""After reclassification, num_prefills > 0 so the prefill kernel path
|
||||
should compute has_initial_state. For the reclassified request with
|
||||
|
||||
@@ -101,6 +101,7 @@ def test_flash_attention_geometry_comes_from_the_group():
|
||||
num_kv_heads=2,
|
||||
head_size=64,
|
||||
dtype=torch.bfloat16,
|
||||
dcp_sharded=True,
|
||||
)
|
||||
|
||||
with (
|
||||
|
||||
@@ -60,10 +60,43 @@ class TestGetMLAPrefillBackend:
|
||||
|
||||
def test_cpu_uses_sdpa_prefill(self):
|
||||
vllm_config = _make_vllm_config()
|
||||
zen_cls = MLAPrefillBackendEnum.ZEN_CPU.get_class()
|
||||
|
||||
# A Zen CPU with zentorch selects the Zen backend, so expect whichever
|
||||
# one this host offers.
|
||||
expected = (
|
||||
zen_cls if zen_cls.is_available() else MLAPrefillBackendEnum.CPU.get_class()
|
||||
)
|
||||
|
||||
with patch("vllm.platforms.current_platform") as mock_platform:
|
||||
mock_platform.is_cpu.return_value = True
|
||||
|
||||
backend = get_mla_prefill_backend(vllm_config)
|
||||
assert backend is expected
|
||||
|
||||
def test_zen_cpu_prefers_zentorch_prefill(self):
|
||||
vllm_config = _make_vllm_config()
|
||||
zen_cls = MLAPrefillBackendEnum.ZEN_CPU.get_class()
|
||||
|
||||
with (
|
||||
patch("vllm.platforms.current_platform") as mock_platform,
|
||||
patch.object(zen_cls, "is_available", return_value=True),
|
||||
):
|
||||
mock_platform.is_cpu.return_value = True
|
||||
|
||||
backend = get_mla_prefill_backend(vllm_config)
|
||||
assert backend is zen_cls
|
||||
|
||||
def test_zen_cpu_falls_back_when_zentorch_unavailable(self):
|
||||
vllm_config = _make_vllm_config()
|
||||
zen_cls = MLAPrefillBackendEnum.ZEN_CPU.get_class()
|
||||
|
||||
with (
|
||||
patch("vllm.platforms.current_platform") as mock_platform,
|
||||
patch.object(zen_cls, "is_available", return_value=False),
|
||||
):
|
||||
mock_platform.is_cpu.return_value = True
|
||||
|
||||
backend = get_mla_prefill_backend(vllm_config)
|
||||
assert backend is MLAPrefillBackendEnum.CPU.get_class()
|
||||
|
||||
|
||||
@@ -1599,19 +1599,142 @@ def test_project_kv_cache_groups_to_worker():
|
||||
assert set(proj_spec.kv_cache_specs.keys()) == {"layer1", "layer3"}
|
||||
|
||||
|
||||
def test_dcp_world_size_for_kv_cache_spec_shards_full_attention_only():
|
||||
dcp = 8
|
||||
full = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=1, dtype=torch.float32
|
||||
@pytest.mark.parametrize("sliding_window", [None, 256])
|
||||
@pytest.mark.parametrize("disable_hybrid", [False, True])
|
||||
@pytest.mark.parametrize("pcp_size", [1, 4])
|
||||
def test_dcp_target_allocates_replicated_draft_independently(
|
||||
monkeypatch, sliding_window, disable_hybrid, pcp_size
|
||||
):
|
||||
"""A draft must retain all positions even when the target shards them."""
|
||||
from vllm.v1.attention.backends.utils import resolve_kv_cache_layout
|
||||
|
||||
monkeypatch.delenv("VLLM_KV_CACHE_LAYOUT", raising=False)
|
||||
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
|
||||
config.parallel_config.decode_context_parallel_size = 4
|
||||
config.parallel_config.prefill_context_parallel_size = pcp_size
|
||||
config.scheduler_config.disable_hybrid_kv_cache_manager = disable_hybrid
|
||||
config.cache_config.block_size = 16
|
||||
config.cache_config.kv_cache_layout = None
|
||||
draft_args = dict(
|
||||
block_size=16,
|
||||
num_kv_heads=8,
|
||||
head_size=64,
|
||||
dtype=torch.bfloat16,
|
||||
dcp_sharded=False,
|
||||
)
|
||||
mla = new_mla_spec()
|
||||
mamba = new_mamba_spec()
|
||||
uniform_mla = UniformTypeKVCacheSpecs(block_size=16, kv_cache_specs={"layer": mla})
|
||||
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(full, dcp) == dcp
|
||||
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(mla, dcp) == dcp
|
||||
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(uniform_mla, dcp) == dcp
|
||||
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(mamba, dcp) == 1
|
||||
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(full, 1) == 1
|
||||
draft = (
|
||||
FullAttentionSpec(**draft_args)
|
||||
if sliding_window is None
|
||||
else SlidingWindowSpec(**draft_args, sliding_window=sliding_window)
|
||||
)
|
||||
specs = {"target": new_mla_spec(), "draft": draft}
|
||||
layout = resolve_kv_cache_layout(config, [["LBHNC", "BLHNC"]], specs.values())
|
||||
assert layout == KVCacheLayout.BLHNC
|
||||
groups = get_kv_cache_groups(config, specs)
|
||||
assert [group.layer_names for group in groups] == [["target"], ["draft"]]
|
||||
widths = [g.kv_cache_spec.max_num_blocks_per_req(config, 1024) for g in groups]
|
||||
assert widths == [16, 64]
|
||||
|
||||
cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
config, groups, available_memory=16 * 1024 * 1024
|
||||
)
|
||||
scheduler_config = generate_scheduler_kv_cache_config([cache_config])
|
||||
assert get_max_concurrency_for_kv_cache_config(config, cache_config) > 0
|
||||
assert get_max_concurrency_for_kv_cache_config(config, cache_config) == (
|
||||
get_max_concurrency_for_kv_cache_config(config, scheduler_config)
|
||||
)
|
||||
manager = KVCacheManager(
|
||||
scheduler_config,
|
||||
max_model_len=1024,
|
||||
hash_block_size=16,
|
||||
scheduler_block_size=64,
|
||||
dcp_world_size=4,
|
||||
pcp_world_size=pcp_size,
|
||||
enable_caching=True,
|
||||
)
|
||||
assert manager.coordinator.group_block_sizes == (64, 16)
|
||||
request = make_request("replicated-draft", [1] * 65, block_size=16, hash_fn=sha256)
|
||||
blocks = manager.allocate_slots(request, 65)
|
||||
assert blocks is not None
|
||||
assert [len(group) for group in blocks.blocks] == [2, 5]
|
||||
manager.cache_blocks(request, 64)
|
||||
cached_request = make_request(
|
||||
"cached-draft", [1] * 65, block_size=16, hash_fn=sha256
|
||||
)
|
||||
cached_blocks, num_cached, _ = manager.get_computed_blocks(cached_request)
|
||||
assert num_cached == 64
|
||||
assert [len(group) for group in cached_blocks.blocks] == [1, 4]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_mla", [False, True])
|
||||
def test_full_attention_merge_preserves_replicated_cache_geometry(use_mla):
|
||||
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
|
||||
config.parallel_config.decode_context_parallel_size = 4
|
||||
draft = replace(
|
||||
new_mla_spec() if use_mla else new_kv_cache_spec(),
|
||||
block_size=16,
|
||||
dcp_sharded=False,
|
||||
)
|
||||
merged = type(draft).merge([draft, draft])
|
||||
wrapped = UniformTypeKVCacheSpecs.from_specs({"draft": merged})
|
||||
assert wrapped is not None and not wrapped.dcp_sharded
|
||||
assert merged.max_num_blocks_per_req(config, 1024) == 64
|
||||
assert merged.max_memory_usage_bytes(config) == 64 * draft.page_size_bytes
|
||||
assert kv_cache_utils.resolve_dcp_kv_block_size(merged, 4) == 16
|
||||
with pytest.raises(AssertionError):
|
||||
type(draft).merge([draft, replace(draft, dcp_sharded=True)])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("with_draft", [False, True])
|
||||
@pytest.mark.parametrize("indexer_alignment", [None, 512])
|
||||
def test_sparse_mla_preserves_physical_row_addressing(with_draft, indexer_alignment):
|
||||
from vllm.v1.attention.backends.mla.sparse_utils import flat_kv_row_view
|
||||
from vllm.v1.worker.utils import allocate_kv_cache
|
||||
|
||||
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
|
||||
config.cache_config.kv_cache_layout = "BLHNC"
|
||||
config.parallel_config.decode_context_parallel_size = 4
|
||||
common = dict(block_size=64, num_kv_heads=1, dtype=torch.uint8)
|
||||
specs = {
|
||||
"target": MLAAttentionSpec(
|
||||
**common,
|
||||
head_size=576,
|
||||
state_content_bytes=656,
|
||||
block_stride_alignment=656,
|
||||
cache_dtype_str="fp8_ds_mla",
|
||||
is_index_group_leader=True,
|
||||
),
|
||||
"indexer": MLAAttentionSpec(
|
||||
**common,
|
||||
head_size=132,
|
||||
cache_role=SparseCacheRole.INDEXER,
|
||||
block_stride_alignment=indexer_alignment,
|
||||
),
|
||||
"draft": SlidingWindowSpec(
|
||||
block_size=64,
|
||||
num_kv_heads=64,
|
||||
head_size=64,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=2048,
|
||||
dcp_sharded=False,
|
||||
),
|
||||
}
|
||||
if not with_draft:
|
||||
del specs["draft"]
|
||||
groups = get_kv_cache_groups(config, specs)
|
||||
cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
config, groups, available_memory=8 * 1024 * 1024
|
||||
)
|
||||
caches = allocate_kv_cache(
|
||||
cache_config, torch.device("cpu"), KVCacheLayout.BLHNC, [64] * len(groups)
|
||||
)
|
||||
cache = caches["target"].squeeze(1)
|
||||
if indexer_alignment is not None:
|
||||
assert caches["indexer"].stride(0) % indexer_alignment == 0
|
||||
rows, stride_rows = flat_kv_row_view(cache, 64)
|
||||
cache[1, 0].fill_(7)
|
||||
torch.testing.assert_close(rows[stride_rows], cache[1, 0])
|
||||
assert torch.all(rows[stride_rows] == 7)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -3815,6 +3938,8 @@ def test_unify_hybrid_kv_cache_specs():
|
||||
"layer_1": before_spec_1,
|
||||
"layer_2": before_spec_2,
|
||||
}
|
||||
kv_cache_spec["draft_layer_1"] = replace(before_spec_1, dcp_sharded=False)
|
||||
kv_cache_spec["draft_layer_2"] = replace(before_spec_2, dcp_sharded=False)
|
||||
kv_cache_utils.unify_hybrid_kv_cache_specs(kv_cache_spec)
|
||||
expected_spec_1 = new_kv_cache_spec(block_size=64)
|
||||
expected_spec_2 = new_kv_cache_spec(
|
||||
@@ -3823,6 +3948,7 @@ def test_unify_hybrid_kv_cache_specs():
|
||||
assert kv_cache_spec["layer_1"] == expected_spec_1
|
||||
assert kv_cache_spec["layer_2"] == expected_spec_2
|
||||
assert kv_cache_spec["layer_2"].page_size_bytes == 64 * 1024
|
||||
assert kv_cache_spec["draft_layer_2"] == replace(expected_spec_2, dcp_sharded=False)
|
||||
|
||||
# 2. has_full_attention and has_chunked_local_attention
|
||||
before_spec_1 = new_kv_cache_spec()
|
||||
@@ -3874,6 +4000,11 @@ def test_unify_hybrid_kv_cache_specs():
|
||||
with pytest.raises(ValueError):
|
||||
kv_cache_utils.unify_hybrid_kv_cache_specs(kv_cache_spec)
|
||||
|
||||
# Replicated Mamba state still requires the hybrid cache manager.
|
||||
kv_cache_spec = {"attention": new_kv_cache_spec(), "mamba": new_mamba_spec()}
|
||||
with pytest.raises(ValueError):
|
||||
kv_cache_utils.unify_hybrid_kv_cache_specs(kv_cache_spec)
|
||||
|
||||
|
||||
def test_unify_kv_cache_spec_page_size_mamba():
|
||||
"""Regression test for https://github.com/vllm-project/vllm/issues/43626.
|
||||
|
||||
@@ -1336,12 +1336,12 @@ def make_kv_cache_config_three_types(
|
||||
)
|
||||
|
||||
|
||||
def test_prefix_cache_hit_uses_per_group_dcp_geometry():
|
||||
@pytest.mark.parametrize("draft_sharded", [True, False])
|
||||
def test_prefix_cache_hit_uses_per_group_dcp_geometry(draft_sharded):
|
||||
"""Prefix lookup must use each group's DCP size, not the process-wide one.
|
||||
|
||||
Target and draft MLA are both sharded (DCP=8); Mamba stays replicated
|
||||
(DCP=1). Hits then align to the sharded full-attention block, not the
|
||||
unsharded page size.
|
||||
Target MLA is sharded; draft MLA can opt out and Mamba stays replicated.
|
||||
Hits must align to the target block in either case.
|
||||
"""
|
||||
block_size = 16
|
||||
dcp = 8
|
||||
@@ -1362,6 +1362,7 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry():
|
||||
KVCacheGroupSpec(
|
||||
["draft_mla"],
|
||||
MLAAttentionSpec(
|
||||
dcp_sharded=draft_sharded,
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
@@ -1388,10 +1389,10 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry():
|
||||
)
|
||||
target_mgr, draft_mgr, mamba_mgr = manager.coordinator.single_type_managers
|
||||
assert target_mgr.dcp_world_size == dcp
|
||||
assert draft_mgr.dcp_world_size == dcp
|
||||
assert draft_mgr.dcp_world_size == (dcp if draft_sharded else 1)
|
||||
assert mamba_mgr.dcp_world_size == 1
|
||||
assert target_mgr.block_size == sharded_block
|
||||
assert draft_mgr.block_size == sharded_block
|
||||
assert draft_mgr.block_size == (sharded_block if draft_sharded else block_size)
|
||||
assert mamba_mgr.block_size == block_size
|
||||
|
||||
hash_fn = sha256
|
||||
@@ -1407,7 +1408,11 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry():
|
||||
req1 = make_request("1", common_token_ids + [100] * 5, block_size, hash_fn)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
|
||||
assert num_computed_tokens == 2 * sharded_block
|
||||
assert [len(group) for group in computed_blocks.blocks] == [2, 2, 16]
|
||||
assert [len(group) for group in computed_blocks.blocks] == [
|
||||
2,
|
||||
2 if draft_sharded else 16,
|
||||
16,
|
||||
]
|
||||
|
||||
manager.free(req0)
|
||||
manager.free(req1)
|
||||
|
||||
@@ -11,6 +11,7 @@ from vllm.config import CompilationConfig, VllmConfig, replace
|
||||
from vllm.config.kernel import MoEBackend
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.spec_decode.draft_model import DraftModelProposer
|
||||
|
||||
from ..utils import (
|
||||
Messages,
|
||||
@@ -196,6 +197,38 @@ def test_draft_model_engine_args_tensor_parallelism():
|
||||
assert draft_config.quant_config is None
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize("dcp_size", [1, 2])
|
||||
def test_draft_model_inherits_target_dcp_config(monkeypatch, dcp_size):
|
||||
monkeypatch.setattr(current_platform, "device_count", lambda: 16)
|
||||
target_config = EngineArgs(
|
||||
model="Qwen/Qwen3-1.7B",
|
||||
tensor_parallel_size=16,
|
||||
decode_context_parallel_size=dcp_size,
|
||||
cp_kv_cache_interleave_size=16,
|
||||
dcp_comm_backend="ag_rs",
|
||||
distributed_executor_backend="mp",
|
||||
speculative_config={
|
||||
"model": "Qwen/Qwen3-0.6B",
|
||||
"method": "draft_model",
|
||||
"num_speculative_tokens": 3,
|
||||
},
|
||||
).create_engine_config()
|
||||
proposer = object.__new__(DraftModelProposer)
|
||||
proposer.vllm_config = target_config
|
||||
proposer.speculative_config = target_config.speculative_config
|
||||
draft_config = proposer._create_draft_vllm_config()
|
||||
draft_parallel = draft_config.parallel_config
|
||||
assert draft_parallel.decode_context_parallel_size == dcp_size
|
||||
assert draft_parallel.cp_kv_cache_interleave_size == 16
|
||||
assert draft_parallel.dcp_comm_backend == "ag_rs"
|
||||
assert (
|
||||
draft_parallel.dcp_q_replicate == target_config.parallel_config.dcp_q_replicate
|
||||
)
|
||||
assert draft_parallel is not target_config.parallel_config
|
||||
assert target_config.parallel_config.decode_context_parallel_size == dcp_size
|
||||
|
||||
|
||||
def _apply_draft_moe_backend(vllm_config: VllmConfig) -> VllmConfig:
|
||||
"""Replicate SpecDecodeBaseProposer._create_draft_vllm_config logic
|
||||
so we can test it without instantiating a full proposer."""
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.config import CompilationConfig
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ..utils import (
|
||||
_skip_if_insufficient_gpus_for_tp,
|
||||
assert_request_outputs_match,
|
||||
evaluate_llm_for_gsm8k,
|
||||
get_test_prompts,
|
||||
)
|
||||
|
||||
|
||||
def check_mtp_correctness(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
sampling_config: SamplingParams,
|
||||
model_setup: tuple[str, str, int, str | None],
|
||||
mm_enabled: bool,
|
||||
expected_accuracy_threshold: float,
|
||||
vllm_runner,
|
||||
):
|
||||
"""Compare the outputs of a original LLM and a speculative LLM
|
||||
which should be the same when using MTP speculative decoding. Due to some variance
|
||||
in the engine, it is possible for some outputs to differ, so we expect that at least
|
||||
6/10 output tokens match exactly, and that the GSM8k accuracy is above a precomputed
|
||||
reference threshold for each model.
|
||||
"""
|
||||
method, model_name, tp_size, draft_model = model_setup
|
||||
_skip_if_insufficient_gpus_for_tp(tp_size)
|
||||
|
||||
# Generate test prompts inside the function instead of using fixture
|
||||
test_prompts = get_test_prompts(mm_enabled)
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_MLA_DISABLE", "1")
|
||||
|
||||
attn_backend = "TRITON_ATTN" if current_platform.is_rocm() else "auto"
|
||||
|
||||
# Skip multimodal profiling for models that don't need it in this test.
|
||||
extra_kwargs: dict[str, Any] = {}
|
||||
if "Qwen3.5" in model_name:
|
||||
extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "video": 0}
|
||||
elif "gemma-4" in model_name:
|
||||
extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "audio": 0}
|
||||
|
||||
if draft_model is not None and "gemma-4" in draft_model:
|
||||
import transformers
|
||||
from packaging.version import Version
|
||||
|
||||
if Version(transformers.__version__) < Version("5.8.0"):
|
||||
pytest.skip(
|
||||
"Gemma4 MTP assistant requires transformers>=5.8.0, "
|
||||
f"got {transformers.__version__}"
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
model_name,
|
||||
block_size=None,
|
||||
max_model_len=2048,
|
||||
tensor_parallel_size=tp_size,
|
||||
trust_remote_code=True,
|
||||
attention_backend=attn_backend,
|
||||
enable_chunked_prefill=None,
|
||||
compilation_config=CompilationConfig(),
|
||||
**extra_kwargs,
|
||||
) as ref_runner:
|
||||
ref_outputs = ref_runner.llm.chat(test_prompts, sampling_config)
|
||||
evaluate_llm_for_gsm8k(
|
||||
ref_runner.llm,
|
||||
expected_accuracy_threshold=expected_accuracy_threshold,
|
||||
)
|
||||
|
||||
speculative_config: dict[str, Any] = {
|
||||
"method": method,
|
||||
"num_speculative_tokens": 1,
|
||||
"max_model_len": 2048,
|
||||
}
|
||||
if draft_model is not None:
|
||||
speculative_config["model"] = draft_model
|
||||
speculative_config["num_speculative_tokens"] = 2
|
||||
|
||||
with vllm_runner(
|
||||
model_name,
|
||||
block_size=None,
|
||||
trust_remote_code=True,
|
||||
tensor_parallel_size=tp_size,
|
||||
speculative_config=speculative_config,
|
||||
max_model_len=2048,
|
||||
attention_backend=attn_backend,
|
||||
enable_chunked_prefill=None,
|
||||
compilation_config=CompilationConfig(),
|
||||
**extra_kwargs,
|
||||
) as spec_runner:
|
||||
# MTP supports async scheduling by default.
|
||||
has_async = (
|
||||
spec_runner.llm.llm_engine.vllm_config.scheduler_config.async_scheduling
|
||||
)
|
||||
assert has_async, (
|
||||
f"Expected async scheduling for {method}: target={model_name}, "
|
||||
f"draft={draft_model}; got {has_async}"
|
||||
)
|
||||
evaluate_llm_for_gsm8k(
|
||||
spec_runner.llm,
|
||||
expected_accuracy_threshold=expected_accuracy_threshold,
|
||||
)
|
||||
spec_outputs = spec_runner.llm.chat(test_prompts, sampling_config)
|
||||
|
||||
# Heuristic: expect at least 80% of the prompts to match exactly
|
||||
# Upon failure, inspect the outputs to check for inaccuracy.
|
||||
assert_request_outputs_match(
|
||||
ref_outputs,
|
||||
spec_outputs,
|
||||
required_matches=int(0.8 * len(ref_outputs)) + 1,
|
||||
context=f"{method} target={model_name}, draft={draft_model}",
|
||||
)
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,44 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.utils import single_gpu_only
|
||||
from vllm import SamplingParams
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .._correctness import check_mtp_correctness
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
["model_setup", "mm_enabled", "expected_accuracy_threshold"],
|
||||
[
|
||||
pytest.param(
|
||||
("mtp", "ZixiQi/DeepSeek-V3-4layers-MTP-FP8", 1, None),
|
||||
False,
|
||||
0.0,
|
||||
marks=pytest.mark.skipif(
|
||||
current_platform.is_device_capability_family(100),
|
||||
reason="DeepSeek MTP: TRTLLM MoE top_k check fails on Blackwell",
|
||||
),
|
||||
), # dummy model
|
||||
],
|
||||
ids=["deepseek"],
|
||||
)
|
||||
@single_gpu_only
|
||||
def test_mtp_correctness(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
sampling_config: SamplingParams,
|
||||
model_setup: tuple[str, str, int, str | None],
|
||||
mm_enabled: bool,
|
||||
expected_accuracy_threshold: float,
|
||||
vllm_runner,
|
||||
):
|
||||
check_mtp_correctness(
|
||||
monkeypatch,
|
||||
sampling_config,
|
||||
model_setup,
|
||||
mm_enabled,
|
||||
expected_accuracy_threshold,
|
||||
vllm_runner,
|
||||
)
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,39 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.utils import single_gpu_only
|
||||
from vllm import SamplingParams
|
||||
|
||||
from .._correctness import check_mtp_correctness
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
["model_setup", "mm_enabled", "expected_accuracy_threshold"],
|
||||
[
|
||||
(
|
||||
("mtp", "google/gemma-4-E4B-it", 1, "google/gemma-4-E4B-it-assistant"),
|
||||
False,
|
||||
0.50,
|
||||
), # gemma4 MTP with assistant model, ref: ~62%
|
||||
],
|
||||
ids=["gemma4-e4b"],
|
||||
)
|
||||
@single_gpu_only
|
||||
def test_mtp_correctness(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
sampling_config: SamplingParams,
|
||||
model_setup: tuple[str, str, int, str | None],
|
||||
mm_enabled: bool,
|
||||
expected_accuracy_threshold: float,
|
||||
vllm_runner,
|
||||
):
|
||||
check_mtp_correctness(
|
||||
monkeypatch,
|
||||
sampling_config,
|
||||
model_setup,
|
||||
mm_enabled,
|
||||
expected_accuracy_threshold,
|
||||
vllm_runner,
|
||||
)
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,36 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.utils import single_gpu_only
|
||||
from vllm import SamplingParams
|
||||
|
||||
from .._correctness import check_mtp_correctness
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
["model_setup", "mm_enabled", "expected_accuracy_threshold"],
|
||||
[
|
||||
# Reference accuracy: 65%-70%.
|
||||
(("mtp", "XiaomiMiMo/MiMo-7B-Base", 1, None), False, 0.5),
|
||||
],
|
||||
ids=["mimo"],
|
||||
)
|
||||
@single_gpu_only
|
||||
def test_mtp_correctness(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
sampling_config: SamplingParams,
|
||||
model_setup: tuple[str, str, int, str | None],
|
||||
mm_enabled: bool,
|
||||
expected_accuracy_threshold: float,
|
||||
vllm_runner,
|
||||
):
|
||||
check_mtp_correctness(
|
||||
monkeypatch,
|
||||
sampling_config,
|
||||
model_setup,
|
||||
mm_enabled,
|
||||
expected_accuracy_threshold,
|
||||
vllm_runner,
|
||||
)
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,39 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.utils import single_gpu_only
|
||||
from vllm import SamplingParams
|
||||
|
||||
from .._correctness import check_mtp_correctness
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
["model_setup", "mm_enabled", "expected_accuracy_threshold"],
|
||||
[
|
||||
(
|
||||
("mtp", "Qwen/Qwen3.5-0.8B-Base", 1, None),
|
||||
False,
|
||||
0.20,
|
||||
), # hybrid + MTP, ref: ~34%-35%
|
||||
],
|
||||
ids=["qwen3_5-hybrid"],
|
||||
)
|
||||
@single_gpu_only
|
||||
def test_mtp_correctness(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
sampling_config: SamplingParams,
|
||||
model_setup: tuple[str, str, int, str | None],
|
||||
mm_enabled: bool,
|
||||
expected_accuracy_threshold: float,
|
||||
vllm_runner,
|
||||
):
|
||||
check_mtp_correctness(
|
||||
monkeypatch,
|
||||
sampling_config,
|
||||
model_setup,
|
||||
mm_enabled,
|
||||
expected_accuracy_threshold,
|
||||
vllm_runner,
|
||||
)
|
||||
@@ -0,0 +1,115 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.utils import single_gpu_only
|
||||
from vllm import SamplingParams
|
||||
from vllm.config import CompilationConfig, CUDAGraphMode
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ...utils import get_spec_decode_metric_value
|
||||
|
||||
PLACEHOLDER_REGRESSION_MODEL = "Qwen/Qwen3.5-0.8B-Base"
|
||||
PLACEHOLDER_REGRESSION_PROMPT = "The capital of France is"
|
||||
PLACEHOLDER_REGRESSION_MAX_TOKENS = 16
|
||||
PLACEHOLDER_REGRESSION_ALLOWED_TOKEN_ID = 42
|
||||
PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS = 3
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cudagraph_mode",
|
||||
[CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL_AND_PIECEWISE],
|
||||
ids=["piecewise", "full-and-piecewise"],
|
||||
)
|
||||
@single_gpu_only
|
||||
def test_mtp_rejected_placeholders_match_greedy_token_ids(
|
||||
cudagraph_mode: CUDAGraphMode,
|
||||
vllm_runner,
|
||||
):
|
||||
"""Rejected MTP padding must not alter the non-speculative token stream."""
|
||||
common_kwargs: dict[str, Any] = {
|
||||
"block_size": None,
|
||||
"max_model_len": 256,
|
||||
"max_num_seqs": 1,
|
||||
"trust_remote_code": True,
|
||||
"enable_chunked_prefill": None,
|
||||
"limit_mm_per_prompt": {"image": 0, "video": 0},
|
||||
"attention_backend": ("TRITON_ATTN" if current_platform.is_rocm() else "auto"),
|
||||
}
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0.0,
|
||||
max_tokens=PLACEHOLDER_REGRESSION_MAX_TOKENS,
|
||||
ignore_eos=True,
|
||||
allowed_token_ids=[PLACEHOLDER_REGRESSION_ALLOWED_TOKEN_ID],
|
||||
logprobs=1,
|
||||
)
|
||||
|
||||
def generate_token_ids(runner) -> tuple[int, ...]:
|
||||
outputs = runner.llm.generate([PLACEHOLDER_REGRESSION_PROMPT], sampling_params)
|
||||
assert outputs and outputs[0].outputs, "Expected one completion"
|
||||
token_ids = tuple(outputs[0].outputs[0].token_ids)
|
||||
assert len(token_ids) == PLACEHOLDER_REGRESSION_MAX_TOKENS
|
||||
return token_ids
|
||||
|
||||
def compilation_config() -> CompilationConfig:
|
||||
return CompilationConfig(
|
||||
cudagraph_mode=cudagraph_mode,
|
||||
cudagraph_capture_sizes=[1, PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS + 1],
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
PLACEHOLDER_REGRESSION_MODEL,
|
||||
compilation_config=compilation_config(),
|
||||
**common_kwargs,
|
||||
) as ref_runner:
|
||||
ref_mode = (
|
||||
ref_runner.llm.llm_engine.vllm_config.compilation_config.cudagraph_mode
|
||||
)
|
||||
assert ref_mode == cudagraph_mode
|
||||
ref_token_ids = generate_token_ids(ref_runner)
|
||||
expected_token_ids = (PLACEHOLDER_REGRESSION_ALLOWED_TOKEN_ID,) * (
|
||||
PLACEHOLDER_REGRESSION_MAX_TOKENS
|
||||
)
|
||||
assert ref_token_ids == expected_token_ids
|
||||
|
||||
with vllm_runner(
|
||||
PLACEHOLDER_REGRESSION_MODEL,
|
||||
speculative_config={
|
||||
"method": "mtp",
|
||||
"num_speculative_tokens": PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS,
|
||||
"max_model_len": 256,
|
||||
"rejection_sample_method": "synthetic",
|
||||
"synthetic_acceptance_rates": [0.0]
|
||||
* PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS,
|
||||
},
|
||||
disable_log_stats=False,
|
||||
compilation_config=compilation_config(),
|
||||
**common_kwargs,
|
||||
) as spec_runner:
|
||||
spec_mode = (
|
||||
spec_runner.llm.llm_engine.vllm_config.compilation_config.cudagraph_mode
|
||||
)
|
||||
assert spec_mode == cudagraph_mode
|
||||
spec_token_ids = generate_token_ids(spec_runner)
|
||||
metrics = spec_runner.llm.get_metrics()
|
||||
num_drafts = get_spec_decode_metric_value(
|
||||
metrics, "vllm:spec_decode_num_drafts"
|
||||
)
|
||||
num_draft_tokens = get_spec_decode_metric_value(
|
||||
metrics, "vllm:spec_decode_num_draft_tokens"
|
||||
)
|
||||
num_accepted_tokens = get_spec_decode_metric_value(
|
||||
metrics, "vllm:spec_decode_num_accepted_tokens"
|
||||
)
|
||||
|
||||
assert num_drafts > 0, "MTP drafter did not run"
|
||||
assert num_draft_tokens > 0, "MTP drafter did not produce draft tokens"
|
||||
assert num_accepted_tokens == 0, "Synthetic sampler accepted a draft token"
|
||||
assert spec_token_ids == ref_token_ids, (
|
||||
f"MTP changed greedy token ids under {cudagraph_mode.name}:\n"
|
||||
f" reference={ref_token_ids}\n"
|
||||
f" speculative={spec_token_ids}"
|
||||
)
|
||||
@@ -1,252 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.utils import single_gpu_only
|
||||
from vllm import SamplingParams
|
||||
from vllm.config import CompilationConfig, CUDAGraphMode
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ..utils import (
|
||||
_skip_if_insufficient_gpus_for_tp,
|
||||
assert_request_outputs_match,
|
||||
evaluate_llm_for_gsm8k,
|
||||
get_spec_decode_metric_value,
|
||||
get_test_prompts,
|
||||
)
|
||||
|
||||
PLACEHOLDER_REGRESSION_MODEL = "Qwen/Qwen3.5-0.8B-Base"
|
||||
PLACEHOLDER_REGRESSION_PROMPT = "The capital of France is"
|
||||
PLACEHOLDER_REGRESSION_MAX_TOKENS = 16
|
||||
PLACEHOLDER_REGRESSION_ALLOWED_TOKEN_ID = 42
|
||||
PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS = 3
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cudagraph_mode",
|
||||
[CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL_AND_PIECEWISE],
|
||||
ids=["piecewise", "full-and-piecewise"],
|
||||
)
|
||||
@single_gpu_only
|
||||
def test_mtp_rejected_placeholders_match_greedy_token_ids(
|
||||
cudagraph_mode: CUDAGraphMode,
|
||||
vllm_runner,
|
||||
):
|
||||
"""Rejected MTP padding must not alter the non-speculative token stream."""
|
||||
common_kwargs: dict[str, Any] = {
|
||||
"block_size": None,
|
||||
"max_model_len": 256,
|
||||
"max_num_seqs": 1,
|
||||
"trust_remote_code": True,
|
||||
"enable_chunked_prefill": None,
|
||||
"limit_mm_per_prompt": {"image": 0, "video": 0},
|
||||
"attention_backend": ("TRITON_ATTN" if current_platform.is_rocm() else "auto"),
|
||||
}
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0.0,
|
||||
max_tokens=PLACEHOLDER_REGRESSION_MAX_TOKENS,
|
||||
ignore_eos=True,
|
||||
allowed_token_ids=[PLACEHOLDER_REGRESSION_ALLOWED_TOKEN_ID],
|
||||
logprobs=1,
|
||||
)
|
||||
|
||||
def generate_token_ids(runner) -> tuple[int, ...]:
|
||||
outputs = runner.llm.generate([PLACEHOLDER_REGRESSION_PROMPT], sampling_params)
|
||||
assert outputs and outputs[0].outputs, "Expected one completion"
|
||||
token_ids = tuple(outputs[0].outputs[0].token_ids)
|
||||
assert len(token_ids) == PLACEHOLDER_REGRESSION_MAX_TOKENS
|
||||
return token_ids
|
||||
|
||||
def compilation_config() -> CompilationConfig:
|
||||
return CompilationConfig(
|
||||
cudagraph_mode=cudagraph_mode,
|
||||
cudagraph_capture_sizes=[1, PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS + 1],
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
PLACEHOLDER_REGRESSION_MODEL,
|
||||
compilation_config=compilation_config(),
|
||||
**common_kwargs,
|
||||
) as ref_runner:
|
||||
ref_mode = (
|
||||
ref_runner.llm.llm_engine.vllm_config.compilation_config.cudagraph_mode
|
||||
)
|
||||
assert ref_mode == cudagraph_mode
|
||||
ref_token_ids = generate_token_ids(ref_runner)
|
||||
expected_token_ids = (PLACEHOLDER_REGRESSION_ALLOWED_TOKEN_ID,) * (
|
||||
PLACEHOLDER_REGRESSION_MAX_TOKENS
|
||||
)
|
||||
assert ref_token_ids == expected_token_ids
|
||||
|
||||
with vllm_runner(
|
||||
PLACEHOLDER_REGRESSION_MODEL,
|
||||
speculative_config={
|
||||
"method": "mtp",
|
||||
"num_speculative_tokens": PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS,
|
||||
"max_model_len": 256,
|
||||
"rejection_sample_method": "synthetic",
|
||||
"synthetic_acceptance_rates": [0.0]
|
||||
* PLACEHOLDER_REGRESSION_NUM_SPEC_TOKENS,
|
||||
},
|
||||
disable_log_stats=False,
|
||||
compilation_config=compilation_config(),
|
||||
**common_kwargs,
|
||||
) as spec_runner:
|
||||
spec_mode = (
|
||||
spec_runner.llm.llm_engine.vllm_config.compilation_config.cudagraph_mode
|
||||
)
|
||||
assert spec_mode == cudagraph_mode
|
||||
spec_token_ids = generate_token_ids(spec_runner)
|
||||
metrics = spec_runner.llm.get_metrics()
|
||||
num_drafts = get_spec_decode_metric_value(
|
||||
metrics, "vllm:spec_decode_num_drafts"
|
||||
)
|
||||
num_draft_tokens = get_spec_decode_metric_value(
|
||||
metrics, "vllm:spec_decode_num_draft_tokens"
|
||||
)
|
||||
num_accepted_tokens = get_spec_decode_metric_value(
|
||||
metrics, "vllm:spec_decode_num_accepted_tokens"
|
||||
)
|
||||
|
||||
assert num_drafts > 0, "MTP drafter did not run"
|
||||
assert num_draft_tokens > 0, "MTP drafter did not produce draft tokens"
|
||||
assert num_accepted_tokens == 0, "Synthetic sampler accepted a draft token"
|
||||
assert spec_token_ids == ref_token_ids, (
|
||||
f"MTP changed greedy token ids under {cudagraph_mode.name}:\n"
|
||||
f" reference={ref_token_ids}\n"
|
||||
f" speculative={spec_token_ids}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
["model_setup", "mm_enabled", "expected_accuracy_threshold"],
|
||||
[
|
||||
# Reference accuracy: 65%-70%.
|
||||
(("mtp", "XiaomiMiMo/MiMo-7B-Base", 1, None), False, 0.5),
|
||||
pytest.param(
|
||||
("mtp", "ZixiQi/DeepSeek-V3-4layers-MTP-FP8", 1, None),
|
||||
False,
|
||||
0.0,
|
||||
marks=pytest.mark.skipif(
|
||||
current_platform.is_device_capability_family(100),
|
||||
reason="DeepSeek MTP: TRTLLM MoE top_k check fails on Blackwell",
|
||||
),
|
||||
), # dummy model
|
||||
(
|
||||
("mtp", "Qwen/Qwen3.5-0.8B-Base", 1, None),
|
||||
False,
|
||||
0.20,
|
||||
), # hybrid + MTP, ref: ~34%-35%
|
||||
(
|
||||
("mtp", "google/gemma-4-E4B-it", 1, "google/gemma-4-E4B-it-assistant"),
|
||||
False,
|
||||
0.50,
|
||||
), # gemma4 MTP with assistant model, ref: ~62%
|
||||
],
|
||||
ids=["mimo", "deepseek", "qwen3_5-hybrid", "gemma4-e4b"],
|
||||
)
|
||||
@single_gpu_only
|
||||
def test_mtp_correctness(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
sampling_config: SamplingParams,
|
||||
model_setup: tuple[str, str, int, str | None],
|
||||
mm_enabled: bool,
|
||||
expected_accuracy_threshold: float,
|
||||
vllm_runner,
|
||||
):
|
||||
"""Compare the outputs of a original LLM and a speculative LLM
|
||||
which should be the same when using MTP speculative decoding. Due to some variance
|
||||
in the engine, it is possible for some outputs to differ, so we expect that at least
|
||||
6/10 output tokens match exactly, and that the GSM8k accuracy is above a precomputed
|
||||
reference threshold for each model.
|
||||
"""
|
||||
method, model_name, tp_size, draft_model = model_setup
|
||||
_skip_if_insufficient_gpus_for_tp(tp_size)
|
||||
|
||||
# Generate test prompts inside the function instead of using fixture
|
||||
test_prompts = get_test_prompts(mm_enabled)
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_MLA_DISABLE", "1")
|
||||
|
||||
attn_backend = "TRITON_ATTN" if current_platform.is_rocm() else "auto"
|
||||
|
||||
# Skip multimodal profiling for models that don't need it in this test.
|
||||
extra_kwargs: dict[str, Any] = {}
|
||||
if "Qwen3.5" in model_name:
|
||||
extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "video": 0}
|
||||
elif "gemma-4" in model_name:
|
||||
extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "audio": 0}
|
||||
|
||||
if draft_model is not None and "gemma-4" in draft_model:
|
||||
import transformers
|
||||
from packaging.version import Version
|
||||
|
||||
if Version(transformers.__version__) < Version("5.8.0"):
|
||||
pytest.skip(
|
||||
"Gemma4 MTP assistant requires transformers>=5.8.0, "
|
||||
f"got {transformers.__version__}"
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
model_name,
|
||||
block_size=None,
|
||||
max_model_len=2048,
|
||||
tensor_parallel_size=tp_size,
|
||||
trust_remote_code=True,
|
||||
attention_backend=attn_backend,
|
||||
enable_chunked_prefill=None,
|
||||
compilation_config=CompilationConfig(),
|
||||
**extra_kwargs,
|
||||
) as ref_runner:
|
||||
ref_outputs = ref_runner.llm.chat(test_prompts, sampling_config)
|
||||
evaluate_llm_for_gsm8k(
|
||||
ref_runner.llm,
|
||||
expected_accuracy_threshold=expected_accuracy_threshold,
|
||||
)
|
||||
|
||||
speculative_config: dict[str, Any] = {
|
||||
"method": method,
|
||||
"num_speculative_tokens": 1,
|
||||
"max_model_len": 2048,
|
||||
}
|
||||
if draft_model is not None:
|
||||
speculative_config["model"] = draft_model
|
||||
speculative_config["num_speculative_tokens"] = 2
|
||||
|
||||
with vllm_runner(
|
||||
model_name,
|
||||
block_size=None,
|
||||
trust_remote_code=True,
|
||||
tensor_parallel_size=tp_size,
|
||||
speculative_config=speculative_config,
|
||||
max_model_len=2048,
|
||||
attention_backend=attn_backend,
|
||||
enable_chunked_prefill=None,
|
||||
compilation_config=CompilationConfig(),
|
||||
**extra_kwargs,
|
||||
) as spec_runner:
|
||||
# MTP supports async scheduling by default.
|
||||
has_async = (
|
||||
spec_runner.llm.llm_engine.vllm_config.scheduler_config.async_scheduling
|
||||
)
|
||||
assert has_async, (
|
||||
f"Expected async scheduling for {method}: target={model_name}, "
|
||||
f"draft={draft_model}; got {has_async}"
|
||||
)
|
||||
evaluate_llm_for_gsm8k(
|
||||
spec_runner.llm,
|
||||
expected_accuracy_threshold=expected_accuracy_threshold,
|
||||
)
|
||||
spec_outputs = spec_runner.llm.chat(test_prompts, sampling_config)
|
||||
|
||||
# Heuristic: expect at least 80% of the prompts to match exactly
|
||||
# Upon failure, inspect the outputs to check for inaccuracy.
|
||||
assert_request_outputs_match(
|
||||
ref_outputs,
|
||||
spec_outputs,
|
||||
required_matches=int(0.8 * len(ref_outputs)) + 1,
|
||||
context=f"{method} target={model_name}, draft={draft_model}",
|
||||
)
|
||||
@@ -4,6 +4,7 @@
|
||||
import copy
|
||||
import time
|
||||
import uuid
|
||||
from collections import deque
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from unittest.mock import MagicMock, PropertyMock, patch
|
||||
|
||||
@@ -395,6 +396,38 @@ def test_engine_core_concurrent_batches():
|
||||
req_id = (req_id + 1) % 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("encoder_only", [True, False])
|
||||
def test_async_step_skips_sampling_for_mm_encoder_only(encoder_only):
|
||||
"""Encoder outputs are complete; only language-model steps need sampling."""
|
||||
core = MagicMock()
|
||||
core.batch_queue = deque()
|
||||
core.batch_queue_size = 2
|
||||
core.is_mm_encoder_only = encoder_only
|
||||
core.is_pooling_model = False
|
||||
core.scheduler.has_requests.side_effect = [True, False, False]
|
||||
scheduled = core.scheduler.schedule.return_value
|
||||
scheduled.total_num_scheduled_tokens = 16
|
||||
scheduled.pending_structured_output_tokens = False
|
||||
|
||||
output = MagicMock()
|
||||
executed: Future[ModelRunnerOutput | None] = Future()
|
||||
executed.set_result(output if encoder_only else None)
|
||||
sampled: Future[ModelRunnerOutput | None] = Future()
|
||||
sampled.set_result(None if encoder_only else output)
|
||||
core.model_executor.execute_model.return_value = executed
|
||||
core.model_executor.sample_tokens.return_value = sampled
|
||||
|
||||
EngineCore.step_with_batch_queue(core)
|
||||
if core.batch_queue:
|
||||
EngineCore.step_with_batch_queue(core)
|
||||
|
||||
core.scheduler.update_from_output.assert_called_once_with(scheduled, output)
|
||||
if encoder_only:
|
||||
core.model_executor.sample_tokens.assert_not_called()
|
||||
else:
|
||||
core.model_executor.sample_tokens.assert_called_once()
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_engine_core_tp():
|
||||
"""Test engine can initialize worker in tp properly."""
|
||||
|
||||
@@ -500,7 +500,7 @@ def test_decode_bench_connector_skips_null_padding():
|
||||
|
||||
|
||||
def test_decode_bench_connector_uses_per_group_dcp_block_sizes():
|
||||
"""DCP scales full-attention blocks but not sliding-window blocks."""
|
||||
"""DCP scales target blocks but not the replicated draft's window."""
|
||||
block_size = 16
|
||||
dcp_world_size = 2
|
||||
vllm_config = create_vllm_config(
|
||||
@@ -515,6 +515,7 @@ def test_decode_bench_connector_uses_per_group_dcp_block_sizes():
|
||||
dtype=torch.float32,
|
||||
)
|
||||
sliding_window = SlidingWindowSpec(
|
||||
dcp_sharded=False,
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
|
||||
@@ -99,6 +99,7 @@ def _run_engine_core_handshake(
|
||||
),
|
||||
speculative_config=None,
|
||||
ec_transfer_config=None,
|
||||
is_mm_encoder_only=False,
|
||||
max_concurrent_batches=1,
|
||||
model_config=SimpleNamespace(
|
||||
runner_type="generate",
|
||||
|
||||
@@ -3,10 +3,12 @@
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import threading
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import torch
|
||||
import zmq.asyncio
|
||||
@@ -14,6 +16,7 @@ import zmq.asyncio
|
||||
from tests.v1.attention.utils import dense_kv_cache_views
|
||||
from vllm import envs
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import mooncake_connector
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector import (
|
||||
KVConnectorRole,
|
||||
MooncakeConnector,
|
||||
@@ -33,6 +36,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector im
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_utils import (
|
||||
MooncakeBootstrapServer,
|
||||
RegisterWorkerPayload,
|
||||
)
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
@@ -538,6 +542,46 @@ def bootstrap_server():
|
||||
server.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_register_worker_recovers_from_slow_bootstrap_server(monkeypatch):
|
||||
"""End-to-end: a real HTTP server that responds slower than the configured
|
||||
timeout must be retried on the wire, not treated as fatal."""
|
||||
original = MooncakeBootstrapServer.register_worker
|
||||
state = {"done": False, "calls": 0}
|
||||
|
||||
async def slow_register(self, payload: RegisterWorkerPayload):
|
||||
state["calls"] += 1
|
||||
if not state["done"]:
|
||||
state["done"] = True
|
||||
# Record the worker first, then stall past the client timeout, so
|
||||
# the retry exercises the duplicate-registration path.
|
||||
response = await original(self, payload)
|
||||
await asyncio.sleep(1.5)
|
||||
return response
|
||||
return await original(self, payload)
|
||||
|
||||
monkeypatch.setattr(MooncakeBootstrapServer, "register_worker", slow_register)
|
||||
|
||||
port = get_open_port()
|
||||
monkeypatch.setenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", str(port))
|
||||
monkeypatch.setenv("VLLM_MOONCAKE_CONNECTOR_TIMEOUT", "0.5")
|
||||
monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 5)
|
||||
|
||||
server = MooncakeBootstrapServer("127.0.0.1", port)
|
||||
server.start()
|
||||
try:
|
||||
worker = _make_local_register_worker_stub()
|
||||
await MooncakeConnectorWorker.register_worker_with_bootstrap(worker)
|
||||
assert state["calls"] == 2
|
||||
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.get(f"http://127.0.0.1:{port}/query")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["0"]["engine_id"] == "eng-1"
|
||||
finally:
|
||||
server.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bootstrap_server(bootstrap_server: MooncakeBootstrapServer):
|
||||
"""Tests the bootstrap server's api for worker registration and querying.
|
||||
@@ -589,9 +633,16 @@ async def test_bootstrap_server(bootstrap_server: MooncakeBootstrapServer):
|
||||
assert data["0"]["worker_addr"]["0"]["0"] == "tcp://1.1.1.1:1111"
|
||||
assert data["0"]["worker_addr"]["0"]["1"] == "tcp://2.2.2.2:2222"
|
||||
|
||||
# Test failure: re-registering the same worker
|
||||
# Re-registering the identical payload is idempotent.
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.post(f"{base_url}/register", json=payload1)
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"status": "ok"}
|
||||
|
||||
# Test failure: same ranks, conflicting address
|
||||
payload1_conflict = dict(payload1, addr="tcp://9.9.9.9:9999")
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.post(f"{base_url}/register", json=payload1_conflict)
|
||||
assert response.status_code == 400
|
||||
assert "is already registered" in response.text
|
||||
|
||||
@@ -628,6 +679,137 @@ def _make_bootstrap_vllm_config(
|
||||
)
|
||||
|
||||
|
||||
def _make_local_register_worker_stub():
|
||||
return SimpleNamespace(
|
||||
vllm_config=SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(
|
||||
local_engines_only=False,
|
||||
data_parallel_rank_local=0,
|
||||
data_parallel_index=0,
|
||||
nnodes_within_dp=1,
|
||||
master_addr="127.0.0.1",
|
||||
data_parallel_master_ip="127.0.0.1",
|
||||
)
|
||||
),
|
||||
hostname="127.0.0.1",
|
||||
side_channel_port=1234,
|
||||
engine_id="eng-1",
|
||||
dp_rank=0,
|
||||
tp_rank=0,
|
||||
pp_rank=0,
|
||||
)
|
||||
|
||||
|
||||
def _make_register_worker_stub():
|
||||
return SimpleNamespace(
|
||||
vllm_config=_make_bootstrap_vllm_config(),
|
||||
hostname="127.0.0.1",
|
||||
side_channel_port=1234,
|
||||
engine_id="eng-1",
|
||||
dp_rank=0,
|
||||
tp_rank=0,
|
||||
pp_rank=0,
|
||||
)
|
||||
|
||||
|
||||
class _FlakyAsyncClient:
|
||||
"""httpx.AsyncClient stub that read-times-out `failures` times first."""
|
||||
|
||||
def __init__(self, failures: int, calls: list[int], **kwargs):
|
||||
self.failures = failures
|
||||
self.calls = calls
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *exc_info):
|
||||
return False
|
||||
|
||||
async def post(self, url, json=None):
|
||||
self.calls.append(1)
|
||||
if len(self.calls) <= self.failures:
|
||||
raise httpx.ReadTimeout("simulated slow bootstrap server")
|
||||
response = MagicMock()
|
||||
response.raise_for_status = MagicMock()
|
||||
return response
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_register_worker_retries_on_read_timeout(monkeypatch):
|
||||
"""A slow rank-0 bootstrap response must be retried, not treated as fatal."""
|
||||
monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 5)
|
||||
monkeypatch.setattr(asyncio, "sleep", AsyncMock())
|
||||
|
||||
calls: list[int] = []
|
||||
monkeypatch.setattr(
|
||||
httpx,
|
||||
"AsyncClient",
|
||||
lambda **kwargs: _FlakyAsyncClient(2, calls),
|
||||
)
|
||||
|
||||
worker = _make_register_worker_stub()
|
||||
await MooncakeConnectorWorker.register_worker_with_bootstrap(worker)
|
||||
|
||||
assert len(calls) == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_register_worker_raises_after_max_attempts(monkeypatch):
|
||||
"""Terminal registration failure must raise, not loop forever."""
|
||||
monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 3)
|
||||
monkeypatch.setattr(asyncio, "sleep", AsyncMock())
|
||||
|
||||
calls: list[int] = []
|
||||
monkeypatch.setattr(
|
||||
httpx,
|
||||
"AsyncClient",
|
||||
lambda **kwargs: _FlakyAsyncClient(99, calls),
|
||||
)
|
||||
|
||||
worker = _make_register_worker_stub()
|
||||
with pytest.raises(RuntimeError, match="after 3 attempts"):
|
||||
await MooncakeConnectorWorker.register_worker_with_bootstrap(worker)
|
||||
|
||||
assert len(calls) == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sender_listener_failure_propagates_to_caller(monkeypatch):
|
||||
"""A terminal registration failure must surface via the Future rather than
|
||||
leaving register_kv_caches blocked on its ready event."""
|
||||
monkeypatch.setenv("VLLM_MOONCAKE_CONNECTOR_TIMEOUT", "0.1")
|
||||
monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 1)
|
||||
|
||||
loop = asyncio.new_event_loop()
|
||||
thread = threading.Thread(target=loop.run_forever, daemon=True)
|
||||
thread.start()
|
||||
|
||||
async def failing_listener(ready_event):
|
||||
raise RuntimeError("simulated terminal registration failure")
|
||||
|
||||
try:
|
||||
ready_event = threading.Event()
|
||||
fut = asyncio.run_coroutine_threadsafe(failing_listener(ready_event), loop)
|
||||
|
||||
deadline = time.monotonic() + 10.0
|
||||
raised = None
|
||||
while not ready_event.wait(timeout=0.1):
|
||||
if fut.done():
|
||||
try:
|
||||
fut.result()
|
||||
except RuntimeError as e:
|
||||
raised = e
|
||||
break
|
||||
assert time.monotonic() < deadline, "caller blocked past its deadline"
|
||||
|
||||
assert raised is not None
|
||||
assert "simulated terminal registration failure" in str(raised)
|
||||
finally:
|
||||
loop.call_soon_threadsafe(loop.stop)
|
||||
thread.join(timeout=5)
|
||||
loop.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"tp_rank",
|
||||
|
||||
@@ -290,6 +290,8 @@ class _FakeKVTransferConfig:
|
||||
class _FakeModelConfig:
|
||||
model = "test-model"
|
||||
use_mla = False
|
||||
enable_sleep_mode = False
|
||||
enable_cumem_allocator = False
|
||||
|
||||
def get_num_layers(self, parallel_config) -> int:
|
||||
return 1
|
||||
@@ -432,6 +434,64 @@ def _patch_worker_runtime(
|
||||
monkeypatch.setattr(worker, "LookupKeyServer", MagicMock())
|
||||
|
||||
|
||||
def test_create_mem_pool_returns_none_when_unconfigured():
|
||||
assert worker.MooncakeStoreWorker._create_mem_pool({}, _FakeModelConfig()) is None
|
||||
|
||||
|
||||
def test_create_mem_pool_rejects_unknown_pool():
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="Unsupported custom_mem_pool='UNKNOWN'",
|
||||
):
|
||||
worker.MooncakeStoreWorker._create_mem_pool(
|
||||
{"custom_mem_pool": "unknown"}, _FakeModelConfig()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"conflicting_option", ["enable_sleep_mode", "enable_cumem_allocator"]
|
||||
)
|
||||
def test_create_mem_pool_rejects_cumem_conflicts(conflicting_option):
|
||||
model_config = _FakeModelConfig()
|
||||
setattr(model_config, conflicting_option, True)
|
||||
|
||||
with pytest.raises(ValueError, match="custom_mem_pool is incompatible"):
|
||||
worker.MooncakeStoreWorker._create_mem_pool(
|
||||
{"custom_mem_pool": "NVLINK"}, model_config
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("pool_type", "allocator_name"),
|
||||
[("nvlink", "NVLinkAllocator"), ("barex", "BarexAllocator")],
|
||||
)
|
||||
def test_create_mem_pool_uses_mooncake_allocator(
|
||||
monkeypatch, pool_type, allocator_name
|
||||
):
|
||||
allocator_cls = MagicMock()
|
||||
allocator = allocator_cls.get_allocator.return_value
|
||||
allocator_module = types.ModuleType("mooncake.allocator")
|
||||
setattr(allocator_module, allocator_name, allocator_cls)
|
||||
mooncake_module = types.ModuleType("mooncake")
|
||||
mooncake_module.allocator = allocator_module # type: ignore[attr-defined]
|
||||
monkeypatch.setitem(sys.modules, "mooncake", mooncake_module)
|
||||
monkeypatch.setitem(sys.modules, "mooncake.allocator", allocator_module)
|
||||
|
||||
monkeypatch.setattr(torch.accelerator, "current_device_index", lambda: 2)
|
||||
mem_pool = MagicMock()
|
||||
mem_pool_ctor = MagicMock(return_value=mem_pool)
|
||||
monkeypatch.setattr(torch.cuda, "MemPool", mem_pool_ctor)
|
||||
|
||||
result = worker.MooncakeStoreWorker._create_mem_pool(
|
||||
{"custom_mem_pool": pool_type}, _FakeModelConfig()
|
||||
)
|
||||
|
||||
assert result is mem_pool
|
||||
allocator_cls.get_allocator.assert_called_once_with(torch.device("cuda", 2))
|
||||
allocator.allocator.assert_called_once_with()
|
||||
mem_pool_ctor.assert_called_once_with(allocator.allocator.return_value)
|
||||
|
||||
|
||||
def test_pool_key_to_string_without_prefix_is_unchanged():
|
||||
"""Default (empty) cache_prefix keeps keys byte-identical to the
|
||||
historical unprefixed format so existing deployments keep their hits."""
|
||||
|
||||
@@ -68,6 +68,7 @@ class MockConnector(KVConnectorBase_V1):
|
||||
# Override just build_kv_connector_stats
|
||||
mock.build_kv_connector_stats = cls.build_kv_connector_stats
|
||||
mock.get_kv_connector_stats.return_value = None
|
||||
mock.get_mem_pool_context.return_value = None
|
||||
return mock
|
||||
|
||||
@classmethod
|
||||
@@ -201,6 +202,36 @@ def mc() -> MultiConnector:
|
||||
return mc
|
||||
|
||||
|
||||
def test_multi_connector_mem_pool_context_none(mc: MultiConnector):
|
||||
assert mc.get_mem_pool_context() is None
|
||||
|
||||
|
||||
def test_multi_connector_forwards_mem_pool_context(mc: MultiConnector):
|
||||
context = MagicMock()
|
||||
provider = MagicMock(spec_set=KVConnectorBase_V1)
|
||||
provider.get_mem_pool_context.return_value = context
|
||||
mc._connectors = [mc._connectors[0], provider]
|
||||
|
||||
assert mc.get_mem_pool_context() is context
|
||||
provider.get_mem_pool_context.assert_called_once_with()
|
||||
|
||||
|
||||
def test_multi_connector_rejects_multiple_mem_pool_contexts(mc: MultiConnector):
|
||||
providers = [
|
||||
MagicMock(spec_set=KVConnectorBase_V1),
|
||||
MagicMock(spec_set=KVConnectorBase_V1),
|
||||
]
|
||||
for provider in providers:
|
||||
provider.get_mem_pool_context.return_value = MagicMock()
|
||||
mc._connectors = providers
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="Multiple connectors provide a KV cache memory pool",
|
||||
):
|
||||
mc.get_mem_pool_context()
|
||||
|
||||
|
||||
# Helper function to compare directories recursively
|
||||
def _compare_directories(dir1: Path, dir2: Path) -> bool:
|
||||
"""Compares two directories recursively for identical content."""
|
||||
@@ -320,6 +351,7 @@ def test_multi_example_connector_consistency():
|
||||
# First three events are from initialization. Layer hooks run before the
|
||||
# deferred load starts after the forward pass.
|
||||
expected_worker_prefix = [
|
||||
"get_mem_pool_context",
|
||||
"register_kv_caches",
|
||||
"set_host_xfer_buffer_ops",
|
||||
"get_handshake_metadata",
|
||||
|
||||
@@ -7,6 +7,7 @@ import os
|
||||
import queue
|
||||
import tempfile
|
||||
import textwrap
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from collections import defaultdict
|
||||
@@ -49,6 +50,8 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl import (
|
||||
NixlKVConnectorStats,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
RemoteMeta,
|
||||
ReqMeta,
|
||||
compute_nixl_compatibility_hash,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_transfer_state import (
|
||||
@@ -2087,6 +2090,10 @@ def recv_worker():
|
||||
worker._recving_transfers = defaultdict(list)
|
||||
worker._failed_recv_reqs = queue.Queue()
|
||||
worker._recv_failures = set()
|
||||
worker._handshake_lock = threading.RLock()
|
||||
worker._handshake_futures = {}
|
||||
worker._remote_agents = {}
|
||||
worker._engine_by_address = {}
|
||||
worker._replicated_pcp_done_sending = set()
|
||||
worker._invalid_block_ids = queue.Queue()
|
||||
worker._pending_recv_notifs = {}
|
||||
@@ -2681,6 +2688,174 @@ def test_engine_ttl_disabled(default_vllm_config, dist_init):
|
||||
assert engine_id in worker.dst_xfer_side_handles
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
class TestPeerReplacement:
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup(self, monkeypatch):
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker as bw
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
||||
|
||||
config = create_vllm_config(kv_connector_extra_config={"engine_ttl": 0})
|
||||
config.kv_transfer_config.kv_buffer_device = "cpu"
|
||||
platform = SimpleNamespace(
|
||||
device_type="cpu",
|
||||
discover_numa_topology=lambda: [],
|
||||
get_nixl_memory_type=lambda: "DRAM",
|
||||
is_rocm=lambda: False,
|
||||
)
|
||||
with (
|
||||
patch.object(bw, "NixlWrapper", FakeNixlWrapper),
|
||||
patch.object(bw, "current_platform", platform),
|
||||
patch.object(bw, "get_tensor_model_parallel_rank", return_value=0),
|
||||
patch.object(bw, "get_tensor_model_parallel_world_size", return_value=1),
|
||||
patch.object(
|
||||
bw, "get_current_attn_backends", return_value=[FlashAttentionBackend]
|
||||
),
|
||||
):
|
||||
self.worker = FakeNixlConnectorWorker(config, "local", hand_shake_latency=0)
|
||||
self.nixl = self.worker.nixl_wrapper
|
||||
self.transport = MagicMock(wraps=self.nixl)
|
||||
self.worker.nixl_wrapper = self.transport
|
||||
monkeypatch.setattr(
|
||||
self.worker._handshake_initiation_executor,
|
||||
"submit",
|
||||
lambda *args: Future(),
|
||||
)
|
||||
self._connect("old")
|
||||
yield
|
||||
self.worker.shutdown()
|
||||
|
||||
def _connect(self, engine_id="new", host="localhost", port=1234):
|
||||
future = self.worker._ensure_handshake(engine_id, host, port, 1)
|
||||
self.worker.REMOTE_ENGINE_ID = engine_id
|
||||
self.nixl.REMOTE_AGENT_NAME = engine_id
|
||||
future.set_result(self.worker._nixl_handshake(host, port, 1, engine_id))
|
||||
|
||||
def _request(self, engine_id="new", blocks=(1,), awaiting_kvs=True):
|
||||
metadata = NixlConnectorMetadata()
|
||||
metadata.reqs_to_recv[f"{engine_id}-req"] = ReqMeta(
|
||||
local_block_ids=(blocks,),
|
||||
local_physical_block_ids=(blocks,),
|
||||
tp_size=1,
|
||||
remote=RemoteMeta(
|
||||
([2],), "localhost", 1234, engine_id, f"{engine_id}-prefill"
|
||||
),
|
||||
awaiting_kvs=awaiting_kvs,
|
||||
)
|
||||
return metadata
|
||||
|
||||
def test_cleans_old_peer_after_confirmed_handshake(self):
|
||||
self._connect("healthy", host="other-host")
|
||||
old_handle = self.worker.dst_xfer_side_handles["old"][0]
|
||||
self.worker.start_load_kv(self._request())
|
||||
self.worker.get_transfer_results()
|
||||
self.transport.remove_remote_agent.assert_not_called()
|
||||
|
||||
self._connect()
|
||||
self.worker.get_transfer_results()
|
||||
self.transport.release_dlist_handle.assert_called_once_with(old_handle)
|
||||
self.transport.remove_remote_agent.assert_called_once_with("old")
|
||||
assert set(self.worker._remote_agents) == {"new", "healthy"}
|
||||
assert self.worker._engine_by_address == {
|
||||
("localhost", 1234): "new",
|
||||
("other-host", 1234): "healthy",
|
||||
}
|
||||
with pytest.raises(KeyError):
|
||||
self.worker.transfer_topo.get_engine_info("old")
|
||||
|
||||
self.worker.start_load_kv(NixlConnectorMetadata())
|
||||
assert self.worker.get_transfer_results().finished_recving == {"new-req"}
|
||||
self.transport.remove_remote_agent.assert_called_once_with("old")
|
||||
|
||||
@pytest.mark.parametrize("host,port", [("other-host", 1234), ("localhost", 5678)])
|
||||
def test_preserves_other_addresses(self, host, port):
|
||||
self._connect(host=host, port=port)
|
||||
self.worker.get_transfer_results()
|
||||
assert "old" in self.worker._remote_agents
|
||||
self.transport.remove_remote_agent.assert_not_called()
|
||||
|
||||
def test_preserves_old_peer_when_handshake_fails(self):
|
||||
self.worker.start_load_kv(self._request())
|
||||
self.worker._handshake_futures["new"].set_exception(
|
||||
RuntimeError("handshake failed")
|
||||
)
|
||||
assert self.worker.get_transfer_results().failed_recving == {"new-req"}
|
||||
assert "old" in self.worker._remote_agents
|
||||
assert self.worker._engine_by_address == {("localhost", 1234): "old"}
|
||||
self.transport.remove_remote_agent.assert_not_called()
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"states,failed",
|
||||
[
|
||||
(["PROC", "DONE"], False),
|
||||
(["ERR", "PROC", "DONE"], True),
|
||||
(["ERR", "PROC", "ERR"], True),
|
||||
],
|
||||
ids=["healthy", "failed-sibling-completes", "failed-sibling-fails"],
|
||||
)
|
||||
def test_waits_for_reads_before_cleanup_and_failure_reporting(self, states, failed):
|
||||
self.worker._recving_metadata.update(self._request("old").reqs_to_recv)
|
||||
self.worker._recving_transfers["old-req"] = [1, 2] if failed else [1]
|
||||
self.transport.check_xfer_state.side_effect = states
|
||||
self._connect()
|
||||
|
||||
result = self.worker.get_transfer_results()
|
||||
assert result.finished_recving == result.failed_recving == set()
|
||||
assert self.worker.get_block_ids_with_load_errors() == set()
|
||||
self.transport.remove_remote_agent.assert_not_called()
|
||||
|
||||
result = self.worker.get_transfer_results()
|
||||
assert result.finished_recving == {"old-req"}
|
||||
assert result.failed_recving == ({"old-req"} if failed else set())
|
||||
assert self.worker.get_block_ids_with_load_errors() == (
|
||||
{1} if failed else set()
|
||||
)
|
||||
assert self.transport.release_xfer_handle.call_count == (2 if failed else 1)
|
||||
self.transport.remove_remote_agent.assert_called_once_with("old")
|
||||
|
||||
def test_waits_for_queued_notification(self):
|
||||
metadata = self._request("old", blocks=())
|
||||
self.worker._recving_metadata.update(metadata.reqs_to_recv)
|
||||
self.worker._background_nixl_handshake(
|
||||
"old-req", "old", metadata.reqs_to_recv["old-req"]
|
||||
)
|
||||
self._connect()
|
||||
self.worker.get_transfer_results()
|
||||
self.transport.remove_remote_agent.assert_not_called()
|
||||
|
||||
self.worker.start_load_kv(NixlConnectorMetadata())
|
||||
self.transport.send_notif.assert_called_once_with(
|
||||
"old", notif_msg=b"old-prefill:1"
|
||||
)
|
||||
result = self.worker.get_transfer_results()
|
||||
assert result.finished_recving == {"old-req"}
|
||||
assert result.failed_recving == set()
|
||||
self.transport.remove_remote_agent.assert_called_once_with("old")
|
||||
|
||||
def test_waits_for_other_handshakes(self):
|
||||
self.worker._ensure_handshake("other", "other-host", 1234, 1)
|
||||
self._connect()
|
||||
self.worker.get_transfer_results()
|
||||
self.transport.remove_remote_agent.assert_not_called()
|
||||
|
||||
self._connect("other", host="other-host")
|
||||
self.worker.get_transfer_results()
|
||||
self.transport.remove_remote_agent.assert_called_once_with("old")
|
||||
|
||||
def test_rehandshakes_queued_request_for_released_engine(self):
|
||||
# A request queued after its engine's handshake, but drained only after
|
||||
# that engine was replaced and released, must not read from it.
|
||||
metadata = self._request("old", blocks=(), awaiting_kvs=False)
|
||||
self.worker._ready_requests.put(("old-req", metadata.reqs_to_recv["old-req"]))
|
||||
self._connect()
|
||||
self.worker.get_transfer_results()
|
||||
self.transport.remove_remote_agent.assert_called_once_with("old")
|
||||
|
||||
self.worker.start_load_kv(NixlConnectorMetadata())
|
||||
assert "old" in self.worker._handshake_futures
|
||||
self.transport.send_notif.assert_not_called()
|
||||
|
||||
|
||||
def test_transfer_topology_unregister():
|
||||
"""TransferTopology.unregister_remote_engine removes the engine."""
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
import gc
|
||||
import queue
|
||||
import threading
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
@@ -54,6 +55,9 @@ def region_pull_worker():
|
||||
worker.enable_permute_local_kv = False
|
||||
worker.enable_heterogeneous_attn_post_process = False
|
||||
worker._engine_last_active = {}
|
||||
worker._handshake_lock = threading.RLock()
|
||||
worker._handshake_futures = {}
|
||||
worker._transfer_layer_group_ids = ()
|
||||
worker._bidirectional_kv_xfer_enabled = False
|
||||
worker._recving_transfers = {}
|
||||
worker.use_mla, worker._has_mamba = True, False
|
||||
@@ -94,6 +98,7 @@ def region_pull_worker():
|
||||
worker.dst_num_blocks = {"P": 100, "D": 100}
|
||||
worker.dst_region_num_blocks = {"P": [100, 100], "D": [100, 100]}
|
||||
worker._remote_agents = {"P": {(0, 0): "P-rank0"}}
|
||||
worker._engine_by_address = {("localhost", 1): "P"}
|
||||
worker._read_blocks_mixed = MagicMock()
|
||||
worker.nixl_wrapper = MagicMock()
|
||||
return worker
|
||||
|
||||
@@ -395,6 +395,7 @@ class _StubWriterWorker(NixlPushConnectorWorker):
|
||||
w._group_spec_types = (FullAttentionSpec,)
|
||||
w._engine_ttl = 0.0
|
||||
w._engine_last_active = {}
|
||||
w._engine_by_address = {}
|
||||
|
||||
# Track _do_start_push_kv invocations.
|
||||
calls: list[tuple[str, Any, dict[str, Any]]] = []
|
||||
|
||||
@@ -0,0 +1,466 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for SimpleCPUOffloadConnector metrics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from prometheus_client import Counter, Gauge, Histogram
|
||||
|
||||
from tests.v1.kv_connector.unit.test_simple_cpu_offload_connector import (
|
||||
_make_connector,
|
||||
)
|
||||
from tests.v1.simple_kv_offload.test_scheduler import (
|
||||
make_request,
|
||||
make_scheduler,
|
||||
make_scheduler_output,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector import (
|
||||
SimpleCPUOffloadConnector,
|
||||
)
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.simple_kv_offload.manager import (
|
||||
BoundaryStoreStats,
|
||||
LoadRequestState,
|
||||
TransferMeta,
|
||||
)
|
||||
from vllm.v1.simple_kv_offload.metrics import (
|
||||
INFO_LABELS,
|
||||
OUTCOME_TO_FIELD,
|
||||
MetricName,
|
||||
SimpleCPUOffloadStats,
|
||||
_MetricType,
|
||||
_StatsKey,
|
||||
)
|
||||
|
||||
SAVE_OUTCOMES = MetricName.SAVE_OUTCOMES
|
||||
LOAD_BLOCKS = MetricName.LOAD_BLOCKS
|
||||
USED_BLOCKS = MetricName.USED_BLOCKS
|
||||
PENDING_STORE_BLOCKS = MetricName.PENDING_STORE_BLOCKS
|
||||
INFO = MetricName.INFO
|
||||
|
||||
|
||||
class _FakeMetric:
|
||||
def __init__(self, **kwargs: Any):
|
||||
self.kwargs = kwargs
|
||||
self.children: list[_FakeMetric] = []
|
||||
self.observed: list[int | float] = []
|
||||
self.increments: list[int | float] = []
|
||||
self.set_values: list[int | float] = []
|
||||
self.labelvalues: tuple[object, ...] = ()
|
||||
|
||||
def labels(self, *labelvalues: object) -> _FakeMetric:
|
||||
child = _FakeMetric(**self.kwargs)
|
||||
child.labelvalues = labelvalues
|
||||
self.children.append(child)
|
||||
return child
|
||||
|
||||
def observe(self, value: int | float) -> None:
|
||||
self.observed.append(value)
|
||||
|
||||
def inc(self, value: int | float) -> None:
|
||||
self.increments.append(value)
|
||||
|
||||
def set(self, value: int | float) -> None:
|
||||
self.set_values.append(value)
|
||||
|
||||
|
||||
def _fake_metric_types() -> dict[type, type]:
|
||||
return {Gauge: _FakeMetric, Counter: _FakeMetric, Histogram: _FakeMetric}
|
||||
|
||||
|
||||
def _make_prom_metrics(per_engine: dict[int, list[object]] | None = None) -> Any:
|
||||
from vllm.v1.simple_kv_offload.metrics import SimpleCPUOffloadPromMetrics
|
||||
|
||||
return SimpleCPUOffloadPromMetrics(
|
||||
vllm_config=SimpleNamespace(kv_transfer_config=None),
|
||||
metric_types=_fake_metric_types(),
|
||||
labelnames=["model_name", "engine"],
|
||||
per_engine_labelvalues=per_engine or {0: ["model", "0"]},
|
||||
)
|
||||
|
||||
|
||||
def _stats_payload(types: dict, data: dict) -> dict[str, Any]:
|
||||
return {_StatsKey.TYPES: types, _StatsKey.DATA: data}
|
||||
|
||||
|
||||
def _values(stats: SimpleCPUOffloadStats, name: str) -> dict:
|
||||
return stats.data[_StatsKey.DATA][name]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stats class
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_build_kv_connector_stats_none_and_empty() -> None:
|
||||
stats = SimpleCPUOffloadConnector.build_kv_connector_stats(data=None)
|
||||
assert isinstance(stats, SimpleCPUOffloadStats)
|
||||
assert stats.is_empty()
|
||||
|
||||
stats = SimpleCPUOffloadConnector.build_kv_connector_stats(data={})
|
||||
assert isinstance(stats, SimpleCPUOffloadStats)
|
||||
assert stats.is_empty()
|
||||
|
||||
|
||||
def test_stats_round_trip_through_serialized_dict() -> None:
|
||||
stats = SimpleCPUOffloadStats()
|
||||
stats.increase_counter(SAVE_OUTCOMES, 3, ("stored",))
|
||||
stats.increase_counter(LOAD_BLOCKS, 4)
|
||||
stats.set_gauge(USED_BLOCKS, 7)
|
||||
stats.set_gauge(INFO, 1, ("cpu", "false", "false", "64"))
|
||||
|
||||
rebuilt = SimpleCPUOffloadConnector.build_kv_connector_stats(data=stats.to_dict())
|
||||
assert isinstance(rebuilt, SimpleCPUOffloadStats)
|
||||
assert _values(rebuilt, SAVE_OUTCOMES) == {("stored",): 3}
|
||||
assert _values(rebuilt, LOAD_BLOCKS) == {(): 4}
|
||||
assert _values(rebuilt, USED_BLOCKS) == {(): 7}
|
||||
assert _values(rebuilt, INFO) == {("cpu", "false", "false", "64"): 1}
|
||||
|
||||
|
||||
def test_stats_aggregate_sums_counters_and_keeps_latest_gauge() -> None:
|
||||
stats1 = SimpleCPUOffloadStats()
|
||||
stats1.increase_counter(SAVE_OUTCOMES, 3, ("stored",))
|
||||
stats1.increase_counter(SAVE_OUTCOMES, 1, ("dropped_cpu_full",))
|
||||
stats1.set_gauge(USED_BLOCKS, 7)
|
||||
|
||||
stats2 = SimpleCPUOffloadStats()
|
||||
stats2.increase_counter(SAVE_OUTCOMES, 4, ("stored",))
|
||||
stats2.set_gauge(USED_BLOCKS, 2)
|
||||
|
||||
result = stats1.aggregate(stats2)
|
||||
assert result is stats1
|
||||
outcomes = _values(stats1, SAVE_OUTCOMES)
|
||||
assert outcomes == {("stored",): 7, ("dropped_cpu_full",): 1}
|
||||
assert _values(stats1, USED_BLOCKS) == {(): 2}
|
||||
|
||||
|
||||
def test_stats_aggregate_merges_types_from_other() -> None:
|
||||
stats1 = SimpleCPUOffloadStats()
|
||||
stats1.set_gauge(USED_BLOCKS, 1)
|
||||
|
||||
stats2 = SimpleCPUOffloadStats()
|
||||
stats2.increase_counter(LOAD_BLOCKS, 2)
|
||||
|
||||
stats1.aggregate(stats2)
|
||||
assert stats1.data[_StatsKey.TYPES][LOAD_BLOCKS] == _MetricType.COUNTER
|
||||
assert _values(stats1, LOAD_BLOCKS) == {(): 2}
|
||||
|
||||
|
||||
def test_stats_reduce_and_reset() -> None:
|
||||
stats = SimpleCPUOffloadStats()
|
||||
stats.increase_counter(SAVE_OUTCOMES, 3, ("stored",))
|
||||
stats.set_gauge(PENDING_STORE_BLOCKS, 4)
|
||||
stats.increase_counter(LOAD_BLOCKS, 9)
|
||||
stats.set_gauge(INFO, 1, ("cpu", "false", "false", "64"))
|
||||
|
||||
reduced = stats.reduce()
|
||||
assert reduced[f"{SAVE_OUTCOMES}:{('stored',)}"] == 3
|
||||
assert reduced[PENDING_STORE_BLOCKS] == 4
|
||||
assert reduced[LOAD_BLOCKS] == 9
|
||||
assert not any(key.startswith(INFO) for key in reduced)
|
||||
|
||||
stats.reset()
|
||||
assert stats.is_empty()
|
||||
|
||||
|
||||
def test_outcome_labels_match_boundary_store_stats_fields() -> None:
|
||||
"""Every outcome label maps 1:1 to a BoundaryStoreStats counter field."""
|
||||
boundary_fields = {
|
||||
f.name for f in dataclasses.fields(BoundaryStoreStats) if f.name != "published"
|
||||
}
|
||||
assert set(OUTCOME_TO_FIELD.values()) == boundary_fields
|
||||
assert len(set(OUTCOME_TO_FIELD.values())) == len(OUTCOME_TO_FIELD)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Prom metrics class
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_prom_metrics_registers_tier1_metrics() -> None:
|
||||
prom = _make_prom_metrics()
|
||||
defs = prom._defs
|
||||
assert set(defs) == {
|
||||
SAVE_OUTCOMES,
|
||||
LOAD_BLOCKS,
|
||||
USED_BLOCKS,
|
||||
PENDING_STORE_BLOCKS,
|
||||
INFO,
|
||||
}
|
||||
# The prom class registers literal names (required by the docs
|
||||
# generator); they must stay in sync with the emission-side constants.
|
||||
assert {m.kwargs["name"] for m in defs.values()} == {
|
||||
SAVE_OUTCOMES,
|
||||
LOAD_BLOCKS,
|
||||
USED_BLOCKS,
|
||||
PENDING_STORE_BLOCKS,
|
||||
INFO,
|
||||
}
|
||||
for metric in defs.values():
|
||||
assert metric.kwargs["documentation"]
|
||||
assert metric.kwargs["labelnames"][:2] == ["model_name", "engine"]
|
||||
|
||||
assert defs[SAVE_OUTCOMES].kwargs["labelnames"] == [
|
||||
"model_name",
|
||||
"engine",
|
||||
"outcome",
|
||||
]
|
||||
assert defs[LOAD_BLOCKS].kwargs["labelnames"] == ["model_name", "engine"]
|
||||
assert defs[INFO].kwargs["labelnames"] == [
|
||||
"model_name",
|
||||
"engine",
|
||||
*INFO_LABELS,
|
||||
]
|
||||
assert defs[USED_BLOCKS].kwargs["labelnames"] == ["model_name", "engine"]
|
||||
|
||||
# mostrecent: multiprocess scrapes expose one series per label set, not
|
||||
# one pid-tagged series per writer process.
|
||||
for gauge in (USED_BLOCKS, PENDING_STORE_BLOCKS, INFO):
|
||||
assert defs[gauge].kwargs["multiprocess_mode"] == "mostrecent"
|
||||
|
||||
|
||||
def test_prom_metrics_observe_routes_each_series() -> None:
|
||||
prom = _make_prom_metrics()
|
||||
prom.observe(
|
||||
_stats_payload(
|
||||
types={
|
||||
SAVE_OUTCOMES: _MetricType.COUNTER,
|
||||
LOAD_BLOCKS: _MetricType.COUNTER,
|
||||
USED_BLOCKS: _MetricType.GAUGE,
|
||||
PENDING_STORE_BLOCKS: _MetricType.GAUGE,
|
||||
INFO: _MetricType.GAUGE,
|
||||
},
|
||||
data={
|
||||
SAVE_OUTCOMES: {("stored",): 3},
|
||||
LOAD_BLOCKS: {(): 4},
|
||||
USED_BLOCKS: {(): 5},
|
||||
PENDING_STORE_BLOCKS: {(): 1},
|
||||
INFO: {("cpu", "false", "false", "64"): 1},
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
assert prom._metrics[(0, SAVE_OUTCOMES, ("stored",))].increments == [3]
|
||||
assert prom._metrics[(0, LOAD_BLOCKS, ())].increments == [4]
|
||||
assert prom._metrics[(0, USED_BLOCKS, ())].set_values == [5]
|
||||
assert prom._metrics[(0, PENDING_STORE_BLOCKS, ())].set_values == [1]
|
||||
assert prom._metrics[(0, INFO, ("cpu", "false", "false", "64"))].set_values == [1]
|
||||
assert prom._metrics[(0, SAVE_OUTCOMES, ("stored",))].labelvalues == (
|
||||
"model",
|
||||
"0",
|
||||
"stored",
|
||||
)
|
||||
assert prom._metrics[(0, INFO, ("cpu", "false", "false", "64"))].labelvalues == (
|
||||
"model",
|
||||
"0",
|
||||
"cpu",
|
||||
"false",
|
||||
"false",
|
||||
"64",
|
||||
)
|
||||
|
||||
|
||||
def test_prom_metrics_reuses_bound_children() -> None:
|
||||
prom = _make_prom_metrics()
|
||||
payload = _stats_payload(
|
||||
types={SAVE_OUTCOMES: _MetricType.COUNTER},
|
||||
data={SAVE_OUTCOMES: {("stored",): 1}},
|
||||
)
|
||||
prom.observe(payload)
|
||||
prom.observe(payload)
|
||||
|
||||
child = prom._metrics[(0, SAVE_OUTCOMES, ("stored",))]
|
||||
assert child.increments == [1, 1]
|
||||
assert len(prom._defs[SAVE_OUTCOMES].children) == 1
|
||||
|
||||
|
||||
def test_prom_metrics_rejects_unknown_metric() -> None:
|
||||
prom = _make_prom_metrics()
|
||||
with pytest.raises(AssertionError, match="Unknown"):
|
||||
prom.observe(
|
||||
_stats_payload(
|
||||
types={"vllm:not_a_metric": _MetricType.COUNTER},
|
||||
data={"vllm:not_a_metric": {(): 1}},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_prom_metrics_rejects_wrong_label_count() -> None:
|
||||
prom = _make_prom_metrics()
|
||||
with pytest.raises(AssertionError, match="labels"):
|
||||
prom.observe(
|
||||
_stats_payload(
|
||||
types={SAVE_OUTCOMES: _MetricType.COUNTER},
|
||||
data={SAVE_OUTCOMES: {("stored", "extra"): 1}},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_prom_metrics_routes_per_engine() -> None:
|
||||
prom = _make_prom_metrics(per_engine={0: ["model", "0"], 1: ["model", "1"]})
|
||||
prom.observe(
|
||||
_stats_payload(
|
||||
types={USED_BLOCKS: _MetricType.GAUGE}, data={USED_BLOCKS: {(): 3}}
|
||||
),
|
||||
engine_idx=1,
|
||||
)
|
||||
assert (0, USED_BLOCKS, ()) not in prom._metrics
|
||||
assert prom._metrics[(1, USED_BLOCKS, ())].set_values == [3]
|
||||
|
||||
|
||||
def test_connector_build_prom_metrics() -> None:
|
||||
from vllm.v1.simple_kv_offload.metrics import SimpleCPUOffloadPromMetrics
|
||||
|
||||
prom = SimpleCPUOffloadConnector.build_prom_metrics(
|
||||
vllm_config=SimpleNamespace(kv_transfer_config=None),
|
||||
metric_types=_fake_metric_types(),
|
||||
labelnames=["model_name", "engine"],
|
||||
per_engine_labelvalues={0: ["model", "0"]},
|
||||
)
|
||||
assert isinstance(prom, SimpleCPUOffloadPromMetrics)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Manager: interval stats
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_manager_get_stats_reports_outcome_deltas_per_interval() -> None:
|
||||
fixture = make_scheduler()
|
||||
sched = fixture.scheduler
|
||||
|
||||
sched.boundary_store_stats.stored += 3
|
||||
sched.boundary_store_stats.dropped_cpu_full += 2
|
||||
sched.boundary_store_stats.skipped_in_flight += 1
|
||||
|
||||
stats = sched.get_stats()
|
||||
assert _values(stats, SAVE_OUTCOMES) == {
|
||||
("stored",): 3,
|
||||
("dropped_cpu_full",): 2,
|
||||
("skipped_in_flight",): 1,
|
||||
}
|
||||
|
||||
# Deltas are per-interval: a second drain reports no new outcomes.
|
||||
stats2 = sched.get_stats()
|
||||
assert SAVE_OUTCOMES not in stats2.data[_StatsKey.DATA]
|
||||
|
||||
|
||||
def test_manager_reset_preserves_undrained_outcome_deltas() -> None:
|
||||
fixture = make_scheduler()
|
||||
sched = fixture.scheduler
|
||||
|
||||
sched.boundary_store_stats.stored += 3
|
||||
sched.reset()
|
||||
|
||||
stats = sched.get_stats()
|
||||
assert _values(stats, SAVE_OUTCOMES) == {("stored",): 3}
|
||||
|
||||
|
||||
def test_manager_get_stats_gauges_and_info_labels() -> None:
|
||||
fixture = make_scheduler(num_cpu_blocks=8)
|
||||
sched = fixture.scheduler
|
||||
|
||||
sched._store_event_to_blocks[0] = TransferMeta([10], [11])
|
||||
sched._pending_finished_stores.append(TransferMeta([12], [13]))
|
||||
sched._abandoned_store_event_to_blocks[1] = TransferMeta([14], [15])
|
||||
|
||||
stats = sched.get_stats()
|
||||
assert _values(stats, PENDING_STORE_BLOCKS) == {(): 3}
|
||||
used = _values(stats, USED_BLOCKS)[()]
|
||||
assert used == sched.num_cpu_blocks - sched.cpu_block_pool.get_num_free_blocks()
|
||||
|
||||
info = _values(stats, INFO)
|
||||
assert info == {("cpu", "false", "false", str(sched.num_cpu_blocks)): 1}
|
||||
|
||||
# Gauges are re-reported every drain.
|
||||
stats2 = sched.get_stats()
|
||||
assert _values(stats2, PENDING_STORE_BLOCKS) == {(): 3}
|
||||
|
||||
|
||||
def test_manager_info_labels_reflect_disk_mode() -> None:
|
||||
connector = _make_connector(
|
||||
extra_config={
|
||||
"kv_offload_backend": "disk",
|
||||
"disk_path": "/tmp/fake-disk-metrics",
|
||||
"disk_capacity_bytes": 1024**3,
|
||||
"use_page_cache": True,
|
||||
}
|
||||
)
|
||||
sched = connector.scheduler_manager
|
||||
assert sched is not None
|
||||
assert sched._info_labelvalues[0] == "disk"
|
||||
assert sched._info_labelvalues[1] == "true"
|
||||
stats = sched.get_stats()
|
||||
assert set(_values(stats, INFO)) == {sched._info_labelvalues}
|
||||
|
||||
|
||||
def test_manager_info_labels_page_cache_false_for_cpu_backend() -> None:
|
||||
"""CPU backend ignores use_page_cache; the info gauge must not claim it."""
|
||||
connector = _make_connector(extra_config={"use_page_cache": True})
|
||||
sched = connector.scheduler_manager
|
||||
assert sched is not None
|
||||
assert sched._info_labelvalues[:2] == ("cpu", "false")
|
||||
stats = sched.get_stats()
|
||||
assert set(_values(stats, INFO)) == {sched._info_labelvalues}
|
||||
|
||||
|
||||
def test_manager_counts_load_blocks_completed() -> None:
|
||||
fixture = make_scheduler(num_cpu_blocks=8)
|
||||
sched = fixture.scheduler
|
||||
|
||||
request = make_request(num_blocks=2)
|
||||
gpu_blocks = fixture.gpu_block_pool.get_new_blocks(2)
|
||||
cpu_blocks = sched.cpu_block_pool.get_new_blocks(2)
|
||||
sched._reqs_to_load[request.request_id] = LoadRequestState(
|
||||
request=request,
|
||||
transfer_meta=TransferMeta(
|
||||
[b.block_id for b in gpu_blocks], [b.block_id for b in cpu_blocks]
|
||||
),
|
||||
)
|
||||
|
||||
sched.build_connector_meta(make_scheduler_output({}))
|
||||
stats = sched.get_stats()
|
||||
assert LOAD_BLOCKS not in stats.data[_StatsKey.DATA]
|
||||
|
||||
sched.update_connector_output(
|
||||
KVConnectorOutput(finished_recving={request.request_id})
|
||||
)
|
||||
stats = sched.get_stats()
|
||||
assert _values(stats, LOAD_BLOCKS) == {(): 2}
|
||||
|
||||
|
||||
def test_manager_abandoned_load_counts_as_completed_after_reset() -> None:
|
||||
"""Loads abandoned by reset() that still finish on the worker are
|
||||
counted as loaded blocks.
|
||||
"""
|
||||
fixture = make_scheduler(num_cpu_blocks=8)
|
||||
sched = fixture.scheduler
|
||||
|
||||
request = make_request(num_blocks=2)
|
||||
gpu_blocks = fixture.gpu_block_pool.get_new_blocks(2)
|
||||
cpu_blocks = sched.cpu_block_pool.get_new_blocks(2)
|
||||
sched._reqs_to_load[request.request_id] = LoadRequestState(
|
||||
request=request,
|
||||
transfer_meta=TransferMeta(
|
||||
[b.block_id for b in gpu_blocks], [b.block_id for b in cpu_blocks]
|
||||
),
|
||||
)
|
||||
|
||||
sched.build_connector_meta(make_scheduler_output({}))
|
||||
stats = sched.get_stats()
|
||||
assert LOAD_BLOCKS not in stats.data[_StatsKey.DATA]
|
||||
|
||||
assert sched.reset() is False
|
||||
assert set(sched._abandoned_reqs_to_load) == {request.request_id}
|
||||
|
||||
sched.update_connector_output(
|
||||
KVConnectorOutput(finished_recving={request.request_id})
|
||||
)
|
||||
assert sched._abandoned_reqs_to_load == {}
|
||||
stats = sched.get_stats()
|
||||
assert _values(stats, LOAD_BLOCKS) == {(): 2}
|
||||
@@ -2316,8 +2316,8 @@ def _make_hybrid_attention_mamba_scheduler(
|
||||
def test_hybrid_store_uses_resolved_group_block_sizes() -> None:
|
||||
"""Replicated groups must not be scaled by the DCP world size.
|
||||
|
||||
``dcp_world_size_for_kv_cache_spec`` gives full attention the process DCP
|
||||
size and every other spec 1. Reconstructing a group's geometry as
|
||||
The spec's ``dcp_sharded`` flag controls its effective block size.
|
||||
Reconstructing a group's geometry as
|
||||
``spec.block_size * cp_world_size`` over-scales the replicated groups, so
|
||||
the eager store scan believes far fewer of their blocks are ready and
|
||||
silently offloads only a fraction of them.
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
@@ -9,7 +10,7 @@ import torch
|
||||
|
||||
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import (
|
||||
prepare_dflash_inputs,
|
||||
DFlashSpeculator,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
@@ -24,6 +25,7 @@ def _run_prepare(
|
||||
cp_rank: int = 0,
|
||||
cp_size: int = 1,
|
||||
cp_interleave: int = 1,
|
||||
draft_dcp_size: int | None = None,
|
||||
):
|
||||
device = torch.device("cuda")
|
||||
max_num_reqs = 4
|
||||
@@ -40,7 +42,9 @@ def _run_prepare(
|
||||
)
|
||||
input_batch = SimpleNamespace(
|
||||
num_reqs=1,
|
||||
num_tokens=4,
|
||||
num_scheduled_tokens=np.array([4], dtype=np.int32),
|
||||
seq_lens_cpu_upper_bound=torch.tensor([target_positions[-1] + 1]),
|
||||
positions=torch.tensor(target_positions, dtype=torch.int64, device=device),
|
||||
query_start_loc=torch.tensor([0, 4], dtype=torch.int32, device=device),
|
||||
idx_mapping=torch.tensor([2], dtype=torch.int32, device=device),
|
||||
@@ -74,36 +78,65 @@ def _run_prepare(
|
||||
next_prefill_tokens = torch.zeros_like(last_sampled)
|
||||
block_table = torch.tensor([block_table_values], dtype=torch.int32, device=device)
|
||||
|
||||
prepare_dflash_inputs(
|
||||
input_buffers,
|
||||
query_slot_mapping,
|
||||
context_positions,
|
||||
context_slot_mapping,
|
||||
sample_indices,
|
||||
sample_pos,
|
||||
sample_idx_mapping,
|
||||
temperature,
|
||||
seeds,
|
||||
input_batch,
|
||||
torch.tensor([1], dtype=torch.int32, device=device),
|
||||
torch.tensor([2], dtype=torch.int32, device=device),
|
||||
last_sampled,
|
||||
next_prefill_tokens,
|
||||
input_temperature,
|
||||
input_seeds,
|
||||
block_table,
|
||||
4,
|
||||
cp_rank,
|
||||
cp_size,
|
||||
cp_interleave,
|
||||
123,
|
||||
num_speculative_steps,
|
||||
num_speculative_steps,
|
||||
max_num_reqs,
|
||||
max_num_tokens,
|
||||
128,
|
||||
class InputsPrepared(Exception):
|
||||
pass
|
||||
|
||||
draft = SimpleNamespace(
|
||||
input_buffers=input_buffers,
|
||||
context_positions=context_positions,
|
||||
_context_slot_mappings=context_slot_mapping.unsqueeze(0),
|
||||
sample_indices=sample_indices,
|
||||
sample_pos=sample_pos,
|
||||
sample_idx_mapping=sample_idx_mapping,
|
||||
temperature=temperature,
|
||||
seeds=seeds,
|
||||
hidden_states=torch.zeros(4, 1, device=device),
|
||||
prepare_context_anchor=lambda *args: None,
|
||||
query_cudagraph_manager=None,
|
||||
dp_size=1,
|
||||
dp_rank=0,
|
||||
pcp_manager=None,
|
||||
draft_kv_cache_group_id=0,
|
||||
draft_kv_cache_group_ids=[0],
|
||||
_layer_group_idx=None,
|
||||
block_tables=SimpleNamespace(
|
||||
slot_mappings=query_slot_mapping.unsqueeze(0),
|
||||
input_block_tables=[block_table],
|
||||
kernel_block_sizes=[4],
|
||||
cp_rank=cp_rank,
|
||||
cp_size=cp_size,
|
||||
cp_interleave=cp_interleave,
|
||||
),
|
||||
dcp_size=cp_size if draft_dcp_size is None else draft_dcp_size,
|
||||
parallel_drafting_token_id=123,
|
||||
num_query_per_req=num_speculative_steps,
|
||||
num_speculative_steps=num_speculative_steps,
|
||||
max_num_reqs=max_num_reqs,
|
||||
max_num_tokens=max_num_tokens,
|
||||
max_model_len=128,
|
||||
sample_from_anchor=True,
|
||||
)
|
||||
with (
|
||||
patch(
|
||||
"vllm.v1.worker.gpu.spec_decode.dflash.speculator.dispatch_cg_and_sync_dp",
|
||||
side_effect=InputsPrepared,
|
||||
),
|
||||
pytest.raises(InputsPrepared),
|
||||
):
|
||||
DFlashSpeculator.propose(
|
||||
draft,
|
||||
input_batch,
|
||||
{},
|
||||
{},
|
||||
draft.hidden_states,
|
||||
None,
|
||||
torch.tensor([1], dtype=torch.int32, device=device),
|
||||
torch.tensor([2], dtype=torch.int32, device=device),
|
||||
last_sampled,
|
||||
next_prefill_tokens,
|
||||
input_temperature,
|
||||
input_seeds,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
return SimpleNamespace(
|
||||
input_buffers=input_buffers,
|
||||
@@ -118,12 +151,17 @@ def _run_prepare(
|
||||
)
|
||||
|
||||
|
||||
def test_prepare_dflash_inputs_excludes_rejected_context_suffix():
|
||||
@pytest.mark.parametrize("cp_rank,cp_size", [(0, 1), (0, 2), (1, 2)])
|
||||
def test_prepare_dflash_inputs_excludes_rejected_context_suffix(cp_rank, cp_size):
|
||||
# Positions 10/11 use physical block 7. Rejected positions 12/13 would use
|
||||
# block 8, but must be PAD context rather than contaminating draft KV.
|
||||
out = _run_prepare(
|
||||
target_positions=[10, 11, 12, 13],
|
||||
block_table_values=[0, 0, 7, 8, 9, 10, 11, 12],
|
||||
cp_rank=cp_rank,
|
||||
cp_size=cp_size,
|
||||
cp_interleave=2,
|
||||
draft_dcp_size=1,
|
||||
)
|
||||
|
||||
assert out.context_positions[:4].tolist() == [10, 11, 0, 0]
|
||||
|
||||
@@ -17,6 +17,7 @@ from types import MethodType, SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
@@ -53,6 +54,7 @@ def _make_fake_speculator(
|
||||
arange_np=np.arange(max_num_reqs + 1, dtype=np.int32),
|
||||
draft_is_prefilling=torch.zeros(max_num_reqs, dtype=torch.bool),
|
||||
block_tables=fake_block_tables,
|
||||
dcp_size=1,
|
||||
input_buffers=fake_input_buffers,
|
||||
attn_groups=[],
|
||||
kv_cache_config=SimpleNamespace(kv_cache_groups=[]),
|
||||
@@ -152,14 +154,17 @@ def test_build_draft_attn_metadata_clamps_to_max_model_len():
|
||||
assert torch.equal(bound, torch.tensor([1024, 503], dtype=torch.int32))
|
||||
|
||||
|
||||
def test_build_draft_attn_metadata_recomputes_dcp_local_seq_lens():
|
||||
@pytest.mark.parametrize("draft_dcp_size", [1, 2])
|
||||
def test_build_draft_attn_metadata_recomputes_dcp_local_seq_lens(draft_dcp_size):
|
||||
fake = _make_fake_speculator()
|
||||
fake.dcp_size = draft_dcp_size
|
||||
fake.block_tables.cp_size = 2
|
||||
fake.block_tables.cp_rank = 1
|
||||
fake.block_tables.cp_interleave = 4
|
||||
fake.input_buffers.seq_lens[:3] = torch.tensor([5, 9, 16])
|
||||
|
||||
def fake_prepare(out, seq_lens, num_reqs, dcp_size, dcp_rank, cp_interleave):
|
||||
assert draft_dcp_size > 1
|
||||
assert seq_lens is fake.input_buffers.seq_lens
|
||||
assert (num_reqs, dcp_size, dcp_rank, cp_interleave) == (3, 2, 1, 4)
|
||||
out[:num_reqs].copy_(torch.tensor([1, 4, 8], dtype=torch.int32))
|
||||
@@ -176,6 +181,9 @@ def test_build_draft_attn_metadata_recomputes_dcp_local_seq_lens():
|
||||
)
|
||||
|
||||
local = captured["dcp_local_seq_lens"]
|
||||
if draft_dcp_size == 1:
|
||||
assert local is None
|
||||
return
|
||||
assert isinstance(local, torch.Tensor)
|
||||
assert local.data_ptr() == fake.input_buffers.dcp_local_seq_lens.data_ptr()
|
||||
assert local.tolist() == [1, 4, 8, 0]
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -7,6 +9,38 @@ from vllm.v1.attention.backends.utils import get_dcp_local_seq_lens
|
||||
from vllm.v1.worker.cp_utils import should_skip_dcp_context_attention
|
||||
|
||||
|
||||
def test_cp_checks_allow_a_replicated_draft_backend_without_dcp(monkeypatch):
|
||||
from vllm.v1.worker import cp_utils
|
||||
|
||||
config = SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(
|
||||
prefill_context_parallel_size=1,
|
||||
decode_context_parallel_size=4,
|
||||
cp_kv_cache_interleave_size=16,
|
||||
),
|
||||
speculative_config=object(),
|
||||
)
|
||||
target = SimpleNamespace(
|
||||
impl=SimpleNamespace(
|
||||
dcp_world_size=4,
|
||||
supports_mtp_with_cp_non_trivial_interleave_size=True,
|
||||
need_to_return_lse_for_decode=True,
|
||||
)
|
||||
)
|
||||
draft = SimpleNamespace(
|
||||
impl=SimpleNamespace(
|
||||
dcp_world_size=1,
|
||||
supports_mtp_with_cp_non_trivial_interleave_size=False,
|
||||
need_to_return_lse_for_decode=False,
|
||||
)
|
||||
)
|
||||
layers = {"target": target, "draft": draft}
|
||||
monkeypatch.setattr(cp_utils, "get_layers_from_vllm_config", lambda *args: layers)
|
||||
cp_utils.check_attention_cp_compatibility(config, target_layer_names={"target"})
|
||||
with pytest.raises(AssertionError, match="MTP with cp_kv_cache_interleave_size"):
|
||||
cp_utils.check_attention_cp_compatibility(config)
|
||||
|
||||
|
||||
def test_skip_gate_only_for_zero_context():
|
||||
assert should_skip_dcp_context_attention(torch.zeros(3, dtype=torch.int32))
|
||||
assert not should_skip_dcp_context_attention(
|
||||
|
||||
@@ -2,8 +2,17 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec
|
||||
from vllm.v1.worker.gpu import model_runner
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator
|
||||
from vllm.v1.worker.gpu.spec_decode.dspark.utils import _get_dspark_parallel_config
|
||||
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -49,3 +58,86 @@ def test_dspark_parallel_config_disables_eplb_atomically():
|
||||
assert draft_config.eplb_config.num_redundant_experts == 0
|
||||
assert not draft_config.enable_elastic_ep
|
||||
assert draft_config.eplb_config is not target_config.eplb_config
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pcp_size", [1, 4])
|
||||
@pytest.mark.parametrize("dcp_size", [1, 4])
|
||||
@pytest.mark.parametrize("use_mla", [False, True])
|
||||
def test_draft_context_parallelism_without_changing_target(
|
||||
monkeypatch, pcp_size, dcp_size, use_mla
|
||||
):
|
||||
target_parallel = ParallelConfig(
|
||||
tensor_parallel_size=4,
|
||||
prefill_context_parallel_size=pcp_size,
|
||||
decode_context_parallel_size=dcp_size,
|
||||
cp_kv_cache_interleave_size=16,
|
||||
distributed_executor_backend="mp",
|
||||
)
|
||||
target_config = SimpleNamespace(
|
||||
parallel_config=target_parallel,
|
||||
speculative_config=SimpleNamespace(
|
||||
draft_model_config=SimpleNamespace(use_mla=use_mla)
|
||||
),
|
||||
)
|
||||
|
||||
class CapturedConfig(Exception):
|
||||
pass
|
||||
|
||||
def capture_init(self, config, device):
|
||||
raise CapturedConfig(config)
|
||||
|
||||
monkeypatch.setattr(DraftModelSpeculator, "__init__", capture_init)
|
||||
with pytest.raises(CapturedConfig) as captured:
|
||||
DFlashSpeculator(target_config, device=None)
|
||||
draft_parallel = captured.value.args[0].parallel_config
|
||||
assert draft_parallel.tensor_parallel_size == 4
|
||||
assert draft_parallel.prefill_context_parallel_size == 1
|
||||
assert draft_parallel.decode_context_parallel_size == (dcp_size if use_mla else 1)
|
||||
assert draft_parallel.cp_kv_cache_interleave_size == 16
|
||||
assert target_parallel.prefill_context_parallel_size == pcp_size
|
||||
assert target_parallel.decode_context_parallel_size == dcp_size
|
||||
assert target_parallel.cp_kv_cache_interleave_size == 16
|
||||
|
||||
|
||||
@pytest.mark.parametrize("configured_dcp", [1, 4])
|
||||
def test_attention_uses_draft_dcp_setting_inside_target_process_group(
|
||||
monkeypatch, configured_dcp
|
||||
):
|
||||
import vllm.config as config_module
|
||||
from vllm.distributed import parallel_state
|
||||
from vllm.v1.attention.backend import AttentionImplBase
|
||||
|
||||
config = SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=configured_dcp)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
config_module, "get_current_vllm_config_or_none", lambda: config
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
parallel_state, "_DCP", SimpleNamespace(world_size=4, rank_in_group=2)
|
||||
)
|
||||
impl = AttentionImplBase()
|
||||
assert impl.dcp_world_size == configured_dcp
|
||||
assert impl.dcp_rank == (0 if configured_dcp == 1 else 2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("draft_dcp_size", [1, 4])
|
||||
def test_runner_marks_only_replicated_draft_caches(monkeypatch, draft_dcp_size):
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=64, dtype=torch.bfloat16
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
model_runner, "get_kv_cache_spec", lambda _: {"target": spec, "draft": spec}
|
||||
)
|
||||
runner = object.__new__(model_runner.GPUModelRunner)
|
||||
runner.vllm_config = None
|
||||
runner.dcp_size = 4
|
||||
runner.speculator = object.__new__(DFlashSpeculator)
|
||||
runner.speculator.dcp_size = draft_dcp_size
|
||||
runner.speculator.draft_attn_layer_names = {"draft"}
|
||||
|
||||
specs = runner.get_kv_cache_spec()
|
||||
|
||||
assert specs["target"].dcp_sharded
|
||||
assert specs["draft"].dcp_sharded == (draft_dcp_size == 4)
|
||||
|
||||
@@ -88,6 +88,7 @@ def test_pcp_draft_metadata_keeps_graph_padding_in_decode(cg_mode):
|
||||
query_start_loc=torch.tensor([0, 1, 2, 2, 2], dtype=torch.int32),
|
||||
seq_lens=torch.tensor([11, 21, 0, 0], dtype=torch.int32),
|
||||
)
|
||||
speculator.dcp_size = 1
|
||||
speculator.block_tables = SimpleNamespace(
|
||||
cp_size=1,
|
||||
input_block_tables=[torch.zeros(4, 1, dtype=torch.int32)],
|
||||
@@ -97,7 +98,9 @@ def test_pcp_draft_metadata_keeps_graph_padding_in_decode(cg_mode):
|
||||
speculator.attn_groups = [
|
||||
[
|
||||
SimpleNamespace(
|
||||
get_metadata_builder=lambda _: SimpleNamespace(build=build),
|
||||
get_metadata_builder=lambda _: SimpleNamespace(
|
||||
build=build, supports_update_block_table=False
|
||||
),
|
||||
layer_names=["draft"],
|
||||
)
|
||||
]
|
||||
@@ -183,6 +186,7 @@ def test_speculator_uses_draft_model_hidden_size(monkeypatch, hc_mult, expected)
|
||||
parallel_config=SimpleNamespace(
|
||||
data_parallel_size=1,
|
||||
data_parallel_rank=0,
|
||||
decode_context_parallel_size=1,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -171,22 +171,23 @@ def test_block_tables_skip_custom_slot_mapping_groups():
|
||||
|
||||
@pytest.mark.parametrize("cp_rank", range(4))
|
||||
def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
|
||||
"""DCP interleave is expressed in logical-block token coordinates."""
|
||||
"""Only sharded groups use DCP interleave in logical-block coordinates."""
|
||||
device = torch.device("cuda")
|
||||
block_tables = BlockTables(
|
||||
block_sizes=[128],
|
||||
block_sizes=[128, 128],
|
||||
max_num_reqs=1,
|
||||
max_num_batched_tokens=1024,
|
||||
max_num_blocks_per_group=[2],
|
||||
max_num_blocks_per_group=[2, 8],
|
||||
device=device,
|
||||
kernel_block_sizes=[64],
|
||||
kernel_block_sizes=[64, 64],
|
||||
dcp_sharded=[True, False],
|
||||
cp_size=4,
|
||||
cp_rank=cp_rank,
|
||||
cp_interleave=128,
|
||||
)
|
||||
block_tables.append_block_ids(
|
||||
req_index=0,
|
||||
new_block_ids=([5, 9],),
|
||||
new_block_ids=([5, 9], list(range(10, 18))),
|
||||
overwrite=True,
|
||||
)
|
||||
block_tables.apply_staged_writes()
|
||||
@@ -199,7 +200,7 @@ def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
|
||||
query_start_loc,
|
||||
positions,
|
||||
num_tokens_padded=1024,
|
||||
)[0]
|
||||
)
|
||||
|
||||
expected = torch.full((1024,), -1, dtype=torch.int64, device=device)
|
||||
first_start = cp_rank * 128
|
||||
@@ -210,7 +211,8 @@ def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
|
||||
expected[second_start : second_start + 128] = torch.arange(
|
||||
9 * 128, 10 * 128, dtype=torch.int64, device=device
|
||||
)
|
||||
assert torch.equal(actual, expected)
|
||||
assert torch.equal(actual[0], expected)
|
||||
assert torch.equal(actual[1], positions + 10 * 128)
|
||||
|
||||
|
||||
def test_v1_block_table_move_row_clears_vacated_row():
|
||||
|
||||
@@ -372,6 +372,62 @@ def test_swap_states_in_input_batch(device: str, batch_size: int, swap_list: lis
|
||||
_compare_objs(input_batch, ref_input_batch)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device", DEVICES)
|
||||
def test_condense_clears_stale_allowed_token_ids_mask(device: str):
|
||||
"""condense() must clear the mask row a constrained request is moved out
|
||||
of. Otherwise a later request that reuses that row without
|
||||
allowed_token_ids inherits the stale whitelist.
|
||||
See https://github.com/vllm-project/vllm/issues/43894.
|
||||
"""
|
||||
batch_size = 4
|
||||
allowed_token_id = 13
|
||||
input_batch = InputBatch(
|
||||
max_num_reqs=batch_size,
|
||||
max_model_len=1024,
|
||||
max_num_batched_tokens=1024,
|
||||
device=torch.device(device),
|
||||
vocab_size=VOCAB_SIZE,
|
||||
block_sizes=[1],
|
||||
kernel_block_sizes=[1],
|
||||
max_num_blocks_per_req=[1024],
|
||||
)
|
||||
|
||||
def _make_req(suffix: int, allowed_token_ids=None) -> CachedRequestState:
|
||||
return CachedRequestState(
|
||||
req_id=f"req_id_{suffix}",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(allowed_token_ids=allowed_token_ids),
|
||||
pooling_params=None,
|
||||
mm_features=[],
|
||||
block_ids=([],),
|
||||
generator=None,
|
||||
num_computed_tokens=0,
|
||||
output_token_ids=[],
|
||||
)
|
||||
|
||||
# Only req_id_2 is constrained, and it lands on the highest row.
|
||||
assert input_batch.add_request(_make_req(0)) == 0
|
||||
assert input_batch.add_request(_make_req(1)) == 1
|
||||
assert input_batch.add_request(_make_req(2, [allowed_token_id])) == 2
|
||||
|
||||
mask = input_batch.allowed_token_ids_mask_cpu_tensor
|
||||
assert mask is not None
|
||||
# The constrained row masks every token except the single allowed id.
|
||||
assert not mask[2][allowed_token_id].item()
|
||||
assert int(mask[2].sum().item()) == VOCAB_SIZE - 1
|
||||
|
||||
# Free row 0, then condense: req_id_2 slides from row 2 down into row 0.
|
||||
input_batch.remove_request("req_id_0")
|
||||
input_batch.condense()
|
||||
assert input_batch.req_id_to_index["req_id_2"] == 0
|
||||
assert int(mask[2].sum().item()) == 0
|
||||
|
||||
# A new unrestricted request reuses row 2 and must stay unconstrained.
|
||||
assert input_batch.add_request(_make_req(3)) == 2
|
||||
assert "req_id_3" not in input_batch.has_allowed_token_ids
|
||||
assert int(mask[2].sum().item()) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device", DEVICES)
|
||||
def test_swap_states_preserves_allowed_token_ids_mask(device: str):
|
||||
"""swap_states() must exchange the two allowed_token_ids_mask rows.
|
||||
|
||||
@@ -75,6 +75,24 @@ def test_validate_config_rejects_full_graph_for_prefills():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("draft_uses_mla", [False, True])
|
||||
@pytest.mark.parametrize("dcp_size", [1, 2, 4])
|
||||
def test_validate_config_dcp_for_dspark(draft_uses_mla, dcp_size):
|
||||
config = _make_config(CUDAGraphMode.NONE)
|
||||
config.parallel_config.prefill_context_parallel_size = 4
|
||||
config.parallel_config.decode_context_parallel_size = dcp_size
|
||||
config.model_config.hf_text_config.index_topk = 1
|
||||
config.speculative_config = SimpleNamespace(
|
||||
use_dspark=lambda: True,
|
||||
draft_model_config=SimpleNamespace(use_mla=draft_uses_mla),
|
||||
)
|
||||
if draft_uses_mla and dcp_size == 2:
|
||||
with pytest.raises(NotImplementedError, match="requires DCP=1 or DCP=PCP"):
|
||||
PCPManager.validate_config(config, supports_mm_inputs=False)
|
||||
return
|
||||
PCPManager.validate_config(config, supports_mm_inputs=False)
|
||||
|
||||
|
||||
def test_replicated_decode_piecewise_graph_padding(monkeypatch):
|
||||
manager = PCPManager(
|
||||
pcp_world_size=2,
|
||||
|
||||
@@ -113,7 +113,6 @@ SEPARATE_GROUPS = [
|
||||
|
||||
EXCLUDE = [
|
||||
r"vllm/model_executor/models/[kK]",
|
||||
r"vllm/model_executor/models/[uU]",
|
||||
r"vllm/model_executor/models/[vV]",
|
||||
r"vllm/model_executor/models/[wW]",
|
||||
r"vllm/model_executor/models/[zZ]",
|
||||
|
||||
+51
-23
@@ -1870,28 +1870,39 @@ def _mhc_delayed_pre_tail(
|
||||
_OPS_REGISTERED = False
|
||||
|
||||
|
||||
_AITER_SITUV2_ACT_ENV = {
|
||||
"a8w4": "AITER_SITUV2_A8W4",
|
||||
"a4w4": "AITER_SITUV2_A4W4",
|
||||
}
|
||||
|
||||
|
||||
def _resolve_situv2_activation() -> str:
|
||||
"""VLLM_ROCM_USE_AITER_MOE_SITUV2 -> a4w4 | a8w4 | a16w4.
|
||||
|
||||
auto (default) and the legacy 1 mean a4w4; legacy 0 means a16w4.
|
||||
"""
|
||||
value = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2.lower()
|
||||
if value in ("auto", "1"):
|
||||
return "a4w4"
|
||||
if value == "0":
|
||||
return "a16w4"
|
||||
return value
|
||||
|
||||
|
||||
def _sync_aiter_situv2_moe_env() -> None:
|
||||
"""Mirror the SiTUv2 MoE toggle into AITER's a4w4 dispatch env.
|
||||
"""Set the AITER_SITUV2_* env matching VLLM_ROCM_USE_AITER_MOE_SITUV2.
|
||||
|
||||
AITER selects afp8 vs afp4 activation kernels via AITER_SITUV2_A8W4 /
|
||||
AITER_SITUV2_A4W4 (see ROCm/aiter fused_moe.py, A8W4 checked first).
|
||||
When VLLM_ROCM_USE_AITER_MOE_SITUV2 is enabled we route to a4w4
|
||||
(afp4_wfp4_fp4 kernels) and clear any legacy AITER_SITUV2_A8W4 override.
|
||||
|
||||
Requires AITER with ROCm/aiter#4463 (first tagged in v0.1.20): a4w4
|
||||
dispatch plus kimik3_a4w4_{un,}tuned_fmoe.csv. Older AITER still runs
|
||||
afp4 FlyDSL but falls back to heuristic configs, which is not the
|
||||
tuned a4w4 path this recipe is meant to use.
|
||||
AITER reads AITER_SITUV2_A8W4 / AITER_SITUV2_A4W4 and defaults to a16w4,
|
||||
so exactly one is set for a8w4/a4w4 and both are cleared for a16w4.
|
||||
"""
|
||||
import os
|
||||
|
||||
import vllm.envs as envs
|
||||
|
||||
if envs.VLLM_ROCM_USE_AITER_MOE_SITUV2:
|
||||
os.environ["AITER_SITUV2_A4W4"] = "1"
|
||||
os.environ.pop("AITER_SITUV2_A8W4", None)
|
||||
else:
|
||||
os.environ.pop("AITER_SITUV2_A4W4", None)
|
||||
selected = _AITER_SITUV2_ACT_ENV.get(_resolve_situv2_activation())
|
||||
for name in _AITER_SITUV2_ACT_ENV.values():
|
||||
if name == selected:
|
||||
os.environ[name] = "1"
|
||||
else:
|
||||
os.environ.pop(name, None)
|
||||
|
||||
|
||||
class rocm_aiter_ops:
|
||||
@@ -1917,7 +1928,8 @@ class rocm_aiter_ops:
|
||||
VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: Controls FP4 assembly GEMM.
|
||||
VLLM_ROCM_USE_AITER_TRITON_ROPE: Controls Triton rotary embeddings.
|
||||
VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: Controls shared expert fusion.
|
||||
VLLM_ROCM_USE_AITER_MOE_SITUV2: Controls SiTUv2 FlyDSL MoE (a4w4).
|
||||
VLLM_ROCM_USE_AITER_MOE_SITUV2: SiTUv2 FlyDSL MoE activation
|
||||
dtype (a16w4 | a8w4 | a4w4).
|
||||
VLLM_ROCM_USE_AITER_TRITON_GEMM: Controls Triton unquantized GEMM.
|
||||
|
||||
Note:
|
||||
@@ -1987,7 +1999,7 @@ class rocm_aiter_ops:
|
||||
# TODO: Consolidate under VLLM_ROCM_USE_AITER_ROPE
|
||||
_TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE
|
||||
_MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
|
||||
_MOE_SITUV2 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2
|
||||
_MOE_SITUV2 = _resolve_situv2_activation()
|
||||
# TODO: Consolidate under _LINEAR_ENABLED
|
||||
_TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM
|
||||
# Lazily probed: whether aiter.topk_softmax supports the
|
||||
@@ -2016,7 +2028,7 @@ class rocm_aiter_ops:
|
||||
cls._FP4_GEMM_DYNAMIC_QUANT_ASM = envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM
|
||||
cls._TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE
|
||||
cls._MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
|
||||
cls._MOE_SITUV2 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2
|
||||
cls._MOE_SITUV2 = _resolve_situv2_activation()
|
||||
_sync_aiter_situv2_moe_env()
|
||||
cls._TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM
|
||||
cls._MOE_DISPATCH_POLICY = envs.VLLM_ROCM_AITER_MOE_DISPATCH_POLICY
|
||||
@@ -2132,9 +2144,25 @@ class rocm_aiter_ops:
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
def is_fused_moe_situv2_enabled(cls) -> bool:
|
||||
# _MOE_SITUV2 is a variant of aiter fused moe, so aiter
|
||||
# fused moe must be enabled as well.
|
||||
return cls.is_fused_moe_enabled() and cls._MOE_SITUV2
|
||||
"""True when a low-precision (a8w4/a4w4) SiTUv2 activation is selected."""
|
||||
return cls.is_fused_moe_enabled() and cls._MOE_SITUV2 != "a16w4"
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
def get_fused_moe_situv2_activation(cls) -> str:
|
||||
"""SiTUv2 activation dtype name: a16w4, a8w4 or a4w4."""
|
||||
return cls._MOE_SITUV2
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
def is_fused_moe_situv2_gate_up_interleaved(cls) -> bool:
|
||||
"""Whether the SiTUv2 kernels expect gate/up-interleaved w13.
|
||||
|
||||
The a8w4 FlyDSL stage-1 kernels (_gui_) are interleaved; a16w4 and
|
||||
a4w4 are separated. Weight shuffle (oracle/mxfp4.py) and gate_mode
|
||||
(experts/rocm_aiter_moe.py) must both use this.
|
||||
"""
|
||||
return cls._MOE_SITUV2 == "a8w4"
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
|
||||
@@ -2148,7 +2148,9 @@ def get_samples(
|
||||
dataset_path=args.dataset_path, disable_shuffle=args.disable_shuffle
|
||||
)
|
||||
# For the "sonnet" dataset, formatting depends on the backend.
|
||||
if args.backend == "openai-chat":
|
||||
# Chat-style backends leave templating to the server; completions
|
||||
# backends need the prompt rendered client side.
|
||||
if args.backend in ("openai-chat", "openai-responses"):
|
||||
input_requests = sonnet_dataset.sample(
|
||||
num_requests=args.num_prompts,
|
||||
input_len=args.sonnet_input_len,
|
||||
|
||||
@@ -447,6 +447,178 @@ async def async_request_openai_chat_completions(
|
||||
return output
|
||||
|
||||
|
||||
def _extract_sse_data(message: str) -> str | None:
|
||||
"""Extract the ``data`` payload of a single SSE message.
|
||||
|
||||
The Responses API stream labels every message with an ``event:`` line
|
||||
before its ``data:`` line, so the payload cannot be recovered by
|
||||
stripping a leading ``data: `` the way the Completions and Chat
|
||||
Completions streams allow.
|
||||
|
||||
Args:
|
||||
message: One complete SSE message, without its trailing blank line.
|
||||
|
||||
Returns:
|
||||
The payload, with multi-line payloads joined by newlines as required
|
||||
by the SSE spec, or None if the message carries no ``data:`` line.
|
||||
|
||||
"""
|
||||
# NOTE: split on "\n" rather than using splitlines(), which also breaks on
|
||||
# U+2028, U+2029 and U+0085. Those are legal inside a JSON string and the
|
||||
# server emits them unescaped, so splitlines() would truncate the payload.
|
||||
data_lines = [
|
||||
line.removeprefix("data:").removeprefix(" ")
|
||||
for line in message.split("\n")
|
||||
if line.startswith("data:")
|
||||
]
|
||||
if not data_lines:
|
||||
return None
|
||||
|
||||
return "\n".join(data_lines)
|
||||
|
||||
|
||||
# Responses events that carry generated tokens. Reasoning deltas are included
|
||||
# because the server is already decoding when it emits them, so excluding them
|
||||
# would report the whole reasoning phase as time to first token.
|
||||
_RESPONSES_TOKEN_DELTA_EVENTS = frozenset(
|
||||
{
|
||||
"response.output_text.delta",
|
||||
"response.reasoning_text.delta",
|
||||
}
|
||||
)
|
||||
|
||||
# Responses events that end a request. Truncation at the output length limit
|
||||
# is a normal benchmark outcome: vLLM reports it as `response.completed` with
|
||||
# `status: "incomplete"`, while the spec also defines a dedicated
|
||||
# `response.incomplete` terminal event.
|
||||
_RESPONSES_TERMINAL_EVENTS = frozenset(
|
||||
{
|
||||
"response.completed",
|
||||
"response.incomplete",
|
||||
}
|
||||
)
|
||||
|
||||
# The spec's failure terminals. vLLM does not emit them today, but
|
||||
# `vllm bench serve` can be pointed at other Responses servers.
|
||||
_RESPONSES_ERROR_EVENTS = frozenset({"error", "response.failed"})
|
||||
|
||||
|
||||
async def async_request_openai_responses(
|
||||
request_func_input: RequestFuncInput,
|
||||
session: aiohttp.ClientSession,
|
||||
pbar: tqdm | None = None,
|
||||
) -> RequestFuncOutput:
|
||||
api_url = request_func_input.api_url
|
||||
_validate_api_url(api_url, "OpenAI Responses API", "responses")
|
||||
|
||||
payload = {
|
||||
"model": request_func_input.model_name
|
||||
if request_func_input.model_name
|
||||
else request_func_input.model,
|
||||
"input": request_func_input.prompt,
|
||||
"max_output_tokens": request_func_input.output_len,
|
||||
"stream": True,
|
||||
# `store` defaults to true. A server started with
|
||||
# VLLM_ENABLE_RESPONSES_API_STORE=1 keeps every stored response for
|
||||
# the lifetime of the process, which would grow its heap over a long
|
||||
# benchmark run.
|
||||
"store": False,
|
||||
}
|
||||
_update_payload_common(payload, request_func_input)
|
||||
|
||||
headers = _get_headers("application/json")
|
||||
_update_headers_common(headers, request_func_input)
|
||||
|
||||
output = RequestFuncOutput()
|
||||
output.prompt_len = request_func_input.prompt_len
|
||||
|
||||
# Only output text is accumulated. Reasoning text is timed but not
|
||||
# returned, matching how `openai-chat` handles `DeltaMessage.reasoning`.
|
||||
generated_text = ""
|
||||
stream_error: str | None = None
|
||||
saw_terminal_event = False
|
||||
first_chunk_received = False
|
||||
st = time.perf_counter()
|
||||
output.start_time = st
|
||||
most_recent_timestamp = st
|
||||
try:
|
||||
async with session.post(url=api_url, json=payload, headers=headers) as response:
|
||||
if response.status == 200:
|
||||
handler = StreamedResponseHandler()
|
||||
|
||||
async for chunk_bytes in response.content.iter_any():
|
||||
for message in handler.add_chunk(chunk_bytes):
|
||||
# NOTE: `_extract_sse_data` returns None for SSE
|
||||
# comments, which servers may send as keep-alives.
|
||||
chunk = _extract_sse_data(message)
|
||||
if not chunk or chunk == "[DONE]":
|
||||
continue
|
||||
|
||||
timestamp = time.perf_counter()
|
||||
data = json.loads(chunk)
|
||||
event_type = data.get("type")
|
||||
|
||||
if event_type in _RESPONSES_TOKEN_DELTA_EVENTS:
|
||||
# First token
|
||||
if not first_chunk_received:
|
||||
first_chunk_received = True
|
||||
output.ttft = timestamp - st
|
||||
|
||||
# Decoding phase
|
||||
else:
|
||||
output.itl.append(timestamp - most_recent_timestamp)
|
||||
|
||||
if event_type == "response.output_text.delta":
|
||||
generated_text += data.get("delta") or ""
|
||||
|
||||
# Only token events advance the end of the
|
||||
# request, so `latency - ttft` stays equal to
|
||||
# `sum(itl)` and TPOT means what serve.py says.
|
||||
most_recent_timestamp = timestamp
|
||||
elif event_type in _RESPONSES_TERMINAL_EVENTS:
|
||||
saw_terminal_event = True
|
||||
usage = (data.get("response") or {}).get("usage") or {}
|
||||
if (n_output := usage.get("output_tokens")) is not None:
|
||||
output.output_tokens = n_output
|
||||
if (n_input := usage.get("input_tokens")) is not None:
|
||||
output.prompt_len = n_input
|
||||
elif event_type in _RESPONSES_ERROR_EVENTS or data.get("error"):
|
||||
stream_error = json.dumps(data.get("error") or data)
|
||||
|
||||
output.generated_text = generated_text
|
||||
output.latency = most_recent_timestamp - st
|
||||
if stream_error is not None:
|
||||
output.success = False
|
||||
output.error = stream_error
|
||||
elif not first_chunk_received:
|
||||
output.success = False
|
||||
output.error = (
|
||||
"Never received a token delta to calculate TTFT."
|
||||
"This response will be marked as failed!"
|
||||
)
|
||||
elif not saw_terminal_event:
|
||||
# Without the terminal event the response is truncated and
|
||||
# its usage block is missing.
|
||||
output.success = False
|
||||
output.error = (
|
||||
"Stream ended without a terminal event."
|
||||
"This response will be marked as failed!"
|
||||
)
|
||||
else:
|
||||
output.success = True
|
||||
else:
|
||||
output.error = response.reason or ""
|
||||
output.success = False
|
||||
except Exception:
|
||||
output.success = False
|
||||
exc_info = sys.exc_info()
|
||||
output.error = "".join(traceback.format_exception(*exc_info))
|
||||
|
||||
if pbar:
|
||||
pbar.update(1)
|
||||
return output
|
||||
|
||||
|
||||
async def async_request_openai_audio(
|
||||
request_func_input: RequestFuncInput,
|
||||
session: aiohttp.ClientSession,
|
||||
@@ -920,6 +1092,7 @@ ASYNC_REQUEST_FUNCS: dict[str, RequestFunc] = {
|
||||
"vllm": async_request_openai_completions,
|
||||
"openai": async_request_openai_completions,
|
||||
"openai-chat": async_request_openai_chat_completions,
|
||||
"openai-responses": async_request_openai_responses,
|
||||
"openai-audio": async_request_openai_audio,
|
||||
"openai-embeddings": async_request_openai_embeddings,
|
||||
"openai-embeddings-chat": async_request_openai_embeddings_chat,
|
||||
@@ -947,5 +1120,10 @@ POOLING_BACKENDS = {
|
||||
OPENAI_COMPATIBLE_BACKENDS = [
|
||||
k
|
||||
for k, v in ASYNC_REQUEST_FUNCS.items()
|
||||
if v in (async_request_openai_completions, async_request_openai_chat_completions)
|
||||
if v
|
||||
in (
|
||||
async_request_openai_completions,
|
||||
async_request_openai_chat_completions,
|
||||
async_request_openai_responses,
|
||||
)
|
||||
]
|
||||
|
||||
@@ -2216,6 +2216,11 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
|
||||
"Sampling parameters are only supported by openai-compatible backends."
|
||||
)
|
||||
|
||||
# The Responses API accepts every sampling parameter above except
|
||||
# min_p, which it would silently drop as an unknown field.
|
||||
if args.backend == "openai-responses" and "min_p" in sampling_params:
|
||||
raise ValueError("--min-p is not supported by the Responses API.")
|
||||
|
||||
if "temperature" not in sampling_params:
|
||||
print(
|
||||
"WARNING: vllm bench serve no longer sets temperature==0 (greedy) "
|
||||
|
||||
@@ -1718,6 +1718,26 @@ class VllmConfig:
|
||||
)
|
||||
self.compilation_config.mode = CompilationMode.NONE
|
||||
|
||||
# TODO: This is a stopgap that turns sequence parallelism / async TP
|
||||
# off under batch invariance. Make the sequence-parallel reduce-scatter
|
||||
# path batch-invariant instead so they can stay enabled (#56370).
|
||||
# Sequence parallelism / async TP rewrite all_reduce + rms_norm into a
|
||||
# reduce-scatter whose reduction order depends on the batch, so they
|
||||
# are not batch-invariant. Decide this before the breakable-CUDA-graph
|
||||
# default below, which declines to auto-enable when they are on.
|
||||
pass_config = self.compilation_config.pass_config
|
||||
if envs.VLLM_BATCH_INVARIANT and (
|
||||
pass_config.enable_sp or pass_config.fuse_gemm_comms
|
||||
):
|
||||
logger.warning_once(
|
||||
"Disabling sequence parallelism and async TP "
|
||||
"(pass_config.enable_sp / fuse_gemm_comms) when "
|
||||
"VLLM_BATCH_INVARIANT is enabled: the reduce-scatter path "
|
||||
"is not batch-invariant (see vllm-project/vllm#56370)."
|
||||
)
|
||||
pass_config.enable_sp = False
|
||||
pass_config.fuse_gemm_comms = False
|
||||
|
||||
breakable_cudagraph_enabled = self._maybe_enable_breakable_cudagraph()
|
||||
|
||||
if not breakable_cudagraph_enabled and (
|
||||
|
||||
@@ -42,6 +42,7 @@ The class provides the following primitives:
|
||||
import enum
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Callable, Iterable
|
||||
from contextlib import AbstractContextManager
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any, Literal
|
||||
|
||||
@@ -227,6 +228,13 @@ class KVConnectorBase_V1(ABC):
|
||||
# Worker-side methods
|
||||
# ==============================
|
||||
|
||||
def get_mem_pool_context(self) -> AbstractContextManager | None:
|
||||
"""Return a custom KV cache allocation context, if configured.
|
||||
|
||||
Returning None uses the engine's default memory pool.
|
||||
"""
|
||||
return None
|
||||
|
||||
def bind_connector_metadata(self, connector_metadata: KVConnectorMetadata) -> None:
|
||||
"""Set the connector metadata from the scheduler.
|
||||
|
||||
|
||||
@@ -59,7 +59,6 @@ from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
dcp_world_size_for_kv_cache_spec,
|
||||
resolve_dcp_kv_block_size,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
@@ -259,13 +258,7 @@ class DecodeBenchConnectorScheduler:
|
||||
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
|
||||
self.kv_cache_groups = kv_cache_config.kv_cache_groups
|
||||
self.group_block_sizes = tuple(
|
||||
resolve_dcp_kv_block_size(
|
||||
group.kv_cache_spec,
|
||||
dcp_world_size_for_kv_cache_spec(
|
||||
group.kv_cache_spec,
|
||||
dcp_world_size,
|
||||
),
|
||||
)
|
||||
resolve_dcp_kv_block_size(group.kv_cache_spec, dcp_world_size)
|
||||
for group in self.kv_cache_groups
|
||||
)
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ from collections.abc import Collection
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from enum import IntEnum
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
import msgspec
|
||||
import numpy as np
|
||||
@@ -70,6 +70,8 @@ from vllm.v1.worker.utils import select_common_block_size
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_BOOTSTRAP_MAX_ATTEMPTS: Final[int] = 3
|
||||
|
||||
try:
|
||||
from mooncake.engine import TransferEngine
|
||||
except ImportError:
|
||||
@@ -1147,24 +1149,44 @@ class MooncakeConnectorWorker:
|
||||
pp_rank=self.pp_rank,
|
||||
addr=worker_addr,
|
||||
)
|
||||
while True:
|
||||
timeout = envs.VLLM_MOONCAKE_CONNECTOR_TIMEOUT
|
||||
max_attempts = _BOOTSTRAP_MAX_ATTEMPTS
|
||||
backoff = 1.0
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
async with httpx.AsyncClient(timeout=timeout) as client:
|
||||
response = await client.post(url, json=payload.model_dump())
|
||||
response.raise_for_status()
|
||||
logger.debug("Successfully registered with bootstrap server at %s", url)
|
||||
break
|
||||
except httpx.ConnectError:
|
||||
# Bootstrap server not ready, wait for a while and retry.
|
||||
await asyncio.sleep(1)
|
||||
return
|
||||
except (httpx.ConnectError, httpx.TimeoutException) as e:
|
||||
# Rank 0 may be busy registering a large memory segment.
|
||||
if attempt == max_attempts:
|
||||
break
|
||||
logger.warning(
|
||||
"Bootstrap registration attempt %d/%d for %s failed with %s: %s. "
|
||||
"Retrying in %.1fs.",
|
||||
attempt,
|
||||
max_attempts,
|
||||
payload,
|
||||
type(e).__name__,
|
||||
e,
|
||||
backoff,
|
||||
)
|
||||
await asyncio.sleep(backoff)
|
||||
backoff = min(backoff * 2, 10.0)
|
||||
except Exception as e:
|
||||
err_msg = (
|
||||
e.response.text if isinstance(e, httpx.HTTPStatusError) else str(e)
|
||||
)
|
||||
logger.error(
|
||||
logger.exception(
|
||||
"Error registering %s with bootstrap server: %s", payload, err_msg
|
||||
)
|
||||
raise e
|
||||
raise
|
||||
raise RuntimeError(
|
||||
f"Failed to register {payload} with the Mooncake bootstrap server at "
|
||||
f"{url} after {max_attempts} attempts."
|
||||
)
|
||||
|
||||
async def _mooncake_sender_listener(self, ready_event: threading.Event):
|
||||
"""Background thread that listens for Mooncake requests, dispatches them
|
||||
@@ -1800,11 +1822,29 @@ class MooncakeConnectorWorker:
|
||||
if self.is_kv_consumer:
|
||||
return
|
||||
|
||||
# httpx applies the timeout to each phase (connect, read, write, pool)
|
||||
# rather than to the whole request, so budget for all four.
|
||||
register_timeout = envs.VLLM_MOONCAKE_CONNECTOR_TIMEOUT
|
||||
max_attempts = _BOOTSTRAP_MAX_ATTEMPTS
|
||||
backoff_total = sum(min(2.0**i, 10.0) for i in range(max_attempts - 1))
|
||||
ready_timeout = max_attempts * 4 * register_timeout + backoff_total + 30.0
|
||||
ready_event = threading.Event()
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
fut = asyncio.run_coroutine_threadsafe(
|
||||
self._mooncake_sender_listener(ready_event), self.sender_loop
|
||||
)
|
||||
ready_event.wait() # Wait for listener ZMQ socket to be ready.
|
||||
deadline = time.monotonic() + ready_timeout
|
||||
while not ready_event.wait(timeout=1.0):
|
||||
if fut.done():
|
||||
fut.result()
|
||||
raise RuntimeError(
|
||||
"Mooncake sender listener exited before becoming ready."
|
||||
)
|
||||
if time.monotonic() > deadline:
|
||||
fut.cancel()
|
||||
raise RuntimeError(
|
||||
"Mooncake sender listener did not become ready within "
|
||||
f"{ready_timeout:.0f}s."
|
||||
)
|
||||
|
||||
async def fetch_finished_recving_reqs(self) -> set[ReqId]:
|
||||
finished_recving_reqs = self.finished_recving_reqs
|
||||
|
||||
@@ -97,14 +97,19 @@ class MooncakeBootstrapServer:
|
||||
dp_entry.worker_addr[payload.tp_rank] = {}
|
||||
|
||||
tp_entry = dp_entry.worker_addr[payload.tp_rank]
|
||||
if payload.pp_rank in tp_entry:
|
||||
existing = tp_entry.get(payload.pp_rank)
|
||||
if existing is not None:
|
||||
# A client timeout can fire after the server recorded the
|
||||
# registration so an identical retry must not be an error.
|
||||
if existing == payload.addr:
|
||||
return {"status": "ok"}
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=(
|
||||
f"Worker with dp_rank={payload.dp_rank}, "
|
||||
f"tp_rank={payload.tp_rank}, pp_rank={payload.pp_rank} "
|
||||
f"is already registered at "
|
||||
f"{tp_entry[payload.pp_rank]}, "
|
||||
f"{existing}, "
|
||||
f"but still want to register at {payload.addr}"
|
||||
),
|
||||
)
|
||||
|
||||
@@ -13,6 +13,7 @@ enabling prefix caching via hash-based deduplication.
|
||||
|
||||
from collections import Counter
|
||||
from collections.abc import Iterable, Sequence
|
||||
from contextlib import AbstractContextManager
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
@@ -322,6 +323,17 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
# Worker-side methods
|
||||
# ============================================================
|
||||
|
||||
def get_mem_pool_context(self) -> AbstractContextManager | None:
|
||||
"""Return a context manager for the custom MemPool, or None.
|
||||
|
||||
Called by the Worker before ``initialize_kv_cache`` so that KV
|
||||
cache is allocated from the Mooncake-managed pool when
|
||||
``custom_mem_pool`` is set in ``kv_connector_extra_config``.
|
||||
"""
|
||||
if self.connector_worker is None:
|
||||
return None
|
||||
return self.connector_worker.get_mem_pool_context()
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.register_kv_caches(kv_caches)
|
||||
|
||||
@@ -19,13 +19,14 @@ import threading
|
||||
import time
|
||||
from collections.abc import Callable, Sequence
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from contextlib import AbstractContextManager
|
||||
from typing import Any, TypeVar
|
||||
|
||||
import torch
|
||||
import zmq
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_dcp_group,
|
||||
get_pcp_group,
|
||||
@@ -1414,6 +1415,41 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
class MooncakeStoreWorker:
|
||||
"""Worker-side component for MooncakeStoreConnector."""
|
||||
|
||||
@staticmethod
|
||||
def _create_mem_pool(
|
||||
extra_config: dict[str, Any], model_config: ModelConfig
|
||||
) -> torch.cuda.MemPool | None:
|
||||
pool_type = str(extra_config.get("custom_mem_pool") or "").upper()
|
||||
if not pool_type:
|
||||
return None
|
||||
if pool_type not in ("NVLINK", "BAREX"):
|
||||
raise ValueError(
|
||||
f"Unsupported custom_mem_pool={pool_type!r}, "
|
||||
"expected 'NVLINK' or 'BAREX'"
|
||||
)
|
||||
if model_config.enable_sleep_mode or model_config.enable_cumem_allocator:
|
||||
raise ValueError(
|
||||
"custom_mem_pool is incompatible with enable_sleep_mode "
|
||||
"or enable_cumem_allocator; CuMemAllocator cannot manage "
|
||||
"allocations from the custom pool."
|
||||
)
|
||||
try:
|
||||
if pool_type == "NVLINK":
|
||||
from mooncake.allocator import NVLinkAllocator as allocator_cls
|
||||
else: # pool_type == "BAREX"
|
||||
from mooncake.allocator import BarexAllocator as allocator_cls
|
||||
except ImportError as e:
|
||||
raise ImportError(
|
||||
f"custom_mem_pool={pool_type!r} requires "
|
||||
"mooncake-transfer-engine>=0.3.8. Please upgrade Mooncake."
|
||||
) from e
|
||||
|
||||
device = torch.device("cuda", torch.accelerator.current_device_index())
|
||||
allocator = allocator_cls.get_allocator(device)
|
||||
mem_pool = torch.cuda.MemPool(allocator.allocator())
|
||||
logger.info("Using Mooncake custom memory pool: %s", pool_type)
|
||||
return mem_pool
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
@@ -1471,6 +1507,10 @@ class MooncakeStoreWorker:
|
||||
|
||||
# Initialize MooncakeDistributedStore with its own TransferEngine
|
||||
store_config = MooncakeStoreConfig.load_from_config()
|
||||
# Validate and create the custom memory pool before initializing the
|
||||
# store so failures cannot leak a partially initialized store handle.
|
||||
self._mem_pool = self._create_mem_pool(extra_config, model_config)
|
||||
|
||||
self.store = MooncakeDistributedStore()
|
||||
local_ip = get_ip()
|
||||
local_hostname = rdma_utils.get_requester_local_hostname(local_ip)
|
||||
@@ -1741,6 +1781,13 @@ class MooncakeStoreWorker:
|
||||
)
|
||||
return token_dbs
|
||||
|
||||
def get_mem_pool_context(self) -> AbstractContextManager | None:
|
||||
"""Return a context manager for the custom MemPool, or None if
|
||||
no custom pool is configured."""
|
||||
if self._mem_pool is None:
|
||||
return None
|
||||
return torch.cuda.use_mem_pool(self._mem_pool)
|
||||
|
||||
def _spec_tp_replication_factor(self, spec: KVCacheSpec) -> int:
|
||||
if self.dcp_size > 1:
|
||||
return 1
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import copy
|
||||
from collections.abc import Callable, Iterable
|
||||
from contextlib import AbstractContextManager
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
@@ -294,6 +295,26 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
# ==============================
|
||||
# Worker-side methods
|
||||
# ==============================
|
||||
def get_mem_pool_context(self) -> AbstractContextManager | None:
|
||||
"""Forward a custom KV cache memory pool from a child connector.
|
||||
|
||||
KV cache is allocated once and can only live in one pool, so at
|
||||
most one child may provide a context.
|
||||
"""
|
||||
found: list[tuple[str, AbstractContextManager]] = []
|
||||
for connector in self._connectors:
|
||||
if (ctx := connector.get_mem_pool_context()) is not None:
|
||||
found.append((type(connector).__name__, ctx))
|
||||
|
||||
if len(found) > 1:
|
||||
names = [name for name, _ in found]
|
||||
raise ValueError(
|
||||
f"Multiple connectors provide a KV cache memory pool {names}; "
|
||||
"KV cache is allocated once and can only live in one pool. "
|
||||
"Configure custom_mem_pool on at most one connector."
|
||||
)
|
||||
return found[0][1] if found else None
|
||||
|
||||
def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None:
|
||||
for c in self._connectors:
|
||||
c.start_load_kv(forward_context, **kwargs)
|
||||
|
||||
@@ -839,6 +839,8 @@ class NixlBaseConnectorWorker:
|
||||
self._engine_ttl: float = vllm_config.kv_transfer_config.get_from_extra_config(
|
||||
"engine_ttl", 3600.0
|
||||
)
|
||||
# Map of pull peer (host, port) -> engine_id currently serving it.
|
||||
self._engine_by_address: dict[tuple[str, int], EngineId] = {}
|
||||
|
||||
self.model_config = vllm_config.model_config
|
||||
|
||||
@@ -1300,6 +1302,7 @@ class NixlBaseConnectorWorker:
|
||||
self._remote_agents[eid] = remote_agents
|
||||
self._engine_clock_offset[eid] = clock_offset
|
||||
self._engine_last_active[eid] = time.perf_counter()
|
||||
self._track_remote_engine_replacement(eid, host, port)
|
||||
except Exception as e:
|
||||
self._log_failure(
|
||||
failure_type="handshake_setup_failed",
|
||||
@@ -2953,6 +2956,7 @@ class NixlBaseConnectorWorker:
|
||||
# Handle timeout to avoid stranding blocks on remote.
|
||||
self._reap_expired_send_leases(done_sending)
|
||||
|
||||
self._cleanup_replaced_remote_engines()
|
||||
return KVConnectorTransferResults(
|
||||
finished_sending=done_sending,
|
||||
finished_recving=done_recving,
|
||||
@@ -3541,14 +3545,53 @@ class NixlBaseConnectorWorker:
|
||||
and meta.remote is not None
|
||||
}
|
||||
|
||||
def _track_remote_engine_replacement(
|
||||
self, engine_id: EngineId, host: str, port: int
|
||||
) -> None:
|
||||
"""Record the engine currently serving a pull peer address."""
|
||||
# TODO: Also handle push mode, which handshakes in both directions.
|
||||
if self._TRANSFER_MODE != "pull":
|
||||
return
|
||||
previous = self._engine_by_address.get((host, port))
|
||||
self._engine_by_address[(host, port)] = engine_id
|
||||
if previous is None or previous == engine_id:
|
||||
return
|
||||
logger.info(
|
||||
"Remote engine at %s:%d changed from %s to %s.",
|
||||
host,
|
||||
port,
|
||||
previous,
|
||||
engine_id,
|
||||
)
|
||||
|
||||
def _cleanup_replaced_remote_engines(self) -> None:
|
||||
"""Release replaced pull peers once requests and handshakes have drained."""
|
||||
if self._TRANSFER_MODE != "pull":
|
||||
return
|
||||
with self._handshake_lock:
|
||||
if self._handshake_futures:
|
||||
return
|
||||
replaced = self._remote_agents.keys() - self._engine_by_address.values()
|
||||
if not replaced:
|
||||
return
|
||||
# _recving_metadata is only accessed from this (main) thread.
|
||||
busy = {
|
||||
meta.remote.engine_id
|
||||
for meta in self._recving_metadata.values()
|
||||
if meta.remote is not None
|
||||
}
|
||||
for engine_id in replaced - busy:
|
||||
self._cleanup_remote_engine(engine_id, log_eviction=False)
|
||||
logger.info("Released NIXL state for replaced remote engine %s.", engine_id)
|
||||
|
||||
def _cleanup_remote_engine(
|
||||
self, engine_id: EngineId, *, log_eviction: bool = True
|
||||
) -> None:
|
||||
"""Remove all state for a single remote engine.
|
||||
|
||||
Releases NIXL resources (dlist handles, remote agents) and clears
|
||||
all per-engine data structures. Used by both TTL eviction and
|
||||
shutdown.
|
||||
all per-engine data structures. Used by TTL eviction, replaced peer
|
||||
cleanup and shutdown.
|
||||
"""
|
||||
assert engine_id in self._remote_agents
|
||||
|
||||
@@ -3558,6 +3601,9 @@ class NixlBaseConnectorWorker:
|
||||
# Pop under the handshake lock; NIXL teardown stays outside it.
|
||||
with self._handshake_lock:
|
||||
agents = self._remote_agents.pop(engine_id)
|
||||
for address, eid in list(self._engine_by_address.items()):
|
||||
if eid == engine_id:
|
||||
del self._engine_by_address[address]
|
||||
for agent_name in agents.values():
|
||||
self.nixl_wrapper.remove_remote_agent(agent_name)
|
||||
|
||||
|
||||
@@ -84,7 +84,14 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
|
||||
# Start transfers for requests whose handshakes have now finished.
|
||||
while not self._ready_requests.empty():
|
||||
self._read_blocks_for_req(*self._ready_requests.get_nowait())
|
||||
req_id, meta = self._ready_requests.get_nowait()
|
||||
assert meta.remote is not None
|
||||
if meta.remote.engine_id not in self._remote_agents:
|
||||
# The engine was released after its handshake completed, so
|
||||
# handshake again. This fails if the engine is gone.
|
||||
self._background_nixl_handshake(req_id, meta.remote.engine_id, meta)
|
||||
continue
|
||||
self._read_blocks_for_req(req_id, meta)
|
||||
|
||||
if self.pcp_rank > 0 and not self.pcp_dcp_sharded:
|
||||
# Replicated-KV PCP: only PCP rank 0 serves the KV, so this rank
|
||||
|
||||
@@ -15,6 +15,12 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorRole,
|
||||
SupportsHMA,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorPromMetrics,
|
||||
KVConnectorStats,
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
@@ -25,6 +31,10 @@ from vllm.v1.simple_kv_offload.manager import (
|
||||
from vllm.v1.simple_kv_offload.metadata import (
|
||||
SimpleCPUOffloadMetadata,
|
||||
)
|
||||
from vllm.v1.simple_kv_offload.metrics import (
|
||||
SimpleCPUOffloadPromMetrics,
|
||||
SimpleCPUOffloadStats,
|
||||
)
|
||||
from vllm.v1.simple_kv_offload.worker import (
|
||||
SimpleCPUOffloadWorker,
|
||||
)
|
||||
@@ -162,6 +172,7 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
hash_block_size=hash_block_size,
|
||||
lazy_offload=lazy_offload,
|
||||
disk_capacity_bytes=disk_capacity_bytes if disk_mode else 0,
|
||||
use_page_cache=use_page_cache if disk_mode else False,
|
||||
)
|
||||
elif role == KVConnectorRole.WORKER:
|
||||
self.worker_handler = SimpleCPUOffloadWorker(
|
||||
@@ -307,6 +318,38 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
return self.scheduler_manager.get_boundary_store_stats()
|
||||
return None
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
# Worker-side callers may invoke this hook on connectors constructed
|
||||
# via __new__ (as in test_worker.py), which lack scheduler_manager.
|
||||
scheduler_manager: SimpleCPUOffloadScheduler | None = getattr(
|
||||
self, "scheduler_manager", None
|
||||
)
|
||||
if scheduler_manager is not None:
|
||||
return scheduler_manager.get_stats()
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def build_kv_connector_stats(
|
||||
cls, data: dict[str, Any] | None = None
|
||||
) -> KVConnectorStats | None:
|
||||
return (
|
||||
SimpleCPUOffloadStats(data=data)
|
||||
if data is not None
|
||||
else SimpleCPUOffloadStats()
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def build_prom_metrics(
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
metric_types: dict[type[PromMetric], type[PromMetricT]],
|
||||
labelnames: list[str],
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
) -> KVConnectorPromMetrics:
|
||||
return SimpleCPUOffloadPromMetrics(
|
||||
vllm_config, metric_types, labelnames, per_engine_labelvalues
|
||||
)
|
||||
|
||||
def take_events(self) -> Iterable[KVCacheEvent]:
|
||||
if self.scheduler_manager is not None:
|
||||
return self.scheduler_manager.take_events()
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user