Merge remote-tracking branch 'upstream/main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-09-29 12:54:18 +07:00
264 changed files with 13095 additions and 1541 deletions
+23 -2
View File
@@ -1,6 +1,6 @@
---
name: kernel-microbenchmark
description: Build, debug, and interpret vLLM GPU kernel microbenchmarks for CUDA, Triton, and CuteDSL, including CUPTI timing, correctness checks, generated-code inspection, multi-GPU measurements, and SOL sanity checks.
description: Build, debug, and interpret vLLM GPU kernel microbenchmarks for CUDA, ROCm/HIP, Triton, and CuteDSL, including CUPTI timing, correctness checks, generated-code inspection, multi-GPU measurements, and SOL sanity checks.
---
# Kernel Microbenchmark
@@ -15,7 +15,7 @@ description: Build, debug, and interpret vLLM GPU kernel microbenchmarks for CUD
4. Compare against a baseline and report enough metadata to reproduce the
result: GPU, dtype, shape, command, branch/commit, and relevant env vars.
5. Treat explanations as hypotheses until backed by an artifact: ablation,
generated PTX/SASS, profiler output, or controlled benchmark.
generated PTX/SASS or AMD ISA, profiler output, or controlled benchmark.
6. If the result changes the conclusion, preserve the compact lesson in a note,
comment, benchmark table, or final summary. For experiments, a short
`Question / Change / Correctness / Result / Observation / Next` note is
@@ -25,6 +25,12 @@ description: Build, debug, and interpret vLLM GPU kernel microbenchmarks for CUD
- Use FlashInfer CUPTI timing by default, with CUDA graph and cold L2 cache:
`from flashinfer.testing import bench_gpu_time_with_cupti`.
- ROCm: CUPTI and FlashInfer do not exist. Use HIP graph replay over rotating
buffers whose total footprint exceeds the last-level cache (e.g. 256 MB on
MI355X), so every call reads cold from HBM, timed with HIP events — see
[benchmarks/graph_replay_benchmark.py](benchmarks/graph_replay_benchmark.py)
(platform-agnostic). Prefer this over `do_bench`, whose per-rep
launch/event overhead inflates small/decode-size kernels.
- For compute-heavy kernels, report TFLOPS with the FLOP formula in the
benchmark. For memory-heavy kernels, report estimated bytes moved and GB/s.
For mixed kernels, report the most honest metric available and call out the
@@ -38,6 +44,11 @@ description: Build, debug, and interpret vLLM GPU kernel microbenchmarks for CUD
formulas, skipped work, sparsity, caching, and whether the baseline is doing
the same operation.
- Force compilation/autotuning before measuring compiled kernels.
- Tune every arm of a comparison before trusting it. A variant's default or
inherited config can be far from its optimum, so conclusions drawn from
untuned variants — especially negative ones — are unreliable. A coarse
sweep of the cheap knobs at a couple of representative shapes is enough;
report the winning config with the numbers.
- Seed inputs when correctness comparisons matter.
- Keep metadata setup, plan construction, allocation, random input generation,
and logging outside the timed region unless that overhead is the experiment.
@@ -98,11 +109,21 @@ throughput conventions.
a two-node TP run as equivalent to a local NVLink-domain run without checking.
- Stabilize GPU clocks or run enough untimed work to reach a steady state.
Alternate candidate order so clock, thermal, and rank-skew effects are shared.
- ROCm: capturing ProcessGroupNCCL collectives in HIP graphs trips the NCCL
watchdog (HIP forbids querying an event recorded in a capturing stream;
the watchdog aborts the process). `TORCH_NCCL_BLOCKING_WAIT=1` before
`import torch` disables the watchdog — benchmark-only knob. Ask for backend
`"nccl"`; torch routes it to RCCL automatically (there is no `"rccl"`
backend string).
## Included Examples
- Use [benchmarks/cupti_microbenchmark.py](benchmarks/cupti_microbenchmark.py)
as a minimal single-GPU FlashInfer CUPTI timing pattern.
- Use
[benchmarks/graph_replay_benchmark.py](benchmarks/graph_replay_benchmark.py)
as a graph-replay timing pattern over rotating buffers sized past the
last-level cache — the ROCm/HIP default, also usable on CUDA.
- Use
[benchmarks/multi_gpu_gemm_rs.py](benchmarks/multi_gpu_gemm_rs.py) as a
minimal distributed CUDA-graph timing pattern.
@@ -0,0 +1,107 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Graph-replay microbenchmark with rotating weight buffers (CUDA and ROCm).
Platform-agnostic: HIP implements the torch.cuda.* API, including CUDAGraph.
Pattern: capture one graph (HIP graph on ROCm) containing R*INNER calls
cycling over R rotating weight buffers, then time replays with CUDA/HIP
events. The rotating buffers keep the captured footprint above the GPU's
last-level cache, so every call reads weights cold from HBM. Prefer this over
do_bench on ROCm: do_bench's per-rep launch/event overhead inflates
small/decode-size kernels, and its default flush buffer is only as large as
the LLC — on a 256 MB LLC that evicts marginally and leaves the cache full of
dirty flush-buffer lines. The same pattern extends to multi-GPU benchmarks
(capture per rank, collectives in identical order every replay); see the
Multi-GPU section of SKILL.md.
"""
import statistics
import pandas as pd
import torch
LLC_TARGET_BYTES = 400 * 1024**2 # above 256 MB MI355X LLC and B200 L2 (126 MB)
INNER = 2 # repeat slots within one graph to amortize per-replay overhead
REPLAYS = 10 # replays per round: makes each measurement large vs overhead
ROUNDS = 5 # median across rounds: replay time is not always stable
WARMUP = 5 # warmup replays to reach steady clocks before timing
MATMUL_CASES = [
("compute-bound", 4096, 4096, 4096),
("decode-GEMM", 16, 16384, 8192),
]
def timeit_graph(calls, inner=INNER, replays=REPLAYS, rounds=ROUNDS):
"""Time a list of per-slot callables as one graph (zero launch overhead);
weights rotate across slots so they stay HBM-cold. Returns the median over
rounds of per-round means, ms per call."""
for c in calls:
c()
torch.accelerator.synchronize()
g = torch.cuda.CUDAGraph()
with torch.cuda.graph(g): # capture runs on a side stream
for i in range(len(calls) * inner):
calls[i % len(calls)]()
torch.accelerator.synchronize()
events = [(torch.cuda.Event(True), torch.cuda.Event(True)) for _ in range(rounds)]
for _ in range(WARMUP):
g.replay()
# No sync between warmup and the first round: queued replays keep the GPU
# busy, so each round's launch latency stays outside its event window.
for s, e in events:
s.record()
for _ in range(replays):
g.replay()
e.record()
torch.accelerator.synchronize()
samples = [s.elapsed_time(e) / (replays * len(calls) * inner) for s, e in events]
return statistics.median(samples) # ms per call
def main() -> None:
if not torch.accelerator.is_available():
raise RuntimeError("No GPU visible (torch.accelerator covers HIP on ROCm).")
torch.set_default_device("cuda")
torch.manual_seed(0)
rows = []
for name, m, n, k in MATMUL_CASES:
a = torch.randn(m, k, dtype=torch.bfloat16) # shared: producer-warm
out = torch.empty(m, n, dtype=torch.bfloat16)
b_bytes = k * n * 2
r = max(2, -(-LLC_TARGET_BYTES // b_bytes)) # ceil
bs = [torch.randn(k, n, dtype=torch.bfloat16) for _ in range(r)]
def make_call(i, a=a, bs=bs, out=out):
def call():
torch.mm(a, bs[i], out=out)
return call
# correctness before timing, per slot (the graph would freeze a wrong
# result into every replay)
for i in range(r):
make_call(i)()
torch.accelerator.synchronize()
torch.testing.assert_close(out, torch.mm(a, bs[i]), atol=1e-1, rtol=1e-1)
us = timeit_graph([make_call(i) for i in range(r)]) * 1e3
rows.append(
{
"case": name,
"shape": f"{m}x{n}x{k}",
"slots": r,
"us": us,
"tflops": 2 * m * n * k / (us * 1e6),
"gbps": 2 * (m * k + k * n + m * n) / (us * 1e3),
}
)
df = pd.DataFrame(rows)
print(df.to_string(index=False, float_format=lambda x: f"{x:.3f}"))
if __name__ == "__main__":
main()
+23
View File
@@ -25,3 +25,26 @@ steps:
'cd tests &&
pytest -v -s ir &&
pytest -v -s kernels/ir'
- label: UVA Kernel Tests
key: uva-kernel-tests
timeout_in_minutes: 20
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 16+
no_plugin: true
working_dir: "."
env:
REGISTRY: "public.ecr.aws/q9t5s3a7"
REPO: "vllm-ci-test-repo"
VLLM_TEST_DEVICE: "xpu"
source_file_dependencies:
- tests/kernels/core/test_uva.py
- vllm/utils/torch_utils.py
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'cd tests &&
pytest -v -s kernels/core/test_uva.py'
+1 -1
View File
@@ -153,7 +153,7 @@ steps:
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'cd tests &&
pytest -v -s quantization/test_auto_round.py &&
pytest -v -s quantization/test_online.py'
pytest -v -s quantization/test_online.py --deselect=tests/quantization/test_online.py::test_online_quantization_loads_real_weights'
- label: "XPU compressed tensors FP8 test"
key: xpu-compressed-tensors-fp8-test
depends_on:
+83 -17
View File
@@ -79,12 +79,22 @@ if [[ -z "${OIDC_TOKEN}" ]]; then
fi
BUILD_JSON="$(mktemp)"
trap 'rm -f "${BUILD_JSON}"' EXIT
BUILD_JSON_STAGING="$(mktemp)"
trap 'rm -f "${BUILD_JSON}" "${BUILD_JSON_STAGING}"' EXIT
BUILD_URL="https://api.buildkite.com/v2/organizations/${BUILDKITE_ORGANIZATION_SLUG}/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}"
# Staged so a failed request cannot clobber the last good snapshot: curl -o
# writes the error body too, and the deadline path reports from whatever
# snapshot it has.
fetch_build() {
curl -sS -w '%{http_code}' -o "${BUILD_JSON}" \
-H "Authorization: Bearer ${BK_TOKEN}" "${BUILD_URL}"
local code
code="$(curl -sS -w '%{http_code}' -o "${BUILD_JSON_STAGING}" \
-H "Authorization: Bearer ${BK_TOKEN}" "${BUILD_URL}")"
if [[ "${code}" == "200" ]]; then
mv -f "${BUILD_JSON_STAGING}" "${BUILD_JSON}"
BUILD_JSON_STAGING="$(mktemp)"
fi
echo "${code}"
}
# Count jobs that have not reached a terminal state, ignoring this job: the
@@ -113,22 +123,78 @@ PY
# finished and none of the eventual 152 hard failures visible, and HUD recorded
# that nightly as green.
#
# Polling rather than a full `depends_on` barrier: Buildkite groups carry no
# key, so a barrier means naming all ~250 step keys, which silently rots when a
# step is added and fails the pipeline upload when one is removed or excluded
# from this lane (AMD, retired A100). The pipeline does gate this step on the
# long pole so it is scheduled near the end of the build -- that keeps the agent
# from idling for hours, but it is only a hint, and this loop is what actually
# guarantees the snapshot is complete.
POLL_INTERVAL_S="${CRCR_POLL_INTERVAL_S:-60}"
# Under the step's own timeout, so a stuck build still gets a partial report
# rather than the job being killed with nothing sent.
WAIT_DEADLINE=$(( $(date +%s) + ${CRCR_MAX_WAIT_S:-46800} ))
# Polling rather than a `depends_on` barrier: Buildkite groups carry no key, so
# a barrier means naming all ~250 step keys, which silently rots when a step is
# added and fails the pipeline upload when one is removed or excluded from this
# lane (AMD, retired A100). Gating on a single step was tried and is worse: a
# dependency that fails or is cancelled takes the report with it despite
# allow_dependency_failure (builds 91191 and 91461 both lost their report that
# way), so this step depends only on the image it runs in.
# 15 min: the wait now spans the whole build, so a 60s poll made ~420 requests
# against an API shared with the rest of CI to learn something that changes on
# the order of minutes. The cost is up to one interval of extra latency after
# the last job lands.
POLL_INTERVAL_S="${CRCR_POLL_INTERVAL_S:-900}"
# The deadline is measured from when the *build* started, not from when this
# script did, so "report no later than N hours into the build" holds however
# late this step is scheduled. Anchoring it to script start made the bound
# meaningless whenever the step was itself delayed.
#
# Resolved after the first fetch, from the build's own created_at.
MAX_BUILD_AGE_S="${CRCR_MAX_BUILD_AGE_S:-25200}"
WAIT_DEADLINE=""
# Used until created_at is known. Without it, an API outage on the very first
# request leaves the deadline unresolved and the loop spins to the step timeout.
FALLBACK_DEADLINE=$(( $(date +%s) + MAX_BUILD_AGE_S ))
build_deadline() {
python3 - "${BUILD_JSON}" "${MAX_BUILD_AGE_S}" <<'PY'
import datetime, json, sys
build = json.load(open(sys.argv[1]))
created = build.get("created_at")
if not created:
raise SystemExit(1)
started = datetime.datetime.fromisoformat(created.replace("Z", "+00:00"))
print(int(started.timestamp()) + int(sys.argv[2]))
PY
}
while :; do
http_code="$(fetch_build)"
if [[ "${http_code}" != "200" ]]; then
echo "buildkite API returned ${http_code} -- skipping report"
exit 0
# A transient 429/5xx must not cost the whole report: treating one
# failure as fatal would drop a nightly for a blip. Keep polling on the
# last good snapshot; only give up if the deadline passes having never
# fetched one.
echo "buildkite API returned ${http_code}"
if [[ ! -s "${BUILD_JSON}" ]]; then
if (( $(date +%s) >= ${WAIT_DEADLINE:-$FALLBACK_DEADLINE} )); then
echo "no build data was ever fetched -- nothing to report"
exit 0
fi
sleep "${POLL_INTERVAL_S}"
continue
fi
if (( $(date +%s) >= ${WAIT_DEADLINE:-$FALLBACK_DEADLINE} )); then
echo "deadline reached while the API is unavailable --" \
"reporting from the last good snapshot"
break
fi
sleep "${POLL_INTERVAL_S}"
continue
fi
if [[ -z "${WAIT_DEADLINE}" ]]; then
WAIT_DEADLINE="$(build_deadline)" || WAIT_DEADLINE=""
if [[ -z "${WAIT_DEADLINE}" ]]; then
# No created_at to anchor to; fall back to script start so the loop
# still has a bound rather than running until the step times out.
WAIT_DEADLINE=$(( $(date +%s) + MAX_BUILD_AGE_S ))
echo "build created_at unavailable -- deadline measured from now"
fi
echo "reporting deadline: $(date -u -d "@${WAIT_DEADLINE}" '+%Y-%m-%dT%H:%M:%SZ' 2>/dev/null \
|| date -u -r "${WAIT_DEADLINE}" '+%Y-%m-%dT%H:%M:%SZ')" \
"(build age limit ${MAX_BUILD_AGE_S}s)"
fi
remaining="$(outstanding_jobs)" || remaining=""
if [[ -z "${remaining}" ]]; then
@@ -140,7 +206,7 @@ while :; do
break
fi
if (( $(date +%s) >= WAIT_DEADLINE )); then
echo "still ${remaining} job(s) running at the wait deadline --" \
echo "still ${remaining} job(s) running at the build-age deadline --" \
"reporting a partial view"
break
fi
+426 -17
View File
@@ -200,9 +200,9 @@ steps:
- pytest -v -s test_sampling_params.py
- pytest -v -s -m 'cpu_test' multimodal
- pytest -v -s renderers
- pytest -v -s tokenizers_
- pytest -v -s reasoning
- pytest -v -s tool_parsers
- pytest -v -s tokenizers_
- pytest -v -s parser
- pytest -v -s transformers_utils
- pytest -v -s config --ignore=config/test_mp_reducer.py
@@ -262,6 +262,24 @@ steps:
commands:
- pytest -v -s tools/test_docker_build_metadata_args.py
- label: ":amd: (MI250) Torch Stable ABI Audit" # TBD
timeout_in_minutes: 35
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
dind: false
agent_pool: mi300_1
no_gpu: true
optional: true
working_dir: "/vllm-workspace/"
source_file_dependencies:
- .buildkite/check-torch-abi.py
- csrc/
- cmake/
- setup.py
- vllm/platforms/rocm.py
commands:
- uv pip install --system torch-abi-audit==0.0.1
- python3 /vllm-workspace/.buildkite/check-torch-abi.py
#########################################################################################################################################
# #
# MI250 (gfx90a) tests #
@@ -415,9 +433,9 @@ steps:
- vllm/v1/cudagraph_dispatcher.py
- vllm/config/compilation.py
- vllm/compilation
- vllm/platforms/rocm.py
- vllm/v1/worker/encoder_cudagraph.py
- vllm/v1/worker/encoder_cudagraph_defs.py
- vllm/platforms/rocm.py
commands:
- pytest -v -s v1/cudagraph/test_cudagraph_dispatch.py
- pytest -v -s v1/cudagraph/test_cudagraph_mode.py
@@ -614,6 +632,7 @@ steps:
- vllm/v1/
commands:
- "find compile/ -maxdepth 1 -name 'test_*.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'"
- pytest -s -v compile/dynamic_shapes/
- label: ":amd: (MI300) Fusion E2E Config Sweep" # TBD
timeout_in_minutes: 40
@@ -659,6 +678,8 @@ steps:
- "pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k 'inductor_partition and not +rms_norm and not +quant_fp8'"
# Different from CUDA, Qwen requires +rms_norm and +quant_fp8 as rms+quant fusion is only supported on AITER
- "pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k 'inductor_partition and +rms_norm and +quant_fp8 and qwen3'"
# DeepSeek-Coder uses the portable static-FP8 MLA fusion path on ROCm.
- "pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k 'inductor_partition and not +rms_norm and +quant_fp8 and DeepSeek-Coder and TRITON_MLA and not True'"
- label: ":amd: (MI300) PyTorch Compilation Passes" # TBD
timeout_in_minutes: 80
@@ -767,7 +788,7 @@ steps:
- vllm/platforms/rocm.py
commands:
- export VLLM_TEST_CLEAN_GPU_MEMORY=1
- VLLM_TEST_CLEAN_GPU_MEMORY=1 pytest -v -s tests/compile/passes/distributed/test_async_tp.py
- pytest -s -v tests/compile/passes/distributed
- pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py::test_tp2_ar_rms_fusions -k "inductor_partition"
- label: ":amd: (MI300) Distributed Compile (Dynamo Partition)" # TBD
@@ -789,6 +810,25 @@ steps:
- VLLM_TEST_CLEAN_GPU_MEMORY=1 pytest -v -s tests/compile/passes/distributed/test_async_tp.py
- pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py::test_tp2_ar_rms_fusions -k "dynamo_partition"
# - label: ":amd: (MI300) Sequence Parallel Correctness" # TBD
# timeout_in_minutes: 180
# mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
# dind: false
# agent_pool: mi300_2
# num_gpus: 2
# optional: true
# working_dir: "/vllm-workspace/"
# source_file_dependencies:
# - vllm/model_executor/layers/
# - vllm/compilation/
# - vllm/v1/worker/
# - vllm/v1/cudagraph_dispatcher.py
# - tests/compile/correctness_e2e/test_sequence_parallel.py
# - vllm/platforms/rocm.py
# commands:
# - export VLLM_TEST_CLEAN_GPU_MEMORY=1
# - pytest -v -s tests/compile/correctness_e2e/test_sequence_parallel.py
- label: ":amd: (MI300) Distributed Compile + RPC" # TBD
timeout_in_minutes: 65
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
@@ -999,7 +1039,7 @@ steps:
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 ../examples/rl/rlhf_http_ipc.py
- label: ":amd: (MI300) Elastic EP Scaling" # TBD
timeout_in_minutes: 50
timeout_in_minutes: 55
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
dind: false
agent_pool: mi300_4
@@ -1014,6 +1054,7 @@ steps:
- tests/distributed/
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- pytest -v -s distributed/test_elastic_ep.py
- label: ":amd: (MI300) RayExecutorV2" # TBD
@@ -1107,7 +1148,7 @@ steps:
#-------------------------------------------------------- mi300 · entrypoints --------------------------------------------------------#
- label: ":amd: (MI300) Entrypoints Unit" # TBD
timeout_in_minutes: 50
timeout_in_minutes: 60
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
dind: false
agent_pool: mi300_1
@@ -1247,6 +1288,59 @@ steps:
commands:
- pytest -s -v evals/mrcr/test_mrcr_correctness.py --config-list-file=evals/mrcr/configs/models-small.txt
- label: ":amd: (MI300) LM Eval KV-Offload" # TBD
timeout_in_minutes: 55
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
dind: false
agent_pool: mi300_1
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/offloading/
- vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py
- vllm/v1/kv_offload/
- vllm/v1/simple_kv_offload/
- tests/evals/gsm8k/test_gsm8k_offloading.py
- vllm/platforms/rocm.py
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_offloading.py -k "nemotron-h-8b or gemma-4-e4b-it"
- label: ":amd: (MI300) LM Eval KV-Offload Medium" # TBD
timeout_in_minutes: 70
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
dind: false
agent_pool: mi300_2
num_gpus: 2
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/offloading/
- vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py
- vllm/v1/kv_offload/
- vllm/v1/simple_kv_offload/
- tests/evals/gsm8k/test_gsm8k_offloading.py
- vllm/platforms/rocm.py
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_offloading.py -k "qwen3.5-35b or deepseek-v2-lite"
- label: ":amd: (MI300) LM Eval KV-Offload Large" # TBD
timeout_in_minutes: 65
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
dind: false
agent_pool: mi300_4
num_gpus: 4
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/offloading/
- vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py
- vllm/v1/kv_offload/
- vllm/v1/simple_kv_offload/
- tests/evals/gsm8k/test_gsm8k_offloading.py
- vllm/platforms/rocm.py
commands:
- VLLM_ROCM_USE_AITER=1 pytest -s -v evals/gsm8k/test_gsm8k_offloading.py -k "deepseek-v4-flash"
- label: ":amd: (MI300) LM Eval Small Models Harness" # TBD
timeout_in_minutes: 35
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
@@ -1398,6 +1492,9 @@ steps:
commands:
- bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_dp4_async_eplb.sh 0.8 200 8050
# TODO(akaratza): Retire this standalone ROCm coverage after
# tests/distributed/test_eplb_spec_decode.py is enabled and validated on ROCm.
# The upstream parity job currently skips that entire module on ROCm.
- label: ":amd: (MI300) Qwen3-Next-80B-A3B-Instruct MTP Async EPLB Accuracy" # TBD
timeout_in_minutes: 45
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
@@ -1446,7 +1543,7 @@ steps:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx.txt
- label: ":amd: (MI300) LM Eval Large Models ROCm Harness" # TBD
timeout_in_minutes: 55
timeout_in_minutes: 40
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
dind: false
agent_pool: mi300_8
@@ -1465,7 +1562,7 @@ steps:
- vllm/platforms/rocm.py
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm.txt --tp-size=8
- pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm.txt --tp-size=8 --timeout=5400
- label: ":amd: (MI300) LM Eval Large Models FP8" # TBD
timeout_in_minutes: 40
@@ -1491,7 +1588,7 @@ steps:
#---------------------------------------------------------- mi300 · kernels ----------------------------------------------------------#
- label: ":amd: (MI300) Core Operation Kernels Shard %N" # TBD
timeout_in_minutes: 40
timeout_in_minutes: 45
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
dind: false
agent_pool: mi300_1
@@ -1629,8 +1726,8 @@ steps:
commands:
- pytest -v -s kernels/core/test_minimax_reduce_rms.py
- label: ":amd: (MI300) KDA Kernels" # TBD
timeout_in_minutes: 30
- label: ":amd: (MI300) Kimi K3" # TBD
timeout_in_minutes: 65
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
dind: false
agent_pool: mi300_1
@@ -1643,11 +1740,20 @@ steps:
- vllm/models/kimi_k3/nvidia/kda.py
- vllm/models/kimi_k3/nvidia/kda_metadata.py
- vllm/models/kimi_k3/nvidia/ops/third_party/kda/
- vllm/models/kimi_k3/amd/
- CMakeLists.txt
- csrc/libtorch_stable/kimi_k3/fused_kda_decode_kernel_rocm.cu
- csrc/libtorch_stable/ops.h
- csrc/libtorch_stable/torch_bindings.cpp
- tests/models/kimi_k3/test_amd_attn_res.py
- tests/models/kimi_k3/test_amd_kda_decode.py
- tests/models/kimi_k3/test_kda.py
- tests/models/kimi_k3/test_kda_metadata.py
- vllm/_custom_ops.py
- vllm/platforms/rocm.py
commands:
- pytest -v -s models/kimi_k3/test_kda.py models/kimi_k3/test_kda_metadata.py
# Run the ROCm AttnRes/KDA decode counterparts plus existing KDA coverage.
- pytest -v -s models/kimi_k3/test_amd_attn_res.py models/kimi_k3/test_amd_kda_decode.py models/kimi_k3/test_kda.py models/kimi_k3/test_kda_metadata.py
- label: ":amd: (MI355) Kimi K3" # TBD
timeout_in_minutes: 60
@@ -1781,6 +1887,135 @@ steps:
- pytest -v -s model_executor -m '(not slow_test)' --timeout=900 --timeout-method=thread
- pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py --timeout=900 --timeout-method=thread
#---------------------------------------------------- mi300 · model_runner_v2 -------------------------------------------------------#
- label: ":amd: (MI300) Model Runner V2 Core" # TBD
timeout_in_minutes: 55
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
dind: false
agent_pool: mi300_1
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/v1/worker/gpu/
- vllm/v1/worker/gpu_worker.py
- vllm/v1/core/sched/
- vllm/v1/attention/
- tests/v1/engine/test_llm_engine.py
- tests/v1/e2e/
- tests/entrypoints/llm/test_struct_output_generate.py
- vllm/platforms/rocm.py
commands:
- set -x
- export VLLM_USE_V2_MODEL_RUNNER=1
- pytest -v -s v1/engine/test_llm_engine.py -k "not test_engine_metrics"
- pytest -v -s v1/e2e/general/test_async_scheduling.py -k "not ngram"
- pytest -v -s v1/e2e/general/test_context_length.py
- pytest -v -s v1/e2e/general/test_min_tokens.py
- pytest -v -s entrypoints/llm/test_struct_output_generate.py -k "xgrammar and not speculative_config6 and not speculative_config7 and not speculative_config8 and not speculative_config0"
- label: ":amd: (MI300) Model Runner V2 Examples" # TBD
timeout_in_minutes: 40
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
dind: false
agent_pool: mi300_1
optional: true
working_dir: "/vllm-workspace/examples"
source_file_dependencies:
- vllm/v1/worker/gpu/
- vllm/v1/core/sched/
- vllm/v1/worker/gpu_worker.py
- examples/basic/offline_inference/
- examples/generate/multimodal/
- examples/features/
- examples/pooling/embed/vision_embedding_offline.py
- examples/features/tensorize_vllm_model.py
- examples/deployment/llm_engine_example.py
- vllm/platforms/rocm.py
commands:
- set -x
- export VLLM_USE_V2_MODEL_RUNNER=1
- pip install tensorizer
- python3 basic/offline_inference/chat.py
- python3 basic/offline_inference/generate.py --model facebook/opt-125m
- python3 generate/multimodal/audio_language_offline.py --seed 0
- python3 generate/multimodal/vision_language_offline.py --seed 0
- python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0
- python3 generate/multimodal/encoder_decoder_multimodal_offline.py --model-type whisper --seed 0
- python3 pooling/embed/vision_embedding_offline.py --seed 0
- python3 features/automatic_prefix_caching/prefix_caching_offline.py
- python3 deployment/llm_engine_example.py
- python3 features/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 features/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
- python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048
- python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
- label: ":amd: (MI300) Model Runner V2 Distributed" # TBD
timeout_in_minutes: 45
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
dind: false
agent_pool: mi300_2
num_gpus: 2
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/v1/worker/gpu/
- vllm/v1/worker/gpu_worker.py
- tests/basic_correctness/test_basic_correctness.py
- tests/v1/distributed/test_async_llm_dp.py
- tests/v1/distributed/test_eagle_dp.py
- vllm/platforms/rocm.py
commands:
- set -x
- export VLLM_USE_V2_MODEL_RUNNER=1
- TARGET_TEST_SUITE=MI300 pytest -v -s basic_correctness/test_basic_correctness.py -m 'distributed(num_gpus=2)' -k "not ray and not True"
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py -k "not ray"
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
- label: ":amd: (MI300) Model Runner V2 Pipeline Parallelism" # TBD
timeout_in_minutes: 65
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
dind: false
agent_pool: mi300_4
num_gpus: 4
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/v1/worker/gpu/
- vllm/v1/worker/gpu_worker.py
- tests/distributed/test_pipeline_parallel.py
- tests/distributed/test_pp_cudagraph.py
- vllm/platforms/rocm.py
commands:
- set -x
- export VLLM_USE_V2_MODEL_RUNNER=1
- pytest -v -s distributed/test_pipeline_parallel.py -k "not ray"
- pytest -v -s distributed/test_pp_cudagraph.py -k "not ray"
- label: ":amd: (MI300) Model Runner V2 Spec Decode" # TBD
timeout_in_minutes: 90
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
dind: false
agent_pool: mi300_1
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/v1/worker/gpu/
- vllm/v1/worker/gpu_worker.py
- tests/v1/spec_decode/test_max_len.py
- tests/v1/spec_decode/test_rejection_sampler_utils.py
- tests/v1/spec_decode/test_synthetic_rejection_sampler_utils.py
- tests/v1/e2e/spec_decode/
- vllm/platforms/rocm.py
commands:
- set -x
- export VLLM_USE_V2_MODEL_RUNNER=1
- pytest -v -s v1/spec_decode/test_max_len.py -k "eagle or mtp"
- pytest -v -s v1/spec_decode/test_rejection_sampler_utils.py
- pytest -v -s v1/spec_decode/test_synthetic_rejection_sampler_utils.py
- pytest -v -s v1/e2e/spec_decode/eagle/
- pytest -v -s v1/e2e/spec_decode/speculators/
- pytest -v -s v1/e2e/spec_decode/mtp/
#------------------------------------------------------ mi300 · models / basic -------------------------------------------------------#
- label: ":amd: (MI300) Basic Models (Initialization)" # TBD
@@ -2258,6 +2493,23 @@ steps:
- VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --ignore quantization/test_rocm_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s rocm/test_moe_weight_replay.py; fi
- label: ":amd: (MI300) Quantized Fusions" # TBD
timeout_in_minutes: 45
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
dind: false
agent_pool: mi300_1
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- tests/fusion
- vllm/model_executor/layers/fusion
- vllm/model_executor/kernels/linear
- vllm/model_executor/layers/quantization/compressed_tensors
- vllm/model_executor/layers/quantization/modelopt.py
- vllm/platforms/rocm.py
commands:
- pytest -v -s fusion/
#--------------------------------------------------------- mi300 · samplers ----------------------------------------------------------#
- label: ":amd: (MI300) Samplers" # TBD
@@ -2411,8 +2663,8 @@ steps:
source_file_dependencies:
- vllm/v1/engine/
- tests/v1/engine/
- vllm/platforms/rocm.py
- tests/v1/test_tensor_ipc_queue.py
- vllm/platforms/rocm.py
commands:
- pytest -v -s v1/engine/test_preprocess_error_handling.py
- pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py
@@ -2625,6 +2877,23 @@ steps:
- export BASELINE_FILE='/tmp/mooncake-ec-e2e/baseline.json'
- bash tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh
- label: ":amd: (MI300) NixlConnector PD edge case" # TBD
timeout_in_minutes: 65
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
dind: false
agent_pool: mi300_2
num_gpus: 2
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- vllm/v1/core/sched/
- tests/v1/kv_connector/nixl_integration/
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- PREFILL_GPU_ID=0 DECODE_GPU_ID=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_edge_case_test.sh
- label: ":amd: (MI300) CrossLayer KV layout Distributed NixlConnector PD accuracy" # TBD
timeout_in_minutes: 60
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
@@ -2665,6 +2934,55 @@ steps:
- pytest -v -s v1/engine/test_engine_core_client.py::test_kv_cache_events_dp
- pytest -v -s distributed/test_utils.py
- label: ":amd: (MI300) Distributed NixlConnector PD accuracy" # TBD
timeout_in_minutes: 45
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
dind: false
agent_pool: mi300_4
num_gpus: 4
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- tests/v1/kv_connector/nixl_integration/
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
- label: ":amd: (MI300) Push NixlConnector PP prefill PD accuracy" # TBD
timeout_in_minutes: 55
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
dind: false
agent_pool: mi300_4
num_gpus: 4
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- tests/v1/kv_connector/nixl_integration/
- tests/v1/kv_connector/nixl_push_integration/
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_push_integration/config_sweep_accuracy_test.sh
- label: ":amd: (MI300) DP EP Distributed NixlConnector PD accuracy" # TBD
timeout_in_minutes: 30
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
dind: false
agent_pool: mi300_4
num_gpus: 4
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- tests/v1/kv_connector/nixl_integration/
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- DP_EP=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
- label: ":amd: (MI300) Hybrid SSM NixlConnector PD accuracy" # TBD
timeout_in_minutes: 80
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
@@ -3647,7 +3965,11 @@ steps:
- vllm/model_executor/models/qwen3.py
- vllm/model_executor/models/qwen3_next.py
- vllm/model_executor/models/qwen3_next_mtp.py
- vllm/distributed/
- vllm/model_executor/layers/fused_moe/
- vllm/third_party/flash_linear_attention/ops/
- tests/evals/gsm8k/
- tests/utils.py
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
@@ -3779,14 +4101,44 @@ steps:
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-t3nc-rocm.txt
- label: ":amd: (MI355) LM Eval Small Models Distributed"
timeout_in_minutes: 55
- label: ":amd: (MI355) LM Eval Spec Decode (Fixed-length FP8)" # TBD
timeout_in_minutes: 145
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
dind: false
agent_pool: mi355_4
num_gpus: 4
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- tests/evals/gsm8k/
- tests/utils.py
- vllm/distributed/
- vllm/models/deepseek_v4/
- vllm/model_executor/models/qwen3_dspark.py
- vllm/model_executor/layers/fused_moe/
- vllm/v1/spec_decode/metrics.py
- vllm/v1/worker/gpu/spec_decode/dspark/
- vllm/v1/worker/gpu/spec_decode/rejection_sampler.py
- vllm/v1/worker/gpu/spec_decode/speculator.py
- vllm/model_executor/layers/quantization/
- vllm/v1/attention/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-spec-decode-rocm.txt
- label: ':amd: (MI355) LM Eval Small Models Distributed'
timeout_in_minutes: 145
mirror_hardwares:
- amdexperimental
- amdproduction
- amdgfx950nightly
- amdmi355
dind: false
agent_pool: mi355_2
num_gpus: 2
optional: true
working_dir: "/vllm-workspace/tests"
working_dir: /vllm-workspace/tests
source_file_dependencies:
- csrc/
- vllm/model_executor/layers/quantization
@@ -3794,11 +4146,22 @@ steps:
- vllm/model_executor/model_loader/
- vllm/v1/attention/backends/
- vllm/v1/attention/selector.py
- vllm/v1/worker/
- vllm/v1/core/
- vllm/config/
- tests/evals/gsm8k/configs/models-mi3xx-fp8-and-mixed.txt
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- tests/evals/gsm8k/configs/models-mi3xx-fp8-and-mixed.txt
- tests/evals/gsm8k/
- tests/utils.py
- vllm/distributed/
- vllm/transformers_utils/configs/diffusion_gemma.py
- vllm/v1/attention/ops/
- tests/evals/gsm8k/configs/models-small-tp-mi355.txt
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx-fp8-and-mixed.txt
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small-tp.txt
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small-tp-mi355.txt
- label: ":amd: (MI355) MoE Refactor Integration Shard %N"
timeout_in_minutes: 105
@@ -3819,6 +4182,7 @@ steps:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor/config-rocm-shard-$$BUILDKITE_PARALLEL_JOB.txt
- label: ":amd: (MI355) Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy" # TBD
timeout_in_minutes: 25
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
@@ -4350,6 +4714,22 @@ steps:
commands:
- pytest -v -s models/test_initialization.py -k 'not test_can_initialize_small_subset' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
- label: ":amd: (MI355) Inkling" # TBD
timeout_in_minutes: 65
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
dind: false
agent_pool: mi355_1
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/models/inkling/
- vllm/cute_utils/
- cmake/external_projects/tml_fa4.cmake
- tests/models/inkling/
- vllm/platforms/rocm.py
commands:
- pytest -v -s models/inkling/rocm
#----------------------------------------------------- mi355 · models / language -----------------------------------------------------#
- label: ":amd: (MI355 DPX) Language Models (Standard)" # TBD
@@ -5104,7 +5484,8 @@ steps:
- vllm/platforms/rocm.py
optional: true
commands:
- pytest -v -s v1/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- pytest -v -s v1/attention --ignore=v1/attention/test_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- VLLM_ROCM_USE_AITER=1 pytest -v -s v1/attention/test_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- label: ":amd: (MI355 DPX) V1 Core (MI355 suite)"
timeout_in_minutes: 60
@@ -5500,6 +5881,34 @@ steps:
- pip install 'modelscope<1.38'
- pytest -v -s test_regression.py
- label: ':amd: (MI355) DeepSeek V4 MoE Integration'
timeout_in_minutes: 20
source_file_dependencies:
- tests/models/test_deepseek_v4_vl_rocm.py
- vllm/models/deepseek_v4/amd/
- vllm/models/deepseek_v4/common/
- vllm/model_executor/layers/fused_moe/
- vllm/model_executor/layers/activation.py
- vllm/model_executor/layers/linear.py
- vllm/config/
- vllm/forward_context.py
- vllm/v1/worker/workspace.py
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- vllm/envs.py
commands:
- VLLM_ROCM_USE_AITER_MOE=0 pytest -v -s models/test_deepseek_v4_vl_rocm.py
dind: false
working_dir: /vllm-workspace/tests
agent_pool: mi355_1
num_gpus: 1
mirror_hardwares:
- amdexperimental
- amdproduction
- amdgfx950nightly
- amdmi355
optional: true
- label: ':amd: (MI355) Quantized MoE'
timeout_in_minutes: 45
working_dir: /vllm-workspace/
+19
View File
@@ -43,3 +43,22 @@ steps:
commands:
- pytest -v -s v1/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
parallelism: 2
mirror:
amd:
label: ":amd: (MI355 DPX) V1 Attention Shard %N"
dind: false
device: mi355_dpx
timeout_in_minutes: 125
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/config/attention.py
- vllm/model_executor/layers/attention
- vllm/v1/attention
- tests/v1/attention
- vllm/_aiter_ops.py
- vllm/envs.py
- vllm/platforms/rocm.py
commands:
- pytest -v -s v1/attention --ignore=v1/attention/test_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- VLLM_ROCM_USE_AITER=1 pytest -v -s v1/attention/test_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
+17 -18
View File
@@ -1,31 +1,30 @@
group: CRCR Report
# The report waits for the rest of the build (see crcr-report.sh), so gate it on
# the long pole rather than letting it wait on an agent. A torch-nightly build
# is tail-heavy: of 347 jobs in build 90830, 344 finished inside 3.5h and three
# stragglers stretched it to 12.7h, with lm-eval-large-models-4xh100 last in
# both 90830 (+760 min) and 90640 (+799 min) by a margin of hours.
# Only image-build-cpu: this step must always run.
#
# Depending on it means Buildkite does not schedule this step until the build is
# nearly over, so the agent polls briefly instead of idling ~13h. It is a
# scheduling hint, not the correctness mechanism: the script still waits for
# every job, so if this stops being the slowest step the report is late to start
# but still correct.
# It was briefly gated on the build's long pole (lm-eval-large-models-4xh100)
# so the agent would not idle. That backfired on the first nightly to use it,
# build 91191: the long pole OOMed, hung, and was cancelled -- and a cancelled
# dependency propagates past `allow_dependency_failure`, so the step went
# `waiting_failed` and that nightly got no CRCR row at all. Nothing in this
# pipeline sets timeout_in_minutes (the pipeline default and org maximum are
# both unset), so a hung job has no terminal state to reach either: gating on
# any single job can suppress the report indefinitely.
#
# The bound now lives in the script, measured from the build's created_at, so
# the report lands within CRCR_MAX_BUILD_AGE_S of the build starting whatever
# else happens.
depends_on:
- image-build-cpu
- lm-eval-large-models-4xh100
steps:
- label: ":satellite: Report nightly results to CRCR"
key: crcr-report
if: build.env("TORCH_NIGHTLY") == "1"
soft_fail: true
allow_dependency_failure: true
# depends_on defers the start to near the end of the build, so in the normal
# case the script polls for minutes. This only has to cover the case where
# the long pole finishes early and something else is still running, so it is
# sized against the whole build rather than the expected wait. The script's
# own deadline (CRCR_MAX_WAIT_S, 13h) trips first and still sends a partial
# report; this is the backstop that stops the job outliving the build.
timeout_in_minutes: 840
# The script polls from the start of the build now, so this covers the whole
# wait. It sits above the script's own build-age deadline (7h) so the script
# sends its partial report rather than being killed with nothing sent.
timeout_in_minutes: 480
device: cpu-small
commands:
- bash /vllm-workspace/.buildkite/scripts/crcr-report.sh
+32
View File
@@ -79,6 +79,22 @@ steps:
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- bash v1/kv_connector/nixl_push_integration/config_sweep_accuracy_test.sh
mirror:
amd:
label: ":amd: (MI300) Push NixlConnector PP prefill PD accuracy"
dind: false
device: mi300_4
timeout_in_minutes: 55
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- tests/v1/kv_connector/nixl_integration/
- tests/v1/kv_connector/nixl_push_integration/
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_push_integration/config_sweep_accuracy_test.sh
- label: ":nvidia: (L4) DP EP Distributed NixlConnector PD accuracy"
key: dp-ep-distributed-nixlconnector-pd-accuracy-tests-4-gpus
@@ -180,6 +196,22 @@ steps:
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- bash v1/kv_connector/nixl_integration/run_edge_case_test.sh
mirror:
amd:
label: ":amd: (MI300) NixlConnector PD edge case"
dind: false
device: mi300_2
timeout_in_minutes: 65
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- vllm/v1/core/sched/
- tests/v1/kv_connector/nixl_integration/
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- PREFILL_GPU_ID=0 DECODE_GPU_ID=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_edge_case_test.sh
- label: ":nvidia: (L4) Hybrid SSM NixlConnector PD prefix cache"
key: hybrid-ssm-nixlconnector-pd-prefix-cache-2-gpus
+30 -4
View File
@@ -47,7 +47,7 @@ steps:
label: ":amd: (MI300) Core Operation Kernels Shard %N"
dind: false
device: mi300_1
timeout_in_minutes: 40
timeout_in_minutes: 45
working_dir: "/vllm-workspace/tests"
depends_on:
- image-build-amd
@@ -143,6 +143,33 @@ steps:
commands:
- pytest -v -s kernels/test_fused_deepseek_v4_*.py
- pytest -v -s models/test_deepseek_v4_mega_moe.py
mirror:
amd:
label: ":amd: (MI355) DeepSeek V4 MoE Integration"
dind: false
device: mi355_1
num_devices: 1
timeout_in_minutes: 20
working_dir: /vllm-workspace/tests
depends_on:
- image-build-amd
source_file_dependencies:
- tests/models/test_deepseek_v4_vl_rocm.py
- vllm/models/deepseek_v4/amd/
- vllm/models/deepseek_v4/common/
- vllm/model_executor/layers/fused_moe/
- vllm/model_executor/layers/activation.py
- vllm/model_executor/layers/linear.py
- vllm/config/
- vllm/forward_context.py
- vllm/v1/worker/workspace.py
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- vllm/envs.py
commands:
# Common qnorm/RoPE/cache kernels already run in the H100 AMD mirror.
# Pin the tested BF16 Triton routing and shared-expert path before imports.
- VLLM_ROCM_USE_AITER_MOE=0 pytest -v -s models/test_deepseek_v4_vl_rocm.py
# Own job so mHC never shares a process with FlashInfer's import-time
# ptxas override.
@@ -173,8 +200,8 @@ steps:
# Catch-all for test files at the tests/kernels root. This job collects
# the whole root so new files are wired by default.
# Files with dedicated jobs elsewhere in this file are excluded via --ignore
# (test_kda, test_bf16x3_router_gemm_cutedsl and test_ll_bf16_gemm run in
# their own jobs / Kernels (B200)).
# (test_bf16x3_router_gemm_cutedsl and test_ll_bf16_gemm run in their own jobs /
# Kernels (B200)).
- label: ":nvidia: (B200) Miscellaneous Kernels"
key: kernels-root-misc-test-b200
timeout_in_minutes: 30
@@ -199,7 +226,6 @@ steps:
--ignore=kernels/test_fused_qk_norm_rope_gate.py
--ignore=kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py
--ignore=kernels/test_top_k_per_row.py
--ignore=kernels/test_kda.py
--ignore=kernels/test_bf16x3_router_gemm_cutedsl.py
--ignore=kernels/test_ll_bf16_gemm.py
--ignore=kernels/test_shuffle_rows.py
+67 -3
View File
@@ -186,7 +186,7 @@ steps:
label: ":amd: (MI355) LM Eval Small Models Distributed"
dind: false
device: mi355_2
timeout_in_minutes: 55
timeout_in_minutes: 145
depends_on:
- image-build-amd
source_file_dependencies:
@@ -196,11 +196,22 @@ steps:
- vllm/model_executor/model_loader/
- vllm/v1/attention/backends/
- vllm/v1/attention/selector.py
- vllm/v1/worker/
- vllm/v1/core/
- vllm/config/
- tests/evals/gsm8k/configs/models-mi3xx-fp8-and-mixed.txt
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- tests/evals/gsm8k/
- tests/utils.py
- vllm/distributed/
- vllm/transformers_utils/configs/diffusion_gemma.py
- vllm/v1/attention/ops/
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx-fp8-and-mixed.txt
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small-tp.txt
# Dense MLA PCP-only accuracy; DP1/DCP1.
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small-tp-mi355.txt
- label: ":nvidia: (B200) LM Eval HiSparse Nightly"
key: lm-eval-hisparse-nightly-4xb200
@@ -319,10 +330,10 @@ steps:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-spec-decode.txt
mirror:
amd:
label: ":amd: (MI355) LM Eval Spec Decode"
label: ":amd: (MI355) LM Eval Spec Decode (Fixed-length FP8)"
dind: false
device: mi355_4
timeout_in_minutes: 120
timeout_in_minutes: 145
depends_on:
- image-build-amd
source_file_dependencies:
@@ -355,6 +366,9 @@ steps:
- vllm/v1/worker/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- tests/evals/gsm8k/
- tests/utils.py
- vllm/distributed/
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-spec-decode-rocm.txt
@@ -951,6 +965,22 @@ steps:
- tests/evals/gsm8k/test_gsm8k_offloading.py
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_offloading.py -k "nemotron-h-8b or gemma-4-e4b-it"
mirror:
amd:
label: ":amd: (MI300) LM Eval KV-Offload"
dind: false
device: mi300_1
timeout_in_minutes: 55
depends_on:
- image-build-amd
working_dir: /vllm-workspace/tests
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/offloading/
- vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py
- vllm/v1/kv_offload/
- vllm/v1/simple_kv_offload/
- tests/evals/gsm8k/test_gsm8k_offloading.py
- vllm/platforms/rocm.py
- label: ":nvidia: (H100) LM Eval KV-Offload Medium"
key: kv-offload-medium
@@ -965,6 +995,22 @@ steps:
- tests/evals/gsm8k/test_gsm8k_offloading.py
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_offloading.py -k "qwen3.5-35b or deepseek-v2-lite"
mirror:
amd:
label: ":amd: (MI300) LM Eval KV-Offload Medium"
dind: false
device: mi300_2
timeout_in_minutes: 70
depends_on:
- image-build-amd
working_dir: /vllm-workspace/tests
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/offloading/
- vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py
- vllm/v1/kv_offload/
- vllm/v1/simple_kv_offload/
- tests/evals/gsm8k/test_gsm8k_offloading.py
- vllm/platforms/rocm.py
- label: ":nvidia: (H100) LM Eval KV-Offload Large"
key: kv-offload-large
@@ -979,6 +1025,24 @@ steps:
- tests/evals/gsm8k/test_gsm8k_offloading.py
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_offloading.py -k "deepseek-v4-flash"
mirror:
amd:
label: ":amd: (MI300) LM Eval KV-Offload Large"
dind: false
device: mi300_4
timeout_in_minutes: 65
depends_on:
- image-build-amd
working_dir: /vllm-workspace/tests
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/offloading/
- vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py
- vllm/v1/kv_offload/
- vllm/v1/simple_kv_offload/
- tests/evals/gsm8k/test_gsm8k_offloading.py
- vllm/platforms/rocm.py
commands:
- VLLM_ROCM_USE_AITER=1 pytest -s -v evals/gsm8k/test_gsm8k_offloading.py -k "deepseek-v4-flash"
- label: ":nvidia: (H200 MIG 35GB) MRCR Eval Small Models"
key: mrcr-eval-small-models
+46
View File
@@ -496,15 +496,22 @@ steps:
- vllm/assets/
- vllm/config/
- vllm/device_allocator/
- vllm/connections.py
- vllm/engine/arg_utils.py
- vllm/entrypoints/chat_utils.py
- vllm/entrypoints/mcp/
- vllm/entrypoints/openai/chat_completion/protocol.py
- vllm/entrypoints/openai/completion/protocol.py
- vllm/entrypoints/openai/responses/protocol.py
- vllm/entrypoints/generate/base/protocol.py
- vllm/env_override.py
- vllm/envs.py
- vllm/exceptions.py
- vllm/inputs/
- vllm/logging_utils/
- vllm/logprobs.py
- vllm/model_executor/layers/quantization/quark/
- vllm/model_executor/model_loader/tensorizer.py
- vllm/multimodal/
- vllm/outputs.py
- vllm/parser/
@@ -514,17 +521,38 @@ steps:
- vllm/reasoning/
- vllm/renderers/
- vllm/sampling_params.py
- vllm/scalar_type.py
- vllm/tokenizers/
- vllm/tool_parsers/
- vllm/transformers_utils/
- vllm/triton_utils/
- vllm/utils/
- vllm/v1/
- vllm/version.py
- cmake/utils.cmake
- tests/test_access_log_filter.py
- tests/test_attention_backend_registry.py
- tests/test_audio_media_size_precheck.py
- tests/test_cmake_utils.py
- tests/test_connections.py
- tests/test_embedded_commit.py
- tests/test_envs.py
- tests/test_force_first_config.py
- tests/test_fxgraphcache_pickle_patch.py
- tests/test_logprobs.py
- tests/test_outputs.py
- tests/test_pcp_dp.py
- tests/test_pooling_params.py
- tests/test_quantization_revision_pin.py
- tests/test_ray_env.py
- tests/test_ray_env_utils.py
- tests/test_request_input_bounds.py
- tests/test_sampling_params.py
- tests/test_scalartype.py
- tests/test_tensorizer_revision_pin.py
- tests/test_triton_utils.py
- tests/test_version.py
- tests/test_zen_cpu_platform_detection.py
- tests/multimodal
- tests/renderers
- tests/standalone_tests/lazy_imports.py
@@ -540,14 +568,32 @@ steps:
- tests/device_allocator
device: cpu-small
commands:
- pip install cmake # test_cmake_utils.py shells out to cmake; the CPU image has none
- python3 standalone_tests/lazy_imports.py
- pytest -v -s device_allocator
- pytest -v -s test_access_log_filter.py
- pytest -v -s test_attention_backend_registry.py
- pytest -v -s test_audio_media_size_precheck.py
- pytest -v -s test_cmake_utils.py
- pytest -v -s test_connections.py
- pytest -v -s test_embedded_commit.py
- pytest -v -s test_envs.py
- pytest -v -s test_force_first_config.py
- pytest -v -s test_fxgraphcache_pickle_patch.py
- pytest -v -s test_logprobs.py
- pytest -v -s test_outputs.py
- pytest -v -s test_pcp_dp.py
- pytest -v -s test_pooling_params.py
- pytest -v -s test_quantization_revision_pin.py
- pytest -v -s test_ray_env.py
- pytest -v -s test_ray_env_utils.py
- pytest -v -s test_request_input_bounds.py
- pytest -v -s test_sampling_params.py
- pytest -v -s test_scalartype.py
- pytest -v -s test_tensorizer_revision_pin.py
- pytest -v -s test_triton_utils.py
- pytest -v -s test_version.py
- pytest -v -s test_zen_cpu_platform_detection.py
- pytest -v -s tokenizers_
- pytest -v -s parser
- pytest -v -s transformers_utils
+17
View File
@@ -95,6 +95,23 @@ steps:
commands:
# FA4 kernel tests require SM100; the suite skips them elsewhere.
- pytest -v -s models/inkling
mirror:
amd:
label: ":amd: (MI355) Inkling"
dind: false
device: mi355_1
timeout_in_minutes: 65
working_dir: "/vllm-workspace/tests"
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/models/inkling/
- vllm/cute_utils/
- cmake/external_projects/tml_fa4.cmake
- tests/models/inkling/
- vllm/platforms/rocm.py
commands:
- pytest -v -s models/inkling/rocm
- label: ":nvidia: (B200) Kimi K3"
key: kimi-k3-unit-tests-b200
+22 -3
View File
@@ -5,8 +5,8 @@ steps:
- label: ":nvidia: (H200 MIG 35GB) PyTorch Compilation"
device: h200_35gb
key: pytorch-compilation-unit-tests
timeout_in_minutes: 60
source_file_dependencies:
timeout_in_minutes: 30
source_file_dependencies: &pytorch-compilation-deps
- vllm/__init__.py
- vllm/_aiter_ops.py
- vllm/_custom_ops.py
@@ -47,7 +47,26 @@ steps:
label: ":amd: (MI300) PyTorch Compilation"
dind: false
device: mi300_1
timeout_in_minutes: 90
timeout_in_minutes: 45
depends_on:
- image-build-amd
source_file_dependencies:
- csrc/
- vllm/platforms/rocm.py
- label: ":nvidia: (H200 MIG 35GB) PyTorch Compilation / Dynamic Shapes"
device: h200_35gb
key: pytorch-compilation-dynamic-shapes
timeout_in_minutes: 30
source_file_dependencies: *pytorch-compilation-deps
commands:
- pytest -s -v compile/dynamic_shapes/
mirror:
amd:
label: ":amd: (MI300) PyTorch Compilation / Dynamic Shapes"
dind: false
device: mi300_1
timeout_in_minutes: 45
depends_on:
- image-build-amd
source_file_dependencies:
+16
View File
@@ -58,6 +58,22 @@ steps:
- vllm/model_executor/layers/quantization/modelopt.py
commands:
- pytest -v -s fusion/
mirror:
amd:
label: ":amd: (MI300) Quantized Fusions"
dind: false
device: mi300_1
timeout_in_minutes: 45
working_dir: "/vllm-workspace/tests"
depends_on:
- image-build-amd
source_file_dependencies:
- tests/fusion
- vllm/model_executor/layers/fusion
- vllm/model_executor/kernels/linear
- vllm/model_executor/layers/quantization/compressed_tensors
- vllm/model_executor/layers/quantization/modelopt.py
- vllm/platforms/rocm.py
- label: ":nvidia: (B200) Quantized MoE"
key: quantized-moe-test-b200
+19
View File
@@ -12,3 +12,22 @@ steps:
- setup.py
commands:
- python3 /vllm-workspace/.buildkite/check-torch-abi.py
mirror:
amd:
label: ":amd: (MI250) Torch Stable ABI Audit"
dind: false
device: mi300_1
timeout_in_minutes: 35
depends_on:
- image-build-amd
no_gpu: true
working_dir: /vllm-workspace/
source_file_dependencies:
- .buildkite/check-torch-abi.py
- csrc/
- cmake/
- setup.py
- vllm/platforms/rocm.py
commands:
- uv pip install --system torch-abi-audit==0.0.1
- python3 /vllm-workspace/.buildkite/check-torch-abi.py
+6 -3
View File
@@ -880,12 +880,15 @@ RUN if [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ]; then \
# `vllm snapshot`. CRIU and its CUDA plugin are built from the same pinned
# source. The `cuda-checkpoint`-compatible helper is built from NVIDIA Dynamo's
# Apache-2.0 source against the public CUDA Driver checkpoint APIs instead of
# redistributing NVIDIA's proprietary cuda-checkpoint binary.
# redistributing NVIDIA's proprietary cuda-checkpoint binary. The helper uses
# checkpoint APIs (CUcheckpointGpuPair) that first appear in the CUDA 13
# headers, so CUDA 12.x images skip the snapshot runtime.
COPY tools/install_snapshot_runtime.sh /usr/local/bin/install_snapshot_runtime.sh
RUN if [ "${TARGETPLATFORM}" = "linux/amd64" ]; then \
RUN if [ "${TARGETPLATFORM}" = "linux/amd64" ] \
&& [ "${CUDA_VERSION%%.*}" -ge 13 ]; then \
MAX_JOBS="${max_jobs}" /usr/local/bin/install_snapshot_runtime.sh; \
else \
echo "Initialized snapshots are not installed for ${TARGETPLATFORM}"; \
echo "Initialized snapshots are not installed for ${TARGETPLATFORM} with CUDA ${CUDA_VERSION}"; \
fi
ENV CRIU_CUDA_PLUGIN_DIR=/usr/local/lib/criu
+3
View File
@@ -433,6 +433,8 @@ ADD ./benchmarks/ ./benchmarks/
ADD ./vllm/collect_env.py .
ADD ./docker/ ./docker/
ADD ./.buildkite/ ./.buildkite/
# cmake/ is needed by tests/test_cmake_utils.py, which includes cmake/utils.cmake.
ADD ./cmake/ ./cmake/
# tools/ carries repo-tooling code that a few tests exercise directly
# (e.g. tests/tools/test_check_test_tethering.py imports
# tools.pre_commit.check_test_tethering).
@@ -480,5 +482,6 @@ LABEL ai.vllm.build.python-version="${PYTHON_VERSION:-3.12}"
# stage above adds examples/ for testing only, so without this the published
# vllm-openai-cpu image would not ship examples/*.jinja.
COPY examples examples
COPY tools/recipes tools/recipes
ENTRYPOINT ["vllm", "serve"]
+5 -2
View File
@@ -926,12 +926,15 @@ RUN --mount=type=cache,target=/root/.cache/uv \
# Install vLLM using uv (inherited from base stage)
# Note: No -U flag to avoid upgrading PyTorch ROCm to CUDA version
# Keep OpenTelemetry within LMCache's cap.
RUN --mount=type=bind,from=export_vllm,src=/,target=/install \
--mount=type=cache,target=/root/.cache/uv \
cd /install \
&& uv pip install --system -c /etc/rocm-constraints.txt -r requirements/rocm.txt \
&& echo "opentelemetry-api<=1.40.0" > /tmp/otel-constraints.txt \
&& uv pip install --system -c /etc/rocm-constraints.txt -c /tmp/otel-constraints.txt -r requirements/rocm.txt \
&& pip uninstall -y vllm \
&& uv pip install --system -c /etc/rocm-constraints.txt *.whl
&& uv pip install --system -c /etc/rocm-constraints.txt -c /tmp/otel-constraints.txt *.whl \
&& rm /tmp/otel-constraints.txt
# Install NIXL ROCm wheel
RUN --mount=type=bind,from=build_nixl,src=/app/install,target=/nixl_install \
+5 -2
View File
@@ -948,12 +948,15 @@ RUN --mount=type=cache,target=/root/.cache/uv \
# Install vLLM using uv (inherited from base stage)
# Note: No -U flag to avoid upgrading PyTorch ROCm to CUDA version
# Keep OpenTelemetry within LMCache's cap.
RUN --mount=type=bind,from=export_vllm,src=/,target=/install \
--mount=type=cache,target=/root/.cache/uv \
cd /install \
&& uv pip install --system -r requirements/rocm.txt \
&& echo "opentelemetry-api<=1.40.0" > /tmp/otel-constraints.txt \
&& uv pip install --system -c /tmp/otel-constraints.txt -r requirements/rocm.txt \
&& pip uninstall -y vllm \
&& uv pip install --system *.whl
&& uv pip install --system -c /tmp/otel-constraints.txt *.whl \
&& rm /tmp/otel-constraints.txt
# Install NIXL ROCm wheel
RUN --mount=type=bind,from=build_nixl,src=/app/install,target=/nixl_install \
+42
View File
@@ -141,6 +141,41 @@ RUN --mount=type=cache,target=/root/.cache/uv \
sed -i '/#include "internal\/pycore_atomic.h"/i\#include "dynamic_annotations.h"' numba/_dispatcher.cpp; \
fi && python setup.py bdist_wheel
# Shared FFmpeg for torchcodec video decoding (UBI has no ffmpeg package).
FROM base AS ffmpeg-build
ARG FFMPEG_VERSION=n6.1.2
WORKDIR /tmp
RUN microdnf install -y pkgconf-pkg-config xz && \
curl -LO https://github.com/FFmpeg/FFmpeg/archive/refs/tags/${FFMPEG_VERSION}.tar.gz && \
tar -xzf ${FFMPEG_VERSION}.tar.gz && \
cd FFmpeg-${FFMPEG_VERSION} && \
./configure --prefix=/usr/local --libdir=/usr/local/lib \
--enable-shared --disable-static --enable-pic \
--disable-doc --disable-debug --disable-ffplay --disable-ffprobe \
--extra-cflags="-Wno-incompatible-pointer-types" && \
make -j"$(nproc)" && \
make install
# Torchcodec video decoder (no s390x wheels; CPU / FFmpeg only s390x does not support audio decoding).
FROM python-install AS torchcodec-builder
ARG TORCHCODEC_VERSION=v0.14.0
WORKDIR /tmp
ENV LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH"
ENV PKG_CONFIG_PATH="/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH"
RUN --mount=type=cache,target=/root/.cache/uv \
--mount=type=bind,from=ffmpeg-build,source=/usr/local,target=/usr/local \
git clone --depth 1 --branch ${TORCHCODEC_VERSION} \
https://github.com/pytorch/torchcodec.git && \
cd torchcodec && \
uv pip install torch==2.13.0 --index-url https://download.pytorch.org/whl/cpu && \
uv pip install pybind11 setuptools wheel && \
pybind11_DIR="$(python -c 'import pybind11; print(pybind11.get_cmake_dir())')" && \
export pybind11_DIR CMAKE_PREFIX_PATH="${pybind11_DIR}:${CMAKE_PREFIX_PATH}" && \
export I_CONFIRM_THIS_IS_NOT_A_LICENSE_VIOLATION=1 && \
export TORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR=1 && \
export TORCHCODEC_CMAKE_BUILD_DIR="${PWD}/build" && \
python setup.py bdist_wheel
# Build OpenCV from source for s390x
FROM python-install AS opencv-builder
WORKDIR /tmp
@@ -179,6 +214,10 @@ RUN --mount=type=bind,from=gperftools-build,src=/tmp/gperftools-2.16,target=/gpe
make -C /gperftools install-libLTLIBRARIES && \
/sbin/ldconfig
RUN --mount=type=bind,from=ffmpeg-build,source=/usr/local,target=/ffmpeg-prefix \
cp -a /ffmpeg-prefix/lib/. /usr/local/lib/ && \
/sbin/ldconfig
ENV LD_PRELOAD=/usr/local/lib/libtcmalloc_minimal.so.4
# Install dependencies, including PyTorch and Apache Arrow
@@ -190,17 +229,20 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--mount=type=bind,from=numba-builder,source=/tmp/llvmlite/dist,target=/tmp/llvmlite-wheels/ \
--mount=type=bind,from=numba-builder,source=/tmp/numba/dist,target=/tmp/numba-wheels/ \
--mount=type=bind,from=opencv-builder,source=/tmp/opencv-python/dist,target=/tmp/opencv-wheels/ \
--mount=type=bind,from=torchcodec-builder,source=/tmp/torchcodec/dist,target=/tmp/torchcodec-wheels/ \
VISION_WHL_FILE=$(ls /tmp/vision-wheels/*.whl) && \
HF_XET_WHL_FILE=$(ls /tmp/hf-xet-wheels/*.whl) && \
LLVM_WHL_FILE=$(ls /tmp/llvmlite-wheels/*.whl) && \
NUMBA_WHL_FILE=$(ls /tmp/numba-wheels/*.whl) && \
OPENCV_WHL_FILE=$(ls /tmp/opencv-wheels/*.whl) && \
TORCHCODEC_WHL_FILE=$(ls /tmp/torchcodec-wheels/*.whl) && \
uv pip install -v \
$VISION_WHL_FILE \
$HF_XET_WHL_FILE \
$LLVM_WHL_FILE \
$NUMBA_WHL_FILE \
$OPENCV_WHL_FILE \
$TORCHCODEC_WHL_FILE \
--torch-backend cpu \
--index-strategy unsafe-best-match \
-r requirements/build/cpu.txt \
+11 -4
View File
@@ -78,10 +78,17 @@ GPU-side fusion is controlled by `multimodal_config.mm_device_do_normalize`:
- The flag is **enabled by default** for all models that support it.
- Currently, it’s on by default for these architectures:
| name | Architecture | Example HF Models |
|--------------|--------------------------------------|-------------------------------------|
| `qwen2-vl` | `Qwen2VLForConditionalGeneration` | `Qwen/Qwen2-VL-2B-Instruct`, etc. |
| `qwen2.5-vl` | `Qwen2_5_VLForConditionalGeneration` | `Qwen/Qwen2.5-VL-3B-Instruct`, etc. |
| name | Architecture | Example HF Models |
|--------------------|------------------------------------------|----------------------------------------|
| `intern-s1-pro` | `InternS1ProForConditionalGeneration` | `internlm/Intern-S1-Pro` |
| `intern-s2-mobius` | `InternS2MobiusForConditionalGeneration` | `internlm/Intern-S2-Mobius` |
| `qwen2-vl` | `Qwen2VLForConditionalGeneration` | `Qwen/Qwen2-VL-2B-Instruct`, etc. |
| `qwen2.5-vl` | `Qwen2_5_VLForConditionalGeneration` | `Qwen/Qwen2.5-VL-3B-Instruct`, etc. |
| `qwen3-vl` | `Qwen3VLForConditionalGeneration` | `Qwen/Qwen3-VL-4B-Instruct`, etc. |
| `qwen3-vl-moe` | `Qwen3VLMoeForConditionalGeneration` | `Qwen/Qwen3-VL-30B-A3B-Instruct`, etc. |
| `qwen3.5` | `Qwen3_5ForConditionalGeneration` | `Qwen/Qwen3.5-0.8B`, etc. |
| `qwen3.5-moe` | `Qwen3_5MoeForConditionalGeneration` | `Qwen/Qwen3.5-35B-A3B`, etc. |
| `qwen4-exp` | `Qwen4ExpForConditionalGeneration` | `Qwen/Qwen3.8-Flash-Next` |
#### Key Properties and Gains
+29 -11
View File
@@ -195,35 +195,53 @@ combined with a hybrid (HMA) KV layout:
that represents a pooled region can differ between P and D.
`NixlPushConnector` handles this for PP-sharded producers by routing
**by layer-name (member) identity** instead of by region index. Each
**by layer name** instead of by region index. Each
worker advertises which layer names back each of its NIXL regions in the
handshake metadata (`NixlAgentMetadata.region_members`). A producer that
needs member routing derives its member-major layout once, when it
needs layer routing derives its transfer layer order once, when it
registers its KV caches, and every transfer it issues uses that order.
`add_remote_agent` then selects exactly the remote regions this stage
owns and reorders them to match, so both sides stay paired regardless of
how each remote rank happens to order its metadata.
Addresses, block lengths, strides, and per-region capacities follow the same
member order. Descriptor offsets use each member's region capacity, so P and
layer order. Descriptor offsets use each layer's region capacity, so P and
D need not allocate the same number of blocks. Physical allocations are still
registered once, even when multiple members share them.
registered once, even when multiple layers share them.
Packed MLA caches (block-outermost layouts such as DeepSeek-V4's) store the
pages of every layer for one block in a single row. A PP stage holds only its
own layers, so its rows are shorter and each layer sits at a different offset:
```text
D (PP=1), block b: | L0 | L1 | L2 | L3 | L4 | L5 |
P (stage 2), block b: | L3 | L4 | L5 |
```
`PP=1` peers transfer whole rows. They also advertise where each layer's page
sits in the row (`NixlAgentMetadata.packed_member_layouts`), so a PP producer
writes each of its pages to the matching slot in D's row. Page sizes must
match; offsets, row strides, and block counts may differ. Because PP stages can
list the same attention backends in a different order, packed push compares the
sorted set of backend names during the handshake. Pull mode is unchanged.
Invariants enforced when the remote regions are aligned:
* every locally owned member must be advertised exactly once by the
remote; a missing member fails the handshake rather than silently
leaving that layer's KV stale, and remote-only members (owned by other
* every locally owned layer must be advertised exactly once by the
remote; a missing layer fails the handshake rather than silently
leaving that layer's KV stale, and remote-only layers (owned by other
PP stages) are ignored;
* a remote that omits member metadata while the local layout requires
member routing fails the handshake instead of falling back to
* a remote that omits layer metadata while the local layout requires
layer routing fails the handshake instead of falling back to
region-index routing;
* member order is a property of the local layout alone, so the same local
* layer order is a property of the local layout alone, so the same local
source descriptors serve every remote engine and TP rank; only the
remote descriptor list is rebuilt per rank.
Decode-side PP is unsupported because completions are counted per
consumer rank. Mamba/SSM hybrids are unsupported under PP.
consumer rank. Mamba/SSM hybrids are unsupported under PP. Packed PP push
requires MLA caches and equal P/D block sizes. MLA pages are replicated
across TP ranks, so producer TP1 to consumer TP2 is supported.
## Scheduler-side responsibilities
+2
View File
@@ -77,6 +77,8 @@ decode = client.chat.completions.create(
)
```
If `messages` has non-text content or `echo` is set, the ids are ignored and `messages` is rendered instead, so it must match the prefill request. Otherwise `kv_transfer_params["prompt_token_ids"]` must be a non-empty list of non-negative integers, or the request fails with HTTP 400, as it always does on `/v1/chat/completions/batch`.
## Development
We implement disaggregated prefilling by running 2 vLLM instances. One for prefill (we call it prefill instance) and one for decode (we call it decode instance), and then use a connector to transfer the prefill KV caches and results from prefill instance to decode instance.
+8 -1
View File
@@ -428,9 +428,16 @@ vLLM supports PEFT sequence-classification adapters that save a complete, single
See [classification_with_lora_offline.py](../../examples/pooling/classify/classification_with_lora_offline.py) for an offline classification example using a LoRA adapter.
To batch adapters with different `num_labels`, set the maximum number of labels:
```bash
vllm serve model --enable-lora --max-lora-cls-labels 8
```
The equivalent `LLM` argument is `max_lora_cls_labels`. It defaults to the base model's `num_labels`, and each request returns its adapter's number of labels.
This support has the following limitations:
- All adapters in one engine must have the same `num_labels` and hidden size as the base classification head. An incompatible adapter is rejected when it is loaded.
- A classification head stored as float32 is converted to the runtime head dtype when it is loaded.
- Token-classification adapters are not supported by this feature.
@@ -120,7 +120,8 @@ Current push PP + HMA limitations:
- Only the prefiller (producer) may be PP-sharded; decode-side PP is not supported.
- Hybrid SSM/Mamba layouts are not supported under PP.
- HMA requires the same block size on P and D.
- Attention-HMA member routing requires decode TP to be no greater than prefill TP.
- Non-MLA attention-HMA layer routing requires decode TP to be no greater than prefill TP.
- Packed layouts under PP require MLA caches. Local and remote packed block strides may differ; layer pages must have equal sizes. MLA supports decode TP greater than prefill TP because its KV is replicated.
### Quantized KV cache
@@ -11,7 +11,7 @@ Currently, the CPU implementation for s390x architecture supports FP32, BF16 and
- OS: `Linux`
- SDK: `gcc/g++ >= 14.0.0` or later with Command Line Tools
- Instruction Set Architecture (ISA): VXE support is required. Works with Z15 and above.
- Build from source python packages (no pre-built s390x wheels): `torchvision`, `llvmlite`, `numba`, `opencv-python-headless`, `hf-xet`
- Build from source python packages (no pre-built s390x wheels): `torchvision`, `llvmlite`, `numba`, `opencv-python-headless`, `hf-xet`, `torchcodec`
--8<-- [end:requirements]
--8<-- [start:set-up-using-python]
@@ -55,7 +55,8 @@ curl https://sh.rustup.rs -sSf | sh -s -- -y && \
Execute the following commands to build and install vLLM from source.
!!! tip
Pre-built wheels are not available for s390x for the following packages. Build them from source before building vLLM: `torchvision`, `llvmlite`, `numba`, `opencv-python-headless`, `hf-xet`.
Pre-built wheels are not available for s390x for the following packages. Build them from source before building vLLM: `torchvision`, `llvmlite`, `numba`, `opencv-python-headless`, `hf-xet`, `torchcodec`.
`torchcodec` also needs a shared FFmpeg (built in `Dockerfile.s390x`). Video decoding only; there is no s390x NVDEC path.
See `docker/Dockerfile.s390x` for exact versions and build commands used in each multi-stage build.
!!! note "LLVM 20 required for llvmlite"
@@ -94,6 +95,7 @@ uv pip install -v \
/path/to/numba.whl \
/path/to/opencv_python_headless.whl \
/path/to/hf_xet.whl \
/path/to/torchcodec.whl \
-r requirements/build/cpu.txt \
-r requirements/cpu.txt \
--torch-backend cpu \
@@ -111,6 +113,7 @@ VLLM_TARGET_DEVICE=cpu VLLM_CPU_MOE_PREPACK=0 python setup.py bdist_wheel && \
/path/to/numba.whl \
/path/to/opencv_python_headless.whl \
/path/to/hf_xet.whl \
/path/to/torchcodec.whl \
-r requirements/build/cpu.txt \
-r requirements/cpu.txt && \
VLLM_TARGET_DEVICE=cpu VLLM_CPU_MOE_PREPACK=0 python setup.py bdist_wheel && \
@@ -183,6 +183,40 @@ docker run \
vllm/vllm-openai-cpu:latest-x86_64 <args...>
```
### Serve with vLLM Recipes
The CPU image includes the [vLLM Recipes](https://recipes.vllm.ai/) deployment
tool. Recipe-assisted serving queries the site at runtime, so the image can use
the latest published recipe for the selected model and hardware.
To find the value for `--hardware`, open vLLM Recipes, select the model, and
choose the target in the **Hardware** picker. The selected page URL also
contains the hardware key; for example, `?hardware=xeon6` maps to
`--hardware xeon6`.
Keep the normal image entrypoint for regular serving. For recipe-assisted
serving, override it with `serve_with_recipe.sh`:
```bash
docker run --rm \
--shm-size=4g \
-v ~/.cache/huggingface:/root/.cache/huggingface \
-p 8000:8000 \
--env "HF_TOKEN=<secret>" \
--entrypoint tools/recipes/serve_with_recipe.sh \
vllm/vllm-openai-cpu:latest-x86_64 \
--model meta-llama/Llama-3.1-8B-Instruct \
--hardware xeon6
```
For Xeon 6, the script enables hardware detection automatically, generates
`config.yml` and `env.sh`, and starts `vllm serve`. The image keeps
`/vllm-workspace` as its working directory so relative recipe assets under
`examples/` continue to resolve.
See the [Recipes tool documentation](../../../tools/recipes/README.md) for
interactive discovery and additional options.
--8<-- [end:pre-built-images]
--8<-- [start:build-image-from-source]
+9 -9
View File
@@ -12,17 +12,17 @@
## Deploy from a vLLM Recipe
The **Recipe** column below links to validated Xeon 6 configurations when
available. Use the Recipe conversion tool to generate `config.yaml` and
`env.sh`; see the
[Recipes conversion tool README](../../../tools/recipes/README.md) for usage.
The **Recipe** column below links to published Xeon 6 configurations when
available. Open a recipe to review the latest deployment configuration for the
model and hardware.
Load the generated environment before starting vLLM:
For one-step deployment with the pre-built CPU image, see
[Serve with vLLM Recipes](../../getting_started/installation/cpu.md#serve-with-vllm-recipes).
The image includes the Recipes tool, which can retrieve the latest published
recipe, apply Xeon hardware detection, and start `vllm serve`.
```bash
source env.sh
vllm serve --config config.yaml
```
For converter usage and interactive model/hardware discovery, see the
[Recipes tool documentation](../../../tools/recipes/README.md).
## Recommended Models
+64
View File
@@ -0,0 +1,64 @@
# Prompt Token ID Logprobs (Teacher Scoring)
In on-policy distillation, the student records its top-K candidate tokens at
every response position, and the teacher provides the log probabilities of
exactly those candidates as the distillation target. The caller passes the
candidates as `prompt_logprob_token_ids`; the teacher runs a single prefill
over the prompt plus response and gathers their log probabilities on the GPU at
every scored row, starting from `prompt_logprob_start` (typically the prompt
length minus one, so only response positions are scored). The result is a
`[rows, K]` matrix whose row `i` scores prompt token
`prompt_logprob_start + i + 1` and whose columns follow the candidate order, so
it lines up position by position with the student's top-K.
## Quick start
```python
from vllm import LLM, SamplingParams
from vllm.inputs import TokensPrompt
llm = LLM(model)
output = llm.generate(
TokensPrompt(prompt_token_ids=prompt_ids + response_ids),
SamplingParams(
max_tokens=1,
prompt_logprob_token_ids=candidate_ids,
prompt_logprob_start=len(prompt_ids) - 1,
),
)
scores = output[0].prompt_token_id_logprobs
# scores.shape == (len(response_ids), len(candidate_ids))
# scores[i, j] = log p(candidate_ids[j] | prompt_ids + response_ids[:i])
```
The same parameters are accepted by the `/inference/v1/generate` HTTP endpoint
(Python and Rust frontends), which returns the matrix as a base64-encoded
`.npy` float32 array:
```python
import base64, io
import numpy as np
scores = np.load(io.BytesIO(base64.b64decode(response["prompt_token_id_logprobs"])))
```
The matrix has `max(prompt_len - 1 - prompt_logprob_start, 0)` rows. With a
logits `--logprobs-mode`, it holds logits instead of log probabilities.
## Requirements
- The V2 model runner (`VLLM_USE_V2_MODEL_RUNNER=1`).
- `len(prompt_logprob_token_ids)` at most `--max-logprobs`.
- No `--kv-sharing-fast-prefill`.
The request skips reading the prefix cache (local and KV connector), since
cached rows have no logits to score.
## Limitations
- One candidate set per request, shared by every scored row; per-position
candidates are planned ([#56860](https://github.com/vllm-project/vllm/issues/56860)).
- Non-streaming only; `stream=true` requests are rejected.
- Not exposed through the OpenAI-compatible endpoints.
- If a prefill starts past the first scored row (e.g. `skip_reading_prefix_cache=False`
with a cache hit), the result is `None` rather than a partial matrix.
+3 -3
View File
@@ -3,7 +3,7 @@
setuptools==77.0.3 # this version can reuse CMake build dir
numba == 0.65.0; platform_machine != "s390x" # Required for N-gram speculative decoding
numba == 0.65.0 # Required for N-gram speculative decoding
# Dependencies for CPUs
torch==2.13.0+cpu; platform_machine == "x86_64" or platform_machine == "s390x" or platform_machine == "aarch64"
@@ -13,10 +13,10 @@ torch==2.13.0; platform_system == "Darwin" or platform_machine == "ppc64le" or p
torchaudio; platform_machine != "s390x" and platform_machine != "riscv64"
# required for the image processor of phi3v, this must be updated alongside torch
torchvision; platform_machine != "s390x" and platform_machine != "riscv64"
torchvision; platform_machine != "riscv64"
# required for the torchcodec video decoding backend
torchcodec >= 0.14; platform_machine != "s390x" and platform_machine != "riscv64" and platform_machine != "ppc64le"
torchcodec >= 0.14; platform_machine != "riscv64" and platform_machine != "ppc64le"
# Intel Extension for PyTorch, only for x86_64 CPUs
intel-openmp==2024.2.1; platform_machine == "x86_64"
+1
View File
@@ -5968,6 +5968,7 @@ dependencies = [
"asynk-strim-attr",
"auto_enums",
"axum",
"base64 0.22.1",
"bytes",
"clap",
"educe",
@@ -17,6 +17,7 @@ use crate::error::{Error, Result, ext_value_decode};
use crate::protocol::logprobs::MaybeWireLogprobs;
use crate::protocol::sampling_mask::MaybeWireSamplingMask;
use crate::protocol::stats::{PrefillStats, SchedulerStats};
use crate::protocol::tensor::WireNdArray;
use crate::protocol::{OpaqueValue, decode_msgpack};
/// The stop reason associated with a finished output.
@@ -133,6 +134,10 @@ pub struct EngineCoreOutput {
/// output when `--per-request-spec-decode-metrics` is enabled.
#[serde(default)]
pub spec_decode_metrics: Option<RequestSpecDecodeMetrics>,
/// Log probabilities of `SamplingParams.prompt_logprob_token_ids`, set on
/// the first output of a request that asked for them.
#[serde(default)]
pub prompt_token_id_logprobs: Option<WireNdArray>,
}
/// Raw per-sequence speculative-decoding accumulator.
@@ -180,6 +185,11 @@ impl EngineCoreOutput {
self.new_sampling_mask = (self.new_sampling_mask.take())
.map(|value| value.resolve(frames, "new_sampling_mask"))
.transpose()?;
if let Some(value) = self.prompt_token_id_logprobs.as_mut() {
value
.resolve_aux_frame(frames)
.map_err(|message| ext_value_decode!("prompt_token_id_logprobs: {message}"))?;
}
Ok(())
}
}
@@ -531,6 +541,7 @@ mod tests {
mm_cache_miss_hashes: None,
new_sampling_mask: None,
spec_decode_metrics: None,
prompt_token_id_logprobs: None,
},
],
scheduler_stats: None,
@@ -101,6 +101,11 @@ pub struct EngineCoreSamplingParams {
///
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
pub prompt_logprobs: Option<i32>,
/// Candidate token IDs scored at every scored causal prompt row, where row
/// `i` scores them as predictions of prompt token `i + 1`.
pub prompt_logprob_token_ids: Option<Vec<u32>>,
/// First causal prompt row to score; `None` scores from the first row.
pub prompt_logprob_start: Option<u32>,
/// Minimum probability threshold for token sampling.
pub min_p: f32,
/// Frequency penalty applied by the sampler.
@@ -171,6 +176,8 @@ impl EngineCoreSamplingParams {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: 0.0,
frequency_penalty: 0.0,
presence_penalty: 0.0,
@@ -278,9 +278,9 @@ pub use kv_connector::*;
pub struct SchedulerStats {
/// Number of requests in model execution batches.
pub num_running_reqs: u64,
/// Length of the "waiting" request queue.
/// Number of waiting requests not deferred.
pub num_waiting_reqs: u64,
/// Length of the "skipped waiting" queue.
/// Number of waiting requests deferred by transient constraints or a blocked status.
#[serde(default)]
pub num_skipped_waiting_reqs: u64,
/// Internal DP load-balancing step counter.
@@ -2684,6 +2684,8 @@ fn python_msgpack_fixtures_match_rust_encoding() {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: 0.0,
frequency_penalty: 0.0,
presence_penalty: 0.0,
@@ -2770,6 +2772,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
mm_cache_miss_hashes: None,
new_sampling_mask: None,
spec_decode_metrics: None,
prompt_token_id_logprobs: None,
},
],
scheduler_stats: None,
@@ -116,6 +116,7 @@ class EngineCoreOutput(
mm_cache_miss_hashes: list[str] | None = None
new_sampling_mask: object | None = None
spec_decode_metrics: object | None = None
prompt_token_id_logprobs: object | None = None
class ExtendedEngineCoreOutput(EngineCoreOutput):
+16
View File
@@ -15,6 +15,7 @@ use vllm_engine_core_client::protocol::output::{
EngineCoreFinishReason, RequestSpecDecodeMetrics, StopReason,
};
use vllm_engine_core_client::protocol::sampling_mask::SamplingMask;
use vllm_engine_core_client::protocol::tensor::WireNdArray;
use vllm_engine_core_client::{AbortCause, EngineCoreOutputStream};
use crate::error::Result;
@@ -38,6 +39,7 @@ pub struct CollectedGenerateOutput {
pub request_id: String,
pub prompt_token_ids: Vec<u32>,
pub prompt_logprobs: Option<Logprobs>,
pub prompt_token_id_logprobs: Option<WireNdArray>,
pub token_ids: Vec<u32>,
pub logprobs: Option<Logprobs>,
pub finish_reason: FinishReason,
@@ -62,6 +64,8 @@ pub struct GeneratePromptInfo {
/// Prompt logprobs returned by engine-core for scored prompt positions,
/// when requested.
pub prompt_logprobs: Option<Logprobs>,
/// Log probabilities of `prompt_logprob_token_ids`, when requested.
pub prompt_token_id_logprobs: Option<WireNdArray>,
}
/// The reason a request finished.
@@ -206,6 +210,7 @@ impl GenerateOutput {
prompt_info: prompt_token_ids.map(|ids| GeneratePromptInfo {
prompt_token_ids: ids,
prompt_logprobs: None,
prompt_token_id_logprobs: None,
}),
token_ids,
logprobs: None,
@@ -247,6 +252,7 @@ impl GenerateOutputStream {
pending_prompt_info: Some(GeneratePromptInfo {
prompt_token_ids,
prompt_logprobs: None,
prompt_token_id_logprobs: None,
}),
raw_stream,
request_metrics,
@@ -282,6 +288,11 @@ impl Stream for GenerateOutputStream {
info.prompt_logprobs =
raw.new_prompt_logprobs_tensors.map(|value| value.into_direct().unwrap());
}
if let Some(info) = &mut self.pending_prompt_info
&& info.prompt_token_id_logprobs.is_none()
{
info.prompt_token_id_logprobs = raw.prompt_token_id_logprobs;
}
let logprobs = raw.new_logprobs.map(|value| value.into_direct().unwrap());
let sampling_mask = raw.new_sampling_mask.map(|value| value.into_direct().unwrap());
@@ -361,6 +372,7 @@ impl<T: Stream<Item = Result<GenerateOutput>> + Send> T {
pin_mut!(stream);
let mut prompt_token_ids = None;
let mut prompt_logprobs = None;
let mut prompt_token_id_logprobs = None;
let mut cached_token_count = 0;
let mut collected: Option<CollectedGenerateOutput> = None;
@@ -374,6 +386,9 @@ impl<T: Stream<Item = Result<GenerateOutput>> + Send> T {
if prompt_logprobs.is_none() {
prompt_logprobs = info.prompt_logprobs;
}
if prompt_token_id_logprobs.is_none() {
prompt_token_id_logprobs = info.prompt_token_id_logprobs;
}
}
if let Some(existing) = collected.as_mut() {
@@ -393,6 +408,7 @@ impl<T: Stream<Item = Result<GenerateOutput>> + Send> T {
request_id: output.request_id,
prompt_token_ids: prompt_token_ids.take().unwrap_or_default(),
prompt_logprobs: prompt_logprobs.take(),
prompt_token_id_logprobs: prompt_token_id_logprobs.take(),
token_ids: output.token_ids,
logprobs: output.logprobs,
finish_reason: FinishReason::Error,
+2
View File
@@ -277,6 +277,7 @@ async fn generate_streams_outputs() {
Some(GeneratePromptInfo {
prompt_token_ids: vec![11, 22].into(),
prompt_logprobs: Some(prompt_logprobs()),
prompt_token_id_logprobs: None,
})
);
assert_eq!(first.token_ids, vec![1, 2]);
@@ -405,6 +406,7 @@ async fn collect_output_rejects_partial_sampling_mask() {
prompt_info: Some(GeneratePromptInfo {
prompt_token_ids: Arc::from([11_u32, 22]),
prompt_logprobs: None,
prompt_token_id_logprobs: None,
}),
token_ids: vec![33, 44],
logprobs: None,
+136 -9
View File
@@ -150,7 +150,33 @@ impl ToolSchema {
impl JsonParamType {
/// Normalize one parameter property schema.
///
/// `"nullable": true` (the OpenAPI spelling of an optional parameter) adds
/// `null` to the accepted types.
fn from_schema(schema: &Value) -> Option<Self> {
let param_type = Self::from_schema_ignoring_nullable(schema)?;
let nullable = schema.get("nullable").and_then(Value::as_bool) == Some(true);
Some(match param_type {
_ if !nullable || param_type.accepts_null() => param_type,
Self::OneOf(mut types) => {
types.push(Self::Null);
Self::OneOf(types)
}
param_type => Self::OneOf(vec![param_type, Self::Null]),
})
}
/// Return whether `null` is one of the accepted types.
fn accepts_null(&self) -> bool {
match self {
Self::Null => true,
Self::OneOf(types) => types.iter().any(Self::accepts_null),
_ => false,
}
}
/// Normalize one parameter property schema, without `nullable`.
fn from_schema_ignoring_nullable(schema: &Value) -> Option<Self> {
let schema = schema.as_object()?;
if let Some(type_value) = schema.get("type") {
@@ -284,8 +310,10 @@ impl JsonParamType {
}
}
/// Recognize JSON and Python null spellings emitted by chat templates.
/// Recognize JSON and Python null spellings emitted by chat templates,
/// ignoring surrounding whitespace.
fn is_null_literal(value: &str) -> bool {
let value = value.trim();
value.eq_ignore_ascii_case("null") || value.eq_ignore_ascii_case("none")
}
@@ -327,18 +355,24 @@ fn try_convert_value(param_type: &JsonParamType, input: &ParamInput) -> Option<V
}
/// Convert one raw string value to a normalized JSON type.
///
/// Only `string` keeps the value verbatim; the other types ignore surrounding
/// whitespace.
fn try_convert_text_value(param_type: &JsonParamType, value: &str) -> Option<Value> {
match param_type {
JsonParamType::String => Some(Value::String(value.to_string())),
JsonParamType::Integer => value.parse::<i64>().ok().map(Number::from).map(Value::Number),
JsonParamType::Number => try_convert_number(value),
JsonParamType::Integer => without_digit_separators(value.trim())?
.parse::<i64>()
.ok()
.map(Number::from)
.map(Value::Number),
JsonParamType::Number => try_convert_number(&without_digit_separators(value.trim())?),
JsonParamType::Boolean => try_convert_boolean(value),
JsonParamType::Object { .. } if value.is_empty() => Some(Value::Object(Map::new())),
JsonParamType::Array { .. } if value.is_empty() => Some(Value::Array(Vec::new())),
JsonParamType::Object { .. } | JsonParamType::Array { .. } => {
// For composite types with string input, simply interpret the string as JSON.
serde_json::from_str(value).ok()
}
JsonParamType::Object { .. } if value.trim().is_empty() => Some(Value::Object(Map::new())),
JsonParamType::Array { .. } if value.trim().is_empty() => Some(Value::Array(Vec::new())),
// For composite types with string input, interpret the string as JSON of that type.
JsonParamType::Object { .. } => serde_json::from_str(value).ok().filter(Value::is_object),
JsonParamType::Array { .. } => serde_json::from_str(value).ok().filter(Value::is_array),
JsonParamType::Null => is_null_literal(value).then_some(Value::Null),
JsonParamType::OneOf(types) => {
types.iter().find_map(|param_type| try_convert_text_value(param_type, value))
@@ -411,6 +445,22 @@ fn insert_object_value(object: &mut Map<String, Value>, key: String, value: Valu
}
}
/// Remove single underscores between digits (`1_000`), as Python numeric
/// literals allow; any other underscore makes the value invalid.
fn without_digit_separators(value: &str) -> Option<std::borrow::Cow<'_, str>> {
if !value.contains('_') {
return Some(value.into());
}
let bytes = value.as_bytes();
let valid = bytes.iter().enumerate().all(|(index, byte)| {
*byte != b'_'
|| (index > 0
&& bytes[index - 1].is_ascii_digit()
&& bytes.get(index + 1).is_some_and(u8::is_ascii_digit))
});
valid.then(|| value.replace('_', "").into())
}
/// Convert one raw string value to a JSON number.
fn try_convert_number(value: &str) -> Option<Value> {
serde_json::from_str::<Number>(value)
@@ -555,6 +605,83 @@ mod tests {
ParamInput::Elements(elements)
}
#[test]
fn non_string_values_ignore_surrounding_whitespace() {
let params = ToolSchema::from_schema(&json!({
"type": "object",
"properties": {
"text": { "type": "string" },
"count": { "type": "integer" },
"size": { "type": "number" },
"enabled": { "type": "boolean" },
"payload": { "type": "object" },
"items": { "type": "array" }
}
}));
assert_eq!(params.convert("text", text(" 7 ")), json!(" 7 "));
assert_eq!(params.convert("count", text(" 7\n")), json!(7));
assert_eq!(params.convert("size", text("\t2.5 ")), json!(2.5));
assert_eq!(params.convert("enabled", text(" false ")), json!(false));
assert_eq!(params.convert("count", text(" None ")), json!(null));
assert_eq!(params.convert("payload", text(" \n")), json!({}));
assert_eq!(params.convert("items", text(" [1] ")), json!([1]));
}
#[test]
fn numbers_accept_digit_separators() {
let params = ToolSchema::from_schema(&json!({
"type": "object",
"properties": {
"count": { "type": "integer" },
"size": { "type": "number" }
}
}));
assert_eq!(params.convert("count", text("1_000")), json!(1000));
assert_eq!(params.convert("size", text("1_000.5")), json!(1000.5));
for invalid in ["_1", "1_", "1__0", "1_.5"] {
assert_eq!(params.convert("size", text(invalid)), json!(invalid));
}
}
#[test]
fn nullable_schemas_accept_null() {
let params = ToolSchema::from_schema(&json!({
"type": "object",
"properties": {
"text": { "type": "string", "nullable": true },
"count": { "type": ["integer", "string"], "nullable": true }
}
}));
// Without `nullable`, a string parameter keeps the literal text "null".
assert_eq!(params.convert("text", text("null")), json!(null));
assert_eq!(params.convert("text", text("x")), json!("x"));
assert_eq!(params.convert("count", text("None")), json!(null));
assert_eq!(params.convert("count", text("7")), json!(7));
}
#[test]
fn composite_values_must_decode_to_their_type() {
let params = ToolSchema::from_schema(&json!({
"type": "object",
"properties": {
"payload": { "type": "object" },
"items": { "type": "array" },
"either": { "type": ["object", "array"] }
}
}));
assert_eq!(params.convert("payload", text("[1]")), json!("[1]"));
assert_eq!(params.convert("payload", text("5")), json!("5"));
assert_eq!(
params.convert("items", text(r#"{"k":1}"#)),
json!(r#"{"k":1}"#)
);
assert_eq!(params.convert("either", text("[1]")), json!([1]));
}
#[test]
fn converts_upstream_aliases() {
let params = ToolSchema::from_schema(&json!({
+1
View File
@@ -9,6 +9,7 @@ anyhow.workspace = true
asynk-strim-attr.workspace = true
auto_enums.workspace = true
axum.workspace = true
base64.workspace = true
educe.workspace = true
futures.workspace = true
http-body.workspace = true
@@ -16,12 +16,15 @@ use axum::extract::State;
use axum::http::HeaderMap;
use axum::response::sse::Event;
use axum::response::{IntoResponse, Response};
use base64::Engine as _;
use futures::{Stream, StreamExt as _, pin_mut};
use thiserror_ext::AsReport as _;
use tracing::{error, info, trace};
use tracing_futures::Instrument as _;
use vllm_engine_core_client::protocol::dtype::TensorDtype;
use vllm_engine_core_client::protocol::logprobs::{Logprobs, PositionLogprobs};
use vllm_engine_core_client::protocol::multimodal::MmFeatureSpec;
use vllm_engine_core_client::protocol::tensor::WireNdArray;
use vllm_llm::{
CollectedGenerateOutput, FinishReason, GenerateOutput, GenerateOutputStreamExt as _, TokenUsage,
};
@@ -330,6 +333,11 @@ fn collect_generate(
token_ids: collected.token_ids,
}],
prompt_logprobs,
prompt_token_id_logprobs: collected
.prompt_token_id_logprobs
.as_ref()
.map(npy_base64)
.transpose()?,
prompt_token_ids: return_token_ids.then_some(collected.prompt_token_ids),
mm_placeholders: return_token_ids.then_some(mm_placeholders).flatten(),
kv_transfer_params: collected.kv_transfer_params,
@@ -432,6 +440,29 @@ fn position_to_logprob_map(position: &PositionLogprobs) -> HashMap<u32, Generate
.collect()
}
/// Encode a float32 matrix as Python `numpy2base64` does.
fn npy_base64(array: &WireNdArray) -> Result<String, ApiError> {
let (TensorDtype::F32, Some(data), [rows, cols]) = (
array.dtype.scalar,
array.data.as_raw_view(),
array.shape.as_slice(),
) else {
bail_server_error!("prompt_token_id_logprobs: expected a float32 matrix");
};
let mut header =
format!("{{'descr': '<f4', 'fortran_order': False, 'shape': ({rows}, {cols}), }}");
header.push_str(&" ".repeat(21 - rows.to_string().len()));
header.push_str(&" ".repeat(64 - (11 + header.len()) % 64));
header.push('\n');
let mut npy = Vec::with_capacity(10 + header.len() + data.len());
npy.extend_from_slice(b"\x93NUMPY\x01\x00");
npy.extend_from_slice(&(header.len() as u16).to_le_bytes());
npy.extend_from_slice(header.as_bytes());
npy.extend_from_slice(data);
Ok(base64::engine::general_purpose::STANDARD.encode(npy))
}
fn format_token_id(token_id: u32) -> String {
format!("token_id:{token_id}")
}
@@ -509,6 +540,7 @@ mod tests {
prompt_info: Some(GeneratePromptInfo {
prompt_token_ids: Arc::from([11_u32, 22_u32]),
prompt_logprobs: None,
prompt_token_id_logprobs: None,
}),
token_ids: vec![33],
logprobs: None,
@@ -592,6 +624,7 @@ mod tests {
prompt_info: prompt_token_ids.map(|ids| GeneratePromptInfo {
prompt_token_ids: Arc::from(ids),
prompt_logprobs: None,
prompt_token_id_logprobs: None,
}),
token_ids,
logprobs: None,
@@ -747,6 +780,7 @@ mod tests {
let output = CollectedGenerateOutput {
request_id: "raw-1".to_string(),
prompt_logprobs: None,
prompt_token_id_logprobs: None,
token_ids: vec![30],
logprobs: None,
finish_reason: FinishReason::stop_eos(),
@@ -781,6 +815,17 @@ mod tests {
assert!(response.mm_placeholders.is_none());
}
#[test]
fn prompt_token_id_logprobs_match_numpy2base64() {
let scores =
WireNdArray::from_f32(vec![2, 3], vec![-0.5, -1.5, -2.5, -3.5, -4.5, -5.5]).unwrap();
// numpy2base64(np.array([[-0.5, -1.5, -2.5], [-3.5, -4.5, -5.5]], np.float32))
assert_eq!(
npy_base64(&scores).unwrap(),
"k05VTVBZAQB2AHsnZGVzY3InOiAnPGY0JywgJ2ZvcnRyYW5fb3JkZXInOiBGYWxzZSwgJ3NoYXBlJzogKDIsIDMpLCB9ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAoAAAC/AADAvwAAIMAAAGDAAACQwAAAsMA="
);
}
#[test]
fn collect_generate_returns_spec_decode_metrics() {
let metrics = RequestSpecDecodeMetrics {
@@ -792,6 +837,7 @@ mod tests {
let output = CollectedGenerateOutput {
request_id: "raw-metrics".to_string(),
prompt_logprobs: None,
prompt_token_id_logprobs: None,
token_ids: vec![30],
logprobs: None,
finish_reason: FinishReason::stop_eos(),
@@ -879,6 +925,7 @@ mod tests {
let output = CollectedGenerateOutput {
request_id: "raw-1".to_string(),
prompt_logprobs: None,
prompt_token_id_logprobs: None,
token_ids: vec![30],
logprobs: None,
finish_reason: FinishReason::stop_eos(),
@@ -914,6 +961,7 @@ mod tests {
let output_without_payload = |prompt_token_ids: Vec<u32>| CollectedGenerateOutput {
request_id: "raw-1".to_string(),
prompt_logprobs: None,
prompt_token_id_logprobs: None,
token_ids: vec![3],
logprobs: None,
finish_reason: FinishReason::stop_eos(),
@@ -94,6 +94,8 @@ pub(super) struct GenerateResponse {
pub request_id: String,
pub choices: Vec<GenerateResponseChoice>,
pub prompt_logprobs: Option<Vec<Option<HashMap<u32, GenerateLogprob>>>>,
/// Base64 `.npy` float32 array, as Python `numpy2base64` encodes it.
pub prompt_token_id_logprobs: Option<String>,
pub prompt_token_ids: Option<Vec<u32>>,
pub mm_placeholders: Option<MultiModalPlaceholders>,
pub kv_transfer_params: Option<Value>,
@@ -57,6 +57,13 @@ pub(crate) fn validate_request_compat(
}
}
if request.stream && request.sampling_params.inner.prompt_logprob_token_ids.is_some() {
bail_invalid_request!(
param = "sampling_params",
"`prompt_logprob_token_ids` are not available when `stream=true`."
);
}
Ok(())
}
@@ -169,6 +176,17 @@ mod tests {
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
}
#[test]
fn validate_request_compat_rejects_streaming_prompt_logprob_token_ids() {
let request: GenerateRequest = serde_json::from_value(json!({
"token_ids": [11, 22],
"stream": true,
"sampling_params": {"prompt_logprob_token_ids": [1, 2]}
}))
.expect("parse request");
assert!(validate_request_compat(&request, &served(&["test-model"])).is_err());
}
#[test]
fn validate_request_compat_accepts_non_stream_prompt_logprobs() {
let request: GenerateRequest = serde_json::from_value(json!({
@@ -159,6 +159,8 @@ pub(super) fn prepare_chat_request(
thinking_token_budget: request.thinking_token_budget,
logprobs: request.logprobs.then_some(top_logprobs),
prompt_logprobs,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: request.min_p,
frequency_penalty: request.frequency_penalty,
presence_penalty: request.presence_penalty,
@@ -124,6 +124,8 @@ pub(super) fn prepare_completion_request(
thinking_token_budget: request.thinking_token_budget,
logprobs,
prompt_logprobs,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: request.min_p,
frequency_penalty: request.frequency_penalty,
presence_penalty: request.presence_penalty,
+56
View File
@@ -103,6 +103,8 @@ pub fn lower_sampling_params(
thinking_token_budget,
logprobs,
prompt_logprobs,
prompt_logprob_token_ids,
prompt_logprob_start,
min_p,
frequency_penalty,
presence_penalty,
@@ -123,6 +125,8 @@ pub fn lower_sampling_params(
logprobs,
prompt_logprobs,
logprob_token_ids.as_deref(),
prompt_logprob_token_ids.as_deref(),
prompt_logprob_start,
sampling_limits,
)?;
validate_repetition_detection(repetition_detection.as_ref())?;
@@ -176,6 +180,8 @@ pub fn lower_sampling_params(
thinking_token_budget,
logprobs,
prompt_logprobs,
prompt_logprob_token_ids,
prompt_logprob_start,
min_p,
frequency_penalty,
presence_penalty,
@@ -646,6 +652,8 @@ mod tests {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: 0.0,
frequency_penalty: 0.0,
presence_penalty: 0.0,
@@ -701,6 +709,8 @@ mod tests {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: 0.0,
frequency_penalty: 0.0,
presence_penalty: 0.0,
@@ -870,6 +880,8 @@ mod tests {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: 0.0,
frequency_penalty: 0.0,
presence_penalty: 0.0,
@@ -935,6 +947,8 @@ mod tests {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: 0.0,
frequency_penalty: 0.0,
presence_penalty: 0.0,
@@ -1008,6 +1022,8 @@ mod tests {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: 0.1,
frequency_penalty: 0.0,
presence_penalty: 0.0,
@@ -1122,6 +1138,44 @@ mod tests {
));
}
#[test]
fn lower_sampling_params_validates_prompt_logprob_token_ids() {
let lower = |ids: Option<Vec<u32>>, start: Option<u32>| {
lower_sampling_params_with_limits(
SamplingParams {
prompt_logprob_token_ids: ids,
prompt_logprob_start: start,
..Default::default()
},
sample_sampling_limits(),
)
};
let rejects = |ids, start| match lower(ids, start) {
Err(error) => error.to_string(),
Ok(_) => panic!("expected rejection"),
};
let params = lower(Some(vec![1, 2]), Some(1)).unwrap();
assert_eq!(params.prompt_logprob_token_ids, Some(vec![1, 2]));
assert_eq!(params.prompt_logprob_start, Some(1));
expect_test::expect![[r#"
[
"prompt_logprob_token_ids must not be empty.",
"requested prompt_logprob_token_ids of 21, which is greater than max allowed: 20",
"token_id(s) [1000] in prompt_logprob_token_ids are out of vocabulary. Vocabulary size: 1000",
"prompt_logprob_token_ids must not contain duplicates.",
"prompt_logprob_start requires prompt_logprob_token_ids.",
]
"#]]
.assert_debug_eq(&[
rejects(Some(vec![]), None),
rejects(Some((0..21).collect()), None),
rejects(Some(vec![1000]), None),
rejects(Some(vec![1, 1]), None),
rejects(None, Some(0)),
]);
}
#[test]
fn lower_sampling_params_rejects_out_of_vocab_stop_token_ids() {
let error = lower_sampling_params_with_limits(
@@ -1259,6 +1313,8 @@ mod tests {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: 0.1,
frequency_penalty: 0.0,
presence_penalty: 0.0,
+28
View File
@@ -6,6 +6,7 @@
//! `-1` is expanded only for bounds checks. The original request values are
//! passed through to engine-core.
use itertools::Itertools as _;
use thiserror::Error;
use crate::backend::SamplingLimits;
@@ -35,6 +36,12 @@ pub enum LogprobsError {
len(logprob_token_ids). Got logprobs={logprobs}, len(logprob_token_ids)={num_token_ids}."
)]
TokenIdsMismatch { logprobs: i32, num_token_ids: usize },
#[error("prompt_logprob_token_ids must not be empty.")]
EmptyPromptLogprobTokenIds,
#[error("prompt_logprob_token_ids must not contain duplicates.")]
DuplicatePromptLogprobTokenIds,
#[error("prompt_logprob_start requires prompt_logprob_token_ids.")]
PromptLogprobStartWithoutTokenIds,
}
/// Validate logprobs count sampling parameters.
@@ -42,6 +49,8 @@ pub(super) fn validate_logprobs(
logprobs: Option<i32>,
prompt_logprobs: Option<i32>,
logprob_token_ids: Option<&[u32]>,
prompt_logprob_token_ids: Option<&[u32]>,
prompt_logprob_start: Option<u32>,
sampling_limits: SamplingLimits,
) -> Result<(), LogprobsError> {
let vocab_size = sampling_limits.model_vocab_size;
@@ -50,9 +59,28 @@ pub(super) fn validate_logprobs(
validate_logprobs_count(logprobs, max_logprobs, vocab_size, "logprobs")?;
validate_logprobs_count(prompt_logprobs, max_logprobs, vocab_size, "prompt_logprobs")?;
validate_logprobs_count(
prompt_logprob_token_ids.map(|ids| ids.len() as i32),
max_logprobs,
vocab_size,
"prompt_logprob_token_ids",
)?;
validate_prompt_logprob_token_ids(prompt_logprob_token_ids, prompt_logprob_start)?;
validate_logprob_token_ids(logprobs, logprob_token_ids)
}
fn validate_prompt_logprob_token_ids(
token_ids: Option<&[u32]>,
start: Option<u32>,
) -> Result<(), LogprobsError> {
match token_ids {
Some([]) => Err(LogprobsError::EmptyPromptLogprobTokenIds),
Some(ids) if !ids.iter().all_unique() => Err(LogprobsError::DuplicatePromptLogprobTokenIds),
None if start.is_some() => Err(LogprobsError::PromptLogprobStartWithoutTokenIds),
_ => Ok(()),
}
}
fn validate_logprobs_count(
requested: Option<i32>,
max_logprobs: usize,
+8
View File
@@ -95,6 +95,14 @@ pub(crate) fn validate_vocab_range(
)?;
}
if let Some(token_ids) = params.prompt_logprob_token_ids.as_deref() {
validate_param(
"prompt_logprob_token_ids",
token_ids.iter().copied(),
limits.model_vocab_size,
)?;
}
if let Some(bad_words_token_ids) = params.bad_words_token_ids.as_deref() {
validate_param(
"bad_words",
+7
View File
@@ -97,6 +97,11 @@ pub struct SamplingParams {
///
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
pub prompt_logprobs: Option<i32>,
/// Candidate token IDs scored at every scored causal prompt row, where row
/// `i` scores them as predictions of prompt token `i + 1`.
pub prompt_logprob_token_ids: Option<Vec<u32>>,
/// First causal prompt row to score; `None` scores from the first row.
pub prompt_logprob_start: Option<u32>,
/// Minimum probability threshold for token sampling. `None` means no
/// explicit user override.
pub min_p: Option<f32>,
@@ -156,6 +161,8 @@ impl Default for SamplingParams {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: None,
frequency_penalty: None,
presence_penalty: None,
@@ -56,8 +56,10 @@ class TestSetting:
),
],
)
@pytest.mark.parametrize("use_v2_model_runner", [False, True])
def test_compile_correctness(
test_setting: TestSetting,
use_v2_model_runner: bool,
):
model = test_setting.model
model_args = test_setting.model_args
@@ -75,6 +77,10 @@ def test_compile_correctness(
f"--attention-backend={attn_backend}",
]
# Pin every compared setting to the same model runner so that only the
# compilation mode differs within a comparison.
runner_env = {"VLLM_USE_V2_MODEL_RUNNER": str(int(use_v2_model_runner))}
all_args: list[list[str]] = []
all_envs: list[dict[str, str] | None] = []
@@ -86,7 +92,7 @@ def test_compile_correctness(
CompilationMode.VLLM_COMPILE,
]:
all_args.append(final_args + [f"-cc.mode={mode.name}", "-cc.backend=inductor"])
all_envs.append({})
all_envs.append(dict(runner_env))
# inductor will change the output, so we only compare if the output
# is close, not exactly the same.
compare_all_settings(
@@ -94,7 +100,6 @@ def test_compile_correctness(
all_args,
all_envs,
method=method if method != "generate" else "generate_close",
force_v1_runner=True,
)
all_envs.clear()
@@ -108,5 +113,5 @@ def test_compile_correctness(
CompilationMode.VLLM_COMPILE,
]:
all_args.append(final_args + [f"-cc.mode={mode.name}", "-cc.backend=eager"])
all_envs.append({})
compare_all_settings(model, all_args, all_envs, method=method, force_v1_runner=True)
all_envs.append(dict(runner_env))
compare_all_settings(model, all_args, all_envs, method=method)
+214 -6
View File
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import operator
import pytest
import torch
@@ -29,6 +31,33 @@ class SplitCoalescingModel(torch.nn.Module):
return q + 1, k + 2, v + 3
class SliceCoalescingModel(torch.nn.Module):
"""Model with a complete QKV partition represented as contiguous slices."""
def __init__(self, q_size: int, kv_size: int) -> None:
super().__init__()
self.q_size = q_size
self.kv_size = kv_size
def forward(self, qkv: torch.Tensor):
q_end = self.q_size
k_end = q_end + self.kv_size
v_end = k_end + self.kv_size
q = torch.ops.aten.slice.Tensor(qkv, -1, 0, q_end)
k = torch.ops.aten.slice.Tensor(qkv, -1, q_end, k_end)
v = torch.ops.aten.slice.Tensor(qkv, -1, k_end, v_end)
return q + 1, k + 2, v + 3
def make_vllm_config() -> VllmConfig:
return VllmConfig(
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
pass_config=PassConfig(),
)
)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
def test_split_coalescing(dtype):
torch.set_default_device(DEVICE_TYPE)
@@ -37,12 +66,7 @@ def test_split_coalescing(dtype):
q_size, kv_size = 2048, 512
vllm_config = VllmConfig(
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
pass_config=PassConfig(),
)
)
vllm_config = make_vllm_config()
with vllm.config.set_current_vllm_config(vllm_config):
coalesce_pass = SplitCoalescingPass(vllm_config)
backend = TestBackend(coalesce_pass)
@@ -63,3 +87,187 @@ def test_split_coalescing(dtype):
torch.testing.assert_close(eager, compiled, atol=ATOL, rtol=RTOL)
assert backend.op_count(torch.ops.aten.split_with_sizes.default) == 1
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
def test_slice_partition_canonicalization(dtype):
torch.set_default_device(DEVICE_TYPE)
torch.set_default_dtype(dtype)
torch.manual_seed(0)
q_size, kv_size = 2048, 512
vllm_config = make_vllm_config()
with vllm.config.set_current_vllm_config(vllm_config):
coalesce_pass = SplitCoalescingPass(vllm_config)
backend = TestBackend(coalesce_pass)
model = SliceCoalescingModel(q_size, kv_size)
qkv = torch.randn(5, q_size + 2 * kv_size)
torch._dynamo.mark_dynamic(qkv, 0)
result_eager = model(qkv)
result_compiled = torch.compile(model, backend=backend, fullgraph=True)(qkv)
for eager, compiled in zip(result_eager, result_compiled):
torch.testing.assert_close(eager, compiled, atol=2e-3, rtol=2e-3)
assert backend.op_count(torch.ops.aten.slice.Tensor, before=True) == 3
assert backend.op_count(torch.ops.aten.slice.Tensor) == 0
assert backend.op_count(torch.ops.aten.split_with_sizes.default) == 1
split = next(
node
for node in backend.final_graph.nodes
if node.target == torch.ops.aten.split_with_sizes.default
)
assert list(split.args[1]) == [q_size, kv_size, kv_size]
assert split.args[2] == -1
getitems = [
node
for node in split.users
if node.op == "call_function" and node.target == operator.getitem
]
assert sorted(node.args[1] for node in getitems) == [0, 1, 2]
assert isinstance(split.meta["val"], list)
for getitem in getitems:
index = getitem.args[1]
assert getitem.meta["val"] is split.meta["val"][index]
def make_slice_graph(
ranges: list[tuple[int, int, int]],
) -> torch.fx.Graph:
graph = torch.fx.Graph()
source = graph.placeholder("qkv")
source.meta["val"] = torch.empty((2, 8), device="meta")
slices = []
for start, end, step in ranges:
node = graph.call_function(
torch.ops.aten.slice.Tensor,
args=(source, -1, start, end, step),
)
node.meta["val"] = torch.empty(
(2, len(range(start, min(end, 8), step))), device="meta"
)
slices.append(node)
graph.output(tuple(slices))
return graph
def test_slice_partition_accepts_safe_group():
graph = make_slice_graph([(0, 4, 1), (4, 8, 1)])
vllm_config = make_vllm_config()
with vllm.config.set_current_vllm_config(vllm_config):
SplitCoalescingPass(vllm_config)(graph)
graph.lint()
assert sum(node.target == torch.ops.aten.slice.Tensor for node in graph.nodes) == 0
assert (
sum(
node.target == torch.ops.aten.split_with_sizes.default
for node in graph.nodes
)
== 1
)
@pytest.mark.parametrize(
"ranges",
[
[(0, 3, 1), (4, 8, 1)], # gap
[(0, 5, 1), (4, 8, 1)], # overlap
[(1, 4, 1), (4, 8, 1)], # missing prefix
[(0, 4, 1), (4, 7, 1)], # missing suffix
[(0, 8, 1)], # single full-range slice
[(0, 8, 2)], # non-unit step
],
)
def test_slice_partition_rejects_unsafe_groups(ranges):
graph = make_slice_graph(ranges)
vllm_config = make_vllm_config()
with vllm.config.set_current_vllm_config(vllm_config):
SplitCoalescingPass(vllm_config)(graph)
slice_count = sum(
node.target == torch.ops.aten.slice.Tensor for node in graph.nodes
)
split_count = sum(
node.target == torch.ops.aten.split_with_sizes.default for node in graph.nodes
)
assert slice_count == len(ranges)
assert split_count == 0
def test_nested_slice_partitions_use_live_source():
"""Use the rewired source when a slice partition consumes another slice."""
graph = torch.fx.Graph()
source = graph.placeholder("input")
source.meta["val"] = torch.empty((2, 8), device="meta")
outer_left = graph.call_function(
torch.ops.aten.slice.Tensor, args=(source, -1, 0, 4)
)
outer_left.meta["val"] = torch.empty((2, 4), device="meta")
outer_right = graph.call_function(
torch.ops.aten.slice.Tensor, args=(source, -1, 4, 8)
)
outer_right.meta["val"] = torch.empty((2, 4), device="meta")
inner_left = graph.call_function(
torch.ops.aten.slice.Tensor, args=(outer_left, -1, 0, 2)
)
inner_left.meta["val"] = torch.empty((2, 2), device="meta")
inner_right = graph.call_function(
torch.ops.aten.slice.Tensor, args=(outer_left, -1, 2, 4)
)
inner_right.meta["val"] = torch.empty((2, 2), device="meta")
graph.output((inner_left, inner_right, outer_right))
vllm_config = make_vllm_config()
with vllm.config.set_current_vllm_config(vllm_config):
SplitCoalescingPass(vllm_config)(graph)
graph.lint()
assert sum(node.target == torch.ops.aten.slice.Tensor for node in graph.nodes) == 0
splits = [
node
for node in graph.nodes
if node.target == torch.ops.aten.split_with_sizes.default
]
assert len(splits) == 2
outer_split = next(node for node in splits if list(node.args[1]) == [4, 4])
inner_split = next(node for node in splits if list(node.args[1]) == [2, 2])
assert outer_split.args[0] is source
assert inner_split.args[0].target == operator.getitem
assert inner_split.args[0].args == (outer_split, 0)
def test_split_coalescing_preserves_different_dimensions():
"""Keep equal-sized splits on different dimensions as separate nodes."""
graph = torch.fx.Graph()
source = graph.placeholder("input")
source.meta["val"] = torch.empty((8, 8), device="meta")
outputs = []
for dim in (0, 1):
split = graph.call_function(
torch.ops.aten.split_with_sizes.default,
args=(source, [4, 4], dim),
)
for index in range(2):
outputs.append(graph.call_function(operator.getitem, args=(split, index)))
graph.output(tuple(outputs))
vllm_config = make_vllm_config()
with vllm.config.set_current_vllm_config(vllm_config):
SplitCoalescingPass(vllm_config)(graph)
splits = [
node
for node in graph.nodes
if node.target == torch.ops.aten.split_with_sizes.default
]
assert len(splits) == 2
assert all(split.args[0] is source for split in splits)
assert {split.args[2] for split in splits} == {0, 1}
+6 -3
View File
@@ -215,7 +215,9 @@ def test_use_cudagraphs(
# forked needed to workaround https://github.com/vllm-project/vllm/issues/21073
@pytest.mark.forked
def test_stock_torch_compile(vllm_runner, monkeypatch):
@pytest.mark.parametrize("use_v2_model_runner", [False, True])
def test_stock_torch_compile(vllm_runner, monkeypatch, use_v2_model_runner):
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", str(int(use_v2_model_runner)))
# Disable multiprocessing so that the counter is in the same process
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
@@ -226,9 +228,10 @@ def test_stock_torch_compile(vllm_runner, monkeypatch):
"facebook/opt-125m",
compilation_config={"mode": CompilationMode.STOCK_TORCH_COMPILE},
gpu_memory_utilization=0.4,
) as _,
) as runner,
):
pass
outputs = runner.generate_greedy(["Hello, my name is"], max_tokens=5)
assert outputs[0][0]
# forked needed to workaround https://github.com/vllm-project/vllm/issues/21073
+43 -1
View File
@@ -7,7 +7,11 @@ import os
import pytest
import torch
from vllm.compilation.wrapper import TorchCompileWithNoGuardsWrapper
from vllm.compilation.counter import compilation_counter
from vllm.compilation.wrapper import (
TorchCompileWithNoGuardsWrapper,
compile_model_with_stock_torch,
)
from vllm.config import (
CompilationConfig,
CompilationMode,
@@ -117,6 +121,44 @@ def test_torch_compile_wrapper(use_bytecode_hook, monkeypatch):
raise AssertionError("expected an exception to be raised")
def test_compile_model_with_stock_torch(monkeypatch):
"""The whole model is compiled in place with the configured backend."""
graphs: list[torch.fx.GraphModule] = []
def recording_backend(gm: torch.fx.GraphModule, example_inputs):
graphs.append(gm)
return gm.forward
monkeypatch.setattr(
CompilationConfig,
"init_backend",
lambda self, vllm_config, *args, **kwargs: recording_backend,
)
class Model(torch.nn.Module):
def __init__(self):
super().__init__()
self.linear = torch.nn.Linear(4, 2)
def forward(self, x: torch.Tensor):
return self.linear(x).relu()
vllm_config = VllmConfig()
vllm_config.compilation_config = CompilationConfig(
mode=CompilationMode.STOCK_TORCH_COMPILE
)
model = Model()
x = torch.randn(3, 4)
expected = model(x)
torch._dynamo.reset()
with compilation_counter.expect(stock_torch_compile_count=1):
compile_model_with_stock_torch(model, vllm_config)
assert not graphs # compilation is deferred to the first call
torch.testing.assert_close(model(x), expected)
assert len(graphs) == 1
if __name__ == "__main__":
# Run with both parameter values
+244
View File
@@ -188,6 +188,250 @@ def test_mm_processor_device_type_normalizes(device: object, expected: str | Non
assert config.get_mm_processor_device_type() == expected
def test_merge_mm_processor_kwargs_scoped_precedence():
configured = {
"size": {
"shortest_edge": 64,
"longest_edge": 512,
},
"do_resize": True,
"padding": False,
"images_kwargs": {
"size": {"longest_edge": 1024},
"do_resize": False,
},
"videos_kwargs": {
"size": {"longest_edge": 2048},
},
}
inference = {
"size": {"shortest_edge": 128},
"do_resize": True,
"images_kwargs": {
"size": {"shortest_edge": 256},
},
}
config = MultiModalConfig(
mm_processor_kwargs=configured,
mm_device_do_normalize=False,
)
assert config.merge_mm_processor_kwargs(inference) == {
"size": {
"shortest_edge": 128,
"longest_edge": 512,
},
"do_resize": True,
"padding": False,
"images_kwargs": {
"size": {
"shortest_edge": 256,
"longest_edge": 1024,
},
"do_resize": True,
},
"videos_kwargs": {
"size": {
"shortest_edge": 128,
"longest_edge": 2048,
},
},
}
def test_merge_mm_processor_kwargs_mapping_vs_replacement():
config = MultiModalConfig(
mm_processor_kwargs={
"size": {"shortest_edge": 64},
"images_kwargs": {"size": {"longest_edge": 1024}},
},
mm_device_do_normalize=False,
)
assert config.merge_mm_processor_kwargs({"size": 7}) == {
"size": 7,
"images_kwargs": {"size": 7},
}
assert config.merge_mm_processor_kwargs({"size": {}}) == {
"size": {"shortest_edge": 64},
"images_kwargs": {
"size": {
"shortest_edge": 64,
"longest_edge": 1024,
},
},
}
_EMPTY_IMAGE_SCOPE_KWARGS: tuple[dict[str, object], ...] = (
{},
{"images_kwargs": None},
{"images_kwargs": {}},
)
@pytest.mark.parametrize("configured", _EMPTY_IMAGE_SCOPE_KWARGS)
@pytest.mark.parametrize("inference", _EMPTY_IMAGE_SCOPE_KWARGS)
def test_merge_mm_processor_kwargs_empty_scopes_are_absent(
configured: dict[str, object],
inference: dict[str, object],
):
config = MultiModalConfig(
mm_processor_kwargs=configured,
mm_device_do_normalize=False,
)
assert config.merge_mm_processor_kwargs(inference) == {}
@pytest.mark.parametrize("empty_inference", _EMPTY_IMAGE_SCOPE_KWARGS)
def test_merge_mm_processor_kwargs_empty_inference_scope_preserves_config_merge(
empty_inference: dict[str, object],
):
config = MultiModalConfig(
mm_processor_kwargs={
"images_kwargs": {
"size": {
"shortest_edge": 64,
"longest_edge": 512,
},
},
},
mm_device_do_normalize=False,
)
inference = {
"size": {"shortest_edge": 128},
**empty_inference,
}
assert config.merge_mm_processor_kwargs(inference) == {
"size": {"shortest_edge": 128},
"images_kwargs": {
"size": {
"shortest_edge": 128,
"longest_edge": 512,
},
},
}
@pytest.mark.parametrize("empty_config", _EMPTY_IMAGE_SCOPE_KWARGS)
def test_merge_mm_processor_kwargs_empty_config_scope_allows_inference_merge(
empty_config: dict[str, object],
):
config = MultiModalConfig(
mm_processor_kwargs={
"size": {"longest_edge": 512},
**empty_config,
},
mm_device_do_normalize=False,
)
assert config.merge_mm_processor_kwargs(
{
"size": {"shortest_edge": 128},
"images_kwargs": {"size": {"longest_edge": 1024}},
}
) == {
"size": {
"shortest_edge": 128,
"longest_edge": 512,
},
"images_kwargs": {
"size": {
"shortest_edge": 128,
"longest_edge": 1024,
},
},
}
@pytest.mark.parametrize("non_mapping_scope", [False, 0, "", [], 123])
def test_merge_mm_processor_kwargs_falsey_non_mapping_scopes_are_not_empty(
non_mapping_scope: object,
):
config = MultiModalConfig(
mm_processor_kwargs={"images_kwargs": non_mapping_scope},
mm_device_do_normalize=False,
)
for empty_inference in _EMPTY_IMAGE_SCOPE_KWARGS:
assert config.merge_mm_processor_kwargs(empty_inference) == {
"images_kwargs": non_mapping_scope
}
@pytest.mark.parametrize(
("configured_scope", "inference_scope", "expected_scope"),
[
(123, {"size": {"shortest_edge": 128}}, {"size": {"shortest_edge": 128}}),
({"size": {"shortest_edge": 64}}, 123, 123),
(123, 456, 456),
],
)
def test_merge_mm_processor_kwargs_non_mapping_scope_precedence(
configured_scope: object,
inference_scope: object,
expected_scope: object,
):
config = MultiModalConfig(
mm_processor_kwargs={"images_kwargs": configured_scope},
mm_device_do_normalize=False,
)
merged = config.merge_mm_processor_kwargs({"images_kwargs": inference_scope})
assert merged["images_kwargs"] == expected_scope
def test_merge_mm_processor_kwargs_none_leaf_is_explicit():
config = MultiModalConfig(mm_device_do_normalize=False)
assert config.merge_mm_processor_kwargs({"images_kwargs": {"do_resize": None}}) == {
"images_kwargs": {"do_resize": None}
}
def test_merge_mm_processor_kwargs_normalize_and_ownership():
configured: dict[str, object] = {
"images_kwargs": {
"size": {"shortest_edge": 64},
},
}
inference: dict[str, object] = {
"do_normalize": True,
"do_rescale": True,
"images_kwargs": {
"size": {"longest_edge": 1024},
},
}
configured_before = copy.deepcopy(configured)
inference_before = copy.deepcopy(inference)
config = MultiModalConfig(
mm_processor_kwargs=configured,
mm_device_do_normalize=True,
)
merged = config.merge_mm_processor_kwargs(inference)
assert merged["do_normalize"] is False
assert merged["do_rescale"] is False
assert configured == configured_before
assert inference == inference_before
assert config.mm_processor_kwargs == configured_before
images_kwargs = merged["images_kwargs"]
assert isinstance(images_kwargs, dict)
size = images_kwargs["size"]
assert isinstance(size, dict)
size["shortest_edge"] = 999
assert configured == configured_before
assert inference == inference_before
assert config.mm_processor_kwargs == configured_before
def _validate_mm_processor_device(*, device: str, ec_role: ECRole | None) -> None:
ec_config = (
None
+30
View File
@@ -389,6 +389,36 @@ def test_flashinfer_standalone_workspace_size(
assert create_workspace.call_args.args[3] == expected
def test_flashinfer_workspace_failure_is_not_retried(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Creation is collective; retrying it per all-reduce desyncs ranks."""
create_workspace = Mock(return_value=None)
monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_workspace", None)
monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_quant_workspace", None)
monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_workspace_failed", False)
monkeypatch.setattr(
flashinfer_all_reduce,
"_resolve_fi_ar_backend",
Mock(return_value=("mnnvl", True)),
)
monkeypatch.setattr(flashinfer_all_reduce, "get_node_count", lambda: 1)
monkeypatch.setattr(
flashinfer_all_reduce, "_get_tuned_standalone_max_size", Mock(return_value=None)
)
monkeypatch.setattr(flashinfer_all_reduce, "_create_workspace", create_workspace)
args = (2, 0, 128, 4096, torch.bfloat16, Mock())
assert flashinfer_all_reduce.get_fi_ar_workspace(*args) is None
assert flashinfer_all_reduce.get_fi_ar_workspace(*args) is None
# One mnnvl attempt plus one trtllm fallback, never repeated.
assert create_workspace.call_count == 2
flashinfer_all_reduce.destroy_fi_ar_workspace()
flashinfer_all_reduce.get_fi_ar_workspace(*args)
assert create_workspace.call_count == 4
def test_flashinfer_all_reduce_precedes_nccl(monkeypatch: pytest.MonkeyPatch) -> None:
output = torch.empty(2)
fi_ar_comm = Mock(disabled=False)
@@ -26,6 +26,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
BatchChatCompletionRequest,
)
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.exceptions import VLLMValidationError
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.v1.engine.async_llm import AsyncLLM
@@ -385,3 +386,19 @@ async def test_batched_harmony_response_format_uses_structural_tag() -> None:
assert len(calls) == 2
for call in calls:
assert call.args[1].structured_outputs.structural_tag is not None
@pytest.mark.skip_global_cleanup
def test_batch_rejects_kv_transfer_prompt_token_ids():
"""One pre-tokenized prompt cannot stand in for every conversation."""
with pytest.raises(
VLLMValidationError, match=r"parameter=kv_transfer_params\.prompt_token_ids"
):
BatchChatCompletionRequest(
model="test-model",
messages=[
[{"role": "user", "content": "first"}],
[{"role": "user", "content": "second"}],
],
kv_transfer_params={"prompt_token_ids": [10, 20, 30]},
)
@@ -595,6 +595,15 @@ def _build_online_renderer(
)
def _build_mock_engine() -> MagicMock:
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
mock_engine.model_config = MockModelConfig()
mock_engine.input_processor = MagicMock()
mock_engine.renderer = _build_renderer(mock_engine.model_config)
return mock_engine
def _build_serving_chat(
engine: AsyncLLM,
*,
@@ -2817,3 +2826,132 @@ def test_make_request_with_harmony_reuses_kv_transfer_prompt_token_ids():
assert engine_input["prompt_token_ids"] == [10, 20, 30]
# The reuse key is consumed and other kv_transfer_params are preserved.
assert request.kv_transfer_params == {"do_remote_prefill": True}
@pytest.mark.parametrize("ids", [[-1], [1.5]])
def test_make_request_with_harmony_rejects_invalid_kv_transfer_prompt_token_ids(ids):
engine = MockEngine()
engine.model_config.hf_config = MockHFConfig(model_type="gpt_oss")
models = OpenAIServingModels(engine, BASE_MODEL_PATHS)
online_renderer = _build_online_renderer(engine, models.registry)
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "hi"}],
kv_transfer_params={"prompt_token_ids": ids},
)
with pytest.raises(VLLMValidationError, match="non-negative integers"):
online_renderer._make_request_with_harmony(request)
@pytest.mark.asyncio
@pytest.mark.parametrize("ids", [[], [-1], [1.5], [1.0], ["1"], [True], "abc", 5])
async def test_chat_kv_transfer_prompt_token_ids_rejects_invalid_ids(ids):
"""``kv_transfer_params`` is untyped, so the ids are checked on reuse."""
serving_chat = _build_serving_chat(_build_mock_engine())
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "hi"}],
kv_transfer_params={"prompt_token_ids": ids},
)
with pytest.raises(VLLMValidationError, match="non-negative integers"):
await serving_chat.render_chat_request(request)
@pytest.mark.asyncio
async def test_chat_kv_transfer_prompt_token_ids_rejection_notifies_kv_connector():
"""A render-time 400 on a decode request notifies the KV connector."""
mock_engine = _build_mock_engine()
mock_engine.notify_kv_transfer_request_rejected = AsyncMock()
serving_chat = _build_serving_chat(mock_engine)
serving_chat.has_kv_connector = True
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "hi"}],
kv_transfer_params={"prompt_token_ids": [1.5], "do_remote_prefill": True},
)
with pytest.raises(VLLMValidationError, match="non-negative integers"):
await serving_chat.create_chat_completion(request)
mock_engine.notify_kv_transfer_request_rejected.assert_awaited_once_with(
request.request_id, {"do_remote_prefill": True}, data_parallel_rank=None
)
@pytest.mark.asyncio
@pytest.mark.parametrize("ids", [[10, 20, 30], [1.5]])
async def test_chat_kv_transfer_prompt_token_ids_ignored_with_echo(ids):
"""``echo`` needs the rendered conversation, so the ids are not used."""
serving_chat = _build_serving_chat(_build_mock_engine())
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "hi"}],
kv_transfer_params={"prompt_token_ids": ids, "do_remote_prefill": True},
echo=True,
)
result = await serving_chat.render_chat_request(request)
assert not isinstance(result, ErrorResponse)
conversation, engine_inputs = result
assert [msg["role"] for msg in conversation] == ["user"]
assert engine_inputs[0]["prompt_token_ids"] != ids
assert request.kv_transfer_params == {"do_remote_prefill": True}
@pytest.mark.parametrize(
"content",
[
[
{"type": "text", "text": "what is in this image?"},
{"type": "image_url", "image_url": {"url": "https://example.com/a.png"}},
],
# A part written without a "type" is identified by its media key.
[{"image_url": "https://example.com/a.png"}],
# A media key counts even when the part claims to be text, which is
# how chat_utils reads a part carrying a "uuid".
[
{
"type": "text",
"text": "look",
"uuid": "u1",
"image_url": "https://example.com/a.png",
}
],
[{"type": "hologram", "hologram": "https://a/b.holo"}],
[{"type": ["image_url"]}],
],
)
def test_chat_kv_transfer_prompt_token_ids_ignored_with_non_text_content(content):
"""The ids are dropped so that ``messages`` is rendered with its media."""
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": content}],
kv_transfer_params={
"prompt_token_ids": [10, 20, 30],
"do_remote_prefill": True,
},
)
assert request.kv_transfer_params == {"do_remote_prefill": True}
def test_chat_kv_transfer_prompt_token_ids_allows_text_only_parts():
"""Text-bearing part types are not multimodal, so the ids are kept."""
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[
{"role": "user", "content": [{"type": "text", "text": "hi"}]},
{
"role": "assistant",
"content": [
{"type": "thinking", "thinking": "let me think"},
{"type": "refusal", "refusal": "no"},
],
},
{"role": "user", "content": "plain string"},
],
kv_transfer_params={"prompt_token_ids": [10, 20, 30]},
)
assert request.kv_transfer_params == {"prompt_token_ids": [10, 20, 30]}
@@ -1,9 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import io
import json
import httpx
import numpy as np
import pybase64 as base64
import pytest
import pytest_asyncio
from transformers import AutoTokenizer
@@ -311,6 +314,52 @@ async def test_generate_logprobs(client, logprobs_value):
assert len(entry["top_logprobs"]) == max(logprobs_value, 1)
@pytest.mark.asyncio
async def test_generate_prompt_token_id_logprobs(client):
token_ids = [11, 22, 33, 44, 55]
candidates = [22, 33, 44, 55]
sampling_params = {
"max_tokens": 1,
"prompt_logprobs": 0,
"prompt_logprob_token_ids": candidates,
"prompt_logprob_start": 1,
}
payload = {
"model": MODEL_NAME,
"token_ids": token_ids,
"sampling_params": sampling_params,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
data = resp.json()
scores = np.load(io.BytesIO(base64.b64decode(data["prompt_token_id_logprobs"])))
assert scores.shape == (len(token_ids) - 2, len(candidates))
for row, target in enumerate(token_ids[2:]):
expected = data["prompt_logprobs"][row + 2][str(target)]["logprob"]
assert scores[row, candidates.index(target)] == pytest.approx(
expected, abs=1e-3
)
stream_only_ids = {"prompt_logprob_token_ids": candidates}
resp = await client.post(
GEN_ENDPOINT,
json={**payload, "sampling_params": stream_only_ids, "stream": True},
)
assert resp.status_code == 400
assert "prompt_logprob_token_ids" in resp.text
past_end = {**sampling_params, "prompt_logprob_start": len(token_ids)}
resp = await client.post(
GEN_ENDPOINT, json={**payload, "sampling_params": past_end}
)
resp.raise_for_status()
empty = np.load(
io.BytesIO(base64.b64decode(resp.json()["prompt_token_id_logprobs"]))
)
assert empty.shape == (0, len(candidates))
@pytest.mark.asyncio
async def test_same_response_as_chat_completions(client, tokenizer, messages):
token_ids = tokenizer.apply_chat_template(
@@ -0,0 +1,24 @@
model_name: "RedHatAI/DeepSeek-Coder-V2-Lite-Instruct-FP8"
accuracy_threshold: 0.72
num_questions: 1319
num_fewshot: 5
rocm_request_timeout_seconds: 1800
# Dense MLA PCP on MI355; DP1/DCP1, eager execution, and Triton MoE.
server_args: >-
--revision efe1ced428db63e7ccbcf367334596f77e9af140
--code-revision e434a23f91ba5b4923cf6c9d9a238eb4a08e3a11
--dtype bfloat16
--enforce-eager
--max-model-len 4096
--max-num-seqs 8
--max-num-batched-tokens 4096
--tensor-parallel-size 1
--prefill-context-parallel-size 2
--enable-expert-parallel
--moe-backend triton
--attention-config '{"backend":"ROCM_AITER_MLA","mla_prefill_backend":"ROCM_AITER_FA"}'
--kv-cache-dtype auto
env:
VLLM_USE_V2_MODEL_RUNNER: "1"
VLLM_ROCM_USE_AITER: "1"
VLLM_WORKER_MULTIPROC_METHOD: "spawn"
@@ -1,3 +1,4 @@
Qwen3.5-35B-A3B-DEP2.yaml
Qwen3.5-35B-A3B-FP8-DEP2.yaml
Qwen3.5-35B-A3B-MXFP4-AITER-TP2.yaml
Qwen3.5-35B-A3B-MXFP4-EMU-TP2.yaml
@@ -0,0 +1,2 @@
# Dense MLA PCP accuracy on MI355.
DeepSeek-V2-Lite-Instruct-FP8-TP1-PCP2-ROCm.yaml
@@ -0,0 +1,109 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""fp8 query and index-K cache on the Triton indexer must agree with bf16.
The MiniMax M3 lightning-indexer scores only feed a top-k block ranking (the
attention itself reads the main KV cache), so an fp8 (e4m3) side cache trades
~2% relative score error for half the per-step index-K read bandwidth — the
dominant linear-in-context decode cost at long context. #45892 enabled fp8
index caches on the SM100 MSA path; this covers the Triton impl used
everywhere else (e.g. SM120), for both the decode and prefill scorers.
"""
import pytest
import torch
from vllm.models.minimax_m3.common.ops.index_topk import (
SPARSE_BLOCK_SIZE,
minimax_m3_index_decode,
minimax_m3_index_score,
minimax_m3_index_topk,
)
from vllm.platforms import current_platform
FP8_CUDA_SUPPORTED = current_platform.is_cuda() and current_platform.supports_fp8()
PAGE = SPARSE_BLOCK_SIZE
HEAD_DIM = 128
HEADS = 1
TOPK = 16
NUM_PAGES = 100
SEQ_LEN = 96 * PAGE + 37
PLANTED = (7, 40, 88)
def _make_inputs(device: str) -> tuple[torch.Tensor, torch.Tensor]:
torch.manual_seed(3)
keys = torch.randn(NUM_PAGES * PAGE, HEAD_DIM, dtype=torch.bfloat16, device=device)
# RMS-normalize like the real (post index_k_norm) keys.
keys = keys * keys.float().pow(2).mean(-1, keepdim=True).add(1e-6).rsqrt().to(
torch.bfloat16
)
q = torch.randn(1, HEADS, HEAD_DIM, dtype=torch.bfloat16, device=device)
for b in PLANTED:
keys[b * PAGE + 5] = (q[0, 0] * 3).to(torch.bfloat16)
return q, keys
def _check_selection(top_bf16: set[int], top_fp8: set[int]) -> None:
# Planted (clearly relevant) blocks must never be lost.
assert all(b in top_fp8 for b in PLANTED)
# Selection may differ only at the noise floor (near-tied filler blocks).
assert len(top_bf16 & top_fp8) >= TOPK - 2
@pytest.mark.skipif(not FP8_CUDA_SUPPORTED, reason="CUDA FP8 support required")
@torch.inference_mode()
def test_fp8_index_query_and_cache_decode_topk_matches_bf16() -> None:
dev = "cuda"
q, keys = _make_inputs(dev)
block_table = torch.arange(NUM_PAGES, dtype=torch.int32, device=dev).unsqueeze(0)
seq_lens = torch.tensor([SEQ_LEN], dtype=torch.int32, device=dev)
def select(cache: torch.Tensor) -> set[int]:
query = q.to(cache.dtype)
idx = minimax_m3_index_decode(
query,
cache.view(NUM_PAGES, PAGE, HEAD_DIM).contiguous(),
block_table,
seq_lens,
SEQ_LEN,
TOPK,
0,
0,
HEADS,
1,
1,
)
return {int(b) for b in idx[0, 0].tolist() if b >= 0}
_check_selection(select(keys), select(keys.to(torch.float8_e4m3fn)))
@pytest.mark.skipif(not FP8_CUDA_SUPPORTED, reason="CUDA FP8 support required")
@torch.inference_mode()
def test_fp8_index_query_and_cache_prefill_topk_matches_bf16() -> None:
dev = "cuda"
q, keys = _make_inputs(dev)
block_table = torch.arange(NUM_PAGES, dtype=torch.int32, device=dev).unsqueeze(0)
seq_lens = torch.tensor([SEQ_LEN], dtype=torch.int32, device=dev)
# single decode-like query token at the end of the sequence
cu_seqlens_q = torch.tensor([0, 1], dtype=torch.int32, device=dev)
prefix_lens = torch.tensor([SEQ_LEN - 1], dtype=torch.int32, device=dev)
def select(cache: torch.Tensor) -> set[int]:
query = q.to(cache.dtype)
score = minimax_m3_index_score(
query,
cache.view(NUM_PAGES, PAGE, HEAD_DIM).contiguous(),
block_table,
cu_seqlens_q,
seq_lens,
prefix_lens,
1,
SEQ_LEN,
HEADS,
)
idx = minimax_m3_index_topk(score, cu_seqlens_q, prefix_lens, 1, TOPK, 0, 0)
return {int(b) for b in idx[0, 0].tolist() if b >= 0}
_check_selection(select(keys), select(keys.to(torch.float8_e4m3fn)))
@@ -8,6 +8,7 @@ import pytest
import torch
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(), reason="Only used by ROCm"
@@ -424,6 +425,37 @@ def test_compute_global_topk_ragged_indices_and_indptr() -> None:
torch.testing.assert_close(actual_lens, expected_lens)
@pytest.mark.parametrize("width", [16, 640, 2176])
@pytest.mark.parametrize("num_queries", [1, 70, 1000])
@torch.inference_mode()
def test_build_ragged_indices_from_dense_drops_invalid_entries(
width: int, num_queries: int
) -> None:
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
build_ragged_indices_from_dense,
)
device = torch.device("cuda")
set_random_seed(width + num_queries)
num_rows = 5000
indices = torch.randint(0, num_rows + 100, (num_queries, width))
indices[torch.rand(num_queries, width) < 0.3] = -1
lengths = torch.randint(0, width + 8, (num_queries,))
flat, indptr = build_ragged_indices_from_dense(
indices.to(device, torch.int32),
lengths.to(device, torch.int32),
num_rows=num_rows,
)
# Each row keeps its in-range entries within its length, in order.
expected = [
[x for x in row[:n] if 0 <= x < num_rows]
for row, n in zip(indices.tolist(), lengths.tolist())
]
assert _rows_from_ragged(flat, indptr) == expected
@torch.inference_mode()
def test_combine_topk_swa_indices_adds_image_visibility() -> None:
from vllm.models.deepseek_v4.amd.rocm import combine_topk_swa_indices
+18 -5
View File
@@ -233,9 +233,15 @@ def test_apply_rotary_emb_xpu_matches_native(
def test_packed_qk_rope_correctness(
num_tokens: int, dtype: torch.dtype, default_vllm_config
):
"""packed_qk_rope_ must be bitwise identical to the per-tensor
ApplyRotaryEmb path with enable_fp32_compute=True, and must leave the V
slice untouched."""
"""packed_qk_rope_ must match the per-tensor ApplyRotaryEmb path with
enable_fp32_compute=True to within one ULP, and must leave the V slice
untouched.
Both sides compute in fp32 on identical inputs. The only gap is which
product the backend keeps exact inside the o1 multiply-add, and that
follows the register layout, so it is decided by codegen rather than by
the source and holds neither across backends nor across Triton versions.
"""
from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb
from vllm.model_executor.layers.rotary_embedding.packed_qk_rope import (
packed_qk_rope_,
@@ -265,6 +271,13 @@ def test_packed_qk_rope_correctness(
xk_ref = op(xk, cos, sin)
packed_qk_rope_(xqkv, freqs_cis)
torch.testing.assert_close(xqkv[:, 0], xq_ref, atol=0, rtol=0)
torch.testing.assert_close(xqkv[:, 1], xk_ref, atol=0, rtol=0)
# One ULP at the largest magnitude present, not relative: o1 can cancel,
# so the error scales with the inputs to the sum, not with the result.
eps = torch.finfo(dtype).eps
q_atol = eps * xq_ref.abs().max().item()
k_atol = eps * xk_ref.abs().max().item()
torch.testing.assert_close(xqkv[:, 0], xq_ref, atol=q_atol, rtol=0)
torch.testing.assert_close(xqkv[:, 1], xk_ref, atol=k_atol, rtol=0)
# V is never read or written by the kernel, so it stays exact.
torch.testing.assert_close(xqkv[:, 2], xv, atol=0, rtol=0)
+93 -30
View File
@@ -4,54 +4,57 @@ import numpy as np
import pytest
import torch
from vllm.platforms import current_platform
from vllm.utils.platform_utils import is_uva_available
from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor
from vllm.v1.worker.gpu import buffer_utils
from vllm.v1.worker.gpu.buffer_utils import StagedWriteTensor
CUDA_DEVICES = [
f"cuda:{i}" for i in range(1 if torch.accelerator.device_count() == 1 else 2)
DEVICE_TYPE = current_platform.device_type
DEVICES = [
f"{DEVICE_TYPE}:{i}"
for i in range(1 if torch.accelerator.device_count() == 1 else 2)
]
@pytest.mark.skipif(not is_uva_available(), reason="UVA is not available.")
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("device", DEVICES)
def test_cpu_write(device):
torch.set_default_device(device)
cpu_tensor = torch.zeros(10, 10, device="cpu", pin_memory=True, dtype=torch.int32)
cuda_view = get_accelerator_view_from_cpu_tensor(cpu_tensor)
assert cuda_view.device.type == "cuda"
gpu_view = get_accelerator_view_from_cpu_tensor(cpu_tensor)
assert gpu_view.device.type == DEVICE_TYPE
assert cuda_view[0, 0] == 0
assert cuda_view[2, 3] == 0
assert cuda_view[4, 5] == 0
assert gpu_view[0, 0] == 0
assert gpu_view[2, 3] == 0
assert gpu_view[4, 5] == 0
cpu_tensor[0, 0] = 1
cpu_tensor[2, 3] = 2
cpu_tensor[4, 5] = -1
cuda_view.mul_(2)
assert cuda_view[0, 0] == 2
assert cuda_view[2, 3] == 4
assert cuda_view[4, 5] == -2
gpu_view.mul_(2)
assert gpu_view[0, 0] == 2
assert gpu_view[2, 3] == 4
assert gpu_view[4, 5] == -2
@pytest.mark.skipif(not is_uva_available(), reason="UVA is not available.")
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("device", DEVICES)
def test_gpu_write(device):
torch.set_default_device(device)
cpu_tensor = torch.zeros(10, 10, device="cpu", pin_memory=True, dtype=torch.int32)
cuda_view = get_accelerator_view_from_cpu_tensor(cpu_tensor)
assert cuda_view.device.type == "cuda"
gpu_view = get_accelerator_view_from_cpu_tensor(cpu_tensor)
assert gpu_view.device.type == DEVICE_TYPE
assert cuda_view[0, 0] == 0
assert cuda_view[2, 3] == 0
assert cuda_view[4, 5] == 0
assert gpu_view[0, 0] == 0
assert gpu_view[2, 3] == 0
assert gpu_view[4, 5] == 0
cuda_view[0, 0] = 1
cuda_view[2, 3] = 2
cuda_view[4, 5] = -1
cuda_view.mul_(2)
gpu_view[0, 0] = 1
gpu_view[2, 3] = 2
gpu_view[4, 5] = -1
gpu_view.mul_(2)
assert cpu_tensor[0, 0] == 2
assert cpu_tensor[2, 3] == 4
@@ -59,7 +62,7 @@ def test_gpu_write(device):
@pytest.mark.skipif(not is_uva_available(), reason="UVA is not available.")
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("device", DEVICES)
def test_staged_write_uses_uva_contents_for_uva_target(device, monkeypatch):
def fail_async_tensor_h2d(*args, **kwargs):
pytest.fail("UVA-backed targets should not copy write contents to the GPU")
@@ -146,7 +149,7 @@ def test_uva_pool_copy_to_gpu_preserves_shape_and_out(use_out, input_type):
)
values = expected.numpy() if input_type == "numpy" else expected
out = (
torch.empty(expected.shape, dtype=torch.int32, device="cuda")
torch.empty(expected.shape, dtype=torch.int32, device=DEVICE_TYPE)
if use_out
else None
)
@@ -161,7 +164,7 @@ def test_uva_pool_copy_to_gpu_preserves_shape_and_out(use_out, input_type):
@pytest.mark.parametrize("dtype", [torch.int32, torch.int64, torch.float32])
def test_staged_write_inflight(uva_target, dtype):
"""Preserve every generation until its consumer finishes before slot reuse."""
device = torch.device("cuda:0")
device = torch.device(f"{DEVICE_TYPE}:0")
with torch.accelerator.device_index(device.index):
state = StagedWriteTensor(
(4, 4096),
@@ -171,9 +174,9 @@ def test_staged_write_inflight(uva_target, dtype):
uva_instead_of_gpu=uva_target,
)
assert (state.write_contents is not None) == uva_target
stream = torch.cuda.Stream()
stream.wait_stream(torch.cuda.current_stream())
pending: list[tuple[torch.cuda.Event, torch.Tensor, torch.Tensor]] = []
stream = torch.Stream(device=device)
stream.wait_stream(torch.accelerator.current_stream())
pending: list[tuple[torch.Event, torch.Tensor, torch.Tensor]] = []
expected = torch.zeros((4, 4096), dtype=dtype, device="cpu")
for step in range(24):
if len(pending) == 2:
@@ -188,15 +191,75 @@ def test_staged_write_inflight(uva_target, dtype):
values += 0.25
expected[row, 2 : 2 + length] = values
expected[3, 1:4] = step
with torch.cuda.stream(stream):
with stream:
state.stage_write(row, 2, values.tolist())
state.stage_write(3, 1, [step] * 3)
state.apply_write()
# A GPU consumer observes this generation before the next update.
snapshot = state.gpu.clone()
event = torch.cuda.Event()
event = torch.Event()
event.record(stream)
pending.append((event, snapshot, expected.clone()))
for event, snapshot, reference in pending:
event.synchronize()
torch.testing.assert_close(snapshot.cpu(), reference, rtol=0, atol=0)
@pytest.mark.skipif(not is_uva_available(), reason="UVA is not available.")
@pytest.mark.parametrize("device", DEVICES)
def test_non_pinned_cpu_tensor(device):
# Non-pinned CPU tensors are internally copied into a pinned buffer,
# so the resulting gpu view reflects the values at creation time but
# is decoupled from further writes to the original `cpu_tensor`.
torch.set_default_device(device)
cpu_tensor = torch.arange(100, dtype=torch.int32, device="cpu").view(10, 10)
assert not cpu_tensor.is_pinned()
gpu_view = get_accelerator_view_from_cpu_tensor(cpu_tensor)
assert gpu_view.device.type == DEVICE_TYPE
assert gpu_view[0, 0] == 0
assert gpu_view[2, 3] == 23
assert gpu_view[9, 9] == 99
# Writes to the original (unpinned) CPU tensor must not affect the view,
# since a private pinned copy was made.
cpu_tensor[0, 0] = -1
assert gpu_view[0, 0] == 0
# The view itself remains writable and independently usable.
gpu_view.mul_(2)
assert gpu_view[2, 3] == 46
assert gpu_view[9, 9] == 198
@pytest.mark.skipif(not is_uva_available(), reason="UVA is not available.")
@pytest.mark.skipif(
not current_platform.is_xpu(), reason="XPU non-contiguous UVA test."
)
@pytest.mark.parametrize("pinned", [False, True])
@pytest.mark.parametrize("device", DEVICES)
def test_non_contiguous_strided_view(device, pinned):
torch.set_default_device(device)
# Simulate scale_kn: a [32, 16] contiguous tensor
scale_kn = torch.arange(
512, dtype=torch.float32, device="cpu", pin_memory=pinned
).view(32, 16)
# Transposed view: shape [16, 32], non-contiguous stride (1, 16)
cpu_view = scale_kn.t()
assert cpu_view.shape == (16, 32)
assert cpu_view.stride() == (1, 16)
assert not cpu_view.is_contiguous()
gpu_view = get_accelerator_view_from_cpu_tensor(cpu_view)
assert gpu_view.device.type == DEVICE_TYPE
assert gpu_view.shape == (16, 32)
assert gpu_view.stride() == (1, 16)
assert not gpu_view.is_contiguous()
# Transposing tensor_view back should yield a contiguous [32, 16] tensor
gpu_view_t = gpu_view.t()
assert gpu_view_t.shape == (32, 16)
assert gpu_view_t.is_contiguous()
# Correctness check: values match original transposed CPU tensor
assert torch.equal(gpu_view.cpu(), cpu_view)
+115 -7
View File
@@ -27,6 +27,7 @@ from vllm.model_executor.layers.mhc import (
HAS_AITER_MHC,
HAS_AITER_MHC_FUSED,
HAS_AITER_MHC_FUSED_NORM,
HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM,
HAS_AITER_MHC_PRE_NORM,
HAS_TILELANG_MHC,
MHCFusedPostPreOp,
@@ -1300,18 +1301,109 @@ def test_mhc_pre_delayed_rocm_aiter(num_tokens, carried):
torch.testing.assert_close(actual[2], expected[2], atol=0, rtol=0)
@pytest.mark.skipif(
not (current_platform.is_rocm() and HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM),
reason="AITER fused delayed-seam kernel required (gfx950)",
)
@pytest.mark.parametrize("num_tokens", [1, 2, 7, 128, 1024])
@pytest.mark.parametrize("seam", ["post", "no_post", "identity_pre"])
def test_mhc_pre_delayed_rocm_aiter_fused_rms_norm(num_tokens, seam):
"""One fused kernel must match the whole reference seam: the post block
(the previous seam's post/comb gates fold ``sublayer_out`` into the
streams), the delayed pre block (projection + Sinkhorn producing the next
seam's post/comb/pre gates, plus the collapse with the carried pre-mix),
and the RMSNorm folded onto the collapse.
``num_tokens`` spans the kernel's split-K config buckets. ``seam`` covers
the three call shapes the model makes: a regular seam (post folded in), an
Engram seam (post already applied), and the draft entry seam
(``pre_mix=None`` = identity pre-mix).
"""
set_random_seed(0)
hc_mult, hidden_size = 4, 5120
residual, fn, hc_scale, hc_base, norm_weight = _rocm_mhc_inputs(
num_tokens=num_tokens, hidden_size=hidden_size, hc_mult=hc_mult
)
rms_eps = hc_pre_eps = hc_sinkhorn_eps = norm_eps = 1e-6
args = (
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
1.0,
20,
)
pre_mix = (
None
if seam == "identity_pre"
else torch.rand(num_tokens, hc_mult, dtype=torch.float32, device=DEVICE) + 0.5
)
post_kwargs = {}
residual_ref = residual
if seam == "post":
sublayer_out = torch.randn(
num_tokens, hidden_size, dtype=torch.bfloat16, device=DEVICE
)
post_layer_mix = 2 * torch.rand(
num_tokens, hc_mult, 1, dtype=torch.float32, device=DEVICE
)
comb_res_mix = torch.softmax(
torch.randn(num_tokens, hc_mult, hc_mult, device=DEVICE), dim=-1
)
post_kwargs = dict(
sublayer_out=sublayer_out,
post_layer_mix=post_layer_mix,
comb_res_mix=comb_res_mix,
)
residual_ref = mhc_post_torch(
sublayer_out, residual, post_layer_mix, comb_res_mix
)
expected = mhc_pre_delayed_torch(residual_ref, *args[1:], pre_mix=pre_mix)
expected_layer_input = F.rms_norm(
expected[2], (hidden_size,), norm_weight, norm_eps
)
residual_out, *actual = object.__new__(MHCPreDelayedOp).forward_hip(
*args,
pre_mix=pre_mix,
norm_weight=norm_weight,
norm_eps=norm_eps,
**post_kwargs,
)
if seam == "post":
torch.testing.assert_close(residual_out, residual_ref)
else:
# No post was requested, so the residual comes straight back.
assert residual_out is residual
# Gates: post_mix, comb_mix, next_pre_mix (bf16 MFMA vs the fp32 reference).
for i in (0, 1, 3):
torch.testing.assert_close(actual[i], expected[i], atol=5e-4, rtol=1e-3)
# The collapse is bf16 and the kernel projects the once-rounded R', so
# near-zero values need an absolute tolerance (same bound as aiter's UT).
torch.testing.assert_close(actual[2], expected_layer_input, atol=2e-2, rtol=1e-2)
@pytest.mark.skipif(
not (current_platform.is_rocm() and HAS_AITER_MHC),
reason="AITER mHC required",
)
def test_mhc_pre_delayed_rocm_aiter_declines_unsupported(monkeypatch):
"""The broadcast seam and a fused norm must not take the AITER path.
"""The broadcast seam must not take the AITER path; a fused norm must not
take the unfused AITER op.
Neither is expressible with AITER's pre kernels: the broadcast projects a
narrower ``x``, and ``mhc_pre_gemm_sqrsum`` folds no RMSNorm. What is
asserted here is the routing decision, which is what this gate owns; the
numerics of whichever fallback it lands on are covered by
``test_deepseek_v41_mhc_pre_delayed``.
The broadcast projects a narrower ``x``, which no AITER pre kernel takes;
``mhc_pre_gemm_sqrsum`` folds no RMSNorm, so a fused norm routes to the
fused seam kernel where it exists (gfx950) and away from AITER otherwise.
What is asserted here is the routing decision, which is what this gate
owns; the numerics of the paths are covered by
``test_deepseek_v41_mhc_pre_delayed`` and
``test_mhc_pre_delayed_rocm_aiter_fused_rms_norm``.
"""
set_random_seed(0)
hc_mult, hidden_size = 4, 5120
@@ -1331,6 +1423,18 @@ def test_mhc_pre_delayed_rocm_aiter_declines_unsupported(monkeypatch):
monkeypatch.setattr(torch.ops.vllm, "mhc_pre_delayed_aiter", spy)
fused_op = torch.ops.vllm.mhc_fused_post_pre_delayed_rms_norm_aiter
took_fused = False
def fused_spy(*spy_args, **spy_kwargs):
nonlocal took_fused
took_fused = True
return fused_op(*spy_args, **spy_kwargs)
monkeypatch.setattr(
torch.ops.vllm, "mhc_fused_post_pre_delayed_rms_norm_aiter", fused_spy
)
x = residual[:, 0].contiguous()
broadcast_fn = fn.view(-1, hc_mult, hidden_size).sum(1)
broadcast_residual = x.unsqueeze(1).expand(-1, hc_mult, -1).contiguous()
@@ -1342,11 +1446,15 @@ def test_mhc_pre_delayed_rocm_aiter_declines_unsupported(monkeypatch):
)
assert residual_out is broadcast_residual
assert not took_aiter, "the broadcast seam must not reach AITER"
assert not took_fused, "the broadcast seam must not reach AITER"
for i in range(4):
torch.testing.assert_close(actual[i], expected[i], atol=1e-4, rtol=1e-3)
# A fused norm must not reach the unfused AITER op; where the fused seam
# kernel exists (gfx950) it routes there instead.
op.forward_hip(*args, norm_weight=norm_weight, norm_eps=1e-6)
assert not took_aiter, "a fused norm must not reach AITER"
assert not took_aiter, "a fused norm must not reach the unfused AITER op"
assert took_fused == mhc_layers.HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM
# Positive control: the same inputs without a norm do take the AITER path,
# so the two declines above are the gate discriminating rather than the
@@ -240,6 +240,29 @@ def _make_awq_moe_weights(E, K, N, group_size, asym=False):
)
def _canonicalize_gptq_moe_inputs(
w13: torch.Tensor,
w2: torch.Tensor,
w13_scale: torch.Tensor,
w2_scale: torch.Tensor,
w13_qzeros: torch.Tensor | None,
w2_qzeros: torch.Tensor | None,
):
"""Convert GPTQ K-first tensors to the oracle's canonical N-first layout."""
def transpose(value: torch.Tensor | None) -> torch.Tensor | None:
return value.transpose(1, 2).contiguous() if value is not None else None
return (
w13.transpose(1, 2).contiguous(),
w2.transpose(1, 2).contiguous(),
w13_scale.transpose(1, 2).contiguous(),
w2_scale.transpose(1, 2).contiguous(),
transpose(w13_qzeros),
transpose(w2_qzeros),
)
def _run_emulation_forward(
experts, w13_bf16, w2_bf16, hidden_states, topk_weights, topk_ids, E, K, N
):
@@ -478,7 +501,8 @@ def test_gptq_process_weights_shapes_and_values(E, K, N, group_size, asym):
w13, w13s, w13z, w13_ref, w2, w2s, w2z, w2_ref = _make_gptq_moe_weights(
E, K, N, group_size, asym
)
result = _process_weights_emulation_gptq(w13, w2, w13s, w2s, w13z, w2z)
gptq_inputs = _canonicalize_gptq_moe_inputs(w13, w2, w13s, w2s, w13z, w2z)
result = _process_weights_emulation_gptq(*gptq_inputs)
w13_out, w2_out = result[0], result[1]
assert w13_out.shape == (E, 2 * N, K)
@@ -561,7 +585,7 @@ def test_gptq_awq_process_weights_agree(E, K, N, group_size, asym):
g2s_list.append(s2)
a2s_list.append(s2)
gptq_res = _process_weights_emulation_gptq(
gptq_inputs = _canonicalize_gptq_moe_inputs(
torch.stack(g13_list),
torch.stack(g2_list),
torch.stack(g13s_list),
@@ -569,6 +593,7 @@ def test_gptq_awq_process_weights_agree(E, K, N, group_size, asym):
torch.stack(g13z_list) if asym else None,
torch.stack(g2z_list) if asym else None,
)
gptq_res = _process_weights_emulation_gptq(*gptq_inputs)
awq_res = _process_weights_emulation_awq(
torch.stack(a13_list),
torch.stack(a2_list),
@@ -615,7 +640,7 @@ def test_gptq_vs_awq_forward_agree(E, K, N, top_k, group_size, num_tokens):
g2s_list.append(s2)
a2s_list.append(s2.clone())
gptq_res = _process_weights_emulation_gptq(
gptq_inputs = _canonicalize_gptq_moe_inputs(
torch.stack(g13_list),
torch.stack(g2_list),
torch.stack(g13s_list),
@@ -623,6 +648,7 @@ def test_gptq_vs_awq_forward_agree(E, K, N, top_k, group_size, num_tokens):
None,
None,
)
gptq_res = _process_weights_emulation_gptq(*gptq_inputs)
awq_res = _process_weights_emulation_awq(
torch.stack(a13_list),
torch.stack(a2_list),
@@ -750,7 +776,7 @@ def test_ep_output_matches_no_ep(E, K, N, top_k, group_size, num_tokens, ep_size
if fmt == "gptq"
else _process_weights_emulation_awq
)
res = process_fn(
process_inputs = (
torch.stack(packed13_list),
torch.stack(packed2_list),
torch.stack(scales13_list),
@@ -758,6 +784,9 @@ def test_ep_output_matches_no_ep(E, K, N, top_k, group_size, num_tokens, ep_size
None,
None,
)
if fmt == "gptq":
process_inputs = _canonicalize_gptq_moe_inputs(*process_inputs)
res = process_fn(*process_inputs)
w13_all, w2_all = res[0], res[1] # [E, 2N, K], [E, K, N]
hidden_states = torch.randn(num_tokens, K, dtype=torch.bfloat16, device=device)
@@ -855,7 +884,7 @@ def test_ep_gptq_awq_agree(E, K, N, top_k, group_size, num_tokens, ep_size):
g2s_list.append(s2)
a2s_list.append(s2.clone())
gptq_res = _process_weights_emulation_gptq(
gptq_inputs = _canonicalize_gptq_moe_inputs(
torch.stack(g13_list),
torch.stack(g2_list),
torch.stack(g13s_list),
@@ -863,6 +892,7 @@ def test_ep_gptq_awq_agree(E, K, N, top_k, group_size, num_tokens, ep_size):
None,
None,
)
gptq_res = _process_weights_emulation_gptq(*gptq_inputs)
awq_res = _process_weights_emulation_awq(
torch.stack(a13_list),
torch.stack(a2_list),
@@ -954,7 +984,7 @@ def test_ep_partial_rank_no_active_experts(
scales13_list.append(s13)
scales2_list.append(s2)
res = _process_weights_emulation_gptq(
gptq_inputs = _canonicalize_gptq_moe_inputs(
torch.stack(packed13_list),
torch.stack(packed2_list),
torch.stack(scales13_list),
@@ -962,6 +992,7 @@ def test_ep_partial_rank_no_active_experts(
None,
None,
)
res = _process_weights_emulation_gptq(*gptq_inputs)
w13_all, w2_all = res[0], res[1]
# Force topk_ids to only use experts in [0, num_local) — rank 0's slice
@@ -1029,7 +1060,7 @@ def test_ep_sum_equals_full_forward(E, K, N, top_k, group_size, num_tokens, ep_s
scales13_list.append(s13)
scales2_list.append(s2)
res = _process_weights_emulation_gptq(
gptq_inputs = _canonicalize_gptq_moe_inputs(
torch.stack(packed13_list),
torch.stack(packed2_list),
torch.stack(scales13_list),
@@ -1037,6 +1068,7 @@ def test_ep_sum_equals_full_forward(E, K, N, top_k, group_size, num_tokens, ep_s
None,
None,
)
res = _process_weights_emulation_gptq(*gptq_inputs)
w13_all, w2_all = res[0], res[1]
# Fix routing so every token uses exactly 2 consecutive experts (round-robin)
@@ -1143,7 +1175,7 @@ def test_emulation_output_close_to_bf16_reference(
if fmt == "gptq"
else _process_weights_emulation_awq
)
res = process_fn(
process_inputs = (
torch.stack(packed13_list),
torch.stack(packed2_list),
torch.stack(scales13_list),
@@ -1151,6 +1183,9 @@ def test_emulation_output_close_to_bf16_reference(
None,
None,
)
if fmt == "gptq":
process_inputs = _canonicalize_gptq_moe_inputs(*process_inputs)
res = process_fn(*process_inputs)
w13_bf16, w2_bf16 = res[0], res[1]
dummy_scale = torch.ones(1, dtype=torch.float16, device=device)
+2 -1
View File
@@ -120,12 +120,13 @@ def _test_batched_loras(
model=MODEL_NAME,
runner="pooling",
convert="classify",
hf_overrides={"num_labels": 2},
hf_overrides={"num_labels": 1},
dtype="float16",
enable_lora=True,
max_loras=2,
max_cpu_loras=2,
max_lora_rank=16,
max_lora_cls_labels=2,
enforce_eager=True,
max_model_len=512,
gpu_memory_utilization=0.5,
@@ -23,6 +23,7 @@ from vllm.distributed import get_tp_group
from vllm.model_executor.layers.fused_moe.runner import moe_runner
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.models.kimi_k3.amd import latent_moe_runner
from vllm.models.kimi_k3.amd.latent_moe_runner import ROCmLatentMoERunner
from vllm.models.kimi_k3.amd.linear import KimiRoutedOutputTransform
from vllm.platforms import current_platform
@@ -56,21 +57,26 @@ def _build_transform(device: torch.device) -> KimiRoutedOutputTransform:
def _tail_runner(
transform: KimiRoutedOutputTransform, tp_size: int
transform: KimiRoutedOutputTransform, tp_world: int, use_ep: bool = False
) -> ROCmLatentMoERunner:
"""A runner carrying only what the tail reads, so no engine is needed.
``_maybe_reduce_final_output`` is left as the real base-class method, so the
all-reduce that stitches the shards is the real collective.
``tp_world`` is the size of the TP process group, which is what the hidden
dim is split by. Under expert parallelism the MoE config reports
``tp_size=1``/``ep_size=tp_world`` for that same group, so ``use_ep`` pins
that the final reduce still fires.
"""
runner = object.__new__(ROCmLatentMoERunner)
attrs = {
"routed_output_transform": transform,
"_up_proj_shard_size": HIDDEN_SIZE // tp_size,
"_up_proj_shard_size": HIDDEN_SIZE // tp_world,
"_logged_sharded_tail": False,
"moe_config": SimpleNamespace(
tp_size=tp_size,
ep_size=1,
tp_size=1 if use_ep else tp_world,
ep_size=tp_world if use_ep else 1,
is_sequence_parallel=False,
skip_final_all_reduce=False,
),
@@ -94,9 +100,11 @@ def _rank_partials(
return routed.mul_(0.01), shared
def _check_matches_replicated(device: torch.device, tp_size: int, rank: int) -> None:
def _check_matches_replicated(
device: torch.device, tp_world: int, rank: int, use_ep: bool = False
) -> None:
transform = _build_transform(device)
runner = _tail_runner(transform, tp_size)
runner = _tail_runner(transform, tp_world, use_ep)
group = get_tp_group().device_group
for iteration, num_tokens in enumerate((1, 5, 8, 16, 5)):
@@ -119,13 +127,22 @@ def _check_matches_replicated(device: torch.device, tp_size: int, rank: int) ->
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
def _check_matches_replicated_under_ep(
device: torch.device, tp_world: int, rank: int
) -> None:
"""The served config: the routed experts are expert-parallel, so the MoE
config reports no tensor parallelism at all for a group that still splits
the hidden dim ``tp_world`` ways."""
_check_matches_replicated(device, tp_world, rank, use_ep=True)
def _check_writes_only_its_own_shard(
device: torch.device, tp_size: int, rank: int
device: torch.device, tp_world: int, rank: int
) -> None:
"""Two ranks that swapped both weight rows and write offsets still sum to
the right total, so inspect each slice before the final collective."""
transform = _build_transform(device)
runner = _tail_runner(transform, tp_size)
runner = _tail_runner(transform, tp_world)
group = get_tp_group().device_group
torch.manual_seed(rank + 1)
@@ -149,7 +166,7 @@ def _check_writes_only_its_own_shard(
assert captured["output_is_reduced"] is False
shard = HIDDEN_SIZE // tp_size
shard = HIDDEN_SIZE // tp_world
start, end = rank * shard, (rank + 1) * shard
local = captured["states"]
projected = F.linear(latent, transform.up_proj.weight[start:end])
@@ -163,6 +180,7 @@ def _check_writes_only_its_own_shard(
_CHECKS = {
"matches_replicated": _check_matches_replicated,
"matches_replicated_ep": _check_matches_replicated_under_ep,
"own_shard_only": _check_writes_only_its_own_shard,
}
@@ -196,6 +214,11 @@ def test_sharded_tail_tp8_matches_replicated_projection() -> None:
_run_ranks("matches_replicated", 8)
@multi_gpu_test(num_gpus=8)
def test_sharded_tail_tp8_ep8_matches_replicated_projection() -> None:
_run_ranks("matches_replicated_ep", 8)
@multi_gpu_test(num_gpus=4)
def test_sharded_tail_tp4_writes_only_its_own_shard() -> None:
_run_ranks("own_shard_only", 4)
@@ -241,12 +264,19 @@ def test_forward_shards_only_when_the_tail_is_valid(
@pytest.fixture
def build_runner(monkeypatch: pytest.MonkeyPatch):
"""Construct through the real subclass ``__init__``, stubbing only the base."""
"""Construct through the real subclass ``__init__``, stubbing only the base.
``tp_world`` is the size of the TP process group and is varied independently
of ``moe_config.tp_size``, because under expert parallelism those two are
different numbers for the same eight ranks.
"""
world = {"size": 8}
def _base_init(
self,
*,
tp_size: int = 8,
ep_size: int = 1,
hidden: int = HIDDEN_SIZE,
is_sequence_parallel: bool = False,
has_up_proj: bool = True,
@@ -254,7 +284,9 @@ def build_runner(monkeypatch: pytest.MonkeyPatch):
routed_scaling_factor: float = 1.0,
) -> None:
self.moe_config = SimpleNamespace(
tp_size=tp_size, is_sequence_parallel=is_sequence_parallel
tp_size=tp_size,
ep_size=ep_size,
is_sequence_parallel=is_sequence_parallel,
)
self.routed_output_transform = SimpleNamespace(
norm=None,
@@ -266,20 +298,63 @@ def build_runner(monkeypatch: pytest.MonkeyPatch):
self._shared_experts = object() if has_shared_experts else None
monkeypatch.setattr(moe_runner.MoERunner, "__init__", _base_init)
return ROCmLatentMoERunner
monkeypatch.setattr(
latent_moe_runner,
"get_tensor_model_parallel_world_size",
lambda: world["size"],
)
def _build(*, tp_world: int = 8, **kwargs) -> ROCmLatentMoERunner:
world["size"] = tp_world
return ROCmLatentMoERunner(**kwargs)
return _build
def test_shards_under_the_kimi_k3_serving_config(build_runner) -> None:
runner = build_runner()
"""``--tensor-parallel-size 8``, MoE experts tensor-parallel."""
runner = build_runner(tp_world=8, tp_size=8, ep_size=1)
assert runner._tail_shardable
assert runner._up_proj_shard_size == HIDDEN_SIZE // 8
def test_shards_under_the_kimi_k3_serving_config_with_expert_parallel(
build_runner,
) -> None:
"""``--tensor-parallel-size 8 --enable-expert-parallel``, what is served.
EP reports ``moe_config.tp_size == 1`` because each rank owns whole experts,
but the hidden dim is still split across all eight TP ranks, so the tail
must shard exactly as it does without EP.
"""
runner = build_runner(tp_world=8, tp_size=1, ep_size=8)
assert runner._tail_shardable
assert runner._up_proj_shard_size == HIDDEN_SIZE // 8
def test_shards_by_the_process_group_not_the_dp_flattened_tp_size(
build_runner,
) -> None:
"""Without EP, ``moe_config.tp_size`` is flattened across DP: TP=2 x DP=2
reports 4 for a TP process group of 2.
Only those 2 ranks share a hidden dim and take part in the final
all-reduce, so sharding by 4 would leave half of it with no routed
contribution on either rank.
"""
runner = build_runner(tp_world=2, tp_size=4, ep_size=1)
assert runner._tail_shardable
assert runner._up_proj_shard_size == HIDDEN_SIZE // 2
@pytest.mark.parametrize(
"override",
[
pytest.param({"tp_size": 1}, id="no-tp"),
pytest.param({"tp_world": 1, "tp_size": 1}, id="single-rank"),
pytest.param({"tp_world": 1, "tp_size": 4}, id="dp-without-tp"),
pytest.param({"hidden": HIDDEN_SIZE + 2}, id="hidden-not-divisible-by-tp"),
pytest.param({"has_up_proj": False}, id="no-up-proj"),
pytest.param({"has_shared_experts": False}, id="no-shared-partial"),
@@ -290,7 +365,12 @@ def test_shards_under_the_kimi_k3_serving_config(build_runner) -> None:
def test_falls_back_when_the_config_breaks_the_shard(
build_runner, override: dict
) -> None:
"""Each of these makes the sharded tail wrong, not merely slower."""
"""Each of these makes the sharded tail wrong, not merely slower.
``dp-without-tp`` is DP=4 with no TP: the hidden dim is not split at all,
so there is nothing to shard and no all-reduce to stitch shards back
together, however large ``moe_config.tp_size`` claims to be.
"""
runner = build_runner(**override)
assert not runner._tail_shardable
@@ -16,7 +16,9 @@ from vllm import LLM, SamplingParams
from vllm.assets.audio import AudioAsset
from vllm.config import CUDAGraphMode
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.engine.protocol import StreamingInput
from vllm.platforms import current_platform
from vllm.renderers.inputs.preprocess import parse_model_prompt
from vllm.utils.math_utils import cdiv
from vllm.v1.engine.async_llm import AsyncLLM
from vllm.v1.kv_cache_interface import SlidingWindowSpec
@@ -222,6 +224,14 @@ async def test_voxtral_realtime_generator(audio_assets, tokenizer, async_engine)
sampling_params = SamplingParams(temperature=0.0, max_tokens=1)
audio_config = tokenizer.instruct_tokenizer.audio_encoder.audio_config
async def input_stream(audio_buffer):
async for prompt in audio_buffer.get_input_stream():
parsed_prompt = parse_model_prompt(async_engine.model_config, prompt)
(engine_input,) = await async_engine.renderer.render_cmpl_async(
[parsed_prompt]
)
yield StreamingInput(prompt=engine_input)
output_tokens_list = []
for i, audio_asset in enumerate(audio_assets):
output_tokens = []
@@ -241,7 +251,7 @@ async def test_voxtral_realtime_generator(audio_assets, tokenizer, async_engine)
request_id = f"session-{i}"
async for resp in async_engine.generate(
prompt=buffer.get_input_stream(),
prompt=input_stream(buffer),
sampling_params=sampling_params,
request_id=request_id,
):
@@ -34,6 +34,7 @@ def vqa_ppl_test(
mm_processor_kwargs = {}
processor = AutoProcessor.from_pretrained(model_info.name)
image_token_id = processor.image_token_id
images = []
prompts = []
@@ -99,8 +100,10 @@ def vqa_ppl_test(
for token_data in token_datas[1:]:
assert token_data is not None
assert len(token_data) == 1
token_log_prob = list(token_data.values())[0].logprob
token_log_probs.append(token_log_prob)
token_id, token_log_prob = next(iter(token_data.items()))
if token_id == image_token_id:
continue
token_log_probs.append(token_log_prob.logprob)
neg_log_likelihood = -torch.tensor(
token_log_probs, dtype=torch.float32, device="cpu"
@@ -132,11 +135,12 @@ def vqa_ppl_test(
inputs = inputs.to("cuda")
input_ids = inputs["input_ids"]
outputs = hf_model.model(**inputs, labels=input_ids)
labels = input_ids.masked_fill(input_ids == image_token_id, -100)
outputs = hf_model.model(**inputs, labels=labels)
neg_log_likelihood = outputs.loss
neg_log_likelihood = neg_log_likelihood.to(torch.float32).cpu()
num_loss_tokens = input_ids.shape[1] - 1
num_loss_tokens = int((labels[:, 1:] != -100).sum())
nll_sum += neg_log_likelihood * num_loss_tokens
n_tokens += num_loss_tokens
@@ -8,26 +8,51 @@ from tests.models.utils import GenerateModelInfo
from .ppl_utils import vqa_ppl_test
MODELS = [
GenerateModelInfo("Qwen/Qwen2-VL-2B-Instruct", hf_ppl=41081356.0),
GenerateModelInfo("Qwen/Qwen2.5-VL-3B-Instruct", hf_ppl=18330016.0),
(
GenerateModelInfo("Qwen/Qwen2-VL-2B-Instruct", hf_ppl=9.810060501098633),
28,
{},
),
(
GenerateModelInfo("Qwen/Qwen2.5-VL-3B-Instruct", hf_ppl=22.17719078063965),
28,
{},
),
(
GenerateModelInfo("Qwen/Qwen3-VL-4B-Instruct", hf_ppl=11.916606903076172),
32,
{},
),
(
GenerateModelInfo("Qwen/Qwen3.5-4B", hf_ppl=6.478141784667969),
32,
{"enable_prefix_caching": False},
),
]
mm_processor_kwargs = {
"min_pixels": 28 * 28,
"max_pixels": 1280 * 28 * 28,
}
def _pixel_factor(factor: int):
"""`factor = patch_size * merge_size` for Qwen models"""
return {"min_pixels": factor**2, "max_pixels": 1280 * factor**2}
@pytest.mark.parametrize("model_info", MODELS)
@pytest.mark.parametrize("model_info,factor,extra_kwargs", MODELS)
@pytest.mark.parametrize("mm_device_do_normalize", [True, False])
def test_ppl(
hf_runner, vllm_runner, model_info: GenerateModelInfo, mm_device_do_normalize: bool
hf_runner,
vllm_runner,
model_info: GenerateModelInfo,
factor: int,
extra_kwargs: dict,
mm_device_do_normalize: bool,
):
vqa_ppl_test(
hf_runner,
vllm_runner,
model_info,
vllm_extra_kwargs={"mm_device_do_normalize": mm_device_do_normalize},
mm_processor_kwargs=mm_processor_kwargs,
vllm_extra_kwargs={
"mm_device_do_normalize": mm_device_do_normalize,
**extra_kwargs,
},
mm_processor_kwargs=_pixel_factor(factor),
)
@@ -202,19 +202,19 @@ def test_get_prompt_updates_respects_nested_max_soft_tokens(model_id: str):
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
@pytest.mark.parametrize("kwargs_on_init", [False, True])
@pytest.mark.parametrize(
"image_kwargs", [{"rescale_factor": 1 / 127.5}, {"max_soft_tokens": 560}]
)
@pytest.mark.parametrize("video_uuid", [None, "same-video"])
def test_video_cache_is_independent_of_image_kwargs(
model_id: str,
kwargs_on_init: bool,
image_kwargs: dict[str, object],
video_uuid: str | None,
):
"""Image overrides must not change video frames or depend on cache warmth."""
kwargs = {"images_kwargs": image_kwargs}
"""Request image overrides must not change video frames or make results
depend on cache warmth.
"""
request_kwargs = {"images_kwargs": image_kwargs}
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1, "video": 1},
@@ -239,18 +239,11 @@ def test_video_cache_is_independent_of_image_kwargs(
cache=cache,
)
baseline = process({}, cache=None)
if kwargs_on_init:
ctx = build_model_context(
model_id,
mm_processor_kwargs=kwargs,
limit_mm_per_prompt={"image": 1, "video": 1},
mm_processor_cache_gb=1,
)
cache = MultiModalProcessorOnlyCache(ctx.model_config)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
request_kwargs = {} if kwargs_on_init else kwargs
process(request_kwargs, cache=cache)
# Populate the video cache without image overrides, then change only
# images_kwargs on the next request. Since images_kwargs are excluded from
# the video hash, and must not affect the processed video, the cached result
# must match a fresh recomputation.
baseline = process({}, cache=cache)
cached = process(request_kwargs, cache=cache)
cache.clear_cache()
fresh = process(request_kwargs, cache=cache)
@@ -100,6 +100,11 @@ def _expected_placeholder_tokens_per_image(
[
({}, (448, 448), 256),
({"size": {"longest_edge": 1008}}, (1540, 1540), 1296),
(
{"images_kwargs": {"size": {"longest_edge": 1008}}},
(1540, 1540),
1296,
),
({"size": {"longest_edge": 1288}}, (1536, 1187), 1656),
({"size": {"longest_edge": 1008}}, (29, 29), 4),
({"size": {"longest_edge": 1000}}, (1540, 1700), 1188),
@@ -148,6 +153,40 @@ def test_processor_size_override(
assert prompt_update_tokens == hf_placeholder_tokens
def test_scoped_request_size_overrides_configured_flat_size():
ctx = build_model_context(
_MODEL_ID,
mm_processor_kwargs={"size": {"longest_edge": 1008}},
limit_mm_per_prompt={"image": 1},
model_config_kwargs=_MODEL_CONFIG_KWARGS,
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor = AutoProcessor.from_pretrained(
_MODEL_ID,
fix_mistral_regex=True,
)
hf_processor_mm_kwargs: dict[str, object] = {
"images_kwargs": {"size": {"longest_edge": 1288}}
}
image = Image.new("RGB", (1536, 1187), color=(127, 127, 127))
merged_mm_kwargs = processor.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
pixel_values, hf_placeholder_tokens = _process_images_with_hf(
hf_processor, [image], merged_mm_kwargs
)
prompt_update_tokens = _placeholder_tokens_from_prompt_updates(
processor, [image], pixel_values, hf_processor_mm_kwargs
)
expected_from_pixel_values = _expected_placeholder_tokens_per_image(
hf_processor, pixel_values[0]
)
assert expected_from_pixel_values == 1656
assert hf_placeholder_tokens.count(hf_processor.image_token_id) == 1656
assert prompt_update_tokens == hf_placeholder_tokens
def test_lightonocr_keeps_vision_config_image_size():
ctx = build_model_context(
_LIGHTON_MODEL_ID,
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from copy import deepcopy
from types import SimpleNamespace
from typing import Any
import numpy as np
@@ -8,16 +10,238 @@ import pytest
import torch
from PIL import Image
from vllm.config.multimodal import MultiModalConfig
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.model_executor.models.qwen2_vl import Qwen2VLProcessingInfo
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
from vllm.multimodal.inputs import batched_tensors_equal
from vllm.multimodal.processing.context import InputProcessingContext
from vllm.platforms import current_platform
from ....conftest import ImageTestAssets
from ...utils import build_model_context
@pytest.mark.parametrize(
("mm_kwargs", "expected"),
[
(
{"size": {"shortest_edge": 64, "longest_edge": 1024}},
{
"size": {"shortest_edge": 64, "longest_edge": 1024},
"min_pixels": 64,
"max_pixels": 1024,
},
),
(
{"min_pixels": 64, "max_pixels": 1024},
{
"min_pixels": 64,
"max_pixels": 1024,
"size": {"shortest_edge": 64, "longest_edge": 1024},
},
),
(
{
"size": {"shortest_edge": 64, "longest_edge": 1024},
"min_pixels": 128,
"max_pixels": 2048,
},
{
"size": {"shortest_edge": 64, "longest_edge": 1024},
"min_pixels": 128,
"max_pixels": 2048,
},
),
(
{
"min_pixels": None,
"size": {"shortest_edge": 0},
"images_kwargs": {"max_pixels": 4096},
"videos_kwargs": {"size": {"longest_edge": 8192}},
},
{
"min_pixels": None,
"size": {"shortest_edge": 0},
"images_kwargs": {
"max_pixels": 4096,
"size": {"longest_edge": 4096},
},
"videos_kwargs": {
"size": {"longest_edge": 8192},
"max_pixels": 8192,
},
},
),
(
{"min_pixels": 0},
{"min_pixels": 0, "size": {"shortest_edge": 0}},
),
(
{"size": None, "min_pixels": 64},
{"size": {"shortest_edge": 64}, "min_pixels": 64},
),
],
)
def test_complete_mm_processor_size_aliases(
mm_kwargs: dict[str, object],
expected: dict[str, object],
) -> None:
original = deepcopy(mm_kwargs)
actual = Qwen2VLProcessingInfo._complete_mm_processor_size_aliases(mm_kwargs)
assert actual == expected
assert mm_kwargs == original
def test_merge_and_resolve_mm_processor_kwargs_preserves_request_alias_precedence(
monkeypatch: pytest.MonkeyPatch,
) -> None:
mm_config = MultiModalConfig(
mm_processor_kwargs={
"images_kwargs": {"min_pixels": 64},
},
mm_device_do_normalize=False,
)
model_config = SimpleNamespace(get_multimodal_config=lambda: mm_config)
ctx = InputProcessingContext(model_config, tokenizer=None) # type: ignore[arg-type]
info = Qwen2VLProcessingInfo(ctx)
monkeypatch.setattr(
info,
"get_supported_mm_processor_kwargs",
lambda: {"images_kwargs": {"size", "min_pixels"}},
)
assert info._merge_and_resolve_mm_processor_kwargs(
{"size": {"shortest_edge": 128}}
) == {
"images_kwargs": {
"size": {"shortest_edge": 128},
"min_pixels": 128,
},
}
@pytest.mark.parametrize(
("mm_kwargs", "default_size", "expected"),
[
(
{},
{"shortest_edge": 32, "longest_edge": 1024},
{"shortest_edge": 32, "longest_edge": 1024},
),
(
{"size": {"shortest_edge": 64}},
{"shortest_edge": 32, "longest_edge": 1024},
{"shortest_edge": 64, "longest_edge": 1024},
),
(
{
"size": {"shortest_edge": 64, "longest_edge": 2048},
"min_pixels": 128,
"max_pixels": 4096,
},
{"shortest_edge": 32, "longest_edge": 1024},
{"shortest_edge": 128, "longest_edge": 4096},
),
(
{
"size": {"shortest_edge": 64},
"max_pixels": 4096,
},
None,
{"shortest_edge": 64, "longest_edge": 4096},
),
(
{
"size": {"shortest_edge": 64, "longest_edge": 2048},
"min_pixels": None,
"max_pixels": 0,
},
{"shortest_edge": 32, "longest_edge": 1024},
{"shortest_edge": 64, "longest_edge": 0},
),
],
)
def test_get_vision_size(
mm_kwargs: dict[str, object],
default_size: dict[str, object] | None,
expected: dict[str, object],
) -> None:
original_mm_kwargs = {
key: dict(value) if isinstance(value, dict) else value
for key, value in mm_kwargs.items()
}
original_default_size = dict(default_size) if default_size is not None else None
actual = Qwen2VLProcessingInfo._get_vision_size(
mm_kwargs,
default_size=default_size,
)
assert actual == expected
assert mm_kwargs == original_mm_kwargs
assert default_size == original_default_size
@pytest.mark.parametrize(
("modality", "scope"),
[
("image", "images_kwargs"),
("video", "videos_kwargs"),
],
)
def test_get_vision_info_reads_merged_modality_scope(
monkeypatch: pytest.MonkeyPatch,
modality: str,
scope: str,
) -> None:
vision_config = SimpleNamespace(
patch_size=14,
spatial_merge_size=2,
temporal_patch_size=2,
)
ctx = SimpleNamespace(
get_hf_config=lambda *_: SimpleNamespace(vision_config=vision_config)
)
info = Qwen2VLProcessingInfo(ctx)
merged = {
"images_kwargs": {"size": {"shortest_edge": 111}},
"videos_kwargs": {"size": {"shortest_edge": 222}},
}
monkeypatch.setattr(
info,
"_merge_and_resolve_mm_processor_kwargs",
lambda _: merged,
)
seen: list[dict[str, object]] = []
def get_size(
mm_kwargs: dict[str, object],
default_size: dict[str, object] | None = None,
) -> dict[str, object]:
seen.append(mm_kwargs)
return dict(default_size or {})
monkeypatch.setattr(info, "_get_vision_size", get_size)
image_processor = SimpleNamespace(size={"shortest_edge": 32, "longest_edge": 1024})
info._get_vision_info(
image_width=28,
image_height=28,
num_frames=2,
do_resize=False,
image_processor=image_processor,
mm_kwargs={},
modality=modality, # type: ignore[arg-type]
)
assert seen == [merged[scope]]
def test_jina_vl_processing_order() -> None:
"""Jina's document-first prompt keeps cached features and hashes aligned."""
ctx = build_model_context(
@@ -1,19 +1,21 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regression tests for Qwen3-VL processor.
Covers the fix for num_frames-based timestamp calculation
(issue vllm-project/vllm#35909).
"""
"""Regression tests for Qwen3-VL multimodal processing."""
from copy import deepcopy
from types import SimpleNamespace
from typing import Any
import numpy as np
import pytest
import torch
from transformers.models.qwen3_vl import Qwen3VLVideoProcessor
from vllm.config import ModelConfig
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.model_executor.models.qwen3_vl import Qwen3VLProcessingInfo
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.processing.context import BaseProcessingInfo
from ...registry import HF_EXAMPLE_MODELS
from ...utils import build_model_context
@@ -21,6 +23,32 @@ from ...utils import build_model_context
MODEL_ID = "Qwen/Qwen3-VL-4B-Instruct"
def test_qwen3_video_routing_schema_adds_vllm_compatibility_kwargs(
monkeypatch: pytest.MonkeyPatch,
) -> None:
native_video_kwargs = set(Qwen3VLVideoProcessor.valid_kwargs.__annotations__)
assert "size" in native_video_kwargs
assert "min_pixels" not in native_video_kwargs
assert "max_pixels" not in native_video_kwargs
info = Qwen3VLProcessingInfo(SimpleNamespace())
parent_supported = {
"text_kwargs": {"padding"},
"images_kwargs": {"size", "min_pixels", "max_pixels"},
"videos_kwargs": set(native_video_kwargs),
}
monkeypatch.setattr(
BaseProcessingInfo,
"get_supported_mm_processor_kwargs",
lambda self: {scope: set(kwargs) for scope, kwargs in parent_supported.items()},
)
supported = info.supported_mm_processor_kwargs
assert {"min_pixels", "max_pixels"} <= supported["videos_kwargs"]
assert parent_supported["videos_kwargs"] == native_video_kwargs
def _build_video_mm_data(
num_frames: int,
width: int = 128,
@@ -45,6 +73,122 @@ def _build_video_mm_data(
return {"video": [(video, metadata)]}
@pytest.mark.parametrize("model_id", [MODEL_ID])
def test_processor_partial_scoped_image_size(model_id: str) -> None:
"""Partial image ``size`` must be completed and applied before the HF call.
Keep video compatibility and per-video kwargs in an image-only call as well:
Qwen3 must sanitize the video scope before HF validates it. Regression for
#56363.
"""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1, "video": 0},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
image = np.zeros((128, 128, 3), dtype=np.uint8)
min_pixels = 256 * 32 * 32
default_size = dict(processor.info.get_image_processor().size)
complete_size = default_size | {"shortest_edge": min_pixels}
def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
mm_items = processor.info.parse_mm_data({"image": [image]})
return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
"image_grid_thw"
]
stock_grid = process({})
complete_grid = process({"images_kwargs": {"size": complete_size}})
partial_grid = process(
{
"images_kwargs": {
"size": {"shortest_edge": min_pixels},
},
"videos_kwargs": {
"min_pixels": 4096,
"max_pixels": 469_762_048,
"fps": [2.0],
"num_frames": [8],
},
}
)
assert torch.equal(partial_grid, complete_grid)
assert not torch.equal(partial_grid, stock_grid)
@pytest.mark.parametrize("model_id", [MODEL_ID])
def test_processor_partial_scoped_video_size(model_id: str) -> None:
"""Partial scoped video ``size`` must be completed and applied before HF."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 0, "video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
min_pixels = 512 * 32 * 32
default_size = dict(processor.info.get_video_processor().size)
complete_size = default_size | {"shortest_edge": min_pixels}
def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
mm_items = processor.info.parse_mm_data(_build_video_mm_data(num_frames=8))
return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
"video_grid_thw"
]
stock_grid = process({})
complete_grid = process({"videos_kwargs": {"size": complete_size}})
partial_grid = process(
{
"videos_kwargs": {
"size": {"shortest_edge": min_pixels},
},
}
)
assert torch.equal(partial_grid, complete_grid)
assert not torch.equal(partial_grid, stock_grid)
@pytest.mark.parametrize("model_id", [MODEL_ID])
def test_processor_flat_video_pixel_aliases_match_scoped_size(model_id: str) -> None:
"""Flat min/max pixel aliases keep their historical Qwen3 video effect."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 0, "video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
pixel_budget = 512 * 32 * 32
def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
mm_items = processor.info.parse_mm_data(_build_video_mm_data(num_frames=8))
return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
"video_grid_thw"
]
stock_grid = process({})
scoped_grid = process(
{
"videos_kwargs": {
"size": {
"shortest_edge": pixel_budget,
"longest_edge": pixel_budget,
},
},
}
)
flat_grid = process(
{
"min_pixels": pixel_budget,
"max_pixels": pixel_budget,
}
)
assert torch.equal(flat_grid, scoped_grid)
assert not torch.equal(flat_grid, stock_grid)
@pytest.mark.parametrize("model_id", [MODEL_ID])
@pytest.mark.parametrize(
"num_frames",
@@ -189,20 +333,21 @@ def test_processor_kwargs_videos_kwargs_does_not_leak_into_image_budget(
@pytest.mark.parametrize("model_id", [MODEL_ID])
@pytest.mark.parametrize(
"hf_mm_kwargs",
[{"num_frames": [8, 16]}, {"fps": [2.0, 4.0]}],
("hf_mm_kwargs", "scalar_kwargs"),
[
({"num_frames": [8, 16]}, [{"num_frames": 8}, {"num_frames": 16}]),
({"fps": [4.0, 12.0]}, [{"fps": 4.0}, {"fps": 12.0}]),
],
)
def test_processor_multi_video_list_kwargs(
model_id: str,
hf_mm_kwargs: dict[str, Any],
scalar_kwargs: list[dict[str, Any]],
) -> None:
"""Regression test: a multi-video request with list-valued per-video
``mm_processor_kwargs`` (one ``fps``/``num_frames`` per video) must not
crash.
"""List-valued video kwargs must be scalarized per item without mutation.
Before the fix, ``_apply_hf_processor_main`` copied the whole kwargs to every
video without slicing, so ``_get_video_second_idx`` received the list
where a scalar was expected and raised ``TypeError``.
Each video in the multi-video request must match processing the same input
independently with its corresponding scalar ``fps``/``num_frames`` value.
"""
ctx = build_model_context(
model_id,
@@ -216,17 +361,36 @@ def test_processor_multi_video_list_kwargs(
)
mm_data = {
"video": [
_build_video_mm_data(num_frames=16)["video"][0],
_build_video_mm_data(num_frames=32)["video"][0],
_build_video_mm_data(num_frames=32)["video"][0],
]
}
hf_mm_kwargs_before = deepcopy(hf_mm_kwargs)
processed = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=hf_mm_kwargs,
)
def process_single(hf_kwargs: dict[str, Any]) -> torch.Tensor:
single_prompt = "<|vision_start|><|video_pad|><|vision_end|>"
single_mm_data = _build_video_mm_data(num_frames=32)
single = processor(
single_prompt,
mm_items=processor.info.parse_mm_data(single_mm_data),
hf_processor_mm_kwargs=hf_kwargs,
)
return single["mm_kwargs"].get_data()["video_grid_thw"][0]
expected_grids = [process_single(kwargs) for kwargs in scalar_kwargs]
multi_grids = processed["mm_kwargs"].get_data()["video_grid_thw"]
assert not torch.equal(expected_grids[0], expected_grids[1])
assert torch.equal(multi_grids[0], expected_grids[0])
assert torch.equal(multi_grids[1], expected_grids[1])
assert hf_mm_kwargs == hf_mm_kwargs_before
video_phs = processed["mm_placeholders"].get("video", [])
assert len(video_phs) == 2, (
f"Expected exactly 2 video placeholders, got {len(video_phs)}"
@@ -336,7 +500,12 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non
capped_ctx = build_model_context(
model_id,
mm_processor_kwargs={"size": size, "cap_pixels_per_frame": True},
mm_processor_kwargs={
"videos_kwargs": {
"size": size,
"cap_pixels_per_frame": True,
},
},
limit_mm_per_prompt={"image": 0, "video": 1},
)
capped = MULTIMODAL_REGISTRY.create_processor(capped_ctx.model_config)
@@ -350,7 +519,7 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non
uncapped_ctx = build_model_context(
model_id,
mm_processor_kwargs={"size": size},
mm_processor_kwargs={"videos_kwargs": {"size": size}},
limit_mm_per_prompt={"image": 0, "video": 1},
)
uncapped = MULTIMODAL_REGISTRY.create_processor(uncapped_ctx.model_config)
+30
View File
@@ -4,11 +4,13 @@
import pytest
import torch
from vllm.models.qwen4_exp.nvidia.ops.cute_dsl.hc_down_silu import hc_down_silu
from vllm.models.qwen4_exp.nvidia.ops.hc import (
grouped_gemma_rmsnorm,
hc_combine,
hc_combine_norm,
hc_gate_mix,
hc_silu,
)
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
@@ -22,6 +24,13 @@ HC = 4
HIDDEN_SIZE = 2560
HYPER_HIDDEN_SIZE = HC * HIDDEN_SIZE
EPS = 1e-6
LORA_RANK = 320
DOWN_N = LORA_RANK + HC + 12 # merged down+inject weight, 16-row padded
requires_sm90 = pytest.mark.skipif(
not current_platform.has_device_capability(90),
reason="fused HC down+SiLU requires SM90+",
)
def test_grouped_gemma_rmsnorm() -> None:
@@ -125,3 +134,24 @@ def test_hc_combine_norm_unit_injection(num_tokens: int) -> None:
expected_norm = grouped_gemma_rmsnorm(expected, weight, EPS, HC)
assert torch.equal(actual, expected)
torch.testing.assert_close(actual_norm, expected_norm)
@requires_sm90
@pytest.mark.parametrize("num_tokens", [1, 3, 5, 17, 48])
def test_hc_down_silu_fused(num_tokens: int) -> None:
# Compare computed columns with the unfused ll_bf16 + hc_silu reference.
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import ll_bf16_gemm
torch.manual_seed(0)
x = torch.randn(num_tokens, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(DOWN_N, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
lora, injection = hc_down_silu(x, weight, LORA_RANK, HC)
down = ll_bf16_gemm(x, weight).to(torch.bfloat16)
torch.testing.assert_close(
lora, hc_silu(down[:, :LORA_RANK], HC), rtol=0.01, atol=0.01
)
torch.testing.assert_close(
injection, down[:, LORA_RANK : LORA_RANK + HC], rtol=0.01, atol=0.01
)
+125
View File
@@ -20,6 +20,9 @@ import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module
import vllm.model_executor.parameter as parameter_module
import vllm.models.qwen4_exp.common.ngram_embedding as ngram_embedding_module
import vllm.models.qwen4_exp.nvidia.ngram_embedding as nvidia_ngram_embedding_module
from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
from vllm.config import SpeculativeConfig, VllmConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.config.quantization import QuantizationConfigArgs
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.model_executor.layers.quantization.modelopt import (
@@ -50,8 +53,125 @@ from vllm.models.qwen4_exp.nvidia.ple_layer import Qwen4ExpPLELayer
from vllm.utils.torch_utils import weak_ref_tensor
from vllm.v1.attention.backends.short_conv_attn import (
PleShortConvAttentionMetadata,
PleShortConvAttentionMetadataBuilder,
)
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
from vllm.v1.kv_cache_interface import MambaSpec
@pytest.mark.parametrize("device", ["cpu", "cuda"])
@pytest.mark.parametrize("full_graph", [False, True])
@pytest.mark.parametrize(
"query_lens,drafts,spec_rows,non_spec_rows,token_order",
[
pytest.param([4, 2, 0], [3, 1, -1], [0, 1], [], None, id="spec-padding"),
pytest.param(
[3, 1, 2, 5, 0],
[2, -1, 1, -1, -1],
[0, 2],
[1, 3],
[0, 1, 2, 4, 5, 3, 6, 7, 8, 9, 10],
id="mixed-padding",
),
pytest.param([1, 5], [-1, -1], [], [0, 1], None, id="non-spec"),
],
)
def test_ple_metadata_preserves_request_and_token_order(
device, full_graph, query_lens, drafts, spec_rows, non_spec_rows, token_order
):
"""Preserve state routing, acceptance and prefill offsets without GPU sync."""
if device == "cuda" and not torch.cuda.is_available():
pytest.skip("CUDA required")
device = torch.device(device)
config = VllmConfig()
config.speculative_config = SpeculativeConfig(
method="ngram", num_speculative_tokens=3
)
config.compilation_config.cudagraph_mode = (
CUDAGraphMode.FULL if full_graph else CUDAGraphMode.NONE
)
config.compilation_config.max_cudagraph_capture_size = 32
builder = PleShortConvAttentionMetadataBuilder(
MambaSpec(block_size=16, shapes=((16, 64),), dtypes=(torch.float32,)),
["ple"],
config,
device,
)
common = create_common_attn_metadata(
BatchSpec(seq_lens=[q + 16 for q in query_lens], query_lens=query_lens),
16,
device,
arange_block_indices=True,
)
common.is_prefilling = torch.tensor(
[q > 1 and d < 0 for q, d in zip(query_lens, drafts)]
)
# Token padding is independent of the sum of per-request query lengths.
common.num_actual_tokens += 3
accepted = torch.arange(1, len(query_lens) + 1, dtype=torch.int32, device=device)
drafts_cpu = torch.tensor(drafts, dtype=torch.int32)
sync_mode = torch.cuda.get_sync_debug_mode() if device.type == "cuda" else None
if sync_mode is not None:
torch.cuda.set_sync_debug_mode("error")
try:
metadata = builder.build(
0,
common,
num_accepted_tokens=accepted,
num_decode_draft_tokens_cpu=drafts_cpu,
)
finally:
if sync_mode is not None:
torch.cuda.set_sync_debug_mode(sync_mode)
def check(actual, expected):
torch.testing.assert_close(
actual, torch.tensor(expected, dtype=actual.dtype, device=device)
)
assert metadata.num_spec_decodes == len(spec_rows)
assert metadata.num_decodes == sum(query_lens[i] == 1 for i in non_spec_rows)
assert metadata.num_prefills == sum(query_lens[i] > 1 for i in non_spec_rows)
if spec_rows:
torch.testing.assert_close(
metadata.spec_state_indices_tensor[: len(spec_rows)],
common.block_table_tensor[spec_rows, 0],
)
torch.testing.assert_close(
metadata.num_accepted_tokens[: len(spec_rows)], accepted[spec_rows]
)
check(
metadata.spec_query_start_loc[: len(spec_rows) + 1],
[0, *accumulate(query_lens[i] for i in spec_rows)],
)
if full_graph and not non_spec_rows:
check(metadata.spec_state_indices_tensor[len(spec_rows) :], [NULL_BLOCK_ID])
check(metadata.num_accepted_tokens[len(spec_rows) :], [1])
check(
metadata.spec_query_start_loc[len(spec_rows) + 1 :], [sum(query_lens)]
)
assert (
metadata.spec_state_indices_tensor.data_ptr()
== builder.spec_state_indices_tensor.data_ptr()
)
if non_spec_rows:
torch.testing.assert_close(
metadata.state_indices_tensor, common.block_table_tensor[non_spec_rows, 0]
)
prefill_lens = [query_lens[i] for i in non_spec_rows if query_lens[i] > 1]
check(metadata.query_start_loc_p, [0, *accumulate(prefill_lens)])
check(metadata.has_initial_states_p, [True] * len(prefill_lens))
if token_order is not None:
check(
torch.cat((metadata.spec_token_indx, metadata.non_spec_token_indx)),
token_order,
)
else:
assert metadata.spec_token_indx is None
assert metadata.non_spec_token_indx is None
assert metadata.nums_dict is None
assert metadata.batch_ptr is None
assert metadata.token_chunk_offset_ptr is None
def _mock_etp_group(
@@ -1892,6 +2012,11 @@ def _make_conv_metadata(
else None
),
non_spec_query_start_loc=(non_spec_query_start_loc if has_non_spec else None),
query_start_loc_p=(
non_spec_query_start_loc[case.num_decodes :] - case.num_decodes
if num_prefills
else None
),
has_initial_states_p=(
torch.arange(num_prefills, device=device) % 2 == 0 if num_prefills else None
),
+22 -1
View File
@@ -17,6 +17,7 @@ from vllm.models.qwen4_exp.nvidia.ops import qsa as qsa_ops
from vllm.models.qwen4_exp.nvidia.ops import qsa_indexer as qsa_indexer_ops
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
from vllm.v1.worker.utils import clear_layer_kv_caches
requires_qsa_kernels = pytest.mark.skipif(
not current_platform.is_cuda() or not HAS_TRITON,
@@ -365,7 +366,9 @@ def test_qsa_circular_buffer_survives_one_speculative_step(chunk_start: int) ->
assert set(slots.tolist()).isdisjoint((committed % capacity).tolist())
def _qsa_key_cache(block_size: int, compress_ratio: int) -> qsa_cache.QSAKeyStateCache:
def _qsa_key_cache(
block_size: int, compress_ratio: int, **kwargs
) -> qsa_cache.QSAKeyStateCache:
return qsa_cache.QSAKeyStateCache(
head_size=64,
dtype=torch.bfloat16,
@@ -375,6 +378,7 @@ def _qsa_key_cache(block_size: int, compress_ratio: int) -> qsa_cache.QSAKeyStat
compilation_config=SimpleNamespace(static_forward_context={})
),
compress_ratio=compress_ratio,
**kwargs,
)
@@ -402,6 +406,23 @@ def test_qsa_state_caches_adapt_the_unified_logical_layout() -> None:
assert compressed_cache.kv_cache.data_ptr() == compressed_view.data_ptr()
def test_clearing_qsa_key_cache_releases_its_storage() -> None:
"""Profiling teardown clears `kv_cache`; no derived view may keep it alive."""
raw_cache = _qsa_key_cache(
block_size=32, compress_ratio=4, cache_rope_positions=True
)
kv = torch.zeros(2, 1, 8, raw_cache.head_size, dtype=torch.bfloat16)
references_before_bind = torch._C._storage_Use_Count(kv.untyped_storage()._cdata)
raw_cache.bind_kv_cache(kv)
assert raw_cache.key_cache.shape[-1] == 64
assert raw_cache.rope_position_cache.dtype == torch.int64
clear_layer_kv_caches([raw_cache])
references = torch._C._storage_Use_Count(kv.untyped_storage()._cdata)
assert references == references_before_bind
@pytest.mark.parametrize(
("compress_ratio", "num_spec", "expected"),
[(4, 0, 4), (4, 1, 8), (4, 3, 8), (4, 4, 8), (4, 5, 12), (2, 3, 6)],
+295 -43
View File
@@ -3,18 +3,23 @@
import time
from contextlib import nullcontext
from copy import deepcopy
from types import SimpleNamespace
from typing import TypedDict
import numpy as np
import pytest
from vllm.config import ModelConfig, SchedulerConfig
from vllm.config.multimodal import MultiModalConfig
from vllm.exceptions import VLLMValidationError
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.hasher import MultiModalHasher
from vllm.multimodal.parse import MultiModalDataParser
from vllm.multimodal.processing.context import (
BaseProcessingInfo,
InputProcessingContext,
overlay_modality_mm_kwargs,
_resolve_mm_processor_kwargs,
)
from vllm.multimodal.processing.inputs import ProcessorInputs
from vllm.multimodal.processing.processor import (
@@ -1207,58 +1212,305 @@ def test_apply_prompt_updates_falls_back_with_index_targets():
assert [p.tokens for p in placeholders["image"]] == [[200, 201], [9]]
@pytest.mark.skip_global_cleanup
def test_overlay_modality_mm_kwargs_scoped_video_does_not_leak_to_image():
"""HF-style videos_kwargs must overlay only when modality is video."""
video_size = {"longest_edge": 469762048, "shortest_edge": 4096}
kwargs = {"videos_kwargs": {"size": video_size}}
assert overlay_modality_mm_kwargs(kwargs, None) == kwargs
assert "size" not in overlay_modality_mm_kwargs(kwargs, "image")
assert overlay_modality_mm_kwargs(kwargs, "video")["size"] == video_size
class _TextProcessorKwargs(TypedDict, total=False):
padding: bool
@pytest.mark.skip_global_cleanup
def test_overlay_modality_mm_kwargs_flat_size_stays_shared():
"""A flat size override keeps the current shared-namespace behavior."""
size = {"longest_edge": 469762048, "shortest_edge": 4096}
kwargs = {"size": size}
for modality in (None, "image", "video"):
assert overlay_modality_mm_kwargs(kwargs, modality)["size"] == size
class _AudioProcessorKwargs(TypedDict, total=False):
sampling_rate: int
@pytest.mark.skip_global_cleanup
def test_overlay_modality_mm_kwargs_scoped_wins_over_flat_for_modality():
"""A nested videos_kwargs size wins over a flat size for video reads."""
class _ProcessorKwargs(TypedDict, total=False):
text_kwargs: _TextProcessorKwargs
audio_kwargs: _AudioProcessorKwargs
class _ImageProcessorKwargs(TypedDict, total=False):
size: dict[str, int]
min_pixels: int
class _VideoProcessorKwargs(TypedDict, total=False):
size: dict[str, int]
fps: float
@pytest.mark.parametrize(
("supported_mm_limits", "processor", "expected"),
[
(
{"image": None, "video": None},
SimpleNamespace(
valid_processor_kwargs=_ProcessorKwargs,
image_processor=SimpleNamespace(valid_kwargs=_ImageProcessorKwargs),
video_processor=SimpleNamespace(valid_kwargs=_VideoProcessorKwargs),
),
{
"text_kwargs": {"padding"},
"images_kwargs": {"size", "min_pixels"},
"videos_kwargs": {"size", "fps"},
},
),
(
{"image": None},
SimpleNamespace(
valid_processor_kwargs=_ProcessorKwargs,
image_processor=SimpleNamespace(valid_kwargs=_ImageProcessorKwargs),
),
{
"text_kwargs": {"padding"},
"images_kwargs": {"size", "min_pixels"},
},
),
(
{"audio": None},
SimpleNamespace(valid_processor_kwargs=_ProcessorKwargs),
{
"text_kwargs": {"padding"},
"audio_kwargs": {"sampling_rate"},
},
),
],
)
def test_get_supported_mm_processor_kwargs_uses_supported_modalities(
monkeypatch: pytest.MonkeyPatch,
supported_mm_limits: dict[str, int | None],
processor: SimpleNamespace,
expected: dict[str, set[str]],
) -> None:
info = BaseProcessingInfo(SimpleNamespace())
info.__dict__["supported_mm_limits"] = supported_mm_limits
monkeypatch.setattr(info, "get_hf_processor", lambda **_: processor)
assert info.get_supported_mm_processor_kwargs() == expected
def test_supported_mm_processor_kwargs_is_cached(
monkeypatch: pytest.MonkeyPatch,
) -> None:
info = BaseProcessingInfo(SimpleNamespace())
expected = {"images_kwargs": {"size"}}
calls = 0
def get_supported_mm_processor_kwargs() -> dict[str, set[str]]:
nonlocal calls
calls += 1
return expected
monkeypatch.setattr(
info,
"get_supported_mm_processor_kwargs",
get_supported_mm_processor_kwargs,
)
first = info.supported_mm_processor_kwargs
second = info.supported_mm_processor_kwargs
assert first is expected
assert second is first
assert calls == 1
def test_resolve_mm_processor_kwargs_routes_supported_flat_kwargs():
kwargs = {
"size": {"longest_edge": 1},
"videos_kwargs": {"size": {"longest_edge": 2}},
"images_kwargs": {"size": {"longest_edge": 3}},
"size": {"shortest_edge": 64},
"padding": True,
"sampling_rate": 16000,
"unknown": "keep",
"images_kwargs": {
"size": {"longest_edge": 1024},
"custom_image_kwarg": "keep",
},
}
kwargs_before = deepcopy(kwargs)
resolved = _resolve_mm_processor_kwargs(
kwargs,
supported_mm_processor_kwargs={
"text_kwargs": {"padding"},
"images_kwargs": {"size"},
"videos_kwargs": {"size"},
"audio_kwargs": {"sampling_rate"},
},
)
assert resolved == {
"unknown": "keep",
"text_kwargs": {"padding": True},
"images_kwargs": {
"size": {
"shortest_edge": 64,
"longest_edge": 1024,
},
"custom_image_kwarg": "keep",
},
"videos_kwargs": {
"size": {"shortest_edge": 64},
},
"audio_kwargs": {"sampling_rate": 16000},
}
assert kwargs == kwargs_before
video_kwargs = resolved["videos_kwargs"]
image_kwargs = resolved["images_kwargs"]
assert isinstance(video_kwargs, dict)
assert isinstance(image_kwargs, dict)
video_size = video_kwargs["size"]
image_size = image_kwargs["size"]
assert isinstance(video_size, dict)
assert isinstance(image_size, dict)
video_size["shortest_edge"] = 999
assert image_size["shortest_edge"] == 64
assert kwargs == kwargs_before
def test_resolve_mm_processor_kwargs_rejects_non_mapping_destination_scope():
with pytest.raises(TypeError, match="images_kwargs"):
_resolve_mm_processor_kwargs(
{
"size": {"shortest_edge": 64},
"images_kwargs": 123,
},
supported_mm_processor_kwargs={
"images_kwargs": {"size"},
},
)
def test_resolve_mm_processor_kwargs_without_schema_deduplicates_existing_scopes():
kwargs = {
"size": {"shortest_edge": 64},
"num_frames": 16,
"fps": 4,
"images_kwargs": {
"size": {"longest_edge": 1024},
},
"videos_kwargs": {
"num_frames": 8,
},
"audio_kwargs": 123,
"common_kwargs": {
"fps": 2,
},
}
kwargs_before = deepcopy(kwargs)
resolved = _resolve_mm_processor_kwargs(kwargs)
assert resolved == {
"fps": 4,
"images_kwargs": {
"size": {"longest_edge": 1024},
},
"videos_kwargs": {
"num_frames": 8,
},
"audio_kwargs": 123,
"common_kwargs": {
"fps": 2,
},
}
assert kwargs == kwargs_before
assert "text_kwargs" not in resolved
@pytest.mark.parametrize(
"inference_kwargs",
[
{"size": {"shortest_edge": 128}},
{"size": {"shortest_edge": 128}, "images_kwargs": None},
{"size": {"shortest_edge": 128}, "images_kwargs": {}},
],
)
def test_get_merged_mm_kwargs_treats_empty_scopes_as_absent_before_routing(
inference_kwargs: dict[str, object],
):
mm_config = MultiModalConfig(
mm_processor_kwargs={
"size": {
"shortest_edge": 64,
"longest_edge": 512,
},
"images_kwargs": {
"size": {"longest_edge": 1024},
},
},
mm_device_do_normalize=False,
)
model_config = SimpleNamespace(get_multimodal_config=lambda: mm_config)
ctx = InputProcessingContext(model_config, tokenizer=None) # type: ignore[arg-type]
merged = ctx.get_merged_mm_kwargs(
inference_kwargs,
supported_mm_processor_kwargs={
"images_kwargs": {"size"},
"videos_kwargs": {"size"},
},
)
assert merged == {
"images_kwargs": {
"size": {
"shortest_edge": 128,
"longest_edge": 1024,
},
},
"videos_kwargs": {
"size": {
"shortest_edge": 128,
"longest_edge": 512,
},
},
}
assert overlay_modality_mm_kwargs(kwargs, "video")["size"] == {"longest_edge": 2}
assert overlay_modality_mm_kwargs(kwargs, "image")["size"] == {"longest_edge": 3}
assert overlay_modality_mm_kwargs(kwargs, None)["size"] == {"longest_edge": 1}
def test_get_merged_mm_kwargs_merges_before_routing():
mm_config = MultiModalConfig(
mm_processor_kwargs={
"size": {
"shortest_edge": 64,
"longest_edge": 512,
},
"images_kwargs": {
"size": {"longest_edge": 1024},
},
},
mm_device_do_normalize=False,
)
model_config = SimpleNamespace(get_multimodal_config=lambda: mm_config)
ctx = InputProcessingContext(model_config, tokenizer=None) # type: ignore[arg-type]
@pytest.mark.skip_global_cleanup
def test_overlay_modality_mm_kwargs_ignores_non_mapping_scoped_value():
kwargs = {"images_kwargs": "not-a-dict", "size": {"longest_edge": 1}}
assert overlay_modality_mm_kwargs(kwargs, "image")["size"] == {"longest_edge": 1}
merged = ctx.get_merged_mm_kwargs(
{
"size": {
"shortest_edge": 128,
"longest_edge": 768,
},
"images_kwargs": {
"size": {"longest_edge": 2048},
},
},
supported_mm_processor_kwargs={
"images_kwargs": {"size"},
"videos_kwargs": {"size"},
},
)
@pytest.mark.skip_global_cleanup
def test_mm_processor_kwargs_merge_then_overlay_preserves_scoping():
"""Configured videos_kwargs overlay only for video reads after merge."""
from vllm.config.multimodal import MultiModalConfig
size = {"longest_edge": 469762048, "shortest_edge": 4096}
mm_config = MultiModalConfig(mm_processor_kwargs={"videos_kwargs": {"size": size}})
merged = mm_config.merge_mm_processor_kwargs({})
assert overlay_modality_mm_kwargs(merged, "video")["size"] == size
assert "size" not in overlay_modality_mm_kwargs(merged, "image")
assert "size" not in overlay_modality_mm_kwargs(merged, None)
assert merged == {
"images_kwargs": {
"size": {
"shortest_edge": 128,
"longest_edge": 2048,
},
},
"videos_kwargs": {
"size": {
"shortest_edge": 128,
"longest_edge": 768,
},
},
}
def test_processor_inputs_hashes_partial_uuids():
+556
View File
@@ -0,0 +1,556 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""PLaMo3 grammar, truncation, and serving-adapter regressions."""
from __future__ import annotations
import json
import pytest
import regex as re
from tests.parser.engine.conftest import make_mock_tokenizer
from tests.parser.engine.replay_harness import (
DUMMY_TOOLS,
_test_request,
collect_output,
replay_streaming,
replay_with_text_holdback,
)
from vllm.parser.parser_manager import ParserManager
from vllm.parser.plamo3 import (
BEGIN_THINK,
BEGIN_TOOL_ARGUMENTS,
BEGIN_TOOL_NAME,
BEGIN_TOOL_REQUEST,
BEGIN_TOOL_REQUESTS,
END_THINK,
END_TOOL_ARGUMENTS,
END_TOOL_NAME,
END_TOOL_REQUEST,
END_TOOL_REQUESTS,
EOT,
PLAMO_MARKER_TOKENS,
Plamo3Parser,
)
def _tool_header(name="weather"):
return (
BEGIN_TOOL_REQUEST
+ BEGIN_TOOL_NAME
+ name
+ END_TOOL_NAME
+ BEGIN_TOOL_ARGUMENTS
)
def _tool_call(name, arguments):
return _tool_header(name) + arguments + END_TOOL_ARGUMENTS + END_TOOL_REQUEST
@pytest.fixture
def mock_tokenizer():
special = [*sorted(PLAMO_MARKER_TOKENS), EOT, "<|plamo:bos|>"]
atoms = [*special, "<|plamo:constrain|>", "<|plamo:msg|>"]
vocab = {token: 100 + i for i, token in enumerate(atoms)}
inverse = {i: token for token, i in vocab.items()}
pattern = re.compile("|".join(map(re.escape, atoms)) + r"|[\s\S]")
tokenizer = make_mock_tokenizer(vocab, special_tokens=special)
tokenizer.encode.side_effect = lambda text, **kwargs: [
vocab[piece] if piece in vocab else 1000 + ord(piece)
for piece in pattern.findall(text)
]
tokenizer.decode.side_effect = lambda ids, **kwargs: "".join(
inverse[i] if i in inverse else chr(i - 1000) for i in ids
)
return tokenizer
@pytest.fixture(params=["direct", "registered"])
def parser_cls(request):
if request.param == "direct":
return Plamo3Parser
return ParserManager.get_parser(
tool_parser_name="plamo3",
reasoning_parser_name="plamo3",
enable_auto_tools=True,
)
def _stream(parser, tokenizer, output, chunk_size=1, finished_on_last=False):
tokens = [
(i, tokenizer.decode([i]))
for i in tokenizer.encode(output, add_special_tokens=False)
]
deltas = replay_streaming(
parser,
tokens,
chunk_size=chunk_size,
finished_on_last=finished_on_last,
tools=DUMMY_TOOLS,
)
if not finished_on_last:
deltas.append(
parser.parse_delta("", [], _test_request(DUMMY_TOOLS), finished=True)
)
return deltas
@pytest.mark.parametrize("thinking", [False, True])
@pytest.mark.parametrize("with_markers", [False, True])
def test_reasoning_mode(
parser_cls, mock_tokenizer, mock_request, thinking, with_markers
):
output = (
BEGIN_THINK + "Reasoning." + END_THINK + "Answer."
if with_markers
else "Answer."
)
if thinking:
expected = ("Reasoning.", "Answer.") if with_markers else ("Answer.", "")
else:
expected = ("", output)
kwargs = {"chat_template_kwargs": {"enable_thinking": thinking}}
reasoning, content, calls = parser_cls(mock_tokenizer, **kwargs).parse(
output, mock_request
)
assert (reasoning or "", content or "") == expected
assert not calls
result = collect_output(
_stream(parser_cls(mock_tokenizer, **kwargs), mock_tokenizer, output)
)
assert (result.reasoning, result.content) == expected
assert not result.tool_calls
def test_duplicate_think_end_does_not_leak_into_content(
parser_cls, mock_tokenizer, mock_request
):
output = END_THINK + "A" + END_THINK + "B"
reasoning, content, calls = parser_cls(mock_tokenizer).parse(output, mock_request)
assert (reasoning, content) == (None, "AB")
assert not calls
def test_content_can_start_a_new_reasoning_block(
parser_cls, mock_tokenizer, mock_request
):
output = END_THINK + "A" + BEGIN_THINK + "R" + END_THINK
reasoning, content, calls = parser_cls(mock_tokenizer).parse(output, mock_request)
assert (reasoning, content) == ("R", "A")
assert not calls
@pytest.mark.parametrize("chunk_size", [1, 7, None])
@pytest.mark.parametrize("finished_on_last", [False, True])
@pytest.mark.parametrize(
"closing",
[
"",
END_TOOL_ARGUMENTS + END_TOOL_REQUEST,
END_TOOL_ARGUMENTS + END_TOOL_REQUEST + END_TOOL_REQUESTS + EOT,
],
)
def test_complete_arguments_survive_missing_closers(
parser_cls, mock_tokenizer, mock_request, chunk_size, finished_on_last, closing
):
output = (
BEGIN_THINK
+ "Reasoning. "
+ END_THINK
+ "Before tools. "
+ BEGIN_TOOL_REQUESTS
+ _tool_header()
+ '{"city":"東京"}'
+ closing
)
reasoning, content, calls = parser_cls(mock_tokenizer).parse(
output, mock_request, enable_auto_tools=True
)
assert (reasoning, content) == ("Reasoning. ", "Before tools. ")
assert calls and [(c.name, json.loads(c.arguments)) for c in calls] == [
("weather", {"city": "東京"})
]
result = collect_output(
_stream(
parser_cls(mock_tokenizer),
mock_tokenizer,
output,
chunk_size,
finished_on_last,
)
)
assert (result.reasoning, result.content) == (reasoning, content)
assert result.tool_calls == [{"name": "weather", "arguments": '{"city":"東京"}'}]
def test_tool_name_cutoffs_do_not_leak_markers(
parser_cls, mock_tokenizer, mock_request
):
header = BEGIN_TOOL_REQUEST + BEGIN_TOOL_NAME + "weather" + END_TOOL_NAME
header_ids = mock_tokenizer.encode(header, add_special_tokens=False)
name_start = len(
mock_tokenizer.encode(
BEGIN_TOOL_REQUEST + BEGIN_TOOL_NAME, add_special_tokens=False
)
)
for end in range(len(header_ids)):
partial = mock_tokenizer.decode(header_ids[:end])
output = END_THINK + "Before tools." + BEGIN_TOOL_REQUESTS + partial
reasoning, content, calls = parser_cls(mock_tokenizer).parse(
output, mock_request, enable_auto_tools=True
)
assert (reasoning, content) == (None, "Before tools."), end
if end <= name_start:
expected_name = ""
else:
name_text = mock_tokenizer.decode(header_ids[name_start:end])
if "<|plamo:" in name_text:
name_text = name_text[: name_text.rfind("<|plamo:")]
expected_name = name_text[: len("weather")]
assert [call.name for call in calls or []] == (
[expected_name] if expected_name else []
), end
result = collect_output(
_stream(parser_cls(mock_tokenizer), mock_tokenizer, output)
)
assert [call["name"] for call in result.tool_calls] == (
[expected_name] if expected_name else []
), end
assert all("<" not in call["name"] for call in result.tool_calls), end
assert result.content == "Before tools.", end
@pytest.mark.parametrize("chunk_size", [1, 7, None])
@pytest.mark.parametrize("finished_on_last", [False, True])
def test_all_argument_closer_cutoffs_do_not_leak(
parser_cls, mock_tokenizer, mock_request, chunk_size, finished_on_last
):
head = END_THINK + BEGIN_TOOL_REQUESTS + _tool_header()
closer_ids = mock_tokenizer.encode(END_TOOL_ARGUMENTS, add_special_tokens=False)
for end in range(len(closer_ids)):
partial = mock_tokenizer.decode(closer_ids[:end])
output = head + '{"city":"東京"}' + partial
_, _, calls = parser_cls(mock_tokenizer).parse(
output, mock_request, enable_auto_tools=True
)
assert calls and calls[0].arguments == '{"city":"東京"}', end
result = collect_output(
_stream(
parser_cls(mock_tokenizer),
mock_tokenizer,
output,
chunk_size,
finished_on_last,
)
)
assert result.tool_calls == [
{"name": "weather", "arguments": '{"city":"東京"}'}
], end
@pytest.mark.parametrize(
"arguments",
[
"{}",
'{"name":"Alice","arguments":{"x":1},"parameters":true}',
'{"text":"literal <|plamo:end_',
],
)
def test_arguments_are_not_rewritten(
parser_cls, mock_tokenizer, mock_request, arguments
):
output = END_THINK + BEGIN_TOOL_REQUESTS + _tool_header() + arguments
_, _, calls = parser_cls(mock_tokenizer).parse(
output, mock_request, enable_auto_tools=True
)
assert calls and calls[0].arguments == arguments
result = collect_output(_stream(parser_cls(mock_tokenizer), mock_tokenizer, output))
assert result.tool_calls == [{"name": "weather", "arguments": arguments}]
@pytest.mark.parametrize("truncated_second", [False, True])
def test_multiple_calls_keep_indices_ids_and_arguments_separate(
parser_cls, mock_tokenizer, mock_request, truncated_second
):
second = (
BEGIN_TOOL_REQUEST + BEGIN_TOOL_NAME + "clo"
if truncated_second
else _tool_call("clock", '{"timezone":"Asia/Tokyo"}')
)
output = (
END_THINK
+ BEGIN_TOOL_REQUESTS
+ _tool_call("weather", '{"city":"東京"}')
+ second
)
expected = [{"name": "weather", "arguments": '{"city":"東京"}'}]
if truncated_second:
expected.append({"name": "clo", "arguments": "{}"})
else:
output += END_TOOL_REQUESTS
expected.append({"name": "clock", "arguments": '{"timezone":"Asia/Tokyo"}'})
stream_expected = expected
_, content, calls = parser_cls(mock_tokenizer).parse(
output, mock_request, enable_auto_tools=True
)
assert (
calls
and [{"name": c.name, "arguments": c.arguments} for c in calls] == expected
)
deltas = _stream(parser_cls(mock_tokenizer), mock_tokenizer, output)
result = collect_output(deltas)
assert result.tool_calls == stream_expected
assert result.content == (content or "") == ""
announced = [
call for delta in deltas if delta for call in delta.tool_calls or [] if call.id
]
assert [call.index for call in announced] == list(range(len(expected)))
assert len({call.id for call in announced}) == len(expected)
def test_arguments_are_available_before_outer_closer(mock_tokenizer, mock_request):
parser = Plamo3Parser(mock_tokenizer)
output = END_THINK + BEGIN_TOOL_REQUESTS + _tool_call("weather", '{"city":"Tokyo"}')
first = parser.parse_delta(output, [], mock_request, finished=False)
assert collect_output([first]).tool_calls == [
{"name": "weather", "arguments": '{"city":"Tokyo"}'}
]
assert (
parser.parse_delta(END_TOOL_REQUESTS[:10], [], mock_request, finished=False)
is None
)
assert (
parser.parse_delta(END_TOOL_REQUESTS[10:], [], mock_request, finished=True)
is None
)
def test_missing_tool_request_start_does_not_break_streaming(
parser_cls, mock_tokenizer, mock_request
):
output = (
BEGIN_TOOL_REQUESTS
+ BEGIN_TOOL_NAME
+ "weather"
+ END_TOOL_NAME
+ BEGIN_TOOL_ARGUMENTS
+ "{}"
+ END_TOOL_ARGUMENTS
+ END_TOOL_REQUESTS
)
delta = parser_cls(mock_tokenizer).parse_delta(
output, [], mock_request, finished=False
)
assert delta is not None
assert collect_output([delta]).tool_calls == [
{"name": "weather", "arguments": "{}"}
]
def test_reusing_parser_drops_pending_names(mock_tokenizer, mock_request):
parser = Plamo3Parser(mock_tokenizer)
parser.parse(
END_THINK
+ BEGIN_TOOL_REQUESTS
+ BEGIN_TOOL_REQUEST
+ BEGIN_TOOL_NAME
+ "partial",
mock_request,
)
output = (
END_THINK
+ BEGIN_TOOL_REQUESTS
+ _tool_call("weather", "{}")
+ END_TOOL_REQUESTS
)
_, _, calls = parser.parse(output, mock_request)
assert calls and calls[0].name == "weather"
@pytest.mark.parametrize(
"text", ["", "a < b", "literal <", "literal <|plamo:unknown", " text\n"]
)
def test_plain_content_is_preserved(parser_cls, mock_tokenizer, mock_request, text):
output = END_THINK + text + EOT
_, content, calls = parser_cls(mock_tokenizer).parse(output, mock_request)
assert (content or "") == text
assert not calls
result = collect_output(_stream(parser_cls(mock_tokenizer), mock_tokenizer, output))
assert (result.content, result.tool_calls) == (text, [])
@pytest.mark.parametrize(
"text, ended",
[
("", False),
(BEGIN_THINK + "reasoning", False),
(END_THINK + "answer", True),
(END_THINK + "answer" + BEGIN_THINK + "new reasoning", False),
(END_THINK + "answer" + EOT + "assistant", False),
],
)
def test_reasoning_end_checks_the_latest_turn(mock_tokenizer, text, ended):
parser = Plamo3Parser(mock_tokenizer)
ids = mock_tokenizer.encode(text)
assert parser.is_reasoning_end(ids) == ended
if ended:
assert mock_tokenizer.decode(parser.extract_content_ids(ids)) == "answer"
@pytest.mark.parametrize("suffix", ["<|plamo:end_", "<|plamo:end_think"])
def test_partial_end_think_is_consistent_between_streaming_and_non_streaming(
parser_cls, mock_tokenizer, mock_request, suffix
):
output = "visible" + suffix
reasoning, content, calls = parser_cls(mock_tokenizer).parse(output, mock_request)
assert (reasoning, content) == ("visible", None)
assert not calls
result = collect_output(_stream(parser_cls(mock_tokenizer), mock_tokenizer, output))
assert (result.reasoning, result.content, result.tool_calls) == ("visible", "", [])
@pytest.mark.parametrize(
"suffix",
[
"<|plamo:begin_",
"<|plamo:begin_tool_name",
"<|plamo:end_tool",
],
)
def test_unfinished_marker_prefixes_are_stripped(mock_tokenizer, suffix):
parser = Plamo3Parser(mock_tokenizer)
assert parser._strip_unfinished_marker("visible" + suffix) == "visible"
@pytest.mark.parametrize("suffix", ["<|plamo:", "<|plamo:b"])
def test_non_token_prefix_is_not_stripped(mock_tokenizer, suffix):
parser = Plamo3Parser(mock_tokenizer)
text = "visible" + suffix
assert parser._strip_unfinished_marker(text) == text
def test_complete_markers_are_not_stripped(mock_tokenizer):
parser = Plamo3Parser(mock_tokenizer)
text = "visible" + BEGIN_TOOL_NAME
assert parser._strip_unfinished_marker(text) == text
def test_short_prefix_is_not_stripped(mock_tokenizer):
parser = Plamo3Parser(mock_tokenizer)
text = "visible<"
assert parser._strip_unfinished_marker(text) == text
def test_tool_block_can_end_reasoning_implicitly(
parser_cls, mock_tokenizer, mock_request
):
output = (
"reasoning"
+ BEGIN_TOOL_REQUESTS
+ _tool_call("weather", "{}")
+ END_TOOL_REQUESTS
)
reasoning, content, calls = parser_cls(mock_tokenizer).parse(
output, mock_request, enable_auto_tools=True
)
assert (reasoning, content) == ("reasoning", None)
assert calls and calls[0].name == "weather"
result = collect_output(_stream(parser_cls(mock_tokenizer), mock_tokenizer, output))
assert (result.reasoning, result.content) == ("reasoning", "")
assert result.tool_calls == [{"name": "weather", "arguments": "{}"}]
def test_every_cut_point_matches_non_streaming(
parser_cls, mock_tokenizer, mock_request
):
output = (
BEGIN_THINK
+ "Reasoning. "
+ END_THINK
+ "Before tools. "
+ BEGIN_TOOL_REQUESTS
+ _tool_call("weather", '{"city":"東京"}')
+ END_TOOL_REQUESTS
+ EOT
)
ids = mock_tokenizer.encode(output, add_special_tokens=False)
for k in range(1, len(ids) + 1):
prefix = mock_tokenizer.decode(ids[:k])
reasoning, content, calls = parser_cls(mock_tokenizer).parse(
prefix, mock_request, enable_auto_tools=True
)
result = collect_output(
_stream(parser_cls(mock_tokenizer), mock_tokenizer, prefix)
)
assert (result.reasoning, result.content) == (
(reasoning or ""),
(content or ""),
), k
expected = [{"name": c.name, "arguments": c.arguments} for c in (calls or [])]
assert result.tool_calls == expected, (k, expected, result.tool_calls)
@pytest.mark.parametrize(
"choice",
[
"auto",
"required",
"none",
{"type": "function", "function": {"name": "weather"}},
],
)
def test_tool_choice_uses_the_plamo_format(parser_cls, mock_tokenizer, choice):
request = _test_request(
[
{
"type": "function",
"function": {"name": "weather", "parameters": {"type": "object"}},
},
]
)
request = type(request).model_validate(
request.model_dump() | {"tool_choice": choice}
)
output = (
END_THINK
+ "Before tools."
+ BEGIN_TOOL_REQUESTS
+ _tool_call("weather", "{}")
+ END_TOOL_REQUESTS
)
_, content, calls = parser_cls(mock_tokenizer).parse(
output, request, enable_auto_tools=True
)
assert content == "Before tools."
assert [c.name for c in calls or []] == ([] if choice == "none" else ["weather"])
parser = parser_cls(mock_tokenizer)
delta = parser.parse_delta(output, [], request, finished=True)
result = collect_output([delta])
assert result.content == "Before tools."
assert result.tool_calls == (
[] if choice == "none" else [{"name": "weather", "arguments": "{}"}]
)
@pytest.mark.parametrize("delay", [1, 3])
def test_delayed_text_keeps_split_markers_intact(parser_cls, mock_tokenizer, delay):
output = (
BEGIN_THINK
+ "reasoning"
+ END_THINK
+ "content"
+ BEGIN_TOOL_REQUESTS
+ _tool_call("weather", "{}")
+ END_TOOL_REQUESTS
+ EOT
)
tokens = [(i, mock_tokenizer.decode([i])) for i in mock_tokenizer.encode(output)]
result = collect_output(
replay_with_text_holdback(
parser_cls(mock_tokenizer), tokens, text_delay=delay, tools=DUMMY_TOOLS
)
)
assert (result.reasoning, result.content) == ("reasoning", "content")
assert result.tool_calls == [{"name": "weather", "arguments": "{}"}]
+4 -3
View File
@@ -74,9 +74,10 @@ def _discover_parsers() -> list[_ParserInfo]:
if cfg.name not in _BUILDERS:
missing_builders.append(f"{obj.__name__} (config.name={cfg.name!r})")
continue
if cfg.name == "inkling":
# Inkling opts out of token-id terminal matching and has typed
# structural blocks; its replay coverage lives in test_inkling.py.
if cfg.name in ("inkling", "plamo3"):
# Inkling uses typed structural blocks. PLaMo markers can span
# several token IDs. Neither fits this TOOL_END-token harness;
# their direct and delegating coverage lives in dedicated tests.
continue
if cfg.name == "granite":
# Granite has a JSON-array tool body with no TOOL_END terminal, so
+67
View File
@@ -28,6 +28,7 @@ from tests.parser.engine.replay_harness import (
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionToolsParam,
)
from vllm.parser import plamo3
from vllm.parser.engine.registered_adapters import (
DeepSeekV4Parser,
DeepSeekV32Parser,
@@ -40,6 +41,7 @@ from vllm.parser.engine.registered_adapters import (
KimiK2Parser,
MinimaxM2Parser,
NemotronV3Parser,
Plamo3Parser,
Qwen3Parser,
SeedOssParser,
)
@@ -1068,6 +1070,70 @@ def _build_inkling(scenario: Scenario, validate: bool = True) -> Sample:
return sample
# ── PLaMo3 (literal nested tool-request wrappers) ───────────────────
_PLAMO3_MARKERS = (
plamo3.BEGIN_THINK,
plamo3.END_THINK,
plamo3.BEGIN_TOOL_REQUESTS,
plamo3.END_TOOL_REQUESTS,
plamo3.BEGIN_TOOL_REQUEST,
plamo3.END_TOOL_REQUEST,
plamo3.BEGIN_TOOL_NAME,
plamo3.END_TOOL_NAME,
plamo3.BEGIN_TOOL_ARGUMENTS,
plamo3.END_TOOL_ARGUMENTS,
plamo3.EOT,
)
_PLAMO3_VOCAB = {marker: 300 + i for i, marker in enumerate(_PLAMO3_MARKERS)}
def _plamo3_tool_segments(tc: ToolCallSpec) -> list[tuple[str, bool]]:
args = json.dumps(tc.arguments, ensure_ascii=False, separators=(",", ":"))
return [
(plamo3.BEGIN_TOOL_REQUEST, True),
(plamo3.BEGIN_TOOL_NAME, True),
(tc.name, False),
(plamo3.END_TOOL_NAME, True),
(plamo3.BEGIN_TOOL_ARGUMENTS, True),
(args, False),
(plamo3.END_TOOL_ARGUMENTS, True),
(plamo3.END_TOOL_REQUEST, True),
]
def _plamo3_segments(scenario: Scenario) -> list[tuple[str, bool]]:
segs: list[tuple[str, bool]] = [
(plamo3.BEGIN_THINK, True),
(scenario.reasoning or "", False),
(plamo3.END_THINK, True),
]
if scenario.content is not None:
segs.append((scenario.content, False))
if scenario.tool_calls is not None:
segs.append((plamo3.BEGIN_TOOL_REQUESTS, True))
for tool_call in scenario.tool_calls:
segs.extend(_plamo3_tool_segments(tool_call))
segs.append((plamo3.END_TOOL_REQUESTS, True))
return segs
def _build_plamo3(scenario: Scenario, validate: bool = True) -> Sample:
sample = _make_sample(
sample_id=f"plamo3-{scenario.id}",
description=scenario.description,
vocab=_PLAMO3_VOCAB,
segments=_plamo3_segments(scenario),
expected_reasoning=scenario.reasoning or "",
expected_content=_qwen3_expected_content(scenario),
expected_tool_calls=_expected_tc(scenario),
tools=_expected_tools(scenario),
)
if validate:
_validate_sample(sample, Plamo3Parser)
return sample
# ── Granite (JSON-array tool bodies, no reasoning) ───────────────────────
_GRANITE_VOCAB: dict[str, int] = {
@@ -1167,6 +1233,7 @@ _BUILDERS: dict[str, Any] = {
"kimi_k2": _build_kimi_k2,
"qwen3": _build_qwen3,
"inkling": _build_inkling,
"plamo3": _build_plamo3,
}
+118 -9
View File
@@ -149,7 +149,11 @@ def test_wna16_oracle_rejects_incompatible_quant_structures(
assert expected in reason
def test_compressed_tensors_weights_are_transposed_for_triton():
def test_compressed_tensors_nfirst_weights_pass_through_for_triton():
"""CT provides N-first [E, N, K_packed] int32; Triton expects N-first uint8.
No transpose needed — just a view from int32 to uint8.
"""
quant_config = QuantizationArgs(
num_bits=4,
type=QuantizationType.INT,
@@ -158,10 +162,11 @@ def test_compressed_tensors_weights_are_transposed_for_triton():
dynamic=False,
group_size=32,
)
w13 = torch.arange(16, dtype=torch.int32).reshape(1, 2, 8)
w2 = torch.arange(12, dtype=torch.int32).reshape(1, 2, 6)
w13_scale = torch.arange(32, dtype=torch.float16).reshape(1, 4, 8)
w2_scale = torch.arange(18, dtype=torch.float16).reshape(1, 3, 6)
# N-first layout: [E, N, K_packed]
w13 = torch.arange(16, dtype=torch.int32).reshape(1, 8, 2)
w2 = torch.arange(12, dtype=torch.int32).reshape(1, 6, 2)
w13_scale = torch.arange(32, dtype=torch.float16).reshape(1, 8, 4)
w2_scale = torch.arange(18, dtype=torch.float16).reshape(1, 6, 3)
converted = convert_to_wna16_moe_kernel_format(
backend=WNA16MoEBackend.TRITON,
@@ -175,10 +180,40 @@ def test_compressed_tensors_weights_are_transposed_for_triton():
)
assert converted is not None
assert torch.equal(converted[0], w13.transpose(1, 2).contiguous().view(torch.uint8))
assert torch.equal(converted[1], w2.transpose(1, 2).contiguous().view(torch.uint8))
assert torch.equal(converted[2], w13_scale.transpose(1, 2).contiguous())
assert torch.equal(converted[3], w2_scale.transpose(1, 2).contiguous())
assert torch.equal(converted[0], w13.contiguous().view(torch.uint8))
assert torch.equal(converted[1], w2.contiguous().view(torch.uint8))
assert torch.equal(converted[2], w13_scale)
assert torch.equal(converted[3], w2_scale)
def test_gptq_nfirst_weights_pass_through_for_triton():
"""GPTQ frontend normalizes to N-first before calling oracle.
Verifies the oracle handles N-first GPTQ inputs the same way as CT.
"""
quant_config = AutoGPTQConfig(4, 128, False, True, False, {}, {})
# N-first layout (after GPTQ frontend transpose): [E, N, K_packed]
w13 = torch.arange(16, dtype=torch.int32).reshape(1, 8, 2)
w2 = torch.arange(12, dtype=torch.int32).reshape(1, 6, 2)
w13_scale = torch.arange(32, dtype=torch.float16).reshape(1, 8, 4)
w2_scale = torch.arange(18, dtype=torch.float16).reshape(1, 6, 3)
converted = convert_to_wna16_moe_kernel_format(
backend=WNA16MoEBackend.TRITON,
layer=torch.nn.Module(),
quant_config=quant_config,
input_dtype=None,
w13=w13,
w2=w2,
w13_scale=w13_scale,
w2_scale=w2_scale,
)
assert converted is not None
assert torch.equal(converted[0], w13.contiguous().view(torch.uint8))
assert torch.equal(converted[1], w2.contiguous().view(torch.uint8))
assert torch.equal(converted[2], w13_scale)
assert torch.equal(converted[3], w2_scale)
def test_moe_wna16_setup_forwards_selected_backend(monkeypatch):
@@ -511,3 +546,77 @@ def test_compressed_tensors_wna16_moe_marlin_prep_with_unset_group_size():
)
assert layer.w13_weight_scale.shape == (num_experts, 1, 2 * intermediate_size)
assert layer.w2_weight_scale.shape == (num_experts, 1, hidden_size)
def test_moe_wna16_w13_qzeros_shard_split_int8_asym_gptq(monkeypatch):
"""8-bit asymmetric GPTQ MoE must split w13 zero-points by pack factor.
``w13_qzeros`` is packed along the intermediate dim by ``bit8_pack_factor``,
so the gate/up (w1/w3) boundary in the weight loader is
``shard_size // bit8_pack_factor``. Hardcoding ``shard_size // 2`` only
holds for 4-bit and raises a shape mismatch when loading an 8-bit
(unpacked) checkpoint.
"""
from tests.kernels.moe.utils import make_dummy_moe_config
monkeypatch.setattr(
moe_wna16, "get_tp_group", lambda: SimpleNamespace(device="cpu")
)
monkeypatch.setattr(moe_wna16, "get_tensor_model_parallel_rank", lambda: 0)
num_experts, hidden, intermediate, group_size = 2, 256, 512, 128
moe_config = make_dummy_moe_config(
num_experts=num_experts,
hidden_dim=hidden,
intermediate_size=intermediate,
)
quant_config = MoeWNA16Config(
linear_quant_method="gptq",
weight_bits=8,
group_size=group_size,
has_zp=True,
lm_head_quantized=False,
modules_to_not_convert=[],
full_config={},
)
method = object.__new__(MoeWNA16Method)
method.quant_config = quant_config
method.moe = moe_config
layer = torch.nn.Module()
layer.intermediate_size_per_partition = intermediate
layer.moe_config = moe_config
method.create_weights(
layer,
num_experts=num_experts,
hidden_size=hidden,
intermediate_size_per_partition=intermediate,
params_dtype=torch.float16,
weight_loader=lambda *a, **k: True,
)
# 8-bit is unpacked along the intermediate dim: dim1 == 2 * intermediate.
assert layer.w13_qzeros.shape == (
num_experts,
2 * intermediate,
hidden // group_size,
)
loader = layer.w13_qzeros.weight_loader
# Raw GPTQ 8-bit qzeros per shard: int32 (num_groups, intermediate // 4);
# the loader views them as uint8 -> (num_groups, intermediate), then
# transposes and offsets by 1.
num_groups = hidden // group_size
raw_w1 = torch.arange(num_groups * (intermediate // 4), dtype=torch.int32).reshape(
num_groups, intermediate // 4
)
raw_w3 = raw_w1 + 1000
# Must not raise (old code raised a shape mismatch here for 8-bit).
loader(layer.w13_qzeros, raw_w1, "w13_qzeros", "w1", 0, False)
loader(layer.w13_qzeros, raw_w3, "w13_qzeros", "w3", 0, False)
expected_w1 = raw_w1.view(torch.uint8).T + 1
expected_w3 = raw_w3.view(torch.uint8).T + 1
assert torch.equal(layer.w13_qzeros[0, :intermediate].cpu(), expected_w1)
assert torch.equal(layer.w13_qzeros[0, intermediate:].cpu(), expected_w3)
+41 -5
View File
@@ -5,6 +5,7 @@ import json
import logging
import os
from dataclasses import MISSING, Field, asdict, dataclass, field
from datetime import timedelta
from pathlib import Path
from types import SimpleNamespace
from typing import cast
@@ -1464,6 +1465,20 @@ def test_all2all_backend_has_portable_default():
assert ParallelConfig().all2all_backend == "allgather_reducescatter"
def test_dp_group_uses_configured_timeout_without_current_config(monkeypatch):
monkeypatch.setattr(vllm_config_module, "_current_vllm_config", None)
config = ParallelConfig(cpu_distributed_timeout_seconds=30)
with (
patch(
"vllm.distributed.utils.rendezvous",
return_value=iter([(torch.distributed.HashStore(), 0, 1)]),
),
patch("vllm.distributed.utils.init_gloo_process_group") as init_group,
):
config.stateless_init_dp_group()
assert init_group.call_args.kwargs["timeout"] == timedelta(seconds=30)
@pytest.mark.parametrize(
"dp_size, across_dp, expected",
[(1, False, 4), (1, True, 4), (2, False, 4), (2, True, 8)],
@@ -1497,21 +1512,42 @@ def test_engram_dp_shared_memory_requires_cpu_offload():
EngramConfig(cpu_offload=False, dp_shared_memory=True)
@pytest.mark.skip_global_cleanup
@pytest.mark.parametrize(
"cpu_offload,dp_size,elastic_ep,expected",
[(True, 2, False, True), (False, 2, False, False), (True, 1, False, False)],
"cpu_offload,use_thp,dp_shared_memory,dp_size,elastic_ep,expected",
[
(True, False, None, 2, False, True),
(False, False, None, 2, False, False),
(True, False, None, 1, False, False),
(True, False, None, 2, True, False),
# use_thp backs private tables; sharing would silently ignore it.
(True, True, None, 2, False, False),
(True, True, False, 2, False, False),
],
)
def test_engram_dp_shared_memory_defaults_when_supported(
cpu_offload, dp_size, elastic_ep, expected
cpu_offload, use_thp, dp_shared_memory, dp_size, elastic_ep, expected
):
"""Unset dp_shared_memory enables sharing only for offloaded, non-elastic DP."""
"""Resolve sharing only where supported, preserving an explicit false."""
parallel = ParallelConfig(data_parallel_size=dp_size)
parallel.enable_elastic_ep = elastic_ep
config = EngramConfig(cpu_offload=cpu_offload)
config = EngramConfig(
cpu_offload=cpu_offload,
use_thp=use_thp,
dp_shared_memory=dp_shared_memory,
)
config.resolve_dp_shared_memory(parallel)
assert config.dp_shared_memory is expected
@pytest.mark.skip_global_cleanup
def test_engram_thp_rejects_explicit_shared_memory():
with pytest.raises(
ValueError, match="use_thp requires cpu_offload=True and dp_shared_memory=False"
):
EngramConfig(use_thp=True, dp_shared_memory=True)
@pytest.mark.parametrize(
"dp_size,load_format,multithread,error",
[
+41
View File
@@ -165,6 +165,45 @@ def test_json_logging(monkeypatch, tmp_path):
assert log["process"] == os.getpid()
@pytest.mark.parametrize("factory_order", ["before", "after", "replacement"])
def test_configure_logging_preserves_application_record_factory(
monkeypatch, factory_order
):
"""Application fields survive initial configuration and reconfiguration."""
original_factory = logging.getLogRecordFactory()
monkeypatch.setattr(vllm_logger, "dictConfig", lambda _: None)
monkeypatch.setattr(vllm_logger, "_last_configured_logging_config", None)
monkeypatch.setattr(vllm_logger, "_vllm_process_info", None)
config = LoggingConfig()
formatter = logging.Formatter("%(request_id)s %(vllm_process_name)s %(message)s")
try:
logging.setLogRecordFactory(logging.LogRecord)
if factory_order != "before":
configure_logging(config)
base_factory = (
logging.LogRecord
if factory_order == "replacement"
else logging.getLogRecordFactory()
)
def application_factory(*args, **kwargs):
record = base_factory(*args, **kwargs)
record.request_id = "request-123"
return record
logging.setLogRecordFactory(application_factory)
decorate_logs("Worker_DP0")
for _ in range(2):
configure_logging(config)
record = logging.getLogger("application").makeRecord(
"application", logging.INFO, __file__, 1, "probe", (), None
)
assert formatter.format(record) == "request-123 Worker_DP0 probe"
finally:
logging.setLogRecordFactory(original_factory)
def test_use_color_force_color(monkeypatch):
"""FORCE_COLOR forces colored logs without a TTY, while NO_COLOR and an
explicit VLLM_LOGGING_COLOR=0 take precedence over it."""
@@ -223,9 +262,11 @@ def test_logger_configuring_can_be_disabled(monkeypatch):
monkeypatch.setenv("VLLM_CONFIGURE_LOGGING", "0")
monkeypatch.delenv("VLLM_LOGGING_CONFIG_PATH", raising=False)
original_factory = logging.getLogRecordFactory()
with patch("vllm.logger.dictConfig") as dict_config_mock:
configure_logging(LoggingConfig(configure_logging=False))
dict_config_mock.assert_not_called()
assert logging.getLogRecordFactory() is original_factory
def test_an_error_is_raised_when_custom_logging_config_file_does_not_exist(monkeypatch):
+2 -3
View File
@@ -43,6 +43,7 @@ def prompt_embeds_processor():
logits_processors=None,
is_diffusion=False,
return_sampling_mask=False,
supports_multimodal_inputs=False,
)
model_config.get_vocab_size.return_value = 32
model_config.try_get_generation_config.return_value = {}
@@ -53,9 +54,7 @@ def prompt_embeds_processor():
)
renderer = Mock(tokenizer=None)
renderer.get_eos_token_id.return_value = None
registry = Mock()
registry.supports_multimodal_inputs.return_value = False
return InputProcessor(config, renderer, mm_registry=registry)
return InputProcessor(config, renderer, mm_registry=Mock())
@pytest.mark.parametrize("mask_len", [0, 1, 2, 4])
+49 -14
View File
@@ -10,10 +10,13 @@ from vllm.triton_utils import importing as triton_importing
from vllm.triton_utils.importing import TritonLanguagePlaceholder, TritonPlaceholder
def _has_triton_for_backends(**drivers: bool) -> bool:
def _has_triton_for_backends(*, vllm_version: str = "0.0.0", **drivers: bool) -> bool:
"""Re-evaluate ``HAS_TRITON`` against a synthetic ``triton.backends`` map.
``drivers`` maps backend name to whether its driver reports itself active.
``vllm_version`` selects the build type: importing.py takes its Triton-CPU
branch only when "cpu" is in the installed vLLM version, so pin it rather
than depending on the real install (the CPU CI image is a "+cpu" build).
"""
backends = {}
for name, is_active in drivers.items():
@@ -34,6 +37,7 @@ def _has_triton_for_backends(**drivers: bool) -> bool:
with (
mock.patch.dict(sys.modules, patched_modules),
mock.patch.dict("os.environ", {}, clear=True),
mock.patch("importlib.metadata.version", return_value=vllm_version),
):
return importlib.reload(triton_importing).HAS_TRITON
finally:
@@ -129,19 +133,50 @@ def test_cpu_backend_alone_disables_triton():
assert _has_triton_for_backends(cpu=True) is False
def test_cpu_build_with_cpu_backend_keeps_triton():
assert _has_triton_for_backends(vllm_version="0.0.0+cpu", cpu=True) is True
def test_cpu_build_without_cpu_backend_disables_triton():
assert _has_triton_for_backends(vllm_version="0.0.0+cpu", amd=True) is False
def test_no_triton_fallback():
# clear existing triton modules
sys.modules.pop("triton", None)
sys.modules.pop("triton.language", None)
sys.modules.pop("vllm.triton_utils", None)
sys.modules.pop("vllm.triton_utils.importing", None)
# Save the real modules so they can be put back afterward - popping them
# (rather than mock.patch.dict alone) is what forces vllm.triton_utils to
# be freshly, unmemoized re-imported under the "triton is absent"
# condition below.
saved_modules = {
name: sys.modules.get(name)
for name in (
"triton",
"triton.language",
"vllm.triton_utils",
"vllm.triton_utils.importing",
)
}
for name in saved_modules:
sys.modules.pop(name, None)
# mock triton not being installed
with mock.patch.dict(sys.modules, {"triton": None}):
from vllm.triton_utils import HAS_TRITON, tl, triton
try:
# mock triton not being installed
with mock.patch.dict(sys.modules, {"triton": None}):
from vllm.triton_utils import HAS_TRITON, tl, triton
assert HAS_TRITON is False
assert triton.__class__.__name__ == "TritonPlaceholder"
assert triton.language.__class__.__name__ == "TritonLanguagePlaceholder"
assert tl.__class__.__name__ == "TritonLanguagePlaceholder"
assert tl.constexpr(2**31 - 1) == 2**31 - 1
assert HAS_TRITON is False
assert triton.__class__.__name__ == "TritonPlaceholder"
assert triton.language.__class__.__name__ == "TritonLanguagePlaceholder"
assert tl.__class__.__name__ == "TritonLanguagePlaceholder"
assert tl.constexpr(2**31 - 1) == 2**31 - 1
finally:
# The pops above are outside mock.patch.dict's scope, so exiting the
# `with` restores "triton" to absent, not to the real module - unlike
# _has_triton_for_backends above, nothing here puts it back. Left
# alone, every subsequent import of vllm.triton_utils in this process
# (including other test files, and other tests' cleanup fixtures that
# need real triton) would get this fake, HAS_TRITON=False module.
for name, module in saved_modules.items():
if module is not None:
sys.modules[name] = module
else:
sys.modules.pop(name, None)
+4 -1
View File
@@ -13,7 +13,10 @@ def test_version_is_defined():
def test_version_tuple():
assert len(version.__version_tuple__) in (3, 4, 5)
# (major, minor, patch) plus optional pre-release tag, dev-distance, and
# git-hash components - setuptools_scm can emit any combination of them,
# e.g. 6 when a dev build sits on top of an rc-tagged commit.
assert len(version.__version_tuple__) >= 3
@pytest.mark.parametrize(
+28 -11
View File
@@ -13,6 +13,23 @@ from vllm.platforms import (
)
@pytest.fixture
def caplog_vllm(caplog):
"""`caplog`, but also captures vLLM's loggers.
vllm/logger.py configures the "vllm" logger with propagate=False, so
records from child loggers like "vllm.platforms" reach vLLM's own
handler but never bubble up to the root logger where plain `caplog`
listens. Attach caplog's handler directly to "vllm" to see them too.
"""
logger = logging.getLogger("vllm")
logger.addHandler(caplog.handler)
try:
yield caplog
finally:
logger.removeHandler(caplog.handler)
def test_is_amd_zen_cpu_detects_amd_with_avx512():
cpuinfo = "vendor_id: AuthenticAMD\nflags: avx avx2 avx512f avx512bw"
with (
@@ -65,7 +82,7 @@ def test_cpu_target_selects_cpu_platform_from_non_cpu_wheel(
rocm_plugin.assert_not_called()
def test_broken_zentorch_falls_back_to_cpu_platform(caplog):
def test_broken_zentorch_falls_back_to_cpu_platform(caplog_vllm):
original_import = builtins.__import__
def import_with_broken_zentorch(name, *args, **kwargs):
@@ -84,16 +101,16 @@ def test_broken_zentorch_falls_back_to_cpu_platform(caplog):
patch("vllm.platforms.vllm_version_matches_substr", return_value=True),
patch("vllm.platforms._is_amd_zen_cpu", return_value=True),
patch.object(builtins, "__import__", side_effect=import_with_broken_zentorch),
caplog.at_level(logging.DEBUG, logger="vllm.platforms"),
caplog_vllm.at_level(logging.DEBUG, logger="vllm.platforms"),
):
platform = resolve_current_platform_cls_qualname()
assert platform == "vllm.platforms.cpu.CpuPlatform"
assert "zentorch failed to import" in caplog.text
assert "OSError: incompatible shared library" in caplog.text
assert "zentorch failed to import" in caplog_vllm.text
assert "OSError: incompatible shared library" in caplog_vllm.text
def test_zentorch_failure_other_than_oserror_is_not_recovered(caplog):
def test_zentorch_failure_other_than_oserror_is_not_recovered(caplog_vllm):
original_import = builtins.__import__
def import_with_broken_zentorch(name, *args, **kwargs):
@@ -112,15 +129,15 @@ def test_zentorch_failure_other_than_oserror_is_not_recovered(caplog):
patch("vllm.platforms.vllm_version_matches_substr", return_value=True),
patch("vllm.platforms._is_amd_zen_cpu", return_value=True),
patch.object(builtins, "__import__", side_effect=import_with_broken_zentorch),
caplog.at_level(logging.DEBUG, logger="vllm.platforms"),
caplog_vllm.at_level(logging.DEBUG, logger="vllm.platforms"),
):
platform = resolve_current_platform_cls_qualname()
assert platform == "vllm.platforms.interface.UnspecifiedPlatform"
assert "RuntimeError: unknown zentorch failure" in caplog.text
assert "RuntimeError: unknown zentorch failure" in caplog_vllm.text
def test_platform_plugin_failure_is_logged(caplog):
def test_platform_plugin_failure_is_logged(caplog_vllm):
def failing_plugin():
raise RuntimeError("plugin exploded")
@@ -132,10 +149,10 @@ def test_platform_plugin_failure_is_logged(caplog):
clear=True,
),
patch("vllm.platforms.load_plugins_by_group", return_value={}),
caplog.at_level(logging.DEBUG, logger="vllm.platforms"),
caplog_vllm.at_level(logging.DEBUG, logger="vllm.platforms"),
):
platform = resolve_current_platform_cls_qualname()
assert platform == "vllm.platforms.interface.UnspecifiedPlatform"
assert "Platform plugin cpu failed during detection" in caplog.text
assert "RuntimeError: plugin exploded" in caplog.text
assert "Platform plugin cpu failed during detection" in caplog_vllm.text
assert "RuntimeError: plugin exploded" in caplog_vllm.text
@@ -16,6 +16,17 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionToolsParam,
)
from vllm.parser.abstract_parser import DelegatingParser
from vllm.parser.plamo3 import (
BEGIN_TOOL_ARGUMENTS,
BEGIN_TOOL_NAME,
BEGIN_TOOL_REQUEST,
BEGIN_TOOL_REQUESTS,
END_TOOL_ARGUMENTS,
END_TOOL_NAME,
END_TOOL_REQUEST,
END_TOOL_REQUESTS,
EOT,
)
from vllm.tool_parsers.abstract_tool_parser import ToolParser
from vllm.tool_parsers.deepseekv3_tool_parser import DeepSeekV3ToolParser
from vllm.tool_parsers.deepseekv4_engine_tool_parser import DeepSeekV4EngineToolParser
@@ -30,6 +41,7 @@ from vllm.tool_parsers.kimi_k2_tool_parser import KimiK2ToolParser
from vllm.tool_parsers.kimi_k3_tool_parser import KimiK3ToolParser
from vllm.tool_parsers.llama_tool_parser import Llama3JsonToolParser
from vllm.tool_parsers.minimax_m2_tool_parser import MinimaxM2ToolParser
from vllm.tool_parsers.plamo3_engine_tool_parser import Plamo3EngineToolParser
from vllm.tool_parsers.qwen3_engine_tool_parser import Qwen3EngineToolParser
from vllm.tool_parsers.structural_tag_registry import (
SUPPORTED_STRUCTURAL_TAG_MODELS,
@@ -271,6 +283,131 @@ def test_hermes_required_tool_calls_use_empty_separator():
assert tag.format.separator == ""
def _plamo3_call(name: str, arguments: str) -> str:
return (
BEGIN_TOOL_REQUEST
+ BEGIN_TOOL_NAME
+ name
+ END_TOOL_NAME
+ BEGIN_TOOL_ARGUMENTS
+ arguments
+ END_TOOL_ARGUMENTS
+ END_TOOL_REQUEST
)
def _plamo3_requests(*calls: str, eot: bool = False) -> str:
return (
BEGIN_TOOL_REQUESTS + "".join(calls) + END_TOOL_REQUESTS + (EOT if eot else "")
)
def _plamo3_grammar(tool_choice, tools):
tag = get_model_structural_tag(
model="plamo3",
tools=tools,
tool_choice=tool_choice,
reasoning=False,
)
assert isinstance(tag, StructuralTag)
return Grammar.from_structural_tag(tag)
def _plamo3_tools() -> list[ChatCompletionToolsParam]:
return [
ChatCompletionToolsParam(
type="function",
function={
"name": "get_weather",
"strict": True,
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
},
},
),
ChatCompletionToolsParam(
type="function",
function={
"name": "get_time",
"strict": True,
"parameters": {
"type": "object",
"properties": {"timezone": {"type": "string"}},
"required": ["timezone"],
},
},
),
]
def test_plamo3_registered_as_vllm_structural_tag_model():
assert "plamo3" in VLLM_BUILTIN_STRUCTURAL_TAG_MODELS
assert Plamo3EngineToolParser.structural_tag_model == "plamo3"
assert Plamo3EngineToolParser.supports_required_and_named is False
@pytest.mark.parametrize(
"body",
[
_plamo3_requests(_plamo3_call("get_weather", '{"city":"Tokyo"}')),
_plamo3_requests(
_plamo3_call("get_weather", '{"city":"Tokyo"}'),
_plamo3_call("get_time", '{"timezone":"Asia/Tokyo"}'),
eot=True,
),
],
)
def test_plamo3_required_accepts_complete_request_blocks(body: str):
assert _is_grammar_accept_string(_plamo3_grammar("required", _plamo3_tools()), body)
@pytest.mark.parametrize(
"body",
[
_plamo3_requests(_plamo3_call("unknown", '{"city":"Tokyo"}')),
_plamo3_requests(_plamo3_call("get_weather", "{}")),
BEGIN_TOOL_REQUESTS + _plamo3_call("get_weather", '{"city":"Tokyo"}'),
],
)
def test_plamo3_required_rejects_invalid_request_blocks(body: str):
assert not _is_grammar_accept_string(
_plamo3_grammar("required", _plamo3_tools()),
body,
)
def test_plamo3_auto_allows_text_or_a_tool_request_block(sample_tools_strict):
grammar = _plamo3_grammar("auto", sample_tools_strict)
assert _is_grammar_accept_string(grammar, "Plain response")
assert _is_grammar_accept_string(
grammar,
_plamo3_requests(_plamo3_call("get_weather", '{"city":"Tokyo"}')),
)
def test_plamo3_forced_stops_after_the_named_tool_call():
tools = _plamo3_tools()
tool_choice = ChatCompletionNamedToolChoiceParam(
function=ChatCompletionNamedFunction(name="get_weather")
)
grammar = _plamo3_grammar(tool_choice, tools)
assert _is_grammar_accept_string(
grammar,
_plamo3_requests(_plamo3_call("get_weather", '{"city":"Tokyo"}')),
)
assert not _is_grammar_accept_string(
grammar,
_plamo3_requests(
_plamo3_call("get_weather", '{"city":"Tokyo"}'),
_plamo3_call("get_time", '{"timezone":"Asia/Tokyo"}'),
),
)
# ---------------------------------------------------------------------------
# Kimi K3 (XTML channel format) structural tag
# ---------------------------------------------------------------------------
+60
View File
@@ -0,0 +1,60 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import importlib.util
from collections.abc import Iterator
from pathlib import Path
import pytest
import vllm.utils.flashinfer as fi
def _make_exe(path: Path) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.touch(mode=0o755)
@pytest.fixture
def default_cuda_home(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path:
"""Hide any real toolkit, keep ninja on PATH, redirect /usr/local/cuda."""
for var in ("CUDA_HOME", "CUDA_PATH", "FLASHINFER_NVCC"):
monkeypatch.delenv(var, raising=False)
_make_exe(tmp_path / "venv" / "bin" / "ninja")
monkeypatch.setenv("PATH", str(tmp_path / "venv" / "bin"))
home = tmp_path / "usr_local_cuda"
monkeypatch.setattr(fi, "_DEFAULT_CUDA_HOME", str(home))
return home
@pytest.fixture(autouse=True)
def flashinfer_without_cubin(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
"""Pretend flashinfer-python is installed without flashinfer-cubin."""
real_find_spec = importlib.util.find_spec
monkeypatch.setattr(
importlib.util,
"find_spec",
lambda name, *args: (
object() if name == "flashinfer" else real_find_spec(name, *args)
),
)
monkeypatch.setattr(fi, "has_flashinfer_cubin", lambda: False)
fi.has_flashinfer.cache_clear()
yield
fi.has_flashinfer.cache_clear()
def test_has_flashinfer_finds_toolkit_off_path(default_cuda_home: Path):
"""Bare-metal installs keep nvcc in /usr/local/cuda, off PATH, which is
where FlashInfer's JIT finds it."""
_make_exe(default_cuda_home / "bin" / "nvcc")
assert fi.has_flashinfer()
def test_has_flashinfer_requires_ninja(
default_cuda_home: Path, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
):
"""FlashInfer runs `ninja` from PATH, which lacks the venv's bin directory
when vLLM is launched without activating the venv."""
_make_exe(tmp_path / "cuda" / "bin" / "nvcc")
monkeypatch.setenv("PATH", str(tmp_path / "cuda" / "bin"))
assert not fi.has_flashinfer()
@@ -0,0 +1,152 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Routing/selection tests for the AITER ASM cprr DCP-verify route.
Companion to the upstream segmented-verify tests. Everything here is pure CPU:
it pins the *decisions* (which route a KV group takes, which head count the
kernel runs at, which configurations are refused) rather than kernel numerics.
The numerics parity test against the segmented route needs a GPU and both
routes built; it is marked and skipped when unavailable.
"""
from __future__ import annotations
import pytest
rocm_aiter_mla = pytest.importorskip(
"vllm.v1.attention.backends.mla.rocm_aiter_mla",
reason="ROCm AITER MLA backend not available",
)
_heads = rocm_aiter_mla._asm_dcp_verify_heads
_configured = rocm_aiter_mla._asm_dcp_verify_configured
_select_route = rocm_aiter_mla._select_dcp_decode_route
Route = rocm_aiter_mla._DCPDecodeRoute
NATIVE = rocm_aiter_mla._NATIVE_CPRR_HEADS
MIN_QLEN = rocm_aiter_mla._MIN_CPRR_QLEN
def test_dcp_verify_env_defaults_to_segmented(monkeypatch):
monkeypatch.delenv("VLLM_ROCM_AITER_MLA_DCP_VERIFY", raising=False)
assert rocm_aiter_mla.envs.VLLM_ROCM_AITER_MLA_DCP_VERIFY == "segmented"
def test_dcp_verify_env_rejects_unknown_route(monkeypatch):
monkeypatch.setenv("VLLM_ROCM_AITER_MLA_DCP_VERIFY", "unknown")
with pytest.raises(ValueError, match="Invalid value"):
_ = rocm_aiter_mla.envs.VLLM_ROCM_AITER_MLA_DCP_VERIFY
# --------------------------------------------------------------------------
# _asm_dcp_verify_heads -- which head count the cprr kernel runs at
# --------------------------------------------------------------------------
@pytest.mark.parametrize("n", NATIVE)
def test_native_head_counts_are_used_as_is(n):
assert _heads(n) == n
def test_non_native_head_count_pads_up_to_the_next_native():
"""K3 at TP8/DCP8 gathers 96 heads for the target; the kernel has no 96
variant, so it must run at 128, not silently fall out of the asm path."""
assert _heads(96) == 128
assert _heads(1) == 16
assert _heads(17) == 32
assert _heads(65) == 128
def test_head_count_above_the_largest_native_is_unservable():
"""0 means 'cannot serve'; anything else would route a shape the kernel
has no build for."""
assert _heads(max(NATIVE) + 1) == 0
assert _heads(256) == 0
# --------------------------------------------------------------------------
# _asm_dcp_verify_configured -- reachability
# --------------------------------------------------------------------------
@pytest.fixture
def on_gfx950(monkeypatch):
"""The cprr kernels are built only for gfx950, so the gate checks the arch.
Patched rather than detected, so both branches run on any runner."""
import vllm.platforms.rocm as rocm
monkeypatch.setattr(rocm, "on_gfx950", lambda: True)
def test_route_needs_dcp(on_gfx950):
assert _configured(dcp_world_size=1, cp_interleave=1, multi_token_decode=True) is (
False
)
assert _configured(dcp_world_size=8, cp_interleave=1, multi_token_decode=True) is (
True
)
def test_round_robin_interleave_other_than_one_is_excluded(on_gfx950):
"""The kernel's global-position causal window assumes interleave 1, the
same restriction the segmented gate carries."""
assert _configured(dcp_world_size=8, cp_interleave=2, multi_token_decode=True) is (
False
)
assert _configured(dcp_world_size=8, cp_interleave=16, multi_token_decode=True) is (
False
)
def test_route_needs_multi_token_decode(on_gfx950):
"""Single-token decode is already served. Enabling the route for it would
pad the head count on every step and buy nothing."""
assert _configured(dcp_world_size=8, cp_interleave=1, multi_token_decode=False) is (
False
)
def test_route_needs_gfx950(monkeypatch):
"""hsa/gfx942/mla and hsa/gfx1250/mla carry no cprr rows, so off gfx950 the
kernel lookup fails at the first verify step rather than falling back."""
import vllm.platforms.rocm as rocm
monkeypatch.setattr(rocm, "on_gfx950", lambda: False)
assert _configured(dcp_world_size=8, cp_interleave=1, multi_token_decode=True) is (
False
)
# --------------------------------------------------------------------------
# qlen floor
# --------------------------------------------------------------------------
def test_min_cprr_qlen_is_above_two():
"""Qlen 2 is below CPRR's floor and must use segmented verification."""
assert MIN_QLEN > 2
@pytest.mark.parametrize(
"supports,causal,qlen,asm,expected",
[
(True, True, 1, True, Route.PLAIN),
(True, True, 2, True, Route.SEGMENTED),
(True, True, 3, True, Route.CPRR),
(True, True, 4, True, Route.CPRR),
(True, True, 4, False, Route.SEGMENTED),
(True, False, 2, True, Route.PLAIN),
(True, False, 4, False, Route.PLAIN),
(False, True, 3, True, Route.CPRR),
],
)
def test_dcp_decode_route(supports, causal, qlen, asm, expected):
assert _select_route(supports, causal, qlen, asm) is expected
def test_causal_multi_token_batch_without_a_valid_route_fails():
with pytest.raises(RuntimeError, match="requires either segmented MLA"):
_select_route(False, True, 2, True)
# Kernel numerics live in test_rocm_aiter_mla_dcp_cprr_numerics.py, which runs
# the real builder and the real asm kernel on a gfx950 GPU.
@@ -0,0 +1,540 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""GPU numerics for the AITER ASM cprr DCP-verify route.
Runs the REAL metadata builder (``AiterMLAMetadataBuilder.build``, so
``get_mla_metadata_v1`` is exercised with the cprr kwargs and the persistent
schedule buffers are produced) and the REAL asm kernel, on ONE GPU with the DCP
ranks simulated one after another.
That is legitimate because the aiter MLA backend contains no collective on this
path: the query all-gather happens in the *layer*, strictly before
``forward_mqa``, so the head fold, the round-robin kernel call and the un-fold
are pure per-rank local arithmetic in which ``cp_world_size``/``cp_rank`` are
just numbers feeding the causal-mask maths. Restricting a shard to a residue
class of global positions is exactly what DCP rank r sees.
Three properties are checked:
* each shard matches an exact torch reference over the positions it holds,
with causality applied on the GLOBAL position (the thing cprr reconstructs
in-kernel and the segmented route instead carries in per-row lengths);
* the LSE merge of all shards reproduces full-context attention;
* a shard told the wrong ``cp_rank`` fails both -- without this positive
control a passing run proves nothing, since a kernel that ignored global
positions still produces plausible-looking output.
"""
from __future__ import annotations
from types import SimpleNamespace
import pytest
import torch
from vllm.platforms import current_platform
if not current_platform.is_rocm():
pytest.skip("ROCm AITER MLA tests", allow_module_level=True)
from vllm._aiter_ops import is_aiter_found # noqa: E402
from vllm.v1.attention.backends.mla import rocm_aiter_mla # noqa: E402
KV_LORA_RANK = 512
QK_ROPE_HEAD_DIM = 64
QK_NOPE_HEAD_DIM = 128
HEAD_SIZE = KV_LORA_RANK + QK_ROPE_HEAD_DIM
SCALE = (QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM) ** -0.5
DCP = 8
NUM_REQS = 8
CTX = 256
QLEN = 5 # 1 + num_speculative_tokens; must clear _MIN_CPRR_QLEN
# fp8-e4m3 keeps 3 mantissa bits, so the relative error floor is ~6.2e-2. Gates
# are relative for the same reason: an absolute bound would just be a bound on
# the output magnitude.
SHARD_RTOL = 1.0e-1
MERGE_RTOL = 5.0e-2
MODEL = "deepseek-ai/DeepSeek-R1"
def _gpu_available() -> bool:
return is_aiter_found() and torch.accelerator.is_available()
def _on_gfx950() -> bool:
if not is_aiter_found():
return False
from vllm.platforms.rocm import on_gfx950
return on_gfx950()
requires_cprr = pytest.mark.skipif(
not (_gpu_available() and _on_gfx950()),
reason="cprr DCP verify needs ROCm + AITER on gfx950",
)
@pytest.fixture(autouse=True)
def select_asm_route(monkeypatch):
monkeypatch.setenv("VLLM_ROCM_AITER_MLA_DCP_VERIFY", "asm")
# --------------------------------------------------------------------------
# reference maths
# --------------------------------------------------------------------------
def _torch_reference(q_f32, kv_f32, ctx, qlen, scale, positions):
"""Exact absorbed-MLA decode over an explicit set of GLOBAL KV positions.
A shard sees only ``positions``, but causality still uses the global
position: query token i (global position ctx + i) may attend to any p in
positions with p <= ctx + i. Restricting ``positions`` to a round-robin
residue class is precisely DCP rank r's view.
"""
nr, _, heads, _ = q_f32.shape
dev = q_f32.device
o = torch.zeros(nr, qlen, heads, KV_LORA_RANK, dtype=torch.float32, device=dev)
lse = torch.full((nr, qlen, heads), float("-inf"), dtype=torch.float32, device=dev)
for r in range(nr):
kv_sel = kv_f32[r].index_select(0, positions)
for i in range(qlen):
valid = positions <= (ctx + i)
if not bool(valid.any()):
continue
k = kv_sel[valid]
s = (q_f32[r, i] @ k.T) * scale
m = s.max(dim=-1, keepdim=True).values
p = torch.exp(s - m)
denom = p.sum(dim=-1, keepdim=True)
o[r, i] = (p / denom) @ k[:, :KV_LORA_RANK]
lse[r, i] = m.squeeze(-1) + torch.log(denom.squeeze(-1))
return o, lse
def _merge_partials(os_, lses):
"""Cross-shard softmax merge (natural-log LSE)."""
stack_lse = torch.stack(lses)
m = stack_lse.max(dim=0).values
w = torch.exp(stack_lse - m)
num = (w.unsqueeze(-1) * torch.stack(os_)).sum(dim=0)
den = w.sum(dim=0).unsqueeze(-1)
return num / den, m + torch.log(w.sum(dim=0))
# --------------------------------------------------------------------------
# single-process rank simulation
# --------------------------------------------------------------------------
class _FakeGroup:
def __init__(self, world_size: int, rank: int):
self.world_size = world_size
self.rank_in_group = rank
self.rank = rank
self.local_rank = rank
def all_gather(self, tensor, dim=0):
raise RuntimeError(
"collective called in a single-process test -- the path under test "
"is supposed to be collective-free; this indicates a real change"
)
def _install_fake_groups(dcp_size: int, cp_rank: int, tp_size: int = 1) -> None:
"""Point vLLM's parallel_state at rank-simulating stubs (no NCCL)."""
import vllm.distributed.parallel_state as ps
import vllm.model_executor.layers.attention.mla_attention as mla_attention
import vllm.v1.attention.backends.mla.rocm_aiter_mla as backend
dcp = _FakeGroup(dcp_size, cp_rank)
tp = _FakeGroup(tp_size, cp_rank)
ps.get_dcp_group = lambda: dcp
ps.get_tp_group = lambda: tp
# Both of these imported the symbol directly at module import time, so
# patching parallel_state alone leaves them resolving the real group (and
# the builder then reports dcp_world_size 1, silently skipping the route).
backend.get_dcp_group = lambda: dcp
mla_attention.get_dcp_group = lambda: dcp
@pytest.fixture(autouse=True)
def _restore_parallel_groups():
"""Undo _install_fake_groups after every test.
It patches module globals in three modules, so without this the fake DCP
group leaks into the rest of the pytest session and silently breaks
unrelated tests that expect a real (or absent) group.
"""
import vllm.distributed.parallel_state as ps
import vllm.model_executor.layers.attention.mla_attention as mla_attention
import vllm.v1.attention.backends.mla.rocm_aiter_mla as backend
saved = [
(mod, name, getattr(mod, name))
for mod, name in (
(ps, "get_dcp_group"),
(ps, "get_tp_group"),
(backend, "get_dcp_group"),
(mla_attention, "get_dcp_group"),
)
]
yield
for mod, name, value in saved:
setattr(mod, name, value)
class _PrefillBackendStub:
"""The base builder does ``attention_layer.prefill_backend.clone()``.
prefill_backend is consulted only from the chunked-context prefill path,
which a decode-only test never enters.
"""
def clone(self):
return self
def _make_ctx_layer(vllm_config):
"""Stand-in for the registered attention layer in static_forward_context.
``MLACommonMetadataBuilder.__init__`` reads the MLA dims, ``dcp_manager``
and ``prefill_backend`` off this object. ``dcp_manager`` is the Direct-DCP
symmetric-memory workspace, used ONLY by the chunked-prefill KV all-gather;
it is constructed without running __init__ so no peer pointers are exchanged
and no collective is issued. That is what keeps this test single-process.
"""
from vllm.v1.attention.ops.dcp import MLADCPManager
mgr = object.__new__(MLADCPManager)
mgr.init_kv_gather = lambda *a, **k: None
return SimpleNamespace(
non_causal_multi_token_decode=False,
q_lora_rank=None,
kv_lora_rank=KV_LORA_RANK,
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
v_head_dim=QK_NOPE_HEAD_DIM,
dcp_manager=mgr,
prefill_backend=_PrefillBackendStub(),
)
def _ensure_workspace_manager(device: torch.device) -> None:
"""Init the global workspace manager, which GPUModelRunner does in production.
``AiterMLAMetadataBuilder.__init__`` reserves the fp8 prefill
persistent-scheduling scratch through ``current_workspace_manager()``, so
merely constructing a builder outside a serve trips its assert. Idempotent:
one builder is constructed per DCP shard. Deliberately does NOT lock the
workspace -- production locks only after warmup, and this keeps allocating
across shards.
"""
from vllm.v1.worker.workspace import (
init_workspace_manager,
is_workspace_manager_initialized,
)
if not is_workspace_manager_initialized():
init_workspace_manager(device)
class _Layer:
def __init__(self, device):
self._q_scale = torch.tensor(1.0, device=device)
self._k_scale = torch.tensor(1.0, device=device)
self._v_scale = torch.tensor(1.0, device=device)
def _build_impl(num_heads: int, dcp: int, cp_rank: int):
"""Real AiterMLAImpl.forward_mqa bound to a minimally-populated instance.
Constructing the full impl would drag in kv_b_proj and the rest of the
weight-dependent MLA setup -- i.e. model loading, which this test exists to
avoid. forward_mqa's fold path reads only the attributes set below.
"""
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAImpl
impl = object.__new__(AiterMLAImpl)
impl.kv_lora_rank = KV_LORA_RANK
impl.qk_rope_head_dim = QK_ROPE_HEAD_DIM
impl.num_heads = num_heads
impl.dcp_world_size = dcp
impl.dcp_rank = cp_rank
impl.pcp_world_size = 1
impl.kv_cache_dtype = "fp8"
impl.scale = SCALE
# `_decode_num_heads` is a read-only property deriving num_heads *
# dcp_world_size, both set above; assert rather than assign.
assert impl._decode_num_heads == num_heads * dcp
return impl
def _make_vllm_config(heads_per_rank: int):
"""A DCP8 MLA config with no weights and no distributed init.
DeepSeek-R1's config supplies the MLA dims (kv_lora_rank 512, rope 64,
nope 128) that the aiter asm kernel is built for; only the head count is
overridden, to place the GATHERED count (heads_per_rank * DCP) on the
kernel variant under test. ``get_num_attention_heads`` reads
model_arch_config, a snapshot taken at config-creation time, so patching
hf_config alone would be inert.
"""
from tests.v1.attention.utils import create_vllm_config
vllm_config = create_vllm_config(
model_name=MODEL,
tensor_parallel_size=1,
max_model_len=2048,
max_num_seqs=NUM_REQS * 2,
max_num_batched_tokens=4096,
)
vllm_config.model_config.model_arch_config.total_num_attention_heads = (
heads_per_rank
)
# Without a speculative config the batch reorder threshold stays at 1 and a
# qlen-QLEN row is classified as a PREFILL, so the decode path under test is
# never reached. _init_reorder_batch_threshold reads exactly these two
# fields, so a stand-in is enough and avoids pulling in a draft model.
vllm_config.speculative_config = SimpleNamespace(
num_speculative_tokens=QLEN - 1, parallel_drafting=False
)
vllm_config.cache_config.cache_dtype = "fp8"
vllm_config.parallel_config.decode_context_parallel_size = DCP
vllm_config.parallel_config.cp_kv_cache_interleave_size = 1
return vllm_config
def _run_shard(
vllm_config,
spec,
layer_name,
device,
cp_rank,
kern_rank,
q_flat,
kv_flat,
nr,
qlen,
ctx,
dcp,
pos,
):
"""Build real metadata for one DCP rank and run the real asm kernel.
``kern_rank`` is separate from ``cp_rank`` so the positive control can hand
rank r's pages to the kernel while telling it that it is rank r+1.
"""
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAMetadataBuilder
L = ctx + qlen
local_len = int(pos.numel())
# One GLOBAL kv cache: page (req*L + p) holds request req position p. Shards
# select pages by residue class, so every shard reads the same underlying
# bytes and no re-quantisation noise is introduced between arms.
block_table = (
torch.arange(nr, device=device).view(nr, 1) * L + pos.view(1, -1)
).to(torch.int32)
# seq_lens is the GLOBAL context; dcp_local_seq_lens is what this rank
# actually holds. Passing the global length as the local one walks the
# kernel off the end of the page table.
seq_lens_cpu = torch.full((nr,), L, dtype=torch.int32)
local_cpu = torch.full((nr,), local_len, dtype=torch.int32)
common = CommonAttentionMetadata(
query_start_loc=torch.arange(
0, nr * qlen + 1, qlen, dtype=torch.int32, device=device
),
query_start_loc_cpu=torch.arange(0, nr * qlen + 1, qlen, dtype=torch.int32),
seq_lens=seq_lens_cpu.to(device),
num_reqs=nr,
num_actual_tokens=nr * qlen,
max_query_len=qlen,
max_seq_len=L,
block_table_tensor=block_table,
slot_mapping=torch.arange(nr * qlen, dtype=torch.int64, device=device),
causal=True,
dcp_local_seq_lens=local_cpu.to(device),
dcp_local_seq_lens_cpu_upper_bound=local_cpu,
)
_install_fake_groups(dcp, kern_rank)
builder = AiterMLAMetadataBuilder(spec, [layer_name], vllm_config, device)
md = builder.build(0, common)
impl = _build_impl(builder.num_heads, dcp, kern_rank)
o, lse = impl.forward_mqa(q_flat, kv_flat, md, _Layer(device))
torch.accelerator.synchronize()
return md, o, lse, local_len
def _numerics(heads_per_rank: int, sabotage: bool = False):
"""Run every DCP shard and return (per-shard max rel err, merged rel err)."""
import aiter
from vllm.config import set_current_vllm_config
device = torch.device("cuda", 0)
torch.accelerator.set_device_index(device.index)
torch.manual_seed(0)
_ensure_workspace_manager(device)
_install_fake_groups(DCP, 0)
vllm_config = _make_vllm_config(heads_per_rank)
with set_current_vllm_config(vllm_config):
from vllm.v1.kv_cache_interface import MLAAttentionSpec
layer_name = "model.layers.0.self_attn.attn"
vllm_config.compilation_config.static_forward_context[layer_name] = (
_make_ctx_layer(vllm_config)
)
# block_size 1: one page per token, so page ids are positions and the
# residue-class page table is expressible directly.
spec = MLAAttentionSpec(
block_size=1,
num_kv_heads=1,
head_size=HEAD_SIZE,
dtype=vllm_config.model_config.dtype,
non_causal_multi_token_decode=False,
)
nr, ctx, qlen = NUM_REQS, CTX, QLEN
L = ctx + qlen
gathered_heads = heads_per_rank * DCP
kv_dtype = aiter.dtypes.fp8
kv_flat = torch.randn(
nr * L, 1, HEAD_SIZE, dtype=torch.float32, device=device
).to(kv_dtype)
q_flat = (
torch.randn(
nr * qlen,
gathered_heads,
HEAD_SIZE,
dtype=torch.float32,
device=device,
)
.mul_(0.5)
.to(kv_dtype)
)
# Dequantise for the reference so the comparison measures the kernel,
# not the fp8 cast that both sides share.
kv_f32 = kv_flat.float().view(nr, L, HEAD_SIZE)
q_f32 = q_flat.float().view(nr, qlen, gathered_heads, HEAD_SIZE)
ref_o, _ = _torch_reference(
q_f32, kv_f32, ctx, qlen, SCALE, torch.arange(L, device=device)
)
shard_o, shard_lse = [], []
max_shard_rel = 0.0
for r in range(DCP):
pos = torch.arange(r, L, DCP, device=device) # residue class r
kern_rank = (r + 1) % DCP if sabotage else r
md, o, lse, _ = _run_shard(
vllm_config,
spec,
layer_name,
device,
r,
kern_rank,
q_flat,
kv_flat,
nr,
qlen,
ctx,
DCP,
pos,
)
# The route actually under test must have been taken; a silent
# fall-out to Triton/segmented would otherwise pass quietly.
assert md.decode.dcp_route is rocm_aiter_mla._DCPDecodeRoute.CPRR
assert md.decode.asm_decode_num_heads, "cprr asm route was not taken"
o = o.float().view(nr, qlen, gathered_heads, KV_LORA_RANK)
shard_lse.append(lse.float().view(nr, qlen, gathered_heads))
shard_o.append(o)
po, _ = _torch_reference(q_f32, kv_f32, ctx, qlen, SCALE, pos)
rel = (o - po).abs().max().item() / max(po.abs().max().item(), 1e-6)
max_shard_rel = max(max_shard_rel, rel)
merged, _ = _merge_partials(shard_o, shard_lse)
merge_rel = (merged - ref_o).abs().max().item() / max(
ref_o.abs().max().item(), 1e-6
)
return max_shard_rel, merge_rel
# --------------------------------------------------------------------------
# tests
# --------------------------------------------------------------------------
@requires_cprr
@pytest.mark.parametrize(
"heads_per_rank",
[
16, # 128 gathered heads -- a NATIVE cprr kernel, no padding
12, # 96 gathered heads -- K3's DSpark target, must pad up to 128
],
ids=["native-128", "padded-96-to-128"],
)
@torch.inference_mode()
def test_cprr_shards_match_reference_and_merge_to_full_context(heads_per_rank):
"""Every DCP shard must be right, and their LSE merge must reproduce
full-context attention.
A shard that ignored global positions (applying a local causal tail
instead) still produces plausible-looking output; it only diverges from the
reference on the tokens whose causal window is truncated. Checking the merge
alone would hide that, so both are asserted.
"""
shard_rel, merge_rel = _numerics(heads_per_rank)
assert shard_rel < SHARD_RTOL, f"per-shard rel err {shard_rel:.3e}"
assert merge_rel < MERGE_RTOL, f"merged rel err {merge_rel:.3e}"
@requires_cprr
@torch.inference_mode()
def test_wrong_cp_rank_breaks_the_result():
"""Positive control: without it, a passing run above proves nothing.
Feeding shard r's pages to the kernel while telling it that it is rank r+1
shifts every reconstructed global position by one, so both the causal mask
and the round-robin reconstruction are wrong. If the error does NOT blow up,
the kernel is not using cp_rank and the test above has no power.
"""
shard_rel, merge_rel = _numerics(16, sabotage=True)
assert shard_rel > SHARD_RTOL or merge_rel > MERGE_RTOL, (
f"sabotaged run still passed (shard {shard_rel:.3e}, merge "
f"{merge_rel:.3e}) -- cp_rank is not reaching the kernel"
)
@requires_cprr
def test_cprr_kernel_is_selected_for_the_k3_shape():
"""The K3 DSpark target shape must actually reach the cprr route.
Guards against a silent fall-out to segmented/Triton, which is what the
stock configuration does and what this route exists to avoid.
"""
from vllm.v1.attention.backends.mla import rocm_aiter_mla as m
assert (
m._asm_dcp_verify_configured(
dcp_world_size=DCP, cp_interleave=1, multi_token_decode=True
)
is True
)
assert QLEN >= m._MIN_CPRR_QLEN
assert m._select_dcp_decode_route(True, True, QLEN, True) is m._DCPDecodeRoute.CPRR
assert m._asm_dcp_verify_heads(96) in m._NATIVE_CPRR_HEADS
@@ -87,21 +87,31 @@ def _builder(
kv_cache_dtype: str = "auto",
dcp_world_size: int = 1,
dcp_rank: int = 0,
asm_dcp_verify: bool = False,
decode_causal: bool = True,
supports_segmented_dcp_verify: bool | None = None,
):
decode_num_heads = num_heads * dcp_world_size
asm_heads = (
rocm_aiter_mla._asm_dcp_verify_heads(decode_num_heads) if asm_dcp_verify else 0
)
if supports_segmented_dcp_verify is None:
supports_segmented_dcp_verify = rocm_aiter_mla._segmented_dcp_verify_supported(
dcp_world_size, 1
)
stub = SimpleNamespace(
device=torch.device("cpu"),
num_heads=num_heads,
# DCP gathers every rank's head shard before decode, and the routing
# predicates read the gathered count.
_decode_num_heads=num_heads * dcp_world_size,
_decode_num_heads=decode_num_heads,
dcp_world_size=dcp_world_size,
dcp_rank=dcp_rank,
cp_kv_cache_interleave_size=1,
# Mirrors the production constructor: configuration decides whether the
# segmented route is available, query length decides per batch.
_supports_segmented_dcp_verify=rocm_aiter_mla._segmented_dcp_verify_supported(
dcp_world_size, 1
),
# segmented route is available, query length decides per batch. CPRR
# preferred still keeps this True so qlen < _MIN_CPRR_QLEN can fall back.
_supports_segmented_dcp_verify=supports_segmented_dcp_verify,
# Derived once in the real constructor, so derive it once here too.
_segmented_page_size=rocm_aiter_mla._segmented_mla_page_size(kernel_block_size),
_dcp_verify_buffers=None,
@@ -122,7 +132,11 @@ def _builder(
_uniform_padded_mtp_qo_len=(AiterMLAMetadataBuilder._uniform_padded_mtp_qo_len),
_use_persistent_metadata=False,
kernel_block_size=kernel_block_size,
_num_attention_heads=AiterMLAHelper.get_actual_mla_num_heads(num_heads),
_num_attention_heads=(
asm_heads
if asm_dcp_verify
else AiterMLAHelper.get_actual_mla_num_heads(num_heads)
),
_mla_work_meta_data=torch.empty(1, dtype=torch.int32),
_mla_work_info_set=torch.empty(1, dtype=torch.int32),
_mla_work_indptr=torch.empty(1, dtype=torch.int32),
@@ -132,7 +146,19 @@ def _builder(
_mla_q_dtype=torch.bfloat16,
_mla_kv_dtype=torch.bfloat16,
decode_attn_out_dtype=torch.bfloat16,
_decode_causal=True,
_decode_causal=decode_causal,
# Default off so existing cases keep exercising segmented/persistent.
# Pass asm_dcp_verify=True for the CPRR route and its qlen-floor fallback.
_asm_dcp_verify=asm_dcp_verify,
_asm_dcp_verify_heads=asm_heads,
_g_kv_indptr_buf=(
torch.zeros(max_decode_rows + 1, dtype=torch.int32)
if asm_dcp_verify and dcp_world_size > 1
else None
),
# Only read when the asm route is on; the real builder derives it from
# the device CU count.
_mla_max_split_per_batch=256,
)
# Bound to the stub rather than faked: the verify flatten's per-row view is
# part of what _build_decode is being tested for.
@@ -287,6 +313,209 @@ def test_dcp_fp8_verify_build_uses_segmented(monkeypatch):
assert metadata.dcp_verify is not None
def _cprr_decode_batch(builder, *, qlen: int, num_reqs: int = 1):
seq_lens = torch.full((num_reqs,), 16, dtype=torch.int32)
query_start_loc = torch.arange(0, (num_reqs + 1) * qlen, qlen, dtype=torch.int32)
return AiterMLAMetadataBuilder._build_decode(
builder,
block_table_tensor=torch.zeros(num_reqs, 8, dtype=torch.int32),
seq_lens_device=seq_lens,
max_seq_len=16,
query_start_loc_cpu=query_start_loc,
query_start_loc_device=query_start_loc,
num_decode_tokens=num_reqs * qlen,
max_query_len=qlen,
dcp_tot_seq_lens_device=seq_lens * 8,
)
def test_asm_preferred_qlen2_falls_back_to_segmented(monkeypatch):
"""K=4 can clamp to qlen 2; CPRR has no kernel, so use segmented MLA."""
monkeypatch.setattr(rocm_aiter_mla, "_segmented_mla_decode_supported", lambda: True)
get_mla_metadata_v1 = mock.MagicMock()
monkeypatch.setitem(
sys.modules,
"aiter",
SimpleNamespace(get_mla_metadata_v1=get_mla_metadata_v1),
)
metadata = _cprr_decode_batch(
_builder(
mtp_decode_qlen=5,
dcp_world_size=8,
num_heads=12,
asm_dcp_verify=True,
kernel_block_size=2,
),
qlen=2,
)
assert metadata.dcp_verify is not None
assert metadata.g_kv_indptr is None
assert not metadata.has_persistent_metadata
get_mla_metadata_v1.assert_not_called()
def test_asm_preferred_qlen_at_cprr_floor_stays_on_cprr(monkeypatch):
monkeypatch.setattr(rocm_aiter_mla, "_segmented_mla_decode_supported", lambda: True)
get_mla_metadata_v1 = mock.MagicMock()
monkeypatch.setitem(
sys.modules,
"aiter",
SimpleNamespace(get_mla_metadata_v1=get_mla_metadata_v1),
)
monkeypatch.setattr(
rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel()
)
metadata = _cprr_decode_batch(
_builder(
mtp_decode_qlen=5,
dcp_world_size=8,
num_heads=12,
asm_dcp_verify=True,
),
qlen=rocm_aiter_mla._MIN_CPRR_QLEN,
)
assert metadata.dcp_verify is None
assert metadata.g_kv_indptr is not None
assert metadata.has_persistent_metadata
assert get_mla_metadata_v1.call_args.kwargs.get("is_cp_round_robin") is True
def test_cprr_capable_single_token_decode_keeps_persistent_metadata(monkeypatch):
"""The CPRR qlen floor must not suppress the ordinary qlen-1 schedule."""
get_mla_metadata_v1 = mock.MagicMock()
monkeypatch.setitem(
sys.modules,
"aiter",
SimpleNamespace(get_mla_metadata_v1=get_mla_metadata_v1),
)
monkeypatch.setattr(
rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel()
)
metadata = _cprr_decode_batch(
_builder(
mtp_decode_qlen=5,
dcp_world_size=8,
num_heads=12,
asm_dcp_verify=True,
),
qlen=1,
)
assert metadata.g_kv_indptr is None
assert metadata.has_persistent_metadata
assert get_mla_metadata_v1.call_args.kwargs.get("is_cp_round_robin") is None
def test_non_causal_dcp_block_uses_plain_mask0_metadata(monkeypatch):
"""A non-causal block needs no CPRR global-position window or head pad."""
get_mla_metadata_v1 = mock.MagicMock()
monkeypatch.setitem(
sys.modules,
"aiter",
SimpleNamespace(get_mla_metadata_v1=get_mla_metadata_v1),
)
monkeypatch.setattr(
rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel()
)
builder = _builder(
mtp_decode_qlen=5,
dcp_world_size=8,
num_heads=12,
asm_dcp_verify=True,
)
builder._decode_causal = False
metadata = _cprr_decode_batch(builder, qlen=4)
assert metadata.dcp_verify is None
assert metadata.g_kv_indptr is None
assert metadata.has_persistent_metadata
assert get_mla_metadata_v1.call_args.args[3] == 96
assert get_mla_metadata_v1.call_args.args[5] is False
assert get_mla_metadata_v1.call_args.kwargs.get("is_cp_round_robin") is None
def test_asm_qlen2_without_segmented_fails_during_build(monkeypatch):
monkeypatch.setitem(
sys.modules,
"aiter",
SimpleNamespace(get_mla_metadata_v1=mock.MagicMock()),
)
monkeypatch.setattr(
rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel()
)
with pytest.raises(RuntimeError, match="requires either segmented MLA"):
_cprr_decode_batch(
_builder(
mtp_decode_qlen=5,
dcp_world_size=8,
num_heads=12,
asm_dcp_verify=True,
supports_segmented_dcp_verify=False,
),
qlen=2,
)
def test_non_causal_dcp_block_bypasses_cprr_in_forward(monkeypatch):
"""Process-level CPRR capability must not pad or route a non-causal batch."""
captured = {}
def fake_aiter_decode(q, kv_buffer, out, *args, **kwargs):
captured["q_heads"] = q.shape[1]
captured["kwargs"] = kwargs
return None, torch.zeros(q.shape[0], q.shape[1])
monkeypatch.setattr(
rocm_aiter_mla, "_get_aiter_mla_decode", lambda: fake_aiter_decode
)
impl = object.__new__(AiterMLAImpl)
impl.num_heads = 12
impl.dcp_world_size = 8
impl.kv_cache_dtype = "auto"
impl.kv_lora_rank = 512
impl.qk_rope_head_dim = 64
impl.scale = 576**-0.5
decode = SimpleNamespace(
max_qo_len=4,
qo_indptr=torch.tensor([0, 4], dtype=torch.int32),
paged_kv_indptr=torch.tensor([0, 1], dtype=torch.int32),
paged_kv_indices=torch.zeros(1, dtype=torch.int32),
paged_kv_last_page_len=torch.ones(1, dtype=torch.int32),
use_gluon_decode=False,
use_gluon_verify=False,
dcp_route=rocm_aiter_mla._DCPDecodeRoute.PLAIN,
dcp_verify=None,
g_kv_indptr=None,
has_persistent_metadata=False,
attn_out_dtype=torch.bfloat16,
asm_decode_num_heads=0,
mla_num_kv_splits=0,
cp_world_size=8,
cp_rank=0,
min_kv_seq_len=1,
)
attn_metadata = SimpleNamespace(decode=decode, causal=False, work_meta_data=None)
layer = SimpleNamespace(_q_scale=torch.tensor(1.0), _k_scale=torch.tensor(1.0))
q = torch.zeros(4, 96, 576, dtype=torch.bfloat16)
output, lse = impl.forward_mqa(q, torch.zeros(1, 1, 576), attn_metadata, layer)
assert captured["q_heads"] == 96
assert captured["kwargs"]["causal"] is False
assert "g_kv_indptr" not in captured["kwargs"]
assert "num_kv_splits" not in captured["kwargs"]
assert output.shape[1] == 96
assert lse is not None and lse.shape == (4, 96)
def test_single_token_dcp_decode_returns_unpadded_lse(monkeypatch):
"""Single-token DCP decode must come back with an LSE the merge can use.
@@ -325,9 +554,14 @@ def test_single_token_dcp_decode_returns_unpadded_lse(monkeypatch):
paged_kv_last_page_len=torch.ones(num_tokens, dtype=torch.int32),
use_gluon_decode=False,
use_gluon_verify=False,
dcp_route=rocm_aiter_mla._DCPDecodeRoute.PLAIN,
dcp_verify=None,
has_persistent_metadata=False,
attn_out_dtype=torch.bfloat16,
# 0 == the round-robin (cprr) asm route is off, this rank's decode takes
# the existing path. Mirrors the AiterMLADecodeMetadata default.
asm_decode_num_heads=0,
g_kv_indptr=None,
)
attn_metadata = SimpleNamespace(decode=decode, causal=True, work_meta_data=None)
layer = SimpleNamespace(_q_scale=torch.tensor(1.0), _k_scale=torch.tensor(1.0))
@@ -453,6 +687,7 @@ def test_segmented_dcp_verify_matches_causal_attention(monkeypatch):
),
use_gluon_decode=False,
use_gluon_verify=False,
dcp_route=rocm_aiter_mla._DCPDecodeRoute.SEGMENTED,
dcp_verify=view,
attn_out_dtype=torch.bfloat16,
)
@@ -0,0 +1,326 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(), reason="ROCm-only aiter path"
)
def _rows_from_ragged(indices: torch.Tensor, indptr: torch.Tensor) -> list[list[int]]:
ends = indptr.tolist()
return [indices[s:e].tolist() for s, e in zip(ends, ends[1:])]
def test_triton_sparse_mla_gate(monkeypatch) -> None:
import vllm._aiter_ops as aiter_ops
from vllm._aiter_ops import rocm_aiter_ops
from vllm.platforms.rocm import on_gfx950
warnings: list[str] = []
monkeypatch.setattr(
aiter_ops.logger,
"warning_once",
lambda msg, *args, **kwargs: warnings.append(msg % args),
)
def enabled(aiter: bool, flag: bool) -> bool:
monkeypatch.setenv("VLLM_ROCM_USE_AITER", str(int(aiter)))
monkeypatch.setenv("VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA", str(int(flag)))
rocm_aiter_ops.refresh_env_variables()
return bool(rocm_aiter_ops.is_triton_sparse_mla_enabled())
try:
assert not enabled(aiter=True, flag=False) and not warnings
assert not enabled(aiter=False, flag=True)
assert "VLLM_ROCM_USE_AITER is off" in warnings.pop()
# On gfx950 the flag takes effect; elsewhere it warns and falls back.
assert enabled(aiter=True, flag=True) == on_gfx950()
if not on_gfx950():
assert "gfx950" in warnings.pop()
finally:
monkeypatch.undo()
rocm_aiter_ops.refresh_env_variables()
@pytest.mark.parametrize(
"rope_dim",
[0, 64],
ids=["rope_free", "appended_rope"], # GLM-5.3-Flash; GLM-5.1/5.2, V3.2
)
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
@torch.inference_mode()
def test_forward_mqa_prepares_triton_sparse_mla_inputs(
monkeypatch, rope_dim: int, kv_cache_dtype: str
) -> None:
"""What forward_mqa hands aiter: global slots built from the request-local
top-k rows, q quantized with the layer's scale, and the cache as stored."""
from vllm._aiter_ops import rocm_aiter_ops
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
ROCMAiterMLASparseImpl,
)
device = torch.device("cuda")
gen = torch.Generator().manual_seed(0)
kv_lora_rank, num_heads, topk, block_size = 512, 16, 128, 16
head_size = kv_lora_rank + rope_dim
fp8 = kv_cache_dtype == "fp8"
# A decode row, then a short prefill.
query_lens, seq_lens = [1, 5], [40, 20]
num_tokens = sum(query_lens)
block_table = torch.randperm(8, generator=gen).view(2, 4).to(device, torch.int32)
topk_indices = torch.full((num_tokens, topk), -1, dtype=torch.int32)
req_id_per_token: list[int] = []
rows: list[list[int]] = []
for req, (query_len, seq_len) in enumerate(zip(query_lens, seq_lens)):
for pos in range(seq_len - query_len, seq_len):
local = torch.randperm(pos + 1, generator=gen)
topk_indices[len(rows), : pos + 1] = local.int()
slots = block_table[req, local // block_size].cpu() * block_size
rows.append((slots + local % block_size).tolist())
req_id_per_token.append(req)
indptr = torch.tensor([0] + [len(row) for row in rows]).cumsum(0)
impl = ROCMAiterMLASparseImpl.__new__(ROCMAiterMLASparseImpl)
impl.num_heads = num_heads
impl.kv_lora_rank = kv_lora_rank
impl.qk_rope_head_dim = rope_dim
impl.scale = head_size**-0.5
impl.sinks = torch.randn(num_heads, generator=gen).to(device)
impl.kv_cache_dtype = kv_cache_dtype
impl.topk_indices_buffer = topk_indices.to(device)
impl.use_aiter_sparse_mla = True
layer = SimpleNamespace(
_q_scale=torch.tensor([0.02], device=device),
_k_scale=torch.tensor([0.01], device=device),
)
metadata = SimpleNamespace(
num_actual_tokens=num_tokens,
topk_tokens=topk,
req_id_per_token=torch.tensor(req_id_per_token, dtype=torch.int32).to(device),
block_table=block_table,
block_size=block_size,
paged_kv_indptr=indptr.to(device, torch.int32),
paged_kv_indices=torch.zeros(num_tokens * topk, dtype=torch.int32).to(device),
attn_out_dtype=torch.bfloat16,
)
cache_dtype = current_platform.fp8_dtype() if fp8 else torch.bfloat16
kv_cache = torch.zeros(8, block_size, head_size, device=device).to(cache_dtype)
# Two rows of cudagraph padding past the real tokens.
q = torch.randn(num_tokens + 2, num_heads, head_size, generator=gen).to(
device, torch.bfloat16
)
calls = []
monkeypatch.setattr(
rocm_aiter_ops, "triton_sparse_mla_fwd", lambda *a, **k: calls.append((a, k))
)
out, lse = impl.forward_mqa(q, kv_cache, metadata, layer)
assert len(calls) == 1
(q_in, kv_in, o, sm_scale, kv_indptr, kv_indices), kwargs = calls[0]
assert _rows_from_ragged(kv_indices, kv_indptr) == rows
assert q_in.shape[0] == num_tokens
if fp8:
assert q_in.dtype == cache_dtype
torch.testing.assert_close(
q_in.float() * layer._q_scale, q[:num_tokens].float(), atol=1e-3, rtol=0.07
)
else:
assert q_in.data_ptr() == q.data_ptr()
assert kv_in.shape == (8 * block_size, 1, 1, head_size)
assert kv_in.data_ptr() == kv_cache.data_ptr()
assert o.data_ptr() == out.data_ptr() and o.shape[0] == num_tokens
assert sm_scale == impl.scale
assert kwargs["kv_lora_rank"] == kv_lora_rank
assert kwargs["qk_rope_head_dim"] == rope_dim
assert kwargs["q_scale"] is layer._q_scale
assert kwargs["kv_scale"] is layer._k_scale
assert kwargs["attn_sink"] is impl.sinks
assert lse is None
@pytest.mark.parametrize(
("model", "compress_ratio"),
[("deepseek_v41", 1), ("deepseek_v41", 2), ("deepseek_v4", 4)],
)
@torch.inference_mode()
def test_paged_prefill_indices_match_gathered_prefill(
monkeypatch, model: str, compress_ratio: int
) -> None:
"""The aiter prefill reads the paged caches through the SWA builder's rows
and build_prefill_topk_ragged_indices, and must attend exactly the keys the
dequantize-and-gather prefill does."""
import importlib
from tests.v1.attention.utils import create_vllm_config
from vllm._aiter_ops import rocm_aiter_ops
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
build_prefill_topk_ragged_indices,
)
from vllm.v1.kv_cache_interface import SlidingWindowMLASpec
rocm = importlib.import_module(f"vllm.models.{model}.amd.rocm")
device = torch.device("cuda")
gen = torch.Generator().manual_seed(compress_ratio)
window, width, block_size, max_model_len = 16, 32, 16, 128
comp_block = block_size // compress_ratio
num_compressed = max_model_len // compress_ratio
# A decode row, then prefills with and without a prefix.
prefix_lens, query_lens = [20, 0, 13, 40], [1, 7, 20, 9]
seq_lens = [p + q for p, q in zip(prefix_lens, query_lens)]
num_decodes = num_decode_tokens = 1
num_tokens = sum(query_lens)
qsl = torch.tensor([0] + query_lens).cumsum(0).to(torch.int32)
positions = torch.cat([torch.arange(p, s) for p, s in zip(prefix_lens, seq_lens)])
token_to_req = torch.repeat_interleave(
torch.arange(len(query_lens)), torch.tensor(query_lens)
)
swa_bt = torch.randperm(32, generator=gen)[:16].view(4, 4).to(device, torch.int32)
comp_bt = torch.randperm(32, generator=gen)[:16].view(4, 4).to(device, torch.int32)
vllm_config = create_vllm_config(
model_name="facebook/opt-125m",
max_model_len=max_model_len,
block_size=block_size,
max_num_seqs=4,
max_num_batched_tokens=64,
hf_config_override={
"compress_ratios": [compress_ratio],
"index_topk": width,
"sliding_window": window,
},
)
swa_spec = SlidingWindowMLASpec(
block_size=block_size,
num_kv_heads=1,
head_size=512,
dtype=torch.bfloat16,
sliding_window=window,
cache_dtype_str="fp8_ds_mla",
model_version="deepseek_v4",
)
slots = swa_bt.cpu()[token_to_req, positions // block_size] * block_size + (
positions % block_size
)
common = CommonAttentionMetadata(
query_start_loc=qsl.to(device),
query_start_loc_cpu=qsl,
seq_lens=torch.tensor(seq_lens, dtype=torch.int32).to(device),
seq_lens_cpu_upper_bound=torch.tensor(seq_lens, dtype=torch.int32),
num_reqs=len(query_lens),
num_actual_tokens=num_tokens,
max_query_len=max(query_lens),
max_seq_len=max(seq_lens),
block_table_tensor=swa_bt,
slot_mapping=slots.to(device, torch.int64),
positions=positions.to(device),
causal=True,
)
def build(enabled: bool):
monkeypatch.setattr(
rocm_aiter_ops, "is_triton_sparse_mla_enabled", lambda: enabled
)
return rocm.DeepseekV4ROCMAiterSparseSWAMetadataBuilder(
swa_spec, ["swa"], vllm_config, device
).build(0, common)
assert build(enabled=False).prefill_swa_ragged_indices is None
md = build(enabled=True)
assert md.num_decode_tokens == num_decode_tokens
# Local top-k rows. Under each row's causal cap sit a failed candidate and
# one past the pool; past the cap, stale values neither path may read.
num_prefill_tokens = num_tokens - num_decode_tokens
topk = torch.full((num_prefill_tokens, width), -1, dtype=torch.int32)
for row in range(num_prefill_tokens):
visible = (int(positions[row + num_decode_tokens]) + 1) // compress_ratio
cap = min(visible, width)
topk[row, :cap] = torch.randperm(visible, generator=gen)[:cap].int()
if cap >= 2:
topk[row, :2] = torch.tensor([-1, num_compressed + 3])
topk[row, cap:] = torch.randint(
0, num_compressed, (width - cap,), generator=gen
).int()
topk = topk.to(device)
topk_indices, topk_indptr = build_prefill_topk_ragged_indices(
topk,
md.token_to_req_indices,
md.query_start_loc,
md.seq_lens,
md.is_valid_token,
comp_bt,
block_size=comp_block,
compress_ratio=compress_ratio,
num_compressed=num_compressed,
token_offset=num_decode_tokens,
)
paged_keys = [
sorted([("s", x) for x in swa if x >= 0] + [("c", x) for x in top if x >= 0])
for swa, top in zip(
_rows_from_ragged(
md.prefill_swa_ragged_indices, md.prefill_swa_ragged_indptr
),
_rows_from_ragged(topk_indices, topk_indptr),
)
]
# The gathered path's combine, mapped from workspace offsets to cache slots.
M = num_compressed + window + num_tokens
combined, combined_lens = rocm.combine_topk_swa_indices(
topk,
md.query_start_loc[num_decodes:],
md.prefill_seq_lens,
md.prefill_gather_lens,
window,
compress_ratio,
width,
M,
num_compressed,
)
gather_start = (md.prefill_seq_lens - md.prefill_gather_lens).tolist()
swa_bt_cpu, comp_bt_cpu = swa_bt.cpu(), comp_bt.cpu()
def slot(block_table, req, pos, size):
return int(block_table[req, pos // size]) * size + pos % size
gathered_keys = []
for row, (values, n) in enumerate(zip(combined.tolist(), combined_lens.tolist())):
req = int(token_to_req[row + num_decode_tokens])
chunk_req = req - num_decodes
keys = []
for value in values[:n]:
if value < 0:
continue
offset = value - chunk_req * M
if offset < num_compressed:
keys.append(("c", slot(comp_bt_cpu, req, offset, comp_block)))
else:
pos = gather_start[chunk_req] + offset - num_compressed
keys.append(("s", slot(swa_bt_cpu, req, pos, block_size)))
gathered_keys.append(sorted(keys))
assert paged_keys == gathered_keys
@pytest.mark.parametrize("model", ["deepseek_v4", "deepseek_v41"])
def test_aiter_sparse_mla_stays_off_with_kv_connector(monkeypatch, model) -> None:
import importlib
from vllm._aiter_ops import rocm_aiter_ops
rocm = importlib.import_module(f"vllm.models.{model}.amd.rocm")
monkeypatch.setattr(rocm_aiter_ops, "is_triton_sparse_mla_enabled", lambda: True)
assert rocm._aiter_sparse_mla_enabled(has_kv_transfer=False)
assert not rocm._aiter_sparse_mla_enabled(has_kv_transfer=True)
+46 -1
View File
@@ -13,7 +13,7 @@ from vllm.v1.request import RequestStatus
from vllm.v1.structured_output import StructuredOutputGrammar
from vllm.v1.utils import ConstantList
from .utils import create_requests, create_scheduler, mock_kv
from .utils import EOS_TOKEN_ID, create_requests, create_scheduler, mock_kv
pytestmark = pytest.mark.cpu_test
@@ -766,3 +766,48 @@ def test_kv_pressure_preempt_mid_handoff(kv_role: str, defer_free: bool):
else:
assert handoff.is_finished()
assert handoff.num_output_tokens == 1
def test_resumable_request_handoff():
"""New streaming input resumes with the correct context and stale results
are ignored.
"""
scheduler = create_scheduler(
async_scheduling=True,
use_v2_model_runner=True,
)
engine = PipelinedEngine(scheduler, queue_size=3)
engine._next_token = EOS_TOKEN_ID
old_prompt_len = 10
continuation_len = 12
request_id = "resumable-race"
request = create_requests(
1, num_tokens=old_prompt_len, max_tokens=4, req_ids=[request_id]
)[0]
request.resumable = True
continuation = create_requests(
1, num_tokens=continuation_len, max_tokens=4, req_ids=[request_id]
)[0]
continuation.resumable = True
scheduler.add_request(request)
for _ in range(3):
assert engine._schedule()
engine._process_oldest_step()
scheduler.add_request(continuation)
assert engine._schedule()
new_step, _ = engine.queue[0]
assert new_step.scheduled_new_reqs[0].num_computed_tokens == old_prompt_len
all_token_ids_before_stale = list(request.all_token_ids)
for _ in range(2):
engine._process_oldest_step()
assert list(request.output_token_ids) == []
assert list(request.all_token_ids) == all_token_ids_before_stale
engine._process_oldest_step()
assert len(request.output_token_ids) == 1

Some files were not shown because too many files have changed in this diff Show More