diff --git a/.agents/skills/kernel-microbenchmark/SKILL.md b/.agents/skills/kernel-microbenchmark/SKILL.md index f2dd5be929..98542023ff 100644 --- a/.agents/skills/kernel-microbenchmark/SKILL.md +++ b/.agents/skills/kernel-microbenchmark/SKILL.md @@ -1,6 +1,6 @@ --- name: kernel-microbenchmark -description: Build, debug, and interpret vLLM GPU kernel microbenchmarks for CUDA, Triton, and CuteDSL, including CUPTI timing, correctness checks, generated-code inspection, multi-GPU measurements, and SOL sanity checks. +description: Build, debug, and interpret vLLM GPU kernel microbenchmarks for CUDA, ROCm/HIP, Triton, and CuteDSL, including CUPTI timing, correctness checks, generated-code inspection, multi-GPU measurements, and SOL sanity checks. --- # Kernel Microbenchmark @@ -15,7 +15,7 @@ description: Build, debug, and interpret vLLM GPU kernel microbenchmarks for CUD 4. Compare against a baseline and report enough metadata to reproduce the result: GPU, dtype, shape, command, branch/commit, and relevant env vars. 5. Treat explanations as hypotheses until backed by an artifact: ablation, - generated PTX/SASS, profiler output, or controlled benchmark. + generated PTX/SASS or AMD ISA, profiler output, or controlled benchmark. 6. If the result changes the conclusion, preserve the compact lesson in a note, comment, benchmark table, or final summary. For experiments, a short `Question / Change / Correctness / Result / Observation / Next` note is @@ -25,6 +25,12 @@ description: Build, debug, and interpret vLLM GPU kernel microbenchmarks for CUD - Use FlashInfer CUPTI timing by default, with CUDA graph and cold L2 cache: `from flashinfer.testing import bench_gpu_time_with_cupti`. +- ROCm: CUPTI and FlashInfer do not exist. Use HIP graph replay over rotating + buffers whose total footprint exceeds the last-level cache (e.g. 256 MB on + MI355X), so every call reads cold from HBM, timed with HIP events — see + [benchmarks/graph_replay_benchmark.py](benchmarks/graph_replay_benchmark.py) + (platform-agnostic). Prefer this over `do_bench`, whose per-rep + launch/event overhead inflates small/decode-size kernels. - For compute-heavy kernels, report TFLOPS with the FLOP formula in the benchmark. For memory-heavy kernels, report estimated bytes moved and GB/s. For mixed kernels, report the most honest metric available and call out the @@ -38,6 +44,11 @@ description: Build, debug, and interpret vLLM GPU kernel microbenchmarks for CUD formulas, skipped work, sparsity, caching, and whether the baseline is doing the same operation. - Force compilation/autotuning before measuring compiled kernels. +- Tune every arm of a comparison before trusting it. A variant's default or + inherited config can be far from its optimum, so conclusions drawn from + untuned variants — especially negative ones — are unreliable. A coarse + sweep of the cheap knobs at a couple of representative shapes is enough; + report the winning config with the numbers. - Seed inputs when correctness comparisons matter. - Keep metadata setup, plan construction, allocation, random input generation, and logging outside the timed region unless that overhead is the experiment. @@ -98,11 +109,21 @@ throughput conventions. a two-node TP run as equivalent to a local NVLink-domain run without checking. - Stabilize GPU clocks or run enough untimed work to reach a steady state. Alternate candidate order so clock, thermal, and rank-skew effects are shared. +- ROCm: capturing ProcessGroupNCCL collectives in HIP graphs trips the NCCL + watchdog (HIP forbids querying an event recorded in a capturing stream; + the watchdog aborts the process). `TORCH_NCCL_BLOCKING_WAIT=1` before + `import torch` disables the watchdog — benchmark-only knob. Ask for backend + `"nccl"`; torch routes it to RCCL automatically (there is no `"rccl"` + backend string). ## Included Examples - Use [benchmarks/cupti_microbenchmark.py](benchmarks/cupti_microbenchmark.py) as a minimal single-GPU FlashInfer CUPTI timing pattern. +- Use + [benchmarks/graph_replay_benchmark.py](benchmarks/graph_replay_benchmark.py) + as a graph-replay timing pattern over rotating buffers sized past the + last-level cache — the ROCm/HIP default, also usable on CUDA. - Use [benchmarks/multi_gpu_gemm_rs.py](benchmarks/multi_gpu_gemm_rs.py) as a minimal distributed CUDA-graph timing pattern. diff --git a/.agents/skills/kernel-microbenchmark/benchmarks/graph_replay_benchmark.py b/.agents/skills/kernel-microbenchmark/benchmarks/graph_replay_benchmark.py new file mode 100644 index 0000000000..ec051ba789 --- /dev/null +++ b/.agents/skills/kernel-microbenchmark/benchmarks/graph_replay_benchmark.py @@ -0,0 +1,107 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Graph-replay microbenchmark with rotating weight buffers (CUDA and ROCm). + +Platform-agnostic: HIP implements the torch.cuda.* API, including CUDAGraph. +Pattern: capture one graph (HIP graph on ROCm) containing R*INNER calls +cycling over R rotating weight buffers, then time replays with CUDA/HIP +events. The rotating buffers keep the captured footprint above the GPU's +last-level cache, so every call reads weights cold from HBM. Prefer this over +do_bench on ROCm: do_bench's per-rep launch/event overhead inflates +small/decode-size kernels, and its default flush buffer is only as large as +the LLC — on a 256 MB LLC that evicts marginally and leaves the cache full of +dirty flush-buffer lines. The same pattern extends to multi-GPU benchmarks +(capture per rank, collectives in identical order every replay); see the +Multi-GPU section of SKILL.md. +""" + +import statistics + +import pandas as pd +import torch + +LLC_TARGET_BYTES = 400 * 1024**2 # above 256 MB MI355X LLC and B200 L2 (126 MB) +INNER = 2 # repeat slots within one graph to amortize per-replay overhead +REPLAYS = 10 # replays per round: makes each measurement large vs overhead +ROUNDS = 5 # median across rounds: replay time is not always stable +WARMUP = 5 # warmup replays to reach steady clocks before timing +MATMUL_CASES = [ + ("compute-bound", 4096, 4096, 4096), + ("decode-GEMM", 16, 16384, 8192), +] + + +def timeit_graph(calls, inner=INNER, replays=REPLAYS, rounds=ROUNDS): + """Time a list of per-slot callables as one graph (zero launch overhead); + weights rotate across slots so they stay HBM-cold. Returns the median over + rounds of per-round means, ms per call.""" + for c in calls: + c() + torch.accelerator.synchronize() + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g): # capture runs on a side stream + for i in range(len(calls) * inner): + calls[i % len(calls)]() + torch.accelerator.synchronize() + events = [(torch.cuda.Event(True), torch.cuda.Event(True)) for _ in range(rounds)] + for _ in range(WARMUP): + g.replay() + # No sync between warmup and the first round: queued replays keep the GPU + # busy, so each round's launch latency stays outside its event window. + for s, e in events: + s.record() + for _ in range(replays): + g.replay() + e.record() + torch.accelerator.synchronize() + samples = [s.elapsed_time(e) / (replays * len(calls) * inner) for s, e in events] + return statistics.median(samples) # ms per call + + +def main() -> None: + if not torch.accelerator.is_available(): + raise RuntimeError("No GPU visible (torch.accelerator covers HIP on ROCm).") + + torch.set_default_device("cuda") + torch.manual_seed(0) + + rows = [] + for name, m, n, k in MATMUL_CASES: + a = torch.randn(m, k, dtype=torch.bfloat16) # shared: producer-warm + out = torch.empty(m, n, dtype=torch.bfloat16) + b_bytes = k * n * 2 + r = max(2, -(-LLC_TARGET_BYTES // b_bytes)) # ceil + bs = [torch.randn(k, n, dtype=torch.bfloat16) for _ in range(r)] + + def make_call(i, a=a, bs=bs, out=out): + def call(): + torch.mm(a, bs[i], out=out) + + return call + + # correctness before timing, per slot (the graph would freeze a wrong + # result into every replay) + for i in range(r): + make_call(i)() + torch.accelerator.synchronize() + torch.testing.assert_close(out, torch.mm(a, bs[i]), atol=1e-1, rtol=1e-1) + + us = timeit_graph([make_call(i) for i in range(r)]) * 1e3 + rows.append( + { + "case": name, + "shape": f"{m}x{n}x{k}", + "slots": r, + "us": us, + "tflops": 2 * m * n * k / (us * 1e6), + "gbps": 2 * (m * k + k * n + m * n) / (us * 1e3), + } + ) + + df = pd.DataFrame(rows) + print(df.to_string(index=False, float_format=lambda x: f"{x:.3f}")) + + +if __name__ == "__main__": + main() diff --git a/.buildkite/intel_jobs/kernels_intel.yaml b/.buildkite/intel_jobs/kernels_intel.yaml index da6086c1dc..3650a729d7 100644 --- a/.buildkite/intel_jobs/kernels_intel.yaml +++ b/.buildkite/intel_jobs/kernels_intel.yaml @@ -25,3 +25,26 @@ steps: 'cd tests && pytest -v -s ir && pytest -v -s kernels/ir' + +- label: UVA Kernel Tests + key: uva-kernel-tests + timeout_in_minutes: 20 + device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 16+ + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - tests/kernels/core/test_uva.py + - vllm/utils/torch_utils.py + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'cd tests && + pytest -v -s kernels/core/test_uva.py' diff --git a/.buildkite/intel_jobs/test-intel.yaml b/.buildkite/intel_jobs/test-intel.yaml index ad1a03e9cf..30e14bd384 100644 --- a/.buildkite/intel_jobs/test-intel.yaml +++ b/.buildkite/intel_jobs/test-intel.yaml @@ -153,7 +153,7 @@ steps: bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && pytest -v -s quantization/test_auto_round.py && - pytest -v -s quantization/test_online.py' + pytest -v -s quantization/test_online.py --deselect=tests/quantization/test_online.py::test_online_quantization_loads_real_weights' - label: "XPU compressed tensors FP8 test" key: xpu-compressed-tensors-fp8-test depends_on: diff --git a/.buildkite/scripts/crcr-report.sh b/.buildkite/scripts/crcr-report.sh index 8812e876ae..3900b80339 100644 --- a/.buildkite/scripts/crcr-report.sh +++ b/.buildkite/scripts/crcr-report.sh @@ -79,12 +79,22 @@ if [[ -z "${OIDC_TOKEN}" ]]; then fi BUILD_JSON="$(mktemp)" -trap 'rm -f "${BUILD_JSON}"' EXIT +BUILD_JSON_STAGING="$(mktemp)" +trap 'rm -f "${BUILD_JSON}" "${BUILD_JSON_STAGING}"' EXIT BUILD_URL="https://api.buildkite.com/v2/organizations/${BUILDKITE_ORGANIZATION_SLUG}/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}" +# Staged so a failed request cannot clobber the last good snapshot: curl -o +# writes the error body too, and the deadline path reports from whatever +# snapshot it has. fetch_build() { - curl -sS -w '%{http_code}' -o "${BUILD_JSON}" \ - -H "Authorization: Bearer ${BK_TOKEN}" "${BUILD_URL}" + local code + code="$(curl -sS -w '%{http_code}' -o "${BUILD_JSON_STAGING}" \ + -H "Authorization: Bearer ${BK_TOKEN}" "${BUILD_URL}")" + if [[ "${code}" == "200" ]]; then + mv -f "${BUILD_JSON_STAGING}" "${BUILD_JSON}" + BUILD_JSON_STAGING="$(mktemp)" + fi + echo "${code}" } # Count jobs that have not reached a terminal state, ignoring this job: the @@ -113,22 +123,78 @@ PY # finished and none of the eventual 152 hard failures visible, and HUD recorded # that nightly as green. # -# Polling rather than a full `depends_on` barrier: Buildkite groups carry no -# key, so a barrier means naming all ~250 step keys, which silently rots when a -# step is added and fails the pipeline upload when one is removed or excluded -# from this lane (AMD, retired A100). The pipeline does gate this step on the -# long pole so it is scheduled near the end of the build -- that keeps the agent -# from idling for hours, but it is only a hint, and this loop is what actually -# guarantees the snapshot is complete. -POLL_INTERVAL_S="${CRCR_POLL_INTERVAL_S:-60}" -# Under the step's own timeout, so a stuck build still gets a partial report -# rather than the job being killed with nothing sent. -WAIT_DEADLINE=$(( $(date +%s) + ${CRCR_MAX_WAIT_S:-46800} )) +# Polling rather than a `depends_on` barrier: Buildkite groups carry no key, so +# a barrier means naming all ~250 step keys, which silently rots when a step is +# added and fails the pipeline upload when one is removed or excluded from this +# lane (AMD, retired A100). Gating on a single step was tried and is worse: a +# dependency that fails or is cancelled takes the report with it despite +# allow_dependency_failure (builds 91191 and 91461 both lost their report that +# way), so this step depends only on the image it runs in. +# 15 min: the wait now spans the whole build, so a 60s poll made ~420 requests +# against an API shared with the rest of CI to learn something that changes on +# the order of minutes. The cost is up to one interval of extra latency after +# the last job lands. +POLL_INTERVAL_S="${CRCR_POLL_INTERVAL_S:-900}" + +# The deadline is measured from when the *build* started, not from when this +# script did, so "report no later than N hours into the build" holds however +# late this step is scheduled. Anchoring it to script start made the bound +# meaningless whenever the step was itself delayed. +# +# Resolved after the first fetch, from the build's own created_at. +MAX_BUILD_AGE_S="${CRCR_MAX_BUILD_AGE_S:-25200}" +WAIT_DEADLINE="" +# Used until created_at is known. Without it, an API outage on the very first +# request leaves the deadline unresolved and the loop spins to the step timeout. +FALLBACK_DEADLINE=$(( $(date +%s) + MAX_BUILD_AGE_S )) + +build_deadline() { + python3 - "${BUILD_JSON}" "${MAX_BUILD_AGE_S}" <<'PY' +import datetime, json, sys +build = json.load(open(sys.argv[1])) +created = build.get("created_at") +if not created: + raise SystemExit(1) +started = datetime.datetime.fromisoformat(created.replace("Z", "+00:00")) +print(int(started.timestamp()) + int(sys.argv[2])) +PY +} + while :; do http_code="$(fetch_build)" if [[ "${http_code}" != "200" ]]; then - echo "buildkite API returned ${http_code} -- skipping report" - exit 0 + # A transient 429/5xx must not cost the whole report: treating one + # failure as fatal would drop a nightly for a blip. Keep polling on the + # last good snapshot; only give up if the deadline passes having never + # fetched one. + echo "buildkite API returned ${http_code}" + if [[ ! -s "${BUILD_JSON}" ]]; then + if (( $(date +%s) >= ${WAIT_DEADLINE:-$FALLBACK_DEADLINE} )); then + echo "no build data was ever fetched -- nothing to report" + exit 0 + fi + sleep "${POLL_INTERVAL_S}" + continue + fi + if (( $(date +%s) >= ${WAIT_DEADLINE:-$FALLBACK_DEADLINE} )); then + echo "deadline reached while the API is unavailable --" \ + "reporting from the last good snapshot" + break + fi + sleep "${POLL_INTERVAL_S}" + continue + fi + if [[ -z "${WAIT_DEADLINE}" ]]; then + WAIT_DEADLINE="$(build_deadline)" || WAIT_DEADLINE="" + if [[ -z "${WAIT_DEADLINE}" ]]; then + # No created_at to anchor to; fall back to script start so the loop + # still has a bound rather than running until the step times out. + WAIT_DEADLINE=$(( $(date +%s) + MAX_BUILD_AGE_S )) + echo "build created_at unavailable -- deadline measured from now" + fi + echo "reporting deadline: $(date -u -d "@${WAIT_DEADLINE}" '+%Y-%m-%dT%H:%M:%SZ' 2>/dev/null \ + || date -u -r "${WAIT_DEADLINE}" '+%Y-%m-%dT%H:%M:%SZ')" \ + "(build age limit ${MAX_BUILD_AGE_S}s)" fi remaining="$(outstanding_jobs)" || remaining="" if [[ -z "${remaining}" ]]; then @@ -140,7 +206,7 @@ while :; do break fi if (( $(date +%s) >= WAIT_DEADLINE )); then - echo "still ${remaining} job(s) running at the wait deadline --" \ + echo "still ${remaining} job(s) running at the build-age deadline --" \ "reporting a partial view" break fi diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 530fd0d27c..16c1e7075b 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -200,9 +200,9 @@ steps: - pytest -v -s test_sampling_params.py - pytest -v -s -m 'cpu_test' multimodal - pytest -v -s renderers - - pytest -v -s tokenizers_ - pytest -v -s reasoning - pytest -v -s tool_parsers + - pytest -v -s tokenizers_ - pytest -v -s parser - pytest -v -s transformers_utils - pytest -v -s config --ignore=config/test_mp_reducer.py @@ -262,6 +262,24 @@ steps: commands: - pytest -v -s tools/test_docker_build_metadata_args.py +- label: ":amd: (MI250) Torch Stable ABI Audit" # TBD + timeout_in_minutes: 35 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_1 + no_gpu: true + optional: true + working_dir: "/vllm-workspace/" + source_file_dependencies: + - .buildkite/check-torch-abi.py + - csrc/ + - cmake/ + - setup.py + - vllm/platforms/rocm.py + commands: + - uv pip install --system torch-abi-audit==0.0.1 + - python3 /vllm-workspace/.buildkite/check-torch-abi.py + ######################################################################################################################################### # # # MI250 (gfx90a) tests # @@ -415,9 +433,9 @@ steps: - vllm/v1/cudagraph_dispatcher.py - vllm/config/compilation.py - vllm/compilation - - vllm/platforms/rocm.py - vllm/v1/worker/encoder_cudagraph.py - vllm/v1/worker/encoder_cudagraph_defs.py + - vllm/platforms/rocm.py commands: - pytest -v -s v1/cudagraph/test_cudagraph_dispatch.py - pytest -v -s v1/cudagraph/test_cudagraph_mode.py @@ -614,6 +632,7 @@ steps: - vllm/v1/ commands: - "find compile/ -maxdepth 1 -name 'test_*.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'" + - pytest -s -v compile/dynamic_shapes/ - label: ":amd: (MI300) Fusion E2E Config Sweep" # TBD timeout_in_minutes: 40 @@ -659,6 +678,8 @@ steps: - "pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k 'inductor_partition and not +rms_norm and not +quant_fp8'" # Different from CUDA, Qwen requires +rms_norm and +quant_fp8 as rms+quant fusion is only supported on AITER - "pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k 'inductor_partition and +rms_norm and +quant_fp8 and qwen3'" + # DeepSeek-Coder uses the portable static-FP8 MLA fusion path on ROCm. + - "pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k 'inductor_partition and not +rms_norm and +quant_fp8 and DeepSeek-Coder and TRITON_MLA and not True'" - label: ":amd: (MI300) PyTorch Compilation Passes" # TBD timeout_in_minutes: 80 @@ -767,7 +788,7 @@ steps: - vllm/platforms/rocm.py commands: - export VLLM_TEST_CLEAN_GPU_MEMORY=1 - - VLLM_TEST_CLEAN_GPU_MEMORY=1 pytest -v -s tests/compile/passes/distributed/test_async_tp.py + - pytest -s -v tests/compile/passes/distributed - pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py::test_tp2_ar_rms_fusions -k "inductor_partition" - label: ":amd: (MI300) Distributed Compile (Dynamo Partition)" # TBD @@ -789,6 +810,25 @@ steps: - VLLM_TEST_CLEAN_GPU_MEMORY=1 pytest -v -s tests/compile/passes/distributed/test_async_tp.py - pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py::test_tp2_ar_rms_fusions -k "dynamo_partition" +# - label: ":amd: (MI300) Sequence Parallel Correctness" # TBD +# timeout_in_minutes: 180 +# mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] +# dind: false +# agent_pool: mi300_2 +# num_gpus: 2 +# optional: true +# working_dir: "/vllm-workspace/" +# source_file_dependencies: +# - vllm/model_executor/layers/ +# - vllm/compilation/ +# - vllm/v1/worker/ +# - vllm/v1/cudagraph_dispatcher.py +# - tests/compile/correctness_e2e/test_sequence_parallel.py +# - vllm/platforms/rocm.py +# commands: +# - export VLLM_TEST_CLEAN_GPU_MEMORY=1 +# - pytest -v -s tests/compile/correctness_e2e/test_sequence_parallel.py + - label: ":amd: (MI300) Distributed Compile + RPC" # TBD timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] @@ -999,7 +1039,7 @@ steps: - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 ../examples/rl/rlhf_http_ipc.py - label: ":amd: (MI300) Elastic EP Scaling" # TBD - timeout_in_minutes: 50 + timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_4 @@ -1014,6 +1054,7 @@ steps: - tests/distributed/ - vllm/platforms/rocm.py commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt - pytest -v -s distributed/test_elastic_ep.py - label: ":amd: (MI300) RayExecutorV2" # TBD @@ -1107,7 +1148,7 @@ steps: #-------------------------------------------------------- mi300 · entrypoints --------------------------------------------------------# - label: ":amd: (MI300) Entrypoints Unit" # TBD - timeout_in_minutes: 50 + timeout_in_minutes: 60 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1247,6 +1288,59 @@ steps: commands: - pytest -s -v evals/mrcr/test_mrcr_correctness.py --config-list-file=evals/mrcr/configs/models-small.txt +- label: ":amd: (MI300) LM Eval KV-Offload" # TBD + timeout_in_minutes: 55 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_1 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/offloading/ + - vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py + - vllm/v1/kv_offload/ + - vllm/v1/simple_kv_offload/ + - tests/evals/gsm8k/test_gsm8k_offloading.py + - vllm/platforms/rocm.py + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_offloading.py -k "nemotron-h-8b or gemma-4-e4b-it" + +- label: ":amd: (MI300) LM Eval KV-Offload Medium" # TBD + timeout_in_minutes: 70 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_2 + num_gpus: 2 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/offloading/ + - vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py + - vllm/v1/kv_offload/ + - vllm/v1/simple_kv_offload/ + - tests/evals/gsm8k/test_gsm8k_offloading.py + - vllm/platforms/rocm.py + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_offloading.py -k "qwen3.5-35b or deepseek-v2-lite" + +- label: ":amd: (MI300) LM Eval KV-Offload Large" # TBD + timeout_in_minutes: 65 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_4 + num_gpus: 4 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/offloading/ + - vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py + - vllm/v1/kv_offload/ + - vllm/v1/simple_kv_offload/ + - tests/evals/gsm8k/test_gsm8k_offloading.py + - vllm/platforms/rocm.py + commands: + - VLLM_ROCM_USE_AITER=1 pytest -s -v evals/gsm8k/test_gsm8k_offloading.py -k "deepseek-v4-flash" + - label: ":amd: (MI300) LM Eval Small Models Harness" # TBD timeout_in_minutes: 35 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] @@ -1398,6 +1492,9 @@ steps: commands: - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_dp4_async_eplb.sh 0.8 200 8050 +# TODO(akaratza): Retire this standalone ROCm coverage after +# tests/distributed/test_eplb_spec_decode.py is enabled and validated on ROCm. +# The upstream parity job currently skips that entire module on ROCm. - label: ":amd: (MI300) Qwen3-Next-80B-A3B-Instruct MTP Async EPLB Accuracy" # TBD timeout_in_minutes: 45 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] @@ -1446,7 +1543,7 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx.txt - label: ":amd: (MI300) LM Eval Large Models ROCm Harness" # TBD - timeout_in_minutes: 55 + timeout_in_minutes: 40 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_8 @@ -1465,7 +1562,7 @@ steps: - vllm/platforms/rocm.py commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm.txt --tp-size=8 + - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm.txt --tp-size=8 --timeout=5400 - label: ":amd: (MI300) LM Eval Large Models FP8" # TBD timeout_in_minutes: 40 @@ -1491,7 +1588,7 @@ steps: #---------------------------------------------------------- mi300 · kernels ----------------------------------------------------------# - label: ":amd: (MI300) Core Operation Kernels Shard %N" # TBD - timeout_in_minutes: 40 + timeout_in_minutes: 45 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1629,8 +1726,8 @@ steps: commands: - pytest -v -s kernels/core/test_minimax_reduce_rms.py -- label: ":amd: (MI300) KDA Kernels" # TBD - timeout_in_minutes: 30 +- label: ":amd: (MI300) Kimi K3" # TBD + timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1643,11 +1740,20 @@ steps: - vllm/models/kimi_k3/nvidia/kda.py - vllm/models/kimi_k3/nvidia/kda_metadata.py - vllm/models/kimi_k3/nvidia/ops/third_party/kda/ + - vllm/models/kimi_k3/amd/ + - CMakeLists.txt + - csrc/libtorch_stable/kimi_k3/fused_kda_decode_kernel_rocm.cu + - csrc/libtorch_stable/ops.h + - csrc/libtorch_stable/torch_bindings.cpp + - tests/models/kimi_k3/test_amd_attn_res.py + - tests/models/kimi_k3/test_amd_kda_decode.py - tests/models/kimi_k3/test_kda.py - tests/models/kimi_k3/test_kda_metadata.py + - vllm/_custom_ops.py - vllm/platforms/rocm.py commands: - - pytest -v -s models/kimi_k3/test_kda.py models/kimi_k3/test_kda_metadata.py + # Run the ROCm AttnRes/KDA decode counterparts plus existing KDA coverage. + - pytest -v -s models/kimi_k3/test_amd_attn_res.py models/kimi_k3/test_amd_kda_decode.py models/kimi_k3/test_kda.py models/kimi_k3/test_kda_metadata.py - label: ":amd: (MI355) Kimi K3" # TBD timeout_in_minutes: 60 @@ -1781,6 +1887,135 @@ steps: - pytest -v -s model_executor -m '(not slow_test)' --timeout=900 --timeout-method=thread - pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py --timeout=900 --timeout-method=thread +#---------------------------------------------------- mi300 · model_runner_v2 -------------------------------------------------------# + +- label: ":amd: (MI300) Model Runner V2 Core" # TBD + timeout_in_minutes: 55 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_1 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/v1/worker/gpu/ + - vllm/v1/worker/gpu_worker.py + - vllm/v1/core/sched/ + - vllm/v1/attention/ + - tests/v1/engine/test_llm_engine.py + - tests/v1/e2e/ + - tests/entrypoints/llm/test_struct_output_generate.py + - vllm/platforms/rocm.py + commands: + - set -x + - export VLLM_USE_V2_MODEL_RUNNER=1 + - pytest -v -s v1/engine/test_llm_engine.py -k "not test_engine_metrics" + - pytest -v -s v1/e2e/general/test_async_scheduling.py -k "not ngram" + - pytest -v -s v1/e2e/general/test_context_length.py + - pytest -v -s v1/e2e/general/test_min_tokens.py + - pytest -v -s entrypoints/llm/test_struct_output_generate.py -k "xgrammar and not speculative_config6 and not speculative_config7 and not speculative_config8 and not speculative_config0" + +- label: ":amd: (MI300) Model Runner V2 Examples" # TBD + timeout_in_minutes: 40 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_1 + optional: true + working_dir: "/vllm-workspace/examples" + source_file_dependencies: + - vllm/v1/worker/gpu/ + - vllm/v1/core/sched/ + - vllm/v1/worker/gpu_worker.py + - examples/basic/offline_inference/ + - examples/generate/multimodal/ + - examples/features/ + - examples/pooling/embed/vision_embedding_offline.py + - examples/features/tensorize_vllm_model.py + - examples/deployment/llm_engine_example.py + - vllm/platforms/rocm.py + commands: + - set -x + - export VLLM_USE_V2_MODEL_RUNNER=1 + - pip install tensorizer + - python3 basic/offline_inference/chat.py + - python3 basic/offline_inference/generate.py --model facebook/opt-125m + - python3 generate/multimodal/audio_language_offline.py --seed 0 + - python3 generate/multimodal/vision_language_offline.py --seed 0 + - python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0 + - python3 generate/multimodal/encoder_decoder_multimodal_offline.py --model-type whisper --seed 0 + - python3 pooling/embed/vision_embedding_offline.py --seed 0 + - python3 features/automatic_prefix_caching/prefix_caching_offline.py + - python3 deployment/llm_engine_example.py + - python3 features/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 features/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors + - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048 + - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536 + +- label: ":amd: (MI300) Model Runner V2 Distributed" # TBD + timeout_in_minutes: 45 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_2 + num_gpus: 2 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/v1/worker/gpu/ + - vllm/v1/worker/gpu_worker.py + - tests/basic_correctness/test_basic_correctness.py + - tests/v1/distributed/test_async_llm_dp.py + - tests/v1/distributed/test_eagle_dp.py + - vllm/platforms/rocm.py + commands: + - set -x + - export VLLM_USE_V2_MODEL_RUNNER=1 + - TARGET_TEST_SUITE=MI300 pytest -v -s basic_correctness/test_basic_correctness.py -m 'distributed(num_gpus=2)' -k "not ray and not True" + - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py -k "not ray" + - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py + +- label: ":amd: (MI300) Model Runner V2 Pipeline Parallelism" # TBD + timeout_in_minutes: 65 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_4 + num_gpus: 4 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/v1/worker/gpu/ + - vllm/v1/worker/gpu_worker.py + - tests/distributed/test_pipeline_parallel.py + - tests/distributed/test_pp_cudagraph.py + - vllm/platforms/rocm.py + commands: + - set -x + - export VLLM_USE_V2_MODEL_RUNNER=1 + - pytest -v -s distributed/test_pipeline_parallel.py -k "not ray" + - pytest -v -s distributed/test_pp_cudagraph.py -k "not ray" + +- label: ":amd: (MI300) Model Runner V2 Spec Decode" # TBD + timeout_in_minutes: 90 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_1 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/v1/worker/gpu/ + - vllm/v1/worker/gpu_worker.py + - tests/v1/spec_decode/test_max_len.py + - tests/v1/spec_decode/test_rejection_sampler_utils.py + - tests/v1/spec_decode/test_synthetic_rejection_sampler_utils.py + - tests/v1/e2e/spec_decode/ + - vllm/platforms/rocm.py + commands: + - set -x + - export VLLM_USE_V2_MODEL_RUNNER=1 + - pytest -v -s v1/spec_decode/test_max_len.py -k "eagle or mtp" + - pytest -v -s v1/spec_decode/test_rejection_sampler_utils.py + - pytest -v -s v1/spec_decode/test_synthetic_rejection_sampler_utils.py + - pytest -v -s v1/e2e/spec_decode/eagle/ + - pytest -v -s v1/e2e/spec_decode/speculators/ + - pytest -v -s v1/e2e/spec_decode/mtp/ + #------------------------------------------------------ mi300 · models / basic -------------------------------------------------------# - label: ":amd: (MI300) Basic Models (Initialization)" # TBD @@ -2258,6 +2493,23 @@ steps: - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --ignore quantization/test_rocm_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s rocm/test_moe_weight_replay.py; fi +- label: ":amd: (MI300) Quantized Fusions" # TBD + timeout_in_minutes: 45 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_1 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - tests/fusion + - vllm/model_executor/layers/fusion + - vllm/model_executor/kernels/linear + - vllm/model_executor/layers/quantization/compressed_tensors + - vllm/model_executor/layers/quantization/modelopt.py + - vllm/platforms/rocm.py + commands: + - pytest -v -s fusion/ + #--------------------------------------------------------- mi300 · samplers ----------------------------------------------------------# - label: ":amd: (MI300) Samplers" # TBD @@ -2411,8 +2663,8 @@ steps: source_file_dependencies: - vllm/v1/engine/ - tests/v1/engine/ - - vllm/platforms/rocm.py - tests/v1/test_tensor_ipc_queue.py + - vllm/platforms/rocm.py commands: - pytest -v -s v1/engine/test_preprocess_error_handling.py - pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py @@ -2625,6 +2877,23 @@ steps: - export BASELINE_FILE='/tmp/mooncake-ec-e2e/baseline.json' - bash tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh +- label: ":amd: (MI300) NixlConnector PD edge case" # TBD + timeout_in_minutes: 65 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_2 + num_gpus: 2 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ + - vllm/v1/core/sched/ + - tests/v1/kv_connector/nixl_integration/ + - vllm/platforms/rocm.py + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + - PREFILL_GPU_ID=0 DECODE_GPU_ID=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_edge_case_test.sh + - label: ":amd: (MI300) CrossLayer KV layout Distributed NixlConnector PD accuracy" # TBD timeout_in_minutes: 60 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] @@ -2665,6 +2934,55 @@ steps: - pytest -v -s v1/engine/test_engine_core_client.py::test_kv_cache_events_dp - pytest -v -s distributed/test_utils.py +- label: ":amd: (MI300) Distributed NixlConnector PD accuracy" # TBD + timeout_in_minutes: 45 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_4 + num_gpus: 4 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ + - tests/v1/kv_connector/nixl_integration/ + - vllm/platforms/rocm.py + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh + +- label: ":amd: (MI300) Push NixlConnector PP prefill PD accuracy" # TBD + timeout_in_minutes: 55 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_4 + num_gpus: 4 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ + - tests/v1/kv_connector/nixl_integration/ + - tests/v1/kv_connector/nixl_push_integration/ + - vllm/platforms/rocm.py + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_push_integration/config_sweep_accuracy_test.sh + +- label: ":amd: (MI300) DP EP Distributed NixlConnector PD accuracy" # TBD + timeout_in_minutes: 30 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_4 + num_gpus: 4 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ + - tests/v1/kv_connector/nixl_integration/ + - vllm/platforms/rocm.py + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + - DP_EP=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh + - label: ":amd: (MI300) Hybrid SSM NixlConnector PD accuracy" # TBD timeout_in_minutes: 80 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] @@ -3647,7 +3965,11 @@ steps: - vllm/model_executor/models/qwen3.py - vllm/model_executor/models/qwen3_next.py - vllm/model_executor/models/qwen3_next_mtp.py + - vllm/distributed/ + - vllm/model_executor/layers/fused_moe/ - vllm/third_party/flash_linear_attention/ops/ + - tests/evals/gsm8k/ + - tests/utils.py - vllm/_aiter_ops.py - vllm/platforms/rocm.py commands: @@ -3779,14 +4101,44 @@ steps: commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-t3nc-rocm.txt -- label: ":amd: (MI355) LM Eval Small Models Distributed" - timeout_in_minutes: 55 +- label: ":amd: (MI355) LM Eval Spec Decode (Fixed-length FP8)" # TBD + timeout_in_minutes: 145 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false + agent_pool: mi355_4 + num_gpus: 4 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - tests/evals/gsm8k/ + - tests/utils.py + - vllm/distributed/ + - vllm/models/deepseek_v4/ + - vllm/model_executor/models/qwen3_dspark.py + - vllm/model_executor/layers/fused_moe/ + - vllm/v1/spec_decode/metrics.py + - vllm/v1/worker/gpu/spec_decode/dspark/ + - vllm/v1/worker/gpu/spec_decode/rejection_sampler.py + - vllm/v1/worker/gpu/spec_decode/speculator.py + - vllm/model_executor/layers/quantization/ + - vllm/v1/attention/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-spec-decode-rocm.txt + +- label: ':amd: (MI355) LM Eval Small Models Distributed' + timeout_in_minutes: 145 + mirror_hardwares: + - amdexperimental + - amdproduction + - amdgfx950nightly + - amdmi355 + dind: false agent_pool: mi355_2 num_gpus: 2 optional: true - working_dir: "/vllm-workspace/tests" + working_dir: /vllm-workspace/tests source_file_dependencies: - csrc/ - vllm/model_executor/layers/quantization @@ -3794,11 +4146,22 @@ steps: - vllm/model_executor/model_loader/ - vllm/v1/attention/backends/ - vllm/v1/attention/selector.py + - vllm/v1/worker/ + - vllm/v1/core/ + - vllm/config/ + - tests/evals/gsm8k/configs/models-mi3xx-fp8-and-mixed.txt - vllm/_aiter_ops.py - vllm/platforms/rocm.py - - tests/evals/gsm8k/configs/models-mi3xx-fp8-and-mixed.txt + - tests/evals/gsm8k/ + - tests/utils.py + - vllm/distributed/ + - vllm/transformers_utils/configs/diffusion_gemma.py + - vllm/v1/attention/ops/ + - tests/evals/gsm8k/configs/models-small-tp-mi355.txt commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx-fp8-and-mixed.txt + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small-tp.txt + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small-tp-mi355.txt - label: ":amd: (MI355) MoE Refactor Integration Shard %N" timeout_in_minutes: 105 @@ -3819,6 +4182,7 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor/config-rocm-shard-$$BUILDKITE_PARALLEL_JOB.txt + - label: ":amd: (MI355) Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy" # TBD timeout_in_minutes: 25 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] @@ -4350,6 +4714,22 @@ steps: commands: - pytest -v -s models/test_initialization.py -k 'not test_can_initialize_small_subset' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB +- label: ":amd: (MI355) Inkling" # TBD + timeout_in_minutes: 65 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_1 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/models/inkling/ + - vllm/cute_utils/ + - cmake/external_projects/tml_fa4.cmake + - tests/models/inkling/ + - vllm/platforms/rocm.py + commands: + - pytest -v -s models/inkling/rocm + #----------------------------------------------------- mi355 · models / language -----------------------------------------------------# - label: ":amd: (MI355 DPX) Language Models (Standard)" # TBD @@ -5104,7 +5484,8 @@ steps: - vllm/platforms/rocm.py optional: true commands: - - pytest -v -s v1/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - pytest -v -s v1/attention --ignore=v1/attention/test_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - VLLM_ROCM_USE_AITER=1 pytest -v -s v1/attention/test_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - label: ":amd: (MI355 DPX) V1 Core (MI355 suite)" timeout_in_minutes: 60 @@ -5500,6 +5881,34 @@ steps: - pip install 'modelscope<1.38' - pytest -v -s test_regression.py +- label: ':amd: (MI355) DeepSeek V4 MoE Integration' + timeout_in_minutes: 20 + source_file_dependencies: + - tests/models/test_deepseek_v4_vl_rocm.py + - vllm/models/deepseek_v4/amd/ + - vllm/models/deepseek_v4/common/ + - vllm/model_executor/layers/fused_moe/ + - vllm/model_executor/layers/activation.py + - vllm/model_executor/layers/linear.py + - vllm/config/ + - vllm/forward_context.py + - vllm/v1/worker/workspace.py + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/envs.py + commands: + - VLLM_ROCM_USE_AITER_MOE=0 pytest -v -s models/test_deepseek_v4_vl_rocm.py + dind: false + working_dir: /vllm-workspace/tests + agent_pool: mi355_1 + num_gpus: 1 + mirror_hardwares: + - amdexperimental + - amdproduction + - amdgfx950nightly + - amdmi355 + optional: true + - label: ':amd: (MI355) Quantized MoE' timeout_in_minutes: 45 working_dir: /vllm-workspace/ diff --git a/.buildkite/test_areas/attention.yaml b/.buildkite/test_areas/attention.yaml index e11108d925..b9f430a20e 100644 --- a/.buildkite/test_areas/attention.yaml +++ b/.buildkite/test_areas/attention.yaml @@ -43,3 +43,22 @@ steps: commands: - pytest -v -s v1/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT parallelism: 2 + mirror: + amd: + label: ":amd: (MI355 DPX) V1 Attention Shard %N" + dind: false + device: mi355_dpx + timeout_in_minutes: 125 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/config/attention.py + - vllm/model_executor/layers/attention + - vllm/v1/attention + - tests/v1/attention + - vllm/_aiter_ops.py + - vllm/envs.py + - vllm/platforms/rocm.py + commands: + - pytest -v -s v1/attention --ignore=v1/attention/test_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - VLLM_ROCM_USE_AITER=1 pytest -v -s v1/attention/test_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT diff --git a/.buildkite/test_areas/crcr_report.yaml b/.buildkite/test_areas/crcr_report.yaml index f6b6582893..19e0e064e8 100644 --- a/.buildkite/test_areas/crcr_report.yaml +++ b/.buildkite/test_areas/crcr_report.yaml @@ -1,31 +1,30 @@ group: CRCR Report -# The report waits for the rest of the build (see crcr-report.sh), so gate it on -# the long pole rather than letting it wait on an agent. A torch-nightly build -# is tail-heavy: of 347 jobs in build 90830, 344 finished inside 3.5h and three -# stragglers stretched it to 12.7h, with lm-eval-large-models-4xh100 last in -# both 90830 (+760 min) and 90640 (+799 min) by a margin of hours. +# Only image-build-cpu: this step must always run. # -# Depending on it means Buildkite does not schedule this step until the build is -# nearly over, so the agent polls briefly instead of idling ~13h. It is a -# scheduling hint, not the correctness mechanism: the script still waits for -# every job, so if this stops being the slowest step the report is late to start -# but still correct. +# It was briefly gated on the build's long pole (lm-eval-large-models-4xh100) +# so the agent would not idle. That backfired on the first nightly to use it, +# build 91191: the long pole OOMed, hung, and was cancelled -- and a cancelled +# dependency propagates past `allow_dependency_failure`, so the step went +# `waiting_failed` and that nightly got no CRCR row at all. Nothing in this +# pipeline sets timeout_in_minutes (the pipeline default and org maximum are +# both unset), so a hung job has no terminal state to reach either: gating on +# any single job can suppress the report indefinitely. +# +# The bound now lives in the script, measured from the build's created_at, so +# the report lands within CRCR_MAX_BUILD_AGE_S of the build starting whatever +# else happens. depends_on: - image-build-cpu - - lm-eval-large-models-4xh100 steps: - label: ":satellite: Report nightly results to CRCR" key: crcr-report if: build.env("TORCH_NIGHTLY") == "1" soft_fail: true allow_dependency_failure: true - # depends_on defers the start to near the end of the build, so in the normal - # case the script polls for minutes. This only has to cover the case where - # the long pole finishes early and something else is still running, so it is - # sized against the whole build rather than the expected wait. The script's - # own deadline (CRCR_MAX_WAIT_S, 13h) trips first and still sends a partial - # report; this is the backstop that stops the job outliving the build. - timeout_in_minutes: 840 + # The script polls from the start of the build now, so this covers the whole + # wait. It sits above the script's own build-age deadline (7h) so the script + # sends its partial report rather than being killed with nothing sent. + timeout_in_minutes: 480 device: cpu-small commands: - bash /vllm-workspace/.buildkite/scripts/crcr-report.sh diff --git a/.buildkite/test_areas/disaggregated.yaml b/.buildkite/test_areas/disaggregated.yaml index 43abe2ce01..b9bd59583b 100644 --- a/.buildkite/test_areas/disaggregated.yaml +++ b/.buildkite/test_areas/disaggregated.yaml @@ -79,6 +79,22 @@ steps: commands: - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh - bash v1/kv_connector/nixl_push_integration/config_sweep_accuracy_test.sh + mirror: + amd: + label: ":amd: (MI300) Push NixlConnector PP prefill PD accuracy" + dind: false + device: mi300_4 + timeout_in_minutes: 55 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ + - tests/v1/kv_connector/nixl_integration/ + - tests/v1/kv_connector/nixl_push_integration/ + - vllm/platforms/rocm.py + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_push_integration/config_sweep_accuracy_test.sh - label: ":nvidia: (L4) DP EP Distributed NixlConnector PD accuracy" key: dp-ep-distributed-nixlconnector-pd-accuracy-tests-4-gpus @@ -180,6 +196,22 @@ steps: commands: - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh - bash v1/kv_connector/nixl_integration/run_edge_case_test.sh + mirror: + amd: + label: ":amd: (MI300) NixlConnector PD edge case" + dind: false + device: mi300_2 + timeout_in_minutes: 65 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ + - vllm/v1/core/sched/ + - tests/v1/kv_connector/nixl_integration/ + - vllm/platforms/rocm.py + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + - PREFILL_GPU_ID=0 DECODE_GPU_ID=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_edge_case_test.sh - label: ":nvidia: (L4) Hybrid SSM NixlConnector PD prefix cache" key: hybrid-ssm-nixlconnector-pd-prefix-cache-2-gpus diff --git a/.buildkite/test_areas/kernels.yaml b/.buildkite/test_areas/kernels.yaml index c53c91cb63..c939553f64 100644 --- a/.buildkite/test_areas/kernels.yaml +++ b/.buildkite/test_areas/kernels.yaml @@ -47,7 +47,7 @@ steps: label: ":amd: (MI300) Core Operation Kernels Shard %N" dind: false device: mi300_1 - timeout_in_minutes: 40 + timeout_in_minutes: 45 working_dir: "/vllm-workspace/tests" depends_on: - image-build-amd @@ -143,6 +143,33 @@ steps: commands: - pytest -v -s kernels/test_fused_deepseek_v4_*.py - pytest -v -s models/test_deepseek_v4_mega_moe.py + mirror: + amd: + label: ":amd: (MI355) DeepSeek V4 MoE Integration" + dind: false + device: mi355_1 + num_devices: 1 + timeout_in_minutes: 20 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - tests/models/test_deepseek_v4_vl_rocm.py + - vllm/models/deepseek_v4/amd/ + - vllm/models/deepseek_v4/common/ + - vllm/model_executor/layers/fused_moe/ + - vllm/model_executor/layers/activation.py + - vllm/model_executor/layers/linear.py + - vllm/config/ + - vllm/forward_context.py + - vllm/v1/worker/workspace.py + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/envs.py + commands: + # Common qnorm/RoPE/cache kernels already run in the H100 AMD mirror. + # Pin the tested BF16 Triton routing and shared-expert path before imports. + - VLLM_ROCM_USE_AITER_MOE=0 pytest -v -s models/test_deepseek_v4_vl_rocm.py # Own job so mHC never shares a process with FlashInfer's import-time # ptxas override. @@ -173,8 +200,8 @@ steps: # Catch-all for test files at the tests/kernels root. This job collects # the whole root so new files are wired by default. # Files with dedicated jobs elsewhere in this file are excluded via --ignore -# (test_kda, test_bf16x3_router_gemm_cutedsl and test_ll_bf16_gemm run in -# their own jobs / Kernels (B200)). +# (test_bf16x3_router_gemm_cutedsl and test_ll_bf16_gemm run in their own jobs / +# Kernels (B200)). - label: ":nvidia: (B200) Miscellaneous Kernels" key: kernels-root-misc-test-b200 timeout_in_minutes: 30 @@ -199,7 +226,6 @@ steps: --ignore=kernels/test_fused_qk_norm_rope_gate.py --ignore=kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py --ignore=kernels/test_top_k_per_row.py - --ignore=kernels/test_kda.py --ignore=kernels/test_bf16x3_router_gemm_cutedsl.py --ignore=kernels/test_ll_bf16_gemm.py --ignore=kernels/test_shuffle_rows.py diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index 9f99e1c53f..971e308d25 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -186,7 +186,7 @@ steps: label: ":amd: (MI355) LM Eval Small Models Distributed" dind: false device: mi355_2 - timeout_in_minutes: 55 + timeout_in_minutes: 145 depends_on: - image-build-amd source_file_dependencies: @@ -196,11 +196,22 @@ steps: - vllm/model_executor/model_loader/ - vllm/v1/attention/backends/ - vllm/v1/attention/selector.py + - vllm/v1/worker/ + - vllm/v1/core/ + - vllm/config/ - tests/evals/gsm8k/configs/models-mi3xx-fp8-and-mixed.txt - vllm/_aiter_ops.py - vllm/platforms/rocm.py + - tests/evals/gsm8k/ + - tests/utils.py + - vllm/distributed/ + - vllm/transformers_utils/configs/diffusion_gemma.py + - vllm/v1/attention/ops/ commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx-fp8-and-mixed.txt + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small-tp.txt + # Dense MLA PCP-only accuracy; DP1/DCP1. + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small-tp-mi355.txt - label: ":nvidia: (B200) LM Eval HiSparse Nightly" key: lm-eval-hisparse-nightly-4xb200 @@ -319,10 +330,10 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-spec-decode.txt mirror: amd: - label: ":amd: (MI355) LM Eval Spec Decode" + label: ":amd: (MI355) LM Eval Spec Decode (Fixed-length FP8)" dind: false device: mi355_4 - timeout_in_minutes: 120 + timeout_in_minutes: 145 depends_on: - image-build-amd source_file_dependencies: @@ -355,6 +366,9 @@ steps: - vllm/v1/worker/ - vllm/_aiter_ops.py - vllm/platforms/rocm.py + - tests/evals/gsm8k/ + - tests/utils.py + - vllm/distributed/ commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-spec-decode-rocm.txt @@ -951,6 +965,22 @@ steps: - tests/evals/gsm8k/test_gsm8k_offloading.py commands: - pytest -s -v evals/gsm8k/test_gsm8k_offloading.py -k "nemotron-h-8b or gemma-4-e4b-it" + mirror: + amd: + label: ":amd: (MI300) LM Eval KV-Offload" + dind: false + device: mi300_1 + timeout_in_minutes: 55 + depends_on: + - image-build-amd + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/offloading/ + - vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py + - vllm/v1/kv_offload/ + - vllm/v1/simple_kv_offload/ + - tests/evals/gsm8k/test_gsm8k_offloading.py + - vllm/platforms/rocm.py - label: ":nvidia: (H100) LM Eval KV-Offload Medium" key: kv-offload-medium @@ -965,6 +995,22 @@ steps: - tests/evals/gsm8k/test_gsm8k_offloading.py commands: - pytest -s -v evals/gsm8k/test_gsm8k_offloading.py -k "qwen3.5-35b or deepseek-v2-lite" + mirror: + amd: + label: ":amd: (MI300) LM Eval KV-Offload Medium" + dind: false + device: mi300_2 + timeout_in_minutes: 70 + depends_on: + - image-build-amd + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/offloading/ + - vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py + - vllm/v1/kv_offload/ + - vllm/v1/simple_kv_offload/ + - tests/evals/gsm8k/test_gsm8k_offloading.py + - vllm/platforms/rocm.py - label: ":nvidia: (H100) LM Eval KV-Offload Large" key: kv-offload-large @@ -979,6 +1025,24 @@ steps: - tests/evals/gsm8k/test_gsm8k_offloading.py commands: - pytest -s -v evals/gsm8k/test_gsm8k_offloading.py -k "deepseek-v4-flash" + mirror: + amd: + label: ":amd: (MI300) LM Eval KV-Offload Large" + dind: false + device: mi300_4 + timeout_in_minutes: 65 + depends_on: + - image-build-amd + working_dir: /vllm-workspace/tests + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/offloading/ + - vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py + - vllm/v1/kv_offload/ + - vllm/v1/simple_kv_offload/ + - tests/evals/gsm8k/test_gsm8k_offloading.py + - vllm/platforms/rocm.py + commands: + - VLLM_ROCM_USE_AITER=1 pytest -s -v evals/gsm8k/test_gsm8k_offloading.py -k "deepseek-v4-flash" - label: ":nvidia: (H200 MIG 35GB) MRCR Eval Small Models" key: mrcr-eval-small-models diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index f4b07ffd97..0c0e8c9a4d 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -496,15 +496,22 @@ steps: - vllm/assets/ - vllm/config/ - vllm/device_allocator/ + - vllm/connections.py - vllm/engine/arg_utils.py - vllm/entrypoints/chat_utils.py - vllm/entrypoints/mcp/ - vllm/entrypoints/openai/chat_completion/protocol.py + - vllm/entrypoints/openai/completion/protocol.py + - vllm/entrypoints/openai/responses/protocol.py - vllm/entrypoints/generate/base/protocol.py + - vllm/env_override.py - vllm/envs.py - vllm/exceptions.py - vllm/inputs/ + - vllm/logging_utils/ + - vllm/logprobs.py - vllm/model_executor/layers/quantization/quark/ + - vllm/model_executor/model_loader/tensorizer.py - vllm/multimodal/ - vllm/outputs.py - vllm/parser/ @@ -514,17 +521,38 @@ steps: - vllm/reasoning/ - vllm/renderers/ - vllm/sampling_params.py + - vllm/scalar_type.py - vllm/tokenizers/ - vllm/tool_parsers/ - vllm/transformers_utils/ + - vllm/triton_utils/ - vllm/utils/ - vllm/v1/ + - vllm/version.py + - cmake/utils.cmake + - tests/test_access_log_filter.py + - tests/test_attention_backend_registry.py + - tests/test_audio_media_size_precheck.py + - tests/test_cmake_utils.py + - tests/test_connections.py + - tests/test_embedded_commit.py - tests/test_envs.py + - tests/test_force_first_config.py + - tests/test_fxgraphcache_pickle_patch.py + - tests/test_logprobs.py - tests/test_outputs.py - tests/test_pcp_dp.py - tests/test_pooling_params.py + - tests/test_quantization_revision_pin.py - tests/test_ray_env.py + - tests/test_ray_env_utils.py + - tests/test_request_input_bounds.py - tests/test_sampling_params.py + - tests/test_scalartype.py + - tests/test_tensorizer_revision_pin.py + - tests/test_triton_utils.py + - tests/test_version.py + - tests/test_zen_cpu_platform_detection.py - tests/multimodal - tests/renderers - tests/standalone_tests/lazy_imports.py @@ -540,14 +568,32 @@ steps: - tests/device_allocator device: cpu-small commands: + - pip install cmake # test_cmake_utils.py shells out to cmake; the CPU image has none - python3 standalone_tests/lazy_imports.py - pytest -v -s device_allocator + - pytest -v -s test_access_log_filter.py + - pytest -v -s test_attention_backend_registry.py + - pytest -v -s test_audio_media_size_precheck.py + - pytest -v -s test_cmake_utils.py + - pytest -v -s test_connections.py + - pytest -v -s test_embedded_commit.py - pytest -v -s test_envs.py + - pytest -v -s test_force_first_config.py + - pytest -v -s test_fxgraphcache_pickle_patch.py + - pytest -v -s test_logprobs.py - pytest -v -s test_outputs.py - pytest -v -s test_pcp_dp.py - pytest -v -s test_pooling_params.py + - pytest -v -s test_quantization_revision_pin.py - pytest -v -s test_ray_env.py + - pytest -v -s test_ray_env_utils.py + - pytest -v -s test_request_input_bounds.py - pytest -v -s test_sampling_params.py + - pytest -v -s test_scalartype.py + - pytest -v -s test_tensorizer_revision_pin.py + - pytest -v -s test_triton_utils.py + - pytest -v -s test_version.py + - pytest -v -s test_zen_cpu_platform_detection.py - pytest -v -s tokenizers_ - pytest -v -s parser - pytest -v -s transformers_utils diff --git a/.buildkite/test_areas/models_basic.yaml b/.buildkite/test_areas/models_basic.yaml index 83a08cfc24..62805e90f6 100644 --- a/.buildkite/test_areas/models_basic.yaml +++ b/.buildkite/test_areas/models_basic.yaml @@ -95,6 +95,23 @@ steps: commands: # FA4 kernel tests require SM100; the suite skips them elsewhere. - pytest -v -s models/inkling + mirror: + amd: + label: ":amd: (MI355) Inkling" + dind: false + device: mi355_1 + timeout_in_minutes: 65 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/models/inkling/ + - vllm/cute_utils/ + - cmake/external_projects/tml_fa4.cmake + - tests/models/inkling/ + - vllm/platforms/rocm.py + commands: + - pytest -v -s models/inkling/rocm - label: ":nvidia: (B200) Kimi K3" key: kimi-k3-unit-tests-b200 diff --git a/.buildkite/test_areas/pytorch.yaml b/.buildkite/test_areas/pytorch.yaml index 38f0f617bc..5b6260b534 100644 --- a/.buildkite/test_areas/pytorch.yaml +++ b/.buildkite/test_areas/pytorch.yaml @@ -5,8 +5,8 @@ steps: - label: ":nvidia: (H200 MIG 35GB) PyTorch Compilation" device: h200_35gb key: pytorch-compilation-unit-tests - timeout_in_minutes: 60 - source_file_dependencies: + timeout_in_minutes: 30 + source_file_dependencies: &pytorch-compilation-deps - vllm/__init__.py - vllm/_aiter_ops.py - vllm/_custom_ops.py @@ -47,7 +47,26 @@ steps: label: ":amd: (MI300) PyTorch Compilation" dind: false device: mi300_1 - timeout_in_minutes: 90 + timeout_in_minutes: 45 + depends_on: + - image-build-amd + source_file_dependencies: + - csrc/ + - vllm/platforms/rocm.py + +- label: ":nvidia: (H200 MIG 35GB) PyTorch Compilation / Dynamic Shapes" + device: h200_35gb + key: pytorch-compilation-dynamic-shapes + timeout_in_minutes: 30 + source_file_dependencies: *pytorch-compilation-deps + commands: + - pytest -s -v compile/dynamic_shapes/ + mirror: + amd: + label: ":amd: (MI300) PyTorch Compilation / Dynamic Shapes" + dind: false + device: mi300_1 + timeout_in_minutes: 45 depends_on: - image-build-amd source_file_dependencies: diff --git a/.buildkite/test_areas/quantization.yaml b/.buildkite/test_areas/quantization.yaml index d1f47cf2a7..122679553f 100644 --- a/.buildkite/test_areas/quantization.yaml +++ b/.buildkite/test_areas/quantization.yaml @@ -58,6 +58,22 @@ steps: - vllm/model_executor/layers/quantization/modelopt.py commands: - pytest -v -s fusion/ + mirror: + amd: + label: ":amd: (MI300) Quantized Fusions" + dind: false + device: mi300_1 + timeout_in_minutes: 45 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd + source_file_dependencies: + - tests/fusion + - vllm/model_executor/layers/fusion + - vllm/model_executor/kernels/linear + - vllm/model_executor/layers/quantization/compressed_tensors + - vllm/model_executor/layers/quantization/modelopt.py + - vllm/platforms/rocm.py - label: ":nvidia: (B200) Quantized MoE" key: quantized-moe-test-b200 diff --git a/.buildkite/test_areas/torch_abi.yaml b/.buildkite/test_areas/torch_abi.yaml index 201f9b6093..ee95203a01 100644 --- a/.buildkite/test_areas/torch_abi.yaml +++ b/.buildkite/test_areas/torch_abi.yaml @@ -12,3 +12,22 @@ steps: - setup.py commands: - python3 /vllm-workspace/.buildkite/check-torch-abi.py + mirror: + amd: + label: ":amd: (MI250) Torch Stable ABI Audit" + dind: false + device: mi300_1 + timeout_in_minutes: 35 + depends_on: + - image-build-amd + no_gpu: true + working_dir: /vllm-workspace/ + source_file_dependencies: + - .buildkite/check-torch-abi.py + - csrc/ + - cmake/ + - setup.py + - vllm/platforms/rocm.py + commands: + - uv pip install --system torch-abi-audit==0.0.1 + - python3 /vllm-workspace/.buildkite/check-torch-abi.py diff --git a/docker/Dockerfile b/docker/Dockerfile index 6609ed7464..5f4c4ab35f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -880,12 +880,15 @@ RUN if [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ]; then \ # `vllm snapshot`. CRIU and its CUDA plugin are built from the same pinned # source. The `cuda-checkpoint`-compatible helper is built from NVIDIA Dynamo's # Apache-2.0 source against the public CUDA Driver checkpoint APIs instead of -# redistributing NVIDIA's proprietary cuda-checkpoint binary. +# redistributing NVIDIA's proprietary cuda-checkpoint binary. The helper uses +# checkpoint APIs (CUcheckpointGpuPair) that first appear in the CUDA 13 +# headers, so CUDA 12.x images skip the snapshot runtime. COPY tools/install_snapshot_runtime.sh /usr/local/bin/install_snapshot_runtime.sh -RUN if [ "${TARGETPLATFORM}" = "linux/amd64" ]; then \ +RUN if [ "${TARGETPLATFORM}" = "linux/amd64" ] \ + && [ "${CUDA_VERSION%%.*}" -ge 13 ]; then \ MAX_JOBS="${max_jobs}" /usr/local/bin/install_snapshot_runtime.sh; \ else \ - echo "Initialized snapshots are not installed for ${TARGETPLATFORM}"; \ + echo "Initialized snapshots are not installed for ${TARGETPLATFORM} with CUDA ${CUDA_VERSION}"; \ fi ENV CRIU_CUDA_PLUGIN_DIR=/usr/local/lib/criu diff --git a/docker/Dockerfile.cpu b/docker/Dockerfile.cpu index 43172fbdcf..ed5dfa5d24 100644 --- a/docker/Dockerfile.cpu +++ b/docker/Dockerfile.cpu @@ -433,6 +433,8 @@ ADD ./benchmarks/ ./benchmarks/ ADD ./vllm/collect_env.py . ADD ./docker/ ./docker/ ADD ./.buildkite/ ./.buildkite/ +# cmake/ is needed by tests/test_cmake_utils.py, which includes cmake/utils.cmake. +ADD ./cmake/ ./cmake/ # tools/ carries repo-tooling code that a few tests exercise directly # (e.g. tests/tools/test_check_test_tethering.py imports # tools.pre_commit.check_test_tethering). @@ -480,5 +482,6 @@ LABEL ai.vllm.build.python-version="${PYTHON_VERSION:-3.12}" # stage above adds examples/ for testing only, so without this the published # vllm-openai-cpu image would not ship examples/*.jinja. COPY examples examples +COPY tools/recipes tools/recipes ENTRYPOINT ["vllm", "serve"] diff --git a/docker/Dockerfile.rock b/docker/Dockerfile.rock index babc4db100..afeac31742 100644 --- a/docker/Dockerfile.rock +++ b/docker/Dockerfile.rock @@ -926,12 +926,15 @@ RUN --mount=type=cache,target=/root/.cache/uv \ # Install vLLM using uv (inherited from base stage) # Note: No -U flag to avoid upgrading PyTorch ROCm to CUDA version +# Keep OpenTelemetry within LMCache's cap. RUN --mount=type=bind,from=export_vllm,src=/,target=/install \ --mount=type=cache,target=/root/.cache/uv \ cd /install \ - && uv pip install --system -c /etc/rocm-constraints.txt -r requirements/rocm.txt \ + && echo "opentelemetry-api<=1.40.0" > /tmp/otel-constraints.txt \ + && uv pip install --system -c /etc/rocm-constraints.txt -c /tmp/otel-constraints.txt -r requirements/rocm.txt \ && pip uninstall -y vllm \ - && uv pip install --system -c /etc/rocm-constraints.txt *.whl + && uv pip install --system -c /etc/rocm-constraints.txt -c /tmp/otel-constraints.txt *.whl \ + && rm /tmp/otel-constraints.txt # Install NIXL ROCm wheel RUN --mount=type=bind,from=build_nixl,src=/app/install,target=/nixl_install \ diff --git a/docker/Dockerfile.rocm b/docker/Dockerfile.rocm index ecc2600a71..74b007e5e7 100644 --- a/docker/Dockerfile.rocm +++ b/docker/Dockerfile.rocm @@ -948,12 +948,15 @@ RUN --mount=type=cache,target=/root/.cache/uv \ # Install vLLM using uv (inherited from base stage) # Note: No -U flag to avoid upgrading PyTorch ROCm to CUDA version +# Keep OpenTelemetry within LMCache's cap. RUN --mount=type=bind,from=export_vllm,src=/,target=/install \ --mount=type=cache,target=/root/.cache/uv \ cd /install \ - && uv pip install --system -r requirements/rocm.txt \ + && echo "opentelemetry-api<=1.40.0" > /tmp/otel-constraints.txt \ + && uv pip install --system -c /tmp/otel-constraints.txt -r requirements/rocm.txt \ && pip uninstall -y vllm \ - && uv pip install --system *.whl + && uv pip install --system -c /tmp/otel-constraints.txt *.whl \ + && rm /tmp/otel-constraints.txt # Install NIXL ROCm wheel RUN --mount=type=bind,from=build_nixl,src=/app/install,target=/nixl_install \ diff --git a/docker/Dockerfile.s390x b/docker/Dockerfile.s390x index 6faba540a9..06be3ad523 100644 --- a/docker/Dockerfile.s390x +++ b/docker/Dockerfile.s390x @@ -141,6 +141,41 @@ RUN --mount=type=cache,target=/root/.cache/uv \ sed -i '/#include "internal\/pycore_atomic.h"/i\#include "dynamic_annotations.h"' numba/_dispatcher.cpp; \ fi && python setup.py bdist_wheel +# Shared FFmpeg for torchcodec video decoding (UBI has no ffmpeg package). +FROM base AS ffmpeg-build +ARG FFMPEG_VERSION=n6.1.2 +WORKDIR /tmp +RUN microdnf install -y pkgconf-pkg-config xz && \ + curl -LO https://github.com/FFmpeg/FFmpeg/archive/refs/tags/${FFMPEG_VERSION}.tar.gz && \ + tar -xzf ${FFMPEG_VERSION}.tar.gz && \ + cd FFmpeg-${FFMPEG_VERSION} && \ + ./configure --prefix=/usr/local --libdir=/usr/local/lib \ + --enable-shared --disable-static --enable-pic \ + --disable-doc --disable-debug --disable-ffplay --disable-ffprobe \ + --extra-cflags="-Wno-incompatible-pointer-types" && \ + make -j"$(nproc)" && \ + make install + +# Torchcodec video decoder (no s390x wheels; CPU / FFmpeg only s390x does not support audio decoding). +FROM python-install AS torchcodec-builder +ARG TORCHCODEC_VERSION=v0.14.0 +WORKDIR /tmp +ENV LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" +ENV PKG_CONFIG_PATH="/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH" +RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=bind,from=ffmpeg-build,source=/usr/local,target=/usr/local \ + git clone --depth 1 --branch ${TORCHCODEC_VERSION} \ + https://github.com/pytorch/torchcodec.git && \ + cd torchcodec && \ + uv pip install torch==2.13.0 --index-url https://download.pytorch.org/whl/cpu && \ + uv pip install pybind11 setuptools wheel && \ + pybind11_DIR="$(python -c 'import pybind11; print(pybind11.get_cmake_dir())')" && \ + export pybind11_DIR CMAKE_PREFIX_PATH="${pybind11_DIR}:${CMAKE_PREFIX_PATH}" && \ + export I_CONFIRM_THIS_IS_NOT_A_LICENSE_VIOLATION=1 && \ + export TORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR=1 && \ + export TORCHCODEC_CMAKE_BUILD_DIR="${PWD}/build" && \ + python setup.py bdist_wheel + # Build OpenCV from source for s390x FROM python-install AS opencv-builder WORKDIR /tmp @@ -179,6 +214,10 @@ RUN --mount=type=bind,from=gperftools-build,src=/tmp/gperftools-2.16,target=/gpe make -C /gperftools install-libLTLIBRARIES && \ /sbin/ldconfig +RUN --mount=type=bind,from=ffmpeg-build,source=/usr/local,target=/ffmpeg-prefix \ + cp -a /ffmpeg-prefix/lib/. /usr/local/lib/ && \ + /sbin/ldconfig + ENV LD_PRELOAD=/usr/local/lib/libtcmalloc_minimal.so.4 # Install dependencies, including PyTorch and Apache Arrow @@ -190,17 +229,20 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=bind,from=numba-builder,source=/tmp/llvmlite/dist,target=/tmp/llvmlite-wheels/ \ --mount=type=bind,from=numba-builder,source=/tmp/numba/dist,target=/tmp/numba-wheels/ \ --mount=type=bind,from=opencv-builder,source=/tmp/opencv-python/dist,target=/tmp/opencv-wheels/ \ + --mount=type=bind,from=torchcodec-builder,source=/tmp/torchcodec/dist,target=/tmp/torchcodec-wheels/ \ VISION_WHL_FILE=$(ls /tmp/vision-wheels/*.whl) && \ HF_XET_WHL_FILE=$(ls /tmp/hf-xet-wheels/*.whl) && \ LLVM_WHL_FILE=$(ls /tmp/llvmlite-wheels/*.whl) && \ NUMBA_WHL_FILE=$(ls /tmp/numba-wheels/*.whl) && \ OPENCV_WHL_FILE=$(ls /tmp/opencv-wheels/*.whl) && \ + TORCHCODEC_WHL_FILE=$(ls /tmp/torchcodec-wheels/*.whl) && \ uv pip install -v \ $VISION_WHL_FILE \ $HF_XET_WHL_FILE \ $LLVM_WHL_FILE \ $NUMBA_WHL_FILE \ $OPENCV_WHL_FILE \ + $TORCHCODEC_WHL_FILE \ --torch-backend cpu \ --index-strategy unsafe-best-match \ -r requirements/build/cpu.txt \ diff --git a/docs/design/mm_processing.md b/docs/design/mm_processing.md index 8e49a4099c..d350836ca8 100644 --- a/docs/design/mm_processing.md +++ b/docs/design/mm_processing.md @@ -78,10 +78,17 @@ GPU-side fusion is controlled by `multimodal_config.mm_device_do_normalize`: - The flag is **enabled by default** for all models that support it. - Currently, it’s on by default for these architectures: -| name | Architecture | Example HF Models | -|--------------|--------------------------------------|-------------------------------------| -| `qwen2-vl` | `Qwen2VLForConditionalGeneration` | `Qwen/Qwen2-VL-2B-Instruct`, etc. | -| `qwen2.5-vl` | `Qwen2_5_VLForConditionalGeneration` | `Qwen/Qwen2.5-VL-3B-Instruct`, etc. | +| name | Architecture | Example HF Models | +|--------------------|------------------------------------------|----------------------------------------| +| `intern-s1-pro` | `InternS1ProForConditionalGeneration` | `internlm/Intern-S1-Pro` | +| `intern-s2-mobius` | `InternS2MobiusForConditionalGeneration` | `internlm/Intern-S2-Mobius` | +| `qwen2-vl` | `Qwen2VLForConditionalGeneration` | `Qwen/Qwen2-VL-2B-Instruct`, etc. | +| `qwen2.5-vl` | `Qwen2_5_VLForConditionalGeneration` | `Qwen/Qwen2.5-VL-3B-Instruct`, etc. | +| `qwen3-vl` | `Qwen3VLForConditionalGeneration` | `Qwen/Qwen3-VL-4B-Instruct`, etc. | +| `qwen3-vl-moe` | `Qwen3VLMoeForConditionalGeneration` | `Qwen/Qwen3-VL-30B-A3B-Instruct`, etc. | +| `qwen3.5` | `Qwen3_5ForConditionalGeneration` | `Qwen/Qwen3.5-0.8B`, etc. | +| `qwen3.5-moe` | `Qwen3_5MoeForConditionalGeneration` | `Qwen/Qwen3.5-35B-A3B`, etc. | +| `qwen4-exp` | `Qwen4ExpForConditionalGeneration` | `Qwen/Qwen3.8-Flash-Next` | #### Key Properties and Gains diff --git a/docs/design/nixl_kv_push_connector.md b/docs/design/nixl_kv_push_connector.md index a6debd7dba..621b0f7c9d 100644 --- a/docs/design/nixl_kv_push_connector.md +++ b/docs/design/nixl_kv_push_connector.md @@ -195,35 +195,53 @@ combined with a hybrid (HMA) KV layout: that represents a pooled region can differ between P and D. `NixlPushConnector` handles this for PP-sharded producers by routing -**by layer-name (member) identity** instead of by region index. Each +**by layer name** instead of by region index. Each worker advertises which layer names back each of its NIXL regions in the handshake metadata (`NixlAgentMetadata.region_members`). A producer that -needs member routing derives its member-major layout once, when it +needs layer routing derives its transfer layer order once, when it registers its KV caches, and every transfer it issues uses that order. `add_remote_agent` then selects exactly the remote regions this stage owns and reorders them to match, so both sides stay paired regardless of how each remote rank happens to order its metadata. Addresses, block lengths, strides, and per-region capacities follow the same -member order. Descriptor offsets use each member's region capacity, so P and +layer order. Descriptor offsets use each layer's region capacity, so P and D need not allocate the same number of blocks. Physical allocations are still -registered once, even when multiple members share them. +registered once, even when multiple layers share them. + +Packed MLA caches (block-outermost layouts such as DeepSeek-V4's) store the +pages of every layer for one block in a single row. A PP stage holds only its +own layers, so its rows are shorter and each layer sits at a different offset: + +```text +D (PP=1), block b: | L0 | L1 | L2 | L3 | L4 | L5 | +P (stage 2), block b: | L3 | L4 | L5 | +``` + +`PP=1` peers transfer whole rows. They also advertise where each layer's page +sits in the row (`NixlAgentMetadata.packed_member_layouts`), so a PP producer +writes each of its pages to the matching slot in D's row. Page sizes must +match; offsets, row strides, and block counts may differ. Because PP stages can +list the same attention backends in a different order, packed push compares the +sorted set of backend names during the handshake. Pull mode is unchanged. Invariants enforced when the remote regions are aligned: -* every locally owned member must be advertised exactly once by the - remote; a missing member fails the handshake rather than silently - leaving that layer's KV stale, and remote-only members (owned by other +* every locally owned layer must be advertised exactly once by the + remote; a missing layer fails the handshake rather than silently + leaving that layer's KV stale, and remote-only layers (owned by other PP stages) are ignored; -* a remote that omits member metadata while the local layout requires - member routing fails the handshake instead of falling back to +* a remote that omits layer metadata while the local layout requires + layer routing fails the handshake instead of falling back to region-index routing; -* member order is a property of the local layout alone, so the same local +* layer order is a property of the local layout alone, so the same local source descriptors serve every remote engine and TP rank; only the remote descriptor list is rebuilt per rank. Decode-side PP is unsupported because completions are counted per -consumer rank. Mamba/SSM hybrids are unsupported under PP. +consumer rank. Mamba/SSM hybrids are unsupported under PP. Packed PP push +requires MLA caches and equal P/D block sizes. MLA pages are replicated +across TP ranks, so producer TP1 to consumer TP2 is supported. ## Scheduler-side responsibilities diff --git a/docs/features/disagg_prefill.md b/docs/features/disagg_prefill.md index 4c5265449c..e6a5797f26 100644 --- a/docs/features/disagg_prefill.md +++ b/docs/features/disagg_prefill.md @@ -77,6 +77,8 @@ decode = client.chat.completions.create( ) ``` +If `messages` has non-text content or `echo` is set, the ids are ignored and `messages` is rendered instead, so it must match the prefill request. Otherwise `kv_transfer_params["prompt_token_ids"]` must be a non-empty list of non-negative integers, or the request fails with HTTP 400, as it always does on `/v1/chat/completions/batch`. + ## Development We implement disaggregated prefilling by running 2 vLLM instances. One for prefill (we call it prefill instance) and one for decode (we call it decode instance), and then use a connector to transfer the prefill KV caches and results from prefill instance to decode instance. diff --git a/docs/features/lora.md b/docs/features/lora.md index 78004722a9..4c95bde8cd 100644 --- a/docs/features/lora.md +++ b/docs/features/lora.md @@ -428,9 +428,16 @@ vLLM supports PEFT sequence-classification adapters that save a complete, single See [classification_with_lora_offline.py](../../examples/pooling/classify/classification_with_lora_offline.py) for an offline classification example using a LoRA adapter. +To batch adapters with different `num_labels`, set the maximum number of labels: + +```bash +vllm serve model --enable-lora --max-lora-cls-labels 8 +``` + +The equivalent `LLM` argument is `max_lora_cls_labels`. It defaults to the base model's `num_labels`, and each request returns its adapter's number of labels. + This support has the following limitations: -- All adapters in one engine must have the same `num_labels` and hidden size as the base classification head. An incompatible adapter is rejected when it is loaded. - A classification head stored as float32 is converted to the runtime head dtype when it is loaded. - Token-classification adapters are not supported by this feature. diff --git a/docs/features/nixl_connector_compatibility.md b/docs/features/nixl_connector_compatibility.md index a41d5868b1..46cb0bc482 100644 --- a/docs/features/nixl_connector_compatibility.md +++ b/docs/features/nixl_connector_compatibility.md @@ -120,7 +120,8 @@ Current push PP + HMA limitations: - Only the prefiller (producer) may be PP-sharded; decode-side PP is not supported. - Hybrid SSM/Mamba layouts are not supported under PP. - HMA requires the same block size on P and D. -- Attention-HMA member routing requires decode TP to be no greater than prefill TP. +- Non-MLA attention-HMA layer routing requires decode TP to be no greater than prefill TP. +- Packed layouts under PP require MLA caches. Local and remote packed block strides may differ; layer pages must have equal sizes. MLA supports decode TP greater than prefill TP because its KV is replicated. ### Quantized KV cache diff --git a/docs/getting_started/installation/cpu.s390x.inc.md b/docs/getting_started/installation/cpu.s390x.inc.md index e1ff4ee3cb..4f2c052fac 100644 --- a/docs/getting_started/installation/cpu.s390x.inc.md +++ b/docs/getting_started/installation/cpu.s390x.inc.md @@ -11,7 +11,7 @@ Currently, the CPU implementation for s390x architecture supports FP32, BF16 and - OS: `Linux` - SDK: `gcc/g++ >= 14.0.0` or later with Command Line Tools - Instruction Set Architecture (ISA): VXE support is required. Works with Z15 and above. -- Build from source python packages (no pre-built s390x wheels): `torchvision`, `llvmlite`, `numba`, `opencv-python-headless`, `hf-xet` +- Build from source python packages (no pre-built s390x wheels): `torchvision`, `llvmlite`, `numba`, `opencv-python-headless`, `hf-xet`, `torchcodec` --8<-- [end:requirements] --8<-- [start:set-up-using-python] @@ -55,7 +55,8 @@ curl https://sh.rustup.rs -sSf | sh -s -- -y && \ Execute the following commands to build and install vLLM from source. !!! tip - Pre-built wheels are not available for s390x for the following packages. Build them from source before building vLLM: `torchvision`, `llvmlite`, `numba`, `opencv-python-headless`, `hf-xet`. + Pre-built wheels are not available for s390x for the following packages. Build them from source before building vLLM: `torchvision`, `llvmlite`, `numba`, `opencv-python-headless`, `hf-xet`, `torchcodec`. + `torchcodec` also needs a shared FFmpeg (built in `Dockerfile.s390x`). Video decoding only; there is no s390x NVDEC path. See `docker/Dockerfile.s390x` for exact versions and build commands used in each multi-stage build. !!! note "LLVM 20 required for llvmlite" @@ -94,6 +95,7 @@ uv pip install -v \ /path/to/numba.whl \ /path/to/opencv_python_headless.whl \ /path/to/hf_xet.whl \ + /path/to/torchcodec.whl \ -r requirements/build/cpu.txt \ -r requirements/cpu.txt \ --torch-backend cpu \ @@ -111,6 +113,7 @@ VLLM_TARGET_DEVICE=cpu VLLM_CPU_MOE_PREPACK=0 python setup.py bdist_wheel && \ /path/to/numba.whl \ /path/to/opencv_python_headless.whl \ /path/to/hf_xet.whl \ + /path/to/torchcodec.whl \ -r requirements/build/cpu.txt \ -r requirements/cpu.txt && \ VLLM_TARGET_DEVICE=cpu VLLM_CPU_MOE_PREPACK=0 python setup.py bdist_wheel && \ diff --git a/docs/getting_started/installation/cpu.x86.inc.md b/docs/getting_started/installation/cpu.x86.inc.md index 260ffe2863..ebae441f93 100644 --- a/docs/getting_started/installation/cpu.x86.inc.md +++ b/docs/getting_started/installation/cpu.x86.inc.md @@ -183,6 +183,40 @@ docker run \ vllm/vllm-openai-cpu:latest-x86_64 ``` +### Serve with vLLM Recipes + +The CPU image includes the [vLLM Recipes](https://recipes.vllm.ai/) deployment +tool. Recipe-assisted serving queries the site at runtime, so the image can use +the latest published recipe for the selected model and hardware. + +To find the value for `--hardware`, open vLLM Recipes, select the model, and +choose the target in the **Hardware** picker. The selected page URL also +contains the hardware key; for example, `?hardware=xeon6` maps to +`--hardware xeon6`. + +Keep the normal image entrypoint for regular serving. For recipe-assisted +serving, override it with `serve_with_recipe.sh`: + +```bash +docker run --rm \ + --shm-size=4g \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + -p 8000:8000 \ + --env "HF_TOKEN=" \ + --entrypoint tools/recipes/serve_with_recipe.sh \ + vllm/vllm-openai-cpu:latest-x86_64 \ + --model meta-llama/Llama-3.1-8B-Instruct \ + --hardware xeon6 +``` + +For Xeon 6, the script enables hardware detection automatically, generates +`config.yml` and `env.sh`, and starts `vllm serve`. The image keeps +`/vllm-workspace` as its working directory so relative recipe assets under +`examples/` continue to resolve. + +See the [Recipes tool documentation](../../../tools/recipes/README.md) for +interactive discovery and additional options. + --8<-- [end:pre-built-images] --8<-- [start:build-image-from-source] diff --git a/docs/models/hardware_supported_models/cpu.md b/docs/models/hardware_supported_models/cpu.md index 10ca8caa0e..6e6da02876 100644 --- a/docs/models/hardware_supported_models/cpu.md +++ b/docs/models/hardware_supported_models/cpu.md @@ -12,17 +12,17 @@ ## Deploy from a vLLM Recipe -The **Recipe** column below links to validated Xeon 6 configurations when -available. Use the Recipe conversion tool to generate `config.yaml` and -`env.sh`; see the -[Recipes conversion tool README](../../../tools/recipes/README.md) for usage. +The **Recipe** column below links to published Xeon 6 configurations when +available. Open a recipe to review the latest deployment configuration for the +model and hardware. -Load the generated environment before starting vLLM: +For one-step deployment with the pre-built CPU image, see +[Serve with vLLM Recipes](../../getting_started/installation/cpu.md#serve-with-vllm-recipes). +The image includes the Recipes tool, which can retrieve the latest published +recipe, apply Xeon hardware detection, and start `vllm serve`. -```bash -source env.sh -vllm serve --config config.yaml -``` +For converter usage and interactive model/hardware discovery, see the +[Recipes tool documentation](../../../tools/recipes/README.md). ## Recommended Models diff --git a/docs/training/prompt_token_id_logprobs.md b/docs/training/prompt_token_id_logprobs.md new file mode 100644 index 0000000000..940facad4f --- /dev/null +++ b/docs/training/prompt_token_id_logprobs.md @@ -0,0 +1,64 @@ +# Prompt Token ID Logprobs (Teacher Scoring) + +In on-policy distillation, the student records its top-K candidate tokens at +every response position, and the teacher provides the log probabilities of +exactly those candidates as the distillation target. The caller passes the +candidates as `prompt_logprob_token_ids`; the teacher runs a single prefill +over the prompt plus response and gathers their log probabilities on the GPU at +every scored row, starting from `prompt_logprob_start` (typically the prompt +length minus one, so only response positions are scored). The result is a +`[rows, K]` matrix whose row `i` scores prompt token +`prompt_logprob_start + i + 1` and whose columns follow the candidate order, so +it lines up position by position with the student's top-K. + +## Quick start + +```python +from vllm import LLM, SamplingParams +from vllm.inputs import TokensPrompt + +llm = LLM(model) +output = llm.generate( + TokensPrompt(prompt_token_ids=prompt_ids + response_ids), + SamplingParams( + max_tokens=1, + prompt_logprob_token_ids=candidate_ids, + prompt_logprob_start=len(prompt_ids) - 1, + ), +) +scores = output[0].prompt_token_id_logprobs +# scores.shape == (len(response_ids), len(candidate_ids)) +# scores[i, j] = log p(candidate_ids[j] | prompt_ids + response_ids[:i]) +``` + +The same parameters are accepted by the `/inference/v1/generate` HTTP endpoint +(Python and Rust frontends), which returns the matrix as a base64-encoded +`.npy` float32 array: + +```python +import base64, io +import numpy as np + +scores = np.load(io.BytesIO(base64.b64decode(response["prompt_token_id_logprobs"]))) +``` + +The matrix has `max(prompt_len - 1 - prompt_logprob_start, 0)` rows. With a +logits `--logprobs-mode`, it holds logits instead of log probabilities. + +## Requirements + +- The V2 model runner (`VLLM_USE_V2_MODEL_RUNNER=1`). +- `len(prompt_logprob_token_ids)` at most `--max-logprobs`. +- No `--kv-sharing-fast-prefill`. + +The request skips reading the prefix cache (local and KV connector), since +cached rows have no logits to score. + +## Limitations + +- One candidate set per request, shared by every scored row; per-position + candidates are planned ([#56860](https://github.com/vllm-project/vllm/issues/56860)). +- Non-streaming only; `stream=true` requests are rejected. +- Not exposed through the OpenAI-compatible endpoints. +- If a prefill starts past the first scored row (e.g. `skip_reading_prefix_cache=False` + with a cache hit), the result is `None` rather than a partial matrix. diff --git a/requirements/cpu.txt b/requirements/cpu.txt index 06bddd4a67..b4867a0ff3 100644 --- a/requirements/cpu.txt +++ b/requirements/cpu.txt @@ -3,7 +3,7 @@ setuptools==77.0.3 # this version can reuse CMake build dir -numba == 0.65.0; platform_machine != "s390x" # Required for N-gram speculative decoding +numba == 0.65.0 # Required for N-gram speculative decoding # Dependencies for CPUs torch==2.13.0+cpu; platform_machine == "x86_64" or platform_machine == "s390x" or platform_machine == "aarch64" @@ -13,10 +13,10 @@ torch==2.13.0; platform_system == "Darwin" or platform_machine == "ppc64le" or p torchaudio; platform_machine != "s390x" and platform_machine != "riscv64" # required for the image processor of phi3v, this must be updated alongside torch -torchvision; platform_machine != "s390x" and platform_machine != "riscv64" +torchvision; platform_machine != "riscv64" # required for the torchcodec video decoding backend -torchcodec >= 0.14; platform_machine != "s390x" and platform_machine != "riscv64" and platform_machine != "ppc64le" +torchcodec >= 0.14; platform_machine != "riscv64" and platform_machine != "ppc64le" # Intel Extension for PyTorch, only for x86_64 CPUs intel-openmp==2024.2.1; platform_machine == "x86_64" diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 5b122ea14c..6a5d13f087 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -5968,6 +5968,7 @@ dependencies = [ "asynk-strim-attr", "auto_enums", "axum", + "base64 0.22.1", "bytes", "clap", "educe", diff --git a/rust/src/engine-core-client/src/protocol/output.rs b/rust/src/engine-core-client/src/protocol/output.rs index 0edd0ce788..c33062eeb3 100644 --- a/rust/src/engine-core-client/src/protocol/output.rs +++ b/rust/src/engine-core-client/src/protocol/output.rs @@ -17,6 +17,7 @@ use crate::error::{Error, Result, ext_value_decode}; use crate::protocol::logprobs::MaybeWireLogprobs; use crate::protocol::sampling_mask::MaybeWireSamplingMask; use crate::protocol::stats::{PrefillStats, SchedulerStats}; +use crate::protocol::tensor::WireNdArray; use crate::protocol::{OpaqueValue, decode_msgpack}; /// The stop reason associated with a finished output. @@ -133,6 +134,10 @@ pub struct EngineCoreOutput { /// output when `--per-request-spec-decode-metrics` is enabled. #[serde(default)] pub spec_decode_metrics: Option, + /// Log probabilities of `SamplingParams.prompt_logprob_token_ids`, set on + /// the first output of a request that asked for them. + #[serde(default)] + pub prompt_token_id_logprobs: Option, } /// Raw per-sequence speculative-decoding accumulator. @@ -180,6 +185,11 @@ impl EngineCoreOutput { self.new_sampling_mask = (self.new_sampling_mask.take()) .map(|value| value.resolve(frames, "new_sampling_mask")) .transpose()?; + if let Some(value) = self.prompt_token_id_logprobs.as_mut() { + value + .resolve_aux_frame(frames) + .map_err(|message| ext_value_decode!("prompt_token_id_logprobs: {message}"))?; + } Ok(()) } } @@ -531,6 +541,7 @@ mod tests { mm_cache_miss_hashes: None, new_sampling_mask: None, spec_decode_metrics: None, + prompt_token_id_logprobs: None, }, ], scheduler_stats: None, diff --git a/rust/src/engine-core-client/src/protocol/sampling.rs b/rust/src/engine-core-client/src/protocol/sampling.rs index 92f354be6f..0cd11b5397 100644 --- a/rust/src/engine-core-client/src/protocol/sampling.rs +++ b/rust/src/engine-core-client/src/protocol/sampling.rs @@ -101,6 +101,11 @@ pub struct EngineCoreSamplingParams { /// /// `None` disables prompt logprobs. `-1` requests the full vocabulary. pub prompt_logprobs: Option, + /// Candidate token IDs scored at every scored causal prompt row, where row + /// `i` scores them as predictions of prompt token `i + 1`. + pub prompt_logprob_token_ids: Option>, + /// First causal prompt row to score; `None` scores from the first row. + pub prompt_logprob_start: Option, /// Minimum probability threshold for token sampling. pub min_p: f32, /// Frequency penalty applied by the sampler. @@ -171,6 +176,8 @@ impl EngineCoreSamplingParams { thinking_token_budget: None, logprobs: None, prompt_logprobs: None, + prompt_logprob_token_ids: None, + prompt_logprob_start: None, min_p: 0.0, frequency_penalty: 0.0, presence_penalty: 0.0, diff --git a/rust/src/engine-core-client/src/protocol/stats.rs b/rust/src/engine-core-client/src/protocol/stats.rs index 021fd7c26a..aa1b9c69ab 100644 --- a/rust/src/engine-core-client/src/protocol/stats.rs +++ b/rust/src/engine-core-client/src/protocol/stats.rs @@ -278,9 +278,9 @@ pub use kv_connector::*; pub struct SchedulerStats { /// Number of requests in model execution batches. pub num_running_reqs: u64, - /// Length of the "waiting" request queue. + /// Number of waiting requests not deferred. pub num_waiting_reqs: u64, - /// Length of the "skipped waiting" queue. + /// Number of waiting requests deferred by transient constraints or a blocked status. #[serde(default)] pub num_skipped_waiting_reqs: u64, /// Internal DP load-balancing step counter. diff --git a/rust/src/engine-core-client/src/tests/client.rs b/rust/src/engine-core-client/src/tests/client.rs index bd3847c483..cdc3f718bd 100644 --- a/rust/src/engine-core-client/src/tests/client.rs +++ b/rust/src/engine-core-client/src/tests/client.rs @@ -2684,6 +2684,8 @@ fn python_msgpack_fixtures_match_rust_encoding() { thinking_token_budget: None, logprobs: None, prompt_logprobs: None, + prompt_logprob_token_ids: None, + prompt_logprob_start: None, min_p: 0.0, frequency_penalty: 0.0, presence_penalty: 0.0, @@ -2770,6 +2772,7 @@ fn python_msgpack_fixtures_match_rust_encoding() { mm_cache_miss_hashes: None, new_sampling_mask: None, spec_decode_metrics: None, + prompt_token_id_logprobs: None, }, ], scheduler_stats: None, diff --git a/rust/src/engine-core-client/src/tests/python_compat.py b/rust/src/engine-core-client/src/tests/python_compat.py index 76978328bb..bd8d40dc32 100755 --- a/rust/src/engine-core-client/src/tests/python_compat.py +++ b/rust/src/engine-core-client/src/tests/python_compat.py @@ -116,6 +116,7 @@ class EngineCoreOutput( mm_cache_miss_hashes: list[str] | None = None new_sampling_mask: object | None = None spec_decode_metrics: object | None = None + prompt_token_id_logprobs: object | None = None class ExtendedEngineCoreOutput(EngineCoreOutput): diff --git a/rust/src/llm/src/output.rs b/rust/src/llm/src/output.rs index 34eae25682..4b6d033539 100644 --- a/rust/src/llm/src/output.rs +++ b/rust/src/llm/src/output.rs @@ -15,6 +15,7 @@ use vllm_engine_core_client::protocol::output::{ EngineCoreFinishReason, RequestSpecDecodeMetrics, StopReason, }; use vllm_engine_core_client::protocol::sampling_mask::SamplingMask; +use vllm_engine_core_client::protocol::tensor::WireNdArray; use vllm_engine_core_client::{AbortCause, EngineCoreOutputStream}; use crate::error::Result; @@ -38,6 +39,7 @@ pub struct CollectedGenerateOutput { pub request_id: String, pub prompt_token_ids: Vec, pub prompt_logprobs: Option, + pub prompt_token_id_logprobs: Option, pub token_ids: Vec, pub logprobs: Option, pub finish_reason: FinishReason, @@ -62,6 +64,8 @@ pub struct GeneratePromptInfo { /// Prompt logprobs returned by engine-core for scored prompt positions, /// when requested. pub prompt_logprobs: Option, + /// Log probabilities of `prompt_logprob_token_ids`, when requested. + pub prompt_token_id_logprobs: Option, } /// The reason a request finished. @@ -206,6 +210,7 @@ impl GenerateOutput { prompt_info: prompt_token_ids.map(|ids| GeneratePromptInfo { prompt_token_ids: ids, prompt_logprobs: None, + prompt_token_id_logprobs: None, }), token_ids, logprobs: None, @@ -247,6 +252,7 @@ impl GenerateOutputStream { pending_prompt_info: Some(GeneratePromptInfo { prompt_token_ids, prompt_logprobs: None, + prompt_token_id_logprobs: None, }), raw_stream, request_metrics, @@ -282,6 +288,11 @@ impl Stream for GenerateOutputStream { info.prompt_logprobs = raw.new_prompt_logprobs_tensors.map(|value| value.into_direct().unwrap()); } + if let Some(info) = &mut self.pending_prompt_info + && info.prompt_token_id_logprobs.is_none() + { + info.prompt_token_id_logprobs = raw.prompt_token_id_logprobs; + } let logprobs = raw.new_logprobs.map(|value| value.into_direct().unwrap()); let sampling_mask = raw.new_sampling_mask.map(|value| value.into_direct().unwrap()); @@ -361,6 +372,7 @@ impl> + Send> T { pin_mut!(stream); let mut prompt_token_ids = None; let mut prompt_logprobs = None; + let mut prompt_token_id_logprobs = None; let mut cached_token_count = 0; let mut collected: Option = None; @@ -374,6 +386,9 @@ impl> + Send> T { if prompt_logprobs.is_none() { prompt_logprobs = info.prompt_logprobs; } + if prompt_token_id_logprobs.is_none() { + prompt_token_id_logprobs = info.prompt_token_id_logprobs; + } } if let Some(existing) = collected.as_mut() { @@ -393,6 +408,7 @@ impl> + Send> T { request_id: output.request_id, prompt_token_ids: prompt_token_ids.take().unwrap_or_default(), prompt_logprobs: prompt_logprobs.take(), + prompt_token_id_logprobs: prompt_token_id_logprobs.take(), token_ids: output.token_ids, logprobs: output.logprobs, finish_reason: FinishReason::Error, diff --git a/rust/src/llm/tests/generate.rs b/rust/src/llm/tests/generate.rs index 370d5c046f..efd48eba77 100644 --- a/rust/src/llm/tests/generate.rs +++ b/rust/src/llm/tests/generate.rs @@ -277,6 +277,7 @@ async fn generate_streams_outputs() { Some(GeneratePromptInfo { prompt_token_ids: vec![11, 22].into(), prompt_logprobs: Some(prompt_logprobs()), + prompt_token_id_logprobs: None, }) ); assert_eq!(first.token_ids, vec![1, 2]); @@ -405,6 +406,7 @@ async fn collect_output_rejects_partial_sampling_mask() { prompt_info: Some(GeneratePromptInfo { prompt_token_ids: Arc::from([11_u32, 22]), prompt_logprobs: None, + prompt_token_id_logprobs: None, }), token_ids: vec![33, 44], logprobs: None, diff --git a/rust/src/parser/src/tool/parameters.rs b/rust/src/parser/src/tool/parameters.rs index 2ae26cd894..71601ca8af 100644 --- a/rust/src/parser/src/tool/parameters.rs +++ b/rust/src/parser/src/tool/parameters.rs @@ -150,7 +150,33 @@ impl ToolSchema { impl JsonParamType { /// Normalize one parameter property schema. + /// + /// `"nullable": true` (the OpenAPI spelling of an optional parameter) adds + /// `null` to the accepted types. fn from_schema(schema: &Value) -> Option { + let param_type = Self::from_schema_ignoring_nullable(schema)?; + let nullable = schema.get("nullable").and_then(Value::as_bool) == Some(true); + Some(match param_type { + _ if !nullable || param_type.accepts_null() => param_type, + Self::OneOf(mut types) => { + types.push(Self::Null); + Self::OneOf(types) + } + param_type => Self::OneOf(vec![param_type, Self::Null]), + }) + } + + /// Return whether `null` is one of the accepted types. + fn accepts_null(&self) -> bool { + match self { + Self::Null => true, + Self::OneOf(types) => types.iter().any(Self::accepts_null), + _ => false, + } + } + + /// Normalize one parameter property schema, without `nullable`. + fn from_schema_ignoring_nullable(schema: &Value) -> Option { let schema = schema.as_object()?; if let Some(type_value) = schema.get("type") { @@ -284,8 +310,10 @@ impl JsonParamType { } } -/// Recognize JSON and Python null spellings emitted by chat templates. +/// Recognize JSON and Python null spellings emitted by chat templates, +/// ignoring surrounding whitespace. fn is_null_literal(value: &str) -> bool { + let value = value.trim(); value.eq_ignore_ascii_case("null") || value.eq_ignore_ascii_case("none") } @@ -327,18 +355,24 @@ fn try_convert_value(param_type: &JsonParamType, input: &ParamInput) -> Option Option { match param_type { JsonParamType::String => Some(Value::String(value.to_string())), - JsonParamType::Integer => value.parse::().ok().map(Number::from).map(Value::Number), - JsonParamType::Number => try_convert_number(value), + JsonParamType::Integer => without_digit_separators(value.trim())? + .parse::() + .ok() + .map(Number::from) + .map(Value::Number), + JsonParamType::Number => try_convert_number(&without_digit_separators(value.trim())?), JsonParamType::Boolean => try_convert_boolean(value), - JsonParamType::Object { .. } if value.is_empty() => Some(Value::Object(Map::new())), - JsonParamType::Array { .. } if value.is_empty() => Some(Value::Array(Vec::new())), - JsonParamType::Object { .. } | JsonParamType::Array { .. } => { - // For composite types with string input, simply interpret the string as JSON. - serde_json::from_str(value).ok() - } + JsonParamType::Object { .. } if value.trim().is_empty() => Some(Value::Object(Map::new())), + JsonParamType::Array { .. } if value.trim().is_empty() => Some(Value::Array(Vec::new())), + // For composite types with string input, interpret the string as JSON of that type. + JsonParamType::Object { .. } => serde_json::from_str(value).ok().filter(Value::is_object), + JsonParamType::Array { .. } => serde_json::from_str(value).ok().filter(Value::is_array), JsonParamType::Null => is_null_literal(value).then_some(Value::Null), JsonParamType::OneOf(types) => { types.iter().find_map(|param_type| try_convert_text_value(param_type, value)) @@ -411,6 +445,22 @@ fn insert_object_value(object: &mut Map, key: String, value: Valu } } +/// Remove single underscores between digits (`1_000`), as Python numeric +/// literals allow; any other underscore makes the value invalid. +fn without_digit_separators(value: &str) -> Option> { + if !value.contains('_') { + return Some(value.into()); + } + let bytes = value.as_bytes(); + let valid = bytes.iter().enumerate().all(|(index, byte)| { + *byte != b'_' + || (index > 0 + && bytes[index - 1].is_ascii_digit() + && bytes.get(index + 1).is_some_and(u8::is_ascii_digit)) + }); + valid.then(|| value.replace('_', "").into()) +} + /// Convert one raw string value to a JSON number. fn try_convert_number(value: &str) -> Option { serde_json::from_str::(value) @@ -555,6 +605,83 @@ mod tests { ParamInput::Elements(elements) } + #[test] + fn non_string_values_ignore_surrounding_whitespace() { + let params = ToolSchema::from_schema(&json!({ + "type": "object", + "properties": { + "text": { "type": "string" }, + "count": { "type": "integer" }, + "size": { "type": "number" }, + "enabled": { "type": "boolean" }, + "payload": { "type": "object" }, + "items": { "type": "array" } + } + })); + + assert_eq!(params.convert("text", text(" 7 ")), json!(" 7 ")); + assert_eq!(params.convert("count", text(" 7\n")), json!(7)); + assert_eq!(params.convert("size", text("\t2.5 ")), json!(2.5)); + assert_eq!(params.convert("enabled", text(" false ")), json!(false)); + assert_eq!(params.convert("count", text(" None ")), json!(null)); + assert_eq!(params.convert("payload", text(" \n")), json!({})); + assert_eq!(params.convert("items", text(" [1] ")), json!([1])); + } + + #[test] + fn numbers_accept_digit_separators() { + let params = ToolSchema::from_schema(&json!({ + "type": "object", + "properties": { + "count": { "type": "integer" }, + "size": { "type": "number" } + } + })); + + assert_eq!(params.convert("count", text("1_000")), json!(1000)); + assert_eq!(params.convert("size", text("1_000.5")), json!(1000.5)); + for invalid in ["_1", "1_", "1__0", "1_.5"] { + assert_eq!(params.convert("size", text(invalid)), json!(invalid)); + } + } + + #[test] + fn nullable_schemas_accept_null() { + let params = ToolSchema::from_schema(&json!({ + "type": "object", + "properties": { + "text": { "type": "string", "nullable": true }, + "count": { "type": ["integer", "string"], "nullable": true } + } + })); + + // Without `nullable`, a string parameter keeps the literal text "null". + assert_eq!(params.convert("text", text("null")), json!(null)); + assert_eq!(params.convert("text", text("x")), json!("x")); + assert_eq!(params.convert("count", text("None")), json!(null)); + assert_eq!(params.convert("count", text("7")), json!(7)); + } + + #[test] + fn composite_values_must_decode_to_their_type() { + let params = ToolSchema::from_schema(&json!({ + "type": "object", + "properties": { + "payload": { "type": "object" }, + "items": { "type": "array" }, + "either": { "type": ["object", "array"] } + } + })); + + assert_eq!(params.convert("payload", text("[1]")), json!("[1]")); + assert_eq!(params.convert("payload", text("5")), json!("5")); + assert_eq!( + params.convert("items", text(r#"{"k":1}"#)), + json!(r#"{"k":1}"#) + ); + assert_eq!(params.convert("either", text("[1]")), json!([1])); + } + #[test] fn converts_upstream_aliases() { let params = ToolSchema::from_schema(&json!({ diff --git a/rust/src/server/Cargo.toml b/rust/src/server/Cargo.toml index 8990ba2f3b..8b22b6344a 100644 --- a/rust/src/server/Cargo.toml +++ b/rust/src/server/Cargo.toml @@ -9,6 +9,7 @@ anyhow.workspace = true asynk-strim-attr.workspace = true auto_enums.workspace = true axum.workspace = true +base64.workspace = true educe.workspace = true futures.workspace = true http-body.workspace = true diff --git a/rust/src/server/src/routes/inference/generate.rs b/rust/src/server/src/routes/inference/generate.rs index 79b68622cf..15caedaf17 100644 --- a/rust/src/server/src/routes/inference/generate.rs +++ b/rust/src/server/src/routes/inference/generate.rs @@ -16,12 +16,15 @@ use axum::extract::State; use axum::http::HeaderMap; use axum::response::sse::Event; use axum::response::{IntoResponse, Response}; +use base64::Engine as _; use futures::{Stream, StreamExt as _, pin_mut}; use thiserror_ext::AsReport as _; use tracing::{error, info, trace}; use tracing_futures::Instrument as _; +use vllm_engine_core_client::protocol::dtype::TensorDtype; use vllm_engine_core_client::protocol::logprobs::{Logprobs, PositionLogprobs}; use vllm_engine_core_client::protocol::multimodal::MmFeatureSpec; +use vllm_engine_core_client::protocol::tensor::WireNdArray; use vllm_llm::{ CollectedGenerateOutput, FinishReason, GenerateOutput, GenerateOutputStreamExt as _, TokenUsage, }; @@ -330,6 +333,11 @@ fn collect_generate( token_ids: collected.token_ids, }], prompt_logprobs, + prompt_token_id_logprobs: collected + .prompt_token_id_logprobs + .as_ref() + .map(npy_base64) + .transpose()?, prompt_token_ids: return_token_ids.then_some(collected.prompt_token_ids), mm_placeholders: return_token_ids.then_some(mm_placeholders).flatten(), kv_transfer_params: collected.kv_transfer_params, @@ -432,6 +440,29 @@ fn position_to_logprob_map(position: &PositionLogprobs) -> HashMap Result { + let (TensorDtype::F32, Some(data), [rows, cols]) = ( + array.dtype.scalar, + array.data.as_raw_view(), + array.shape.as_slice(), + ) else { + bail_server_error!("prompt_token_id_logprobs: expected a float32 matrix"); + }; + let mut header = + format!("{{'descr': ' String { format!("token_id:{token_id}") } @@ -509,6 +540,7 @@ mod tests { prompt_info: Some(GeneratePromptInfo { prompt_token_ids: Arc::from([11_u32, 22_u32]), prompt_logprobs: None, + prompt_token_id_logprobs: None, }), token_ids: vec![33], logprobs: None, @@ -592,6 +624,7 @@ mod tests { prompt_info: prompt_token_ids.map(|ids| GeneratePromptInfo { prompt_token_ids: Arc::from(ids), prompt_logprobs: None, + prompt_token_id_logprobs: None, }), token_ids, logprobs: None, @@ -747,6 +780,7 @@ mod tests { let output = CollectedGenerateOutput { request_id: "raw-1".to_string(), prompt_logprobs: None, + prompt_token_id_logprobs: None, token_ids: vec![30], logprobs: None, finish_reason: FinishReason::stop_eos(), @@ -781,6 +815,17 @@ mod tests { assert!(response.mm_placeholders.is_none()); } + #[test] + fn prompt_token_id_logprobs_match_numpy2base64() { + let scores = + WireNdArray::from_f32(vec![2, 3], vec![-0.5, -1.5, -2.5, -3.5, -4.5, -5.5]).unwrap(); + // numpy2base64(np.array([[-0.5, -1.5, -2.5], [-3.5, -4.5, -5.5]], np.float32)) + assert_eq!( + npy_base64(&scores).unwrap(), + "k05VTVBZAQB2AHsnZGVzY3InOiAnPGY0JywgJ2ZvcnRyYW5fb3JkZXInOiBGYWxzZSwgJ3NoYXBlJzogKDIsIDMpLCB9ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAoAAAC/AADAvwAAIMAAAGDAAACQwAAAsMA=" + ); + } + #[test] fn collect_generate_returns_spec_decode_metrics() { let metrics = RequestSpecDecodeMetrics { @@ -792,6 +837,7 @@ mod tests { let output = CollectedGenerateOutput { request_id: "raw-metrics".to_string(), prompt_logprobs: None, + prompt_token_id_logprobs: None, token_ids: vec![30], logprobs: None, finish_reason: FinishReason::stop_eos(), @@ -879,6 +925,7 @@ mod tests { let output = CollectedGenerateOutput { request_id: "raw-1".to_string(), prompt_logprobs: None, + prompt_token_id_logprobs: None, token_ids: vec![30], logprobs: None, finish_reason: FinishReason::stop_eos(), @@ -914,6 +961,7 @@ mod tests { let output_without_payload = |prompt_token_ids: Vec| CollectedGenerateOutput { request_id: "raw-1".to_string(), prompt_logprobs: None, + prompt_token_id_logprobs: None, token_ids: vec![3], logprobs: None, finish_reason: FinishReason::stop_eos(), diff --git a/rust/src/server/src/routes/inference/generate/types.rs b/rust/src/server/src/routes/inference/generate/types.rs index 86d8ec00f4..78fe085b83 100644 --- a/rust/src/server/src/routes/inference/generate/types.rs +++ b/rust/src/server/src/routes/inference/generate/types.rs @@ -94,6 +94,8 @@ pub(super) struct GenerateResponse { pub request_id: String, pub choices: Vec, pub prompt_logprobs: Option>>>, + /// Base64 `.npy` float32 array, as Python `numpy2base64` encodes it. + pub prompt_token_id_logprobs: Option, pub prompt_token_ids: Option>, pub mm_placeholders: Option, pub kv_transfer_params: Option, diff --git a/rust/src/server/src/routes/inference/generate/validate.rs b/rust/src/server/src/routes/inference/generate/validate.rs index 9ff8d0460e..9b1779d6e9 100644 --- a/rust/src/server/src/routes/inference/generate/validate.rs +++ b/rust/src/server/src/routes/inference/generate/validate.rs @@ -57,6 +57,13 @@ pub(crate) fn validate_request_compat( } } + if request.stream && request.sampling_params.inner.prompt_logprob_token_ids.is_some() { + bail_invalid_request!( + param = "sampling_params", + "`prompt_logprob_token_ids` are not available when `stream=true`." + ); + } + Ok(()) } @@ -169,6 +176,17 @@ mod tests { assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err()); } + #[test] + fn validate_request_compat_rejects_streaming_prompt_logprob_token_ids() { + let request: GenerateRequest = serde_json::from_value(json!({ + "token_ids": [11, 22], + "stream": true, + "sampling_params": {"prompt_logprob_token_ids": [1, 2]} + })) + .expect("parse request"); + assert!(validate_request_compat(&request, &served(&["test-model"])).is_err()); + } + #[test] fn validate_request_compat_accepts_non_stream_prompt_logprobs() { let request: GenerateRequest = serde_json::from_value(json!({ diff --git a/rust/src/server/src/routes/openai/chat_completions/convert.rs b/rust/src/server/src/routes/openai/chat_completions/convert.rs index d4df614270..418ab23ce8 100644 --- a/rust/src/server/src/routes/openai/chat_completions/convert.rs +++ b/rust/src/server/src/routes/openai/chat_completions/convert.rs @@ -159,6 +159,8 @@ pub(super) fn prepare_chat_request( thinking_token_budget: request.thinking_token_budget, logprobs: request.logprobs.then_some(top_logprobs), prompt_logprobs, + prompt_logprob_token_ids: None, + prompt_logprob_start: None, min_p: request.min_p, frequency_penalty: request.frequency_penalty, presence_penalty: request.presence_penalty, diff --git a/rust/src/server/src/routes/openai/completions/convert.rs b/rust/src/server/src/routes/openai/completions/convert.rs index 53a6ff9906..d05cfffc89 100644 --- a/rust/src/server/src/routes/openai/completions/convert.rs +++ b/rust/src/server/src/routes/openai/completions/convert.rs @@ -124,6 +124,8 @@ pub(super) fn prepare_completion_request( thinking_token_budget: request.thinking_token_budget, logprobs, prompt_logprobs, + prompt_logprob_token_ids: None, + prompt_logprob_start: None, min_p: request.min_p, frequency_penalty: request.frequency_penalty, presence_penalty: request.presence_penalty, diff --git a/rust/src/text/src/lower.rs b/rust/src/text/src/lower.rs index e1357ed7f9..01a4fb25b8 100644 --- a/rust/src/text/src/lower.rs +++ b/rust/src/text/src/lower.rs @@ -103,6 +103,8 @@ pub fn lower_sampling_params( thinking_token_budget, logprobs, prompt_logprobs, + prompt_logprob_token_ids, + prompt_logprob_start, min_p, frequency_penalty, presence_penalty, @@ -123,6 +125,8 @@ pub fn lower_sampling_params( logprobs, prompt_logprobs, logprob_token_ids.as_deref(), + prompt_logprob_token_ids.as_deref(), + prompt_logprob_start, sampling_limits, )?; validate_repetition_detection(repetition_detection.as_ref())?; @@ -176,6 +180,8 @@ pub fn lower_sampling_params( thinking_token_budget, logprobs, prompt_logprobs, + prompt_logprob_token_ids, + prompt_logprob_start, min_p, frequency_penalty, presence_penalty, @@ -646,6 +652,8 @@ mod tests { thinking_token_budget: None, logprobs: None, prompt_logprobs: None, + prompt_logprob_token_ids: None, + prompt_logprob_start: None, min_p: 0.0, frequency_penalty: 0.0, presence_penalty: 0.0, @@ -701,6 +709,8 @@ mod tests { thinking_token_budget: None, logprobs: None, prompt_logprobs: None, + prompt_logprob_token_ids: None, + prompt_logprob_start: None, min_p: 0.0, frequency_penalty: 0.0, presence_penalty: 0.0, @@ -870,6 +880,8 @@ mod tests { thinking_token_budget: None, logprobs: None, prompt_logprobs: None, + prompt_logprob_token_ids: None, + prompt_logprob_start: None, min_p: 0.0, frequency_penalty: 0.0, presence_penalty: 0.0, @@ -935,6 +947,8 @@ mod tests { thinking_token_budget: None, logprobs: None, prompt_logprobs: None, + prompt_logprob_token_ids: None, + prompt_logprob_start: None, min_p: 0.0, frequency_penalty: 0.0, presence_penalty: 0.0, @@ -1008,6 +1022,8 @@ mod tests { thinking_token_budget: None, logprobs: None, prompt_logprobs: None, + prompt_logprob_token_ids: None, + prompt_logprob_start: None, min_p: 0.1, frequency_penalty: 0.0, presence_penalty: 0.0, @@ -1122,6 +1138,44 @@ mod tests { )); } + #[test] + fn lower_sampling_params_validates_prompt_logprob_token_ids() { + let lower = |ids: Option>, start: Option| { + lower_sampling_params_with_limits( + SamplingParams { + prompt_logprob_token_ids: ids, + prompt_logprob_start: start, + ..Default::default() + }, + sample_sampling_limits(), + ) + }; + let rejects = |ids, start| match lower(ids, start) { + Err(error) => error.to_string(), + Ok(_) => panic!("expected rejection"), + }; + + let params = lower(Some(vec![1, 2]), Some(1)).unwrap(); + assert_eq!(params.prompt_logprob_token_ids, Some(vec![1, 2])); + assert_eq!(params.prompt_logprob_start, Some(1)); + expect_test::expect![[r#" + [ + "prompt_logprob_token_ids must not be empty.", + "requested prompt_logprob_token_ids of 21, which is greater than max allowed: 20", + "token_id(s) [1000] in prompt_logprob_token_ids are out of vocabulary. Vocabulary size: 1000", + "prompt_logprob_token_ids must not contain duplicates.", + "prompt_logprob_start requires prompt_logprob_token_ids.", + ] + "#]] + .assert_debug_eq(&[ + rejects(Some(vec![]), None), + rejects(Some((0..21).collect()), None), + rejects(Some(vec![1000]), None), + rejects(Some(vec![1, 1]), None), + rejects(None, Some(0)), + ]); + } + #[test] fn lower_sampling_params_rejects_out_of_vocab_stop_token_ids() { let error = lower_sampling_params_with_limits( @@ -1259,6 +1313,8 @@ mod tests { thinking_token_budget: None, logprobs: None, prompt_logprobs: None, + prompt_logprob_token_ids: None, + prompt_logprob_start: None, min_p: 0.1, frequency_penalty: 0.0, presence_penalty: 0.0, diff --git a/rust/src/text/src/lower/logprobs.rs b/rust/src/text/src/lower/logprobs.rs index b2407f06ed..58ad5d694b 100644 --- a/rust/src/text/src/lower/logprobs.rs +++ b/rust/src/text/src/lower/logprobs.rs @@ -6,6 +6,7 @@ //! `-1` is expanded only for bounds checks. The original request values are //! passed through to engine-core. +use itertools::Itertools as _; use thiserror::Error; use crate::backend::SamplingLimits; @@ -35,6 +36,12 @@ pub enum LogprobsError { len(logprob_token_ids). Got logprobs={logprobs}, len(logprob_token_ids)={num_token_ids}." )] TokenIdsMismatch { logprobs: i32, num_token_ids: usize }, + #[error("prompt_logprob_token_ids must not be empty.")] + EmptyPromptLogprobTokenIds, + #[error("prompt_logprob_token_ids must not contain duplicates.")] + DuplicatePromptLogprobTokenIds, + #[error("prompt_logprob_start requires prompt_logprob_token_ids.")] + PromptLogprobStartWithoutTokenIds, } /// Validate logprobs count sampling parameters. @@ -42,6 +49,8 @@ pub(super) fn validate_logprobs( logprobs: Option, prompt_logprobs: Option, logprob_token_ids: Option<&[u32]>, + prompt_logprob_token_ids: Option<&[u32]>, + prompt_logprob_start: Option, sampling_limits: SamplingLimits, ) -> Result<(), LogprobsError> { let vocab_size = sampling_limits.model_vocab_size; @@ -50,9 +59,28 @@ pub(super) fn validate_logprobs( validate_logprobs_count(logprobs, max_logprobs, vocab_size, "logprobs")?; validate_logprobs_count(prompt_logprobs, max_logprobs, vocab_size, "prompt_logprobs")?; + validate_logprobs_count( + prompt_logprob_token_ids.map(|ids| ids.len() as i32), + max_logprobs, + vocab_size, + "prompt_logprob_token_ids", + )?; + validate_prompt_logprob_token_ids(prompt_logprob_token_ids, prompt_logprob_start)?; validate_logprob_token_ids(logprobs, logprob_token_ids) } +fn validate_prompt_logprob_token_ids( + token_ids: Option<&[u32]>, + start: Option, +) -> Result<(), LogprobsError> { + match token_ids { + Some([]) => Err(LogprobsError::EmptyPromptLogprobTokenIds), + Some(ids) if !ids.iter().all_unique() => Err(LogprobsError::DuplicatePromptLogprobTokenIds), + None if start.is_some() => Err(LogprobsError::PromptLogprobStartWithoutTokenIds), + _ => Ok(()), + } +} + fn validate_logprobs_count( requested: Option, max_logprobs: usize, diff --git a/rust/src/text/src/lower/token_ids.rs b/rust/src/text/src/lower/token_ids.rs index 8512a830e6..240b0528e1 100644 --- a/rust/src/text/src/lower/token_ids.rs +++ b/rust/src/text/src/lower/token_ids.rs @@ -95,6 +95,14 @@ pub(crate) fn validate_vocab_range( )?; } + if let Some(token_ids) = params.prompt_logprob_token_ids.as_deref() { + validate_param( + "prompt_logprob_token_ids", + token_ids.iter().copied(), + limits.model_vocab_size, + )?; + } + if let Some(bad_words_token_ids) = params.bad_words_token_ids.as_deref() { validate_param( "bad_words", diff --git a/rust/src/text/src/request.rs b/rust/src/text/src/request.rs index 2c768f4855..9c9f22a38e 100644 --- a/rust/src/text/src/request.rs +++ b/rust/src/text/src/request.rs @@ -97,6 +97,11 @@ pub struct SamplingParams { /// /// `None` disables prompt logprobs. `-1` requests the full vocabulary. pub prompt_logprobs: Option, + /// Candidate token IDs scored at every scored causal prompt row, where row + /// `i` scores them as predictions of prompt token `i + 1`. + pub prompt_logprob_token_ids: Option>, + /// First causal prompt row to score; `None` scores from the first row. + pub prompt_logprob_start: Option, /// Minimum probability threshold for token sampling. `None` means no /// explicit user override. pub min_p: Option, @@ -156,6 +161,8 @@ impl Default for SamplingParams { thinking_token_budget: None, logprobs: None, prompt_logprobs: None, + prompt_logprob_token_ids: None, + prompt_logprob_start: None, min_p: None, frequency_penalty: None, presence_penalty: None, diff --git a/tests/compile/dynamic_shapes/__init__.py b/tests/compile/dynamic_shapes/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/compile/test_dynamic_shapes_compilation.py b/tests/compile/dynamic_shapes/test_dynamic_shapes_compilation.py similarity index 100% rename from tests/compile/test_dynamic_shapes_compilation.py rename to tests/compile/dynamic_shapes/test_dynamic_shapes_compilation.py diff --git a/tests/compile/fullgraph/test_basic_correctness.py b/tests/compile/fullgraph/test_basic_correctness.py index 859c4844d6..304e6d7cae 100644 --- a/tests/compile/fullgraph/test_basic_correctness.py +++ b/tests/compile/fullgraph/test_basic_correctness.py @@ -56,8 +56,10 @@ class TestSetting: ), ], ) +@pytest.mark.parametrize("use_v2_model_runner", [False, True]) def test_compile_correctness( test_setting: TestSetting, + use_v2_model_runner: bool, ): model = test_setting.model model_args = test_setting.model_args @@ -75,6 +77,10 @@ def test_compile_correctness( f"--attention-backend={attn_backend}", ] + # Pin every compared setting to the same model runner so that only the + # compilation mode differs within a comparison. + runner_env = {"VLLM_USE_V2_MODEL_RUNNER": str(int(use_v2_model_runner))} + all_args: list[list[str]] = [] all_envs: list[dict[str, str] | None] = [] @@ -86,7 +92,7 @@ def test_compile_correctness( CompilationMode.VLLM_COMPILE, ]: all_args.append(final_args + [f"-cc.mode={mode.name}", "-cc.backend=inductor"]) - all_envs.append({}) + all_envs.append(dict(runner_env)) # inductor will change the output, so we only compare if the output # is close, not exactly the same. compare_all_settings( @@ -94,7 +100,6 @@ def test_compile_correctness( all_args, all_envs, method=method if method != "generate" else "generate_close", - force_v1_runner=True, ) all_envs.clear() @@ -108,5 +113,5 @@ def test_compile_correctness( CompilationMode.VLLM_COMPILE, ]: all_args.append(final_args + [f"-cc.mode={mode.name}", "-cc.backend=eager"]) - all_envs.append({}) - compare_all_settings(model, all_args, all_envs, method=method, force_v1_runner=True) + all_envs.append(dict(runner_env)) + compare_all_settings(model, all_args, all_envs, method=method) diff --git a/tests/compile/passes/test_split_coalescing.py b/tests/compile/passes/test_split_coalescing.py index ab7a0be1a2..f8c2891045 100644 --- a/tests/compile/passes/test_split_coalescing.py +++ b/tests/compile/passes/test_split_coalescing.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import operator + import pytest import torch @@ -29,6 +31,33 @@ class SplitCoalescingModel(torch.nn.Module): return q + 1, k + 2, v + 3 +class SliceCoalescingModel(torch.nn.Module): + """Model with a complete QKV partition represented as contiguous slices.""" + + def __init__(self, q_size: int, kv_size: int) -> None: + super().__init__() + self.q_size = q_size + self.kv_size = kv_size + + def forward(self, qkv: torch.Tensor): + q_end = self.q_size + k_end = q_end + self.kv_size + v_end = k_end + self.kv_size + q = torch.ops.aten.slice.Tensor(qkv, -1, 0, q_end) + k = torch.ops.aten.slice.Tensor(qkv, -1, q_end, k_end) + v = torch.ops.aten.slice.Tensor(qkv, -1, k_end, v_end) + return q + 1, k + 2, v + 3 + + +def make_vllm_config() -> VllmConfig: + return VllmConfig( + compilation_config=CompilationConfig( + mode=CompilationMode.VLLM_COMPILE, + pass_config=PassConfig(), + ) + ) + + @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) def test_split_coalescing(dtype): torch.set_default_device(DEVICE_TYPE) @@ -37,12 +66,7 @@ def test_split_coalescing(dtype): q_size, kv_size = 2048, 512 - vllm_config = VllmConfig( - compilation_config=CompilationConfig( - mode=CompilationMode.VLLM_COMPILE, - pass_config=PassConfig(), - ) - ) + vllm_config = make_vllm_config() with vllm.config.set_current_vllm_config(vllm_config): coalesce_pass = SplitCoalescingPass(vllm_config) backend = TestBackend(coalesce_pass) @@ -63,3 +87,187 @@ def test_split_coalescing(dtype): torch.testing.assert_close(eager, compiled, atol=ATOL, rtol=RTOL) assert backend.op_count(torch.ops.aten.split_with_sizes.default) == 1 + + +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +def test_slice_partition_canonicalization(dtype): + torch.set_default_device(DEVICE_TYPE) + torch.set_default_dtype(dtype) + torch.manual_seed(0) + + q_size, kv_size = 2048, 512 + vllm_config = make_vllm_config() + with vllm.config.set_current_vllm_config(vllm_config): + coalesce_pass = SplitCoalescingPass(vllm_config) + backend = TestBackend(coalesce_pass) + model = SliceCoalescingModel(q_size, kv_size) + + qkv = torch.randn(5, q_size + 2 * kv_size) + torch._dynamo.mark_dynamic(qkv, 0) + result_eager = model(qkv) + result_compiled = torch.compile(model, backend=backend, fullgraph=True)(qkv) + + for eager, compiled in zip(result_eager, result_compiled): + torch.testing.assert_close(eager, compiled, atol=2e-3, rtol=2e-3) + + assert backend.op_count(torch.ops.aten.slice.Tensor, before=True) == 3 + assert backend.op_count(torch.ops.aten.slice.Tensor) == 0 + assert backend.op_count(torch.ops.aten.split_with_sizes.default) == 1 + + split = next( + node + for node in backend.final_graph.nodes + if node.target == torch.ops.aten.split_with_sizes.default + ) + assert list(split.args[1]) == [q_size, kv_size, kv_size] + assert split.args[2] == -1 + + getitems = [ + node + for node in split.users + if node.op == "call_function" and node.target == operator.getitem + ] + assert sorted(node.args[1] for node in getitems) == [0, 1, 2] + assert isinstance(split.meta["val"], list) + for getitem in getitems: + index = getitem.args[1] + assert getitem.meta["val"] is split.meta["val"][index] + + +def make_slice_graph( + ranges: list[tuple[int, int, int]], +) -> torch.fx.Graph: + graph = torch.fx.Graph() + source = graph.placeholder("qkv") + source.meta["val"] = torch.empty((2, 8), device="meta") + + slices = [] + for start, end, step in ranges: + node = graph.call_function( + torch.ops.aten.slice.Tensor, + args=(source, -1, start, end, step), + ) + node.meta["val"] = torch.empty( + (2, len(range(start, min(end, 8), step))), device="meta" + ) + slices.append(node) + graph.output(tuple(slices)) + return graph + + +def test_slice_partition_accepts_safe_group(): + graph = make_slice_graph([(0, 4, 1), (4, 8, 1)]) + vllm_config = make_vllm_config() + with vllm.config.set_current_vllm_config(vllm_config): + SplitCoalescingPass(vllm_config)(graph) + + graph.lint() + assert sum(node.target == torch.ops.aten.slice.Tensor for node in graph.nodes) == 0 + assert ( + sum( + node.target == torch.ops.aten.split_with_sizes.default + for node in graph.nodes + ) + == 1 + ) + + +@pytest.mark.parametrize( + "ranges", + [ + [(0, 3, 1), (4, 8, 1)], # gap + [(0, 5, 1), (4, 8, 1)], # overlap + [(1, 4, 1), (4, 8, 1)], # missing prefix + [(0, 4, 1), (4, 7, 1)], # missing suffix + [(0, 8, 1)], # single full-range slice + [(0, 8, 2)], # non-unit step + ], +) +def test_slice_partition_rejects_unsafe_groups(ranges): + graph = make_slice_graph(ranges) + vllm_config = make_vllm_config() + with vllm.config.set_current_vllm_config(vllm_config): + SplitCoalescingPass(vllm_config)(graph) + + slice_count = sum( + node.target == torch.ops.aten.slice.Tensor for node in graph.nodes + ) + split_count = sum( + node.target == torch.ops.aten.split_with_sizes.default for node in graph.nodes + ) + assert slice_count == len(ranges) + assert split_count == 0 + + +def test_nested_slice_partitions_use_live_source(): + """Use the rewired source when a slice partition consumes another slice.""" + graph = torch.fx.Graph() + source = graph.placeholder("input") + source.meta["val"] = torch.empty((2, 8), device="meta") + + outer_left = graph.call_function( + torch.ops.aten.slice.Tensor, args=(source, -1, 0, 4) + ) + outer_left.meta["val"] = torch.empty((2, 4), device="meta") + outer_right = graph.call_function( + torch.ops.aten.slice.Tensor, args=(source, -1, 4, 8) + ) + outer_right.meta["val"] = torch.empty((2, 4), device="meta") + inner_left = graph.call_function( + torch.ops.aten.slice.Tensor, args=(outer_left, -1, 0, 2) + ) + inner_left.meta["val"] = torch.empty((2, 2), device="meta") + inner_right = graph.call_function( + torch.ops.aten.slice.Tensor, args=(outer_left, -1, 2, 4) + ) + inner_right.meta["val"] = torch.empty((2, 2), device="meta") + graph.output((inner_left, inner_right, outer_right)) + + vllm_config = make_vllm_config() + with vllm.config.set_current_vllm_config(vllm_config): + SplitCoalescingPass(vllm_config)(graph) + + graph.lint() + assert sum(node.target == torch.ops.aten.slice.Tensor for node in graph.nodes) == 0 + splits = [ + node + for node in graph.nodes + if node.target == torch.ops.aten.split_with_sizes.default + ] + assert len(splits) == 2 + + outer_split = next(node for node in splits if list(node.args[1]) == [4, 4]) + inner_split = next(node for node in splits if list(node.args[1]) == [2, 2]) + assert outer_split.args[0] is source + assert inner_split.args[0].target == operator.getitem + assert inner_split.args[0].args == (outer_split, 0) + + +def test_split_coalescing_preserves_different_dimensions(): + """Keep equal-sized splits on different dimensions as separate nodes.""" + graph = torch.fx.Graph() + source = graph.placeholder("input") + source.meta["val"] = torch.empty((8, 8), device="meta") + + outputs = [] + for dim in (0, 1): + split = graph.call_function( + torch.ops.aten.split_with_sizes.default, + args=(source, [4, 4], dim), + ) + for index in range(2): + outputs.append(graph.call_function(operator.getitem, args=(split, index))) + graph.output(tuple(outputs)) + + vllm_config = make_vllm_config() + with vllm.config.set_current_vllm_config(vllm_config): + SplitCoalescingPass(vllm_config)(graph) + + splits = [ + node + for node in graph.nodes + if node.target == torch.ops.aten.split_with_sizes.default + ] + assert len(splits) == 2 + assert all(split.args[0] is source for split in splits) + assert {split.args[2] for split in splits} == {0, 1} diff --git a/tests/compile/test_config.py b/tests/compile/test_config.py index 74ed0d34db..24cb816b5d 100644 --- a/tests/compile/test_config.py +++ b/tests/compile/test_config.py @@ -215,7 +215,9 @@ def test_use_cudagraphs( # forked needed to workaround https://github.com/vllm-project/vllm/issues/21073 @pytest.mark.forked -def test_stock_torch_compile(vllm_runner, monkeypatch): +@pytest.mark.parametrize("use_v2_model_runner", [False, True]) +def test_stock_torch_compile(vllm_runner, monkeypatch, use_v2_model_runner): + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", str(int(use_v2_model_runner))) # Disable multiprocessing so that the counter is in the same process monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0") @@ -226,9 +228,10 @@ def test_stock_torch_compile(vllm_runner, monkeypatch): "facebook/opt-125m", compilation_config={"mode": CompilationMode.STOCK_TORCH_COMPILE}, gpu_memory_utilization=0.4, - ) as _, + ) as runner, ): - pass + outputs = runner.generate_greedy(["Hello, my name is"], max_tokens=5) + assert outputs[0][0] # forked needed to workaround https://github.com/vllm-project/vllm/issues/21073 diff --git a/tests/compile/test_wrapper.py b/tests/compile/test_wrapper.py index 5e0755ff71..45dc148cda 100644 --- a/tests/compile/test_wrapper.py +++ b/tests/compile/test_wrapper.py @@ -7,7 +7,11 @@ import os import pytest import torch -from vllm.compilation.wrapper import TorchCompileWithNoGuardsWrapper +from vllm.compilation.counter import compilation_counter +from vllm.compilation.wrapper import ( + TorchCompileWithNoGuardsWrapper, + compile_model_with_stock_torch, +) from vllm.config import ( CompilationConfig, CompilationMode, @@ -117,6 +121,44 @@ def test_torch_compile_wrapper(use_bytecode_hook, monkeypatch): raise AssertionError("expected an exception to be raised") +def test_compile_model_with_stock_torch(monkeypatch): + """The whole model is compiled in place with the configured backend.""" + graphs: list[torch.fx.GraphModule] = [] + + def recording_backend(gm: torch.fx.GraphModule, example_inputs): + graphs.append(gm) + return gm.forward + + monkeypatch.setattr( + CompilationConfig, + "init_backend", + lambda self, vllm_config, *args, **kwargs: recording_backend, + ) + + class Model(torch.nn.Module): + def __init__(self): + super().__init__() + self.linear = torch.nn.Linear(4, 2) + + def forward(self, x: torch.Tensor): + return self.linear(x).relu() + + vllm_config = VllmConfig() + vllm_config.compilation_config = CompilationConfig( + mode=CompilationMode.STOCK_TORCH_COMPILE + ) + model = Model() + x = torch.randn(3, 4) + expected = model(x) + + torch._dynamo.reset() + with compilation_counter.expect(stock_torch_compile_count=1): + compile_model_with_stock_torch(model, vllm_config) + assert not graphs # compilation is deferred to the first call + torch.testing.assert_close(model(x), expected) + assert len(graphs) == 1 + + if __name__ == "__main__": # Run with both parameter values diff --git a/tests/config/test_multimodal_config.py b/tests/config/test_multimodal_config.py index b36f5776da..bd9343bd15 100644 --- a/tests/config/test_multimodal_config.py +++ b/tests/config/test_multimodal_config.py @@ -188,6 +188,250 @@ def test_mm_processor_device_type_normalizes(device: object, expected: str | Non assert config.get_mm_processor_device_type() == expected +def test_merge_mm_processor_kwargs_scoped_precedence(): + configured = { + "size": { + "shortest_edge": 64, + "longest_edge": 512, + }, + "do_resize": True, + "padding": False, + "images_kwargs": { + "size": {"longest_edge": 1024}, + "do_resize": False, + }, + "videos_kwargs": { + "size": {"longest_edge": 2048}, + }, + } + inference = { + "size": {"shortest_edge": 128}, + "do_resize": True, + "images_kwargs": { + "size": {"shortest_edge": 256}, + }, + } + + config = MultiModalConfig( + mm_processor_kwargs=configured, + mm_device_do_normalize=False, + ) + + assert config.merge_mm_processor_kwargs(inference) == { + "size": { + "shortest_edge": 128, + "longest_edge": 512, + }, + "do_resize": True, + "padding": False, + "images_kwargs": { + "size": { + "shortest_edge": 256, + "longest_edge": 1024, + }, + "do_resize": True, + }, + "videos_kwargs": { + "size": { + "shortest_edge": 128, + "longest_edge": 2048, + }, + }, + } + + +def test_merge_mm_processor_kwargs_mapping_vs_replacement(): + config = MultiModalConfig( + mm_processor_kwargs={ + "size": {"shortest_edge": 64}, + "images_kwargs": {"size": {"longest_edge": 1024}}, + }, + mm_device_do_normalize=False, + ) + + assert config.merge_mm_processor_kwargs({"size": 7}) == { + "size": 7, + "images_kwargs": {"size": 7}, + } + assert config.merge_mm_processor_kwargs({"size": {}}) == { + "size": {"shortest_edge": 64}, + "images_kwargs": { + "size": { + "shortest_edge": 64, + "longest_edge": 1024, + }, + }, + } + + +_EMPTY_IMAGE_SCOPE_KWARGS: tuple[dict[str, object], ...] = ( + {}, + {"images_kwargs": None}, + {"images_kwargs": {}}, +) + + +@pytest.mark.parametrize("configured", _EMPTY_IMAGE_SCOPE_KWARGS) +@pytest.mark.parametrize("inference", _EMPTY_IMAGE_SCOPE_KWARGS) +def test_merge_mm_processor_kwargs_empty_scopes_are_absent( + configured: dict[str, object], + inference: dict[str, object], +): + config = MultiModalConfig( + mm_processor_kwargs=configured, + mm_device_do_normalize=False, + ) + + assert config.merge_mm_processor_kwargs(inference) == {} + + +@pytest.mark.parametrize("empty_inference", _EMPTY_IMAGE_SCOPE_KWARGS) +def test_merge_mm_processor_kwargs_empty_inference_scope_preserves_config_merge( + empty_inference: dict[str, object], +): + config = MultiModalConfig( + mm_processor_kwargs={ + "images_kwargs": { + "size": { + "shortest_edge": 64, + "longest_edge": 512, + }, + }, + }, + mm_device_do_normalize=False, + ) + inference = { + "size": {"shortest_edge": 128}, + **empty_inference, + } + + assert config.merge_mm_processor_kwargs(inference) == { + "size": {"shortest_edge": 128}, + "images_kwargs": { + "size": { + "shortest_edge": 128, + "longest_edge": 512, + }, + }, + } + + +@pytest.mark.parametrize("empty_config", _EMPTY_IMAGE_SCOPE_KWARGS) +def test_merge_mm_processor_kwargs_empty_config_scope_allows_inference_merge( + empty_config: dict[str, object], +): + config = MultiModalConfig( + mm_processor_kwargs={ + "size": {"longest_edge": 512}, + **empty_config, + }, + mm_device_do_normalize=False, + ) + + assert config.merge_mm_processor_kwargs( + { + "size": {"shortest_edge": 128}, + "images_kwargs": {"size": {"longest_edge": 1024}}, + } + ) == { + "size": { + "shortest_edge": 128, + "longest_edge": 512, + }, + "images_kwargs": { + "size": { + "shortest_edge": 128, + "longest_edge": 1024, + }, + }, + } + + +@pytest.mark.parametrize("non_mapping_scope", [False, 0, "", [], 123]) +def test_merge_mm_processor_kwargs_falsey_non_mapping_scopes_are_not_empty( + non_mapping_scope: object, +): + config = MultiModalConfig( + mm_processor_kwargs={"images_kwargs": non_mapping_scope}, + mm_device_do_normalize=False, + ) + + for empty_inference in _EMPTY_IMAGE_SCOPE_KWARGS: + assert config.merge_mm_processor_kwargs(empty_inference) == { + "images_kwargs": non_mapping_scope + } + + +@pytest.mark.parametrize( + ("configured_scope", "inference_scope", "expected_scope"), + [ + (123, {"size": {"shortest_edge": 128}}, {"size": {"shortest_edge": 128}}), + ({"size": {"shortest_edge": 64}}, 123, 123), + (123, 456, 456), + ], +) +def test_merge_mm_processor_kwargs_non_mapping_scope_precedence( + configured_scope: object, + inference_scope: object, + expected_scope: object, +): + config = MultiModalConfig( + mm_processor_kwargs={"images_kwargs": configured_scope}, + mm_device_do_normalize=False, + ) + + merged = config.merge_mm_processor_kwargs({"images_kwargs": inference_scope}) + + assert merged["images_kwargs"] == expected_scope + + +def test_merge_mm_processor_kwargs_none_leaf_is_explicit(): + config = MultiModalConfig(mm_device_do_normalize=False) + + assert config.merge_mm_processor_kwargs({"images_kwargs": {"do_resize": None}}) == { + "images_kwargs": {"do_resize": None} + } + + +def test_merge_mm_processor_kwargs_normalize_and_ownership(): + configured: dict[str, object] = { + "images_kwargs": { + "size": {"shortest_edge": 64}, + }, + } + inference: dict[str, object] = { + "do_normalize": True, + "do_rescale": True, + "images_kwargs": { + "size": {"longest_edge": 1024}, + }, + } + configured_before = copy.deepcopy(configured) + inference_before = copy.deepcopy(inference) + + config = MultiModalConfig( + mm_processor_kwargs=configured, + mm_device_do_normalize=True, + ) + merged = config.merge_mm_processor_kwargs(inference) + + assert merged["do_normalize"] is False + assert merged["do_rescale"] is False + assert configured == configured_before + assert inference == inference_before + assert config.mm_processor_kwargs == configured_before + + images_kwargs = merged["images_kwargs"] + assert isinstance(images_kwargs, dict) + size = images_kwargs["size"] + assert isinstance(size, dict) + size["shortest_edge"] = 999 + + assert configured == configured_before + assert inference == inference_before + assert config.mm_processor_kwargs == configured_before + + def _validate_mm_processor_device(*, device: str, ec_role: ECRole | None) -> None: ec_config = ( None diff --git a/tests/distributed/test_comm_ops.py b/tests/distributed/test_comm_ops.py index f35ba14c1f..25434a51b5 100644 --- a/tests/distributed/test_comm_ops.py +++ b/tests/distributed/test_comm_ops.py @@ -389,6 +389,36 @@ def test_flashinfer_standalone_workspace_size( assert create_workspace.call_args.args[3] == expected +def test_flashinfer_workspace_failure_is_not_retried( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Creation is collective; retrying it per all-reduce desyncs ranks.""" + create_workspace = Mock(return_value=None) + monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_workspace", None) + monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_quant_workspace", None) + monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_workspace_failed", False) + monkeypatch.setattr( + flashinfer_all_reduce, + "_resolve_fi_ar_backend", + Mock(return_value=("mnnvl", True)), + ) + monkeypatch.setattr(flashinfer_all_reduce, "get_node_count", lambda: 1) + monkeypatch.setattr( + flashinfer_all_reduce, "_get_tuned_standalone_max_size", Mock(return_value=None) + ) + monkeypatch.setattr(flashinfer_all_reduce, "_create_workspace", create_workspace) + args = (2, 0, 128, 4096, torch.bfloat16, Mock()) + + assert flashinfer_all_reduce.get_fi_ar_workspace(*args) is None + assert flashinfer_all_reduce.get_fi_ar_workspace(*args) is None + # One mnnvl attempt plus one trtllm fallback, never repeated. + assert create_workspace.call_count == 2 + + flashinfer_all_reduce.destroy_fi_ar_workspace() + flashinfer_all_reduce.get_fi_ar_workspace(*args) + assert create_workspace.call_count == 4 + + def test_flashinfer_all_reduce_precedes_nccl(monkeypatch: pytest.MonkeyPatch) -> None: output = torch.empty(2) fi_ar_comm = Mock(disabled=False) diff --git a/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py b/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py index f1de27bf75..a6d78c826c 100644 --- a/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py +++ b/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py @@ -26,6 +26,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import ( BatchChatCompletionRequest, ) from vllm.entrypoints.openai.models.serving import OpenAIServingModels +from vllm.exceptions import VLLMValidationError from vllm.outputs import CompletionOutput, RequestOutput from vllm.renderers.online_renderer import OnlineRenderer from vllm.v1.engine.async_llm import AsyncLLM @@ -385,3 +386,19 @@ async def test_batched_harmony_response_format_uses_structural_tag() -> None: assert len(calls) == 2 for call in calls: assert call.args[1].structured_outputs.structural_tag is not None + + +@pytest.mark.skip_global_cleanup +def test_batch_rejects_kv_transfer_prompt_token_ids(): + """One pre-tokenized prompt cannot stand in for every conversation.""" + with pytest.raises( + VLLMValidationError, match=r"parameter=kv_transfer_params\.prompt_token_ids" + ): + BatchChatCompletionRequest( + model="test-model", + messages=[ + [{"role": "user", "content": "first"}], + [{"role": "user", "content": "second"}], + ], + kv_transfer_params={"prompt_token_ids": [10, 20, 30]}, + ) diff --git a/tests/entrypoints/openai/chat_completion/test_serving_chat.py b/tests/entrypoints/openai/chat_completion/test_serving_chat.py index 6022f5ce77..241920d1e3 100644 --- a/tests/entrypoints/openai/chat_completion/test_serving_chat.py +++ b/tests/entrypoints/openai/chat_completion/test_serving_chat.py @@ -595,6 +595,15 @@ def _build_online_renderer( ) +def _build_mock_engine() -> MagicMock: + mock_engine = MagicMock(spec=AsyncLLM) + mock_engine.errored = False + mock_engine.model_config = MockModelConfig() + mock_engine.input_processor = MagicMock() + mock_engine.renderer = _build_renderer(mock_engine.model_config) + return mock_engine + + def _build_serving_chat( engine: AsyncLLM, *, @@ -2817,3 +2826,132 @@ def test_make_request_with_harmony_reuses_kv_transfer_prompt_token_ids(): assert engine_input["prompt_token_ids"] == [10, 20, 30] # The reuse key is consumed and other kv_transfer_params are preserved. assert request.kv_transfer_params == {"do_remote_prefill": True} + + +@pytest.mark.parametrize("ids", [[-1], [1.5]]) +def test_make_request_with_harmony_rejects_invalid_kv_transfer_prompt_token_ids(ids): + engine = MockEngine() + engine.model_config.hf_config = MockHFConfig(model_type="gpt_oss") + models = OpenAIServingModels(engine, BASE_MODEL_PATHS) + online_renderer = _build_online_renderer(engine, models.registry) + + request = ChatCompletionRequest( + model=MODEL_NAME, + messages=[{"role": "user", "content": "hi"}], + kv_transfer_params={"prompt_token_ids": ids}, + ) + with pytest.raises(VLLMValidationError, match="non-negative integers"): + online_renderer._make_request_with_harmony(request) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("ids", [[], [-1], [1.5], [1.0], ["1"], [True], "abc", 5]) +async def test_chat_kv_transfer_prompt_token_ids_rejects_invalid_ids(ids): + """``kv_transfer_params`` is untyped, so the ids are checked on reuse.""" + serving_chat = _build_serving_chat(_build_mock_engine()) + + request = ChatCompletionRequest( + model=MODEL_NAME, + messages=[{"role": "user", "content": "hi"}], + kv_transfer_params={"prompt_token_ids": ids}, + ) + with pytest.raises(VLLMValidationError, match="non-negative integers"): + await serving_chat.render_chat_request(request) + + +@pytest.mark.asyncio +async def test_chat_kv_transfer_prompt_token_ids_rejection_notifies_kv_connector(): + """A render-time 400 on a decode request notifies the KV connector.""" + mock_engine = _build_mock_engine() + mock_engine.notify_kv_transfer_request_rejected = AsyncMock() + serving_chat = _build_serving_chat(mock_engine) + serving_chat.has_kv_connector = True + + request = ChatCompletionRequest( + model=MODEL_NAME, + messages=[{"role": "user", "content": "hi"}], + kv_transfer_params={"prompt_token_ids": [1.5], "do_remote_prefill": True}, + ) + with pytest.raises(VLLMValidationError, match="non-negative integers"): + await serving_chat.create_chat_completion(request) + + mock_engine.notify_kv_transfer_request_rejected.assert_awaited_once_with( + request.request_id, {"do_remote_prefill": True}, data_parallel_rank=None + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("ids", [[10, 20, 30], [1.5]]) +async def test_chat_kv_transfer_prompt_token_ids_ignored_with_echo(ids): + """``echo`` needs the rendered conversation, so the ids are not used.""" + serving_chat = _build_serving_chat(_build_mock_engine()) + + request = ChatCompletionRequest( + model=MODEL_NAME, + messages=[{"role": "user", "content": "hi"}], + kv_transfer_params={"prompt_token_ids": ids, "do_remote_prefill": True}, + echo=True, + ) + result = await serving_chat.render_chat_request(request) + assert not isinstance(result, ErrorResponse) + + conversation, engine_inputs = result + assert [msg["role"] for msg in conversation] == ["user"] + assert engine_inputs[0]["prompt_token_ids"] != ids + assert request.kv_transfer_params == {"do_remote_prefill": True} + + +@pytest.mark.parametrize( + "content", + [ + [ + {"type": "text", "text": "what is in this image?"}, + {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}}, + ], + # A part written without a "type" is identified by its media key. + [{"image_url": "https://example.com/a.png"}], + # A media key counts even when the part claims to be text, which is + # how chat_utils reads a part carrying a "uuid". + [ + { + "type": "text", + "text": "look", + "uuid": "u1", + "image_url": "https://example.com/a.png", + } + ], + [{"type": "hologram", "hologram": "https://a/b.holo"}], + [{"type": ["image_url"]}], + ], +) +def test_chat_kv_transfer_prompt_token_ids_ignored_with_non_text_content(content): + """The ids are dropped so that ``messages`` is rendered with its media.""" + request = ChatCompletionRequest( + model=MODEL_NAME, + messages=[{"role": "user", "content": content}], + kv_transfer_params={ + "prompt_token_ids": [10, 20, 30], + "do_remote_prefill": True, + }, + ) + assert request.kv_transfer_params == {"do_remote_prefill": True} + + +def test_chat_kv_transfer_prompt_token_ids_allows_text_only_parts(): + """Text-bearing part types are not multimodal, so the ids are kept.""" + request = ChatCompletionRequest( + model=MODEL_NAME, + messages=[ + {"role": "user", "content": [{"type": "text", "text": "hi"}]}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "let me think"}, + {"type": "refusal", "refusal": "no"}, + ], + }, + {"role": "user", "content": "plain string"}, + ], + kv_transfer_params={"prompt_token_ids": [10, 20, 30]}, + ) + assert request.kv_transfer_params == {"prompt_token_ids": [10, 20, 30]} diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py b/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py index fe41e651be..18513f83dc 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py @@ -1,9 +1,12 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import io import json import httpx +import numpy as np +import pybase64 as base64 import pytest import pytest_asyncio from transformers import AutoTokenizer @@ -311,6 +314,52 @@ async def test_generate_logprobs(client, logprobs_value): assert len(entry["top_logprobs"]) == max(logprobs_value, 1) +@pytest.mark.asyncio +async def test_generate_prompt_token_id_logprobs(client): + token_ids = [11, 22, 33, 44, 55] + candidates = [22, 33, 44, 55] + sampling_params = { + "max_tokens": 1, + "prompt_logprobs": 0, + "prompt_logprob_token_ids": candidates, + "prompt_logprob_start": 1, + } + payload = { + "model": MODEL_NAME, + "token_ids": token_ids, + "sampling_params": sampling_params, + } + resp = await client.post(GEN_ENDPOINT, json=payload) + resp.raise_for_status() + data = resp.json() + + scores = np.load(io.BytesIO(base64.b64decode(data["prompt_token_id_logprobs"]))) + assert scores.shape == (len(token_ids) - 2, len(candidates)) + for row, target in enumerate(token_ids[2:]): + expected = data["prompt_logprobs"][row + 2][str(target)]["logprob"] + assert scores[row, candidates.index(target)] == pytest.approx( + expected, abs=1e-3 + ) + + stream_only_ids = {"prompt_logprob_token_ids": candidates} + resp = await client.post( + GEN_ENDPOINT, + json={**payload, "sampling_params": stream_only_ids, "stream": True}, + ) + assert resp.status_code == 400 + assert "prompt_logprob_token_ids" in resp.text + + past_end = {**sampling_params, "prompt_logprob_start": len(token_ids)} + resp = await client.post( + GEN_ENDPOINT, json={**payload, "sampling_params": past_end} + ) + resp.raise_for_status() + empty = np.load( + io.BytesIO(base64.b64decode(resp.json()["prompt_token_id_logprobs"])) + ) + assert empty.shape == (0, len(candidates)) + + @pytest.mark.asyncio async def test_same_response_as_chat_completions(client, tokenizer, messages): token_ids = tokenizer.apply_chat_template( diff --git a/tests/evals/gsm8k/configs/DeepSeek-V2-Lite-Instruct-FP8-TP1-PCP2-ROCm.yaml b/tests/evals/gsm8k/configs/DeepSeek-V2-Lite-Instruct-FP8-TP1-PCP2-ROCm.yaml new file mode 100644 index 0000000000..0275c9941a --- /dev/null +++ b/tests/evals/gsm8k/configs/DeepSeek-V2-Lite-Instruct-FP8-TP1-PCP2-ROCm.yaml @@ -0,0 +1,24 @@ +model_name: "RedHatAI/DeepSeek-Coder-V2-Lite-Instruct-FP8" +accuracy_threshold: 0.72 +num_questions: 1319 +num_fewshot: 5 +rocm_request_timeout_seconds: 1800 +# Dense MLA PCP on MI355; DP1/DCP1, eager execution, and Triton MoE. +server_args: >- + --revision efe1ced428db63e7ccbcf367334596f77e9af140 + --code-revision e434a23f91ba5b4923cf6c9d9a238eb4a08e3a11 + --dtype bfloat16 + --enforce-eager + --max-model-len 4096 + --max-num-seqs 8 + --max-num-batched-tokens 4096 + --tensor-parallel-size 1 + --prefill-context-parallel-size 2 + --enable-expert-parallel + --moe-backend triton + --attention-config '{"backend":"ROCM_AITER_MLA","mla_prefill_backend":"ROCM_AITER_FA"}' + --kv-cache-dtype auto +env: + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_ROCM_USE_AITER: "1" + VLLM_WORKER_MULTIPROC_METHOD: "spawn" diff --git a/tests/evals/gsm8k/configs/models-qwen35-mi355.txt b/tests/evals/gsm8k/configs/models-qwen35-mi355.txt index 49925c827e..13ff05f870 100644 --- a/tests/evals/gsm8k/configs/models-qwen35-mi355.txt +++ b/tests/evals/gsm8k/configs/models-qwen35-mi355.txt @@ -1,3 +1,4 @@ Qwen3.5-35B-A3B-DEP2.yaml +Qwen3.5-35B-A3B-FP8-DEP2.yaml Qwen3.5-35B-A3B-MXFP4-AITER-TP2.yaml Qwen3.5-35B-A3B-MXFP4-EMU-TP2.yaml diff --git a/tests/evals/gsm8k/configs/models-small-tp-mi355.txt b/tests/evals/gsm8k/configs/models-small-tp-mi355.txt new file mode 100644 index 0000000000..78283f92af --- /dev/null +++ b/tests/evals/gsm8k/configs/models-small-tp-mi355.txt @@ -0,0 +1,2 @@ +# Dense MLA PCP accuracy on MI355. +DeepSeek-V2-Lite-Instruct-FP8-TP1-PCP2-ROCm.yaml diff --git a/tests/kernels/attention/test_minimax_m3_fp8_triton_indexer.py b/tests/kernels/attention/test_minimax_m3_fp8_triton_indexer.py new file mode 100644 index 0000000000..247faba17f --- /dev/null +++ b/tests/kernels/attention/test_minimax_m3_fp8_triton_indexer.py @@ -0,0 +1,109 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""fp8 query and index-K cache on the Triton indexer must agree with bf16. + +The MiniMax M3 lightning-indexer scores only feed a top-k block ranking (the +attention itself reads the main KV cache), so an fp8 (e4m3) side cache trades +~2% relative score error for half the per-step index-K read bandwidth — the +dominant linear-in-context decode cost at long context. #45892 enabled fp8 +index caches on the SM100 MSA path; this covers the Triton impl used +everywhere else (e.g. SM120), for both the decode and prefill scorers. +""" + +import pytest +import torch + +from vllm.models.minimax_m3.common.ops.index_topk import ( + SPARSE_BLOCK_SIZE, + minimax_m3_index_decode, + minimax_m3_index_score, + minimax_m3_index_topk, +) +from vllm.platforms import current_platform + +FP8_CUDA_SUPPORTED = current_platform.is_cuda() and current_platform.supports_fp8() +PAGE = SPARSE_BLOCK_SIZE +HEAD_DIM = 128 +HEADS = 1 +TOPK = 16 +NUM_PAGES = 100 +SEQ_LEN = 96 * PAGE + 37 +PLANTED = (7, 40, 88) + + +def _make_inputs(device: str) -> tuple[torch.Tensor, torch.Tensor]: + torch.manual_seed(3) + keys = torch.randn(NUM_PAGES * PAGE, HEAD_DIM, dtype=torch.bfloat16, device=device) + # RMS-normalize like the real (post index_k_norm) keys. + keys = keys * keys.float().pow(2).mean(-1, keepdim=True).add(1e-6).rsqrt().to( + torch.bfloat16 + ) + q = torch.randn(1, HEADS, HEAD_DIM, dtype=torch.bfloat16, device=device) + for b in PLANTED: + keys[b * PAGE + 5] = (q[0, 0] * 3).to(torch.bfloat16) + return q, keys + + +def _check_selection(top_bf16: set[int], top_fp8: set[int]) -> None: + # Planted (clearly relevant) blocks must never be lost. + assert all(b in top_fp8 for b in PLANTED) + # Selection may differ only at the noise floor (near-tied filler blocks). + assert len(top_bf16 & top_fp8) >= TOPK - 2 + + +@pytest.mark.skipif(not FP8_CUDA_SUPPORTED, reason="CUDA FP8 support required") +@torch.inference_mode() +def test_fp8_index_query_and_cache_decode_topk_matches_bf16() -> None: + dev = "cuda" + q, keys = _make_inputs(dev) + block_table = torch.arange(NUM_PAGES, dtype=torch.int32, device=dev).unsqueeze(0) + seq_lens = torch.tensor([SEQ_LEN], dtype=torch.int32, device=dev) + + def select(cache: torch.Tensor) -> set[int]: + query = q.to(cache.dtype) + idx = minimax_m3_index_decode( + query, + cache.view(NUM_PAGES, PAGE, HEAD_DIM).contiguous(), + block_table, + seq_lens, + SEQ_LEN, + TOPK, + 0, + 0, + HEADS, + 1, + 1, + ) + return {int(b) for b in idx[0, 0].tolist() if b >= 0} + + _check_selection(select(keys), select(keys.to(torch.float8_e4m3fn))) + + +@pytest.mark.skipif(not FP8_CUDA_SUPPORTED, reason="CUDA FP8 support required") +@torch.inference_mode() +def test_fp8_index_query_and_cache_prefill_topk_matches_bf16() -> None: + dev = "cuda" + q, keys = _make_inputs(dev) + block_table = torch.arange(NUM_PAGES, dtype=torch.int32, device=dev).unsqueeze(0) + seq_lens = torch.tensor([SEQ_LEN], dtype=torch.int32, device=dev) + # single decode-like query token at the end of the sequence + cu_seqlens_q = torch.tensor([0, 1], dtype=torch.int32, device=dev) + prefix_lens = torch.tensor([SEQ_LEN - 1], dtype=torch.int32, device=dev) + + def select(cache: torch.Tensor) -> set[int]: + query = q.to(cache.dtype) + score = minimax_m3_index_score( + query, + cache.view(NUM_PAGES, PAGE, HEAD_DIM).contiguous(), + block_table, + cu_seqlens_q, + seq_lens, + prefix_lens, + 1, + SEQ_LEN, + HEADS, + ) + idx = minimax_m3_index_topk(score, cu_seqlens_q, prefix_lens, 1, TOPK, 0, 0) + return {int(b) for b in idx[0, 0].tolist() if b >= 0} + + _check_selection(select(keys), select(keys.to(torch.float8_e4m3fn))) diff --git a/tests/kernels/attention/test_rocm_triton_attn_dsv4.py b/tests/kernels/attention/test_rocm_triton_attn_dsv4.py index bd41048d9b..387e690e1b 100644 --- a/tests/kernels/attention/test_rocm_triton_attn_dsv4.py +++ b/tests/kernels/attention/test_rocm_triton_attn_dsv4.py @@ -8,6 +8,7 @@ import pytest import torch from vllm.platforms import current_platform +from vllm.utils.torch_utils import set_random_seed pytestmark = pytest.mark.skipif( not current_platform.is_rocm(), reason="Only used by ROCm" @@ -424,6 +425,37 @@ def test_compute_global_topk_ragged_indices_and_indptr() -> None: torch.testing.assert_close(actual_lens, expected_lens) +@pytest.mark.parametrize("width", [16, 640, 2176]) +@pytest.mark.parametrize("num_queries", [1, 70, 1000]) +@torch.inference_mode() +def test_build_ragged_indices_from_dense_drops_invalid_entries( + width: int, num_queries: int +) -> None: + from vllm.v1.attention.ops.rocm_aiter_mla_sparse import ( + build_ragged_indices_from_dense, + ) + + device = torch.device("cuda") + set_random_seed(width + num_queries) + num_rows = 5000 + indices = torch.randint(0, num_rows + 100, (num_queries, width)) + indices[torch.rand(num_queries, width) < 0.3] = -1 + lengths = torch.randint(0, width + 8, (num_queries,)) + + flat, indptr = build_ragged_indices_from_dense( + indices.to(device, torch.int32), + lengths.to(device, torch.int32), + num_rows=num_rows, + ) + + # Each row keeps its in-range entries within its length, in order. + expected = [ + [x for x in row[:n] if 0 <= x < num_rows] + for row, n in zip(indices.tolist(), lengths.tolist()) + ] + assert _rows_from_ragged(flat, indptr) == expected + + @torch.inference_mode() def test_combine_topk_swa_indices_adds_image_visibility() -> None: from vllm.models.deepseek_v4.amd.rocm import combine_topk_swa_indices diff --git a/tests/kernels/core/test_apply_rotary_emb.py b/tests/kernels/core/test_apply_rotary_emb.py index 7d63c3499c..ddb717f394 100644 --- a/tests/kernels/core/test_apply_rotary_emb.py +++ b/tests/kernels/core/test_apply_rotary_emb.py @@ -233,9 +233,15 @@ def test_apply_rotary_emb_xpu_matches_native( def test_packed_qk_rope_correctness( num_tokens: int, dtype: torch.dtype, default_vllm_config ): - """packed_qk_rope_ must be bitwise identical to the per-tensor - ApplyRotaryEmb path with enable_fp32_compute=True, and must leave the V - slice untouched.""" + """packed_qk_rope_ must match the per-tensor ApplyRotaryEmb path with + enable_fp32_compute=True to within one ULP, and must leave the V slice + untouched. + + Both sides compute in fp32 on identical inputs. The only gap is which + product the backend keeps exact inside the o1 multiply-add, and that + follows the register layout, so it is decided by codegen rather than by + the source and holds neither across backends nor across Triton versions. + """ from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb from vllm.model_executor.layers.rotary_embedding.packed_qk_rope import ( packed_qk_rope_, @@ -265,6 +271,13 @@ def test_packed_qk_rope_correctness( xk_ref = op(xk, cos, sin) packed_qk_rope_(xqkv, freqs_cis) - torch.testing.assert_close(xqkv[:, 0], xq_ref, atol=0, rtol=0) - torch.testing.assert_close(xqkv[:, 1], xk_ref, atol=0, rtol=0) + + # One ULP at the largest magnitude present, not relative: o1 can cancel, + # so the error scales with the inputs to the sum, not with the result. + eps = torch.finfo(dtype).eps + q_atol = eps * xq_ref.abs().max().item() + k_atol = eps * xk_ref.abs().max().item() + torch.testing.assert_close(xqkv[:, 0], xq_ref, atol=q_atol, rtol=0) + torch.testing.assert_close(xqkv[:, 1], xk_ref, atol=k_atol, rtol=0) + # V is never read or written by the kernel, so it stays exact. torch.testing.assert_close(xqkv[:, 2], xv, atol=0, rtol=0) diff --git a/tests/kernels/core/test_uva.py b/tests/kernels/core/test_uva.py index f8c297afc6..b7c2984d6a 100644 --- a/tests/kernels/core/test_uva.py +++ b/tests/kernels/core/test_uva.py @@ -4,54 +4,57 @@ import numpy as np import pytest import torch +from vllm.platforms import current_platform from vllm.utils.platform_utils import is_uva_available from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor from vllm.v1.worker.gpu import buffer_utils from vllm.v1.worker.gpu.buffer_utils import StagedWriteTensor -CUDA_DEVICES = [ - f"cuda:{i}" for i in range(1 if torch.accelerator.device_count() == 1 else 2) +DEVICE_TYPE = current_platform.device_type +DEVICES = [ + f"{DEVICE_TYPE}:{i}" + for i in range(1 if torch.accelerator.device_count() == 1 else 2) ] @pytest.mark.skipif(not is_uva_available(), reason="UVA is not available.") -@pytest.mark.parametrize("device", CUDA_DEVICES) +@pytest.mark.parametrize("device", DEVICES) def test_cpu_write(device): torch.set_default_device(device) cpu_tensor = torch.zeros(10, 10, device="cpu", pin_memory=True, dtype=torch.int32) - cuda_view = get_accelerator_view_from_cpu_tensor(cpu_tensor) - assert cuda_view.device.type == "cuda" + gpu_view = get_accelerator_view_from_cpu_tensor(cpu_tensor) + assert gpu_view.device.type == DEVICE_TYPE - assert cuda_view[0, 0] == 0 - assert cuda_view[2, 3] == 0 - assert cuda_view[4, 5] == 0 + assert gpu_view[0, 0] == 0 + assert gpu_view[2, 3] == 0 + assert gpu_view[4, 5] == 0 cpu_tensor[0, 0] = 1 cpu_tensor[2, 3] = 2 cpu_tensor[4, 5] = -1 - cuda_view.mul_(2) - assert cuda_view[0, 0] == 2 - assert cuda_view[2, 3] == 4 - assert cuda_view[4, 5] == -2 + gpu_view.mul_(2) + assert gpu_view[0, 0] == 2 + assert gpu_view[2, 3] == 4 + assert gpu_view[4, 5] == -2 @pytest.mark.skipif(not is_uva_available(), reason="UVA is not available.") -@pytest.mark.parametrize("device", CUDA_DEVICES) +@pytest.mark.parametrize("device", DEVICES) def test_gpu_write(device): torch.set_default_device(device) cpu_tensor = torch.zeros(10, 10, device="cpu", pin_memory=True, dtype=torch.int32) - cuda_view = get_accelerator_view_from_cpu_tensor(cpu_tensor) - assert cuda_view.device.type == "cuda" + gpu_view = get_accelerator_view_from_cpu_tensor(cpu_tensor) + assert gpu_view.device.type == DEVICE_TYPE - assert cuda_view[0, 0] == 0 - assert cuda_view[2, 3] == 0 - assert cuda_view[4, 5] == 0 + assert gpu_view[0, 0] == 0 + assert gpu_view[2, 3] == 0 + assert gpu_view[4, 5] == 0 - cuda_view[0, 0] = 1 - cuda_view[2, 3] = 2 - cuda_view[4, 5] = -1 - cuda_view.mul_(2) + gpu_view[0, 0] = 1 + gpu_view[2, 3] = 2 + gpu_view[4, 5] = -1 + gpu_view.mul_(2) assert cpu_tensor[0, 0] == 2 assert cpu_tensor[2, 3] == 4 @@ -59,7 +62,7 @@ def test_gpu_write(device): @pytest.mark.skipif(not is_uva_available(), reason="UVA is not available.") -@pytest.mark.parametrize("device", CUDA_DEVICES) +@pytest.mark.parametrize("device", DEVICES) def test_staged_write_uses_uva_contents_for_uva_target(device, monkeypatch): def fail_async_tensor_h2d(*args, **kwargs): pytest.fail("UVA-backed targets should not copy write contents to the GPU") @@ -146,7 +149,7 @@ def test_uva_pool_copy_to_gpu_preserves_shape_and_out(use_out, input_type): ) values = expected.numpy() if input_type == "numpy" else expected out = ( - torch.empty(expected.shape, dtype=torch.int32, device="cuda") + torch.empty(expected.shape, dtype=torch.int32, device=DEVICE_TYPE) if use_out else None ) @@ -161,7 +164,7 @@ def test_uva_pool_copy_to_gpu_preserves_shape_and_out(use_out, input_type): @pytest.mark.parametrize("dtype", [torch.int32, torch.int64, torch.float32]) def test_staged_write_inflight(uva_target, dtype): """Preserve every generation until its consumer finishes before slot reuse.""" - device = torch.device("cuda:0") + device = torch.device(f"{DEVICE_TYPE}:0") with torch.accelerator.device_index(device.index): state = StagedWriteTensor( (4, 4096), @@ -171,9 +174,9 @@ def test_staged_write_inflight(uva_target, dtype): uva_instead_of_gpu=uva_target, ) assert (state.write_contents is not None) == uva_target - stream = torch.cuda.Stream() - stream.wait_stream(torch.cuda.current_stream()) - pending: list[tuple[torch.cuda.Event, torch.Tensor, torch.Tensor]] = [] + stream = torch.Stream(device=device) + stream.wait_stream(torch.accelerator.current_stream()) + pending: list[tuple[torch.Event, torch.Tensor, torch.Tensor]] = [] expected = torch.zeros((4, 4096), dtype=dtype, device="cpu") for step in range(24): if len(pending) == 2: @@ -188,15 +191,75 @@ def test_staged_write_inflight(uva_target, dtype): values += 0.25 expected[row, 2 : 2 + length] = values expected[3, 1:4] = step - with torch.cuda.stream(stream): + with stream: state.stage_write(row, 2, values.tolist()) state.stage_write(3, 1, [step] * 3) state.apply_write() # A GPU consumer observes this generation before the next update. snapshot = state.gpu.clone() - event = torch.cuda.Event() + event = torch.Event() event.record(stream) pending.append((event, snapshot, expected.clone())) for event, snapshot, reference in pending: event.synchronize() torch.testing.assert_close(snapshot.cpu(), reference, rtol=0, atol=0) + + +@pytest.mark.skipif(not is_uva_available(), reason="UVA is not available.") +@pytest.mark.parametrize("device", DEVICES) +def test_non_pinned_cpu_tensor(device): + # Non-pinned CPU tensors are internally copied into a pinned buffer, + # so the resulting gpu view reflects the values at creation time but + # is decoupled from further writes to the original `cpu_tensor`. + torch.set_default_device(device) + cpu_tensor = torch.arange(100, dtype=torch.int32, device="cpu").view(10, 10) + assert not cpu_tensor.is_pinned() + gpu_view = get_accelerator_view_from_cpu_tensor(cpu_tensor) + assert gpu_view.device.type == DEVICE_TYPE + + assert gpu_view[0, 0] == 0 + assert gpu_view[2, 3] == 23 + assert gpu_view[9, 9] == 99 + + # Writes to the original (unpinned) CPU tensor must not affect the view, + # since a private pinned copy was made. + cpu_tensor[0, 0] = -1 + assert gpu_view[0, 0] == 0 + + # The view itself remains writable and independently usable. + gpu_view.mul_(2) + assert gpu_view[2, 3] == 46 + assert gpu_view[9, 9] == 198 + + +@pytest.mark.skipif(not is_uva_available(), reason="UVA is not available.") +@pytest.mark.skipif( + not current_platform.is_xpu(), reason="XPU non-contiguous UVA test." +) +@pytest.mark.parametrize("pinned", [False, True]) +@pytest.mark.parametrize("device", DEVICES) +def test_non_contiguous_strided_view(device, pinned): + torch.set_default_device(device) + # Simulate scale_kn: a [32, 16] contiguous tensor + scale_kn = torch.arange( + 512, dtype=torch.float32, device="cpu", pin_memory=pinned + ).view(32, 16) + # Transposed view: shape [16, 32], non-contiguous stride (1, 16) + cpu_view = scale_kn.t() + assert cpu_view.shape == (16, 32) + assert cpu_view.stride() == (1, 16) + assert not cpu_view.is_contiguous() + + gpu_view = get_accelerator_view_from_cpu_tensor(cpu_view) + assert gpu_view.device.type == DEVICE_TYPE + assert gpu_view.shape == (16, 32) + assert gpu_view.stride() == (1, 16) + assert not gpu_view.is_contiguous() + + # Transposing tensor_view back should yield a contiguous [32, 16] tensor + gpu_view_t = gpu_view.t() + assert gpu_view_t.shape == (32, 16) + assert gpu_view_t.is_contiguous() + + # Correctness check: values match original transposed CPU tensor + assert torch.equal(gpu_view.cpu(), cpu_view) diff --git a/tests/kernels/mhc/test_mhc_kernels.py b/tests/kernels/mhc/test_mhc_kernels.py index 4735fb96d1..0554e9ca66 100644 --- a/tests/kernels/mhc/test_mhc_kernels.py +++ b/tests/kernels/mhc/test_mhc_kernels.py @@ -27,6 +27,7 @@ from vllm.model_executor.layers.mhc import ( HAS_AITER_MHC, HAS_AITER_MHC_FUSED, HAS_AITER_MHC_FUSED_NORM, + HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM, HAS_AITER_MHC_PRE_NORM, HAS_TILELANG_MHC, MHCFusedPostPreOp, @@ -1300,18 +1301,109 @@ def test_mhc_pre_delayed_rocm_aiter(num_tokens, carried): torch.testing.assert_close(actual[2], expected[2], atol=0, rtol=0) +@pytest.mark.skipif( + not (current_platform.is_rocm() and HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM), + reason="AITER fused delayed-seam kernel required (gfx950)", +) +@pytest.mark.parametrize("num_tokens", [1, 2, 7, 128, 1024]) +@pytest.mark.parametrize("seam", ["post", "no_post", "identity_pre"]) +def test_mhc_pre_delayed_rocm_aiter_fused_rms_norm(num_tokens, seam): + """One fused kernel must match the whole reference seam: the post block + (the previous seam's post/comb gates fold ``sublayer_out`` into the + streams), the delayed pre block (projection + Sinkhorn producing the next + seam's post/comb/pre gates, plus the collapse with the carried pre-mix), + and the RMSNorm folded onto the collapse. + + ``num_tokens`` spans the kernel's split-K config buckets. ``seam`` covers + the three call shapes the model makes: a regular seam (post folded in), an + Engram seam (post already applied), and the draft entry seam + (``pre_mix=None`` = identity pre-mix). + """ + set_random_seed(0) + hc_mult, hidden_size = 4, 5120 + residual, fn, hc_scale, hc_base, norm_weight = _rocm_mhc_inputs( + num_tokens=num_tokens, hidden_size=hidden_size, hc_mult=hc_mult + ) + rms_eps = hc_pre_eps = hc_sinkhorn_eps = norm_eps = 1e-6 + args = ( + residual, + fn, + hc_scale, + hc_base, + rms_eps, + hc_pre_eps, + hc_sinkhorn_eps, + 1.0, + 20, + ) + + pre_mix = ( + None + if seam == "identity_pre" + else torch.rand(num_tokens, hc_mult, dtype=torch.float32, device=DEVICE) + 0.5 + ) + post_kwargs = {} + residual_ref = residual + if seam == "post": + sublayer_out = torch.randn( + num_tokens, hidden_size, dtype=torch.bfloat16, device=DEVICE + ) + post_layer_mix = 2 * torch.rand( + num_tokens, hc_mult, 1, dtype=torch.float32, device=DEVICE + ) + comb_res_mix = torch.softmax( + torch.randn(num_tokens, hc_mult, hc_mult, device=DEVICE), dim=-1 + ) + post_kwargs = dict( + sublayer_out=sublayer_out, + post_layer_mix=post_layer_mix, + comb_res_mix=comb_res_mix, + ) + residual_ref = mhc_post_torch( + sublayer_out, residual, post_layer_mix, comb_res_mix + ) + + expected = mhc_pre_delayed_torch(residual_ref, *args[1:], pre_mix=pre_mix) + expected_layer_input = F.rms_norm( + expected[2], (hidden_size,), norm_weight, norm_eps + ) + + residual_out, *actual = object.__new__(MHCPreDelayedOp).forward_hip( + *args, + pre_mix=pre_mix, + norm_weight=norm_weight, + norm_eps=norm_eps, + **post_kwargs, + ) + + if seam == "post": + torch.testing.assert_close(residual_out, residual_ref) + else: + # No post was requested, so the residual comes straight back. + assert residual_out is residual + # Gates: post_mix, comb_mix, next_pre_mix (bf16 MFMA vs the fp32 reference). + for i in (0, 1, 3): + torch.testing.assert_close(actual[i], expected[i], atol=5e-4, rtol=1e-3) + # The collapse is bf16 and the kernel projects the once-rounded R', so + # near-zero values need an absolute tolerance (same bound as aiter's UT). + torch.testing.assert_close(actual[2], expected_layer_input, atol=2e-2, rtol=1e-2) + + @pytest.mark.skipif( not (current_platform.is_rocm() and HAS_AITER_MHC), reason="AITER mHC required", ) def test_mhc_pre_delayed_rocm_aiter_declines_unsupported(monkeypatch): - """The broadcast seam and a fused norm must not take the AITER path. + """The broadcast seam must not take the AITER path; a fused norm must not + take the unfused AITER op. - Neither is expressible with AITER's pre kernels: the broadcast projects a - narrower ``x``, and ``mhc_pre_gemm_sqrsum`` folds no RMSNorm. What is - asserted here is the routing decision, which is what this gate owns; the - numerics of whichever fallback it lands on are covered by - ``test_deepseek_v41_mhc_pre_delayed``. + The broadcast projects a narrower ``x``, which no AITER pre kernel takes; + ``mhc_pre_gemm_sqrsum`` folds no RMSNorm, so a fused norm routes to the + fused seam kernel where it exists (gfx950) and away from AITER otherwise. + What is asserted here is the routing decision, which is what this gate + owns; the numerics of the paths are covered by + ``test_deepseek_v41_mhc_pre_delayed`` and + ``test_mhc_pre_delayed_rocm_aiter_fused_rms_norm``. """ set_random_seed(0) hc_mult, hidden_size = 4, 5120 @@ -1331,6 +1423,18 @@ def test_mhc_pre_delayed_rocm_aiter_declines_unsupported(monkeypatch): monkeypatch.setattr(torch.ops.vllm, "mhc_pre_delayed_aiter", spy) + fused_op = torch.ops.vllm.mhc_fused_post_pre_delayed_rms_norm_aiter + took_fused = False + + def fused_spy(*spy_args, **spy_kwargs): + nonlocal took_fused + took_fused = True + return fused_op(*spy_args, **spy_kwargs) + + monkeypatch.setattr( + torch.ops.vllm, "mhc_fused_post_pre_delayed_rms_norm_aiter", fused_spy + ) + x = residual[:, 0].contiguous() broadcast_fn = fn.view(-1, hc_mult, hidden_size).sum(1) broadcast_residual = x.unsqueeze(1).expand(-1, hc_mult, -1).contiguous() @@ -1342,11 +1446,15 @@ def test_mhc_pre_delayed_rocm_aiter_declines_unsupported(monkeypatch): ) assert residual_out is broadcast_residual assert not took_aiter, "the broadcast seam must not reach AITER" + assert not took_fused, "the broadcast seam must not reach AITER" for i in range(4): torch.testing.assert_close(actual[i], expected[i], atol=1e-4, rtol=1e-3) + # A fused norm must not reach the unfused AITER op; where the fused seam + # kernel exists (gfx950) it routes there instead. op.forward_hip(*args, norm_weight=norm_weight, norm_eps=1e-6) - assert not took_aiter, "a fused norm must not reach AITER" + assert not took_aiter, "a fused norm must not reach the unfused AITER op" + assert took_fused == mhc_layers.HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM # Positive control: the same inputs without a norm do take the AITER path, # so the two declines above are the gate discriminating rather than the diff --git a/tests/kernels/quantization/test_int4_emulation_moe.py b/tests/kernels/quantization/test_int4_emulation_moe.py index 7016860d86..cad0dd94d2 100644 --- a/tests/kernels/quantization/test_int4_emulation_moe.py +++ b/tests/kernels/quantization/test_int4_emulation_moe.py @@ -240,6 +240,29 @@ def _make_awq_moe_weights(E, K, N, group_size, asym=False): ) +def _canonicalize_gptq_moe_inputs( + w13: torch.Tensor, + w2: torch.Tensor, + w13_scale: torch.Tensor, + w2_scale: torch.Tensor, + w13_qzeros: torch.Tensor | None, + w2_qzeros: torch.Tensor | None, +): + """Convert GPTQ K-first tensors to the oracle's canonical N-first layout.""" + + def transpose(value: torch.Tensor | None) -> torch.Tensor | None: + return value.transpose(1, 2).contiguous() if value is not None else None + + return ( + w13.transpose(1, 2).contiguous(), + w2.transpose(1, 2).contiguous(), + w13_scale.transpose(1, 2).contiguous(), + w2_scale.transpose(1, 2).contiguous(), + transpose(w13_qzeros), + transpose(w2_qzeros), + ) + + def _run_emulation_forward( experts, w13_bf16, w2_bf16, hidden_states, topk_weights, topk_ids, E, K, N ): @@ -478,7 +501,8 @@ def test_gptq_process_weights_shapes_and_values(E, K, N, group_size, asym): w13, w13s, w13z, w13_ref, w2, w2s, w2z, w2_ref = _make_gptq_moe_weights( E, K, N, group_size, asym ) - result = _process_weights_emulation_gptq(w13, w2, w13s, w2s, w13z, w2z) + gptq_inputs = _canonicalize_gptq_moe_inputs(w13, w2, w13s, w2s, w13z, w2z) + result = _process_weights_emulation_gptq(*gptq_inputs) w13_out, w2_out = result[0], result[1] assert w13_out.shape == (E, 2 * N, K) @@ -561,7 +585,7 @@ def test_gptq_awq_process_weights_agree(E, K, N, group_size, asym): g2s_list.append(s2) a2s_list.append(s2) - gptq_res = _process_weights_emulation_gptq( + gptq_inputs = _canonicalize_gptq_moe_inputs( torch.stack(g13_list), torch.stack(g2_list), torch.stack(g13s_list), @@ -569,6 +593,7 @@ def test_gptq_awq_process_weights_agree(E, K, N, group_size, asym): torch.stack(g13z_list) if asym else None, torch.stack(g2z_list) if asym else None, ) + gptq_res = _process_weights_emulation_gptq(*gptq_inputs) awq_res = _process_weights_emulation_awq( torch.stack(a13_list), torch.stack(a2_list), @@ -615,7 +640,7 @@ def test_gptq_vs_awq_forward_agree(E, K, N, top_k, group_size, num_tokens): g2s_list.append(s2) a2s_list.append(s2.clone()) - gptq_res = _process_weights_emulation_gptq( + gptq_inputs = _canonicalize_gptq_moe_inputs( torch.stack(g13_list), torch.stack(g2_list), torch.stack(g13s_list), @@ -623,6 +648,7 @@ def test_gptq_vs_awq_forward_agree(E, K, N, top_k, group_size, num_tokens): None, None, ) + gptq_res = _process_weights_emulation_gptq(*gptq_inputs) awq_res = _process_weights_emulation_awq( torch.stack(a13_list), torch.stack(a2_list), @@ -750,7 +776,7 @@ def test_ep_output_matches_no_ep(E, K, N, top_k, group_size, num_tokens, ep_size if fmt == "gptq" else _process_weights_emulation_awq ) - res = process_fn( + process_inputs = ( torch.stack(packed13_list), torch.stack(packed2_list), torch.stack(scales13_list), @@ -758,6 +784,9 @@ def test_ep_output_matches_no_ep(E, K, N, top_k, group_size, num_tokens, ep_size None, None, ) + if fmt == "gptq": + process_inputs = _canonicalize_gptq_moe_inputs(*process_inputs) + res = process_fn(*process_inputs) w13_all, w2_all = res[0], res[1] # [E, 2N, K], [E, K, N] hidden_states = torch.randn(num_tokens, K, dtype=torch.bfloat16, device=device) @@ -855,7 +884,7 @@ def test_ep_gptq_awq_agree(E, K, N, top_k, group_size, num_tokens, ep_size): g2s_list.append(s2) a2s_list.append(s2.clone()) - gptq_res = _process_weights_emulation_gptq( + gptq_inputs = _canonicalize_gptq_moe_inputs( torch.stack(g13_list), torch.stack(g2_list), torch.stack(g13s_list), @@ -863,6 +892,7 @@ def test_ep_gptq_awq_agree(E, K, N, top_k, group_size, num_tokens, ep_size): None, None, ) + gptq_res = _process_weights_emulation_gptq(*gptq_inputs) awq_res = _process_weights_emulation_awq( torch.stack(a13_list), torch.stack(a2_list), @@ -954,7 +984,7 @@ def test_ep_partial_rank_no_active_experts( scales13_list.append(s13) scales2_list.append(s2) - res = _process_weights_emulation_gptq( + gptq_inputs = _canonicalize_gptq_moe_inputs( torch.stack(packed13_list), torch.stack(packed2_list), torch.stack(scales13_list), @@ -962,6 +992,7 @@ def test_ep_partial_rank_no_active_experts( None, None, ) + res = _process_weights_emulation_gptq(*gptq_inputs) w13_all, w2_all = res[0], res[1] # Force topk_ids to only use experts in [0, num_local) — rank 0's slice @@ -1029,7 +1060,7 @@ def test_ep_sum_equals_full_forward(E, K, N, top_k, group_size, num_tokens, ep_s scales13_list.append(s13) scales2_list.append(s2) - res = _process_weights_emulation_gptq( + gptq_inputs = _canonicalize_gptq_moe_inputs( torch.stack(packed13_list), torch.stack(packed2_list), torch.stack(scales13_list), @@ -1037,6 +1068,7 @@ def test_ep_sum_equals_full_forward(E, K, N, top_k, group_size, num_tokens, ep_s None, None, ) + res = _process_weights_emulation_gptq(*gptq_inputs) w13_all, w2_all = res[0], res[1] # Fix routing so every token uses exactly 2 consecutive experts (round-robin) @@ -1143,7 +1175,7 @@ def test_emulation_output_close_to_bf16_reference( if fmt == "gptq" else _process_weights_emulation_awq ) - res = process_fn( + process_inputs = ( torch.stack(packed13_list), torch.stack(packed2_list), torch.stack(scales13_list), @@ -1151,6 +1183,9 @@ def test_emulation_output_close_to_bf16_reference( None, None, ) + if fmt == "gptq": + process_inputs = _canonicalize_gptq_moe_inputs(*process_inputs) + res = process_fn(*process_inputs) w13_bf16, w2_bf16 = res[0], res[1] dummy_scale = torch.ones(1, dtype=torch.float16, device=device) diff --git a/tests/lora/test_sequence_classification.py b/tests/lora/test_sequence_classification.py index 86721bfb6e..773e5448c8 100644 --- a/tests/lora/test_sequence_classification.py +++ b/tests/lora/test_sequence_classification.py @@ -120,12 +120,13 @@ def _test_batched_loras( model=MODEL_NAME, runner="pooling", convert="classify", - hf_overrides={"num_labels": 2}, + hf_overrides={"num_labels": 1}, dtype="float16", enable_lora=True, max_loras=2, max_cpu_loras=2, max_lora_rank=16, + max_lora_cls_labels=2, enforce_eager=True, max_model_len=512, gpu_memory_utilization=0.5, diff --git a/tests/models/kimi_k3/test_amd_latent_moe_runner.py b/tests/models/kimi_k3/test_amd_latent_moe_runner.py index ec270c04fe..fd5bcfd22b 100644 --- a/tests/models/kimi_k3/test_amd_latent_moe_runner.py +++ b/tests/models/kimi_k3/test_amd_latent_moe_runner.py @@ -23,6 +23,7 @@ from vllm.distributed import get_tp_group from vllm.model_executor.layers.fused_moe.runner import moe_runner from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ReplicatedLinear +from vllm.models.kimi_k3.amd import latent_moe_runner from vllm.models.kimi_k3.amd.latent_moe_runner import ROCmLatentMoERunner from vllm.models.kimi_k3.amd.linear import KimiRoutedOutputTransform from vllm.platforms import current_platform @@ -56,21 +57,26 @@ def _build_transform(device: torch.device) -> KimiRoutedOutputTransform: def _tail_runner( - transform: KimiRoutedOutputTransform, tp_size: int + transform: KimiRoutedOutputTransform, tp_world: int, use_ep: bool = False ) -> ROCmLatentMoERunner: """A runner carrying only what the tail reads, so no engine is needed. ``_maybe_reduce_final_output`` is left as the real base-class method, so the all-reduce that stitches the shards is the real collective. + + ``tp_world`` is the size of the TP process group, which is what the hidden + dim is split by. Under expert parallelism the MoE config reports + ``tp_size=1``/``ep_size=tp_world`` for that same group, so ``use_ep`` pins + that the final reduce still fires. """ runner = object.__new__(ROCmLatentMoERunner) attrs = { "routed_output_transform": transform, - "_up_proj_shard_size": HIDDEN_SIZE // tp_size, + "_up_proj_shard_size": HIDDEN_SIZE // tp_world, "_logged_sharded_tail": False, "moe_config": SimpleNamespace( - tp_size=tp_size, - ep_size=1, + tp_size=1 if use_ep else tp_world, + ep_size=tp_world if use_ep else 1, is_sequence_parallel=False, skip_final_all_reduce=False, ), @@ -94,9 +100,11 @@ def _rank_partials( return routed.mul_(0.01), shared -def _check_matches_replicated(device: torch.device, tp_size: int, rank: int) -> None: +def _check_matches_replicated( + device: torch.device, tp_world: int, rank: int, use_ep: bool = False +) -> None: transform = _build_transform(device) - runner = _tail_runner(transform, tp_size) + runner = _tail_runner(transform, tp_world, use_ep) group = get_tp_group().device_group for iteration, num_tokens in enumerate((1, 5, 8, 16, 5)): @@ -119,13 +127,22 @@ def _check_matches_replicated(device: torch.device, tp_size: int, rank: int) -> torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2) +def _check_matches_replicated_under_ep( + device: torch.device, tp_world: int, rank: int +) -> None: + """The served config: the routed experts are expert-parallel, so the MoE + config reports no tensor parallelism at all for a group that still splits + the hidden dim ``tp_world`` ways.""" + _check_matches_replicated(device, tp_world, rank, use_ep=True) + + def _check_writes_only_its_own_shard( - device: torch.device, tp_size: int, rank: int + device: torch.device, tp_world: int, rank: int ) -> None: """Two ranks that swapped both weight rows and write offsets still sum to the right total, so inspect each slice before the final collective.""" transform = _build_transform(device) - runner = _tail_runner(transform, tp_size) + runner = _tail_runner(transform, tp_world) group = get_tp_group().device_group torch.manual_seed(rank + 1) @@ -149,7 +166,7 @@ def _check_writes_only_its_own_shard( assert captured["output_is_reduced"] is False - shard = HIDDEN_SIZE // tp_size + shard = HIDDEN_SIZE // tp_world start, end = rank * shard, (rank + 1) * shard local = captured["states"] projected = F.linear(latent, transform.up_proj.weight[start:end]) @@ -163,6 +180,7 @@ def _check_writes_only_its_own_shard( _CHECKS = { "matches_replicated": _check_matches_replicated, + "matches_replicated_ep": _check_matches_replicated_under_ep, "own_shard_only": _check_writes_only_its_own_shard, } @@ -196,6 +214,11 @@ def test_sharded_tail_tp8_matches_replicated_projection() -> None: _run_ranks("matches_replicated", 8) +@multi_gpu_test(num_gpus=8) +def test_sharded_tail_tp8_ep8_matches_replicated_projection() -> None: + _run_ranks("matches_replicated_ep", 8) + + @multi_gpu_test(num_gpus=4) def test_sharded_tail_tp4_writes_only_its_own_shard() -> None: _run_ranks("own_shard_only", 4) @@ -241,12 +264,19 @@ def test_forward_shards_only_when_the_tail_is_valid( @pytest.fixture def build_runner(monkeypatch: pytest.MonkeyPatch): - """Construct through the real subclass ``__init__``, stubbing only the base.""" + """Construct through the real subclass ``__init__``, stubbing only the base. + + ``tp_world`` is the size of the TP process group and is varied independently + of ``moe_config.tp_size``, because under expert parallelism those two are + different numbers for the same eight ranks. + """ + world = {"size": 8} def _base_init( self, *, tp_size: int = 8, + ep_size: int = 1, hidden: int = HIDDEN_SIZE, is_sequence_parallel: bool = False, has_up_proj: bool = True, @@ -254,7 +284,9 @@ def build_runner(monkeypatch: pytest.MonkeyPatch): routed_scaling_factor: float = 1.0, ) -> None: self.moe_config = SimpleNamespace( - tp_size=tp_size, is_sequence_parallel=is_sequence_parallel + tp_size=tp_size, + ep_size=ep_size, + is_sequence_parallel=is_sequence_parallel, ) self.routed_output_transform = SimpleNamespace( norm=None, @@ -266,20 +298,63 @@ def build_runner(monkeypatch: pytest.MonkeyPatch): self._shared_experts = object() if has_shared_experts else None monkeypatch.setattr(moe_runner.MoERunner, "__init__", _base_init) - return ROCmLatentMoERunner + monkeypatch.setattr( + latent_moe_runner, + "get_tensor_model_parallel_world_size", + lambda: world["size"], + ) + + def _build(*, tp_world: int = 8, **kwargs) -> ROCmLatentMoERunner: + world["size"] = tp_world + return ROCmLatentMoERunner(**kwargs) + + return _build def test_shards_under_the_kimi_k3_serving_config(build_runner) -> None: - runner = build_runner() + """``--tensor-parallel-size 8``, MoE experts tensor-parallel.""" + runner = build_runner(tp_world=8, tp_size=8, ep_size=1) assert runner._tail_shardable assert runner._up_proj_shard_size == HIDDEN_SIZE // 8 +def test_shards_under_the_kimi_k3_serving_config_with_expert_parallel( + build_runner, +) -> None: + """``--tensor-parallel-size 8 --enable-expert-parallel``, what is served. + + EP reports ``moe_config.tp_size == 1`` because each rank owns whole experts, + but the hidden dim is still split across all eight TP ranks, so the tail + must shard exactly as it does without EP. + """ + runner = build_runner(tp_world=8, tp_size=1, ep_size=8) + + assert runner._tail_shardable + assert runner._up_proj_shard_size == HIDDEN_SIZE // 8 + + +def test_shards_by_the_process_group_not_the_dp_flattened_tp_size( + build_runner, +) -> None: + """Without EP, ``moe_config.tp_size`` is flattened across DP: TP=2 x DP=2 + reports 4 for a TP process group of 2. + + Only those 2 ranks share a hidden dim and take part in the final + all-reduce, so sharding by 4 would leave half of it with no routed + contribution on either rank. + """ + runner = build_runner(tp_world=2, tp_size=4, ep_size=1) + + assert runner._tail_shardable + assert runner._up_proj_shard_size == HIDDEN_SIZE // 2 + + @pytest.mark.parametrize( "override", [ - pytest.param({"tp_size": 1}, id="no-tp"), + pytest.param({"tp_world": 1, "tp_size": 1}, id="single-rank"), + pytest.param({"tp_world": 1, "tp_size": 4}, id="dp-without-tp"), pytest.param({"hidden": HIDDEN_SIZE + 2}, id="hidden-not-divisible-by-tp"), pytest.param({"has_up_proj": False}, id="no-up-proj"), pytest.param({"has_shared_experts": False}, id="no-shared-partial"), @@ -290,7 +365,12 @@ def test_shards_under_the_kimi_k3_serving_config(build_runner) -> None: def test_falls_back_when_the_config_breaks_the_shard( build_runner, override: dict ) -> None: - """Each of these makes the sharded tail wrong, not merely slower.""" + """Each of these makes the sharded tail wrong, not merely slower. + + ``dp-without-tp`` is DP=4 with no TP: the hidden dim is not split at all, + so there is nothing to shard and no all-reduce to stitch shards back + together, however large ``moe_config.tp_size`` claims to be. + """ runner = build_runner(**override) assert not runner._tail_shardable diff --git a/tests/models/multimodal/generation/test_voxtral_realtime.py b/tests/models/multimodal/generation/test_voxtral_realtime.py index 2c13e07ff3..09432b4494 100644 --- a/tests/models/multimodal/generation/test_voxtral_realtime.py +++ b/tests/models/multimodal/generation/test_voxtral_realtime.py @@ -16,7 +16,9 @@ from vllm import LLM, SamplingParams from vllm.assets.audio import AudioAsset from vllm.config import CUDAGraphMode from vllm.engine.arg_utils import AsyncEngineArgs +from vllm.engine.protocol import StreamingInput from vllm.platforms import current_platform +from vllm.renderers.inputs.preprocess import parse_model_prompt from vllm.utils.math_utils import cdiv from vllm.v1.engine.async_llm import AsyncLLM from vllm.v1.kv_cache_interface import SlidingWindowSpec @@ -222,6 +224,14 @@ async def test_voxtral_realtime_generator(audio_assets, tokenizer, async_engine) sampling_params = SamplingParams(temperature=0.0, max_tokens=1) audio_config = tokenizer.instruct_tokenizer.audio_encoder.audio_config + async def input_stream(audio_buffer): + async for prompt in audio_buffer.get_input_stream(): + parsed_prompt = parse_model_prompt(async_engine.model_config, prompt) + (engine_input,) = await async_engine.renderer.render_cmpl_async( + [parsed_prompt] + ) + yield StreamingInput(prompt=engine_input) + output_tokens_list = [] for i, audio_asset in enumerate(audio_assets): output_tokens = [] @@ -241,7 +251,7 @@ async def test_voxtral_realtime_generator(audio_assets, tokenizer, async_engine) request_id = f"session-{i}" async for resp in async_engine.generate( - prompt=buffer.get_input_stream(), + prompt=input_stream(buffer), sampling_params=sampling_params, request_id=request_id, ): diff --git a/tests/models/multimodal/generation_ppl_test/ppl_utils.py b/tests/models/multimodal/generation_ppl_test/ppl_utils.py index 317c583204..c612eb7141 100644 --- a/tests/models/multimodal/generation_ppl_test/ppl_utils.py +++ b/tests/models/multimodal/generation_ppl_test/ppl_utils.py @@ -34,6 +34,7 @@ def vqa_ppl_test( mm_processor_kwargs = {} processor = AutoProcessor.from_pretrained(model_info.name) + image_token_id = processor.image_token_id images = [] prompts = [] @@ -99,8 +100,10 @@ def vqa_ppl_test( for token_data in token_datas[1:]: assert token_data is not None assert len(token_data) == 1 - token_log_prob = list(token_data.values())[0].logprob - token_log_probs.append(token_log_prob) + token_id, token_log_prob = next(iter(token_data.items())) + if token_id == image_token_id: + continue + token_log_probs.append(token_log_prob.logprob) neg_log_likelihood = -torch.tensor( token_log_probs, dtype=torch.float32, device="cpu" @@ -132,11 +135,12 @@ def vqa_ppl_test( inputs = inputs.to("cuda") input_ids = inputs["input_ids"] - outputs = hf_model.model(**inputs, labels=input_ids) + labels = input_ids.masked_fill(input_ids == image_token_id, -100) + outputs = hf_model.model(**inputs, labels=labels) neg_log_likelihood = outputs.loss neg_log_likelihood = neg_log_likelihood.to(torch.float32).cpu() - num_loss_tokens = input_ids.shape[1] - 1 + num_loss_tokens = int((labels[:, 1:] != -100).sum()) nll_sum += neg_log_likelihood * num_loss_tokens n_tokens += num_loss_tokens diff --git a/tests/models/multimodal/generation_ppl_test/test_qwen.py b/tests/models/multimodal/generation_ppl_test/test_qwen.py index 39f6877cf6..63742b600f 100644 --- a/tests/models/multimodal/generation_ppl_test/test_qwen.py +++ b/tests/models/multimodal/generation_ppl_test/test_qwen.py @@ -8,26 +8,51 @@ from tests.models.utils import GenerateModelInfo from .ppl_utils import vqa_ppl_test MODELS = [ - GenerateModelInfo("Qwen/Qwen2-VL-2B-Instruct", hf_ppl=41081356.0), - GenerateModelInfo("Qwen/Qwen2.5-VL-3B-Instruct", hf_ppl=18330016.0), + ( + GenerateModelInfo("Qwen/Qwen2-VL-2B-Instruct", hf_ppl=9.810060501098633), + 28, + {}, + ), + ( + GenerateModelInfo("Qwen/Qwen2.5-VL-3B-Instruct", hf_ppl=22.17719078063965), + 28, + {}, + ), + ( + GenerateModelInfo("Qwen/Qwen3-VL-4B-Instruct", hf_ppl=11.916606903076172), + 32, + {}, + ), + ( + GenerateModelInfo("Qwen/Qwen3.5-4B", hf_ppl=6.478141784667969), + 32, + {"enable_prefix_caching": False}, + ), ] -mm_processor_kwargs = { - "min_pixels": 28 * 28, - "max_pixels": 1280 * 28 * 28, -} +def _pixel_factor(factor: int): + """`factor = patch_size * merge_size` for Qwen models""" + return {"min_pixels": factor**2, "max_pixels": 1280 * factor**2} -@pytest.mark.parametrize("model_info", MODELS) +@pytest.mark.parametrize("model_info,factor,extra_kwargs", MODELS) @pytest.mark.parametrize("mm_device_do_normalize", [True, False]) def test_ppl( - hf_runner, vllm_runner, model_info: GenerateModelInfo, mm_device_do_normalize: bool + hf_runner, + vllm_runner, + model_info: GenerateModelInfo, + factor: int, + extra_kwargs: dict, + mm_device_do_normalize: bool, ): vqa_ppl_test( hf_runner, vllm_runner, model_info, - vllm_extra_kwargs={"mm_device_do_normalize": mm_device_do_normalize}, - mm_processor_kwargs=mm_processor_kwargs, + vllm_extra_kwargs={ + "mm_device_do_normalize": mm_device_do_normalize, + **extra_kwargs, + }, + mm_processor_kwargs=_pixel_factor(factor), ) diff --git a/tests/models/multimodal/processing/test_gemma4.py b/tests/models/multimodal/processing/test_gemma4.py index 6430f40d8b..47dd0e55b6 100644 --- a/tests/models/multimodal/processing/test_gemma4.py +++ b/tests/models/multimodal/processing/test_gemma4.py @@ -202,19 +202,19 @@ def test_get_prompt_updates_respects_nested_max_soft_tokens(model_id: str): @pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID]) -@pytest.mark.parametrize("kwargs_on_init", [False, True]) @pytest.mark.parametrize( "image_kwargs", [{"rescale_factor": 1 / 127.5}, {"max_soft_tokens": 560}] ) @pytest.mark.parametrize("video_uuid", [None, "same-video"]) def test_video_cache_is_independent_of_image_kwargs( model_id: str, - kwargs_on_init: bool, image_kwargs: dict[str, object], video_uuid: str | None, ): - """Image overrides must not change video frames or depend on cache warmth.""" - kwargs = {"images_kwargs": image_kwargs} + """Request image overrides must not change video frames or make results + depend on cache warmth. + """ + request_kwargs = {"images_kwargs": image_kwargs} ctx = build_model_context( model_id, limit_mm_per_prompt={"image": 1, "video": 1}, @@ -239,18 +239,11 @@ def test_video_cache_is_independent_of_image_kwargs( cache=cache, ) - baseline = process({}, cache=None) - if kwargs_on_init: - ctx = build_model_context( - model_id, - mm_processor_kwargs=kwargs, - limit_mm_per_prompt={"image": 1, "video": 1}, - mm_processor_cache_gb=1, - ) - cache = MultiModalProcessorOnlyCache(ctx.model_config) - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) - request_kwargs = {} if kwargs_on_init else kwargs - process(request_kwargs, cache=cache) + # Populate the video cache without image overrides, then change only + # images_kwargs on the next request. Since images_kwargs are excluded from + # the video hash, and must not affect the processed video, the cached result + # must match a fresh recomputation. + baseline = process({}, cache=cache) cached = process(request_kwargs, cache=cache) cache.clear_cache() fresh = process(request_kwargs, cache=cache) diff --git a/tests/models/multimodal/processing/test_mistral3.py b/tests/models/multimodal/processing/test_mistral3.py index 4d900e627f..3445dd86fe 100644 --- a/tests/models/multimodal/processing/test_mistral3.py +++ b/tests/models/multimodal/processing/test_mistral3.py @@ -100,6 +100,11 @@ def _expected_placeholder_tokens_per_image( [ ({}, (448, 448), 256), ({"size": {"longest_edge": 1008}}, (1540, 1540), 1296), + ( + {"images_kwargs": {"size": {"longest_edge": 1008}}}, + (1540, 1540), + 1296, + ), ({"size": {"longest_edge": 1288}}, (1536, 1187), 1656), ({"size": {"longest_edge": 1008}}, (29, 29), 4), ({"size": {"longest_edge": 1000}}, (1540, 1700), 1188), @@ -148,6 +153,40 @@ def test_processor_size_override( assert prompt_update_tokens == hf_placeholder_tokens +def test_scoped_request_size_overrides_configured_flat_size(): + ctx = build_model_context( + _MODEL_ID, + mm_processor_kwargs={"size": {"longest_edge": 1008}}, + limit_mm_per_prompt={"image": 1}, + model_config_kwargs=_MODEL_CONFIG_KWARGS, + ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + hf_processor = AutoProcessor.from_pretrained( + _MODEL_ID, + fix_mistral_regex=True, + ) + + hf_processor_mm_kwargs: dict[str, object] = { + "images_kwargs": {"size": {"longest_edge": 1288}} + } + image = Image.new("RGB", (1536, 1187), color=(127, 127, 127)) + + merged_mm_kwargs = processor.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs) + pixel_values, hf_placeholder_tokens = _process_images_with_hf( + hf_processor, [image], merged_mm_kwargs + ) + prompt_update_tokens = _placeholder_tokens_from_prompt_updates( + processor, [image], pixel_values, hf_processor_mm_kwargs + ) + + expected_from_pixel_values = _expected_placeholder_tokens_per_image( + hf_processor, pixel_values[0] + ) + assert expected_from_pixel_values == 1656 + assert hf_placeholder_tokens.count(hf_processor.image_token_id) == 1656 + assert prompt_update_tokens == hf_placeholder_tokens + + def test_lightonocr_keeps_vision_config_image_size(): ctx = build_model_context( _LIGHTON_MODEL_ID, diff --git a/tests/models/multimodal/processing/test_qwen2_vl.py b/tests/models/multimodal/processing/test_qwen2_vl.py index 5ea538555e..b957ebf2c8 100644 --- a/tests/models/multimodal/processing/test_qwen2_vl.py +++ b/tests/models/multimodal/processing/test_qwen2_vl.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from copy import deepcopy +from types import SimpleNamespace from typing import Any import numpy as np @@ -8,16 +10,238 @@ import pytest import torch from PIL import Image +from vllm.config.multimodal import MultiModalConfig from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm +from vllm.model_executor.models.qwen2_vl import Qwen2VLProcessingInfo from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.cache import MultiModalProcessorOnlyCache from vllm.multimodal.inputs import batched_tensors_equal +from vllm.multimodal.processing.context import InputProcessingContext from vllm.platforms import current_platform from ....conftest import ImageTestAssets from ...utils import build_model_context +@pytest.mark.parametrize( + ("mm_kwargs", "expected"), + [ + ( + {"size": {"shortest_edge": 64, "longest_edge": 1024}}, + { + "size": {"shortest_edge": 64, "longest_edge": 1024}, + "min_pixels": 64, + "max_pixels": 1024, + }, + ), + ( + {"min_pixels": 64, "max_pixels": 1024}, + { + "min_pixels": 64, + "max_pixels": 1024, + "size": {"shortest_edge": 64, "longest_edge": 1024}, + }, + ), + ( + { + "size": {"shortest_edge": 64, "longest_edge": 1024}, + "min_pixels": 128, + "max_pixels": 2048, + }, + { + "size": {"shortest_edge": 64, "longest_edge": 1024}, + "min_pixels": 128, + "max_pixels": 2048, + }, + ), + ( + { + "min_pixels": None, + "size": {"shortest_edge": 0}, + "images_kwargs": {"max_pixels": 4096}, + "videos_kwargs": {"size": {"longest_edge": 8192}}, + }, + { + "min_pixels": None, + "size": {"shortest_edge": 0}, + "images_kwargs": { + "max_pixels": 4096, + "size": {"longest_edge": 4096}, + }, + "videos_kwargs": { + "size": {"longest_edge": 8192}, + "max_pixels": 8192, + }, + }, + ), + ( + {"min_pixels": 0}, + {"min_pixels": 0, "size": {"shortest_edge": 0}}, + ), + ( + {"size": None, "min_pixels": 64}, + {"size": {"shortest_edge": 64}, "min_pixels": 64}, + ), + ], +) +def test_complete_mm_processor_size_aliases( + mm_kwargs: dict[str, object], + expected: dict[str, object], +) -> None: + original = deepcopy(mm_kwargs) + + actual = Qwen2VLProcessingInfo._complete_mm_processor_size_aliases(mm_kwargs) + + assert actual == expected + assert mm_kwargs == original + + +def test_merge_and_resolve_mm_processor_kwargs_preserves_request_alias_precedence( + monkeypatch: pytest.MonkeyPatch, +) -> None: + mm_config = MultiModalConfig( + mm_processor_kwargs={ + "images_kwargs": {"min_pixels": 64}, + }, + mm_device_do_normalize=False, + ) + model_config = SimpleNamespace(get_multimodal_config=lambda: mm_config) + ctx = InputProcessingContext(model_config, tokenizer=None) # type: ignore[arg-type] + info = Qwen2VLProcessingInfo(ctx) + monkeypatch.setattr( + info, + "get_supported_mm_processor_kwargs", + lambda: {"images_kwargs": {"size", "min_pixels"}}, + ) + + assert info._merge_and_resolve_mm_processor_kwargs( + {"size": {"shortest_edge": 128}} + ) == { + "images_kwargs": { + "size": {"shortest_edge": 128}, + "min_pixels": 128, + }, + } + + +@pytest.mark.parametrize( + ("mm_kwargs", "default_size", "expected"), + [ + ( + {}, + {"shortest_edge": 32, "longest_edge": 1024}, + {"shortest_edge": 32, "longest_edge": 1024}, + ), + ( + {"size": {"shortest_edge": 64}}, + {"shortest_edge": 32, "longest_edge": 1024}, + {"shortest_edge": 64, "longest_edge": 1024}, + ), + ( + { + "size": {"shortest_edge": 64, "longest_edge": 2048}, + "min_pixels": 128, + "max_pixels": 4096, + }, + {"shortest_edge": 32, "longest_edge": 1024}, + {"shortest_edge": 128, "longest_edge": 4096}, + ), + ( + { + "size": {"shortest_edge": 64}, + "max_pixels": 4096, + }, + None, + {"shortest_edge": 64, "longest_edge": 4096}, + ), + ( + { + "size": {"shortest_edge": 64, "longest_edge": 2048}, + "min_pixels": None, + "max_pixels": 0, + }, + {"shortest_edge": 32, "longest_edge": 1024}, + {"shortest_edge": 64, "longest_edge": 0}, + ), + ], +) +def test_get_vision_size( + mm_kwargs: dict[str, object], + default_size: dict[str, object] | None, + expected: dict[str, object], +) -> None: + original_mm_kwargs = { + key: dict(value) if isinstance(value, dict) else value + for key, value in mm_kwargs.items() + } + original_default_size = dict(default_size) if default_size is not None else None + + actual = Qwen2VLProcessingInfo._get_vision_size( + mm_kwargs, + default_size=default_size, + ) + + assert actual == expected + assert mm_kwargs == original_mm_kwargs + assert default_size == original_default_size + + +@pytest.mark.parametrize( + ("modality", "scope"), + [ + ("image", "images_kwargs"), + ("video", "videos_kwargs"), + ], +) +def test_get_vision_info_reads_merged_modality_scope( + monkeypatch: pytest.MonkeyPatch, + modality: str, + scope: str, +) -> None: + vision_config = SimpleNamespace( + patch_size=14, + spatial_merge_size=2, + temporal_patch_size=2, + ) + ctx = SimpleNamespace( + get_hf_config=lambda *_: SimpleNamespace(vision_config=vision_config) + ) + info = Qwen2VLProcessingInfo(ctx) + merged = { + "images_kwargs": {"size": {"shortest_edge": 111}}, + "videos_kwargs": {"size": {"shortest_edge": 222}}, + } + monkeypatch.setattr( + info, + "_merge_and_resolve_mm_processor_kwargs", + lambda _: merged, + ) + + seen: list[dict[str, object]] = [] + + def get_size( + mm_kwargs: dict[str, object], + default_size: dict[str, object] | None = None, + ) -> dict[str, object]: + seen.append(mm_kwargs) + return dict(default_size or {}) + + monkeypatch.setattr(info, "_get_vision_size", get_size) + image_processor = SimpleNamespace(size={"shortest_edge": 32, "longest_edge": 1024}) + + info._get_vision_info( + image_width=28, + image_height=28, + num_frames=2, + do_resize=False, + image_processor=image_processor, + mm_kwargs={}, + modality=modality, # type: ignore[arg-type] + ) + + assert seen == [merged[scope]] + + def test_jina_vl_processing_order() -> None: """Jina's document-first prompt keeps cached features and hashes aligned.""" ctx = build_model_context( diff --git a/tests/models/multimodal/processing/test_qwen3_vl.py b/tests/models/multimodal/processing/test_qwen3_vl.py index 04f87049c9..2428109815 100644 --- a/tests/models/multimodal/processing/test_qwen3_vl.py +++ b/tests/models/multimodal/processing/test_qwen3_vl.py @@ -1,19 +1,21 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Regression tests for Qwen3-VL processor. - -Covers the fix for num_frames-based timestamp calculation -(issue vllm-project/vllm#35909). -""" +"""Regression tests for Qwen3-VL multimodal processing.""" +from copy import deepcopy +from types import SimpleNamespace from typing import Any import numpy as np import pytest +import torch +from transformers.models.qwen3_vl import Qwen3VLVideoProcessor from vllm.config import ModelConfig from vllm.config.multimodal import MultiModalDummyOptions +from vllm.model_executor.models.qwen3_vl import Qwen3VLProcessingInfo from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.processing.context import BaseProcessingInfo from ...registry import HF_EXAMPLE_MODELS from ...utils import build_model_context @@ -21,6 +23,32 @@ from ...utils import build_model_context MODEL_ID = "Qwen/Qwen3-VL-4B-Instruct" +def test_qwen3_video_routing_schema_adds_vllm_compatibility_kwargs( + monkeypatch: pytest.MonkeyPatch, +) -> None: + native_video_kwargs = set(Qwen3VLVideoProcessor.valid_kwargs.__annotations__) + assert "size" in native_video_kwargs + assert "min_pixels" not in native_video_kwargs + assert "max_pixels" not in native_video_kwargs + + info = Qwen3VLProcessingInfo(SimpleNamespace()) + parent_supported = { + "text_kwargs": {"padding"}, + "images_kwargs": {"size", "min_pixels", "max_pixels"}, + "videos_kwargs": set(native_video_kwargs), + } + monkeypatch.setattr( + BaseProcessingInfo, + "get_supported_mm_processor_kwargs", + lambda self: {scope: set(kwargs) for scope, kwargs in parent_supported.items()}, + ) + + supported = info.supported_mm_processor_kwargs + + assert {"min_pixels", "max_pixels"} <= supported["videos_kwargs"] + assert parent_supported["videos_kwargs"] == native_video_kwargs + + def _build_video_mm_data( num_frames: int, width: int = 128, @@ -45,6 +73,122 @@ def _build_video_mm_data( return {"video": [(video, metadata)]} +@pytest.mark.parametrize("model_id", [MODEL_ID]) +def test_processor_partial_scoped_image_size(model_id: str) -> None: + """Partial image ``size`` must be completed and applied before the HF call. + + Keep video compatibility and per-video kwargs in an image-only call as well: + Qwen3 must sanitize the video scope before HF validates it. Regression for + #56363. + """ + ctx = build_model_context( + model_id, + limit_mm_per_prompt={"image": 1, "video": 0}, + ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + + image = np.zeros((128, 128, 3), dtype=np.uint8) + min_pixels = 256 * 32 * 32 + default_size = dict(processor.info.get_image_processor().size) + complete_size = default_size | {"shortest_edge": min_pixels} + + def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor: + mm_items = processor.info.parse_mm_data({"image": [image]}) + return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[ + "image_grid_thw" + ] + + stock_grid = process({}) + complete_grid = process({"images_kwargs": {"size": complete_size}}) + partial_grid = process( + { + "images_kwargs": { + "size": {"shortest_edge": min_pixels}, + }, + "videos_kwargs": { + "min_pixels": 4096, + "max_pixels": 469_762_048, + "fps": [2.0], + "num_frames": [8], + }, + } + ) + + assert torch.equal(partial_grid, complete_grid) + assert not torch.equal(partial_grid, stock_grid) + + +@pytest.mark.parametrize("model_id", [MODEL_ID]) +def test_processor_partial_scoped_video_size(model_id: str) -> None: + """Partial scoped video ``size`` must be completed and applied before HF.""" + ctx = build_model_context( + model_id, + limit_mm_per_prompt={"image": 0, "video": 1}, + ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + + min_pixels = 512 * 32 * 32 + default_size = dict(processor.info.get_video_processor().size) + complete_size = default_size | {"shortest_edge": min_pixels} + + def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor: + mm_items = processor.info.parse_mm_data(_build_video_mm_data(num_frames=8)) + return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[ + "video_grid_thw" + ] + + stock_grid = process({}) + complete_grid = process({"videos_kwargs": {"size": complete_size}}) + partial_grid = process( + { + "videos_kwargs": { + "size": {"shortest_edge": min_pixels}, + }, + } + ) + + assert torch.equal(partial_grid, complete_grid) + assert not torch.equal(partial_grid, stock_grid) + + +@pytest.mark.parametrize("model_id", [MODEL_ID]) +def test_processor_flat_video_pixel_aliases_match_scoped_size(model_id: str) -> None: + """Flat min/max pixel aliases keep their historical Qwen3 video effect.""" + ctx = build_model_context( + model_id, + limit_mm_per_prompt={"image": 0, "video": 1}, + ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + pixel_budget = 512 * 32 * 32 + + def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor: + mm_items = processor.info.parse_mm_data(_build_video_mm_data(num_frames=8)) + return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[ + "video_grid_thw" + ] + + stock_grid = process({}) + scoped_grid = process( + { + "videos_kwargs": { + "size": { + "shortest_edge": pixel_budget, + "longest_edge": pixel_budget, + }, + }, + } + ) + flat_grid = process( + { + "min_pixels": pixel_budget, + "max_pixels": pixel_budget, + } + ) + + assert torch.equal(flat_grid, scoped_grid) + assert not torch.equal(flat_grid, stock_grid) + + @pytest.mark.parametrize("model_id", [MODEL_ID]) @pytest.mark.parametrize( "num_frames", @@ -189,20 +333,21 @@ def test_processor_kwargs_videos_kwargs_does_not_leak_into_image_budget( @pytest.mark.parametrize("model_id", [MODEL_ID]) @pytest.mark.parametrize( - "hf_mm_kwargs", - [{"num_frames": [8, 16]}, {"fps": [2.0, 4.0]}], + ("hf_mm_kwargs", "scalar_kwargs"), + [ + ({"num_frames": [8, 16]}, [{"num_frames": 8}, {"num_frames": 16}]), + ({"fps": [4.0, 12.0]}, [{"fps": 4.0}, {"fps": 12.0}]), + ], ) def test_processor_multi_video_list_kwargs( model_id: str, hf_mm_kwargs: dict[str, Any], + scalar_kwargs: list[dict[str, Any]], ) -> None: - """Regression test: a multi-video request with list-valued per-video - ``mm_processor_kwargs`` (one ``fps``/``num_frames`` per video) must not - crash. + """List-valued video kwargs must be scalarized per item without mutation. - Before the fix, ``_apply_hf_processor_main`` copied the whole kwargs to every - video without slicing, so ``_get_video_second_idx`` received the list - where a scalar was expected and raised ``TypeError``. + Each video in the multi-video request must match processing the same input + independently with its corresponding scalar ``fps``/``num_frames`` value. """ ctx = build_model_context( model_id, @@ -216,17 +361,36 @@ def test_processor_multi_video_list_kwargs( ) mm_data = { "video": [ - _build_video_mm_data(num_frames=16)["video"][0], + _build_video_mm_data(num_frames=32)["video"][0], _build_video_mm_data(num_frames=32)["video"][0], ] } + hf_mm_kwargs_before = deepcopy(hf_mm_kwargs) processed = processor( prompt, mm_items=processor.info.parse_mm_data(mm_data), hf_processor_mm_kwargs=hf_mm_kwargs, ) + def process_single(hf_kwargs: dict[str, Any]) -> torch.Tensor: + single_prompt = "<|vision_start|><|video_pad|><|vision_end|>" + single_mm_data = _build_video_mm_data(num_frames=32) + single = processor( + single_prompt, + mm_items=processor.info.parse_mm_data(single_mm_data), + hf_processor_mm_kwargs=hf_kwargs, + ) + return single["mm_kwargs"].get_data()["video_grid_thw"][0] + + expected_grids = [process_single(kwargs) for kwargs in scalar_kwargs] + multi_grids = processed["mm_kwargs"].get_data()["video_grid_thw"] + + assert not torch.equal(expected_grids[0], expected_grids[1]) + assert torch.equal(multi_grids[0], expected_grids[0]) + assert torch.equal(multi_grids[1], expected_grids[1]) + assert hf_mm_kwargs == hf_mm_kwargs_before + video_phs = processed["mm_placeholders"].get("video", []) assert len(video_phs) == 2, ( f"Expected exactly 2 video placeholders, got {len(video_phs)}" @@ -336,7 +500,12 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non capped_ctx = build_model_context( model_id, - mm_processor_kwargs={"size": size, "cap_pixels_per_frame": True}, + mm_processor_kwargs={ + "videos_kwargs": { + "size": size, + "cap_pixels_per_frame": True, + }, + }, limit_mm_per_prompt={"image": 0, "video": 1}, ) capped = MULTIMODAL_REGISTRY.create_processor(capped_ctx.model_config) @@ -350,7 +519,7 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non uncapped_ctx = build_model_context( model_id, - mm_processor_kwargs={"size": size}, + mm_processor_kwargs={"videos_kwargs": {"size": size}}, limit_mm_per_prompt={"image": 0, "video": 1}, ) uncapped = MULTIMODAL_REGISTRY.create_processor(uncapped_ctx.model_config) diff --git a/tests/models/qwen4_exp/test_hc_ops.py b/tests/models/qwen4_exp/test_hc_ops.py index cb1a85760e..cdad1cec13 100644 --- a/tests/models/qwen4_exp/test_hc_ops.py +++ b/tests/models/qwen4_exp/test_hc_ops.py @@ -4,11 +4,13 @@ import pytest import torch +from vllm.models.qwen4_exp.nvidia.ops.cute_dsl.hc_down_silu import hc_down_silu from vllm.models.qwen4_exp.nvidia.ops.hc import ( grouped_gemma_rmsnorm, hc_combine, hc_combine_norm, hc_gate_mix, + hc_silu, ) from vllm.platforms import current_platform from vllm.triton_utils import HAS_TRITON @@ -22,6 +24,13 @@ HC = 4 HIDDEN_SIZE = 2560 HYPER_HIDDEN_SIZE = HC * HIDDEN_SIZE EPS = 1e-6 +LORA_RANK = 320 +DOWN_N = LORA_RANK + HC + 12 # merged down+inject weight, 16-row padded + +requires_sm90 = pytest.mark.skipif( + not current_platform.has_device_capability(90), + reason="fused HC down+SiLU requires SM90+", +) def test_grouped_gemma_rmsnorm() -> None: @@ -125,3 +134,24 @@ def test_hc_combine_norm_unit_injection(num_tokens: int) -> None: expected_norm = grouped_gemma_rmsnorm(expected, weight, EPS, HC) assert torch.equal(actual, expected) torch.testing.assert_close(actual_norm, expected_norm) + + +@requires_sm90 +@pytest.mark.parametrize("num_tokens", [1, 3, 5, 17, 48]) +def test_hc_down_silu_fused(num_tokens: int) -> None: + # Compare computed columns with the unfused ll_bf16 + hc_silu reference. + from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import ll_bf16_gemm + + torch.manual_seed(0) + x = torch.randn(num_tokens, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda") + weight = torch.randn(DOWN_N, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda") + + lora, injection = hc_down_silu(x, weight, LORA_RANK, HC) + + down = ll_bf16_gemm(x, weight).to(torch.bfloat16) + torch.testing.assert_close( + lora, hc_silu(down[:, :LORA_RANK], HC), rtol=0.01, atol=0.01 + ) + torch.testing.assert_close( + injection, down[:, LORA_RANK : LORA_RANK + HC], rtol=0.01, atol=0.01 + ) diff --git a/tests/models/qwen4_exp/test_ple.py b/tests/models/qwen4_exp/test_ple.py index 88aae40af1..e871bf42d2 100644 --- a/tests/models/qwen4_exp/test_ple.py +++ b/tests/models/qwen4_exp/test_ple.py @@ -20,6 +20,9 @@ import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module import vllm.model_executor.parameter as parameter_module import vllm.models.qwen4_exp.common.ngram_embedding as ngram_embedding_module import vllm.models.qwen4_exp.nvidia.ngram_embedding as nvidia_ngram_embedding_module +from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata +from vllm.config import SpeculativeConfig, VllmConfig +from vllm.config.compilation import CUDAGraphMode from vllm.config.quantization import QuantizationConfigArgs from vllm.model_executor.layers.quantization.fp8 import Fp8Config from vllm.model_executor.layers.quantization.modelopt import ( @@ -50,8 +53,125 @@ from vllm.models.qwen4_exp.nvidia.ple_layer import Qwen4ExpPLELayer from vllm.utils.torch_utils import weak_ref_tensor from vllm.v1.attention.backends.short_conv_attn import ( PleShortConvAttentionMetadata, + PleShortConvAttentionMetadataBuilder, ) from vllm.v1.attention.backends.utils import NULL_BLOCK_ID +from vllm.v1.kv_cache_interface import MambaSpec + + +@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("full_graph", [False, True]) +@pytest.mark.parametrize( + "query_lens,drafts,spec_rows,non_spec_rows,token_order", + [ + pytest.param([4, 2, 0], [3, 1, -1], [0, 1], [], None, id="spec-padding"), + pytest.param( + [3, 1, 2, 5, 0], + [2, -1, 1, -1, -1], + [0, 2], + [1, 3], + [0, 1, 2, 4, 5, 3, 6, 7, 8, 9, 10], + id="mixed-padding", + ), + pytest.param([1, 5], [-1, -1], [], [0, 1], None, id="non-spec"), + ], +) +def test_ple_metadata_preserves_request_and_token_order( + device, full_graph, query_lens, drafts, spec_rows, non_spec_rows, token_order +): + """Preserve state routing, acceptance and prefill offsets without GPU sync.""" + if device == "cuda" and not torch.cuda.is_available(): + pytest.skip("CUDA required") + device = torch.device(device) + config = VllmConfig() + config.speculative_config = SpeculativeConfig( + method="ngram", num_speculative_tokens=3 + ) + config.compilation_config.cudagraph_mode = ( + CUDAGraphMode.FULL if full_graph else CUDAGraphMode.NONE + ) + config.compilation_config.max_cudagraph_capture_size = 32 + builder = PleShortConvAttentionMetadataBuilder( + MambaSpec(block_size=16, shapes=((16, 64),), dtypes=(torch.float32,)), + ["ple"], + config, + device, + ) + common = create_common_attn_metadata( + BatchSpec(seq_lens=[q + 16 for q in query_lens], query_lens=query_lens), + 16, + device, + arange_block_indices=True, + ) + common.is_prefilling = torch.tensor( + [q > 1 and d < 0 for q, d in zip(query_lens, drafts)] + ) + # Token padding is independent of the sum of per-request query lengths. + common.num_actual_tokens += 3 + accepted = torch.arange(1, len(query_lens) + 1, dtype=torch.int32, device=device) + drafts_cpu = torch.tensor(drafts, dtype=torch.int32) + sync_mode = torch.cuda.get_sync_debug_mode() if device.type == "cuda" else None + if sync_mode is not None: + torch.cuda.set_sync_debug_mode("error") + try: + metadata = builder.build( + 0, + common, + num_accepted_tokens=accepted, + num_decode_draft_tokens_cpu=drafts_cpu, + ) + finally: + if sync_mode is not None: + torch.cuda.set_sync_debug_mode(sync_mode) + + def check(actual, expected): + torch.testing.assert_close( + actual, torch.tensor(expected, dtype=actual.dtype, device=device) + ) + + assert metadata.num_spec_decodes == len(spec_rows) + assert metadata.num_decodes == sum(query_lens[i] == 1 for i in non_spec_rows) + assert metadata.num_prefills == sum(query_lens[i] > 1 for i in non_spec_rows) + if spec_rows: + torch.testing.assert_close( + metadata.spec_state_indices_tensor[: len(spec_rows)], + common.block_table_tensor[spec_rows, 0], + ) + torch.testing.assert_close( + metadata.num_accepted_tokens[: len(spec_rows)], accepted[spec_rows] + ) + check( + metadata.spec_query_start_loc[: len(spec_rows) + 1], + [0, *accumulate(query_lens[i] for i in spec_rows)], + ) + if full_graph and not non_spec_rows: + check(metadata.spec_state_indices_tensor[len(spec_rows) :], [NULL_BLOCK_ID]) + check(metadata.num_accepted_tokens[len(spec_rows) :], [1]) + check( + metadata.spec_query_start_loc[len(spec_rows) + 1 :], [sum(query_lens)] + ) + assert ( + metadata.spec_state_indices_tensor.data_ptr() + == builder.spec_state_indices_tensor.data_ptr() + ) + if non_spec_rows: + torch.testing.assert_close( + metadata.state_indices_tensor, common.block_table_tensor[non_spec_rows, 0] + ) + prefill_lens = [query_lens[i] for i in non_spec_rows if query_lens[i] > 1] + check(metadata.query_start_loc_p, [0, *accumulate(prefill_lens)]) + check(metadata.has_initial_states_p, [True] * len(prefill_lens)) + if token_order is not None: + check( + torch.cat((metadata.spec_token_indx, metadata.non_spec_token_indx)), + token_order, + ) + else: + assert metadata.spec_token_indx is None + assert metadata.non_spec_token_indx is None + assert metadata.nums_dict is None + assert metadata.batch_ptr is None + assert metadata.token_chunk_offset_ptr is None def _mock_etp_group( @@ -1892,6 +2012,11 @@ def _make_conv_metadata( else None ), non_spec_query_start_loc=(non_spec_query_start_loc if has_non_spec else None), + query_start_loc_p=( + non_spec_query_start_loc[case.num_decodes :] - case.num_decodes + if num_prefills + else None + ), has_initial_states_p=( torch.arange(num_prefills, device=device) % 2 == 0 if num_prefills else None ), diff --git a/tests/models/qwen4_exp/test_qsa_reference.py b/tests/models/qwen4_exp/test_qsa_reference.py index b3bc0d1cae..a6f3b6557d 100644 --- a/tests/models/qwen4_exp/test_qsa_reference.py +++ b/tests/models/qwen4_exp/test_qsa_reference.py @@ -17,6 +17,7 @@ from vllm.models.qwen4_exp.nvidia.ops import qsa as qsa_ops from vllm.models.qwen4_exp.nvidia.ops import qsa_indexer as qsa_indexer_ops from vllm.platforms import current_platform from vllm.triton_utils import HAS_TRITON +from vllm.v1.worker.utils import clear_layer_kv_caches requires_qsa_kernels = pytest.mark.skipif( not current_platform.is_cuda() or not HAS_TRITON, @@ -365,7 +366,9 @@ def test_qsa_circular_buffer_survives_one_speculative_step(chunk_start: int) -> assert set(slots.tolist()).isdisjoint((committed % capacity).tolist()) -def _qsa_key_cache(block_size: int, compress_ratio: int) -> qsa_cache.QSAKeyStateCache: +def _qsa_key_cache( + block_size: int, compress_ratio: int, **kwargs +) -> qsa_cache.QSAKeyStateCache: return qsa_cache.QSAKeyStateCache( head_size=64, dtype=torch.bfloat16, @@ -375,6 +378,7 @@ def _qsa_key_cache(block_size: int, compress_ratio: int) -> qsa_cache.QSAKeyStat compilation_config=SimpleNamespace(static_forward_context={}) ), compress_ratio=compress_ratio, + **kwargs, ) @@ -402,6 +406,23 @@ def test_qsa_state_caches_adapt_the_unified_logical_layout() -> None: assert compressed_cache.kv_cache.data_ptr() == compressed_view.data_ptr() +def test_clearing_qsa_key_cache_releases_its_storage() -> None: + """Profiling teardown clears `kv_cache`; no derived view may keep it alive.""" + raw_cache = _qsa_key_cache( + block_size=32, compress_ratio=4, cache_rope_positions=True + ) + kv = torch.zeros(2, 1, 8, raw_cache.head_size, dtype=torch.bfloat16) + references_before_bind = torch._C._storage_Use_Count(kv.untyped_storage()._cdata) + raw_cache.bind_kv_cache(kv) + assert raw_cache.key_cache.shape[-1] == 64 + assert raw_cache.rope_position_cache.dtype == torch.int64 + + clear_layer_kv_caches([raw_cache]) + + references = torch._C._storage_Use_Count(kv.untyped_storage()._cdata) + assert references == references_before_bind + + @pytest.mark.parametrize( ("compress_ratio", "num_spec", "expected"), [(4, 0, 4), (4, 1, 8), (4, 3, 8), (4, 4, 8), (4, 5, 12), (2, 3, 6)], diff --git a/tests/multimodal/test_processing.py b/tests/multimodal/test_processing.py index 0bd19a0ea0..485ea31659 100644 --- a/tests/multimodal/test_processing.py +++ b/tests/multimodal/test_processing.py @@ -3,18 +3,23 @@ import time from contextlib import nullcontext +from copy import deepcopy +from types import SimpleNamespace +from typing import TypedDict import numpy as np import pytest from vllm.config import ModelConfig, SchedulerConfig +from vllm.config.multimodal import MultiModalConfig from vllm.exceptions import VLLMValidationError from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.hasher import MultiModalHasher from vllm.multimodal.parse import MultiModalDataParser from vllm.multimodal.processing.context import ( + BaseProcessingInfo, InputProcessingContext, - overlay_modality_mm_kwargs, + _resolve_mm_processor_kwargs, ) from vllm.multimodal.processing.inputs import ProcessorInputs from vllm.multimodal.processing.processor import ( @@ -1207,58 +1212,305 @@ def test_apply_prompt_updates_falls_back_with_index_targets(): assert [p.tokens for p in placeholders["image"]] == [[200, 201], [9]] -@pytest.mark.skip_global_cleanup -def test_overlay_modality_mm_kwargs_scoped_video_does_not_leak_to_image(): - """HF-style videos_kwargs must overlay only when modality is video.""" - video_size = {"longest_edge": 469762048, "shortest_edge": 4096} - kwargs = {"videos_kwargs": {"size": video_size}} - - assert overlay_modality_mm_kwargs(kwargs, None) == kwargs - assert "size" not in overlay_modality_mm_kwargs(kwargs, "image") - assert overlay_modality_mm_kwargs(kwargs, "video")["size"] == video_size +class _TextProcessorKwargs(TypedDict, total=False): + padding: bool -@pytest.mark.skip_global_cleanup -def test_overlay_modality_mm_kwargs_flat_size_stays_shared(): - """A flat size override keeps the current shared-namespace behavior.""" - size = {"longest_edge": 469762048, "shortest_edge": 4096} - kwargs = {"size": size} - - for modality in (None, "image", "video"): - assert overlay_modality_mm_kwargs(kwargs, modality)["size"] == size +class _AudioProcessorKwargs(TypedDict, total=False): + sampling_rate: int -@pytest.mark.skip_global_cleanup -def test_overlay_modality_mm_kwargs_scoped_wins_over_flat_for_modality(): - """A nested videos_kwargs size wins over a flat size for video reads.""" +class _ProcessorKwargs(TypedDict, total=False): + text_kwargs: _TextProcessorKwargs + audio_kwargs: _AudioProcessorKwargs + + +class _ImageProcessorKwargs(TypedDict, total=False): + size: dict[str, int] + min_pixels: int + + +class _VideoProcessorKwargs(TypedDict, total=False): + size: dict[str, int] + fps: float + + +@pytest.mark.parametrize( + ("supported_mm_limits", "processor", "expected"), + [ + ( + {"image": None, "video": None}, + SimpleNamespace( + valid_processor_kwargs=_ProcessorKwargs, + image_processor=SimpleNamespace(valid_kwargs=_ImageProcessorKwargs), + video_processor=SimpleNamespace(valid_kwargs=_VideoProcessorKwargs), + ), + { + "text_kwargs": {"padding"}, + "images_kwargs": {"size", "min_pixels"}, + "videos_kwargs": {"size", "fps"}, + }, + ), + ( + {"image": None}, + SimpleNamespace( + valid_processor_kwargs=_ProcessorKwargs, + image_processor=SimpleNamespace(valid_kwargs=_ImageProcessorKwargs), + ), + { + "text_kwargs": {"padding"}, + "images_kwargs": {"size", "min_pixels"}, + }, + ), + ( + {"audio": None}, + SimpleNamespace(valid_processor_kwargs=_ProcessorKwargs), + { + "text_kwargs": {"padding"}, + "audio_kwargs": {"sampling_rate"}, + }, + ), + ], +) +def test_get_supported_mm_processor_kwargs_uses_supported_modalities( + monkeypatch: pytest.MonkeyPatch, + supported_mm_limits: dict[str, int | None], + processor: SimpleNamespace, + expected: dict[str, set[str]], +) -> None: + info = BaseProcessingInfo(SimpleNamespace()) + info.__dict__["supported_mm_limits"] = supported_mm_limits + monkeypatch.setattr(info, "get_hf_processor", lambda **_: processor) + + assert info.get_supported_mm_processor_kwargs() == expected + + +def test_supported_mm_processor_kwargs_is_cached( + monkeypatch: pytest.MonkeyPatch, +) -> None: + info = BaseProcessingInfo(SimpleNamespace()) + expected = {"images_kwargs": {"size"}} + calls = 0 + + def get_supported_mm_processor_kwargs() -> dict[str, set[str]]: + nonlocal calls + calls += 1 + return expected + + monkeypatch.setattr( + info, + "get_supported_mm_processor_kwargs", + get_supported_mm_processor_kwargs, + ) + + first = info.supported_mm_processor_kwargs + second = info.supported_mm_processor_kwargs + + assert first is expected + assert second is first + assert calls == 1 + + +def test_resolve_mm_processor_kwargs_routes_supported_flat_kwargs(): kwargs = { - "size": {"longest_edge": 1}, - "videos_kwargs": {"size": {"longest_edge": 2}}, - "images_kwargs": {"size": {"longest_edge": 3}}, + "size": {"shortest_edge": 64}, + "padding": True, + "sampling_rate": 16000, + "unknown": "keep", + "images_kwargs": { + "size": {"longest_edge": 1024}, + "custom_image_kwarg": "keep", + }, + } + kwargs_before = deepcopy(kwargs) + + resolved = _resolve_mm_processor_kwargs( + kwargs, + supported_mm_processor_kwargs={ + "text_kwargs": {"padding"}, + "images_kwargs": {"size"}, + "videos_kwargs": {"size"}, + "audio_kwargs": {"sampling_rate"}, + }, + ) + + assert resolved == { + "unknown": "keep", + "text_kwargs": {"padding": True}, + "images_kwargs": { + "size": { + "shortest_edge": 64, + "longest_edge": 1024, + }, + "custom_image_kwarg": "keep", + }, + "videos_kwargs": { + "size": {"shortest_edge": 64}, + }, + "audio_kwargs": {"sampling_rate": 16000}, + } + assert kwargs == kwargs_before + + video_kwargs = resolved["videos_kwargs"] + image_kwargs = resolved["images_kwargs"] + assert isinstance(video_kwargs, dict) + assert isinstance(image_kwargs, dict) + video_size = video_kwargs["size"] + image_size = image_kwargs["size"] + assert isinstance(video_size, dict) + assert isinstance(image_size, dict) + video_size["shortest_edge"] = 999 + + assert image_size["shortest_edge"] == 64 + assert kwargs == kwargs_before + + +def test_resolve_mm_processor_kwargs_rejects_non_mapping_destination_scope(): + with pytest.raises(TypeError, match="images_kwargs"): + _resolve_mm_processor_kwargs( + { + "size": {"shortest_edge": 64}, + "images_kwargs": 123, + }, + supported_mm_processor_kwargs={ + "images_kwargs": {"size"}, + }, + ) + + +def test_resolve_mm_processor_kwargs_without_schema_deduplicates_existing_scopes(): + kwargs = { + "size": {"shortest_edge": 64}, + "num_frames": 16, + "fps": 4, + "images_kwargs": { + "size": {"longest_edge": 1024}, + }, + "videos_kwargs": { + "num_frames": 8, + }, + "audio_kwargs": 123, + "common_kwargs": { + "fps": 2, + }, + } + kwargs_before = deepcopy(kwargs) + + resolved = _resolve_mm_processor_kwargs(kwargs) + + assert resolved == { + "fps": 4, + "images_kwargs": { + "size": {"longest_edge": 1024}, + }, + "videos_kwargs": { + "num_frames": 8, + }, + "audio_kwargs": 123, + "common_kwargs": { + "fps": 2, + }, + } + assert kwargs == kwargs_before + assert "text_kwargs" not in resolved + + +@pytest.mark.parametrize( + "inference_kwargs", + [ + {"size": {"shortest_edge": 128}}, + {"size": {"shortest_edge": 128}, "images_kwargs": None}, + {"size": {"shortest_edge": 128}, "images_kwargs": {}}, + ], +) +def test_get_merged_mm_kwargs_treats_empty_scopes_as_absent_before_routing( + inference_kwargs: dict[str, object], +): + mm_config = MultiModalConfig( + mm_processor_kwargs={ + "size": { + "shortest_edge": 64, + "longest_edge": 512, + }, + "images_kwargs": { + "size": {"longest_edge": 1024}, + }, + }, + mm_device_do_normalize=False, + ) + model_config = SimpleNamespace(get_multimodal_config=lambda: mm_config) + ctx = InputProcessingContext(model_config, tokenizer=None) # type: ignore[arg-type] + + merged = ctx.get_merged_mm_kwargs( + inference_kwargs, + supported_mm_processor_kwargs={ + "images_kwargs": {"size"}, + "videos_kwargs": {"size"}, + }, + ) + + assert merged == { + "images_kwargs": { + "size": { + "shortest_edge": 128, + "longest_edge": 1024, + }, + }, + "videos_kwargs": { + "size": { + "shortest_edge": 128, + "longest_edge": 512, + }, + }, } - assert overlay_modality_mm_kwargs(kwargs, "video")["size"] == {"longest_edge": 2} - assert overlay_modality_mm_kwargs(kwargs, "image")["size"] == {"longest_edge": 3} - assert overlay_modality_mm_kwargs(kwargs, None)["size"] == {"longest_edge": 1} +def test_get_merged_mm_kwargs_merges_before_routing(): + mm_config = MultiModalConfig( + mm_processor_kwargs={ + "size": { + "shortest_edge": 64, + "longest_edge": 512, + }, + "images_kwargs": { + "size": {"longest_edge": 1024}, + }, + }, + mm_device_do_normalize=False, + ) + model_config = SimpleNamespace(get_multimodal_config=lambda: mm_config) + ctx = InputProcessingContext(model_config, tokenizer=None) # type: ignore[arg-type] -@pytest.mark.skip_global_cleanup -def test_overlay_modality_mm_kwargs_ignores_non_mapping_scoped_value(): - kwargs = {"images_kwargs": "not-a-dict", "size": {"longest_edge": 1}} - assert overlay_modality_mm_kwargs(kwargs, "image")["size"] == {"longest_edge": 1} + merged = ctx.get_merged_mm_kwargs( + { + "size": { + "shortest_edge": 128, + "longest_edge": 768, + }, + "images_kwargs": { + "size": {"longest_edge": 2048}, + }, + }, + supported_mm_processor_kwargs={ + "images_kwargs": {"size"}, + "videos_kwargs": {"size"}, + }, + ) - -@pytest.mark.skip_global_cleanup -def test_mm_processor_kwargs_merge_then_overlay_preserves_scoping(): - """Configured videos_kwargs overlay only for video reads after merge.""" - from vllm.config.multimodal import MultiModalConfig - - size = {"longest_edge": 469762048, "shortest_edge": 4096} - mm_config = MultiModalConfig(mm_processor_kwargs={"videos_kwargs": {"size": size}}) - merged = mm_config.merge_mm_processor_kwargs({}) - assert overlay_modality_mm_kwargs(merged, "video")["size"] == size - assert "size" not in overlay_modality_mm_kwargs(merged, "image") - assert "size" not in overlay_modality_mm_kwargs(merged, None) + assert merged == { + "images_kwargs": { + "size": { + "shortest_edge": 128, + "longest_edge": 2048, + }, + }, + "videos_kwargs": { + "size": { + "shortest_edge": 128, + "longest_edge": 768, + }, + }, + } def test_processor_inputs_hashes_partial_uuids(): diff --git a/tests/parser/engine/test_plamo3.py b/tests/parser/engine/test_plamo3.py new file mode 100644 index 0000000000..7b6339cd35 --- /dev/null +++ b/tests/parser/engine/test_plamo3.py @@ -0,0 +1,556 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""PLaMo3 grammar, truncation, and serving-adapter regressions.""" + +from __future__ import annotations + +import json + +import pytest +import regex as re + +from tests.parser.engine.conftest import make_mock_tokenizer +from tests.parser.engine.replay_harness import ( + DUMMY_TOOLS, + _test_request, + collect_output, + replay_streaming, + replay_with_text_holdback, +) +from vllm.parser.parser_manager import ParserManager +from vllm.parser.plamo3 import ( + BEGIN_THINK, + BEGIN_TOOL_ARGUMENTS, + BEGIN_TOOL_NAME, + BEGIN_TOOL_REQUEST, + BEGIN_TOOL_REQUESTS, + END_THINK, + END_TOOL_ARGUMENTS, + END_TOOL_NAME, + END_TOOL_REQUEST, + END_TOOL_REQUESTS, + EOT, + PLAMO_MARKER_TOKENS, + Plamo3Parser, +) + + +def _tool_header(name="weather"): + return ( + BEGIN_TOOL_REQUEST + + BEGIN_TOOL_NAME + + name + + END_TOOL_NAME + + BEGIN_TOOL_ARGUMENTS + ) + + +def _tool_call(name, arguments): + return _tool_header(name) + arguments + END_TOOL_ARGUMENTS + END_TOOL_REQUEST + + +@pytest.fixture +def mock_tokenizer(): + special = [*sorted(PLAMO_MARKER_TOKENS), EOT, "<|plamo:bos|>"] + atoms = [*special, "<|plamo:constrain|>", "<|plamo:msg|>"] + vocab = {token: 100 + i for i, token in enumerate(atoms)} + inverse = {i: token for token, i in vocab.items()} + pattern = re.compile("|".join(map(re.escape, atoms)) + r"|[\s\S]") + tokenizer = make_mock_tokenizer(vocab, special_tokens=special) + tokenizer.encode.side_effect = lambda text, **kwargs: [ + vocab[piece] if piece in vocab else 1000 + ord(piece) + for piece in pattern.findall(text) + ] + tokenizer.decode.side_effect = lambda ids, **kwargs: "".join( + inverse[i] if i in inverse else chr(i - 1000) for i in ids + ) + return tokenizer + + +@pytest.fixture(params=["direct", "registered"]) +def parser_cls(request): + if request.param == "direct": + return Plamo3Parser + return ParserManager.get_parser( + tool_parser_name="plamo3", + reasoning_parser_name="plamo3", + enable_auto_tools=True, + ) + + +def _stream(parser, tokenizer, output, chunk_size=1, finished_on_last=False): + tokens = [ + (i, tokenizer.decode([i])) + for i in tokenizer.encode(output, add_special_tokens=False) + ] + deltas = replay_streaming( + parser, + tokens, + chunk_size=chunk_size, + finished_on_last=finished_on_last, + tools=DUMMY_TOOLS, + ) + if not finished_on_last: + deltas.append( + parser.parse_delta("", [], _test_request(DUMMY_TOOLS), finished=True) + ) + return deltas + + +@pytest.mark.parametrize("thinking", [False, True]) +@pytest.mark.parametrize("with_markers", [False, True]) +def test_reasoning_mode( + parser_cls, mock_tokenizer, mock_request, thinking, with_markers +): + output = ( + BEGIN_THINK + "Reasoning." + END_THINK + "Answer." + if with_markers + else "Answer." + ) + if thinking: + expected = ("Reasoning.", "Answer.") if with_markers else ("Answer.", "") + else: + expected = ("", output) + kwargs = {"chat_template_kwargs": {"enable_thinking": thinking}} + reasoning, content, calls = parser_cls(mock_tokenizer, **kwargs).parse( + output, mock_request + ) + assert (reasoning or "", content or "") == expected + assert not calls + result = collect_output( + _stream(parser_cls(mock_tokenizer, **kwargs), mock_tokenizer, output) + ) + assert (result.reasoning, result.content) == expected + assert not result.tool_calls + + +def test_duplicate_think_end_does_not_leak_into_content( + parser_cls, mock_tokenizer, mock_request +): + output = END_THINK + "A" + END_THINK + "B" + reasoning, content, calls = parser_cls(mock_tokenizer).parse(output, mock_request) + assert (reasoning, content) == (None, "AB") + assert not calls + + +def test_content_can_start_a_new_reasoning_block( + parser_cls, mock_tokenizer, mock_request +): + output = END_THINK + "A" + BEGIN_THINK + "R" + END_THINK + reasoning, content, calls = parser_cls(mock_tokenizer).parse(output, mock_request) + assert (reasoning, content) == ("R", "A") + assert not calls + + +@pytest.mark.parametrize("chunk_size", [1, 7, None]) +@pytest.mark.parametrize("finished_on_last", [False, True]) +@pytest.mark.parametrize( + "closing", + [ + "", + END_TOOL_ARGUMENTS + END_TOOL_REQUEST, + END_TOOL_ARGUMENTS + END_TOOL_REQUEST + END_TOOL_REQUESTS + EOT, + ], +) +def test_complete_arguments_survive_missing_closers( + parser_cls, mock_tokenizer, mock_request, chunk_size, finished_on_last, closing +): + output = ( + BEGIN_THINK + + "Reasoning. " + + END_THINK + + "Before tools. " + + BEGIN_TOOL_REQUESTS + + _tool_header() + + '{"city":"東京"}' + + closing + ) + reasoning, content, calls = parser_cls(mock_tokenizer).parse( + output, mock_request, enable_auto_tools=True + ) + assert (reasoning, content) == ("Reasoning. ", "Before tools. ") + assert calls and [(c.name, json.loads(c.arguments)) for c in calls] == [ + ("weather", {"city": "東京"}) + ] + result = collect_output( + _stream( + parser_cls(mock_tokenizer), + mock_tokenizer, + output, + chunk_size, + finished_on_last, + ) + ) + assert (result.reasoning, result.content) == (reasoning, content) + assert result.tool_calls == [{"name": "weather", "arguments": '{"city":"東京"}'}] + + +def test_tool_name_cutoffs_do_not_leak_markers( + parser_cls, mock_tokenizer, mock_request +): + header = BEGIN_TOOL_REQUEST + BEGIN_TOOL_NAME + "weather" + END_TOOL_NAME + header_ids = mock_tokenizer.encode(header, add_special_tokens=False) + name_start = len( + mock_tokenizer.encode( + BEGIN_TOOL_REQUEST + BEGIN_TOOL_NAME, add_special_tokens=False + ) + ) + for end in range(len(header_ids)): + partial = mock_tokenizer.decode(header_ids[:end]) + output = END_THINK + "Before tools." + BEGIN_TOOL_REQUESTS + partial + reasoning, content, calls = parser_cls(mock_tokenizer).parse( + output, mock_request, enable_auto_tools=True + ) + assert (reasoning, content) == (None, "Before tools."), end + if end <= name_start: + expected_name = "" + else: + name_text = mock_tokenizer.decode(header_ids[name_start:end]) + if "<|plamo:" in name_text: + name_text = name_text[: name_text.rfind("<|plamo:")] + expected_name = name_text[: len("weather")] + assert [call.name for call in calls or []] == ( + [expected_name] if expected_name else [] + ), end + result = collect_output( + _stream(parser_cls(mock_tokenizer), mock_tokenizer, output) + ) + assert [call["name"] for call in result.tool_calls] == ( + [expected_name] if expected_name else [] + ), end + assert all("<" not in call["name"] for call in result.tool_calls), end + assert result.content == "Before tools.", end + + +@pytest.mark.parametrize("chunk_size", [1, 7, None]) +@pytest.mark.parametrize("finished_on_last", [False, True]) +def test_all_argument_closer_cutoffs_do_not_leak( + parser_cls, mock_tokenizer, mock_request, chunk_size, finished_on_last +): + head = END_THINK + BEGIN_TOOL_REQUESTS + _tool_header() + closer_ids = mock_tokenizer.encode(END_TOOL_ARGUMENTS, add_special_tokens=False) + for end in range(len(closer_ids)): + partial = mock_tokenizer.decode(closer_ids[:end]) + output = head + '{"city":"東京"}' + partial + _, _, calls = parser_cls(mock_tokenizer).parse( + output, mock_request, enable_auto_tools=True + ) + assert calls and calls[0].arguments == '{"city":"東京"}', end + result = collect_output( + _stream( + parser_cls(mock_tokenizer), + mock_tokenizer, + output, + chunk_size, + finished_on_last, + ) + ) + assert result.tool_calls == [ + {"name": "weather", "arguments": '{"city":"東京"}'} + ], end + + +@pytest.mark.parametrize( + "arguments", + [ + "{}", + '{"name":"Alice","arguments":{"x":1},"parameters":true}', + '{"text":"literal <|plamo:end_', + ], +) +def test_arguments_are_not_rewritten( + parser_cls, mock_tokenizer, mock_request, arguments +): + output = END_THINK + BEGIN_TOOL_REQUESTS + _tool_header() + arguments + _, _, calls = parser_cls(mock_tokenizer).parse( + output, mock_request, enable_auto_tools=True + ) + assert calls and calls[0].arguments == arguments + result = collect_output(_stream(parser_cls(mock_tokenizer), mock_tokenizer, output)) + assert result.tool_calls == [{"name": "weather", "arguments": arguments}] + + +@pytest.mark.parametrize("truncated_second", [False, True]) +def test_multiple_calls_keep_indices_ids_and_arguments_separate( + parser_cls, mock_tokenizer, mock_request, truncated_second +): + second = ( + BEGIN_TOOL_REQUEST + BEGIN_TOOL_NAME + "clo" + if truncated_second + else _tool_call("clock", '{"timezone":"Asia/Tokyo"}') + ) + output = ( + END_THINK + + BEGIN_TOOL_REQUESTS + + _tool_call("weather", '{"city":"東京"}') + + second + ) + expected = [{"name": "weather", "arguments": '{"city":"東京"}'}] + if truncated_second: + expected.append({"name": "clo", "arguments": "{}"}) + else: + output += END_TOOL_REQUESTS + expected.append({"name": "clock", "arguments": '{"timezone":"Asia/Tokyo"}'}) + stream_expected = expected + _, content, calls = parser_cls(mock_tokenizer).parse( + output, mock_request, enable_auto_tools=True + ) + assert ( + calls + and [{"name": c.name, "arguments": c.arguments} for c in calls] == expected + ) + deltas = _stream(parser_cls(mock_tokenizer), mock_tokenizer, output) + result = collect_output(deltas) + assert result.tool_calls == stream_expected + assert result.content == (content or "") == "" + announced = [ + call for delta in deltas if delta for call in delta.tool_calls or [] if call.id + ] + assert [call.index for call in announced] == list(range(len(expected))) + assert len({call.id for call in announced}) == len(expected) + + +def test_arguments_are_available_before_outer_closer(mock_tokenizer, mock_request): + parser = Plamo3Parser(mock_tokenizer) + output = END_THINK + BEGIN_TOOL_REQUESTS + _tool_call("weather", '{"city":"Tokyo"}') + first = parser.parse_delta(output, [], mock_request, finished=False) + assert collect_output([first]).tool_calls == [ + {"name": "weather", "arguments": '{"city":"Tokyo"}'} + ] + assert ( + parser.parse_delta(END_TOOL_REQUESTS[:10], [], mock_request, finished=False) + is None + ) + assert ( + parser.parse_delta(END_TOOL_REQUESTS[10:], [], mock_request, finished=True) + is None + ) + + +def test_missing_tool_request_start_does_not_break_streaming( + parser_cls, mock_tokenizer, mock_request +): + output = ( + BEGIN_TOOL_REQUESTS + + BEGIN_TOOL_NAME + + "weather" + + END_TOOL_NAME + + BEGIN_TOOL_ARGUMENTS + + "{}" + + END_TOOL_ARGUMENTS + + END_TOOL_REQUESTS + ) + delta = parser_cls(mock_tokenizer).parse_delta( + output, [], mock_request, finished=False + ) + assert delta is not None + assert collect_output([delta]).tool_calls == [ + {"name": "weather", "arguments": "{}"} + ] + + +def test_reusing_parser_drops_pending_names(mock_tokenizer, mock_request): + parser = Plamo3Parser(mock_tokenizer) + parser.parse( + END_THINK + + BEGIN_TOOL_REQUESTS + + BEGIN_TOOL_REQUEST + + BEGIN_TOOL_NAME + + "partial", + mock_request, + ) + output = ( + END_THINK + + BEGIN_TOOL_REQUESTS + + _tool_call("weather", "{}") + + END_TOOL_REQUESTS + ) + _, _, calls = parser.parse(output, mock_request) + assert calls and calls[0].name == "weather" + + +@pytest.mark.parametrize( + "text", ["", "a < b", "literal <", "literal <|plamo:unknown", " text\n"] +) +def test_plain_content_is_preserved(parser_cls, mock_tokenizer, mock_request, text): + output = END_THINK + text + EOT + _, content, calls = parser_cls(mock_tokenizer).parse(output, mock_request) + assert (content or "") == text + assert not calls + result = collect_output(_stream(parser_cls(mock_tokenizer), mock_tokenizer, output)) + assert (result.content, result.tool_calls) == (text, []) + + +@pytest.mark.parametrize( + "text, ended", + [ + ("", False), + (BEGIN_THINK + "reasoning", False), + (END_THINK + "answer", True), + (END_THINK + "answer" + BEGIN_THINK + "new reasoning", False), + (END_THINK + "answer" + EOT + "assistant", False), + ], +) +def test_reasoning_end_checks_the_latest_turn(mock_tokenizer, text, ended): + parser = Plamo3Parser(mock_tokenizer) + ids = mock_tokenizer.encode(text) + assert parser.is_reasoning_end(ids) == ended + if ended: + assert mock_tokenizer.decode(parser.extract_content_ids(ids)) == "answer" + + +@pytest.mark.parametrize("suffix", ["<|plamo:end_", "<|plamo:end_think"]) +def test_partial_end_think_is_consistent_between_streaming_and_non_streaming( + parser_cls, mock_tokenizer, mock_request, suffix +): + output = "visible" + suffix + reasoning, content, calls = parser_cls(mock_tokenizer).parse(output, mock_request) + assert (reasoning, content) == ("visible", None) + assert not calls + result = collect_output(_stream(parser_cls(mock_tokenizer), mock_tokenizer, output)) + assert (result.reasoning, result.content, result.tool_calls) == ("visible", "", []) + + +@pytest.mark.parametrize( + "suffix", + [ + "<|plamo:begin_", + "<|plamo:begin_tool_name", + "<|plamo:end_tool", + ], +) +def test_unfinished_marker_prefixes_are_stripped(mock_tokenizer, suffix): + parser = Plamo3Parser(mock_tokenizer) + assert parser._strip_unfinished_marker("visible" + suffix) == "visible" + + +@pytest.mark.parametrize("suffix", ["<|plamo:", "<|plamo:b"]) +def test_non_token_prefix_is_not_stripped(mock_tokenizer, suffix): + parser = Plamo3Parser(mock_tokenizer) + text = "visible" + suffix + assert parser._strip_unfinished_marker(text) == text + + +def test_complete_markers_are_not_stripped(mock_tokenizer): + parser = Plamo3Parser(mock_tokenizer) + text = "visible" + BEGIN_TOOL_NAME + assert parser._strip_unfinished_marker(text) == text + + +def test_short_prefix_is_not_stripped(mock_tokenizer): + parser = Plamo3Parser(mock_tokenizer) + text = "visible<" + assert parser._strip_unfinished_marker(text) == text + + +def test_tool_block_can_end_reasoning_implicitly( + parser_cls, mock_tokenizer, mock_request +): + output = ( + "reasoning" + + BEGIN_TOOL_REQUESTS + + _tool_call("weather", "{}") + + END_TOOL_REQUESTS + ) + reasoning, content, calls = parser_cls(mock_tokenizer).parse( + output, mock_request, enable_auto_tools=True + ) + assert (reasoning, content) == ("reasoning", None) + assert calls and calls[0].name == "weather" + result = collect_output(_stream(parser_cls(mock_tokenizer), mock_tokenizer, output)) + assert (result.reasoning, result.content) == ("reasoning", "") + assert result.tool_calls == [{"name": "weather", "arguments": "{}"}] + + +def test_every_cut_point_matches_non_streaming( + parser_cls, mock_tokenizer, mock_request +): + output = ( + BEGIN_THINK + + "Reasoning. " + + END_THINK + + "Before tools. " + + BEGIN_TOOL_REQUESTS + + _tool_call("weather", '{"city":"東京"}') + + END_TOOL_REQUESTS + + EOT + ) + ids = mock_tokenizer.encode(output, add_special_tokens=False) + for k in range(1, len(ids) + 1): + prefix = mock_tokenizer.decode(ids[:k]) + reasoning, content, calls = parser_cls(mock_tokenizer).parse( + prefix, mock_request, enable_auto_tools=True + ) + result = collect_output( + _stream(parser_cls(mock_tokenizer), mock_tokenizer, prefix) + ) + assert (result.reasoning, result.content) == ( + (reasoning or ""), + (content or ""), + ), k + expected = [{"name": c.name, "arguments": c.arguments} for c in (calls or [])] + assert result.tool_calls == expected, (k, expected, result.tool_calls) + + +@pytest.mark.parametrize( + "choice", + [ + "auto", + "required", + "none", + {"type": "function", "function": {"name": "weather"}}, + ], +) +def test_tool_choice_uses_the_plamo_format(parser_cls, mock_tokenizer, choice): + request = _test_request( + [ + { + "type": "function", + "function": {"name": "weather", "parameters": {"type": "object"}}, + }, + ] + ) + request = type(request).model_validate( + request.model_dump() | {"tool_choice": choice} + ) + output = ( + END_THINK + + "Before tools." + + BEGIN_TOOL_REQUESTS + + _tool_call("weather", "{}") + + END_TOOL_REQUESTS + ) + _, content, calls = parser_cls(mock_tokenizer).parse( + output, request, enable_auto_tools=True + ) + assert content == "Before tools." + assert [c.name for c in calls or []] == ([] if choice == "none" else ["weather"]) + parser = parser_cls(mock_tokenizer) + delta = parser.parse_delta(output, [], request, finished=True) + result = collect_output([delta]) + assert result.content == "Before tools." + assert result.tool_calls == ( + [] if choice == "none" else [{"name": "weather", "arguments": "{}"}] + ) + + +@pytest.mark.parametrize("delay", [1, 3]) +def test_delayed_text_keeps_split_markers_intact(parser_cls, mock_tokenizer, delay): + output = ( + BEGIN_THINK + + "reasoning" + + END_THINK + + "content" + + BEGIN_TOOL_REQUESTS + + _tool_call("weather", "{}") + + END_TOOL_REQUESTS + + EOT + ) + tokens = [(i, mock_tokenizer.decode([i])) for i in mock_tokenizer.encode(output)] + result = collect_output( + replay_with_text_holdback( + parser_cls(mock_tokenizer), tokens, text_delay=delay, tools=DUMMY_TOOLS + ) + ) + assert (result.reasoning, result.content) == ("reasoning", "content") + assert result.tool_calls == [{"name": "weather", "arguments": "{}"}] diff --git a/tests/parser/engine/test_replay.py b/tests/parser/engine/test_replay.py index cc8fef483f..89791ba29b 100644 --- a/tests/parser/engine/test_replay.py +++ b/tests/parser/engine/test_replay.py @@ -74,9 +74,10 @@ def _discover_parsers() -> list[_ParserInfo]: if cfg.name not in _BUILDERS: missing_builders.append(f"{obj.__name__} (config.name={cfg.name!r})") continue - if cfg.name == "inkling": - # Inkling opts out of token-id terminal matching and has typed - # structural blocks; its replay coverage lives in test_inkling.py. + if cfg.name in ("inkling", "plamo3"): + # Inkling uses typed structural blocks. PLaMo markers can span + # several token IDs. Neither fits this TOOL_END-token harness; + # their direct and delegating coverage lives in dedicated tests. continue if cfg.name == "granite": # Granite has a JSON-array tool body with no TOOL_END terminal, so diff --git a/tests/parser/engine/trace_builder.py b/tests/parser/engine/trace_builder.py index 5b405e6dae..29b9db3f2b 100644 --- a/tests/parser/engine/trace_builder.py +++ b/tests/parser/engine/trace_builder.py @@ -28,6 +28,7 @@ from tests.parser.engine.replay_harness import ( from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionToolsParam, ) +from vllm.parser import plamo3 from vllm.parser.engine.registered_adapters import ( DeepSeekV4Parser, DeepSeekV32Parser, @@ -40,6 +41,7 @@ from vllm.parser.engine.registered_adapters import ( KimiK2Parser, MinimaxM2Parser, NemotronV3Parser, + Plamo3Parser, Qwen3Parser, SeedOssParser, ) @@ -1068,6 +1070,70 @@ def _build_inkling(scenario: Scenario, validate: bool = True) -> Sample: return sample +# ── PLaMo3 (literal nested tool-request wrappers) ─────────────────── + +_PLAMO3_MARKERS = ( + plamo3.BEGIN_THINK, + plamo3.END_THINK, + plamo3.BEGIN_TOOL_REQUESTS, + plamo3.END_TOOL_REQUESTS, + plamo3.BEGIN_TOOL_REQUEST, + plamo3.END_TOOL_REQUEST, + plamo3.BEGIN_TOOL_NAME, + plamo3.END_TOOL_NAME, + plamo3.BEGIN_TOOL_ARGUMENTS, + plamo3.END_TOOL_ARGUMENTS, + plamo3.EOT, +) +_PLAMO3_VOCAB = {marker: 300 + i for i, marker in enumerate(_PLAMO3_MARKERS)} + + +def _plamo3_tool_segments(tc: ToolCallSpec) -> list[tuple[str, bool]]: + args = json.dumps(tc.arguments, ensure_ascii=False, separators=(",", ":")) + return [ + (plamo3.BEGIN_TOOL_REQUEST, True), + (plamo3.BEGIN_TOOL_NAME, True), + (tc.name, False), + (plamo3.END_TOOL_NAME, True), + (plamo3.BEGIN_TOOL_ARGUMENTS, True), + (args, False), + (plamo3.END_TOOL_ARGUMENTS, True), + (plamo3.END_TOOL_REQUEST, True), + ] + + +def _plamo3_segments(scenario: Scenario) -> list[tuple[str, bool]]: + segs: list[tuple[str, bool]] = [ + (plamo3.BEGIN_THINK, True), + (scenario.reasoning or "", False), + (plamo3.END_THINK, True), + ] + if scenario.content is not None: + segs.append((scenario.content, False)) + if scenario.tool_calls is not None: + segs.append((plamo3.BEGIN_TOOL_REQUESTS, True)) + for tool_call in scenario.tool_calls: + segs.extend(_plamo3_tool_segments(tool_call)) + segs.append((plamo3.END_TOOL_REQUESTS, True)) + return segs + + +def _build_plamo3(scenario: Scenario, validate: bool = True) -> Sample: + sample = _make_sample( + sample_id=f"plamo3-{scenario.id}", + description=scenario.description, + vocab=_PLAMO3_VOCAB, + segments=_plamo3_segments(scenario), + expected_reasoning=scenario.reasoning or "", + expected_content=_qwen3_expected_content(scenario), + expected_tool_calls=_expected_tc(scenario), + tools=_expected_tools(scenario), + ) + if validate: + _validate_sample(sample, Plamo3Parser) + return sample + + # ── Granite (JSON-array tool bodies, no reasoning) ─────────────────────── _GRANITE_VOCAB: dict[str, int] = { @@ -1167,6 +1233,7 @@ _BUILDERS: dict[str, Any] = { "kimi_k2": _build_kimi_k2, "qwen3": _build_qwen3, "inkling": _build_inkling, + "plamo3": _build_plamo3, } diff --git a/tests/quantization/test_moe_wna16.py b/tests/quantization/test_moe_wna16.py index 989bf967fa..344b8ecbd7 100644 --- a/tests/quantization/test_moe_wna16.py +++ b/tests/quantization/test_moe_wna16.py @@ -149,7 +149,11 @@ def test_wna16_oracle_rejects_incompatible_quant_structures( assert expected in reason -def test_compressed_tensors_weights_are_transposed_for_triton(): +def test_compressed_tensors_nfirst_weights_pass_through_for_triton(): + """CT provides N-first [E, N, K_packed] int32; Triton expects N-first uint8. + + No transpose needed — just a view from int32 to uint8. + """ quant_config = QuantizationArgs( num_bits=4, type=QuantizationType.INT, @@ -158,10 +162,11 @@ def test_compressed_tensors_weights_are_transposed_for_triton(): dynamic=False, group_size=32, ) - w13 = torch.arange(16, dtype=torch.int32).reshape(1, 2, 8) - w2 = torch.arange(12, dtype=torch.int32).reshape(1, 2, 6) - w13_scale = torch.arange(32, dtype=torch.float16).reshape(1, 4, 8) - w2_scale = torch.arange(18, dtype=torch.float16).reshape(1, 3, 6) + # N-first layout: [E, N, K_packed] + w13 = torch.arange(16, dtype=torch.int32).reshape(1, 8, 2) + w2 = torch.arange(12, dtype=torch.int32).reshape(1, 6, 2) + w13_scale = torch.arange(32, dtype=torch.float16).reshape(1, 8, 4) + w2_scale = torch.arange(18, dtype=torch.float16).reshape(1, 6, 3) converted = convert_to_wna16_moe_kernel_format( backend=WNA16MoEBackend.TRITON, @@ -175,10 +180,40 @@ def test_compressed_tensors_weights_are_transposed_for_triton(): ) assert converted is not None - assert torch.equal(converted[0], w13.transpose(1, 2).contiguous().view(torch.uint8)) - assert torch.equal(converted[1], w2.transpose(1, 2).contiguous().view(torch.uint8)) - assert torch.equal(converted[2], w13_scale.transpose(1, 2).contiguous()) - assert torch.equal(converted[3], w2_scale.transpose(1, 2).contiguous()) + assert torch.equal(converted[0], w13.contiguous().view(torch.uint8)) + assert torch.equal(converted[1], w2.contiguous().view(torch.uint8)) + assert torch.equal(converted[2], w13_scale) + assert torch.equal(converted[3], w2_scale) + + +def test_gptq_nfirst_weights_pass_through_for_triton(): + """GPTQ frontend normalizes to N-first before calling oracle. + + Verifies the oracle handles N-first GPTQ inputs the same way as CT. + """ + quant_config = AutoGPTQConfig(4, 128, False, True, False, {}, {}) + # N-first layout (after GPTQ frontend transpose): [E, N, K_packed] + w13 = torch.arange(16, dtype=torch.int32).reshape(1, 8, 2) + w2 = torch.arange(12, dtype=torch.int32).reshape(1, 6, 2) + w13_scale = torch.arange(32, dtype=torch.float16).reshape(1, 8, 4) + w2_scale = torch.arange(18, dtype=torch.float16).reshape(1, 6, 3) + + converted = convert_to_wna16_moe_kernel_format( + backend=WNA16MoEBackend.TRITON, + layer=torch.nn.Module(), + quant_config=quant_config, + input_dtype=None, + w13=w13, + w2=w2, + w13_scale=w13_scale, + w2_scale=w2_scale, + ) + + assert converted is not None + assert torch.equal(converted[0], w13.contiguous().view(torch.uint8)) + assert torch.equal(converted[1], w2.contiguous().view(torch.uint8)) + assert torch.equal(converted[2], w13_scale) + assert torch.equal(converted[3], w2_scale) def test_moe_wna16_setup_forwards_selected_backend(monkeypatch): @@ -511,3 +546,77 @@ def test_compressed_tensors_wna16_moe_marlin_prep_with_unset_group_size(): ) assert layer.w13_weight_scale.shape == (num_experts, 1, 2 * intermediate_size) assert layer.w2_weight_scale.shape == (num_experts, 1, hidden_size) + + +def test_moe_wna16_w13_qzeros_shard_split_int8_asym_gptq(monkeypatch): + """8-bit asymmetric GPTQ MoE must split w13 zero-points by pack factor. + + ``w13_qzeros`` is packed along the intermediate dim by ``bit8_pack_factor``, + so the gate/up (w1/w3) boundary in the weight loader is + ``shard_size // bit8_pack_factor``. Hardcoding ``shard_size // 2`` only + holds for 4-bit and raises a shape mismatch when loading an 8-bit + (unpacked) checkpoint. + """ + from tests.kernels.moe.utils import make_dummy_moe_config + + monkeypatch.setattr( + moe_wna16, "get_tp_group", lambda: SimpleNamespace(device="cpu") + ) + monkeypatch.setattr(moe_wna16, "get_tensor_model_parallel_rank", lambda: 0) + + num_experts, hidden, intermediate, group_size = 2, 256, 512, 128 + moe_config = make_dummy_moe_config( + num_experts=num_experts, + hidden_dim=hidden, + intermediate_size=intermediate, + ) + quant_config = MoeWNA16Config( + linear_quant_method="gptq", + weight_bits=8, + group_size=group_size, + has_zp=True, + lm_head_quantized=False, + modules_to_not_convert=[], + full_config={}, + ) + method = object.__new__(MoeWNA16Method) + method.quant_config = quant_config + method.moe = moe_config + + layer = torch.nn.Module() + layer.intermediate_size_per_partition = intermediate + layer.moe_config = moe_config + method.create_weights( + layer, + num_experts=num_experts, + hidden_size=hidden, + intermediate_size_per_partition=intermediate, + params_dtype=torch.float16, + weight_loader=lambda *a, **k: True, + ) + + # 8-bit is unpacked along the intermediate dim: dim1 == 2 * intermediate. + assert layer.w13_qzeros.shape == ( + num_experts, + 2 * intermediate, + hidden // group_size, + ) + loader = layer.w13_qzeros.weight_loader + + # Raw GPTQ 8-bit qzeros per shard: int32 (num_groups, intermediate // 4); + # the loader views them as uint8 -> (num_groups, intermediate), then + # transposes and offsets by 1. + num_groups = hidden // group_size + raw_w1 = torch.arange(num_groups * (intermediate // 4), dtype=torch.int32).reshape( + num_groups, intermediate // 4 + ) + raw_w3 = raw_w1 + 1000 + + # Must not raise (old code raised a shape mismatch here for 8-bit). + loader(layer.w13_qzeros, raw_w1, "w13_qzeros", "w1", 0, False) + loader(layer.w13_qzeros, raw_w3, "w13_qzeros", "w3", 0, False) + + expected_w1 = raw_w1.view(torch.uint8).T + 1 + expected_w3 = raw_w3.view(torch.uint8).T + 1 + assert torch.equal(layer.w13_qzeros[0, :intermediate].cpu(), expected_w1) + assert torch.equal(layer.w13_qzeros[0, intermediate:].cpu(), expected_w3) diff --git a/tests/test_config.py b/tests/test_config.py index 609d59ed5b..0b8b4e3f1c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -5,6 +5,7 @@ import json import logging import os from dataclasses import MISSING, Field, asdict, dataclass, field +from datetime import timedelta from pathlib import Path from types import SimpleNamespace from typing import cast @@ -1464,6 +1465,20 @@ def test_all2all_backend_has_portable_default(): assert ParallelConfig().all2all_backend == "allgather_reducescatter" +def test_dp_group_uses_configured_timeout_without_current_config(monkeypatch): + monkeypatch.setattr(vllm_config_module, "_current_vllm_config", None) + config = ParallelConfig(cpu_distributed_timeout_seconds=30) + with ( + patch( + "vllm.distributed.utils.rendezvous", + return_value=iter([(torch.distributed.HashStore(), 0, 1)]), + ), + patch("vllm.distributed.utils.init_gloo_process_group") as init_group, + ): + config.stateless_init_dp_group() + assert init_group.call_args.kwargs["timeout"] == timedelta(seconds=30) + + @pytest.mark.parametrize( "dp_size, across_dp, expected", [(1, False, 4), (1, True, 4), (2, False, 4), (2, True, 8)], @@ -1497,21 +1512,42 @@ def test_engram_dp_shared_memory_requires_cpu_offload(): EngramConfig(cpu_offload=False, dp_shared_memory=True) +@pytest.mark.skip_global_cleanup @pytest.mark.parametrize( - "cpu_offload,dp_size,elastic_ep,expected", - [(True, 2, False, True), (False, 2, False, False), (True, 1, False, False)], + "cpu_offload,use_thp,dp_shared_memory,dp_size,elastic_ep,expected", + [ + (True, False, None, 2, False, True), + (False, False, None, 2, False, False), + (True, False, None, 1, False, False), + (True, False, None, 2, True, False), + # use_thp backs private tables; sharing would silently ignore it. + (True, True, None, 2, False, False), + (True, True, False, 2, False, False), + ], ) def test_engram_dp_shared_memory_defaults_when_supported( - cpu_offload, dp_size, elastic_ep, expected + cpu_offload, use_thp, dp_shared_memory, dp_size, elastic_ep, expected ): - """Unset dp_shared_memory enables sharing only for offloaded, non-elastic DP.""" + """Resolve sharing only where supported, preserving an explicit false.""" parallel = ParallelConfig(data_parallel_size=dp_size) parallel.enable_elastic_ep = elastic_ep - config = EngramConfig(cpu_offload=cpu_offload) + config = EngramConfig( + cpu_offload=cpu_offload, + use_thp=use_thp, + dp_shared_memory=dp_shared_memory, + ) config.resolve_dp_shared_memory(parallel) assert config.dp_shared_memory is expected +@pytest.mark.skip_global_cleanup +def test_engram_thp_rejects_explicit_shared_memory(): + with pytest.raises( + ValueError, match="use_thp requires cpu_offload=True and dp_shared_memory=False" + ): + EngramConfig(use_thp=True, dp_shared_memory=True) + + @pytest.mark.parametrize( "dp_size,load_format,multithread,error", [ diff --git a/tests/test_logger.py b/tests/test_logger.py index 1810eeba16..f194853fa4 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -165,6 +165,45 @@ def test_json_logging(monkeypatch, tmp_path): assert log["process"] == os.getpid() +@pytest.mark.parametrize("factory_order", ["before", "after", "replacement"]) +def test_configure_logging_preserves_application_record_factory( + monkeypatch, factory_order +): + """Application fields survive initial configuration and reconfiguration.""" + original_factory = logging.getLogRecordFactory() + monkeypatch.setattr(vllm_logger, "dictConfig", lambda _: None) + monkeypatch.setattr(vllm_logger, "_last_configured_logging_config", None) + monkeypatch.setattr(vllm_logger, "_vllm_process_info", None) + config = LoggingConfig() + formatter = logging.Formatter("%(request_id)s %(vllm_process_name)s %(message)s") + + try: + logging.setLogRecordFactory(logging.LogRecord) + if factory_order != "before": + configure_logging(config) + base_factory = ( + logging.LogRecord + if factory_order == "replacement" + else logging.getLogRecordFactory() + ) + + def application_factory(*args, **kwargs): + record = base_factory(*args, **kwargs) + record.request_id = "request-123" + return record + + logging.setLogRecordFactory(application_factory) + decorate_logs("Worker_DP0") + for _ in range(2): + configure_logging(config) + record = logging.getLogger("application").makeRecord( + "application", logging.INFO, __file__, 1, "probe", (), None + ) + assert formatter.format(record) == "request-123 Worker_DP0 probe" + finally: + logging.setLogRecordFactory(original_factory) + + def test_use_color_force_color(monkeypatch): """FORCE_COLOR forces colored logs without a TTY, while NO_COLOR and an explicit VLLM_LOGGING_COLOR=0 take precedence over it.""" @@ -223,9 +262,11 @@ def test_logger_configuring_can_be_disabled(monkeypatch): monkeypatch.setenv("VLLM_CONFIGURE_LOGGING", "0") monkeypatch.delenv("VLLM_LOGGING_CONFIG_PATH", raising=False) + original_factory = logging.getLogRecordFactory() with patch("vllm.logger.dictConfig") as dict_config_mock: configure_logging(LoggingConfig(configure_logging=False)) dict_config_mock.assert_not_called() + assert logging.getLogRecordFactory() is original_factory def test_an_error_is_raised_when_custom_logging_config_file_does_not_exist(monkeypatch): diff --git a/tests/test_request_input_bounds.py b/tests/test_request_input_bounds.py index 7f79d4ba47..514a450f73 100644 --- a/tests/test_request_input_bounds.py +++ b/tests/test_request_input_bounds.py @@ -43,6 +43,7 @@ def prompt_embeds_processor(): logits_processors=None, is_diffusion=False, return_sampling_mask=False, + supports_multimodal_inputs=False, ) model_config.get_vocab_size.return_value = 32 model_config.try_get_generation_config.return_value = {} @@ -53,9 +54,7 @@ def prompt_embeds_processor(): ) renderer = Mock(tokenizer=None) renderer.get_eos_token_id.return_value = None - registry = Mock() - registry.supports_multimodal_inputs.return_value = False - return InputProcessor(config, renderer, mm_registry=registry) + return InputProcessor(config, renderer, mm_registry=Mock()) @pytest.mark.parametrize("mask_len", [0, 1, 2, 4]) diff --git a/tests/test_triton_utils.py b/tests/test_triton_utils.py index 7f6422466f..3d7c8aeed2 100644 --- a/tests/test_triton_utils.py +++ b/tests/test_triton_utils.py @@ -10,10 +10,13 @@ from vllm.triton_utils import importing as triton_importing from vllm.triton_utils.importing import TritonLanguagePlaceholder, TritonPlaceholder -def _has_triton_for_backends(**drivers: bool) -> bool: +def _has_triton_for_backends(*, vllm_version: str = "0.0.0", **drivers: bool) -> bool: """Re-evaluate ``HAS_TRITON`` against a synthetic ``triton.backends`` map. ``drivers`` maps backend name to whether its driver reports itself active. + ``vllm_version`` selects the build type: importing.py takes its Triton-CPU + branch only when "cpu" is in the installed vLLM version, so pin it rather + than depending on the real install (the CPU CI image is a "+cpu" build). """ backends = {} for name, is_active in drivers.items(): @@ -34,6 +37,7 @@ def _has_triton_for_backends(**drivers: bool) -> bool: with ( mock.patch.dict(sys.modules, patched_modules), mock.patch.dict("os.environ", {}, clear=True), + mock.patch("importlib.metadata.version", return_value=vllm_version), ): return importlib.reload(triton_importing).HAS_TRITON finally: @@ -129,19 +133,50 @@ def test_cpu_backend_alone_disables_triton(): assert _has_triton_for_backends(cpu=True) is False +def test_cpu_build_with_cpu_backend_keeps_triton(): + assert _has_triton_for_backends(vllm_version="0.0.0+cpu", cpu=True) is True + + +def test_cpu_build_without_cpu_backend_disables_triton(): + assert _has_triton_for_backends(vllm_version="0.0.0+cpu", amd=True) is False + + def test_no_triton_fallback(): - # clear existing triton modules - sys.modules.pop("triton", None) - sys.modules.pop("triton.language", None) - sys.modules.pop("vllm.triton_utils", None) - sys.modules.pop("vllm.triton_utils.importing", None) + # Save the real modules so they can be put back afterward - popping them + # (rather than mock.patch.dict alone) is what forces vllm.triton_utils to + # be freshly, unmemoized re-imported under the "triton is absent" + # condition below. + saved_modules = { + name: sys.modules.get(name) + for name in ( + "triton", + "triton.language", + "vllm.triton_utils", + "vllm.triton_utils.importing", + ) + } + for name in saved_modules: + sys.modules.pop(name, None) - # mock triton not being installed - with mock.patch.dict(sys.modules, {"triton": None}): - from vllm.triton_utils import HAS_TRITON, tl, triton + try: + # mock triton not being installed + with mock.patch.dict(sys.modules, {"triton": None}): + from vllm.triton_utils import HAS_TRITON, tl, triton - assert HAS_TRITON is False - assert triton.__class__.__name__ == "TritonPlaceholder" - assert triton.language.__class__.__name__ == "TritonLanguagePlaceholder" - assert tl.__class__.__name__ == "TritonLanguagePlaceholder" - assert tl.constexpr(2**31 - 1) == 2**31 - 1 + assert HAS_TRITON is False + assert triton.__class__.__name__ == "TritonPlaceholder" + assert triton.language.__class__.__name__ == "TritonLanguagePlaceholder" + assert tl.__class__.__name__ == "TritonLanguagePlaceholder" + assert tl.constexpr(2**31 - 1) == 2**31 - 1 + finally: + # The pops above are outside mock.patch.dict's scope, so exiting the + # `with` restores "triton" to absent, not to the real module - unlike + # _has_triton_for_backends above, nothing here puts it back. Left + # alone, every subsequent import of vllm.triton_utils in this process + # (including other test files, and other tests' cleanup fixtures that + # need real triton) would get this fake, HAS_TRITON=False module. + for name, module in saved_modules.items(): + if module is not None: + sys.modules[name] = module + else: + sys.modules.pop(name, None) diff --git a/tests/test_version.py b/tests/test_version.py index 928f742f1d..67cb999e13 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -13,7 +13,10 @@ def test_version_is_defined(): def test_version_tuple(): - assert len(version.__version_tuple__) in (3, 4, 5) + # (major, minor, patch) plus optional pre-release tag, dev-distance, and + # git-hash components - setuptools_scm can emit any combination of them, + # e.g. 6 when a dev build sits on top of an rc-tagged commit. + assert len(version.__version_tuple__) >= 3 @pytest.mark.parametrize( diff --git a/tests/test_zen_cpu_platform_detection.py b/tests/test_zen_cpu_platform_detection.py index 0c99c6568a..505d2d3bfd 100644 --- a/tests/test_zen_cpu_platform_detection.py +++ b/tests/test_zen_cpu_platform_detection.py @@ -13,6 +13,23 @@ from vllm.platforms import ( ) +@pytest.fixture +def caplog_vllm(caplog): + """`caplog`, but also captures vLLM's loggers. + + vllm/logger.py configures the "vllm" logger with propagate=False, so + records from child loggers like "vllm.platforms" reach vLLM's own + handler but never bubble up to the root logger where plain `caplog` + listens. Attach caplog's handler directly to "vllm" to see them too. + """ + logger = logging.getLogger("vllm") + logger.addHandler(caplog.handler) + try: + yield caplog + finally: + logger.removeHandler(caplog.handler) + + def test_is_amd_zen_cpu_detects_amd_with_avx512(): cpuinfo = "vendor_id: AuthenticAMD\nflags: avx avx2 avx512f avx512bw" with ( @@ -65,7 +82,7 @@ def test_cpu_target_selects_cpu_platform_from_non_cpu_wheel( rocm_plugin.assert_not_called() -def test_broken_zentorch_falls_back_to_cpu_platform(caplog): +def test_broken_zentorch_falls_back_to_cpu_platform(caplog_vllm): original_import = builtins.__import__ def import_with_broken_zentorch(name, *args, **kwargs): @@ -84,16 +101,16 @@ def test_broken_zentorch_falls_back_to_cpu_platform(caplog): patch("vllm.platforms.vllm_version_matches_substr", return_value=True), patch("vllm.platforms._is_amd_zen_cpu", return_value=True), patch.object(builtins, "__import__", side_effect=import_with_broken_zentorch), - caplog.at_level(logging.DEBUG, logger="vllm.platforms"), + caplog_vllm.at_level(logging.DEBUG, logger="vllm.platforms"), ): platform = resolve_current_platform_cls_qualname() assert platform == "vllm.platforms.cpu.CpuPlatform" - assert "zentorch failed to import" in caplog.text - assert "OSError: incompatible shared library" in caplog.text + assert "zentorch failed to import" in caplog_vllm.text + assert "OSError: incompatible shared library" in caplog_vllm.text -def test_zentorch_failure_other_than_oserror_is_not_recovered(caplog): +def test_zentorch_failure_other_than_oserror_is_not_recovered(caplog_vllm): original_import = builtins.__import__ def import_with_broken_zentorch(name, *args, **kwargs): @@ -112,15 +129,15 @@ def test_zentorch_failure_other_than_oserror_is_not_recovered(caplog): patch("vllm.platforms.vllm_version_matches_substr", return_value=True), patch("vllm.platforms._is_amd_zen_cpu", return_value=True), patch.object(builtins, "__import__", side_effect=import_with_broken_zentorch), - caplog.at_level(logging.DEBUG, logger="vllm.platforms"), + caplog_vllm.at_level(logging.DEBUG, logger="vllm.platforms"), ): platform = resolve_current_platform_cls_qualname() assert platform == "vllm.platforms.interface.UnspecifiedPlatform" - assert "RuntimeError: unknown zentorch failure" in caplog.text + assert "RuntimeError: unknown zentorch failure" in caplog_vllm.text -def test_platform_plugin_failure_is_logged(caplog): +def test_platform_plugin_failure_is_logged(caplog_vllm): def failing_plugin(): raise RuntimeError("plugin exploded") @@ -132,10 +149,10 @@ def test_platform_plugin_failure_is_logged(caplog): clear=True, ), patch("vllm.platforms.load_plugins_by_group", return_value={}), - caplog.at_level(logging.DEBUG, logger="vllm.platforms"), + caplog_vllm.at_level(logging.DEBUG, logger="vllm.platforms"), ): platform = resolve_current_platform_cls_qualname() assert platform == "vllm.platforms.interface.UnspecifiedPlatform" - assert "Platform plugin cpu failed during detection" in caplog.text - assert "RuntimeError: plugin exploded" in caplog.text + assert "Platform plugin cpu failed during detection" in caplog_vllm.text + assert "RuntimeError: plugin exploded" in caplog_vllm.text diff --git a/tests/tool_parsers/test_structural_tag_registry.py b/tests/tool_parsers/test_structural_tag_registry.py index 3221d1fb2c..99b18c6736 100644 --- a/tests/tool_parsers/test_structural_tag_registry.py +++ b/tests/tool_parsers/test_structural_tag_registry.py @@ -16,6 +16,17 @@ from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionToolsParam, ) from vllm.parser.abstract_parser import DelegatingParser +from vllm.parser.plamo3 import ( + BEGIN_TOOL_ARGUMENTS, + BEGIN_TOOL_NAME, + BEGIN_TOOL_REQUEST, + BEGIN_TOOL_REQUESTS, + END_TOOL_ARGUMENTS, + END_TOOL_NAME, + END_TOOL_REQUEST, + END_TOOL_REQUESTS, + EOT, +) from vllm.tool_parsers.abstract_tool_parser import ToolParser from vllm.tool_parsers.deepseekv3_tool_parser import DeepSeekV3ToolParser from vllm.tool_parsers.deepseekv4_engine_tool_parser import DeepSeekV4EngineToolParser @@ -30,6 +41,7 @@ from vllm.tool_parsers.kimi_k2_tool_parser import KimiK2ToolParser from vllm.tool_parsers.kimi_k3_tool_parser import KimiK3ToolParser from vllm.tool_parsers.llama_tool_parser import Llama3JsonToolParser from vllm.tool_parsers.minimax_m2_tool_parser import MinimaxM2ToolParser +from vllm.tool_parsers.plamo3_engine_tool_parser import Plamo3EngineToolParser from vllm.tool_parsers.qwen3_engine_tool_parser import Qwen3EngineToolParser from vllm.tool_parsers.structural_tag_registry import ( SUPPORTED_STRUCTURAL_TAG_MODELS, @@ -271,6 +283,131 @@ def test_hermes_required_tool_calls_use_empty_separator(): assert tag.format.separator == "" +def _plamo3_call(name: str, arguments: str) -> str: + return ( + BEGIN_TOOL_REQUEST + + BEGIN_TOOL_NAME + + name + + END_TOOL_NAME + + BEGIN_TOOL_ARGUMENTS + + arguments + + END_TOOL_ARGUMENTS + + END_TOOL_REQUEST + ) + + +def _plamo3_requests(*calls: str, eot: bool = False) -> str: + return ( + BEGIN_TOOL_REQUESTS + "".join(calls) + END_TOOL_REQUESTS + (EOT if eot else "") + ) + + +def _plamo3_grammar(tool_choice, tools): + tag = get_model_structural_tag( + model="plamo3", + tools=tools, + tool_choice=tool_choice, + reasoning=False, + ) + assert isinstance(tag, StructuralTag) + return Grammar.from_structural_tag(tag) + + +def _plamo3_tools() -> list[ChatCompletionToolsParam]: + return [ + ChatCompletionToolsParam( + type="function", + function={ + "name": "get_weather", + "strict": True, + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + ), + ChatCompletionToolsParam( + type="function", + function={ + "name": "get_time", + "strict": True, + "parameters": { + "type": "object", + "properties": {"timezone": {"type": "string"}}, + "required": ["timezone"], + }, + }, + ), + ] + + +def test_plamo3_registered_as_vllm_structural_tag_model(): + assert "plamo3" in VLLM_BUILTIN_STRUCTURAL_TAG_MODELS + assert Plamo3EngineToolParser.structural_tag_model == "plamo3" + assert Plamo3EngineToolParser.supports_required_and_named is False + + +@pytest.mark.parametrize( + "body", + [ + _plamo3_requests(_plamo3_call("get_weather", '{"city":"Tokyo"}')), + _plamo3_requests( + _plamo3_call("get_weather", '{"city":"Tokyo"}'), + _plamo3_call("get_time", '{"timezone":"Asia/Tokyo"}'), + eot=True, + ), + ], +) +def test_plamo3_required_accepts_complete_request_blocks(body: str): + assert _is_grammar_accept_string(_plamo3_grammar("required", _plamo3_tools()), body) + + +@pytest.mark.parametrize( + "body", + [ + _plamo3_requests(_plamo3_call("unknown", '{"city":"Tokyo"}')), + _plamo3_requests(_plamo3_call("get_weather", "{}")), + BEGIN_TOOL_REQUESTS + _plamo3_call("get_weather", '{"city":"Tokyo"}'), + ], +) +def test_plamo3_required_rejects_invalid_request_blocks(body: str): + assert not _is_grammar_accept_string( + _plamo3_grammar("required", _plamo3_tools()), + body, + ) + + +def test_plamo3_auto_allows_text_or_a_tool_request_block(sample_tools_strict): + grammar = _plamo3_grammar("auto", sample_tools_strict) + + assert _is_grammar_accept_string(grammar, "Plain response") + assert _is_grammar_accept_string( + grammar, + _plamo3_requests(_plamo3_call("get_weather", '{"city":"Tokyo"}')), + ) + + +def test_plamo3_forced_stops_after_the_named_tool_call(): + tools = _plamo3_tools() + tool_choice = ChatCompletionNamedToolChoiceParam( + function=ChatCompletionNamedFunction(name="get_weather") + ) + grammar = _plamo3_grammar(tool_choice, tools) + + assert _is_grammar_accept_string( + grammar, + _plamo3_requests(_plamo3_call("get_weather", '{"city":"Tokyo"}')), + ) + assert not _is_grammar_accept_string( + grammar, + _plamo3_requests( + _plamo3_call("get_weather", '{"city":"Tokyo"}'), + _plamo3_call("get_time", '{"timezone":"Asia/Tokyo"}'), + ), + ) + + # --------------------------------------------------------------------------- # Kimi K3 (XTML channel format) structural tag # --------------------------------------------------------------------------- diff --git a/tests/utils_/test_flashinfer_utils.py b/tests/utils_/test_flashinfer_utils.py new file mode 100644 index 0000000000..3d28486ff5 --- /dev/null +++ b/tests/utils_/test_flashinfer_utils.py @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import importlib.util +from collections.abc import Iterator +from pathlib import Path + +import pytest + +import vllm.utils.flashinfer as fi + + +def _make_exe(path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.touch(mode=0o755) + + +@pytest.fixture +def default_cuda_home(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path: + """Hide any real toolkit, keep ninja on PATH, redirect /usr/local/cuda.""" + for var in ("CUDA_HOME", "CUDA_PATH", "FLASHINFER_NVCC"): + monkeypatch.delenv(var, raising=False) + _make_exe(tmp_path / "venv" / "bin" / "ninja") + monkeypatch.setenv("PATH", str(tmp_path / "venv" / "bin")) + home = tmp_path / "usr_local_cuda" + monkeypatch.setattr(fi, "_DEFAULT_CUDA_HOME", str(home)) + return home + + +@pytest.fixture(autouse=True) +def flashinfer_without_cubin(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + """Pretend flashinfer-python is installed without flashinfer-cubin.""" + real_find_spec = importlib.util.find_spec + monkeypatch.setattr( + importlib.util, + "find_spec", + lambda name, *args: ( + object() if name == "flashinfer" else real_find_spec(name, *args) + ), + ) + monkeypatch.setattr(fi, "has_flashinfer_cubin", lambda: False) + fi.has_flashinfer.cache_clear() + yield + fi.has_flashinfer.cache_clear() + + +def test_has_flashinfer_finds_toolkit_off_path(default_cuda_home: Path): + """Bare-metal installs keep nvcc in /usr/local/cuda, off PATH, which is + where FlashInfer's JIT finds it.""" + _make_exe(default_cuda_home / "bin" / "nvcc") + assert fi.has_flashinfer() + + +def test_has_flashinfer_requires_ninja( + default_cuda_home: Path, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +): + """FlashInfer runs `ninja` from PATH, which lacks the venv's bin directory + when vLLM is launched without activating the venv.""" + _make_exe(tmp_path / "cuda" / "bin" / "nvcc") + monkeypatch.setenv("PATH", str(tmp_path / "cuda" / "bin")) + assert not fi.has_flashinfer() diff --git a/tests/v1/attention/test_rocm_aiter_mla_dcp_cprr.py b/tests/v1/attention/test_rocm_aiter_mla_dcp_cprr.py new file mode 100644 index 0000000000..13b002cda0 --- /dev/null +++ b/tests/v1/attention/test_rocm_aiter_mla_dcp_cprr.py @@ -0,0 +1,152 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Routing/selection tests for the AITER ASM cprr DCP-verify route. + +Companion to the upstream segmented-verify tests. Everything here is pure CPU: +it pins the *decisions* (which route a KV group takes, which head count the +kernel runs at, which configurations are refused) rather than kernel numerics. +The numerics parity test against the segmented route needs a GPU and both +routes built; it is marked and skipped when unavailable. +""" + +from __future__ import annotations + +import pytest + +rocm_aiter_mla = pytest.importorskip( + "vllm.v1.attention.backends.mla.rocm_aiter_mla", + reason="ROCm AITER MLA backend not available", +) + +_heads = rocm_aiter_mla._asm_dcp_verify_heads +_configured = rocm_aiter_mla._asm_dcp_verify_configured +_select_route = rocm_aiter_mla._select_dcp_decode_route +Route = rocm_aiter_mla._DCPDecodeRoute +NATIVE = rocm_aiter_mla._NATIVE_CPRR_HEADS +MIN_QLEN = rocm_aiter_mla._MIN_CPRR_QLEN + + +def test_dcp_verify_env_defaults_to_segmented(monkeypatch): + monkeypatch.delenv("VLLM_ROCM_AITER_MLA_DCP_VERIFY", raising=False) + assert rocm_aiter_mla.envs.VLLM_ROCM_AITER_MLA_DCP_VERIFY == "segmented" + + +def test_dcp_verify_env_rejects_unknown_route(monkeypatch): + monkeypatch.setenv("VLLM_ROCM_AITER_MLA_DCP_VERIFY", "unknown") + with pytest.raises(ValueError, match="Invalid value"): + _ = rocm_aiter_mla.envs.VLLM_ROCM_AITER_MLA_DCP_VERIFY + + +# -------------------------------------------------------------------------- +# _asm_dcp_verify_heads -- which head count the cprr kernel runs at +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("n", NATIVE) +def test_native_head_counts_are_used_as_is(n): + assert _heads(n) == n + + +def test_non_native_head_count_pads_up_to_the_next_native(): + """K3 at TP8/DCP8 gathers 96 heads for the target; the kernel has no 96 + variant, so it must run at 128, not silently fall out of the asm path.""" + assert _heads(96) == 128 + assert _heads(1) == 16 + assert _heads(17) == 32 + assert _heads(65) == 128 + + +def test_head_count_above_the_largest_native_is_unservable(): + """0 means 'cannot serve'; anything else would route a shape the kernel + has no build for.""" + assert _heads(max(NATIVE) + 1) == 0 + assert _heads(256) == 0 + + +# -------------------------------------------------------------------------- +# _asm_dcp_verify_configured -- reachability +# -------------------------------------------------------------------------- + + +@pytest.fixture +def on_gfx950(monkeypatch): + """The cprr kernels are built only for gfx950, so the gate checks the arch. + Patched rather than detected, so both branches run on any runner.""" + import vllm.platforms.rocm as rocm + + monkeypatch.setattr(rocm, "on_gfx950", lambda: True) + + +def test_route_needs_dcp(on_gfx950): + assert _configured(dcp_world_size=1, cp_interleave=1, multi_token_decode=True) is ( + False + ) + assert _configured(dcp_world_size=8, cp_interleave=1, multi_token_decode=True) is ( + True + ) + + +def test_round_robin_interleave_other_than_one_is_excluded(on_gfx950): + """The kernel's global-position causal window assumes interleave 1, the + same restriction the segmented gate carries.""" + assert _configured(dcp_world_size=8, cp_interleave=2, multi_token_decode=True) is ( + False + ) + assert _configured(dcp_world_size=8, cp_interleave=16, multi_token_decode=True) is ( + False + ) + + +def test_route_needs_multi_token_decode(on_gfx950): + """Single-token decode is already served. Enabling the route for it would + pad the head count on every step and buy nothing.""" + assert _configured(dcp_world_size=8, cp_interleave=1, multi_token_decode=False) is ( + False + ) + + +def test_route_needs_gfx950(monkeypatch): + """hsa/gfx942/mla and hsa/gfx1250/mla carry no cprr rows, so off gfx950 the + kernel lookup fails at the first verify step rather than falling back.""" + import vllm.platforms.rocm as rocm + + monkeypatch.setattr(rocm, "on_gfx950", lambda: False) + assert _configured(dcp_world_size=8, cp_interleave=1, multi_token_decode=True) is ( + False + ) + + +# -------------------------------------------------------------------------- +# qlen floor +# -------------------------------------------------------------------------- + + +def test_min_cprr_qlen_is_above_two(): + """Qlen 2 is below CPRR's floor and must use segmented verification.""" + assert MIN_QLEN > 2 + + +@pytest.mark.parametrize( + "supports,causal,qlen,asm,expected", + [ + (True, True, 1, True, Route.PLAIN), + (True, True, 2, True, Route.SEGMENTED), + (True, True, 3, True, Route.CPRR), + (True, True, 4, True, Route.CPRR), + (True, True, 4, False, Route.SEGMENTED), + (True, False, 2, True, Route.PLAIN), + (True, False, 4, False, Route.PLAIN), + (False, True, 3, True, Route.CPRR), + ], +) +def test_dcp_decode_route(supports, causal, qlen, asm, expected): + assert _select_route(supports, causal, qlen, asm) is expected + + +def test_causal_multi_token_batch_without_a_valid_route_fails(): + with pytest.raises(RuntimeError, match="requires either segmented MLA"): + _select_route(False, True, 2, True) + + +# Kernel numerics live in test_rocm_aiter_mla_dcp_cprr_numerics.py, which runs +# the real builder and the real asm kernel on a gfx950 GPU. diff --git a/tests/v1/attention/test_rocm_aiter_mla_dcp_cprr_numerics.py b/tests/v1/attention/test_rocm_aiter_mla_dcp_cprr_numerics.py new file mode 100644 index 0000000000..44323dde41 --- /dev/null +++ b/tests/v1/attention/test_rocm_aiter_mla_dcp_cprr_numerics.py @@ -0,0 +1,540 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""GPU numerics for the AITER ASM cprr DCP-verify route. + +Runs the REAL metadata builder (``AiterMLAMetadataBuilder.build``, so +``get_mla_metadata_v1`` is exercised with the cprr kwargs and the persistent +schedule buffers are produced) and the REAL asm kernel, on ONE GPU with the DCP +ranks simulated one after another. + +That is legitimate because the aiter MLA backend contains no collective on this +path: the query all-gather happens in the *layer*, strictly before +``forward_mqa``, so the head fold, the round-robin kernel call and the un-fold +are pure per-rank local arithmetic in which ``cp_world_size``/``cp_rank`` are +just numbers feeding the causal-mask maths. Restricting a shard to a residue +class of global positions is exactly what DCP rank r sees. + +Three properties are checked: + * each shard matches an exact torch reference over the positions it holds, + with causality applied on the GLOBAL position (the thing cprr reconstructs + in-kernel and the segmented route instead carries in per-row lengths); + * the LSE merge of all shards reproduces full-context attention; + * a shard told the wrong ``cp_rank`` fails both -- without this positive + control a passing run proves nothing, since a kernel that ignored global + positions still produces plausible-looking output. +""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest +import torch + +from vllm.platforms import current_platform + +if not current_platform.is_rocm(): + pytest.skip("ROCm AITER MLA tests", allow_module_level=True) + +from vllm._aiter_ops import is_aiter_found # noqa: E402 +from vllm.v1.attention.backends.mla import rocm_aiter_mla # noqa: E402 + +KV_LORA_RANK = 512 +QK_ROPE_HEAD_DIM = 64 +QK_NOPE_HEAD_DIM = 128 +HEAD_SIZE = KV_LORA_RANK + QK_ROPE_HEAD_DIM +SCALE = (QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM) ** -0.5 + +DCP = 8 +NUM_REQS = 8 +CTX = 256 +QLEN = 5 # 1 + num_speculative_tokens; must clear _MIN_CPRR_QLEN + +# fp8-e4m3 keeps 3 mantissa bits, so the relative error floor is ~6.2e-2. Gates +# are relative for the same reason: an absolute bound would just be a bound on +# the output magnitude. +SHARD_RTOL = 1.0e-1 +MERGE_RTOL = 5.0e-2 + +MODEL = "deepseek-ai/DeepSeek-R1" + + +def _gpu_available() -> bool: + return is_aiter_found() and torch.accelerator.is_available() + + +def _on_gfx950() -> bool: + if not is_aiter_found(): + return False + from vllm.platforms.rocm import on_gfx950 + + return on_gfx950() + + +requires_cprr = pytest.mark.skipif( + not (_gpu_available() and _on_gfx950()), + reason="cprr DCP verify needs ROCm + AITER on gfx950", +) + + +@pytest.fixture(autouse=True) +def select_asm_route(monkeypatch): + monkeypatch.setenv("VLLM_ROCM_AITER_MLA_DCP_VERIFY", "asm") + + +# -------------------------------------------------------------------------- +# reference maths +# -------------------------------------------------------------------------- + + +def _torch_reference(q_f32, kv_f32, ctx, qlen, scale, positions): + """Exact absorbed-MLA decode over an explicit set of GLOBAL KV positions. + + A shard sees only ``positions``, but causality still uses the global + position: query token i (global position ctx + i) may attend to any p in + positions with p <= ctx + i. Restricting ``positions`` to a round-robin + residue class is precisely DCP rank r's view. + """ + nr, _, heads, _ = q_f32.shape + dev = q_f32.device + o = torch.zeros(nr, qlen, heads, KV_LORA_RANK, dtype=torch.float32, device=dev) + lse = torch.full((nr, qlen, heads), float("-inf"), dtype=torch.float32, device=dev) + for r in range(nr): + kv_sel = kv_f32[r].index_select(0, positions) + for i in range(qlen): + valid = positions <= (ctx + i) + if not bool(valid.any()): + continue + k = kv_sel[valid] + s = (q_f32[r, i] @ k.T) * scale + m = s.max(dim=-1, keepdim=True).values + p = torch.exp(s - m) + denom = p.sum(dim=-1, keepdim=True) + o[r, i] = (p / denom) @ k[:, :KV_LORA_RANK] + lse[r, i] = m.squeeze(-1) + torch.log(denom.squeeze(-1)) + return o, lse + + +def _merge_partials(os_, lses): + """Cross-shard softmax merge (natural-log LSE).""" + stack_lse = torch.stack(lses) + m = stack_lse.max(dim=0).values + w = torch.exp(stack_lse - m) + num = (w.unsqueeze(-1) * torch.stack(os_)).sum(dim=0) + den = w.sum(dim=0).unsqueeze(-1) + return num / den, m + torch.log(w.sum(dim=0)) + + +# -------------------------------------------------------------------------- +# single-process rank simulation +# -------------------------------------------------------------------------- + + +class _FakeGroup: + def __init__(self, world_size: int, rank: int): + self.world_size = world_size + self.rank_in_group = rank + self.rank = rank + self.local_rank = rank + + def all_gather(self, tensor, dim=0): + raise RuntimeError( + "collective called in a single-process test -- the path under test " + "is supposed to be collective-free; this indicates a real change" + ) + + +def _install_fake_groups(dcp_size: int, cp_rank: int, tp_size: int = 1) -> None: + """Point vLLM's parallel_state at rank-simulating stubs (no NCCL).""" + import vllm.distributed.parallel_state as ps + import vllm.model_executor.layers.attention.mla_attention as mla_attention + import vllm.v1.attention.backends.mla.rocm_aiter_mla as backend + + dcp = _FakeGroup(dcp_size, cp_rank) + tp = _FakeGroup(tp_size, cp_rank) + ps.get_dcp_group = lambda: dcp + ps.get_tp_group = lambda: tp + # Both of these imported the symbol directly at module import time, so + # patching parallel_state alone leaves them resolving the real group (and + # the builder then reports dcp_world_size 1, silently skipping the route). + backend.get_dcp_group = lambda: dcp + mla_attention.get_dcp_group = lambda: dcp + + +@pytest.fixture(autouse=True) +def _restore_parallel_groups(): + """Undo _install_fake_groups after every test. + + It patches module globals in three modules, so without this the fake DCP + group leaks into the rest of the pytest session and silently breaks + unrelated tests that expect a real (or absent) group. + """ + import vllm.distributed.parallel_state as ps + import vllm.model_executor.layers.attention.mla_attention as mla_attention + import vllm.v1.attention.backends.mla.rocm_aiter_mla as backend + + saved = [ + (mod, name, getattr(mod, name)) + for mod, name in ( + (ps, "get_dcp_group"), + (ps, "get_tp_group"), + (backend, "get_dcp_group"), + (mla_attention, "get_dcp_group"), + ) + ] + yield + for mod, name, value in saved: + setattr(mod, name, value) + + +class _PrefillBackendStub: + """The base builder does ``attention_layer.prefill_backend.clone()``. + + prefill_backend is consulted only from the chunked-context prefill path, + which a decode-only test never enters. + """ + + def clone(self): + return self + + +def _make_ctx_layer(vllm_config): + """Stand-in for the registered attention layer in static_forward_context. + + ``MLACommonMetadataBuilder.__init__`` reads the MLA dims, ``dcp_manager`` + and ``prefill_backend`` off this object. ``dcp_manager`` is the Direct-DCP + symmetric-memory workspace, used ONLY by the chunked-prefill KV all-gather; + it is constructed without running __init__ so no peer pointers are exchanged + and no collective is issued. That is what keeps this test single-process. + """ + from vllm.v1.attention.ops.dcp import MLADCPManager + + mgr = object.__new__(MLADCPManager) + mgr.init_kv_gather = lambda *a, **k: None + + return SimpleNamespace( + non_causal_multi_token_decode=False, + q_lora_rank=None, + kv_lora_rank=KV_LORA_RANK, + qk_nope_head_dim=QK_NOPE_HEAD_DIM, + qk_rope_head_dim=QK_ROPE_HEAD_DIM, + v_head_dim=QK_NOPE_HEAD_DIM, + dcp_manager=mgr, + prefill_backend=_PrefillBackendStub(), + ) + + +def _ensure_workspace_manager(device: torch.device) -> None: + """Init the global workspace manager, which GPUModelRunner does in production. + + ``AiterMLAMetadataBuilder.__init__`` reserves the fp8 prefill + persistent-scheduling scratch through ``current_workspace_manager()``, so + merely constructing a builder outside a serve trips its assert. Idempotent: + one builder is constructed per DCP shard. Deliberately does NOT lock the + workspace -- production locks only after warmup, and this keeps allocating + across shards. + """ + from vllm.v1.worker.workspace import ( + init_workspace_manager, + is_workspace_manager_initialized, + ) + + if not is_workspace_manager_initialized(): + init_workspace_manager(device) + + +class _Layer: + def __init__(self, device): + self._q_scale = torch.tensor(1.0, device=device) + self._k_scale = torch.tensor(1.0, device=device) + self._v_scale = torch.tensor(1.0, device=device) + + +def _build_impl(num_heads: int, dcp: int, cp_rank: int): + """Real AiterMLAImpl.forward_mqa bound to a minimally-populated instance. + + Constructing the full impl would drag in kv_b_proj and the rest of the + weight-dependent MLA setup -- i.e. model loading, which this test exists to + avoid. forward_mqa's fold path reads only the attributes set below. + """ + from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAImpl + + impl = object.__new__(AiterMLAImpl) + impl.kv_lora_rank = KV_LORA_RANK + impl.qk_rope_head_dim = QK_ROPE_HEAD_DIM + impl.num_heads = num_heads + impl.dcp_world_size = dcp + impl.dcp_rank = cp_rank + impl.pcp_world_size = 1 + impl.kv_cache_dtype = "fp8" + impl.scale = SCALE + # `_decode_num_heads` is a read-only property deriving num_heads * + # dcp_world_size, both set above; assert rather than assign. + assert impl._decode_num_heads == num_heads * dcp + return impl + + +def _make_vllm_config(heads_per_rank: int): + """A DCP8 MLA config with no weights and no distributed init. + + DeepSeek-R1's config supplies the MLA dims (kv_lora_rank 512, rope 64, + nope 128) that the aiter asm kernel is built for; only the head count is + overridden, to place the GATHERED count (heads_per_rank * DCP) on the + kernel variant under test. ``get_num_attention_heads`` reads + model_arch_config, a snapshot taken at config-creation time, so patching + hf_config alone would be inert. + """ + from tests.v1.attention.utils import create_vllm_config + + vllm_config = create_vllm_config( + model_name=MODEL, + tensor_parallel_size=1, + max_model_len=2048, + max_num_seqs=NUM_REQS * 2, + max_num_batched_tokens=4096, + ) + vllm_config.model_config.model_arch_config.total_num_attention_heads = ( + heads_per_rank + ) + # Without a speculative config the batch reorder threshold stays at 1 and a + # qlen-QLEN row is classified as a PREFILL, so the decode path under test is + # never reached. _init_reorder_batch_threshold reads exactly these two + # fields, so a stand-in is enough and avoids pulling in a draft model. + vllm_config.speculative_config = SimpleNamespace( + num_speculative_tokens=QLEN - 1, parallel_drafting=False + ) + vllm_config.cache_config.cache_dtype = "fp8" + vllm_config.parallel_config.decode_context_parallel_size = DCP + vllm_config.parallel_config.cp_kv_cache_interleave_size = 1 + return vllm_config + + +def _run_shard( + vllm_config, + spec, + layer_name, + device, + cp_rank, + kern_rank, + q_flat, + kv_flat, + nr, + qlen, + ctx, + dcp, + pos, +): + """Build real metadata for one DCP rank and run the real asm kernel. + + ``kern_rank`` is separate from ``cp_rank`` so the positive control can hand + rank r's pages to the kernel while telling it that it is rank r+1. + """ + from vllm.v1.attention.backend import CommonAttentionMetadata + from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAMetadataBuilder + + L = ctx + qlen + local_len = int(pos.numel()) + + # One GLOBAL kv cache: page (req*L + p) holds request req position p. Shards + # select pages by residue class, so every shard reads the same underlying + # bytes and no re-quantisation noise is introduced between arms. + block_table = ( + torch.arange(nr, device=device).view(nr, 1) * L + pos.view(1, -1) + ).to(torch.int32) + + # seq_lens is the GLOBAL context; dcp_local_seq_lens is what this rank + # actually holds. Passing the global length as the local one walks the + # kernel off the end of the page table. + seq_lens_cpu = torch.full((nr,), L, dtype=torch.int32) + local_cpu = torch.full((nr,), local_len, dtype=torch.int32) + + common = CommonAttentionMetadata( + query_start_loc=torch.arange( + 0, nr * qlen + 1, qlen, dtype=torch.int32, device=device + ), + query_start_loc_cpu=torch.arange(0, nr * qlen + 1, qlen, dtype=torch.int32), + seq_lens=seq_lens_cpu.to(device), + num_reqs=nr, + num_actual_tokens=nr * qlen, + max_query_len=qlen, + max_seq_len=L, + block_table_tensor=block_table, + slot_mapping=torch.arange(nr * qlen, dtype=torch.int64, device=device), + causal=True, + dcp_local_seq_lens=local_cpu.to(device), + dcp_local_seq_lens_cpu_upper_bound=local_cpu, + ) + + _install_fake_groups(dcp, kern_rank) + builder = AiterMLAMetadataBuilder(spec, [layer_name], vllm_config, device) + md = builder.build(0, common) + + impl = _build_impl(builder.num_heads, dcp, kern_rank) + o, lse = impl.forward_mqa(q_flat, kv_flat, md, _Layer(device)) + torch.accelerator.synchronize() + return md, o, lse, local_len + + +def _numerics(heads_per_rank: int, sabotage: bool = False): + """Run every DCP shard and return (per-shard max rel err, merged rel err).""" + import aiter + + from vllm.config import set_current_vllm_config + + device = torch.device("cuda", 0) + torch.accelerator.set_device_index(device.index) + torch.manual_seed(0) + _ensure_workspace_manager(device) + + _install_fake_groups(DCP, 0) + vllm_config = _make_vllm_config(heads_per_rank) + + with set_current_vllm_config(vllm_config): + from vllm.v1.kv_cache_interface import MLAAttentionSpec + + layer_name = "model.layers.0.self_attn.attn" + vllm_config.compilation_config.static_forward_context[layer_name] = ( + _make_ctx_layer(vllm_config) + ) + # block_size 1: one page per token, so page ids are positions and the + # residue-class page table is expressible directly. + spec = MLAAttentionSpec( + block_size=1, + num_kv_heads=1, + head_size=HEAD_SIZE, + dtype=vllm_config.model_config.dtype, + non_causal_multi_token_decode=False, + ) + + nr, ctx, qlen = NUM_REQS, CTX, QLEN + L = ctx + qlen + gathered_heads = heads_per_rank * DCP + kv_dtype = aiter.dtypes.fp8 + + kv_flat = torch.randn( + nr * L, 1, HEAD_SIZE, dtype=torch.float32, device=device + ).to(kv_dtype) + q_flat = ( + torch.randn( + nr * qlen, + gathered_heads, + HEAD_SIZE, + dtype=torch.float32, + device=device, + ) + .mul_(0.5) + .to(kv_dtype) + ) + # Dequantise for the reference so the comparison measures the kernel, + # not the fp8 cast that both sides share. + kv_f32 = kv_flat.float().view(nr, L, HEAD_SIZE) + q_f32 = q_flat.float().view(nr, qlen, gathered_heads, HEAD_SIZE) + + ref_o, _ = _torch_reference( + q_f32, kv_f32, ctx, qlen, SCALE, torch.arange(L, device=device) + ) + + shard_o, shard_lse = [], [] + max_shard_rel = 0.0 + for r in range(DCP): + pos = torch.arange(r, L, DCP, device=device) # residue class r + kern_rank = (r + 1) % DCP if sabotage else r + md, o, lse, _ = _run_shard( + vllm_config, + spec, + layer_name, + device, + r, + kern_rank, + q_flat, + kv_flat, + nr, + qlen, + ctx, + DCP, + pos, + ) + # The route actually under test must have been taken; a silent + # fall-out to Triton/segmented would otherwise pass quietly. + assert md.decode.dcp_route is rocm_aiter_mla._DCPDecodeRoute.CPRR + assert md.decode.asm_decode_num_heads, "cprr asm route was not taken" + + o = o.float().view(nr, qlen, gathered_heads, KV_LORA_RANK) + shard_lse.append(lse.float().view(nr, qlen, gathered_heads)) + shard_o.append(o) + + po, _ = _torch_reference(q_f32, kv_f32, ctx, qlen, SCALE, pos) + rel = (o - po).abs().max().item() / max(po.abs().max().item(), 1e-6) + max_shard_rel = max(max_shard_rel, rel) + + merged, _ = _merge_partials(shard_o, shard_lse) + merge_rel = (merged - ref_o).abs().max().item() / max( + ref_o.abs().max().item(), 1e-6 + ) + return max_shard_rel, merge_rel + + +# -------------------------------------------------------------------------- +# tests +# -------------------------------------------------------------------------- + + +@requires_cprr +@pytest.mark.parametrize( + "heads_per_rank", + [ + 16, # 128 gathered heads -- a NATIVE cprr kernel, no padding + 12, # 96 gathered heads -- K3's DSpark target, must pad up to 128 + ], + ids=["native-128", "padded-96-to-128"], +) +@torch.inference_mode() +def test_cprr_shards_match_reference_and_merge_to_full_context(heads_per_rank): + """Every DCP shard must be right, and their LSE merge must reproduce + full-context attention. + + A shard that ignored global positions (applying a local causal tail + instead) still produces plausible-looking output; it only diverges from the + reference on the tokens whose causal window is truncated. Checking the merge + alone would hide that, so both are asserted. + """ + shard_rel, merge_rel = _numerics(heads_per_rank) + assert shard_rel < SHARD_RTOL, f"per-shard rel err {shard_rel:.3e}" + assert merge_rel < MERGE_RTOL, f"merged rel err {merge_rel:.3e}" + + +@requires_cprr +@torch.inference_mode() +def test_wrong_cp_rank_breaks_the_result(): + """Positive control: without it, a passing run above proves nothing. + + Feeding shard r's pages to the kernel while telling it that it is rank r+1 + shifts every reconstructed global position by one, so both the causal mask + and the round-robin reconstruction are wrong. If the error does NOT blow up, + the kernel is not using cp_rank and the test above has no power. + """ + shard_rel, merge_rel = _numerics(16, sabotage=True) + assert shard_rel > SHARD_RTOL or merge_rel > MERGE_RTOL, ( + f"sabotaged run still passed (shard {shard_rel:.3e}, merge " + f"{merge_rel:.3e}) -- cp_rank is not reaching the kernel" + ) + + +@requires_cprr +def test_cprr_kernel_is_selected_for_the_k3_shape(): + """The K3 DSpark target shape must actually reach the cprr route. + + Guards against a silent fall-out to segmented/Triton, which is what the + stock configuration does and what this route exists to avoid. + """ + from vllm.v1.attention.backends.mla import rocm_aiter_mla as m + + assert ( + m._asm_dcp_verify_configured( + dcp_world_size=DCP, cp_interleave=1, multi_token_decode=True + ) + is True + ) + assert QLEN >= m._MIN_CPRR_QLEN + assert m._select_dcp_decode_route(True, True, QLEN, True) is m._DCPDecodeRoute.CPRR + assert m._asm_dcp_verify_heads(96) in m._NATIVE_CPRR_HEADS diff --git a/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py b/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py index 1970dbb54f..309bfd609c 100644 --- a/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py +++ b/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py @@ -87,21 +87,31 @@ def _builder( kv_cache_dtype: str = "auto", dcp_world_size: int = 1, dcp_rank: int = 0, + asm_dcp_verify: bool = False, + decode_causal: bool = True, + supports_segmented_dcp_verify: bool | None = None, ): + decode_num_heads = num_heads * dcp_world_size + asm_heads = ( + rocm_aiter_mla._asm_dcp_verify_heads(decode_num_heads) if asm_dcp_verify else 0 + ) + if supports_segmented_dcp_verify is None: + supports_segmented_dcp_verify = rocm_aiter_mla._segmented_dcp_verify_supported( + dcp_world_size, 1 + ) stub = SimpleNamespace( device=torch.device("cpu"), num_heads=num_heads, # DCP gathers every rank's head shard before decode, and the routing # predicates read the gathered count. - _decode_num_heads=num_heads * dcp_world_size, + _decode_num_heads=decode_num_heads, dcp_world_size=dcp_world_size, dcp_rank=dcp_rank, cp_kv_cache_interleave_size=1, # Mirrors the production constructor: configuration decides whether the - # segmented route is available, query length decides per batch. - _supports_segmented_dcp_verify=rocm_aiter_mla._segmented_dcp_verify_supported( - dcp_world_size, 1 - ), + # segmented route is available, query length decides per batch. CPRR + # preferred still keeps this True so qlen < _MIN_CPRR_QLEN can fall back. + _supports_segmented_dcp_verify=supports_segmented_dcp_verify, # Derived once in the real constructor, so derive it once here too. _segmented_page_size=rocm_aiter_mla._segmented_mla_page_size(kernel_block_size), _dcp_verify_buffers=None, @@ -122,7 +132,11 @@ def _builder( _uniform_padded_mtp_qo_len=(AiterMLAMetadataBuilder._uniform_padded_mtp_qo_len), _use_persistent_metadata=False, kernel_block_size=kernel_block_size, - _num_attention_heads=AiterMLAHelper.get_actual_mla_num_heads(num_heads), + _num_attention_heads=( + asm_heads + if asm_dcp_verify + else AiterMLAHelper.get_actual_mla_num_heads(num_heads) + ), _mla_work_meta_data=torch.empty(1, dtype=torch.int32), _mla_work_info_set=torch.empty(1, dtype=torch.int32), _mla_work_indptr=torch.empty(1, dtype=torch.int32), @@ -132,7 +146,19 @@ def _builder( _mla_q_dtype=torch.bfloat16, _mla_kv_dtype=torch.bfloat16, decode_attn_out_dtype=torch.bfloat16, - _decode_causal=True, + _decode_causal=decode_causal, + # Default off so existing cases keep exercising segmented/persistent. + # Pass asm_dcp_verify=True for the CPRR route and its qlen-floor fallback. + _asm_dcp_verify=asm_dcp_verify, + _asm_dcp_verify_heads=asm_heads, + _g_kv_indptr_buf=( + torch.zeros(max_decode_rows + 1, dtype=torch.int32) + if asm_dcp_verify and dcp_world_size > 1 + else None + ), + # Only read when the asm route is on; the real builder derives it from + # the device CU count. + _mla_max_split_per_batch=256, ) # Bound to the stub rather than faked: the verify flatten's per-row view is # part of what _build_decode is being tested for. @@ -287,6 +313,209 @@ def test_dcp_fp8_verify_build_uses_segmented(monkeypatch): assert metadata.dcp_verify is not None +def _cprr_decode_batch(builder, *, qlen: int, num_reqs: int = 1): + seq_lens = torch.full((num_reqs,), 16, dtype=torch.int32) + query_start_loc = torch.arange(0, (num_reqs + 1) * qlen, qlen, dtype=torch.int32) + return AiterMLAMetadataBuilder._build_decode( + builder, + block_table_tensor=torch.zeros(num_reqs, 8, dtype=torch.int32), + seq_lens_device=seq_lens, + max_seq_len=16, + query_start_loc_cpu=query_start_loc, + query_start_loc_device=query_start_loc, + num_decode_tokens=num_reqs * qlen, + max_query_len=qlen, + dcp_tot_seq_lens_device=seq_lens * 8, + ) + + +def test_asm_preferred_qlen2_falls_back_to_segmented(monkeypatch): + """K=4 can clamp to qlen 2; CPRR has no kernel, so use segmented MLA.""" + monkeypatch.setattr(rocm_aiter_mla, "_segmented_mla_decode_supported", lambda: True) + get_mla_metadata_v1 = mock.MagicMock() + monkeypatch.setitem( + sys.modules, + "aiter", + SimpleNamespace(get_mla_metadata_v1=get_mla_metadata_v1), + ) + + metadata = _cprr_decode_batch( + _builder( + mtp_decode_qlen=5, + dcp_world_size=8, + num_heads=12, + asm_dcp_verify=True, + kernel_block_size=2, + ), + qlen=2, + ) + + assert metadata.dcp_verify is not None + assert metadata.g_kv_indptr is None + assert not metadata.has_persistent_metadata + get_mla_metadata_v1.assert_not_called() + + +def test_asm_preferred_qlen_at_cprr_floor_stays_on_cprr(monkeypatch): + monkeypatch.setattr(rocm_aiter_mla, "_segmented_mla_decode_supported", lambda: True) + get_mla_metadata_v1 = mock.MagicMock() + monkeypatch.setitem( + sys.modules, + "aiter", + SimpleNamespace(get_mla_metadata_v1=get_mla_metadata_v1), + ) + monkeypatch.setattr( + rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel() + ) + + metadata = _cprr_decode_batch( + _builder( + mtp_decode_qlen=5, + dcp_world_size=8, + num_heads=12, + asm_dcp_verify=True, + ), + qlen=rocm_aiter_mla._MIN_CPRR_QLEN, + ) + + assert metadata.dcp_verify is None + assert metadata.g_kv_indptr is not None + assert metadata.has_persistent_metadata + assert get_mla_metadata_v1.call_args.kwargs.get("is_cp_round_robin") is True + + +def test_cprr_capable_single_token_decode_keeps_persistent_metadata(monkeypatch): + """The CPRR qlen floor must not suppress the ordinary qlen-1 schedule.""" + get_mla_metadata_v1 = mock.MagicMock() + monkeypatch.setitem( + sys.modules, + "aiter", + SimpleNamespace(get_mla_metadata_v1=get_mla_metadata_v1), + ) + monkeypatch.setattr( + rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel() + ) + + metadata = _cprr_decode_batch( + _builder( + mtp_decode_qlen=5, + dcp_world_size=8, + num_heads=12, + asm_dcp_verify=True, + ), + qlen=1, + ) + + assert metadata.g_kv_indptr is None + assert metadata.has_persistent_metadata + assert get_mla_metadata_v1.call_args.kwargs.get("is_cp_round_robin") is None + + +def test_non_causal_dcp_block_uses_plain_mask0_metadata(monkeypatch): + """A non-causal block needs no CPRR global-position window or head pad.""" + get_mla_metadata_v1 = mock.MagicMock() + monkeypatch.setitem( + sys.modules, + "aiter", + SimpleNamespace(get_mla_metadata_v1=get_mla_metadata_v1), + ) + monkeypatch.setattr( + rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel() + ) + builder = _builder( + mtp_decode_qlen=5, + dcp_world_size=8, + num_heads=12, + asm_dcp_verify=True, + ) + builder._decode_causal = False + + metadata = _cprr_decode_batch(builder, qlen=4) + + assert metadata.dcp_verify is None + assert metadata.g_kv_indptr is None + assert metadata.has_persistent_metadata + assert get_mla_metadata_v1.call_args.args[3] == 96 + assert get_mla_metadata_v1.call_args.args[5] is False + assert get_mla_metadata_v1.call_args.kwargs.get("is_cp_round_robin") is None + + +def test_asm_qlen2_without_segmented_fails_during_build(monkeypatch): + monkeypatch.setitem( + sys.modules, + "aiter", + SimpleNamespace(get_mla_metadata_v1=mock.MagicMock()), + ) + monkeypatch.setattr( + rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel() + ) + + with pytest.raises(RuntimeError, match="requires either segmented MLA"): + _cprr_decode_batch( + _builder( + mtp_decode_qlen=5, + dcp_world_size=8, + num_heads=12, + asm_dcp_verify=True, + supports_segmented_dcp_verify=False, + ), + qlen=2, + ) + + +def test_non_causal_dcp_block_bypasses_cprr_in_forward(monkeypatch): + """Process-level CPRR capability must not pad or route a non-causal batch.""" + captured = {} + + def fake_aiter_decode(q, kv_buffer, out, *args, **kwargs): + captured["q_heads"] = q.shape[1] + captured["kwargs"] = kwargs + return None, torch.zeros(q.shape[0], q.shape[1]) + + monkeypatch.setattr( + rocm_aiter_mla, "_get_aiter_mla_decode", lambda: fake_aiter_decode + ) + + impl = object.__new__(AiterMLAImpl) + impl.num_heads = 12 + impl.dcp_world_size = 8 + impl.kv_cache_dtype = "auto" + impl.kv_lora_rank = 512 + impl.qk_rope_head_dim = 64 + impl.scale = 576**-0.5 + decode = SimpleNamespace( + max_qo_len=4, + qo_indptr=torch.tensor([0, 4], dtype=torch.int32), + paged_kv_indptr=torch.tensor([0, 1], dtype=torch.int32), + paged_kv_indices=torch.zeros(1, dtype=torch.int32), + paged_kv_last_page_len=torch.ones(1, dtype=torch.int32), + use_gluon_decode=False, + use_gluon_verify=False, + dcp_route=rocm_aiter_mla._DCPDecodeRoute.PLAIN, + dcp_verify=None, + g_kv_indptr=None, + has_persistent_metadata=False, + attn_out_dtype=torch.bfloat16, + asm_decode_num_heads=0, + mla_num_kv_splits=0, + cp_world_size=8, + cp_rank=0, + min_kv_seq_len=1, + ) + attn_metadata = SimpleNamespace(decode=decode, causal=False, work_meta_data=None) + layer = SimpleNamespace(_q_scale=torch.tensor(1.0), _k_scale=torch.tensor(1.0)) + q = torch.zeros(4, 96, 576, dtype=torch.bfloat16) + + output, lse = impl.forward_mqa(q, torch.zeros(1, 1, 576), attn_metadata, layer) + + assert captured["q_heads"] == 96 + assert captured["kwargs"]["causal"] is False + assert "g_kv_indptr" not in captured["kwargs"] + assert "num_kv_splits" not in captured["kwargs"] + assert output.shape[1] == 96 + assert lse is not None and lse.shape == (4, 96) + + def test_single_token_dcp_decode_returns_unpadded_lse(monkeypatch): """Single-token DCP decode must come back with an LSE the merge can use. @@ -325,9 +554,14 @@ def test_single_token_dcp_decode_returns_unpadded_lse(monkeypatch): paged_kv_last_page_len=torch.ones(num_tokens, dtype=torch.int32), use_gluon_decode=False, use_gluon_verify=False, + dcp_route=rocm_aiter_mla._DCPDecodeRoute.PLAIN, dcp_verify=None, has_persistent_metadata=False, attn_out_dtype=torch.bfloat16, + # 0 == the round-robin (cprr) asm route is off, this rank's decode takes + # the existing path. Mirrors the AiterMLADecodeMetadata default. + asm_decode_num_heads=0, + g_kv_indptr=None, ) attn_metadata = SimpleNamespace(decode=decode, causal=True, work_meta_data=None) layer = SimpleNamespace(_q_scale=torch.tensor(1.0), _k_scale=torch.tensor(1.0)) @@ -453,6 +687,7 @@ def test_segmented_dcp_verify_matches_causal_attention(monkeypatch): ), use_gluon_decode=False, use_gluon_verify=False, + dcp_route=rocm_aiter_mla._DCPDecodeRoute.SEGMENTED, dcp_verify=view, attn_out_dtype=torch.bfloat16, ) diff --git a/tests/v1/attention/test_rocm_aiter_triton_sparse_mla.py b/tests/v1/attention/test_rocm_aiter_triton_sparse_mla.py new file mode 100644 index 0000000000..645eec2979 --- /dev/null +++ b/tests/v1/attention/test_rocm_aiter_triton_sparse_mla.py @@ -0,0 +1,326 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace + +import pytest +import torch + +from vllm.platforms import current_platform + +pytestmark = pytest.mark.skipif( + not current_platform.is_rocm(), reason="ROCm-only aiter path" +) + + +def _rows_from_ragged(indices: torch.Tensor, indptr: torch.Tensor) -> list[list[int]]: + ends = indptr.tolist() + return [indices[s:e].tolist() for s, e in zip(ends, ends[1:])] + + +def test_triton_sparse_mla_gate(monkeypatch) -> None: + import vllm._aiter_ops as aiter_ops + from vllm._aiter_ops import rocm_aiter_ops + from vllm.platforms.rocm import on_gfx950 + + warnings: list[str] = [] + monkeypatch.setattr( + aiter_ops.logger, + "warning_once", + lambda msg, *args, **kwargs: warnings.append(msg % args), + ) + + def enabled(aiter: bool, flag: bool) -> bool: + monkeypatch.setenv("VLLM_ROCM_USE_AITER", str(int(aiter))) + monkeypatch.setenv("VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA", str(int(flag))) + rocm_aiter_ops.refresh_env_variables() + return bool(rocm_aiter_ops.is_triton_sparse_mla_enabled()) + + try: + assert not enabled(aiter=True, flag=False) and not warnings + assert not enabled(aiter=False, flag=True) + assert "VLLM_ROCM_USE_AITER is off" in warnings.pop() + # On gfx950 the flag takes effect; elsewhere it warns and falls back. + assert enabled(aiter=True, flag=True) == on_gfx950() + if not on_gfx950(): + assert "gfx950" in warnings.pop() + finally: + monkeypatch.undo() + rocm_aiter_ops.refresh_env_variables() + + +@pytest.mark.parametrize( + "rope_dim", + [0, 64], + ids=["rope_free", "appended_rope"], # GLM-5.3-Flash; GLM-5.1/5.2, V3.2 +) +@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"]) +@torch.inference_mode() +def test_forward_mqa_prepares_triton_sparse_mla_inputs( + monkeypatch, rope_dim: int, kv_cache_dtype: str +) -> None: + """What forward_mqa hands aiter: global slots built from the request-local + top-k rows, q quantized with the layer's scale, and the cache as stored.""" + from vllm._aiter_ops import rocm_aiter_ops + from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import ( + ROCMAiterMLASparseImpl, + ) + + device = torch.device("cuda") + gen = torch.Generator().manual_seed(0) + kv_lora_rank, num_heads, topk, block_size = 512, 16, 128, 16 + head_size = kv_lora_rank + rope_dim + fp8 = kv_cache_dtype == "fp8" + + # A decode row, then a short prefill. + query_lens, seq_lens = [1, 5], [40, 20] + num_tokens = sum(query_lens) + block_table = torch.randperm(8, generator=gen).view(2, 4).to(device, torch.int32) + topk_indices = torch.full((num_tokens, topk), -1, dtype=torch.int32) + req_id_per_token: list[int] = [] + rows: list[list[int]] = [] + for req, (query_len, seq_len) in enumerate(zip(query_lens, seq_lens)): + for pos in range(seq_len - query_len, seq_len): + local = torch.randperm(pos + 1, generator=gen) + topk_indices[len(rows), : pos + 1] = local.int() + slots = block_table[req, local // block_size].cpu() * block_size + rows.append((slots + local % block_size).tolist()) + req_id_per_token.append(req) + indptr = torch.tensor([0] + [len(row) for row in rows]).cumsum(0) + + impl = ROCMAiterMLASparseImpl.__new__(ROCMAiterMLASparseImpl) + impl.num_heads = num_heads + impl.kv_lora_rank = kv_lora_rank + impl.qk_rope_head_dim = rope_dim + impl.scale = head_size**-0.5 + impl.sinks = torch.randn(num_heads, generator=gen).to(device) + impl.kv_cache_dtype = kv_cache_dtype + impl.topk_indices_buffer = topk_indices.to(device) + impl.use_aiter_sparse_mla = True + layer = SimpleNamespace( + _q_scale=torch.tensor([0.02], device=device), + _k_scale=torch.tensor([0.01], device=device), + ) + metadata = SimpleNamespace( + num_actual_tokens=num_tokens, + topk_tokens=topk, + req_id_per_token=torch.tensor(req_id_per_token, dtype=torch.int32).to(device), + block_table=block_table, + block_size=block_size, + paged_kv_indptr=indptr.to(device, torch.int32), + paged_kv_indices=torch.zeros(num_tokens * topk, dtype=torch.int32).to(device), + attn_out_dtype=torch.bfloat16, + ) + cache_dtype = current_platform.fp8_dtype() if fp8 else torch.bfloat16 + kv_cache = torch.zeros(8, block_size, head_size, device=device).to(cache_dtype) + # Two rows of cudagraph padding past the real tokens. + q = torch.randn(num_tokens + 2, num_heads, head_size, generator=gen).to( + device, torch.bfloat16 + ) + calls = [] + monkeypatch.setattr( + rocm_aiter_ops, "triton_sparse_mla_fwd", lambda *a, **k: calls.append((a, k)) + ) + + out, lse = impl.forward_mqa(q, kv_cache, metadata, layer) + + assert len(calls) == 1 + (q_in, kv_in, o, sm_scale, kv_indptr, kv_indices), kwargs = calls[0] + assert _rows_from_ragged(kv_indices, kv_indptr) == rows + assert q_in.shape[0] == num_tokens + if fp8: + assert q_in.dtype == cache_dtype + torch.testing.assert_close( + q_in.float() * layer._q_scale, q[:num_tokens].float(), atol=1e-3, rtol=0.07 + ) + else: + assert q_in.data_ptr() == q.data_ptr() + assert kv_in.shape == (8 * block_size, 1, 1, head_size) + assert kv_in.data_ptr() == kv_cache.data_ptr() + assert o.data_ptr() == out.data_ptr() and o.shape[0] == num_tokens + assert sm_scale == impl.scale + assert kwargs["kv_lora_rank"] == kv_lora_rank + assert kwargs["qk_rope_head_dim"] == rope_dim + assert kwargs["q_scale"] is layer._q_scale + assert kwargs["kv_scale"] is layer._k_scale + assert kwargs["attn_sink"] is impl.sinks + assert lse is None + + +@pytest.mark.parametrize( + ("model", "compress_ratio"), + [("deepseek_v41", 1), ("deepseek_v41", 2), ("deepseek_v4", 4)], +) +@torch.inference_mode() +def test_paged_prefill_indices_match_gathered_prefill( + monkeypatch, model: str, compress_ratio: int +) -> None: + """The aiter prefill reads the paged caches through the SWA builder's rows + and build_prefill_topk_ragged_indices, and must attend exactly the keys the + dequantize-and-gather prefill does.""" + import importlib + + from tests.v1.attention.utils import create_vllm_config + from vllm._aiter_ops import rocm_aiter_ops + from vllm.v1.attention.backend import CommonAttentionMetadata + from vllm.v1.attention.ops.rocm_aiter_mla_sparse import ( + build_prefill_topk_ragged_indices, + ) + from vllm.v1.kv_cache_interface import SlidingWindowMLASpec + + rocm = importlib.import_module(f"vllm.models.{model}.amd.rocm") + device = torch.device("cuda") + gen = torch.Generator().manual_seed(compress_ratio) + window, width, block_size, max_model_len = 16, 32, 16, 128 + comp_block = block_size // compress_ratio + num_compressed = max_model_len // compress_ratio + + # A decode row, then prefills with and without a prefix. + prefix_lens, query_lens = [20, 0, 13, 40], [1, 7, 20, 9] + seq_lens = [p + q for p, q in zip(prefix_lens, query_lens)] + num_decodes = num_decode_tokens = 1 + num_tokens = sum(query_lens) + qsl = torch.tensor([0] + query_lens).cumsum(0).to(torch.int32) + positions = torch.cat([torch.arange(p, s) for p, s in zip(prefix_lens, seq_lens)]) + token_to_req = torch.repeat_interleave( + torch.arange(len(query_lens)), torch.tensor(query_lens) + ) + swa_bt = torch.randperm(32, generator=gen)[:16].view(4, 4).to(device, torch.int32) + comp_bt = torch.randperm(32, generator=gen)[:16].view(4, 4).to(device, torch.int32) + + vllm_config = create_vllm_config( + model_name="facebook/opt-125m", + max_model_len=max_model_len, + block_size=block_size, + max_num_seqs=4, + max_num_batched_tokens=64, + hf_config_override={ + "compress_ratios": [compress_ratio], + "index_topk": width, + "sliding_window": window, + }, + ) + swa_spec = SlidingWindowMLASpec( + block_size=block_size, + num_kv_heads=1, + head_size=512, + dtype=torch.bfloat16, + sliding_window=window, + cache_dtype_str="fp8_ds_mla", + model_version="deepseek_v4", + ) + slots = swa_bt.cpu()[token_to_req, positions // block_size] * block_size + ( + positions % block_size + ) + common = CommonAttentionMetadata( + query_start_loc=qsl.to(device), + query_start_loc_cpu=qsl, + seq_lens=torch.tensor(seq_lens, dtype=torch.int32).to(device), + seq_lens_cpu_upper_bound=torch.tensor(seq_lens, dtype=torch.int32), + num_reqs=len(query_lens), + num_actual_tokens=num_tokens, + max_query_len=max(query_lens), + max_seq_len=max(seq_lens), + block_table_tensor=swa_bt, + slot_mapping=slots.to(device, torch.int64), + positions=positions.to(device), + causal=True, + ) + + def build(enabled: bool): + monkeypatch.setattr( + rocm_aiter_ops, "is_triton_sparse_mla_enabled", lambda: enabled + ) + return rocm.DeepseekV4ROCMAiterSparseSWAMetadataBuilder( + swa_spec, ["swa"], vllm_config, device + ).build(0, common) + + assert build(enabled=False).prefill_swa_ragged_indices is None + md = build(enabled=True) + assert md.num_decode_tokens == num_decode_tokens + + # Local top-k rows. Under each row's causal cap sit a failed candidate and + # one past the pool; past the cap, stale values neither path may read. + num_prefill_tokens = num_tokens - num_decode_tokens + topk = torch.full((num_prefill_tokens, width), -1, dtype=torch.int32) + for row in range(num_prefill_tokens): + visible = (int(positions[row + num_decode_tokens]) + 1) // compress_ratio + cap = min(visible, width) + topk[row, :cap] = torch.randperm(visible, generator=gen)[:cap].int() + if cap >= 2: + topk[row, :2] = torch.tensor([-1, num_compressed + 3]) + topk[row, cap:] = torch.randint( + 0, num_compressed, (width - cap,), generator=gen + ).int() + topk = topk.to(device) + + topk_indices, topk_indptr = build_prefill_topk_ragged_indices( + topk, + md.token_to_req_indices, + md.query_start_loc, + md.seq_lens, + md.is_valid_token, + comp_bt, + block_size=comp_block, + compress_ratio=compress_ratio, + num_compressed=num_compressed, + token_offset=num_decode_tokens, + ) + paged_keys = [ + sorted([("s", x) for x in swa if x >= 0] + [("c", x) for x in top if x >= 0]) + for swa, top in zip( + _rows_from_ragged( + md.prefill_swa_ragged_indices, md.prefill_swa_ragged_indptr + ), + _rows_from_ragged(topk_indices, topk_indptr), + ) + ] + + # The gathered path's combine, mapped from workspace offsets to cache slots. + M = num_compressed + window + num_tokens + combined, combined_lens = rocm.combine_topk_swa_indices( + topk, + md.query_start_loc[num_decodes:], + md.prefill_seq_lens, + md.prefill_gather_lens, + window, + compress_ratio, + width, + M, + num_compressed, + ) + gather_start = (md.prefill_seq_lens - md.prefill_gather_lens).tolist() + swa_bt_cpu, comp_bt_cpu = swa_bt.cpu(), comp_bt.cpu() + + def slot(block_table, req, pos, size): + return int(block_table[req, pos // size]) * size + pos % size + + gathered_keys = [] + for row, (values, n) in enumerate(zip(combined.tolist(), combined_lens.tolist())): + req = int(token_to_req[row + num_decode_tokens]) + chunk_req = req - num_decodes + keys = [] + for value in values[:n]: + if value < 0: + continue + offset = value - chunk_req * M + if offset < num_compressed: + keys.append(("c", slot(comp_bt_cpu, req, offset, comp_block))) + else: + pos = gather_start[chunk_req] + offset - num_compressed + keys.append(("s", slot(swa_bt_cpu, req, pos, block_size))) + gathered_keys.append(sorted(keys)) + + assert paged_keys == gathered_keys + + +@pytest.mark.parametrize("model", ["deepseek_v4", "deepseek_v41"]) +def test_aiter_sparse_mla_stays_off_with_kv_connector(monkeypatch, model) -> None: + import importlib + + from vllm._aiter_ops import rocm_aiter_ops + + rocm = importlib.import_module(f"vllm.models.{model}.amd.rocm") + monkeypatch.setattr(rocm_aiter_ops, "is_triton_sparse_mla_enabled", lambda: True) + assert rocm._aiter_sparse_mla_enabled(has_kv_transfer=False) + assert not rocm._aiter_sparse_mla_enabled(has_kv_transfer=True) diff --git a/tests/v1/core/test_async_scheduler.py b/tests/v1/core/test_async_scheduler.py index 52167a5073..2995537aac 100644 --- a/tests/v1/core/test_async_scheduler.py +++ b/tests/v1/core/test_async_scheduler.py @@ -13,7 +13,7 @@ from vllm.v1.request import RequestStatus from vllm.v1.structured_output import StructuredOutputGrammar from vllm.v1.utils import ConstantList -from .utils import create_requests, create_scheduler, mock_kv +from .utils import EOS_TOKEN_ID, create_requests, create_scheduler, mock_kv pytestmark = pytest.mark.cpu_test @@ -766,3 +766,48 @@ def test_kv_pressure_preempt_mid_handoff(kv_role: str, defer_free: bool): else: assert handoff.is_finished() assert handoff.num_output_tokens == 1 + + +def test_resumable_request_handoff(): + """New streaming input resumes with the correct context and stale results + are ignored. + """ + scheduler = create_scheduler( + async_scheduling=True, + use_v2_model_runner=True, + ) + engine = PipelinedEngine(scheduler, queue_size=3) + engine._next_token = EOS_TOKEN_ID + + old_prompt_len = 10 + continuation_len = 12 + request_id = "resumable-race" + + request = create_requests( + 1, num_tokens=old_prompt_len, max_tokens=4, req_ids=[request_id] + )[0] + request.resumable = True + continuation = create_requests( + 1, num_tokens=continuation_len, max_tokens=4, req_ids=[request_id] + )[0] + continuation.resumable = True + + scheduler.add_request(request) + + for _ in range(3): + assert engine._schedule() + + engine._process_oldest_step() + scheduler.add_request(continuation) + assert engine._schedule() + new_step, _ = engine.queue[0] + assert new_step.scheduled_new_reqs[0].num_computed_tokens == old_prompt_len + + all_token_ids_before_stale = list(request.all_token_ids) + for _ in range(2): + engine._process_oldest_step() + assert list(request.output_token_ids) == [] + assert list(request.all_token_ids) == all_token_ids_before_stale + + engine._process_oldest_step() + assert len(request.output_token_ids) == 1 diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index 505eef5919..91c93ed68f 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -2273,7 +2273,8 @@ def test_spec_decode_padding_skipped_with_prefill_in_batch(): def test_scheduler_stats_waiting_queues(): - """Test that scheduler stats correctly report waiting and skipped_waiting queues.""" + """Test that scheduler stats correctly report capacity-bound and blocked + waiting requests.""" # Create scheduler with limited capacity so we can have waiting requests scheduler = create_scheduler(max_num_batched_tokens=100) @@ -2286,18 +2287,18 @@ def test_scheduler_stats_waiting_queues(): for request in all_requests[:3]: scheduler.add_request(request) - # Manually add 2 more to skipped_waiting to simulate constraint-blocked + # Add 2 more that are blocked on grammar compilation for request in all_requests[3:]: - request.status = RequestStatus.WAITING_FOR_REMOTE_KVS - scheduler.skipped_waiting.add_request(request) + request.status = RequestStatus.WAITING_FOR_STRUCTURED_OUTPUT_GRAMMAR + scheduler.add_request(request) # Schedule - this will schedule 2 requests, leaving 1 in waiting output = scheduler.schedule() # Verify: 2 scheduled, 1 still waiting on capacity, 2 blocked by constraints assert len(output.scheduled_new_reqs) == 2 - assert len(scheduler.waiting) == 1 - assert len(scheduler.skipped_waiting) == 2 + assert len(scheduler.waiting) == 3 + assert len(scheduler.deferred_waiting) == 2 # Call update_from_output() to get frontend-facing stat scheduled_req_ids = list(output.num_scheduled_tokens.keys()) @@ -2395,7 +2396,7 @@ def _step_until_done( def _num_waiting_requests(scheduler: Scheduler) -> int: - return len(scheduler.waiting) + len(scheduler.skipped_waiting) + return len(scheduler.waiting) + len(scheduler.kv_holding_waiting) def _step_until_kv_transfer_finished(scheduler: Scheduler, req_ids: list[str]): @@ -2944,7 +2945,7 @@ def test_kv_connector_handles_preemption( if is_async: waiting_req_ids = [ req.request_id - for req in scheduler.skipped_waiting + for req in scheduler.kv_holding_waiting if req.status == RequestStatus.WAITING_FOR_REMOTE_KVS ] assert len(waiting_req_ids) == 1 @@ -3807,8 +3808,8 @@ def test_schedule_skip_tokenizer_init_structured_output_request(): output = scheduler.schedule() assert len(output.scheduled_new_reqs) == 0 assert len(scheduler.running) == 0 - assert len(scheduler.waiting) == 0 - assert len(scheduler.skipped_waiting) == 1 + assert len(scheduler.waiting) == 1 + assert len(scheduler.deferred_waiting) == 1 @pytest.mark.parametrize("async_grammar", [True, False]) @@ -5227,7 +5228,7 @@ def test_prepend_skipped_requests_order(): req.status = RequestStatus.WAITING_FOR_REMOTE_KVS scheduler.waiting.remove_requests(expected_waiting_reqs[:2]) for req in expected_waiting_reqs[:2]: - scheduler.skipped_waiting.add_request(req) + scheduler.kv_holding_waiting.add_request(req) # schedule step # expect the first 2 waiting to be skipped, the third running, @@ -5237,21 +5238,18 @@ def test_prepend_skipped_requests_order(): # pop the third request which is expected to be running expected_waiting_reqs.pop(2) - # verify waiting order is preserved - waiting_reqs = list(scheduler.skipped_waiting) + list(scheduler.waiting) + # verify waiting order is preserved, with the KV-holding requests first + waiting_reqs = list(scheduler.kv_holding_waiting) + list(scheduler.waiting) assert waiting_reqs == expected_waiting_reqs -def test_remote_kv_promotion_keeps_fcfs_with_grammar_prefix(): +def test_remote_kv_promotion_drains_before_grammar_prefix(): scheduler = create_scheduler(max_num_seqs=1) scheduler.connector = Mock() scheduler.connector.get_num_new_matched_tokens.return_value = (0, False) requests = create_requests(num_requests=4) - for request in requests: - scheduler.add_request(request) - - req_grammar_1, req_grammar_2, req_remote, req_tail = list(scheduler.waiting) + req_grammar_1, req_grammar_2, req_remote, req_tail = requests # simulate two structured-output grammar requests at the waiting head # that become ready now. @@ -5260,31 +5258,28 @@ def test_remote_kv_promotion_keeps_fcfs_with_grammar_prefix(): req_grammar_2.status = RequestStatus.WAITING_FOR_STRUCTURED_OUTPUT_GRAMMAR req_grammar_2.structured_output_request = Mock(grammar=object()) - # simulate a remote-KV request that is ready to be promoted now. + # simulate a remote-KV request that is ready to be promoted now. Parked + # loads hold KV blocks, so they drain ahead of the grammar requests. req_remote.status = RequestStatus.WAITING_FOR_REMOTE_KVS - scheduler.waiting.remove_requests([req_grammar_1, req_grammar_2, req_remote]) - scheduler.skipped_waiting.add_request(req_grammar_1) - scheduler.skipped_waiting.add_request(req_grammar_2) - scheduler.skipped_waiting.add_request(req_remote) + for request in requests: + scheduler.add_request(request) + assert list(scheduler.kv_holding_waiting) == [req_remote] scheduler.finished_recving_kv_req_ids.add(req_remote.request_id) scheduler._update_waiting_for_remote_kv = Mock() output = scheduler.schedule() assert output.scheduled_new_reqs - assert output.scheduled_new_reqs[0].req_id == req_grammar_1.request_id - waiting_req_ids = [ - req.request_id - for req in list(scheduler.skipped_waiting) + list(scheduler.waiting) - ] + assert output.scheduled_new_reqs[0].req_id == req_remote.request_id + waiting_req_ids = [req.request_id for req in scheduler.waiting] assert waiting_req_ids == [ + req_grammar_1.request_id, req_grammar_2.request_id, - req_remote.request_id, req_tail.request_id, ] -def test_fcfs_mixed_skipped_waiting_types_keep_order(): +def test_fcfs_mixed_blocked_types_remote_load_drains_first(): scheduler = create_scheduler(max_num_batched_tokens=20) scheduler._update_waiting_for_remote_kv = Mock() @@ -5305,7 +5300,8 @@ def test_fcfs_mixed_skipped_waiting_types_keep_order(): for req in (req_grammar, req_remote, req_stream, req_regular, req_tail): scheduler.add_request(req) scheduler.schedule() - assert list(scheduler.skipped_waiting) == [req_grammar, req_remote, req_stream] + assert list(scheduler.waiting) == [req_grammar, req_stream, req_tail] + assert list(scheduler.kv_holding_waiting) == [req_remote] scheduler.finish_requests(req_regular.request_id, RequestStatus.FINISHED_ABORTED) assert not scheduler.running @@ -5316,8 +5312,8 @@ def test_fcfs_mixed_skipped_waiting_types_keep_order(): second_output = scheduler.schedule() expected_order = [ - req_grammar.request_id, req_remote.request_id, + req_grammar.request_id, req_stream.request_id, req_tail.request_id, ] diff --git a/tests/v1/core/test_worker_slot_overflow.py b/tests/v1/core/test_worker_slot_overflow.py index 79366b0a23..fcf5cb8a9b 100644 --- a/tests/v1/core/test_worker_slot_overflow.py +++ b/tests/v1/core/test_worker_slot_overflow.py @@ -105,8 +105,8 @@ def test_streaming_pause_does_not_over_admit_worker_slot(): # still holds its slot. scheduler.update_from_output(out, _model_runner_output(["session"], [[STOP_TOKEN]])) assert session.status == RequestStatus.WAITING_FOR_STREAMING_REQ - # A blocked waiting status lands in skipped_waiting, not the main queue. - assert session in scheduler.skipped_waiting + # The paused session keeps its KV blocks, so it lands in kv_holding_waiting. + assert session in scheduler.kv_holding_waiting assert scheduler.num_waiting_for_streaming_input == 1 # A different request arrives while the session is paused. diff --git a/tests/v1/cudagraph/test_cudagraph_manager.py b/tests/v1/cudagraph/test_cudagraph_manager.py index d46476f8d0..7d2948cba4 100644 --- a/tests/v1/cudagraph/test_cudagraph_manager.py +++ b/tests/v1/cudagraph/test_cudagraph_manager.py @@ -224,7 +224,9 @@ def test_speculator_capture_preserves_decode_query_bounds( ) manager = SpeculatorCudaGraphManager.__new__(SpeculatorCudaGraphManager) manager.max_num_reqs = num_reqs - manager.dp_size = 1 + manager.vllm_config = SimpleNamespace( + parallel_config=SimpleNamespace(data_parallel_size=1) + ) buffers = InputBuffers(num_reqs, num_tokens, torch.device("cpu")) block_tables = MagicMock() block_tables.cp_size = 1 diff --git a/tests/v1/e2e/general/test_mamba_prefix_cache.py b/tests/v1/e2e/general/test_mamba_prefix_cache.py index 6433edac37..cae32e44dc 100644 --- a/tests/v1/e2e/general/test_mamba_prefix_cache.py +++ b/tests/v1/e2e/general/test_mamba_prefix_cache.py @@ -55,7 +55,10 @@ async_scheduling_mode = False num_accepted_tokens = 1 prompt_token_ids: list[int] = [] MODEL = "Qwen/Qwen3-Next-80B-A3B-Instruct-FP8" -BLOCK_SIZE = 560 +# Must be a value no platform rewrites, since the step actions below encode +# where block boundaries fall. XPU rounds GDN block sizes up to a multiple of 64 +# (XPUPlatform.update_block_size_for_backend), so use one. +BLOCK_SIZE = 576 DEVICE_TYPE = current_platform.device_type NUM_HIDDEN_LAYERS = 1 cur_step_action_idx = 0 @@ -541,26 +544,27 @@ def get_mamba_prefix_cache_step_configs( async_scheduling: bool = False, ) -> dict[str, TestConfig]: a = async_scheduling + B = BLOCK_SIZE tests = { "accept_1": TestConfig( - num_prompt_tokens=554, + num_prompt_tokens=B - 6, num_generated_tokens=20, num_accepted_tokens=1, step_actions=[ - StepAction(0, 554, [1, 1, 1, 1], (-1, -1), (-1, -1)), - StepAction(554, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(0, B - 6, [1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(B - 6, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)), StepAction( - 555, 4, [1, 1, 1, 1, 1] if a else [1, 1, 1, 1], (-1, -1), (-1, -1) + B - 5, 4, [1, 1, 1, 1, 1] if a else [1, 1, 1, 1], (-1, -1), (-1, -1) ), StepAction( - 556, 4, [1, 1, 1, 1, 1] if a else [1, 1, 1, 1], (-1, -1), (-1, -1) + B - 4, 4, [1, 1, 1, 1, 1] if a else [1, 1, 1, 1], (-1, -1), (-1, -1) ), - StepAction(557, 4, [1, 1, 1, 1, 1], (0, 1), (-1, -1)), - StepAction(558, 4, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)), - StepAction(559, 4, [1, 1, 1, 1, 1], (-1, -1), (1, 0)), - StepAction(560, 4, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(B - 3, 4, [1, 1, 1, 1, 1], (0, 1), (-1, -1)), + StepAction(B - 2, 4, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(B - 1, 4, [1, 1, 1, 1, 1], (-1, -1), (1, 0)), + StepAction(B, 4, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)), StepAction( - 561, + B + 1, 4, [1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1], (-1, -1), @@ -570,19 +574,19 @@ def get_mamba_prefix_cache_step_configs( ), # test case 2.1: no hit, accept 2 tokens "accept_2_1": TestConfig( - num_prompt_tokens=554, + num_prompt_tokens=B - 6, num_generated_tokens=20, num_accepted_tokens=2, step_actions=[ - StepAction(0, 554, [1, 1, 1, 1], (-1, -1), (-1, -1)), - StepAction(554, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(0, B - 6, [1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(B - 6, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)), StepAction( - 556, 4, [1, 1, 1, 1, 1] if a else [1, 1, 1, 1], (-1, -1), (-1, -1) + B - 4, 4, [1, 1, 1, 1, 1] if a else [1, 1, 1, 1], (-1, -1), (-1, -1) ), - StepAction(558, 4, [1, 1, 1, 1, 1], (1, 1), (2, 0)), - StepAction(560, 4, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(B - 2, 4, [1, 1, 1, 1, 1], (1, 1), (2, 0)), + StepAction(B, 4, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)), StepAction( - 562, + B + 2, 4, [1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1], (-1, -1), @@ -592,56 +596,56 @@ def get_mamba_prefix_cache_step_configs( ), # test case 2.2: no hit, accept 2 tokens "accept_2_2": TestConfig( - num_prompt_tokens=555, + num_prompt_tokens=B - 5, num_generated_tokens=20, num_accepted_tokens=2, step_actions=[ - StepAction(0, 555, [1, 1, 1, 1], (-1, -1), (-1, -1)), - StepAction(555, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)), - StepAction(557, 4, [1, 1, 1, 1, 1], (1, 1), (-1, -1)), - StepAction(559, 4, [1, 1, 1, 1, 1], (-1, -1), (1, 0)), + StepAction(0, B - 5, [1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(B - 5, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(B - 3, 4, [1, 1, 1, 1, 1], (1, 1), (-1, -1)), + StepAction(B - 1, 4, [1, 1, 1, 1, 1], (-1, -1), (1, 0)), StepAction( - 561, + B + 1, 4, [1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1], (-1, -1), (-1, -1), ), - StepAction(563, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(B + 3, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)), ], ), "accept_3_1": TestConfig( - num_prompt_tokens=553, + num_prompt_tokens=B - 7, num_generated_tokens=20, num_accepted_tokens=3, step_actions=[ - StepAction(0, 553, [1, 1, 1, 1], (-1, -1), (-1, -1)), - StepAction(553, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(0, B - 7, [1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(B - 7, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)), StepAction( - 556, 4, [1, 1, 1, 1, 1] if a else [1, 1, 1, 1], (-1, -1), (-1, -1) + B - 4, 4, [1, 1, 1, 1, 1] if a else [1, 1, 1, 1], (-1, -1), (-1, -1) ), - StepAction(559, 4, [1, 1, 1, 1, 1], (2, 1), (1, 0)), + StepAction(B - 1, 4, [1, 1, 1, 1, 1], (2, 1), (1, 0)), StepAction( - 562, + B + 2, 4, [1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1], (-1, -1), (-1, -1), ), - StepAction(565, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(B + 5, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)), ], ), "accept_3_2": TestConfig( - num_prompt_tokens=554, + num_prompt_tokens=B - 6, num_generated_tokens=20, num_accepted_tokens=3, step_actions=[ - StepAction(0, 554, [1, 1, 1, 1], (-1, -1), (-1, -1)), - StepAction(554, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)), - StepAction(557, 4, [1, 1, 1, 1, 1], (2, 1), (3, 0)), - StepAction(560, 4, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(0, B - 6, [1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(B - 6, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(B - 3, 4, [1, 1, 1, 1, 1], (2, 1), (3, 0)), + StepAction(B, 4, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)), StepAction( - 563, + B + 3, 4, [1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1], (-1, -1), @@ -650,87 +654,87 @@ def get_mamba_prefix_cache_step_configs( ], ), "accept_3_3": TestConfig( - num_prompt_tokens=555, + num_prompt_tokens=B - 5, num_generated_tokens=20, num_accepted_tokens=3, step_actions=[ - StepAction(0, 555, [1, 1, 1, 1], (-1, -1), (-1, -1)), - StepAction(555, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)), - StepAction(558, 4, [1, 1, 1, 1, 1], (2, 1), (2, 0)), + StepAction(0, B - 5, [1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(B - 5, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(B - 2, 4, [1, 1, 1, 1, 1], (2, 1), (2, 0)), StepAction( - 561, + B + 1, 4, [1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1], (-1, -1), (-1, -1), ), - StepAction(564, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(B + 4, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)), ], ), "accept_4_1": TestConfig( - num_prompt_tokens=553, + num_prompt_tokens=B - 7, num_generated_tokens=20, num_accepted_tokens=4, step_actions=[ - StepAction(0, 553, [1, 1, 1, 1], (-1, -1), (-1, -1)), - StepAction(553, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)), - StepAction(557, 4, [1, 1, 1, 1, 1], (3, 1), (3, 0)), + StepAction(0, B - 7, [1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(B - 7, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(B - 3, 4, [1, 1, 1, 1, 1], (3, 1), (3, 0)), StepAction( - 561, + B + 1, 4, [1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1], (-1, -1), (-1, -1), ), - StepAction(565, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(B + 5, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)), ], ), "accept_4_2": TestConfig( - num_prompt_tokens=554, + num_prompt_tokens=B - 6, num_generated_tokens=25, num_accepted_tokens=4, step_actions=[ - StepAction(0, 554, [1, 1, 1, 1], (-1, -1), (-1, -1)), - StepAction(554, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)), - StepAction(558, 4, [1, 1, 1, 1, 1], (3, 1), (2, 0)), + StepAction(0, B - 6, [1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(B - 6, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(B - 2, 4, [1, 1, 1, 1, 1], (3, 1), (2, 0)), StepAction( - 562, + B + 2, 4, [1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1], (-1, -1), (-1, -1), ), - StepAction(566, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(B + 6, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)), ], ), "accept_4_3": TestConfig( - num_prompt_tokens=555, + num_prompt_tokens=B - 5, num_generated_tokens=25, num_accepted_tokens=4, step_actions=[ - StepAction(0, 555, [1, 1, 1, 1], (-1, -1), (-1, -1)), - StepAction(555, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)), - StepAction(559, 4, [1, 1, 1, 1, 1], (3, 1), (1, 0)), + StepAction(0, B - 5, [1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(B - 5, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(B - 1, 4, [1, 1, 1, 1, 1], (3, 1), (1, 0)), StepAction( - 563, + B + 3, 4, [1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1], (-1, -1), (-1, -1), ), - StepAction(567, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(B + 7, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)), ], ), "accept_4_4": TestConfig( - num_prompt_tokens=556, + num_prompt_tokens=B - 4, num_generated_tokens=25, num_accepted_tokens=4, step_actions=[ - StepAction(0, 556, [1, 1, 1, 1], (-1, -1), (-1, -1)), - StepAction(556, 4, [1, 1, 1, 1], (-1, -1), (3, 0)), - StepAction(560, 4, [1, 1, 1, 1, 1], (0, 1), (-1, -1)), + StepAction(0, B - 4, [1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(B - 4, 4, [1, 1, 1, 1], (-1, -1), (3, 0)), + StepAction(B, 4, [1, 1, 1, 1, 1], (0, 1), (-1, -1)), StepAction( - 564, + B + 4, 4, [1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1], (-1, -1), @@ -739,23 +743,23 @@ def get_mamba_prefix_cache_step_configs( ], ), "prompt_block_size": TestConfig( - num_prompt_tokens=560, + num_prompt_tokens=B, num_generated_tokens=10, num_accepted_tokens=4, step_actions=[ - StepAction(0, 560, [1, 1, 1, 1], (-1, -1), (-1, -1)), - StepAction(560, 4, [1, 1, 1, 1, 1], (0, 1), (-1, -1)), + StepAction(0, B, [1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(B, 4, [1, 1, 1, 1, 1], (0, 1), (-1, -1)), ], ), "prompt_2_block_size": TestConfig( - num_prompt_tokens=560 * 2, + num_prompt_tokens=B * 2, num_generated_tokens=10, num_accepted_tokens=4, step_actions=[ - StepAction(0, 560, [1, 1, 1, 1], (-1, -1), (-1, -1)), - StepAction(560, 560, [1, 1, 1, 1, 1], (0, 1), (-1, -1)), + StepAction(0, B, [1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(B, B, [1, 1, 1, 1, 1], (0, 1), (-1, -1)), StepAction( - 560 * 2, + B * 2, 4, [1, 1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1, 1], (1, 2), @@ -764,14 +768,14 @@ def get_mamba_prefix_cache_step_configs( ], ), "prompt_2_block_size_10": TestConfig( - num_prompt_tokens=560 * 2 + 10, + num_prompt_tokens=B * 2 + 10, num_generated_tokens=10, num_accepted_tokens=4, step_actions=[ - StepAction(0, 560, [1, 1, 1, 1], (-1, -1), (-1, -1)), - StepAction(560, 570, [1, 0, 1, 1, 1, 1], (0, 2), (-1, -1)), + StepAction(0, B, [1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(B, B + 10, [1, 0, 1, 1, 1, 1], (0, 2), (-1, -1)), StepAction( - 560 * 2 + 10, + B * 2 + 10, 4, [1, 0, 1, 1, 1, 1] if a else [0, 0, 1, 1, 1, 1], (-1, -1), @@ -780,14 +784,14 @@ def get_mamba_prefix_cache_step_configs( ], ), "prompt_3_block_size": TestConfig( - num_prompt_tokens=560 * 3, + num_prompt_tokens=B * 3, num_generated_tokens=10, num_accepted_tokens=4, step_actions=[ - StepAction(0, 560 * 2, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)), - StepAction(560 * 2, 560, [0, 1, 1, 1, 1, 1], (1, 2), (-1, -1)), + StepAction(0, B * 2, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(B * 2, B, [0, 1, 1, 1, 1, 1], (1, 2), (-1, -1)), StepAction( - 560 * 3, + B * 3, 4, [0, 1, 1, 1, 1, 1, 1] if a else [0, 0, 1, 1, 1, 1, 1], (2, 3), @@ -796,14 +800,14 @@ def get_mamba_prefix_cache_step_configs( ], ), "prompt_3_block_size_10": TestConfig( - num_prompt_tokens=560 * 3 + 10, + num_prompt_tokens=B * 3 + 10, num_generated_tokens=10, num_accepted_tokens=4, step_actions=[ - StepAction(0, 560 * 2, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)), - StepAction(560 * 2, 570, [0, 1, 0, 1, 1, 1, 1], (1, 3), (-1, -1)), + StepAction(0, B * 2, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(B * 2, B + 10, [0, 1, 0, 1, 1, 1, 1], (1, 3), (-1, -1)), StepAction( - 560 * 3 + 10, + B * 3 + 10, 4, [0, 1, 0, 1, 1, 1, 1] if a else [0, 0, 0, 1, 1, 1, 1], (-1, -1), @@ -812,21 +816,21 @@ def get_mamba_prefix_cache_step_configs( ], ), "prompt_10_block_size": TestConfig( - num_prompt_tokens=560 * 10, + num_prompt_tokens=B * 10, num_generated_tokens=10, num_accepted_tokens=4, step_actions=[ - StepAction(0, 560 * 5, [0, 0, 0, 0, 1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(0, B * 5, [0, 0, 0, 0, 1, 1, 1, 1], (-1, -1), (-1, -1)), StepAction( - 560 * 5, - 560 * 4, + B * 5, + B * 4, [0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1], (4, 8), (-1, -1), ), StepAction( - 560 * 9, - 560, + B * 9, + B, [0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1] if a else [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1], @@ -834,7 +838,7 @@ def get_mamba_prefix_cache_step_configs( (-1, -1), ), StepAction( - 560 * 10, + B * 10, 4, [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1] if a @@ -845,21 +849,21 @@ def get_mamba_prefix_cache_step_configs( ], ), "prompt_10_block_size_10": TestConfig( - num_prompt_tokens=560 * 10 + 10, + num_prompt_tokens=B * 10 + 10, num_generated_tokens=10, num_accepted_tokens=4, step_actions=[ - StepAction(0, 560 * 5, [0, 0, 0, 0, 1, 1, 1, 1], (-1, -1), (-1, -1)), + StepAction(0, B * 5, [0, 0, 0, 0, 1, 1, 1, 1], (-1, -1), (-1, -1)), StepAction( - 560 * 5, - 560 * 4, + B * 5, + B * 4, [0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1], (4, 8), (-1, -1), ), StepAction( - 560 * 9, - 560 + 10, + B * 9, + B + 10, [0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 1, 1, 1] if a else [0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1], diff --git a/tests/v1/engine/test_engine_core_client.py b/tests/v1/engine/test_engine_core_client.py index 15dec72a41..add424d32a 100644 --- a/tests/v1/engine/test_engine_core_client.py +++ b/tests/v1/engine/test_engine_core_client.py @@ -215,6 +215,10 @@ def _make_dplb_client(num_engines: int = 3, client_count: int = 1) -> DPLBAsyncM client.lb_engines = [[0, 0, 0.0] for _ in range(num_engines)] client.eng_start_index = 0 client._kv_event_sources = {} + client.vllm_config = SimpleNamespace( + use_v2_model_runner=True, + parallel_config=SimpleNamespace(all2all_backend="allgather_reducescatter"), + ) return client @@ -248,6 +252,19 @@ def test_dplb_non_late_interaction_still_uses_lb(): assert client.lb_engines[1][0] == 1 +def test_dplb_in_flight_request_stays_on_same_engine(): + client = _make_dplb_client() + request = make_request(SamplingParams(max_tokens=1)) + engine = client.core_engines[2] + client.reqs_in_flight[request.request_id] = engine + client.engine_inflight[engine] = 1 + + chosen_engine = client.get_core_engine_for_request(request) + + assert chosen_engine == engine + assert client.engine_inflight[engine] == 1 + + def test_dplb_burst_round_robins_despite_snapshot_rebinds(): """A stats snapshot rebind wipes the optimistic lb_engines increments; the exact in-flight floor must keep a burst spreading round-robin.""" diff --git a/tests/v1/engine/test_logprobs_processor.py b/tests/v1/engine/test_logprobs_processor.py index edb8cef518..ee404eccfa 100644 --- a/tests/v1/engine/test_logprobs_processor.py +++ b/tests/v1/engine/test_logprobs_processor.py @@ -11,8 +11,10 @@ row), the trailing positions are populated with sentinel values """ import numpy as np +import torch from vllm.logprobs import create_sample_logprobs +from vllm.v1.engine import EngineCoreOutput from vllm.v1.engine.logprobs import LogprobsProcessor from vllm.v1.outputs import LogprobsLists @@ -64,3 +66,24 @@ def test_accepts_exactly_sized_row(): pos = processor.logprobs[0] assert set(pos.keys()) == {7, 11, 13} + + +def test_prompt_token_id_logprobs_are_popped_once(): + """DELTA outputs carry fixed-ID prompt scores exactly once.""" + processor = _make_processor(num_logprobs=1) + assert processor.pop_prompt_token_id_logprobs() is None + + processor.update_from_output( + EngineCoreOutput( + request_id="req-0", + new_token_ids=[], + prompt_token_id_logprobs=torch.tensor( + [[-0.5, -1.5], [-2.5, -3.5]], dtype=torch.float32 + ), + ) + ) + + scores = processor.pop_prompt_token_id_logprobs() + assert scores is not None + assert scores.tolist() == [[-0.5, -1.5], [-2.5, -3.5]] + assert processor.pop_prompt_token_id_logprobs() is None diff --git a/tests/v1/kv_connector/unit/test_error_propagation.py b/tests/v1/kv_connector/unit/test_error_propagation.py index 2f33ab18b9..91ec525f5c 100644 --- a/tests/v1/kv_connector/unit/test_error_propagation.py +++ b/tests/v1/kv_connector/unit/test_error_propagation.py @@ -119,7 +119,7 @@ def test_error_propagation_async_load(fail_scheduler: Scheduler): scheduler_output = fail_scheduler.schedule() - assert len(fail_scheduler.skipped_waiting) == 1 + assert len(fail_scheduler.kv_holding_waiting) == 1 assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS assert request.num_computed_tokens == num_external_computed_tokens @@ -145,4 +145,5 @@ def test_error_propagation_async_load(fail_scheduler: Scheduler): assert output.finish_reason == FinishReason.ERROR assert len(fail_scheduler.waiting) == 0 - assert len(fail_scheduler.skipped_waiting) == 0 + assert len(fail_scheduler.kv_holding_waiting) == 0 + assert not fail_scheduler.deferred_waiting diff --git a/tests/v1/kv_connector/unit/test_invalid_blocks_correctness.py b/tests/v1/kv_connector/unit/test_invalid_blocks_correctness.py index 41fce6e9ca..777bada58b 100644 --- a/tests/v1/kv_connector/unit/test_invalid_blocks_correctness.py +++ b/tests/v1/kv_connector/unit/test_invalid_blocks_correctness.py @@ -333,7 +333,7 @@ def test_async_recompute_blocks_not_cached_when_invalid( scheduler_output = recompute_scheduler.schedule() # request should be waiting for remote KVs - assert len(recompute_scheduler.skipped_waiting) == 1 + assert len(recompute_scheduler.kv_holding_waiting) == 1 assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS assert request.num_computed_tokens == num_external_computed_tokens diff --git a/tests/v1/kv_connector/unit/test_kv_load_failure_recovery.py b/tests/v1/kv_connector/unit/test_kv_load_failure_recovery.py index a8bc2d0e12..108eb049d9 100644 --- a/tests/v1/kv_connector/unit/test_kv_load_failure_recovery.py +++ b/tests/v1/kv_connector/unit/test_kv_load_failure_recovery.py @@ -111,8 +111,8 @@ def test_async_load_failure( scheduler_output = scheduler.schedule() assert len(scheduler.waiting) == 0 - assert len(scheduler.skipped_waiting) == 3 - for request in scheduler.skipped_waiting: + assert len(scheduler.kv_holding_waiting) == 3 + for request in scheduler.kv_holding_waiting: assert request.num_computed_tokens == num_external_computed_tokens assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS assert scheduler.connector.get_num_new_matched_tokens.call_count == 3 @@ -132,8 +132,8 @@ def test_async_load_failure( min_invalid_block_idx = min(invalid_block_idxs) assert len(scheduler.waiting) == 0 - assert len(scheduler.skipped_waiting) == 3 - for request in scheduler.skipped_waiting: + assert len(scheduler.kv_holding_waiting) == 3 + for request in scheduler.kv_holding_waiting: if request.request_id == request2.request_id: assert request.num_computed_tokens == ( min_invalid_block_idx * scheduler.block_size @@ -340,8 +340,9 @@ def test_async_progressive_load_failure( scheduler_output = scheduler.schedule() assert len(scheduler.waiting) == 0 - assert len(scheduler.skipped_waiting) == 1 - assert scheduler.skipped_waiting.peek_request().request_id == request.request_id + assert len(scheduler.kv_holding_waiting) == 1 + peeked = scheduler.kv_holding_waiting.peek_request() + assert peeked.request_id == request.request_id assert request.num_computed_tokens == num_external_computed_tokens assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS assert scheduler.connector.get_num_new_matched_tokens.call_count == 1 @@ -363,8 +364,9 @@ def test_async_progressive_load_failure( min_invalid_block_idx = min(min_invalid_block_idx, invalid_block_idx) assert len(scheduler.waiting) == 0 - assert len(scheduler.skipped_waiting) == 1 - assert scheduler.skipped_waiting.peek_request().request_id == request.request_id + assert len(scheduler.kv_holding_waiting) == 1 + peeked = scheduler.kv_holding_waiting.peek_request() + assert peeked.request_id == request.request_id assert request.num_computed_tokens == ( min_invalid_block_idx * scheduler.block_size ) diff --git a/tests/v1/kv_connector/unit/test_nixl_connector.py b/tests/v1/kv_connector/unit/test_nixl_connector.py index d8b6fcb5a0..017dee10d8 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector.py @@ -3026,7 +3026,7 @@ def test_empty_recv_is_reported_only_when_awaited( request is named in neither _recving_transfers nor _failed_recv_reqs and never reaches finished_recving. The scheduler has no other way to release a WAITING_FOR_REMOTE_KVS request, and there is no timeout, so it sits in - skipped_waiting holding its blocks for the life of the process. + kv_holding_waiting holding its blocks for the life of the process. Notify-only (awaiting_kvs=False): request_finished seeding an empty recv to free P's blocks for a request aborted before it was scheduled, or a readback diff --git a/tests/v1/kv_connector/unit/test_nixl_desc_geometry.py b/tests/v1/kv_connector/unit/test_nixl_desc_geometry.py index f2999bdb13..029ffafd3c 100644 --- a/tests/v1/kv_connector/unit/test_nixl_desc_geometry.py +++ b/tests/v1/kv_connector/unit/test_nixl_desc_geometry.py @@ -16,10 +16,27 @@ from threading import Event, Lock from types import SimpleNamespace from unittest.mock import MagicMock, patch +import msgspec import numpy as np import pytest import torch +from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker as bw +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( + NixlAgentMetadata, +) +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import ( + NixlPushConnectorWorker, +) +from vllm.v1.kv_cache_interface import ( + KVCacheConfig, + KVCacheGroupSpec, + KVCacheLayout, + KVCacheTensor, + MLAAttentionSpec, + create_kv_cache_views, +) + from .utils import create_vllm_config @@ -102,6 +119,189 @@ class _RecordingNixl: pass +def _make_packed_mla_view_worker( + layouts, + block_stride, + *, + num_blocks=4, + pp_size=1, + backend_names=("FLASHMLA",), +): + """Register real strided views; only NIXL and distributed runtime are fake.""" + block_size = 16 + raw = torch.zeros(num_blocks * block_stride, dtype=torch.int8) + tensors, groups, caches = [], [], {} + for name, (offset, page_size) in layouts.items(): + spec = MLAAttentionSpec( + block_size=block_size, + num_kv_heads=1, + head_size=page_size // block_size, + dtype=torch.uint8, + ) + tensor = KVCacheTensor( + size=raw.nbytes, + layers=[name], + layer_stride=page_size, + block_stride=block_stride, + offset=offset, + ) + tensors.append(tensor) + groups.append(KVCacheGroupSpec([name], spec)) + (caches[name],) = create_kv_cache_views( + raw, spec, num_blocks, KVCacheLayout.BLHNC, tensor + ) + + config = create_vllm_config(block_size=block_size) + config.parallel_config.pipeline_parallel_size = pp_size + config.kv_transfer_config.kv_role = "kv_producer" + config.cache_config.kv_cache_layout = "BLHNC" + backends = [] + for name in backend_names: + backend = MagicMock() + backend.get_supported_kernel_block_sizes.return_value = [block_size] + backend.get_name.return_value = name + backend.full_cls_name.return_value = f"fake.{name}" + backends.append(backend) + with ( + patch.object(bw, "NixlWrapper", _RecordingNixl), + patch.object(bw, "get_tensor_model_parallel_rank", return_value=0), + patch.object(bw, "get_tensor_model_parallel_world_size", return_value=1), + patch.object(bw, "get_current_attn_backends", return_value=backends), + patch("threading.Thread"), + ): + worker = NixlPushConnectorWorker( + config, + "producer" if pp_size > 1 else "consumer", + KVCacheConfig(num_blocks, tensors, groups), + ) + worker.use_mla = True + worker.register_kv_caches(caches) + return worker, raw + + +@pytest.mark.cpu_test +@pytest.mark.parametrize("num_layers", [1, 2]) +def test_packed_mla_pp1_push_peer_transfers_whole_rows(num_layers): + layouts = {f"L{i}": (i * 128, 128) for i in range(num_layers)} + stride = num_layers * 128 + worker, raw = _make_packed_mla_view_worker(layouts, stride) + assert worker._registered_descs == [[(raw.data_ptr(), raw.nbytes, 0, "")]] + assert worker.src_blocks_data.tolist() == [ + [raw.data_ptr() + block * stride, stride, 0] for block in range(4) + ] + assert worker._transfer_layer_group_ids == () + metadata = msgspec.msgpack.decode( + worker.xfer_handshake_metadata.agent_metadata_bytes, type=NixlAgentMetadata + ) + assert metadata.packed_member_layouts == layouts + + +@pytest.mark.cpu_test +@pytest.mark.parametrize( + "remote_backends, compatible", + [ + (("INDEXER", "FLASHMLA"), True), + (("FLASHMLA", "OTHER"), False), + (("FLASHMLA",), False), + ], +) +def test_packed_push_compatibility_hash_uses_all_backends_in_stable_order( + remote_backends, compatible +): + producer, _ = _make_packed_mla_view_worker( + {"L0": (0, 128)}, + 128, + pp_size=2, + backend_names=("FLASHMLA", "INDEXER"), + ) + consumer, _ = _make_packed_mla_view_worker( + {"L0": (0, 128)}, 128, backend_names=remote_backends + ) + assert (producer.compat_hash == consumer.compat_hash) is compatible + + +@pytest.mark.cpu_test +@pytest.mark.parametrize("p_num_blocks, d_num_blocks", [(4, 4), (4, 6), (6, 4)]) +def test_packed_mla_pp_pairs_asymmetric_strides_and_overlapping_layers( + p_num_blocks, d_num_blocks +): + # Different cache groups overlay pages of different sizes at the same address. + # PP also changes both the placement and the stride of the matching D pages. + producer, p_raw = _make_packed_mla_view_worker( + {"L2": (0, 128), "L2.swa": (0, 64), "L3": (128, 64)}, + 192, + num_blocks=p_num_blocks, + pp_size=2, + ) + consumer, d_raw = _make_packed_mla_view_worker( + {"L0": (0, 128), "L2": (128, 128), "L2.swa": (0, 64), "L3": (64, 64)}, + 256, + num_blocks=d_num_blocks, + ) + assert producer.block_len_per_layer == [128, 64, 64] + assert producer._transfer_layer_group_ids == (0, 1, 2) + assert producer.num_descs == 3 * p_num_blocks + local_ids = producer._compute_desc_ids( + [[1], [2], [3]], + producer.num_blocks, + None, + 1, + region_num_blocks=producer.dst_region_num_blocks[producer.engine_id], + ) + assert producer.src_blocks_data[local_ids].tolist() == [ + [p_raw.data_ptr() + 192, 128, 0], + [p_raw.data_ptr() + 2 * 192, 64, 0], + [p_raw.data_ptr() + 128 + 3 * 192, 64, 0], + ] + # Each decoder TP rank receives the complete MLA page, without head slicing. + for rank in (0, 1): + metadata = msgspec.msgpack.decode( + consumer.xfer_handshake_metadata.agent_metadata_bytes, + type=NixlAgentMetadata, + ) + producer.add_remote_agent(metadata, remote_tp_rank=rank, remote_tp_size=2) + assert metadata.region_num_blocks == [d_num_blocks] * 3 + assert metadata.region_names == ["L2", "L2.swa", "L3"] + remote_ids = producer._compute_desc_ids( + [[3], [1], [2]], + metadata.num_blocks, + None, + 1, + region_num_blocks=producer.dst_region_num_blocks[consumer.engine_id], + ) + handle = producer.dst_xfer_side_handles[consumer.engine_id][rank] + remote_descs = producer.nixl_wrapper.dlists[handle] + assert len(remote_descs) == 3 * d_num_blocks + assert remote_descs[remote_ids].tolist() == [ + [d_raw.data_ptr() + 128 + 3 * 256, 128, 0], + [d_raw.data_ptr() + 256, 64, 0], + [d_raw.data_ptr() + 64 + 2 * 256, 64, 0], + ] + aligned = msgspec.msgpack.encode(metadata) + producer._align_remote_regions_by_layer(metadata) + assert msgspec.msgpack.encode(metadata) == aligned + + +@pytest.mark.cpu_test +@pytest.mark.parametrize("remote_block_size", [8, 32]) +def test_packed_mla_rejects_unequal_block_sizes_before_peer_registration( + remote_block_size, +): + producer, _ = _make_packed_mla_view_worker({"L0": (0, 128)}, 128, pp_size=2) + metadata = msgspec.msgpack.decode( + producer.xfer_handshake_metadata.agent_metadata_bytes, type=NixlAgentMetadata + ) + metadata.engine_id = "remote" + metadata.block_size = remote_block_size + with pytest.raises(NotImplementedError, match="identical P/D block sizes"): + producer.add_remote_agent(metadata) + assert len(producer.nixl_wrapper.dlists) == 1 # Only the local list exists. + assert producer.tp_mappings == {} + assert "remote" not in producer.dst_num_blocks + with pytest.raises(KeyError): + producer.transfer_topo.get_engine_info("remote") + + @pytest.mark.cpu_test def test_local_descriptors_follow_each_region_pool_capacity(): from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import ( @@ -200,8 +400,8 @@ def _register_overlaid_mla_worker( worker.src_xfer_handles_by_block_size = {} worker.kv_caches_base_addr = defaultdict(dict) worker._mamba_ssm_size = (0, 0) - worker.kv_cache_layout = "NHD" - worker.host_buffer_kv_cache_layout = "NHD" + worker.kv_cache_layout = "LBNHC" + worker.host_buffer_kv_cache_layout = "LBNHC" worker._physical_blocks_per_logical_kv_block = 1 worker._logical_num_blocks = num_blocks worker.region_mem_types = [] @@ -221,6 +421,7 @@ def _register_overlaid_mla_worker( worker.device_kv_caches = {} worker.pp_size = 2 if push_pp else 1 worker._is_hma_required = True + worker._has_packed_cache = False worker.dcp_size = 1 worker.pcp_size = 1 worker.kv_buffer_device = "cuda" diff --git a/tests/v1/kv_connector/unit/test_nixl_push_connector.py b/tests/v1/kv_connector/unit/test_nixl_push_connector.py index 4d8ead28f3..7ce6666ae8 100644 --- a/tests/v1/kv_connector/unit/test_nixl_push_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_push_connector.py @@ -368,6 +368,7 @@ class _StubWriterWorker(NixlPushConnectorWorker): w._recving_transfers = defaultdict(list) w._is_hma_required = False w.xfer_stats = NixlKVConnectorStats() + w._has_packed_cache = False w._reqs_to_process = set() w._reqs_to_send = {} w.consumer_notification_counts_by_req = defaultdict(int) @@ -1843,6 +1844,40 @@ def test_layer_metadata_round_trip(): assert msgspec.msgpack.Decoder(NixlAgentMetadata).decode(encoded) == metadata +@pytest.mark.parametrize( + "layouts, error", + [ + ({"L0": (-1, 64)}, "escapes its block"), + ({"L0": (0, 0)}, "escapes its block"), + ({"L0": (200, 64)}, "escapes its block"), + ({"L1": (0, 64)}, "has no layout"), + ({"L0": (0, 32)}, "page sizes must match"), + ], +) +def test_packed_layer_alignment_rejects_invalid_page_layout(layouts, error): + worker = _layer_routing_worker([["L0"]], {"L0": 0}) + worker.block_len_per_layer = [64] + metadata = _agent_metadata([["L0", "L1"]], [0x10000], [256]) + metadata.packed_member_layouts = layouts + with pytest.raises(AssertionError, match=error): + worker._align_remote_regions_by_layer(metadata) + + +def test_layer_alignment_preserves_nonpacked_regions_alongside_packed_layers(): + worker = _layer_routing_worker([["a"], ["b"]], {"a": 0, "b": 1}) + worker.block_len_per_layer = [64, 128] + metadata = _agent_metadata( + [["a", "other"], ["b"]], [0x1000, 0x2000], [192, 128], [256, 128] + ) + metadata.packed_member_layouts = {"a": (64, 64), "other": (128, 64)} + + worker._align_remote_regions_by_layer(metadata) + + assert metadata.kv_caches_base_addr == [0x1040, 0x2000] + assert metadata.block_lens == [64, 128] + assert metadata.block_strides == [256, 128] + + def test_layer_identity_gate_preserves_the_non_hma_path(): assert _layer_routing_worker([["a"]], {"a": 0})._transfer_layer_region_indices == ( 0, @@ -1850,6 +1885,7 @@ def test_layer_identity_gate_preserves_the_non_hma_path(): # A base worker (pull) never routes by layer name. pull = object.__new__(NixlBaseConnectorWorker) + pull.pp_size = 2 pull._transfer_layer_names = () pull._transfer_layer_region_indices = () pull._transfer_layer_group_ids = () diff --git a/tests/v1/kv_connector/unit/test_remote_prefill_lifecycle.py b/tests/v1/kv_connector/unit/test_remote_prefill_lifecycle.py index caff9c9a49..daddd20b4a 100644 --- a/tests/v1/kv_connector/unit/test_remote_prefill_lifecycle.py +++ b/tests/v1/kv_connector/unit/test_remote_prefill_lifecycle.py @@ -25,7 +25,7 @@ pytestmark = pytest.mark.cpu_test def _num_waiting_requests(scheduler) -> int: - return len(scheduler.waiting) + len(scheduler.skipped_waiting) + return len(scheduler.waiting) + len(scheduler.kv_holding_waiting) def test_basic_lifecycle(): @@ -64,7 +64,7 @@ def test_basic_lifecycle(): # Req waiting for KVs with no computed/scheduled toks ... assert _num_waiting_requests(scheduler) == 1 - assert request in scheduler.skipped_waiting + assert request in scheduler.kv_holding_waiting assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS assert request.num_computed_tokens == NUM_TOKENS @@ -791,3 +791,71 @@ def test_async_load_reserves_blocks_for_promotion_margin(): scheduler_output = scheduler.schedule() assert req_a.status == RequestStatus.RUNNING assert scheduler_output.num_scheduled_tokens[req_a.request_id] > 0 + + +def test_deferred_lookup_does_not_block_parked_load(): + """A request with a deferred connector lookup must not strand an async + load queued behind it. + + req_a's lookup is deferred (connector returns None), so it is parked in + waiting holding no blocks; req_b is then admitted as an async + load holding most of the pool. Once req_a's lookup resolves it cannot + allocate, and the waiting scan stops at it every step. The parked load + must still be promotable: it lives in the KV-holding queue + that is drained first, so it runs, finishes, and frees its blocks for + req_a. + """ + vllm_config = create_vllm_config() + BLOCK_SIZE = vllm_config.cache_config.block_size + scheduler = create_scheduler(vllm_config, num_blocks=8) # usable = 7 + + req_a = create_request( + request_id=1, block_size=BLOCK_SIZE, num_tokens=BLOCK_SIZE * 4, max_tokens=1 + ) + req_b = create_request( + request_id=2, + block_size=BLOCK_SIZE, + num_tokens=BLOCK_SIZE * 5, + do_remote_prefill=True, + max_tokens=1, + ) + scheduler.add_request(req_a) + scheduler.add_request(req_b) + + deferrals = [(None, False)] + + def lookup(request, num_computed_tokens): + if request is req_b: + return BLOCK_SIZE * 5, True + return deferrals.pop() if deferrals else (0, False) + + with patch.object( + scheduler.connector, "get_num_new_matched_tokens", side_effect=lookup + ): + # Step 1: req_a's lookup is deferred; req_b is admitted as an async + # load holding 5 of 7 usable blocks. + scheduler.schedule() + assert req_a.status == RequestStatus.WAITING + assert req_b.status == RequestStatus.WAITING_FOR_REMOTE_KVS + + # req_a's lookup resolves (no hit) while req_b's transfer is in + # flight; req_a needs 4 blocks but only 2 are free, so the waiting + # scan stops at it. req_b's transfer then finishes. + scheduler.update_from_output( + scheduler.schedule(), + create_model_runner_output([], finished_recving={req_b.request_id}), + ) + + # req_b must still be reached, promoted, and scheduled despite the + # scan stopping at req_a. + scheduler_output = scheduler.schedule() + assert req_b.status == RequestStatus.RUNNING + assert scheduler_output.num_scheduled_tokens[req_b.request_id] > 0 + + # req_b finishes and frees its blocks; now req_a fits and runs. + scheduler.update_from_output( + scheduler_output, create_model_runner_output([req_b]) + ) + scheduler_output = scheduler.schedule() + assert req_a.status == RequestStatus.RUNNING + assert scheduler_output.num_scheduled_tokens[req_a.request_id] > 0 diff --git a/tests/v1/kv_connector/unit/utils.py b/tests/v1/kv_connector/unit/utils.py index dda7c0f386..f154816369 100644 --- a/tests/v1/kv_connector/unit/utils.py +++ b/tests/v1/kv_connector/unit/utils.py @@ -54,6 +54,8 @@ def assert_scheduler_empty(scheduler: Scheduler): # Scheduler Metadata. assert len(scheduler.requests) == 0 assert len(scheduler.waiting) == 0 + assert len(scheduler.kv_holding_waiting) == 0 + assert not scheduler.deferred_waiting assert len(scheduler.running) == 0 assert len(scheduler.finished_req_ids) == 0 assert len(scheduler.finished_recving_kv_req_ids) == 0 diff --git a/tests/v1/sample/test_logprobs.py b/tests/v1/sample/test_logprobs.py index d2cb99bc26..85551e80b4 100644 --- a/tests/v1/sample/test_logprobs.py +++ b/tests/v1/sample/test_logprobs.py @@ -6,7 +6,9 @@ import math from collections.abc import Generator from types import SimpleNamespace from typing import get_args +from unittest.mock import patch +import numpy as np import pytest import torch @@ -23,6 +25,7 @@ from vllm.config.model import LogprobsMode from vllm.distributed import cleanup_dist_env_and_memory from vllm.exceptions import VLLMValidationError from vllm.platforms import current_platform +from vllm.v1.engine.input_processor import InputProcessor from ...conftest import HfRunner, VllmRunner @@ -81,9 +84,9 @@ def hf_model(hf_runner) -> Generator[HfRunner, None, None]: yield hf_model -def _model_config(vocab_size: int = 10): +def _model_config(vocab_size: int = 10, max_logprobs: int = 20): return SimpleNamespace( - max_logprobs=20, + max_logprobs=max_logprobs, logits_processors=None, is_diffusion=False, get_vocab_size=lambda: vocab_size, @@ -432,6 +435,57 @@ def test_logprob_token_ids_validate_vocab_bounds_invalid(token_ids: list[int]): ) +def test_prompt_logprob_token_ids_bounded_by_max_logprobs(): + """The candidate count is bounded by max_logprobs.""" + model_config = _model_config(vocab_size=100, max_logprobs=4) + + def verify(**kwargs): + SamplingParams(**kwargs).verify( + model_config, + speculative_config=None, + structured_outputs_config=None, + tokenizer=None, + ) + + verify(prompt_logprob_token_ids=[1, 2, 3, 4]) + + # The error names the value to raise max_logprobs to. + with pytest.raises(VLLMValidationError, match=r"max_logprobs.*at least 5"): + verify(prompt_logprob_token_ids=[1, 2, 3, 4, 5]) + + # prompt_logprob_start alone is a caller mistake, not a silent no-op. + with pytest.raises(VLLMValidationError, match="requires prompt_logprob_token_ids"): + verify(prompt_logprob_start=3) + + +def test_prompt_logprob_token_ids_require_v2_model_runner(): + """Only the V2 runner scores them; the V1 runner would return None, and + kv-sharing fast prefill would score rows the cross-decoder never ran.""" + processor = SimpleNamespace( + model_config=SimpleNamespace( + return_sampling_mask=False, enable_trace_replay=False, is_diffusion=False + ), + vllm_config=SimpleNamespace( + reasoning_config=None, + use_v2_model_runner=False, + cache_config=SimpleNamespace(kv_sharing_fast_prefill=False), + ), + speculative_config=None, + structured_outputs_config=None, + tokenizer=None, + validate_logits_processors_params=lambda params: None, + ) + params = SamplingParams(prompt_logprob_token_ids=[1, 2]) + with patch.object(SamplingParams, "verify"): + with pytest.raises(VLLMValidationError, match="V2 model runner"): + InputProcessor._validate_params(processor, params, ("generate",)) + processor.vllm_config.use_v2_model_runner = True + InputProcessor._validate_params(processor, params, ("generate",)) + processor.vllm_config.cache_config.kv_sharing_fast_prefill = True + with pytest.raises(VLLMValidationError, match="fast-prefill"): + InputProcessor._validate_params(processor, params, ("generate",)) + + def test_none_logprobs(vllm_model, example_prompts): """Engine should return `logprobs` and `prompt_logprobs` as `None`. @@ -1298,6 +1352,127 @@ def test_prompt_logprobs_with_chunking_and_preemption(): print(f"Test passed with {preemptions} preemptions") +def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch): + """Fixed-ID scores stay row-aligned across chunked prefill and preemption.""" + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") + + prompts = [ + "The following numbers of the sequence " + + ", ".join(str(i) for i in range(10)) + + " are:", + "In one word, the capital of France is ", + ] + [f"Tell me about the number {i}: " for i in range(32)] + + start = 2 + candidate_ids = [10, 100, 1000, 10000] + sampling_params = SamplingParams( + temperature=0.0, + max_tokens=40, + min_tokens=20, + prompt_logprob_token_ids=candidate_ids, + prompt_logprob_start=start, + ) + + with VllmRunner( + "Qwen/Qwen3-0.6B", + max_model_len=512, + enable_chunked_prefill=True, + max_num_batched_tokens=48, # Force prefill chunking + num_gpu_blocks_override=33, # Force preemptions (32 usable + null block) + disable_log_stats=False, + gpu_memory_utilization=0.25, + ) as vllm_model: + metrics_before = vllm_model.llm.get_metrics() + outputs = vllm_model.llm.generate(prompts, sampling_params) + + for i, output in enumerate(outputs): + scores = output.prompt_token_id_logprobs + assert scores is not None, f"Output {i} missing fixed-ID scores" + expected_shape = ( + len(output.prompt_token_ids) - 1 - start, + len(candidate_ids), + ) + assert scores.shape == expected_shape, ( + f"Output {i} scored {scores.shape}, expected {expected_shape}" + ) + assert math.isfinite(float(scores.min())) + assert float(scores.max()) <= 1e-3, "logprobs must be <= 0" + + metrics_after = vllm_model.llm.get_metrics() + preemptions_before = next( + (m.value for m in metrics_before if m.name == "vllm:num_preemptions"), 0 + ) + preemptions_after = next( + (m.value for m in metrics_after if m.name == "vllm:num_preemptions"), 0 + ) + assert preemptions_after - preemptions_before > 0, ( + "Test did not trigger any preemptions" + ) + + # Row alignment is numerical: chunked and preempted scores must match an + # unchunked, unpreempted run of the same requests. Batch composition moves + # bf16 tail logprobs by a few percent; a misaligned row differs by nats. + with VllmRunner( + "Qwen/Qwen3-0.6B", + max_model_len=512, + max_logprobs=64, + gpu_memory_utilization=0.25, + ) as reference: + reference_outputs = reference.llm.generate(prompts, sampling_params) + own_ids = [list(dict.fromkeys(o.prompt_token_ids)) for o in reference_outputs] + self_scored = reference.llm.generate( + prompts, + [ + SamplingParams( + max_tokens=1, prompt_logprobs=0, prompt_logprob_token_ids=ids + ) + for ids in own_ids + ], + ) + for output, ref in zip(outputs, reference_outputs): + np.testing.assert_allclose( + output.prompt_token_id_logprobs, ref.prompt_token_id_logprobs, rtol=0.1 + ) + for output, ids in zip(self_scored, own_ids): + scores = output.prompt_token_id_logprobs + for row, target in enumerate(output.prompt_token_ids[1:]): + expected = output.prompt_logprobs[row + 1][target].logprob + assert scores[row, ids.index(target)] == pytest.approx(expected, abs=1e-3) + + +def test_prompt_logprob_token_ids_drop_partially_scored_prefills(monkeypatch): + """A prefill that starts past the first scored row returns no scores. + + Such a prefill (here a prefix-cache hit the caller opted back into) never + computes the leading rows, so emitting the buffer would return values the + model never produced for them. + """ + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") + + prompt = "The capital of France is Paris. " * 20 + sampling_params = SamplingParams( + temperature=0.0, + max_tokens=1, + prompt_logprob_token_ids=[10, 100, 1000], + skip_reading_prefix_cache=False, + ) + + with VllmRunner( + "Qwen/Qwen3-0.6B", + max_model_len=512, + enable_prefix_caching=True, + gpu_memory_utilization=0.25, + ) as vllm_model: + first = vllm_model.llm.generate([prompt], sampling_params)[0] + assert first.prompt_token_id_logprobs is not None + assert math.isfinite(float(first.prompt_token_id_logprobs.min())) + + # The re-send is served from the prefix cache, so its prefill starts + # past row 0 and the leading rows are never computed. + second = vllm_model.llm.generate([prompt], sampling_params)[0] + assert second.prompt_token_id_logprobs is None + + @large_gpu_mark(min_gb=24) def test_token_logprobs_large_batch_int64_row_offset(): """Regression: logprob kernel row offset (row * vocab_size) must use int64. diff --git a/tests/v1/sample/test_trace_replay_params.py b/tests/v1/sample/test_trace_replay_params.py index 6e0d726470..d5736f1773 100644 --- a/tests/v1/sample/test_trace_replay_params.py +++ b/tests/v1/sample/test_trace_replay_params.py @@ -41,6 +41,14 @@ def test_sampling_params_trace_field_rejects_empty_list(): ) +def test_sampling_params_trace_field_rejects_prompt_scoring(): + params = SamplingParams(trace_decode_token_ids=[1], prompt_logprob_token_ids=[2, 3]) + with pytest.raises(ValueError, match="prompt_logprob_token_ids"): + params._validate_trace_replay( + _make_model_config(vocab_size=100), speculative_config=None + ) + + def test_sampling_params_trace_field_requires_single_output(): params = SamplingParams(n=2, trace_decode_token_ids=[1]) with pytest.raises(ValueError, match="requires n=1"): diff --git a/tests/v1/spec_decode/test_dsv4_mtp_local_argmax.py b/tests/v1/spec_decode/test_dsv4_mtp_local_argmax.py new file mode 100644 index 0000000000..2c508864c6 --- /dev/null +++ b/tests/v1/spec_decode/test_dsv4_mtp_local_argmax.py @@ -0,0 +1,163 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""``get_top_tokens`` on the DeepSeek V4 MTP drafter. + +The speculator opts into ``use_local_argmax_reduction`` by probing the draft +model for ``get_top_tokens``, and expects it to select the same greedy draft +token the full-vocab argmax would. V4 reaches its LM head through hc_head and +the shared head's RMSNorm, so both methods have to share that transform. +""" + +from types import SimpleNamespace +from unittest import mock + +import pytest +import torch +from transformers import PreTrainedConfig + +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.mhc import HCHeadOp +from vllm.model_executor.models.deepseek_mtp import SharedHead +from vllm.platforms import current_platform + +if not current_platform.is_rocm(): + pytest.skip( + "get_top_tokens is implemented on the ROCm DeepSeek V4 MTP drafter only.", + allow_module_level=True, + ) + +from vllm.models.deepseek_v4.amd.mtp import ( # noqa: E402 + DeepSeekV4MTP, + DeepSeekV4MultiTokenPredictor, +) + +DEVICE_TYPE = current_platform.device_type +START_LAYER_IDX = 61 +HIDDEN_SIZE = 4096 +HC_MULT = 4 +RMS_NORM_EPS = 1e-6 +HC_EPS = 1e-6 +_EMBEDDING = "vllm.model_executor.layers.vocab_parallel_embedding" + + +def _make_predictor(vocab_size: int) -> DeepSeekV4MultiTokenPredictor: + config = PreTrainedConfig() + config.hidden_size = HIDDEN_SIZE + config.vocab_size = vocab_size + config.rms_norm_eps = RMS_NORM_EPS + + # A single vocab shard, so the reduction returns the local argmax without + # touching the process group. + with ( + mock.patch(f"{_EMBEDDING}.get_tensor_model_parallel_rank", return_value=0), + mock.patch( + f"{_EMBEDDING}.get_tensor_model_parallel_world_size", return_value=1 + ), + ): + shared_head = SharedHead(config=config, prefix="shared_head") + + generator = torch.Generator().manual_seed(7) + with torch.no_grad(): + shared_head.head.weight.normal_(0.0, 0.02, generator=generator) + # The default norm weight is all ones, and the argmax is invariant to + # the norm's per-row rescale, so a uniform gain would hide a skipped + # shared_head. Vary it so the comparison below can see one. + shared_head.norm.weight.normal_(1.0, 0.5, generator=generator) + shared_head = shared_head.to(device=DEVICE_TYPE, dtype=torch.bfloat16) + + hc_head_fn = torch.randn( + HC_MULT, HC_MULT * HIDDEN_SIZE, dtype=torch.float32, generator=generator + ) + # Non-trivial gate scale and bias: hc_head mixes the hc_mult streams with + # these, and defaults of 0 would make a skipped hc_head harder to see. + hc_head_scale = torch.randn(1, dtype=torch.float32, generator=generator) * 0.1 + hc_head_base = torch.randn(HC_MULT, dtype=torch.float32, generator=generator) * 0.1 + + mtp_layer = SimpleNamespace( + config=config, + hc_mult=HC_MULT, + hc_eps=HC_EPS, + rms_norm_eps=RMS_NORM_EPS, + hc_head_op=HCHeadOp(), + hc_head_fn=(hc_head_fn * 1e-4).to(DEVICE_TYPE), + hc_head_scale=hc_head_scale.to(DEVICE_TYPE), + hc_head_base=hc_head_base.to(DEVICE_TYPE), + shared_head=shared_head, + ) + + # Only the attributes the two methods read; a real predictor would also + # build the MLA/MoE draft block, which is irrelevant here. + predictor = object.__new__(DeepSeekV4MultiTokenPredictor) + torch.nn.Module.__init__(predictor) + predictor.layers = {str(START_LAYER_IDX): mtp_layer} + predictor.mtp_start_layer_idx = START_LAYER_IDX + predictor.num_mtp_layers = 1 + predictor.logits_processor = LogitsProcessor(vocab_size) + return predictor + + +def _hidden_states(num_tokens: int, seed: int) -> torch.Tensor: + """The flat pre-hc_head residual the MTP layer returns.""" + generator = torch.Generator().manual_seed(seed) + hidden_states = torch.randn( + num_tokens, HC_MULT * HIDDEN_SIZE, dtype=torch.float32, generator=generator + ) + return hidden_states.to(device=DEVICE_TYPE, dtype=torch.bfloat16) + + +def _assert_selects_max_logit(top: torch.Tensor, logits: torch.Tensor) -> None: + """The selected token must carry the row's maximum logit. + + hc_head emits bf16, so rows can tie for the maximum and then no single + index is the right answer; compare the values the two paths select. + """ + assert top.shape == (logits.shape[0],) + selected = logits.gather(1, top.unsqueeze(1)).squeeze(1) + assert torch.equal(selected, logits.max(dim=-1).values) + + +def test_get_top_tokens_matches_full_argmax(default_vllm_config): + """Same draft token as the path taken when the reduction is off. + + ``get_top_tokens`` has to run the same pre-head transform as + ``compute_logits``: hc_head over the (T, hc_mult, D) residual, then the + shared head's RMSNorm. Dropping either one returns a different draft token + instead of failing, which is what this comparison catches. + """ + predictor = _make_predictor(vocab_size=512) + + for num_tokens in (1, 16, 128): + hidden_states = _hidden_states(num_tokens, seed=11) + + logits = predictor.compute_logits(hidden_states) + top = predictor.get_top_tokens(hidden_states) + + assert top.dtype == torch.int64 + _assert_selects_max_logit(top, logits) + + +def test_get_top_tokens_honors_padded_vocab(default_vllm_config): + """Padding entries above ``org_vocab_size`` must never win the argmax.""" + vocab_size = 500 + predictor = _make_predictor(vocab_size) + head = predictor.layers[str(START_LAYER_IDX)].shared_head.head + assert head.shard_indices.num_org_vocab_padding > 0 + + hidden_states = _hidden_states(32, seed=13) + logits = predictor.compute_logits(hidden_states) + top = predictor.get_top_tokens(hidden_states) + + _assert_selects_max_logit(top, logits) + assert (top < vocab_size).all() + + +def test_wrapper_forwards_spec_step_idx(): + """``DeepSeekV4MTP`` is what the speculator probes, so it needs the method.""" + assert hasattr(DeepSeekV4MTP, "get_top_tokens") + + hidden_states = torch.empty(0) + wrapper = object.__new__(DeepSeekV4MTP) + wrapper.model = SimpleNamespace(get_top_tokens=lambda *args: args) + + assert DeepSeekV4MTP.get_top_tokens(wrapper, hidden_states, 2) == (hidden_states, 2) + assert DeepSeekV4MTP.get_top_tokens(wrapper, hidden_states) == (hidden_states, 0) diff --git a/tests/v1/spec_decode/test_dynamic_sd_cug.py b/tests/v1/spec_decode/test_dynamic_sd_cug.py index 8abbe91c1a..392135f7b0 100644 --- a/tests/v1/spec_decode/test_dynamic_sd_cug.py +++ b/tests/v1/spec_decode/test_dynamic_sd_cug.py @@ -126,7 +126,7 @@ def test_dynamic_sd_full_cudagraph_covers_all_uniform_decode_shapes(monkeypatch) num_reqs, num_tokens, max_query_len, - has_prefill=False, + decode_graph_eligible=True, ) # The scheduler should mark every one of these shapes as a uniform @@ -262,14 +262,14 @@ def test_prompt_chunks_shaped_like_spec_decode_miss_the_full_graph(monkeypatch): # The batch shape is indistinguishable from a full batch of spec decodes. assert ( get_uniform_decode_token_count( - num_reqs, num_tokens, decode_query_len, has_prefill=False + num_reqs, num_tokens, decode_query_len, decode_graph_eligible=True ) == decode_query_len ) # Two of the requests are decode_query_len tokens into a longer prompt. uniform_tok_count = get_uniform_decode_token_count( - num_reqs, num_tokens, decode_query_len, has_prefill=True + num_reqs, num_tokens, decode_query_len, decode_graph_eligible=False ) assert uniform_tok_count is None desc = manager.dispatch( @@ -283,7 +283,7 @@ def test_prompt_chunks_shaped_like_spec_decode_miss_the_full_graph(monkeypatch): # The same shape with every request decoding still gets its FULL graph. uniform_tok_count = get_uniform_decode_token_count( - num_reqs, num_tokens, decode_query_len, has_prefill=False + num_reqs, num_tokens, decode_query_len, decode_graph_eligible=True ) assert uniform_tok_count == decode_query_len desc = manager.dispatch( @@ -336,7 +336,7 @@ def test_basic_sd_does_not_capture_shorter_full_decode_shapes(monkeypatch): num_reqs, num_tokens, max_query_len, - has_prefill=False, + decode_graph_eligible=True, ) assert uniform_tok_count == max_query_len @@ -401,7 +401,7 @@ def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch): num_reqs, num_tokens, max_query_len, - has_prefill=False, + decode_graph_eligible=True, ) assert uniform_tok_count == max_query_len diff --git a/tests/v1/spec_decode/test_rejection_sampler_utils.py b/tests/v1/spec_decode/test_rejection_sampler_utils.py index 9e0052bcc5..c4e704ae24 100644 --- a/tests/v1/spec_decode/test_rejection_sampler_utils.py +++ b/tests/v1/spec_decode/test_rejection_sampler_utils.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import math +from types import SimpleNamespace import pytest import torch @@ -10,6 +11,10 @@ from vllm.v1.watermarking import GumbelWatermarker from vllm.v1.watermarking.spec_decode import watermarked_rejection_sample from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample from vllm.v1.worker.gpu.sample.watermark import philox_gumbel_sample +from vllm.v1.worker.gpu.spec_decode.rejection_sampler import ( + RejectionSampler, + gather_draft_sampled, +) from vllm.v1.worker.gpu.spec_decode.rejection_sampler_utils import ( rejection_sample, ) @@ -669,6 +674,81 @@ def test_placeholder_blocks_later_draft_tokens(use_block_verification: bool): ) +def test_verify_rejects_unproposed_drafts(): + """Zero drafts padded onto a step starting in the prefill must be rejected.""" + torch.manual_seed(0) + device = "cuda" + num_trials = 256 + K = 8 + + # Row 0: token 0 is very unlikely. Rows >= 1 follow the zero inputs and + # predict token 0. The never-written draft logits row is uniform. + target_logits_1d = torch.randn(VOCAB_SIZE, device=device) + target_logits_1d[0] = -20.0 + inputs = _build_rejection_sample_inputs( + target_logits_1d, + torch.zeros(VOCAB_SIZE, device=device), + K, + temperature=1.0, + num_trials=num_trials, + ) + target_rest = torch.randn(VOCAB_SIZE, device=device) + target_rest[0] = 20.0 + inputs["target_logits"].view(num_trials, K + 1, VOCAB_SIZE)[:, 1:] = target_rest + inputs["draft_sampled"].view(num_trials, K + 1)[:, 1:] = 0 + + # Each request's first logit is the last prefill token, as in the first + # decode step after a P/D remote KV load. + first_pos = inputs["pos"].view(num_trials, K + 1)[:, 0] + prefill_len = first_pos + 1 + rejection_sampler = object.__new__(RejectionSampler) + rejection_sampler.sampler = SimpleNamespace( + apply_sampling_params=lambda logits, *args: logits, + sampling_states=SimpleNamespace( + temperature=SimpleNamespace(gpu=inputs["temperature"]), + seeds=SimpleNamespace(gpu=inputs["seed"]), + ), + use_fp64_gumbel=False, + ) + rejection_sampler.num_speculative_steps = K + rejection_sampler.synthetic_conditional_rates = None + rejection_sampler.use_block_verification = True + rejection_sampler.watermark_key = None + + def verify() -> tuple[torch.Tensor, torch.Tensor]: + draft_sampled, pos = gather_draft_sampled( + inputs["draft_sampled"], + inputs["pos"], + torch.arange(num_trials * (K + 1), device=device), + inputs["expanded_idx_mapping"], + inputs["expanded_local_pos"], + prefill_len, + ) + _, sampled, num_sampled = rejection_sampler._verify( + inputs["target_logits"], + inputs["draft_logits"], + draft_sampled, + pos, + inputs["cu_num_logits"], + inputs["idx_mapping"], + None, + inputs["expanded_idx_mapping"], + inputs["expanded_local_pos"], + None, + ) + return sampled, num_sampled + + sampled, num_sampled = verify() + assert (num_sampled == 1).all() + # Resampled from the target at row 0, not from a residual. + assert (sampled[:, 0] != 0).all() + + # Once past the prefill, block verification accepts the same zero drafts. + prefill_len.copy_(first_pos) + _, num_sampled = verify() + assert (num_sampled == K + 1).all() + + @pytest.mark.parametrize( "num_speculative_steps,temperature", [ diff --git a/tests/v1/streaming_input/test_scheduler_streaming.py b/tests/v1/streaming_input/test_scheduler_streaming.py index 0d259c1c69..bfef3dd33c 100644 --- a/tests/v1/streaming_input/test_scheduler_streaming.py +++ b/tests/v1/streaming_input/test_scheduler_streaming.py @@ -4,6 +4,7 @@ import unittest from unittest.mock import MagicMock +import numpy as np import torch from vllm.config import DeviceConfig, VllmConfig @@ -20,7 +21,7 @@ from vllm.v1.kv_cache_interface import ( KVCacheConfig, KVCacheGroupSpec, ) -from vllm.v1.outputs import ModelRunnerOutput +from vllm.v1.outputs import LogprobsLists, ModelRunnerOutput from vllm.v1.request import Request, RequestStatus, StreamingUpdate from vllm.v1.structured_output import StructuredOutputManager @@ -84,6 +85,61 @@ def create_scheduler() -> Scheduler: class TestStreamingScheduler(unittest.TestCase): + def test_chunk_logprobs_do_not_depend_on_continuation_arrival(self): + """A queued continuation must not change the finishing chunk's logprobs.""" + for continuation_queued in (False, True): + with self.subTest(continuation_queued=continuation_queued): + scheduler = create_scheduler() + session = DummyRequest("session", prompt_token_ids=[1, 2, 3]) + session.sampling_params = SamplingParams( + stop_token_ids=[STOP_TOKEN], max_tokens=16, logprobs=0 + ) + scheduler.add_request(session) + scheduled = scheduler.schedule() + + continuation = DummyRequest("session", prompt_token_ids=[4, 5]) + assert continuation.sampling_params.logprobs is None + if continuation_queued: + scheduler.add_request(continuation) + + logprobs = LogprobsLists( + logprob_token_ids=np.array([[STOP_TOKEN]], dtype=np.int32), + logprobs=np.array([[-0.25]], dtype=np.float32), + sampled_token_ranks=np.array([1], dtype=np.int32), + ) + runner_output = ModelRunnerOutput( + req_ids=[session.request_id], + req_id_to_index={session.request_id: 0}, + sampled_token_ids=[[STOP_TOKEN]], + logprobs=logprobs, + prompt_logprobs_dict={}, + pooler_output=[], + ) + outputs = scheduler.update_from_output(scheduled, runner_output) + [output] = outputs[session.client_index].outputs + + if not continuation_queued: + assert session.status == RequestStatus.WAITING_FOR_STREAMING_REQ + scheduler.add_request(continuation) + assert session.status == RequestStatus.WAITING + assert session.sampling_params is continuation.sampling_params + assert output.new_token_ids == [STOP_TOKEN] + assert output.finish_reason == FinishReason.STOP + assert output.new_logprobs is not None, ( + "The finishing chunk requested logprobs; the continuation's " + "settings must not suppress them" + ) + np.testing.assert_array_equal( + output.new_logprobs.logprob_token_ids, logprobs.logprob_token_ids + ) + np.testing.assert_array_equal( + output.new_logprobs.logprobs, logprobs.logprobs + ) + np.testing.assert_array_equal( + output.new_logprobs.sampled_token_ranks, + logprobs.sampled_token_ranks, + ) + def test_add_request(self): scheduler = create_scheduler() @@ -115,8 +171,7 @@ class TestStreamingScheduler(unittest.TestCase): prompt_token_ids=[1, 2, 3], ) session.num_computed_tokens = len(session.prompt_token_ids) - session.max_tokens = 10 # Initial max_tokens - session._output_token_ids = [1] * 10 # reach max_tokens + session.max_tokens = 2 new_request = DummyRequest( request_id="session", @@ -129,26 +184,21 @@ class TestStreamingScheduler(unittest.TestCase): scheduler._update_request_as_session(session, update) assert session.sampling_params.max_tokens == 10 - # _update_request_as_session clears output tokens first, so - # max_tokens = num_output_tokens (0) + update.max_tokens (10) = 10 assert session.max_tokens == 10 session.num_computed_tokens = len(session.prompt_token_ids) - # Simulate generating 5 more output tokens - session._output_token_ids = [1] * 5 new_request2 = DummyRequest( request_id="session", prompt_token_ids=[7, 8, 9], ) - new_request2.sampling_params = SamplingParams(max_tokens=10) - new_request2.max_tokens = 10 + new_request2.sampling_params = SamplingParams(max_tokens=4) + new_request2.max_tokens = 4 update2 = StreamingUpdate.from_request(new_request2) scheduler._update_request_as_session(session, update2) - assert session.sampling_params.max_tokens == 10 - # Again, output tokens are cleared first, so max_tokens = 0 + 10 = 10 - assert session.max_tokens == 10 + assert session.sampling_params.max_tokens == 4 + assert session.max_tokens == 4 def test_update_request_as_session(self): scheduler = create_scheduler() @@ -498,9 +548,9 @@ class TestStreamingScheduler(unittest.TestCase): eco_cycle2 = eco_dict_cycle2[session.client_index].outputs[0] assert eco_cycle2.finish_reason == FinishReason.STOP assert session.status == RequestStatus.WAITING_FOR_STREAMING_REQ - # Sessions paused for streaming input are blocked-waiting, so they - # live in the skipped_waiting queue rather than the main waiting queue. - assert session in scheduler.skipped_waiting + # Sessions paused for streaming input keep their KV blocks, so they + # live in the kv_holding_waiting queue. + assert session in scheduler.kv_holding_waiting assert session._all_token_ids == [1, 2, 3, 10, STOP_TOKEN] # CRITICAL ASSERTION: Cached prompt_token_ids STILL must not have changed diff --git a/tests/v1/structured_output/test_backend_xgrammar_plamo3.py b/tests/v1/structured_output/test_backend_xgrammar_plamo3.py new file mode 100644 index 0000000000..a6506dafaa --- /dev/null +++ b/tests/v1/structured_output/test_backend_xgrammar_plamo3.py @@ -0,0 +1,117 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace + +import pytest +import xgrammar as xgr + +from vllm.config import StructuredOutputsConfig, VllmConfig +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionToolsParam, +) +from vllm.parser.plamo3 import ( + BEGIN_TOOL_ARGUMENTS, + BEGIN_TOOL_NAME, + BEGIN_TOOL_REQUEST, + BEGIN_TOOL_REQUESTS, + END_TOOL_ARGUMENTS, + END_TOOL_NAME, + END_TOOL_REQUEST, + END_TOOL_REQUESTS, +) +from vllm.tool_parsers.structural_tag_registry import get_model_structural_tag +from vllm.v1.structured_output.backend_types import StructuredOutputOptions +from vllm.v1.structured_output.backend_xgrammar import XgrammarBackend + + +class Plamo3Tokenizer: + """Minimal stand-in for PLaMo's remote-code PythonBackend tokenizer.""" + + eos_token_id = 0 + + def __init__(self, tokens: list[str]) -> None: + self._vocab = {token: token_id for token_id, token in enumerate(tokens)} + + def get_vocab(self) -> dict[str, int]: + return self._vocab + + def init_xgrammar(self): + stop_token_ids = [self.eos_token_id] + tokenizer_info = xgr.TokenizerInfo( + encoded_vocab=list(self._vocab), + vocab_type=xgr.VocabType.BYTE_FALLBACK, + vocab_size=len(self._vocab), + stop_token_ids=stop_token_ids, + ) + return tokenizer_info, stop_token_ids + + +def make_vllm_config() -> VllmConfig: + config = VllmConfig( + structured_outputs_config=StructuredOutputsConfig(backend="xgrammar"), + ) + config.model_config = SimpleNamespace( + hf_config=SimpleNamespace(model_type="plamo3") + ) + return config + + +def test_plamo3_python_tokenizer_compiles_and_accepts_structural_tag(): + call_tokens = [ + BEGIN_TOOL_REQUESTS, + BEGIN_TOOL_REQUEST, + BEGIN_TOOL_NAME, + "get_weather", + END_TOOL_NAME, + BEGIN_TOOL_ARGUMENTS, + '{"city":"Tokyo"}', + END_TOOL_ARGUMENTS, + END_TOOL_REQUEST, + END_TOOL_REQUESTS, + ] + tokenizer = Plamo3Tokenizer(["", *call_tokens]) + backend = XgrammarBackend( + make_vllm_config(), + tokenizer=tokenizer, + vocab_size=len(tokenizer.get_vocab()), + ) + tools = [ + ChatCompletionToolsParam( + type="function", + function={ + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + "strict": True, + }, + ) + ] + tag = get_model_structural_tag("plamo3", tools, "required", reasoning=False) + assert tag is not None + + grammar = backend.compile_grammar( + StructuredOutputOptions.STRUCTURAL_TAG, + tag.model_dump_json(), + ) + token_ids = [tokenizer.get_vocab()[token] for token in call_tokens] + assert grammar.accept_tokens("plamo3", token_ids) + + +@pytest.mark.parametrize("value", ['"', "あ"]) +def test_plamo3_grammar_matches_decoded_bytes(value): + pieces = [f"<0x{byte:02X}>" for byte in value.encode("utf-8")] + tokenizer = Plamo3Tokenizer(list(dict.fromkeys(["", value, *pieces]))) + backend = XgrammarBackend( + make_vllm_config(), + tokenizer=tokenizer, + vocab_size=len(tokenizer.get_vocab()), + ) + for tokens in ([value], pieces): + grammar = backend.compile_grammar(StructuredOutputOptions.REGEX, value) + assert grammar.accept_tokens( + "plamo3", [tokenizer.get_vocab()[token] for token in tokens] + ) diff --git a/tests/v1/worker/test_attn_utils.py b/tests/v1/worker/test_attn_utils.py index 1bff5ae466..73a0131915 100644 --- a/tests/v1/worker/test_attn_utils.py +++ b/tests/v1/worker/test_attn_utils.py @@ -77,6 +77,7 @@ def test_get_kv_cache_spec_resolves_hisparse_block_size( layers = {} for name, sizes in zip(specs, [main_sizes, indexer_sizes, [block_size]]): backend = SimpleNamespace( + get_name=lambda name=name: name, customize_spec=AttentionBackend.customize_spec, get_supported_kernel_block_sizes=lambda sizes=sizes: sizes, ) diff --git a/tests/v1/worker/test_gpu_batch_ordering.py b/tests/v1/worker/test_gpu_batch_ordering.py index 5037dcc7a9..db9e080e4e 100644 --- a/tests/v1/worker/test_gpu_batch_ordering.py +++ b/tests/v1/worker/test_gpu_batch_ordering.py @@ -56,6 +56,7 @@ def _make_runner( runner: Any = GPUModelRunner.__new__(GPUModelRunner) runner.decode_query_len = decode_query_len runner.adaptive_verification = None + runner.pcp_manager = None runner.req_states = SimpleNamespace( req_id_to_index={req_id: i for i, req_id in enumerate(req_states)}, # The runner keeps this as min(num_computed_tokens, prefill_len). @@ -128,6 +129,65 @@ def test_prompt_chunk_of_decode_query_len_is_not_uniform_decode(): assert _uniform_token_count({"p0": (0, 8)}, 8) is None +def test_one_new_prompt_token_over_context_runs_as_decode(): + # A prompt tail padded with placeholder drafts and a one-token mid-prompt + # chunk both compute like decodes. A fresh one-token prompt and a real + # multi-token chunk don't. + batch = {"d0": (16, 16), "tail": (128, 129), "mid": (64, 129)} + batch.update({"fresh": (0, 1), "chunk": (120, 129)}) + runner = _make_runner(batch, decode_query_len=4) + scheduler_output = SimpleNamespace( + num_scheduled_tokens={"d0": 4, "tail": 4, "mid": 1, "fresh": 1, "chunk": 4}, + total_num_scheduled_tokens=14, + scheduled_spec_decode_tokens={"d0": [1, 2, 3], "tail": [-1] * 3}, + ) + state, _ = runner.gather_batch_req_state(scheduler_output, False) + assert state is not None + runs_as_decode = dict(zip(state.req_ids, state.prefill_runs_as_decode_np)) + assert runs_as_decode == { + "d0": False, + "tail": True, + "mid": True, + "fresh": False, + "chunk": False, + } + + +def test_one_token_prompt_tail_with_prior_context_is_uniform_decode(): + # Replaying the last prompt token over existing context (full prefix-cache + # hit or P/D KV import) computes exactly like a decode. + batch = {"d0": (16, 16), "d1": (16, 16), "p0": (128, 129)} + assert _uniform_token_count(batch, 1) == 1 + + # A fresh 1-token prompt alongside it has no prior state. + batch["p1"] = (0, 1) + assert _uniform_token_count(batch, 1) is None + + # Only one-token tails qualify, not K+1 chunks with prior context. + assert _uniform_token_count({"d0": (16, 16), "p0": (120, 129)}, 8) is None + + +def test_padded_prompt_tail_with_prior_context_is_uniform_decode(): + # The scheduler pads a one-token prompt tail with K placeholder drafts to + # keep the K+1 spec-decode shape; it verifies like any other decode. + batch = {"d0": (16, 16), "d1": (16, 16), "p0": (128, 129)} + runner = _make_runner(batch, decode_query_len=4) + scheduler_output = SimpleNamespace( + num_scheduled_tokens={req_id: 4 for req_id in batch}, + total_num_scheduled_tokens=12, + scheduled_spec_decode_tokens={req_id: [-1] * 3 for req_id in batch}, + ) + state, uniform_tok_count = runner.gather_batch_req_state(scheduler_output, False) + assert state.has_prefill and state.decode_graph_eligible + assert uniform_tok_count == 4 + + # Without the placeholder drafts, it is a real 4-token prompt chunk. + del scheduler_output.scheduled_spec_decode_tokens["p0"] + state, uniform_tok_count = runner.gather_batch_req_state(scheduler_output, False) + assert not state.decode_graph_eligible + assert uniform_tok_count is None + + def test_dummy_batches_stay_uniform_decode(): # Dummy batches have no request state to consult and must stay classified # by shape alone; rejecting them would make capture and replay disagree. @@ -185,6 +245,7 @@ def test_uniform_decode_uses_state_index_not_batch_position(): runner: Any = GPUModelRunner.__new__(GPUModelRunner) runner.decode_query_len = 8 runner.adaptive_verification = None + runner.pcp_manager = None # State arrays in state-index order: a prefilling request, then two decodes. runner.req_states = SimpleNamespace( req_id_to_index={"prefilling": 0, "decode_a": 1, "decode_b": 2}, @@ -214,11 +275,11 @@ def test_uniform_decode_uses_state_index_not_batch_position(): def test_uniform_decode_predicate(): - # Shape and prefill state must both pass. - assert get_uniform_decode_token_count(2, 16, 8, False) == 8 - assert get_uniform_decode_token_count(2, 16, 8, True) is None + # Shape and decode-graph eligibility must both pass. + assert get_uniform_decode_token_count(2, 16, 8, True) == 8 + assert get_uniform_decode_token_count(2, 16, 8, False) is None # 12 tokens over 2 requests is no shared query length. - assert get_uniform_decode_token_count(2, 12, 8, False) is None + assert get_uniform_decode_token_count(2, 12, 8, True) is None def test_no_speculator_dispatches_on_query_length_alone(): diff --git a/tests/v1/worker/test_gpu_batch_shard.py b/tests/v1/worker/test_gpu_batch_shard.py index ffb0922098..60cf6479a7 100644 --- a/tests/v1/worker/test_gpu_batch_shard.py +++ b/tests/v1/worker/test_gpu_batch_shard.py @@ -96,6 +96,7 @@ def _make_batch( num_computed_prefill_tokens_np=np.zeros(num_reqs, dtype=np.int32), is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_), has_prefill=False, + decode_graph_eligible=True, input_ids=torch.zeros(num_tokens, dtype=torch.int32, device=DEVICE), positions=torch.arange(num_tokens, dtype=torch.int64, device=DEVICE), is_padding=torch.zeros(num_tokens, dtype=torch.bool, device=DEVICE), diff --git a/tests/v1/worker/test_gpu_model_runner.py b/tests/v1/worker/test_gpu_model_runner.py index e77d278116..fb4b7d1e58 100644 --- a/tests/v1/worker/test_gpu_model_runner.py +++ b/tests/v1/worker/test_gpu_model_runner.py @@ -298,6 +298,10 @@ def _make_mock_backend_for_kernel_block_size( supported_sizes: list[int | MultipleOf], ): class _MockBackend: + @staticmethod + def get_name(): + return "MOCK" + @staticmethod def get_supported_kernel_block_sizes(): return supported_sizes diff --git a/tests/v1/worker/test_gpu_model_runner_v2_eplb.py b/tests/v1/worker/test_gpu_model_runner_v2_eplb.py index b9cee16ac4..7c759894fe 100644 --- a/tests/v1/worker/test_gpu_model_runner_v2_eplb.py +++ b/tests/v1/worker/test_gpu_model_runner_v2_eplb.py @@ -7,6 +7,7 @@ from typing import Any import torch +from vllm.config import CompilationConfig, CompilationMode from vllm.model_executor.warmup.jit_warmup import JitWarmupRegistry from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT from vllm.v1.worker.gpu import eplb_utils as eplb @@ -24,7 +25,6 @@ class FakeMemoryProfiler: class FakeEplbState: instances: list["FakeEplbState"] = [] - from_mapping_kwargs: dict[str, Any] | None = None def __init__(self, parallel_config: Any, device: torch.device): self.parallel_config = parallel_config @@ -33,7 +33,7 @@ class FakeEplbState: self.step_calls: list[tuple[bool, bool, bool]] = [] self.async_started = False self.is_async = True - self.built_from_mapping = False + self.update_mapping_calls: list[tuple[Any, torch.Tensor]] = [] FakeEplbState.instances.append(self) def add_model(self, model: Any, model_config: Any) -> None: @@ -45,12 +45,8 @@ class FakeEplbState: def start_async_loop(self) -> None: self.async_started = True - @classmethod - def from_mapping(cls, **kwargs: Any) -> "FakeEplbState": - cls.from_mapping_kwargs = kwargs - state = cls(kwargs["parallel_config"], kwargs["device"]) - state.built_from_mapping = True - return state + def update_mapping(self, model_config: Any, mapping: torch.Tensor) -> None: + self.update_mapping_calls.append((model_config, mapping)) def _make_runner(**overrides: Any) -> Any: @@ -73,7 +69,9 @@ def _make_runner(**overrides: Any) -> Any: enable_batch_sharded_sampling=False, eplb_config=SimpleNamespace(log_balancedness=True), ) + runner.compilation_config = CompilationConfig(mode=CompilationMode.NONE) runner.vllm_config = SimpleNamespace( + compilation_config=runner.compilation_config, load_config=runner.load_config, model_config=runner.model_config, ) @@ -136,7 +134,7 @@ def test_v2_load_model_registers_moe_with_eplb(monkeypatch): assert runner.eplb_state.async_started is True -def test_v2_load_model_with_dummy_weights_skips_eplb_registration(monkeypatch): +def test_v2_load_model_with_dummy_weights_defers_eplb_async_loop(monkeypatch): FakeEplbState.instances.clear() model = SimpleNamespace(is_moe=True) @@ -159,24 +157,20 @@ def test_v2_load_model_with_dummy_weights_skips_eplb_registration(monkeypatch): assert runner.load_config.load_format == "dummy" assert runner.eplb_state is not None - assert runner.eplb_state.add_model_calls == [] + # A scaling-up worker registers so it can join the EPLB communicator. + assert runner.eplb_state.add_model_calls == [(model, runner.model_config)] assert runner.eplb_state.async_started is False -def test_v2_setup_eplb_from_mapping_rebuilds_state(monkeypatch): - FakeEplbState.instances.clear() - FakeEplbState.from_mapping_kwargs = None - monkeypatch.setattr(eplb, "EplbState", FakeEplbState) - monkeypatch.setattr(eplb, "get_mixture_of_experts_model", lambda model: model) - - runner = _make_runner(model=SimpleNamespace(is_moe=True)) +def test_v2_setup_eplb_from_mapping_updates_state_in_place(): + runner = _make_runner() + state = FakeEplbState(runner.parallel_config, runner.device) + runner.eplb_state = state mapping = torch.tensor([[0, 1, 2, 3]], dtype=torch.int64) mrv2.GPUModelRunner.setup_eplb_from_mapping(runner, mapping) - assert runner.eplb_state is not None - assert runner.eplb_state.built_from_mapping is True - assert FakeEplbState.from_mapping_kwargs is not None - assert FakeEplbState.from_mapping_kwargs["expanded_physical_to_logical"] is mapping + assert runner.eplb_state is state + assert state.update_mapping_calls == [(runner.model_config, mapping)] def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch): diff --git a/tests/v1/worker/test_mamba_hybrid_model_state.py b/tests/v1/worker/test_mamba_hybrid_model_state.py index 19f77c13e6..dba5444150 100644 --- a/tests/v1/worker/test_mamba_hybrid_model_state.py +++ b/tests/v1/worker/test_mamba_hybrid_model_state.py @@ -45,6 +45,7 @@ def test_prepare_attn_forwards_positions(monkeypatch: pytest.MonkeyPatch) -> Non seq_lens_cpu_upper_bound=torch.tensor([1537], dtype=torch.int32), seq_lens=torch.tensor([1537], dtype=torch.int32), is_prefilling_np=torch.tensor([False]).numpy(), + prefill_runs_as_decode_np=None, dcp_local_seq_lens=None, positions=positions, prompt_lens=torch.tensor([1024], dtype=torch.int32), @@ -102,8 +103,7 @@ def test_padded_prompt_tail_builds_as_spec_decode( seq_lens_cpu_upper_bound=torch.tensor(seq_lens, dtype=torch.int32), seq_lens=torch.tensor(seq_lens, dtype=torch.int32), is_prefilling_np=np.array(is_prefilling), - prefill_len_np=np.array([40, 129, 100], dtype=np.int32), - num_computed_prefill_tokens_np=np.array([40, 128, 0], dtype=np.int32), + prefill_runs_as_decode_np=np.array([False, True, False]), dcp_local_seq_lens=None, positions=torch.zeros(12, dtype=torch.int64), prompt_lens=None, @@ -121,11 +121,12 @@ def test_padded_prompt_tail_builds_as_spec_decode( mamba_metadata = build_attn_metadata.call_args.kwargs[ "model_specific_attn_metadata" ] + assert mamba_metadata.is_prefilling.tolist() == [False, False, True] builder = _create_gdn_builder(num_speculative_tokens=k) common = create_common_attn_metadata( BatchSpec(seq_lens=seq_lens, query_lens=query_lens), BLOCK_SIZE, DEVICE - ).replace(is_prefilling=torch.tensor(is_prefilling)) + ).replace(**mamba_metadata.get_extra_common_attn_kwargs(0, 3)) meta = builder.build( common_prefix_len=0, common_attn_metadata=common, diff --git a/tools/pre_commit/mypy.py b/tools/pre_commit/mypy.py index 7262c1f2a9..69b36d43b2 100755 --- a/tools/pre_commit/mypy.py +++ b/tools/pre_commit/mypy.py @@ -113,7 +113,6 @@ SEPARATE_GROUPS = [ EXCLUDE = [ r"vllm/model_executor/models/[kK]", - r"vllm/model_executor/models/[vV]", r"vllm/model_executor/models/[wW]", r"vllm/model_executor/models/[zZ]", ] diff --git a/tools/pre_commit/test_tethering_allowlist.txt b/tools/pre_commit/test_tethering_allowlist.txt index 80b4aaec4c..cc5294cf26 100644 --- a/tools/pre_commit/test_tethering_allowlist.txt +++ b/tools/pre_commit/test_tethering_allowlist.txt @@ -59,27 +59,6 @@ tests/models/test_dspark_mla.py # no models_*.yaml job collects this path tests/models/test_language_model_cache_is_weak.py # no models_*.yaml job collects this path tests/models/transformers/test_layer_registry.py # no models_*.yaml job collects this path -# tests/ root-level - misc.yaml runs a hand-picked set of these by name -# (test_regression.py, test_envs.py, test_outputs.py, test_pooling_params.py); -# the rest are unreferenced. -tests/test_access_log_filter.py # root test_*.py not in misc.yaml's by-name list -tests/test_attention_backend_registry.py # root test_*.py not in misc.yaml's by-name list -tests/test_audio_media_size_precheck.py # root test_*.py not in misc.yaml's by-name list -tests/test_cmake_utils.py # root test_*.py not in misc.yaml's by-name list -tests/test_connections.py # root test_*.py not in misc.yaml's by-name list -tests/test_embedded_commit.py # root test_*.py not in misc.yaml's by-name list -tests/test_force_first_config.py # root test_*.py not in misc.yaml's by-name list -tests/test_fxgraphcache_pickle_patch.py # root test_*.py not in misc.yaml's by-name list -tests/test_logprobs.py # root test_*.py not in misc.yaml's by-name list -tests/test_quantization_revision_pin.py # root test_*.py not in misc.yaml's by-name list -tests/test_ray_env_utils.py # root test_*.py not in misc.yaml's by-name list -tests/test_request_input_bounds.py # root test_*.py not in misc.yaml's by-name list -tests/test_scalartype.py # root test_*.py not in misc.yaml's by-name list -tests/test_tensorizer_revision_pin.py # root test_*.py not in misc.yaml's by-name list -tests/test_triton_utils.py # root test_*.py not in misc.yaml's by-name list -tests/test_version.py # root test_*.py not in misc.yaml's by-name list -tests/test_zen_cpu_platform_detection.py # root test_*.py not in misc.yaml's by-name list - # tests/v1/ - the v1 subtree is otherwise covered by many `pytest v1/...` jobs. tests/v1/e2e/test_replayssm_decode.py # ReplaySSM decode test (#52506) wired into no job tests/v1/ec_connector/integration/test_epd_correctness.py # run_epd_correctness_test.sh exists but no Buildkite job invokes it @@ -91,10 +70,3 @@ tests/platforms/test_cpu.py # CPU-only unit test; no job targets tests/platform # tests/tools/ tests/tools/test_config_validator.py # logic covered by the validate-config pre-commit hook; no test_areas job - -# tests/watermarking/ - added by #54053 (2026-09-09), no Buildkite job collects the dir yet -tests/watermarking/test_detection.py # watermarking feature (#54053); no job collects tests/watermarking/ -tests/watermarking/test_gumbel.py # watermarking feature (#54053); no job collects tests/watermarking/ -tests/watermarking/test_prf.py # watermarking feature (#54053); no job collects tests/watermarking/ -tests/watermarking/test_watermarking.py # watermarking feature (#54053); no job collects tests/watermarking/ -tests/watermarking/test_watermarking_e2e.py # watermarking hardening (#56309); no job collects tests/watermarking/ diff --git a/tools/recipes/README.md b/tools/recipes/README.md index 253bd7a726..90ad33b3eb 100644 --- a/tools/recipes/README.md +++ b/tools/recipes/README.md @@ -23,8 +23,9 @@ refine the initial configuration. Sweep tuning is an optional validation step. ## Getting Started -Use `serve_with_recipe.sh` to generate the recipe configuration and start vLLM -in one step: +Use `serve_with_recipe.sh` to fetch the selected deployment from +[vLLM Recipes](https://recipes.vllm.ai/), generate the configuration, and start +vLLM in one step: ```bash tools/recipes/serve_with_recipe.sh \ @@ -32,6 +33,23 @@ tools/recipes/serve_with_recipe.sh \ --hardware xeon6 ``` +The converter queries `recipes.vllm.ai` at runtime, so it can use the latest +published recipe for the requested model and hardware without rebuilding the +vLLM image. + +### Find the hardware name + +Open [vLLM Recipes](https://recipes.vllm.ai/), select the model, and choose the +target in the **Hardware** picker. The selected page URL contains the hardware +key. For example, `?hardware=xeon6` maps to `--hardware xeon6`. + +You can also run the converter without arguments for interactive model, +hardware, and strategy discovery: + +```bash +python3 tools/recipes/recipe_json_to_vllm_config.py +``` + For `xeon6`, the script enables hardware detection automatically. Generated `config.yml` and `env.sh` files remain in the directory where the script was invoked. diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index 0e89b39818..906349af9a 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -1927,6 +1927,7 @@ class rocm_aiter_ops: VLLM_ROCM_USE_AITER_FP8BMM: Controls FP8 batched matrix multiply. VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: Controls FP4 assembly GEMM. VLLM_ROCM_USE_AITER_TRITON_ROPE: Controls Triton rotary embeddings. + VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA: Controls Triton sparse MLA (gfx950). VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: Controls shared expert fusion. VLLM_ROCM_USE_AITER_MOE_SITUV2: SiTUv2 FlyDSL MoE activation dtype (a16w4 | a8w4 | a4w4). @@ -1998,6 +1999,7 @@ class rocm_aiter_ops: _FP4_GEMM_DYNAMIC_QUANT_ASM = envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM # TODO: Consolidate under VLLM_ROCM_USE_AITER_ROPE _TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE + _TRITON_SPARSE_MLA = envs.VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA _MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS _MOE_SITUV2 = _resolve_situv2_activation() # TODO: Consolidate under _LINEAR_ENABLED @@ -2027,6 +2029,7 @@ class rocm_aiter_ops: cls._LINEAR_HIPBMM_ENABLED = envs.VLLM_ROCM_USE_AITER_LINEAR_HIPBMM cls._FP4_GEMM_DYNAMIC_QUANT_ASM = envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM cls._TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE + cls._TRITON_SPARSE_MLA = envs.VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA cls._MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS cls._MOE_SITUV2 = _resolve_situv2_activation() _sync_aiter_situv2_moe_env() @@ -2391,6 +2394,25 @@ class rocm_aiter_ops: def is_triton_rotary_embed_enabled(cls) -> bool: return cls._AITER_ENABLED and cls._TRITON_ROTARY_EMBED + @classmethod + @if_aiter_supported + def is_triton_sparse_mla_enabled(cls) -> bool: + if not cls._TRITON_SPARSE_MLA: + return False + from vllm.platforms.rocm import on_gfx950 + + if not cls._AITER_ENABLED: + reason = "VLLM_ROCM_USE_AITER is off" + elif not on_gfx950(): + reason = "the kernel is gfx950-only" + else: + return True + logger.warning_once( + "VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA is set, but %s; ignoring it.", + reason, + ) + return False + @classmethod @if_aiter_supported def is_triton_gemm_enabled(cls) -> bool: @@ -3137,6 +3159,57 @@ class rocm_aiter_ops: ) -> tuple[torch.Tensor, torch.Tensor]: return torch.ops.vllm.rocm_aiter_fused_topk(x, router_logits, top_k, gate_up) + @staticmethod + def triton_sparse_mla_fwd( + q: torch.Tensor, + kv_buffer: torch.Tensor, + o: torch.Tensor, + sm_scale: float, + kv_indptr: torch.Tensor, + kv_indices: torch.Tensor, + kv_lora_rank: int = 512, + qk_rope_head_dim: int = 64, + q_scale: torch.Tensor | None = None, + kv_scale: torch.Tensor | None = None, + attn_sink: torch.Tensor | None = None, + extra_kv_buffer: torch.Tensor | None = None, + extra_kv_indptr: torch.Tensor | None = None, + extra_kv_indices: torch.Tensor | None = None, + has_invalid: bool = True, + ) -> None: + """Sparse MLA read straight from the KV cache, for prefill and decode. + + kv_indices are ragged global slot ids. With has_invalid they may hold + -1, which the kernel masks; a caller whose stream never holds a + negative slot passes False, which lets the fp8 kernel stage K straight + into LDS. The cache format (bf16, per-tensor fp8, or DeepSeek V4's + paged fp8_ds_mla) is inferred from kv_buffer and kv_scale. An fp8 q + must come with its q_scale and runs both dots in fp8. The extra segment + is DeepSeek V4's second cache: SWA window in kv_buffer, top-k + compressed tokens here. + """ + from aiter.ops.triton.attention.sparse_mla import sparse_mla_fwd + + fp8_q = q.dtype == FP8_DTYPE + sparse_mla_fwd( + q, + kv_buffer, + kv_indptr, + kv_indices, + sm_scale, + kv_scale=kv_scale, + kv_lora_rank=kv_lora_rank, + qk_rope_head_dim=qk_rope_head_dim, + has_invalid=has_invalid, + dot_precision="fp8" if fp8_q else "bf16", + q_scale=q_scale if fp8_q else None, + out=o, + attn_sink=attn_sink, + extra_kv=extra_kv_buffer, + extra_indptr=extra_kv_indptr, + extra_indices=extra_kv_indices, + ) + @staticmethod def mla_decode_fwd( q: torch.Tensor, @@ -4162,6 +4235,57 @@ class rocm_aiter_ops: next_pre_mix.view(*outer_shape, hc_mult), ) + @staticmethod + def mhc_fused_post_pre_delayed_rms_norm( + residual: torch.Tensor, + fn: torch.Tensor, + hc_scale: torch.Tensor, + hc_base: torch.Tensor, + rms_eps: float, + hc_pre_eps: float, + hc_sinkhorn_eps: float, + hc_post_mult_value: float, + sinkhorn_repeat: int, + pre_mix: torch.Tensor | None, + sublayer_out: torch.Tensor | None, + post_layer_mix: torch.Tensor | None, + comb_res_mix: torch.Tensor | None, + norm_weight: torch.Tensor, + norm_eps: float, + residual_out: torch.Tensor | None, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """:meth:`mhc_pre_delayed` with the RMSNorm of the collapse folded in. + + Runs aiter's fused Triton kernel ``mhc_fused_post_pre_delayed_rmsnorm``; + the returned ``layer_input`` is already normalised with ``norm_weight`` / + ``norm_eps``. + """ + from aiter.ops.triton.fusions.mhc_fused_post_pre_delayed_rmsnorm import ( + mhc_fused_post_pre_delayed_rmsnorm, + ) + + _, post_mix, comb_mix, layer_input, next_pre_mix = ( + mhc_fused_post_pre_delayed_rmsnorm( + residual, + fn, + hc_scale, + hc_base, + rms_eps, + hc_pre_eps, + hc_sinkhorn_eps, + hc_post_mult_value, + sinkhorn_repeat, + pre_mix, + sublayer_out, + post_layer_mix, + comb_res_mix, + norm_weight, + norm_eps, + residual_out=residual_out, + ) + ) + return post_mix, comb_mix, layer_input, next_pre_mix + @staticmethod def mhc_fused_post_pre_delayed_prefers_unfused(num_tokens: int) -> bool: """True when AITER's own heuristic favours separate post and pre. diff --git a/vllm/compilation/passes/utility/split_coalescing.py b/vllm/compilation/passes/utility/split_coalescing.py index f16623a1aa..e9423830d1 100644 --- a/vllm/compilation/passes/utility/split_coalescing.py +++ b/vllm/compilation/passes/utility/split_coalescing.py @@ -1,21 +1,24 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Coalesce duplicate ``split_with_sizes`` nodes that operate on the same -input tensor with the same split sizes. +"""Canonicalize tensor partitions to a single ``split_with_sizes`` node. -On certain hardware/dtype combinations (e.g. B200 + FP8) the Inductor -graph may contain multiple ``split_with_sizes`` calls on the same tensor -that CSE fails to merge. This pass detects and replaces the duplicates -so that downstream pattern-matching passes (e.g. QK-Norm+RoPE fusion) -see a single split node with all users attached. +Functionalization may replay the individual outputs of a multi-output split as +contiguous ``slice`` nodes. This pass converts complete slice partitions back +to ``split_with_sizes`` and coalesces equivalent split nodes so downstream +pattern-matching passes see one canonical split with all users attached. See Also: - - vLLM #33295 (original issue) + - vLLM #33295 (original duplicate-split issue) - PyTorch #174472 (upstream CSE gap) + - PyTorch #194037 (multi-output views replayed as individual slices) """ import operator +from collections import defaultdict +from copy import copy +from dataclasses import dataclass +from typing import Any import torch from torch import fx @@ -28,35 +31,177 @@ from ..vllm_inductor_pass import VllmInductorPass logger = init_logger(__name__) +@dataclass(frozen=True) +class _SliceInfo: + node: fx.Node + dim: int + start: int + end: int + + class SplitCoalescingPass(VllmInductorPass): - """Replace duplicate ``split_with_sizes`` nodes with a single canonical - node when they share the same input tensor and split sizes.""" + """Canonicalize tensor partitions to one shared ``split_with_sizes``. - @VllmInductorPass.time_and_log - def __call__(self, graph: fx.Graph) -> None: - count = 0 + A complete contiguous slice partition is rewritten from:: - # Map from input tensor node -> list of split nodes seen so far. - split_nodes: dict[fx.Node, list[fx.Node]] = {} + q = slice(qkv, 0, q_size) + k = slice(qkv, q_size, q_size + kv_size) + v = slice(qkv, q_size + kv_size, q_size + 2 * kv_size) - for node in graph.nodes: - if not is_func(node, torch.ops.aten.split_with_sizes.default): + to:: + + qkv_split = split_with_sizes(qkv, [q_size, kv_size, kv_size]) + q = qkv_split[0] + k = qkv_split[1] + v = qkv_split[2] + + Equivalent splits are also coalesced from:: + + q = split_with_sizes(qkv, sizes)[0] + k = split_with_sizes(qkv, sizes)[1] + + to:: + + qkv_split = split_with_sizes(qkv, sizes) + q = qkv_split[0] + k = qkv_split[1] + """ + + @staticmethod + def _get_arg(node: fx.Node, index: int, name: str, default: Any) -> Any: + """Read an FX argument by position, keyword, or schema default.""" + if len(node.args) > index: + return node.args[index] + return node.kwargs.get(name, default) + + @staticmethod + def _normalize_dim(dim: int, rank: int) -> int: + return dim + rank if dim < 0 else dim + + @staticmethod + def _tensor_val(node: fx.Node) -> torch.Tensor | None: + value = node.meta.get("val") + return value if isinstance(value, torch.Tensor) else None + + def _slice_info(self, node: fx.Node) -> _SliceInfo | None: + source = node.args[0] + source_val = self._tensor_val(source) + slice_val = self._tensor_val(node) + # in case someone forgot to set meta ! + if source_val is None or slice_val is None: + return None + + dim = self._normalize_dim(self._get_arg(node, 1, "dim", 0), source_val.dim()) + extent = source_val.shape[dim] + + # do not support dynamic slices. + if not isinstance(extent, int): + return None + + start = self._get_arg(node, 2, "start", None) + end = self._get_arg(node, 3, "end", None) + step = self._get_arg(node, 4, "step", 1) + start = 0 if start is None else start + end = extent if end is None else end + if not isinstance(start, int) or not isinstance(end, int) or step != 1: + return None + + if start < 0 or end < 0: + return None + + # aten.slice clamps an oversized end to the selected dimension. + end = min(end, extent) + if start >= end: + return None + + return _SliceInfo(node=node, dim=dim, start=start, end=end) + + def _canonicalize_slices(self, graph: fx.Graph) -> int: + groups: dict[tuple[fx.Node, int], list[_SliceInfo]] = defaultdict(list) + + for node in graph.find_nodes( + op="call_function", target=torch.ops.aten.slice.Tensor + ): + slice_info = self._slice_info(node) + if slice_info is None: continue + groups[(node.args[0], slice_info.dim)].append(slice_info) + + count = 0 + for slice_group in groups.values(): + if len(slice_group) < 2: + continue + + source = slice_group[0].node.args[0] + dim = slice_group[0].dim + source_val = self._tensor_val(source) + assert source_val is not None + extent = source_val.shape[dim] + assert isinstance(extent, int) + + ordered = sorted(slice_group, key=lambda info: info.start) + # Ensure they are contiguous partitions. + if ( + ordered[0].start != 0 + or ordered[-1].end != extent + or any( + left.end != right.start for left, right in zip(ordered, ordered[1:]) + ) + ): + continue + + split_sizes = [info.end - info.start for info in ordered] + # Use -1 for the last dimension; otherwise use the positive dimension index. + split_dim = -1 if dim == source_val.dim() - 1 else dim + first_slice = slice_group[0].node + + with graph.inserting_before(first_slice): + split = graph.call_function( + torch.ops.aten.split_with_sizes.default, + args=(source, split_sizes, split_dim), + ) + split.meta["val"] = [info.node.meta["val"] for info in ordered] + + replacements: list[tuple[fx.Node, fx.Node]] = [] + for index, info in enumerate(ordered): + getitem = graph.call_function( + operator.getitem, + args=(split, index), + type_expr=info.node.type, + ) + getitem.meta = copy(info.node.meta) + replacements.append((info.node, getitem)) + + for slice_node, getitem in replacements: + slice_node.replace_all_uses_with(getitem) + graph.erase_node(slice_node) + + count += 1 + + return count + + def _coalesce_splits(self, graph: fx.Graph) -> int: + count = 0 + split_nodes: dict[tuple[fx.Node, int], list[fx.Node]] = defaultdict(list) + + for node in graph.find_nodes( + op="call_function", target=torch.ops.aten.split_with_sizes.default + ): if not all(is_func(user, operator.getitem) for user in node.users): continue - arg_node, split_sizes = node.args[:2] - - if arg_node not in split_nodes: - split_nodes[arg_node] = [node] + source, split_sizes = node.args[:2] + source_val = self._tensor_val(source) + if source_val is None: continue - - # Find existing node with same split_sizes + raw_dim = self._get_arg(node, 2, "dim", 0) + dim = self._normalize_dim(raw_dim, source_val.dim()) + candidates = split_nodes[(source, dim)] canonical = next( ( - n - for n in split_nodes[arg_node] - if list(n.args[1]) == list(split_sizes) + candidate + for candidate in candidates + if list(candidate.args[1]) == list(split_sizes) ), None, ) @@ -65,6 +210,16 @@ class SplitCoalescingPass(VllmInductorPass): graph.erase_node(node) count += 1 else: - split_nodes[arg_node].append(node) + candidates.append(node) - logger.debug("Coalesced %d duplicate split_with_sizes nodes", count) + return count + + @VllmInductorPass.time_and_log + def __call__(self, graph: fx.Graph) -> None: + canonicalized = self._canonicalize_slices(graph) + coalesced = self._coalesce_splits(graph) + logger.debug( + "Canonicalized %d slice partitions and coalesced %d split nodes", + canonicalized, + coalesced, + ) diff --git a/vllm/compilation/wrapper.py b/vllm/compilation/wrapper.py index 1df30a70a8..78b34f8959 100644 --- a/vllm/compilation/wrapper.py +++ b/vllm/compilation/wrapper.py @@ -12,7 +12,13 @@ from typing import Any, ParamSpec, TypeVar import torch import vllm.envs as envs -from vllm.config import CompilationMode, CUDAGraphMode, get_current_vllm_config +from vllm.compilation.counter import compilation_counter +from vllm.config import ( + CompilationMode, + CUDAGraphMode, + VllmConfig, + get_current_vllm_config, +) from vllm.config.compilation import DynamicShapesType from vllm.logger import init_logger from vllm.utils.nvtx_pytorch_hooks import layerwise_nvtx_marker_context @@ -44,6 +50,17 @@ def _compilation_context() -> Generator[None, None, None]: torch._dynamo.config.accumulated_cache_size_limit = original_accumulated_cache +def compile_model_with_stock_torch( + model: torch.nn.Module, vllm_config: VllmConfig +) -> None: + from vllm.env_override import _apply_constrain_to_fx_strides_patch + + _apply_constrain_to_fx_strides_patch() + backend = vllm_config.compilation_config.init_backend(vllm_config) + compilation_counter.stock_torch_compile_count += 1 + model.compile(fullgraph=True, backend=backend) + + class TorchCompileWithNoGuardsWrapper: """A wrapper class for torch.compile, it ensures that all guards are dropped when CompilationMode is not CompilationMode.STOCK_TORCH_COMPILE. diff --git a/vllm/config/engram.py b/vllm/config/engram.py index da2919f97b..33e90e0c04 100644 --- a/vllm/config/engram.py +++ b/vllm/config/engram.py @@ -100,6 +100,7 @@ class EngramConfig: if self.dp_shared_memory is None: self.dp_shared_memory = ( self.cpu_offload + and not self.use_thp and parallel_config.data_parallel_size > 1 and not parallel_config.enable_elastic_ep ) diff --git a/vllm/config/lora.py b/vllm/config/lora.py index 24e42a42fc..e5c1d14f6d 100644 --- a/vllm/config/lora.py +++ b/vllm/config/lora.py @@ -84,6 +84,9 @@ class LoRAConfig: experts (stored once with expert-dim 1) instead of per-expert. The shared factors are broadcast to the expert count at kernel time. Only meaningful for MoE models whose adapters use this layout; ignored otherwise.""" + max_lora_cls_labels: int | None = Field(default=None, ge=1) + """Maximum output size for LoRA classification heads. Defaults to the + base classification head size.""" def compute_hash(self) -> str: """WARNING: Whenever a new field is added to this config, @@ -104,6 +107,7 @@ class LoRAConfig: factors.append(self.enable_tower_connector_lora) factors.append(self.enable_mixed_moe_lora_format) factors.append(self.enable_moe_shared_loras) + factors.append(self.max_lora_cls_labels) # target_modules affects which modules get LoRA applied factors.append( tuple(sorted(self.target_modules)) if self.target_modules else None diff --git a/vllm/config/multimodal.py b/vllm/config/multimodal.py index 5ce309456b..24c534b364 100644 --- a/vllm/config/multimodal.py +++ b/vllm/config/multimodal.py @@ -27,6 +27,186 @@ logger = init_logger(__name__) _T = TypeVar("_T") +# HuggingFace processors accept a shared flat namespace in addition to the +# nested processor kwarg scopes ``text_kwargs`` / ``images_kwargs`` / +# ``videos_kwargs`` / ``audio_kwargs``. vLLM recognizes these scopes when +# resolving ``mm_processor_kwargs``. +_HF_PROCESSOR_KWARG_SCOPES = ( + "text_kwargs", + "images_kwargs", + "videos_kwargs", + "audio_kwargs", +) + + +def _recursively_merge_mm_processor_kwargs( + defaults: Mapping[str, object], + overrides: Mapping[str, object], +) -> dict[str, object]: + """Merge two mappings recursively, with ``overrides`` taking precedence. + + Nested values are merged only when both values are mappings; otherwise the + override replaces the existing value. + """ + merged = dict(defaults) + + for key, value in overrides.items(): + default_value = merged.get(key) + if isinstance(default_value, Mapping) and isinstance(value, Mapping): + merged[key] = _recursively_merge_mm_processor_kwargs(default_value, value) + else: + merged[key] = value + + return merged + + +def _merge_mm_processor_scope( + config_flat: Mapping[str, object], + config_scoped: Mapping[str, object], + inference_flat: Mapping[str, object], + inference_scoped: Mapping[str, object], +) -> dict[str, object]: + """Merge kwargs for one existing HuggingFace processor kwarg scope. + + Flat kwargs contribute only to keys present in either the configured or + inference-time scope. For those keys, precedence is: + configured flat < configured scoped < inference flat < inference scoped. + + Nested mappings are merged recursively; otherwise the higher-priority value + replaces the lower-priority value. + """ + # Flat kwargs contribute only to keys present in this scope in either input. + scoped_keys = config_scoped.keys() | inference_scoped.keys() + config_flat_for_scope = { + key: value for key, value in config_flat.items() if key in scoped_keys + } + inference_flat_for_scope = { + key: value for key, value in inference_flat.items() if key in scoped_keys + } + + # Merge from lowest to highest priority: + # configured flat -> configured scoped -> inference flat -> inference scoped. + merged = _recursively_merge_mm_processor_kwargs( + config_flat_for_scope, config_scoped + ) + merged = _recursively_merge_mm_processor_kwargs(merged, inference_flat_for_scope) + merged = _recursively_merge_mm_processor_kwargs(merged, inference_scoped) + return merged + + +def _merge_mm_processor_kwargs( + config_kwargs: Mapping[str, object], + inference_kwargs: Mapping[str, object], +) -> dict[str, object]: + """Merge configured and inference-time multi-modal processor kwargs. + + The merge is performed in two parts: + + 1. The shared flat kwargs are merged recursively, with inference-time values + taking precedence. + 2. Each existing HuggingFace processor kwarg scope is merged separately. + For keys present in either the configured or inference-time scope, + precedence is: + configured flat < configured scoped < inference flat < inference scoped. + + For example: + + configured = { + "size": {"shortest_edge": 256}, + "videos_kwargs": { + "size": {"longest_edge": 1024}, + "num_frames": 16, + }, + } + inference = { + "size": {"shortest_edge": 512}, + "videos_kwargs": {"fps": 4}, + } + + produces: + + { + "size": {"shortest_edge": 512}, + "videos_kwargs": { + "size": { + "shortest_edge": 512, + "longest_edge": 1024, + }, + "num_frames": 16, + "fps": 4, + }, + } + + Scoped values do not modify the shared flat kwargs or sibling scopes. + Missing scopes, None-valued scopes, and empty scope mappings are treated as + absent and do not create a scope. + """ + # Merge the shared flat kwargs first; nested scopes are handled separately + # below. + config_flat = { + key: value + for key, value in config_kwargs.items() + if key not in _HF_PROCESSOR_KWARG_SCOPES + } + inference_flat = { + key: value + for key, value in inference_kwargs.items() + if key not in _HF_PROCESSOR_KWARG_SCOPES + } + merged = _recursively_merge_mm_processor_kwargs(config_flat, inference_flat) + + # Merge each nested processor kwarg scope present in either input. + for scoped_key in _HF_PROCESSOR_KWARG_SCOPES: + config_scope_value = config_kwargs.get(scoped_key) + inference_scope_value = inference_kwargs.get(scoped_key) + + # Treat missing scopes, None-valued scopes, and empty scope mappings as + # absent. + if isinstance(config_scope_value, Mapping) and not config_scope_value: + config_scope_value = None + if isinstance(inference_scope_value, Mapping) and not inference_scope_value: + inference_scope_value = None + + # Skip adding the scope to the merged result when it is absent from both + # sources. + if config_scope_value is None and inference_scope_value is None: + continue + + # Handle the inference scope first because it has higher precedence; only + # mapping-valued scopes participate in the recursive scoped merge. + inference_scope_kwargs: Mapping[str, object] + if inference_scope_value is None: + inference_scope_kwargs = {} + elif not isinstance(inference_scope_value, Mapping): + # Non-mapping scopes cannot participate in the recursive scoped merge; + # the higher-priority inference value replaces the configured scope. + merged[scoped_key] = inference_scope_value + continue + else: + inference_scope_kwargs = inference_scope_value + + # Handle the configured scope next because it has lower precedence; only + # mapping-valued scopes participate in the recursive scoped merge. + if isinstance(config_scope_value, Mapping): + config_scope_kwargs = config_scope_value + elif inference_scope_value is None: + # Non-mapping scopes cannot participate in the recursive scoped merge; + # preserve the configured value when inference contributes nothing. + merged[scoped_key] = config_scope_value + continue + else: + config_scope_kwargs = {} + + # Merge this scope together with the matching flat kwargs. + merged[scoped_key] = _merge_mm_processor_scope( + config_flat, + config_scope_kwargs, + inference_flat, + inference_scope_kwargs, + ) + + return merged + @dataclass class BaseDummyOptions: @@ -533,8 +713,14 @@ class MultiModalConfig: ) -> dict[str, object]: """Get the keyword arguments to pass to the multi-modal processor according to the extra arguments passed during inference. + + Nested mappings are merged recursively, with inference-time values taking + precedence over configured values. """ - merged = dict(self.mm_processor_kwargs or {}) | dict(inference_kwargs) + merged = _merge_mm_processor_kwargs( + self.mm_processor_kwargs or {}, + inference_kwargs, + ) if self.mm_device_do_normalize: # The model expects raw pixels and normalises on device; # overriding these per request would silently double-normalise. diff --git a/vllm/config/parallel.py b/vllm/config/parallel.py index 3929d3be12..b9136cfbb2 100644 --- a/vllm/config/parallel.py +++ b/vllm/config/parallel.py @@ -4,6 +4,7 @@ import os import socket from collections.abc import Callable +from datetime import timedelta from typing import TYPE_CHECKING, Any, Literal, overload import regex as re @@ -613,6 +614,11 @@ class ParallelConfig: """ return self.data_parallel_external_lb or self.data_parallel_hybrid_lb + @property + def cpu_distributed_timeout(self) -> timedelta | None: + seconds = self.cpu_distributed_timeout_seconds + return timedelta(seconds=seconds) if seconds is not None else None + def get_next_dp_init_port(self) -> int: """We might need to initialize process groups in multiple processes that is related to data parallelism, @@ -694,6 +700,7 @@ class ParallelConfig: backend="gloo", return_store=return_store, listen_socket=listen_socket, + timeout=self.cpu_distributed_timeout, ) except DistNetworkError as e: # We only want to retry when the root cause is EADDRINUSE. diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 220da0c72b..8022cd0b88 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -2041,6 +2041,7 @@ class VllmConfig: self._validate_mm_processor_device() if self.use_v2_model_runner: + self._disable_cudagraphs_for_v2_stock_torch_compile() self._validate_v2_model_runner() else: self._validate_v1_model_runner() @@ -3045,9 +3046,6 @@ class VllmConfig: unsupported: list[str] = [] speculative_config = self.speculative_config - if self.compilation_config.mode == CompilationMode.STOCK_TORCH_COMPILE: - unsupported.append("stock torch.compile") - if ( self.compilation_config.pass_config.enable_sp and self.parallel_config.tensor_parallel_size > 1 @@ -3067,8 +3065,6 @@ class VllmConfig: # https://github.com/vllm-project/vllm/pull/40704 "ngram", "ngram_gpu", - # https://github.com/vllm-project/vllm/pull/43091 - "draft_model", "suffix", "medusa", "mlp_speculator", @@ -3088,9 +3084,6 @@ class VllmConfig: if self.parallel_config.use_ubatching: unsupported.extend(self._get_dbo_unsupported_features()) - if self.parallel_config.enable_elastic_ep: - unsupported.append("elastic expert parallelism") - if self.cache_config.mamba_cache_mode == "all": unsupported.append("mamba cache mode 'all'") @@ -3306,6 +3299,28 @@ class VllmConfig: "Enable attribution or use --enforce-eager." ) + def _disable_cudagraphs_for_v2_stock_torch_compile(self) -> None: + """Run stock torch.compile without CUDA graphs in Model Runner V2. + + V1 never wraps a stock-compiled model in CUDAGraphWrapper, so it runs + without CUDA graphs. V2's CUDA graph manager would otherwise capture + FULL graphs around the stock-compiled model, so disable them to match. + """ + compilation_config = self.compilation_config + if ( + compilation_config.mode != CompilationMode.STOCK_TORCH_COMPILE + or compilation_config.cudagraph_mode == CUDAGraphMode.NONE + ): + return + logger.info_once( + "CUDA graphs are not supported with stock torch.compile in Model " + "Runner V2. Overriding cudagraph_mode %s to NONE.", + compilation_config.cudagraph_mode.name, + ) + compilation_config.cudagraph_mode = CUDAGraphMode.NONE + compilation_config.max_cudagraph_capture_size = 0 + compilation_config.cudagraph_capture_sizes = [] + def _validate_v2_model_runner(self) -> None: """Check for features not yet supported by the V2 model runner.""" if not HAS_TRITON: diff --git a/vllm/distributed/device_communicators/flashinfer_all_reduce.py b/vllm/distributed/device_communicators/flashinfer_all_reduce.py index f1644d699e..9bb17e1528 100644 --- a/vllm/distributed/device_communicators/flashinfer_all_reduce.py +++ b/vllm/distributed/device_communicators/flashinfer_all_reduce.py @@ -20,6 +20,7 @@ from vllm.distributed.device_communicators.all_reduce_utils import ( from vllm.distributed.parallel_state import _node_count, get_node_count from vllm.logger import init_logger from vllm.platforms import current_platform +from vllm.utils.gpu_sync_debug import gpu_sync_allowed logger = init_logger(__name__) @@ -45,6 +46,10 @@ _fi_ar_workspace = None # allreduce backend or a fallback backend when the primary workspace is not # available on the current topology. _fi_ar_quant_workspace = None +# Creation is collective, so a failed attempt must not be retried on every +# all-reduce call. +_fi_ar_workspace_failed = False +_fi_ar_quant_workspace_failed = False _fi_ar_workspace_groups: dict[int, ProcessGroup] = {} @@ -79,16 +84,19 @@ def _create_workspace( rng_state = random.getstate() try: random.seed(int.from_bytes(os.urandom(16), byteorder="big")) - workspace = flashinfer_comm.create_allreduce_fusion_workspace( - backend=backend, - world_size=world_size, - rank=rank, - max_token_num=max_token_num, - hidden_dim=hidden_dim, - dtype=dtype, - comm_backend=comm_backend, - group=group, - ) + # Creation may run lazily inside the first sync-checked forward (e.g. + # with enforce_eager, which skips warmup). + with gpu_sync_allowed(first_only=True): + workspace = flashinfer_comm.create_allreduce_fusion_workspace( + backend=backend, + world_size=world_size, + rank=rank, + max_token_num=max_token_num, + hidden_dim=hidden_dim, + dtype=dtype, + comm_backend=comm_backend, + group=group, + ) if backend == "mnnvl" and not getattr(workspace, "mc_ptr", 0): workspace.destroy() logger.warning_once( @@ -175,8 +183,8 @@ def get_fi_ar_workspace( for standalone allreduce. Backend is controlled by VLLM_FLASHINFER_ALLREDUCE_BACKEND env var. """ - global _fi_ar_workspace - if _fi_ar_workspace is not None: + global _fi_ar_workspace, _fi_ar_workspace_failed + if _fi_ar_workspace is not None or _fi_ar_workspace_failed: return _fi_ar_workspace backend, allow_trtllm_fallback = _resolve_fi_ar_backend() @@ -218,6 +226,7 @@ def get_fi_ar_workspace( f"with backend={backend}" ) else: + _fi_ar_workspace_failed = True logger.error_once( "Failed to initialize FlashInfer Allreduce norm fusion workspace " f"with backend={backend}" @@ -240,8 +249,8 @@ def get_fi_ar_quant_workspace( non-quant fusion. With ``auto`` this prefers mnnvl and falls back to trtllm only on single-node topologies where mnnvl multicast is unavailable. """ - global _fi_ar_quant_workspace - if _fi_ar_quant_workspace is not None: + global _fi_ar_quant_workspace, _fi_ar_quant_workspace_failed + if _fi_ar_quant_workspace is not None or _fi_ar_quant_workspace_failed: return _fi_ar_quant_workspace backend, allow_trtllm_fallback = _resolve_fi_ar_backend() @@ -291,6 +300,7 @@ def get_fi_ar_quant_workspace( f"fusion workspace with backend={backend}" ) else: + _fi_ar_quant_workspace_failed = True logger.error_once( "Failed to initialize FlashInfer Allreduce norm quantization " f"fusion workspace with backend={backend}" @@ -304,6 +314,7 @@ _fi_ar_workspace_lock = threading.Lock() def destroy_fi_ar_workspace(): global _fi_ar_workspace, _fi_ar_quant_workspace + global _fi_ar_workspace_failed, _fi_ar_quant_workspace_failed with _fi_ar_workspace_lock: is_alias = _fi_ar_workspace is _fi_ar_quant_workspace @@ -313,6 +324,7 @@ def destroy_fi_ar_workspace(): _fi_ar_quant_workspace.destroy() _fi_ar_workspace = _fi_ar_quant_workspace = None + _fi_ar_workspace_failed = _fi_ar_quant_workspace_failed = False _fi_ar_workspace_groups.clear() diff --git a/vllm/distributed/elastic_ep/elastic_execute.py b/vllm/distributed/elastic_ep/elastic_execute.py index 2602edf2df..e66bafd817 100644 --- a/vllm/distributed/elastic_ep/elastic_execute.py +++ b/vllm/distributed/elastic_ep/elastic_execute.py @@ -54,6 +54,8 @@ from vllm.platforms import current_platform from vllm.utils import is_moe_layer from vllm.v1.engine import ReconfigureDistributedRequest, ReconfigureRankType from vllm.v1.worker.dp_utils import skip_dp_coordination +from vllm.v1.worker.gpu.cudagraph_utils import CudaGraphManager +from vllm.v1.worker.gpu.eplb_utils import preserve_serving_state from vllm.v1.worker.gpu_ubatch_wrapper import UBatchWrapper from vllm.v1.worker.workspace import lock_workspace, unlock_workspace @@ -171,6 +173,17 @@ class ElasticEPScalingExecutor: self.worker.vllm_config.parallel_config, ) + @contextmanager + def _suppress_eplb(self) -> Iterator[None]: + # A rank warming up alone must not advance the EPLB step counter. + runner = self.worker.model_runner + was_suppressed = runner.eep_eplb_suppressed + runner.eep_eplb_suppressed = True + try: + yield + finally: + runner.eep_eplb_suppressed = was_suppressed + @contextmanager def _disable_flashinfer_autotune(self) -> Iterator[None]: kernel_config = self.worker.vllm_config.kernel_config @@ -421,7 +434,17 @@ class ElasticEPScalingExecutor: self._staged_moe_quant_methods.clear() def _release_cuda_graphs(self) -> None: - if isinstance(self.worker.model_runner.model, CUDAGraphWrapper): + manager = getattr(self.worker.model_runner, "cudagraph_manager", None) + speculator = getattr(self.worker.model_runner, "speculator", None) + if manager is not None: + # MRV2 captures through CudaGraphManager instead of wrapping the + # model, so neither wrapper branch below ever fires. + manager.release_graphs() + for attr in vars(speculator).values() if speculator else (): + if isinstance(attr, CudaGraphManager): + attr.release_graphs() + + elif isinstance(self.worker.model_runner.model, CUDAGraphWrapper): wrapper = self.worker.model_runner.model wrapper.concrete_cudagraph_entries = {} @@ -431,6 +454,8 @@ class ElasticEPScalingExecutor: torch.compiler.reset() with set_current_vllm_config(self.worker.vllm_config): reset_compile_wrapper(self.worker.model_runner.get_model()) + if speculator is not None: + reset_compile_wrapper(speculator.model) gc.collect() torch.accelerator.synchronize() @@ -675,42 +700,28 @@ class ElasticEPScalingExecutor: self.warm_and_capture() def warm_and_capture(self) -> None: - # Save and clear block tables so the dummy MoE forward doesn't - # write dummy slot mappings into real KV-cache blocks. - multi_block_table = self.worker.model_runner.input_batch.block_table - saved_block_tables: list[tuple[torch.Tensor, torch.Tensor]] = [] - for bt in multi_block_table.block_tables: - saved_block_tables.append( - (bt.block_table.gpu.clone(), bt.block_table.cpu.clone()) - ) - multi_block_table.clear() - # _ensure_workspace_size allocates a fresh tensor on grow, leaving # any captured CUDA graph with a stale data pointer; drop graphs # before re-warm so captures realign with the resized buffer. self._release_cuda_graphs() unlock_workspace() - # Grow the MoE workspace at max_num_tokens. compile_or_warm_up_model - # alone only exercises cudagraph-capture sizes and can leave the - # workspace too small for post-reshuffle routing. Use _dummy_run - # directly with skip_eplb=True so dummy routing doesn't pollute the - # just-rebalanced EPLB stats. runner = self.worker.model_runner all2all_manager = get_ep_all2all_manager() reuse_kernel = self._can_reuse_fused_moe_kernel() + serving_state = ( + preserve_serving_state(runner) + if self.worker.use_v2_model_runner + else runner.preserve_serving_state() + ) with ( skip_dp_coordination() if reuse_kernel else nullcontext(), all2all_manager.mask_remote_ranks() if reuse_kernel else nullcontext(), self._disable_flashinfer_autotune() if reuse_kernel else nullcontext(), + self._suppress_eplb(), + serving_state, ): runner._dummy_run(runner.max_num_tokens, is_profile=True, skip_eplb=True) self.worker.compile_or_warm_up_model() lock_workspace() - - for bt, (saved_gpu, saved_cpu) in zip( - multi_block_table.block_tables, saved_block_tables - ): - bt.block_table.gpu.copy_(saved_gpu) - bt.block_table.cpu.copy_(saved_cpu) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py index f4a643d2b7..0cbdd786d8 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py @@ -165,8 +165,8 @@ class NixlBaseConnectorWorker: # Overridden by NixlPushConnectorWorker. _TRANSFER_MODE: str = "pull" - # Layer-name routing is supported only by NixlPushConnector for HMA KV - # caches under PP. + # Layer-name routing is supported only by NixlPushConnector for HMA or + # packed KV caches under PP. _supports_pp_hma = False def _compute_desc_ids( @@ -432,15 +432,22 @@ class NixlBaseConnectorWorker: group_by_layer[name] for name in layer_names ) - def _requires_layer_name_routing(self) -> bool: - """Whether PP push must match HMA layers by name, not region index.""" + def _tracks_region_layers(self) -> bool: + """Whether regions advertise their member layer names. + + Push workers do so for HMA and packed layouts, so a PP producer can + match layers by name instead of by region index. + """ return ( self._supports_pp_hma - and self.pp_size > 1 - and self._is_hma_required + and (self._is_hma_required or self._has_packed_cache) and not self._has_mamba ) + def _requires_layer_name_routing(self) -> bool: + """Whether PP push must match HMA/packed layers by name, not region index.""" + return self.pp_size > 1 and self._tracks_region_layers() + def _align_remote_regions_by_layer( self, nixl_agent_meta: NixlAgentMetadata ) -> None: @@ -479,6 +486,10 @@ class NixlBaseConnectorWorker: ] assert not missing, f"Remote is missing locally owned layers: {missing}" + packed_regions = { + remote_region_by_layer[name] + for name in nixl_agent_meta.packed_member_layouts + } remote_regions = [ remote_region_by_layer[name] for name in self._transfer_layer_names ] @@ -505,6 +516,24 @@ class NixlBaseConnectorWorker: ] if nixl_agent_meta.region_names is not None: nixl_agent_meta.region_names = list(self._transfer_layer_names) + if nixl_agent_meta.packed_member_layouts: + for i, layer_name in enumerate(self._transfer_layer_names): + if remote_regions[i] not in packed_regions: + continue + assert layer_name in nixl_agent_meta.packed_member_layouts, ( + f"Remote packed layer {layer_name!r} has no layout" + ) + offset, page_size = nixl_agent_meta.packed_member_layouts[layer_name] + assert ( + 0 <= offset < offset + page_size <= nixl_agent_meta.block_lens[i] + ), f"Remote packed layer {layer_name!r} escapes its block" + local_region = self._transfer_layer_region_indices[i] + assert page_size == self.block_len_per_layer[local_region], ( + f"Packed MLA page sizes must match for layer {layer_name!r}" + ) + nixl_agent_meta.kv_caches_base_addr[i] += offset + nixl_agent_meta.block_lens[i] = page_size + nixl_agent_meta.packed_member_layouts = {} # One layer per region keeps a second alignment pass a no-op. nixl_agent_meta.region_members = [[name] for name in self._transfer_layer_names] @@ -899,7 +928,10 @@ class NixlBaseConnectorWorker: # within a block (BLHNC/BHLNC), where stride > block_len. self.block_stride_per_layer = list[int]() - # Region layer membership is populated for HMA layouts. + # Block-outermost layouts (BLHNC/BHLNC) pack every layer's page into one + # block row of a shared allocation. + self._has_packed_cache = KVCacheLayout[self.kv_cache_layout].is_block_outermost + # Region layer membership is populated for HMA and packed layouts. self.region_members = [] # Local layer order, set by ``_set_region_layers`` only when # routing by layer name. Empty means region-index routing. @@ -1352,6 +1384,8 @@ class NixlBaseConnectorWorker: def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]): """Register the KV Cache data in nixl.""" + use_layer_name_routing = self._requires_layer_name_routing() + route_packed_layers = self._has_packed_cache and use_layer_name_routing self.transfer_topo = TransferTopology( tp_rank=self.transfer_tp_rank, tp_size=self.transfer_tp_size, @@ -1369,9 +1403,15 @@ class NixlBaseConnectorWorker: else None, is_mamba=self._has_mamba, ) + backend_name = self.backend_name + if self._supports_pp_hma and self._has_packed_cache: + # PP can reorder the same backends across stages of a packed model. + backend_name = ",".join( + sorted(backend.get_name() for backend in self.attn_backends) + ) self.compat_hash = compute_nixl_compatibility_hash( self.vllm_config, - self.backend_name, + backend_name, transfer_mode=self._TRANSFER_MODE, ) @@ -1413,6 +1453,11 @@ class NixlBaseConnectorWorker: # CSA-linear needs separate logical regions for attention and state # aliases even though every view shares one block-major allocation. packed_storage = packed_storage and not self._is_csa_linear + if route_packed_layers and any( + not isinstance(spec, (MLAAttentionSpec, SlidingWindowMLASpec)) + for spec in self._layer_specs.values() + ): + raise NotImplementedError("PP push with packed KV caches requires MLA") layer_specs: dict[str, KVCacheSpec] = {} compressed_region_owners: dict[int, torch.Tensor] = {} @@ -1439,10 +1484,9 @@ class NixlBaseConnectorWorker: ): compressed_region_owners.setdefault(cache.data_ptr(), cache) - track_region_layers = ( - self._supports_pp_hma and self._is_hma_required and not self._has_mamba - ) + track_region_layers = self._tracks_region_layers() region_layers: list[list[str]] = [] + packed_member_layouts: dict[str, tuple[int, int]] = {} # K and V are packed into the content dim, so each attention layer is a # single NIXL region whose block transfers as one unit. Mamba layers instead @@ -1587,11 +1631,34 @@ class NixlBaseConnectorWorker: virtual_transfer_pages = _uses_dense_virtual_transfer_pages( layer_spec, cache, physical_page_size, num_blocks ) + # Push workers address the pages of packed MLA rows by layer. + packed_mla_push = ( + track_region_layers + and self._has_packed_cache + and packed_storage + and storage_is_block_major + and is_mla_region + ) if virtual_transfer_pages: # A compressed kernel row can contain multiple NIXL transfer pages. region_specs = [ (cache.data_ptr(), physical_page_size, physical_page_size) ] + elif packed_mla_push and use_layer_name_routing: + # PP>1 prefill stage: register only this layer's page of each row. + region_specs = [ + (cache.data_ptr(), physical_page_size, block_stride) + ] + elif packed_mla_push: + # PP=1 decode or prefill: register whole rows, and advertise where + # this layer's page sits so a PP>1 prefill can write into it. + storage_block_len = registration_len // num_blocks + region_specs = [ + (registration_base, storage_block_len, storage_block_len) + ] + offset = cache.data_ptr() - storage_addr + assert offset >= 0 and offset + physical_page_size <= block_stride + packed_member_layouts[layer_name] = (offset, physical_page_size) elif storage_is_block_major and ( not page_contiguous and ((packed_storage and is_mla_region) or not self._is_csa_linear) @@ -1633,7 +1700,9 @@ class NixlBaseConnectorWorker: ] for base_addr, block_len, block_stride in region_specs: - if base_addr in seen_base_addresses: + # A packed PP producer keeps one region per layer, so aliases at + # one address (a layer and its SWA view) stay distinct. + if base_addr in seen_base_addresses and not route_packed_layers: region_index = seen_base_addresses.index(base_addr) assert region_mem_types[region_index] == mem_type self._region_is_mla[region_index] |= is_mla_region @@ -1717,7 +1786,7 @@ class NixlBaseConnectorWorker: self._set_region_layers(region_layers) - if self.pp_size > 1 and not self._is_hma_required: + if self.pp_size > 1 and not use_layer_name_routing: start_layer, end_layer = self.model_config.get_layers_start_end_indices( self.vllm_config.parallel_config ) @@ -1807,6 +1876,7 @@ class NixlBaseConnectorWorker: dcp_size=self.dcp_size, pcp_size=self.pcp_size, region_members=self.region_members, + packed_member_layouts=packed_member_layouts, ) # Wrap metadata in payload with hash for defensive decoding assert self.compat_hash is not None diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py index ca6f30d0cf..8d0aad7633 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py @@ -48,8 +48,9 @@ PUSH_REG_NOTIF_PREFIX = b"PUSH_REG:" # 10: Add dense virtual transfer pages for compressed MLA caches # 11: Add per-region transfer geometry and memory types to NixlAgentMetadata # 12: Add per-region member names for PP push +# 13: Add packed-member layouts and order-independent packed-push backend hashes # -NIXL_CONNECTOR_VERSION: int = 12 +NIXL_CONNECTOR_VERSION: int = 13 @dataclass @@ -74,6 +75,8 @@ class NixlAgentMetadata: pcp_size: int = 1 # Layer names sharing each advertised region, in region order. region_members: list[list[str]] = field(default_factory=list) + # Packed member -> (byte offset in its region's block, bytes per page). + packed_member_layouts: dict[str, tuple[int, int]] = field(default_factory=dict) @dataclass diff --git a/vllm/distributed/stateless_coordinator.py b/vllm/distributed/stateless_coordinator.py index 6ce2aa89ac..40957aac3e 100644 --- a/vllm/distributed/stateless_coordinator.py +++ b/vllm/distributed/stateless_coordinator.py @@ -17,6 +17,8 @@ from vllm.distributed.parallel_state import ( ) from vllm.distributed.utils import ( StatelessProcessGroup, + get_cpu_distributed_timeout_or_none, + get_distributed_timeout_or_none, stateless_destroy_torch_distributed_process_group, stateless_init_torch_distributed_process_group, ) @@ -104,6 +106,10 @@ class StatelessGroupCoordinator(GroupCoordinator): backend = str(torch_distributed_backend) self.backend = backend + cpu_timeout = get_cpu_distributed_timeout_or_none() + device_timeout = ( + cpu_timeout if backend == "gloo" else get_distributed_timeout_or_none() + ) for idx, ranks in enumerate(group_ranks): if self.rank in ranks: self.ranks = ranks @@ -130,6 +136,7 @@ class StatelessGroupCoordinator(GroupCoordinator): backend=backend, group_name=f"{self.unique_name}_device", listen_socket=socks[0] if socks else None, + timeout=device_timeout, ) cpu_group = stateless_init_torch_distributed_process_group( host=host, @@ -139,6 +146,7 @@ class StatelessGroupCoordinator(GroupCoordinator): backend="gloo", group_name=f"{self.unique_name}_cpu", listen_socket=socks[1] if socks else None, + timeout=cpu_timeout, ) tcp_store_group = StatelessProcessGroup.create( host=host, diff --git a/vllm/distributed/utils.py b/vllm/distributed/utils.py index b32269ec2b..c780316bb4 100644 --- a/vllm/distributed/utils.py +++ b/vllm/distributed/utils.py @@ -530,8 +530,7 @@ def get_cpu_distributed_timeout_or_none() -> timedelta | None: vllm_config = get_current_vllm_config_or_none() if vllm_config is None: return None - timeout_seconds = vllm_config.parallel_config.cpu_distributed_timeout_seconds - return timedelta(seconds=timeout_seconds) if timeout_seconds is not None else None + return vllm_config.parallel_config.cpu_distributed_timeout def get_distributed_timeout_or_none() -> timedelta | None: @@ -582,6 +581,7 @@ def stateless_init_torch_distributed_process_group( group_name: str | None = None, return_store: bool = False, listen_socket: socket.socket | None = None, + timeout: timedelta | None = None, ) -> ProcessGroup | tuple[ProcessGroup, Store]: """A replacement for `torch.distributed.init_process_group` that does not pollute the global state. The created ProcessGroup object can be used for @@ -617,18 +617,13 @@ def stateless_init_torch_distributed_process_group( is skipped and a ``TCPStore`` server is created directly using the pre-bound socket. This is useful for eliminating TOCTOU races between port allocation and binding. + + If *timeout* is None, use PyTorch's default for the backend. """ init_method = get_tcp_uri(host, port) backend = Backend(backend) # it is basically string - timeout = _get_default_timeout(backend) - if backend == "gloo": - gloo_timeout = get_cpu_distributed_timeout_or_none() - if gloo_timeout is not None: - timeout = gloo_timeout - else: - device_timeout = get_distributed_timeout_or_none() - if device_timeout is not None: - timeout = device_timeout + if timeout is None: + timeout = _get_default_timeout(backend) if listen_socket is not None: store = create_tcp_store( diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 3e84f50997..d641b1fbab 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -676,6 +676,7 @@ class EngineArgs: specialize_active_lora: bool = LoRAConfig.specialize_active_lora enable_mixed_moe_lora_format: bool = LoRAConfig.enable_mixed_moe_lora_format enable_moe_shared_loras: bool = LoRAConfig.enable_moe_shared_loras + max_lora_cls_labels: int | None = LoRAConfig.max_lora_cls_labels ray_workers_use_nsight: bool = ParallelConfig.ray_workers_use_nsight num_gpu_blocks_override: int | None = CacheConfig.num_gpu_blocks_override @@ -1571,6 +1572,10 @@ class EngineArgs: "--enable-moe-shared-loras", **lora_kwargs["enable_moe_shared_loras"], ) + lora_group.add_argument( + "--max-lora-cls-labels", + **lora_kwargs["max_lora_cls_labels"], + ) # Logging arguments logging_group = parser.add_argument_group( @@ -2587,6 +2592,7 @@ class EngineArgs: specialize_active_lora=self.specialize_active_lora, enable_mixed_moe_lora_format=self.enable_mixed_moe_lora_format, enable_moe_shared_loras=self.enable_moe_shared_loras, + max_lora_cls_labels=self.max_lora_cls_labels, max_cpu_loras=self.max_cpu_loras if self.max_cpu_loras and self.max_cpu_loras > 0 else None, diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index f739495a8a..a419ffca81 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -1810,6 +1810,11 @@ PART_TYPES_TO_SKIP_NONE_CONTENT = ( "refusal", ) +# Content part types parsed as text rather than multimodal data. +TEXT_PART_TYPES = frozenset( + {"text", "input_text", "output_text", "refusal", "thinking"} +) + def _parse_chat_message_content_parts( role: str, @@ -1904,7 +1909,7 @@ def _parse_chat_message_content_part( ) return None - if part_type in ("text", "input_text", "output_text", "refusal", "thinking"): + if part_type in TEXT_PART_TYPES: str_content = cast(str, content) _reject_reserved_placeholder_in_text(str_content, mm_parser.model_config) if wrap_dicts: diff --git a/vllm/entrypoints/openai/chat_completion/protocol.py b/vllm/entrypoints/openai/chat_completion/protocol.py index 5c1bc594d7..8ad7534d6d 100644 --- a/vllm/entrypoints/openai/chat_completion/protocol.py +++ b/vllm/entrypoints/openai/chat_completion/protocol.py @@ -20,6 +20,8 @@ from pydantic import ( from vllm.config import ModelConfig from vllm.entrypoints.chat_utils import ( + MM_PARSER_MAP, + TEXT_PART_TYPES, ChatCompletionMessageParam, ChatTemplateContentFormatOption, ) @@ -58,6 +60,11 @@ logger = init_logger(__name__) _INT64_MIN = -(2**63) _INT64_MAX = 2**63 - 1 +# Content part types that carry no multimodal data. +_TEXT_CONTENT_PART_TYPES = TEXT_PART_TYPES | {"tool_reference"} +# Keys that mark a content part as multimodal, whatever its ``type``. +_MEDIA_CONTENT_PART_KEYS = frozenset(MM_PARSER_MAP) - _TEXT_CONTENT_PART_TYPES + class ChatMessage(OpenAIBaseModel): role: str @@ -976,6 +983,39 @@ class ChatCompletionRequest(OpenAIBaseModel): ) return data + @model_validator(mode="before") + @classmethod + def drop_prompt_token_ids_with_media(cls, data): + # The forwarded ids would drop media in ``messages``, so ignore them. Runs + # before validation, which can turn content into a one-shot iterator. + if not isinstance(data, dict): + return data + kv_transfer_params = data.get("kv_transfer_params") + messages = data.get("messages") + if ( + not isinstance(kv_transfer_params, dict) + or kv_transfer_params.get("prompt_token_ids") is None + or not isinstance(messages, list) + ): + return data + for msg in messages: + content = msg.get("content") if isinstance(msg, dict) else None + if not isinstance(content, list): + continue + for part in content: + if isinstance(part, dict) and ( + any(key in part for key in _MEDIA_CONTENT_PART_KEYS) + or not isinstance(part_type := part.get("type", "text"), str) + or part_type not in _TEXT_CONTENT_PART_TYPES + ): + logger.debug( + "Ignoring kv_transfer_params['prompt_token_ids']: " + "messages have non-text content and are rendered instead." + ) + kv_transfer_params.pop("prompt_token_ids") + return data + return data + @model_validator(mode="before") @classmethod def check_system_message_content_type(cls, data): @@ -1117,6 +1157,17 @@ class BatchChatCompletionRequest(OpenAIBaseModel): "when using `logprob_token_ids`, `logprobs` must be set to true.", parameter="logprob_token_ids", ) + kv_transfer_params = data.get("kv_transfer_params") + if ( + isinstance(kv_transfer_params, dict) + and kv_transfer_params.get("prompt_token_ids") is not None + ): + raise VLLMValidationError( + "Batch chat completions do not support " + "`kv_transfer_params['prompt_token_ids']`: one pre-tokenized " + "prompt cannot serve several conversations.", + parameter="kv_transfer_params.prompt_token_ids", + ) response_format = data.get("response_format") if response_format is not None: rf_type = ( diff --git a/vllm/entrypoints/pooling/base/io_processor.py b/vllm/entrypoints/pooling/base/io_processor.py index 05ec69e44a..0d9431f022 100644 --- a/vllm/entrypoints/pooling/base/io_processor.py +++ b/vllm/entrypoints/pooling/base/io_processor.py @@ -207,7 +207,10 @@ class PoolingIOProcessor: else: pooling_params = ctx.pooling_params - params_seq = self._params_to_seq(pooling_params, num_requests) + # Clone to avoid modifying the caller's pooling parameters. + params_seq = [ + param.clone() for param in self._params_to_seq(pooling_params, num_requests) + ] for param in params_seq: if param.task is None: diff --git a/vllm/entrypoints/pooling/pooling/io_processor.py b/vllm/entrypoints/pooling/pooling/io_processor.py index 4b18cfc9e1..d5132f2a66 100644 --- a/vllm/entrypoints/pooling/pooling/io_processor.py +++ b/vllm/entrypoints/pooling/pooling/io_processor.py @@ -168,7 +168,7 @@ class PluginWithIOProcessorPlugins(PoolingIOProcessor): pooling_params = ctx.pooling_params params_seq: list[PoolingParams] = [ - self.io_processor.merge_pooling_params(param) + self.io_processor.merge_pooling_params(param.clone()) for param in self._params_to_seq( pooling_params, num_requests, diff --git a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py index fb2b938723..ad607648bd 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py @@ -381,6 +381,7 @@ class GenerateResponse(BaseModel): choices: list[GenerateResponseChoice] usage: UsageInfo | None = Field(default=None) prompt_logprobs: list[dict[int, Logprob] | None] | None = None + prompt_token_id_logprobs: str | None = None prompt_token_ids: list[int] | None = None mm_placeholders: dict[str, list[PlaceholderRangeInfo]] | None = None metrics: PerRequestMetrics | None = None diff --git a/vllm/entrypoints/scale_out/token_in_token_out/serving.py b/vllm/entrypoints/scale_out/token_in_token_out/serving.py index b480b802ec..8b0fbff99b 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/serving.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/serving.py @@ -138,6 +138,11 @@ class ServingTokens(GenerateBaseServing): f"sampling_params.n must be at most the server's max_num_seqs " f"({max_num_seqs}), got {sampling_params.n}." ) + # The stream schema has no field for the scores. + if request.stream and sampling_params.prompt_logprob_token_ids is not None: + return self.create_error_response( + "prompt_logprob_token_ids are not available when stream=true." + ) if self.force_no_detokenize and sampling_params.stop: # SamplingParams rejects stop with detokenize=False at request # validation, but this server forces detokenize=False afterwards, @@ -389,6 +394,11 @@ class ServingTokens(GenerateBaseServing): choices=choices, usage=usage, prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs), + prompt_token_id_logprobs=( + numpy2base64(final_res.prompt_token_id_logprobs) + if final_res.prompt_token_id_logprobs is not None + else None + ), prompt_token_ids=( final_res.prompt_token_ids if request.return_token_ids else None ), diff --git a/vllm/envs.py b/vllm/envs.py index f663414a91..c8e1a8b708 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -145,8 +145,10 @@ if TYPE_CHECKING: VLLM_ROCM_USE_AITER_RMSNORM: bool = True VLLM_ROCM_USE_AITER_MLA: bool = True VLLM_ROCM_AITER_MLA_ASM_PADDING: Literal["auto", "gluon", "asm"] = "auto" + VLLM_ROCM_AITER_MLA_DCP_VERIFY: Literal["asm", "segmented"] = "segmented" VLLM_ROCM_USE_AITER_MHA: bool = True VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: bool = False + VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA: bool = False VLLM_ROCM_USE_AITER_TRITON_ROPE: bool = False VLLM_ROCM_USE_AITER_FP8BMM: bool = True VLLM_ROCM_USE_AITER_FP4BMM: bool = True @@ -1309,6 +1311,15 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_ROCM_USE_AITER_MLA": lambda: ( os.getenv("VLLM_ROCM_USE_AITER_MLA", "True").lower() in ("true", "1") ), + # Kernel for causal multi-token (spec-decode) verify steps under decode + # context parallelism on gfx950: "asm" uses AITER's round-robin ASM + # decode, "segmented" the Triton segmented MLA path. + "VLLM_ROCM_AITER_MLA_DCP_VERIFY": env_with_choices( + "VLLM_ROCM_AITER_MLA_DCP_VERIFY", + "segmented", + ["asm", "segmented"], + case_sensitive=False, + ), # Small-head (<16) AITER MLA decode kernel selection. Small head counts # (e.g. Kimi-K3: 12 heads/rank at TP8, 6 at TP16) can decode either through # the Gluon small-head kernel or through the padded persistent-scheduling @@ -1333,6 +1344,14 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_ROCM_USE_AITER_FP4_ASM_GEMM": lambda: ( os.getenv("VLLM_ROCM_USE_AITER_FP4_ASM_GEMM", "False").lower() in ("true", "1") ), + # Whether sparse MLA prefill and decode run on aiter's Triton kernel, which + # reads the KV cache as stored (bf16 or fp8, paged or flat). Used by the + # ROCM_AITER_MLA_SPARSE backend (DeepSeek V3.2, GLM-5.x) and DeepSeek + # V4 / V4.1. gfx950 only. By default is disabled. + "VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA": lambda: ( + os.getenv("VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA", "False").lower() + in ("true", "1") + ), # Whether to use aiter rope. # By default is disabled. "VLLM_ROCM_USE_AITER_TRITON_ROPE": lambda: ( diff --git a/vllm/logger.py b/vllm/logger.py index 5e5f63aefc..24692aa392 100644 --- a/vllm/logger.py +++ b/vllm/logger.py @@ -7,7 +7,7 @@ import json import logging import os import sys -from collections.abc import Generator, Hashable +from collections.abc import Callable, Generator, Hashable from contextlib import contextmanager from copy import deepcopy from dataclasses import replace @@ -31,20 +31,24 @@ _FORMAT = ( ) _DATE_FORMAT = "%m-%d %H:%M:%S" -_base_log_record_factory = logging.getLogRecordFactory() _vllm_process_info: tuple[str, int] | None = None -def _vllm_log_record_factory(*args: Any, **kwargs: Any) -> logging.LogRecord: - """Add vLLM process metadata to each log record.""" - record = _base_log_record_factory(*args, **kwargs) - process_info = _vllm_process_info - pid = os.getpid() - if process_info is None or process_info[1] != pid: - record.vllm_process_name = record.processName - else: - record.vllm_process_name = process_info[0] - return record +class _VllmLogRecordFactory: + """Add vLLM process metadata while preserving the wrapped record factory.""" + + def __init__(self, factory: Callable[..., logging.LogRecord]) -> None: + self._factory = factory + + def __call__(self, *args: Any, **kwargs: Any) -> logging.LogRecord: + record = self._factory(*args, **kwargs) + process_info = _vllm_process_info + pid = os.getpid() + if process_info is None or process_info[1] != pid: + record.vllm_process_name = record.processName + else: + record.vllm_process_name = process_info[0] + return record def set_vllm_process_name(process_name: str, *, skip_if_set: bool = False) -> None: @@ -220,7 +224,9 @@ def _configure_vllm_root_logger(config: "LoggingConfig | None" = None) -> None: ) if configure_logging: - logging.setLogRecordFactory(_vllm_log_record_factory) + factory = logging.getLogRecordFactory() + if not isinstance(factory, _VllmLogRecordFactory): + logging.setLogRecordFactory(_VllmLogRecordFactory(factory)) logging_config = deepcopy(DEFAULT_LOGGING_CONFIG) vllm_handler = logging_config["handlers"]["vllm"] diff --git a/vllm/lora/layers/classifier.py b/vllm/lora/layers/classifier.py index 58160d1931..55dd4e2e7d 100644 --- a/vllm/lora/layers/classifier.py +++ b/vllm/lora/layers/classifier.py @@ -20,28 +20,38 @@ class ClassificationHeadWithLoRA(ReplicatedLinearWithLoRA): ) -> None: # Preserve ordinary LoRA A/B support for classification heads. super().create_lora_weights(max_loras, lora_config, model_config) + + self.max_lora_cls_labels = lora_config.max_lora_cls_labels or self.output_size + self.padded_num_labels = max(self.output_size, self.max_lora_cls_labels) self.full_weight_stacked = torch.zeros( max_loras, 1, - self.output_size, + self.padded_num_labels, self.input_size, dtype=lora_config.lora_dtype, device=self.device, ) self.full_bias_stacked = torch.zeros( max_loras, - self.output_size, + self.padded_num_labels, dtype=self.base_layer.params_dtype, device=self.device, ) self.full_module_enabled = torch.zeros( max_loras, dtype=torch.bool, device=self.device ) + # Zero marks an unused slot; positive values are unpadded label counts. + self.full_module_num_labels = [0] * max_loras + self._output_lora_indices: tuple[int, ...] = () + + def set_output_mapping(self, slot_indices: tuple[int, ...]) -> None: + self._output_lora_indices = slot_indices def reset_module_to_save(self, index: int) -> None: self.full_weight_stacked[index].zero_() self.full_bias_stacked[index].zero_() self.full_module_enabled[index] = False + self.full_module_num_labels[index] = 0 def set_module_to_save( self, @@ -50,17 +60,33 @@ class ClassificationHeadWithLoRA(ReplicatedLinearWithLoRA): bias: torch.Tensor | None, ) -> None: self.reset_module_to_save(index) - self.full_weight_stacked[index, 0].copy_(weight, non_blocking=True) + num_labels = weight.size(0) + self.full_weight_stacked[index, 0, :num_labels].copy_(weight, non_blocking=True) if bias is not None: - self.full_bias_stacked[index].copy_(bias, non_blocking=True) + self.full_bias_stacked[index, :num_labels].copy_(bias, non_blocking=True) + self.full_module_num_labels[index] = num_labels self.full_module_enabled[index] = True - def forward(self, input_: torch.Tensor) -> torch.Tensor: + def forward(self, input_: torch.Tensor) -> torch.Tensor | list[torch.Tensor]: # TODO base_result maybe don't need compute truly. base_result = super().forward(input_) - output = base_result[0] if isinstance(base_result, tuple) else base_result + base_result = base_result[0] if isinstance(base_result, tuple) else base_result - full_output = self.punica_wrapper.apply_lora_full_linear( + slot_indices = self._output_lora_indices + assert len(slot_indices) == base_result.size(0), ( + "Classification rows do not match LoRA request mapping" + ) + num_labels = [ + self.full_module_num_labels[index] if index >= 0 else 0 + for index in slot_indices + ] + + if not any(num_labels): + return base_result + + output = base_result.new_zeros(base_result.size(0), self.padded_num_labels) + output[:, : self.output_size].copy_(base_result) + self.punica_wrapper.apply_lora_full_linear( output, input_.to(self.full_weight_stacked.dtype), self.full_weight_stacked, @@ -68,8 +94,12 @@ class ClassificationHeadWithLoRA(ReplicatedLinearWithLoRA): self.full_module_enabled, ) - if full_output is not None: - output = full_output + # Rows without a full head keep the base classifier output size. + num_labels = [size or self.output_size for size in num_labels] + if len(set(num_labels)) == 1: + return output[:, : num_labels[0]] + + output = [output[row, :size] for row, size in enumerate(num_labels)] return output def _apply_lora_to_output( diff --git a/vllm/lora/model_manager.py b/vllm/lora/model_manager.py index b47c22d6aa..adc2353b3e 100644 --- a/vllm/lora/model_manager.py +++ b/vllm/lora/model_manager.py @@ -451,6 +451,16 @@ class LoRAModelManager: self.vocab_size, ) + if self._classification_head is not None: + _, classification_head = self._classification_head + if classification_head.punica_wrapper is punica_wrapper: + classification_head.set_output_mapping( + tuple( + self.lora_index_to_id.index(lora_id) if lora_id > 0 else -1 + for lora_id in mapping.prompt_mapping + ) + ) + def remove_all_adapters(self): """Remove all LoRAModels from the manager.""" self._registered_adapters.clear() @@ -1262,18 +1272,23 @@ class LoRAModelManager: f"classification head {module_name!r}." ) - expected_weight_shape = (wrapper.output_size, wrapper.input_size) received_weight_shape = tuple(full_module.weight.shape) - if received_weight_shape != expected_weight_shape: + if ( + full_module.weight.ndim != 2 + or full_module.weight.size(0) < 1 + or full_module.weight.size(0) > wrapper.max_lora_cls_labels + or full_module.weight.size(1) != wrapper.input_size + ): raise ValueError( f"Full module {saved_module_name!r} for {module_name!r} has " "an incompatible weight shape: expected " - f"{expected_weight_shape}, received {received_weight_shape}." + f"(1..{wrapper.max_lora_cls_labels}, {wrapper.input_size}), " + f"received {received_weight_shape}." ) if full_module.bias is None: return - expected_bias_shape = (wrapper.output_size,) + expected_bias_shape = (full_module.weight.size(0),) received_bias_shape = tuple(full_module.bias.shape) if received_bias_shape != expected_bias_shape: raise ValueError( diff --git a/vllm/model_executor/kernels/mhc/__init__.py b/vllm/model_executor/kernels/mhc/__init__.py index f9df7f6f2c..6fc4919989 100644 --- a/vllm/model_executor/kernels/mhc/__init__.py +++ b/vllm/model_executor/kernels/mhc/__init__.py @@ -13,6 +13,7 @@ __all__ = [ "hc_head_fused_kernel_cuda", "mhc_pre_aiter", "mhc_pre_delayed_aiter", + "mhc_fused_post_pre_delayed_rms_norm_aiter", "mhc_post_aiter", "mhc_fused_post_pre_aiter", "hc_head_fused_aiter", diff --git a/vllm/model_executor/kernels/mhc/aiter.py b/vllm/model_executor/kernels/mhc/aiter.py index 06b4f14984..f102143ab0 100644 --- a/vllm/model_executor/kernels/mhc/aiter.py +++ b/vllm/model_executor/kernels/mhc/aiter.py @@ -207,6 +207,79 @@ def _mhc_pre_delayed_aiter_fake( ) +def mhc_fused_post_pre_delayed_rms_norm_aiter( + residual: torch.Tensor, + fn: torch.Tensor, + hc_scale: torch.Tensor, + hc_base: torch.Tensor, + rms_eps: float, + hc_pre_eps: float, + hc_sinkhorn_eps: float, + hc_post_mult_value: float, + sinkhorn_repeat: int, + pre_mix: torch.Tensor | None, + sublayer_out: torch.Tensor | None, + post_layer_mix: torch.Tensor | None, + comb_res_mix: torch.Tensor | None, + norm_weight: torch.Tensor, + norm_eps: float, + residual_out: torch.Tensor | None, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """:func:`mhc_pre_delayed_aiter` with the RMSNorm of the collapse folded in: + ``layer_input`` comes back normalised with ``norm_weight`` / ``norm_eps``.""" + from vllm._aiter_ops import rocm_aiter_ops + + return rocm_aiter_ops.mhc_fused_post_pre_delayed_rms_norm( + residual, + fn, + hc_scale, + hc_base, + rms_eps, + hc_pre_eps, + hc_sinkhorn_eps, + hc_post_mult_value, + sinkhorn_repeat, + pre_mix, + sublayer_out, + post_layer_mix, + comb_res_mix, + norm_weight, + norm_eps, + residual_out, + ) + + +def _mhc_fused_post_pre_delayed_rms_norm_aiter_fake( + residual: torch.Tensor, + fn: torch.Tensor, + hc_scale: torch.Tensor, + hc_base: torch.Tensor, + rms_eps: float, + hc_pre_eps: float, + hc_sinkhorn_eps: float, + hc_post_mult_value: float, + sinkhorn_repeat: int, + pre_mix: torch.Tensor | None, + sublayer_out: torch.Tensor | None, + post_layer_mix: torch.Tensor | None, + comb_res_mix: torch.Tensor | None, + norm_weight: torch.Tensor, + norm_eps: float, + residual_out: torch.Tensor | None, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + return _mhc_pre_delayed_aiter_fake( + residual, + fn, + hc_scale, + hc_base, + rms_eps, + hc_pre_eps, + hc_sinkhorn_eps, + hc_post_mult_value, + sinkhorn_repeat, + ) + + def mhc_post_aiter( x: torch.Tensor, residual: torch.Tensor, @@ -335,6 +408,12 @@ direct_register_custom_op( mutates_args=["residual_out"], fake_impl=_mhc_pre_delayed_aiter_fake, ) +direct_register_custom_op( + op_name="mhc_fused_post_pre_delayed_rms_norm_aiter", + op_func=mhc_fused_post_pre_delayed_rms_norm_aiter, + mutates_args=["residual_out"], + fake_impl=_mhc_fused_post_pre_delayed_rms_norm_aiter_fake, +) direct_register_custom_op( op_name="mhc_post_aiter", op_func=mhc_post_aiter, diff --git a/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py b/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py index ad591d9ea7..791dc56766 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py +++ b/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py @@ -567,13 +567,26 @@ def _process_weights_marlin( """Standard Marlin weight post-processing shared by MARLIN and BATCHED_MARLIN backends. + Inputs arrive in canonical N-first layout ``[E, N, K_packed]``. + Steps ----- - 1. Optional FP8 preprocessing of packed weights / scales. - 2. Repack weights via ``gptq_marlin_moe_repack``. - 3. Permute scales (and optionally extract INT8 global scales). - 4. Permute bias tensors. + 1. Transpose canonical N-first to K-first for Marlin. + 2. Optional FP8 preprocessing of packed weights / scales. + 3. Repack weights via ``gptq_marlin_moe_repack``. + 4. Permute scales (and optionally extract INT8 global scales). + 5. Permute bias tensors. """ + # Canonical N-first → K-first for Marlin. + w13_qweight = w13_qweight.transpose(1, 2).contiguous() + w2_qweight = w2_qweight.transpose(1, 2).contiguous() + w13_scales = w13_scales.transpose(1, 2).contiguous() + w2_scales = w2_scales.transpose(1, 2).contiguous() + if w13_qzeros is not None: + w13_qzeros = w13_qzeros.transpose(1, 2).contiguous() + if w2_qzeros is not None: + w2_qzeros = w2_qzeros.transpose(1, 2).contiguous() + is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1 marlin_w13_qweight: torch.Tensor @@ -934,7 +947,12 @@ def _process_weights_cpu( torch.Tensor | None, # w13_bias torch.Tensor | None, # w2_bias ]: - """CPU INT4 W4A16 weight post-processing.""" + """CPU INT4 W4A16 weight post-processing. + + Non-AWQ inputs arrive in canonical N-first layout and are transposed + to K-first internally. AWQ uses a different packing axis and arrives + in its native format. + """ from vllm.model_executor.layers.fused_moe.experts.cpu_moe import ( prepare_int4_moe_layer_for_cpu, ) @@ -945,15 +963,9 @@ def _process_weights_cpu( AutoGPTQConfig, ) - # Detect packing format. - # AWQ: qweight is [E, K, 2*N//8] (packed along output/N dim). - # GPTQ: qweight is [E, K//8, 2*N] (packed along input/K dim). - # compressed-tensors: qweight is [E, K//8, 2*N] (packed along input/K dim). if isinstance(quant_config, AutoAWQConfig): - # AWQ: K is stored unpacked in dim 1. cpu_quant_algo = ops.CPUQuantAlgo.AWQ elif isinstance(quant_config, (AutoGPTQConfig, QuantizationArgs)): - # GPTQ / compressed-tensors: K//8 is stored packed in dim 1. if isinstance(quant_config, AutoGPTQConfig) and quant_config.desc_act: raise NotImplementedError( "CPU WNA16 MoE backend does not support GPTQ with " @@ -961,6 +973,15 @@ def _process_weights_cpu( "reordering support." ) cpu_quant_algo = ops.CPUQuantAlgo.GPTQ + # Canonical N-first → K-first for CPU kernel. + w13 = w13.transpose(1, 2).contiguous() + w2 = w2.transpose(1, 2).contiguous() + w13_scale = w13_scale.transpose(1, 2).contiguous() + w2_scale = w2_scale.transpose(1, 2).contiguous() + if w13_qzeros is not None: + w13_qzeros = w13_qzeros.transpose(1, 2).contiguous() + if w2_qzeros is not None: + w2_qzeros = w2_qzeros.transpose(1, 2).contiguous() else: raise TypeError( "CPU WNA16 MoE backend requires AutoAWQConfig, AutoGPTQConfig " @@ -1030,41 +1051,28 @@ def _process_weights_xpu( torch.Tensor | None, # w13_bias torch.Tensor | None, # w2_bias ]: - """Repack GPTQ-format INT4 MoE weights into the layout - `vllm_xpu_kernels.fused_moe_interface.xpu_fused_moe(is_int4=True)` expects: + """Repack INT4 MoE weights into the layout + `vllm_xpu_kernels.fused_moe_interface.xpu_fused_moe(is_int4=True)` expects. - w13: [E, 2*N, K] int4 (uint8 storage [E, 2*N, K // 2]) - w13_scales: [E, 2*N, K // group_size] params_dtype - w2: [E, K, N] int4 (uint8 storage [E, K, N // 2]) - w2_scales: [E, K, N // group_size] params_dtype - - Input GPTQ layout from MoERunner.weight_loader: - w13: [E, K // 8, 2*N] int32 (8 nibbles per int32 along the input dim) - w13_scales: [E, K // group_size, 2*N] params_dtype - w2: [E, N // 8, K] int32 - w2_scales: [E, N // group_size, K] params_dtype - - Transpose dim 1 ↔ dim 2 then view int32 → uint8 to recover sequential - int4-packed bytes along the input dim. Each packed int32 holds 8 nibbles - `(n7<<28)|(n6<<24)|...|(n1<<4)|n0` in ascending K order; on a - little-endian host the int32→uint8 view exposes them as bytes - `[n1<<4|n0, n3<<4|n2, n5<<4|n4, n7<<4|n6]`, i.e. two nibbles per byte - with the lower nibble = lower input-K index. xpu_fused_moe(is_int4=True) - expects this convention; on a big-endian host the byte order reverses - and the kernel would silently miscompute, so we hard-fail. + Inputs arrive in canonical N-first layout ``[E, N, K_packed]`` int32. + The int32 → uint8 view recovers sequential int4-packed bytes along + the input dim. Each packed int32 holds 8 nibbles in ascending K order; + on a little-endian host the view exposes them as two nibbles per byte + with the lower nibble = lower input-K index. xpu_fused_moe(is_int4=True) + expects this convention; big-endian hosts are unsupported. """ del layer, quant_config # unused — kept for parity with the marlin helper if sys.byteorder != "little": raise NotImplementedError( - "_process_weights_xpu requires a little-endian host: the GPTQ " + "_process_weights_xpu requires a little-endian host: the " "int32 → uint8 nibble repack relies on LE byte ordering." ) - w13_xpu = w13_qweight.transpose(1, 2).contiguous().view(torch.uint8) - w2_xpu = w2_qweight.transpose(1, 2).contiguous().view(torch.uint8) - w13_scales_xpu = w13_scales.transpose(1, 2).contiguous() - w2_scales_xpu = w2_scales.transpose(1, 2).contiguous() + w13_xpu = w13_qweight.contiguous().view(torch.uint8) + w2_xpu = w2_qweight.contiguous().view(torch.uint8) + w13_scales_xpu = w13_scales.contiguous() + w2_scales_xpu = w2_scales.contiguous() return ( w13_xpu, @@ -1315,16 +1323,23 @@ def _process_weights_emulation_gptq( ) -> tuple: """Dequantize int4 weights to BF16 for the emulation backend. - Inputs are in GPTQ packed format: - w13: [E, K//8, 2*N] int32 (gate+up proj stacked on dim 2) - w2: [E, N//8, K] int32 - w13_scale: [E, K//gs, 2*N] float16 - w2_scale: [E, N//gs, K] float16 + Inputs arrive in canonical N-first layout and are transposed to + K-first internally for the GPTQ dequantization routines. Outputs (what TritonExperts expects): w13_out: [E, 2*N, K] bfloat16 w2_out: [E, K, N] bfloat16 """ + # Canonical N-first → K-first for dequantization. + w13 = w13.transpose(1, 2).contiguous() + w2 = w2.transpose(1, 2).contiguous() + w13_scale = w13_scale.transpose(1, 2).contiguous() + w2_scale = w2_scale.transpose(1, 2).contiguous() + if w13_qzeros is not None: + w13_qzeros = w13_qzeros.transpose(1, 2).contiguous() + if w2_qzeros is not None: + w2_qzeros = w2_qzeros.transpose(1, 2).contiguous() + # w13: packed along K (dim 1), output cols are 2*N (dim 2) # transpose_output=True yields [E, 2*N, K] w13_bf16 = _unpack_and_dequant_int4_gptq( @@ -1435,6 +1450,10 @@ def convert_to_wna16_moe_kernel_format( ): """Dispatch weight post-processing to the appropriate per-backend handler. + All non-AWQ frontends must supply weights and scales in N-first + canonical layout ``[E, N_out, K_packed]``. AWQ uses a different + packing axis and is handled by dedicated per-backend helpers. + To add a new backend, implement a ``_process_weights_`` helper and add a branch here. Backends that rewrite the layer's parameters in place (e.g. Humming) return ``None``; the caller then skips the param scatter. @@ -1626,67 +1645,33 @@ def convert_to_wna16_moe_kernel_format( w2_qzeros, ) elif backend == WNA16MoEBackend.TRITON: - # Two possible input layouts depending on the quantization source: - # - # MoeWNA16 (uint8): (E, N_out, K // bit8_pack) — N-first - # → just view as uint8 (no-op) - # - # AutoGPTQ/compressed-tensors (int32, K-first): - # (E, K // pack32, N_out) - # → transpose to N-first, then view as uint8 to get - # (E, N_out, K // bit8_pack) [int32 = 4 bytes → 4 uint8s] - # Scales: (E, K // gs, N_out) → transpose → (E, N_out, K // gs) - from vllm.model_executor.layers.quantization.auto_gptq import ( - AutoGPTQConfig, - ) + # All inputs arrive in canonical N-first format; view as uint8. + w13_uint8 = w13.contiguous().view(torch.uint8) + w2_uint8 = w2.contiguous().view(torch.uint8) + + # Repack N-first int32 zero-points to uint8: + # [E, N//pf, K//gs] int32 → [E, N*4//pf, K//gs] uint8 + if w13_qzeros is not None and w13_qzeros.dtype == torch.int32: + E13, Np13, Kg13 = w13_qzeros.shape + w13_qzeros = ( + w13_qzeros.contiguous() + .view(torch.uint8) + .reshape(E13, Np13, Kg13, 4) + .permute(0, 1, 3, 2) + .reshape(E13, Np13 * 4, Kg13) + .contiguous() + ) + if w2_qzeros is not None and w2_qzeros.dtype == torch.int32: + E2, Np2, Kg2 = w2_qzeros.shape + w2_qzeros = ( + w2_qzeros.contiguous() + .view(torch.uint8) + .reshape(E2, Np2, Kg2, 4) + .permute(0, 1, 3, 2) + .reshape(E2, Np2 * 4, Kg2) + .contiguous() + ) - if isinstance(quant_config, (AutoGPTQConfig, QuantizationArgs)): - # These integrations build in K-first format even when the Triton - # backend is selected. Transpose to N-first first. - w13_uint8 = w13.transpose(1, 2).contiguous().view(torch.uint8) - w2_uint8 = w2.transpose(1, 2).contiguous().view(torch.uint8) - w13_scale = w13_scale.transpose(1, 2).contiguous() - w2_scale = w2_scale.transpose(1, 2).contiguous() - # Zero points from compressed-tensors checkpoints are K-first int32 - # with 8 int4 ZPs packed per element: shape (E, K//gs, N//8). - # fused_moe_kernel_gptq_awq expects N-first uint8 with 2 int4 ZPs - # per byte: shape (E, N//2, K//gs), indexed as - # (offs_bn // 2) * stride_bzn + offs_k_group * stride_bzk. - # Conversion steps: - # (E, K//gs, N//8) int32 - # → transpose(1,2) → (E, N//8, K//gs) int32 - # → view(uint8) → (E, N//8, K//gs*4) [each int32 → 4 bytes] - # → reshape(…,4) → (E, N//8, K//gs, 4) [isolate byte index] - # → permute(0,1,3,2) → (E, N//8, 4, K//gs) [byte index before K] - # → reshape → (E, N//2, K//gs) [kernel expected layout] - # After this, element [e, offs_bn//2, k_group] is the uint8 byte - # holding the two int4 ZPs for output channels offs_bn and offs_bn+1. - if w13_qzeros is not None: - E13, Kg13, Np13 = w13_qzeros.shape - w13_qzeros = ( - w13_qzeros.transpose(1, 2) - .contiguous() - .view(torch.uint8) - .reshape(E13, Np13, Kg13, 4) - .permute(0, 1, 3, 2) - .reshape(E13, Np13 * 4, Kg13) - .contiguous() - ) - if w2_qzeros is not None: - E2, Kg2, Np2 = w2_qzeros.shape - w2_qzeros = ( - w2_qzeros.transpose(1, 2) - .contiguous() - .view(torch.uint8) - .reshape(E2, Np2, Kg2, 4) - .permute(0, 1, 3, 2) - .reshape(E2, Np2 * 4, Kg2) - .contiguous() - ) - else: - # MoeWNA16 uses N-first uint8 weights and scales. - w13_uint8 = w13.view(torch.uint8) - w2_uint8 = w2.view(torch.uint8) return ( w13_uint8, w2_uint8, diff --git a/vllm/model_executor/layers/fused_moe/routed_experts.py b/vllm/model_executor/layers/fused_moe/routed_experts.py index 1d411ecc72..3de0aa8dab 100644 --- a/vllm/model_executor/layers/fused_moe/routed_experts.py +++ b/vllm/model_executor/layers/fused_moe/routed_experts.py @@ -676,7 +676,7 @@ class RoutedExperts(PluggableLayer): # Hereafter, `expert_id` is local physical id # is_transposed: if the dim to shard the weight - # should be flipped. Required by GPTQ, compressed-tensors + # should be flipped. Required by GPTQ/AWQ (K-first format). # should be whatever dimension intermediate_size_per_partition is is_transposed = getattr(param, "is_transposed", False) diff --git a/vllm/model_executor/layers/fusion/mm_input_norm.py b/vllm/model_executor/layers/fusion/mm_input_norm.py index a21828033d..f03cfa4c36 100644 --- a/vllm/model_executor/layers/fusion/mm_input_norm.py +++ b/vllm/model_executor/layers/fusion/mm_input_norm.py @@ -18,14 +18,14 @@ where:: """ import math -from typing import Any, NamedTuple +from typing import NamedTuple import torch from torch import nn from vllm.config import ModelConfig from vllm.model_executor.custom_op import CustomOp -from vllm.transformers_utils.processor import cached_get_processor, get_processor_config +from vllm.transformers_utils.processor import get_processor_config from vllm.triton_utils import tl, triton @@ -144,33 +144,19 @@ class NormParams(NamedTuple): def _load_norm_params(model_config: ModelConfig) -> NormParams: """Resolve the per-channel affine parameters ``(image_mean, image_std, - rescale_factor)`` from the processor config, falling back to the image - processor object.""" - model = model_config.model - revision = model_config.revision + rescale_factor)`` from the processor config.""" + config = get_processor_config(model_config.model, revision=model_config.revision) + config = config.get("image_processor", config) - config = get_processor_config(model, revision=revision) - # NOTE: do not use cached_image_processor_from_config here — it merges - # mm_processor_kwargs, which mm_device_do_normalize poisons with - # do_normalize=False. - image_processor = cached_get_processor( - model, revision=revision, trust_remote_code=model_config.trust_remote_code - ).image_processor - - def resolve(key: str) -> Any: - """Processor config value, falling back to the image_processor.""" - if (value := config.get(key)) is not None: - return value - return getattr(image_processor, key, None) - - do_rescale = bool(resolve("do_rescale")) - do_normalize = bool(resolve("do_normalize")) + has_norm_params = "image_mean" in config and "image_std" in config + do_normalize = bool(config.get("do_normalize", has_norm_params)) + do_rescale = bool(config.get("do_rescale", has_norm_params)) # Parameters whose flag is off are unused; default them to no-ops # without resolving them. - rescale_factor = resolve("rescale_factor") if do_rescale else 1.0 - image_mean = resolve("image_mean") if do_normalize else [0.0] * 3 - image_std = resolve("image_std") if do_normalize else [1.0] * 3 + rescale_factor = config.get("rescale_factor", 1 / 255) if do_rescale else 1.0 + image_mean = config.get("image_mean") if do_normalize else [0.0] * 3 + image_std = config.get("image_std") if do_normalize else [1.0] * 3 assert rescale_factor is not None, "rescale_factor is still None after resolution." assert image_mean is not None, "image_mean is still None after resolution." @@ -195,6 +181,10 @@ class IdentityInputNorm(nn.Module): ``visual_dtype`` here. """ + @property + def input_dtype(self) -> torch.dtype | None: + return None + def forward( self, pixel_values: torch.Tensor, visual_dtype: torch.dtype ) -> torch.Tensor: @@ -251,6 +241,10 @@ class FusedMMInputNorm(CustomOp): self.register_buffer("weight", (rescale_factor / std).to(device)) self.register_buffer("bias", (-mean / std).to(device)) + @property + def input_dtype(self) -> torch.dtype | None: + return torch.uint8 + # ------------------------------------------------------------------ # Internal helpers shared by the platform-specific forward_* methods # ------------------------------------------------------------------ diff --git a/vllm/model_executor/layers/mhc.py b/vllm/model_executor/layers/mhc.py index 73993fbc98..19e8b427b4 100644 --- a/vllm/model_executor/layers/mhc.py +++ b/vllm/model_executor/layers/mhc.py @@ -59,9 +59,21 @@ def _aiter_mhc_op_accepts_norm(op_name: str) -> bool: return False +def _has_aiter_mhc_fused_post_pre_delayed_rms_norm() -> bool: + if not HAS_AITER_MHC: + return False + from vllm.platforms.rocm import on_gfx950 + + # The fused delayed seam kernel is only validated on gfx950. + return on_gfx950() + + HAS_AITER_MHC_FUSED = _has_aiter_mhc_fused() HAS_AITER_MHC_PRE_NORM = _aiter_mhc_op_accepts_norm("mhc_pre") HAS_AITER_MHC_FUSED_NORM = _aiter_mhc_op_accepts_norm("mhc_fused_post_pre") +HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM = ( + _has_aiter_mhc_fused_post_pre_delayed_rms_norm() +) def _aiter_mhc_supported( @@ -388,10 +400,42 @@ class MHCPreDelayedOp(CustomOp): post_layer_mix: torch.Tensor | None = None, comb_res_mix: torch.Tensor | None = None, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: - # The AITER delayed path drives mhc_pre_gemm_sqrsum against `residual` - # and folds no RMSNorm, so it cannot serve the model-entry broadcast - # (which projects a narrower `x`) or a fused norm. Both are handled by - # TileLang, or by the reference when TileLang is unavailable. + # aiter's fused delayed seam: post-mix, gate projection, and the + # collapse with the carried pre-mix and its RMSNorm. It always applies + # the norm and projects `residual` itself. + if ( + HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM + and x is None + and norm_weight is not None + and _aiter_mhc_supported(residual, norm_weight, supports_norm=True) + ): + residual_out = ( + torch.empty_like(residual) if sublayer_out is not None else None + ) + rest = torch.ops.vllm.mhc_fused_post_pre_delayed_rms_norm_aiter( + residual, + fn, + hc_scale, + hc_base, + rms_eps, + hc_pre_eps, + hc_sinkhorn_eps, + hc_post_mult_value, + sinkhorn_repeat, + pre_mix, + sublayer_out, + post_layer_mix, + comb_res_mix, + norm_weight, + norm_eps, + residual_out, + ) + return (residual if residual_out is None else residual_out), *rest + # The unfused AITER delayed path drives mhc_pre_gemm_sqrsum against + # `residual` and folds no RMSNorm, so it cannot serve the model-entry + # broadcast (which projects a narrower `x`) or a fused norm the branch + # above did not take. Both are handled by TileLang, or by the reference + # when TileLang is unavailable. if ( x is None and norm_weight is None diff --git a/vllm/model_executor/layers/pooler/seqwise/heads.py b/vllm/model_executor/layers/pooler/seqwise/heads.py index c72532cc10..40624cc17c 100644 --- a/vllm/model_executor/layers/pooler/seqwise/heads.py +++ b/vllm/model_executor/layers/pooler/seqwise/heads.py @@ -171,16 +171,24 @@ class ClassifierPoolerHead(SequencePoolerHead): pooled_data = pooled_data.to(self.head_dtype) if self.classifier is not None: - logits = self.classifier(pooled_data) + logits: SequencePoolerHeadOutput = self.classifier(pooled_data) else: logits = pooled_data # logits shape: [batchsize, num_labels] # Affine score calibration: activation((logit - mean) / sigma) if self.logit_mean is not None: - logits = logits - self.logit_mean + logits = ( + [value - self.logit_mean for value in logits] + if isinstance(logits, list) + else logits - self.logit_mean + ) if self.logit_sigma is not None: - logits = logits / self.logit_sigma + logits = ( + [value / self.logit_sigma for value in logits] + if isinstance(logits, list) + else logits / self.logit_sigma + ) if self.activation is not None: flags = [p.use_activation for p in pooling_params] diff --git a/vllm/model_executor/layers/pooler/seqwise/poolers.py b/vllm/model_executor/layers/pooler/seqwise/poolers.py index 2115e5184e..c2ac967a15 100644 --- a/vllm/model_executor/layers/pooler/seqwise/poolers.py +++ b/vllm/model_executor/layers/pooler/seqwise/poolers.py @@ -130,7 +130,7 @@ def pooler_for_classify( logit_mean=model_config.pooler_config.logit_mean, logit_sigma=model_config.pooler_config.logit_sigma, activation=resolve_classifier_act_fn( - model_config, static_num_labels=True, act_fn=act_fn + model_config, static_num_labels=False, act_fn=act_fn ), ) diff --git a/vllm/model_executor/layers/quantization/auto_gptq.py b/vllm/model_executor/layers/quantization/auto_gptq.py index ff57f3b8c5..ed430f7445 100644 --- a/vllm/model_executor/layers/quantization/auto_gptq.py +++ b/vllm/model_executor/layers/quantization/auto_gptq.py @@ -626,19 +626,31 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase): layer.register_parameter("w2_bias", None) w2_bias = None + # Normalize GPTQ K-first layout to canonical N-first format. + w13 = layer.w13_qweight.data.transpose(1, 2).contiguous() + w2 = layer.w2_qweight.data.transpose(1, 2).contiguous() + w13_scale = layer.w13_scales.data.transpose(1, 2).contiguous() + w2_scale = layer.w2_scales.data.transpose(1, 2).contiguous() + w13_qzeros = getattr(layer, "w13_qzeros", None) + w2_qzeros = getattr(layer, "w2_qzeros", None) + if w13_qzeros is not None: + w13_qzeros = w13_qzeros.data.transpose(1, 2).contiguous() + if w2_qzeros is not None: + w2_qzeros = w2_qzeros.data.transpose(1, 2).contiguous() + converted = convert_to_wna16_moe_kernel_format( backend=self.wna16_moe_backend, layer=layer, quant_config=self.quant_config, input_dtype=self.input_dtype, - w13=layer.w13_qweight, - w2=layer.w2_qweight, - w13_scale=layer.w13_scales, - w2_scale=layer.w2_scales, + w13=w13, + w2=w2, + w13_scale=w13_scale, + w2_scale=w2_scale, w13_bias=w13_bias, w2_bias=w2_bias, - w13_qzeros=getattr(layer, "w13_qzeros", None), - w2_qzeros=getattr(layer, "w2_qzeros", None), + w13_qzeros=w13_qzeros, + w2_qzeros=w2_qzeros, ) if converted is None: diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16.py index 6eb2455d0e..c03d2bfcf4 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16.py @@ -112,10 +112,10 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod): WNA16MoEBackend.MARLIN, WNA16MoEBackend.BATCHED_MARLIN, ] - self.is_transposed = self.wna16_backend not in ( - WNA16MoEBackend.FLASHINFER_TRTLLM, - WNA16MoEBackend.HUMMING, - ) + # CT checkpoints always load in N-first format [E, N, K_packed]. + # Backend-specific transposition is handled in + # convert_to_wna16_moe_kernel_format. + self.is_transposed = False if self.is_marlin: assert check_moe_marlin_supports_config( @@ -141,80 +141,31 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod): num_groups_w2: int | None = None, num_groups_w13: int | None = None, ) -> tuple[int, int, int]: - """Get the shape of the weight based on the weight name, number of experts - hidden size, intermediate size per partition, number of groups for w2, - and number of groups for w13. Pass in num_groups_w2 and num_groups_w13 - for weight scales/zero_points. + """Return N-first buffer shape for the given weight/scale/zp tensor. + + All CT MoE weights are allocated in the checkpoint's native N-first + layout ``[E, N, K_packed]``. Backend-specific transposition (e.g. to + Marlin's K-first layout) is deferred to + ``convert_to_wna16_moe_kernel_format``. """ - if weight_name in ("w13_scale", "w13_zp"): - assert num_groups_w13 is not None, ( - "num_groups_w13 must be provided for weight scales/zero_points" - ) - if weight_name in ("w2_scale", "w2_zp"): - assert num_groups_w2 is not None, ( - "num_groups_w2 must be provided for weight scales/zero_points" - ) w13_num_shards = 2 if self.moe.is_act_and_mul else 1 - shape_map: dict[str, dict[str, tuple[int, int | None, int | None]]] = { - "w13_weight": { - "Flashinfer": ( - num_experts, - w13_num_shards * intermediate_size_per_partition, - self._packed_dim(hidden_size), - ), - "Marlin": ( - num_experts, - self._packed_dim(hidden_size), - w13_num_shards * intermediate_size_per_partition, - ), - }, - "w13_scale": { - "Flashinfer": ( - num_experts, - w13_num_shards * intermediate_size_per_partition, - num_groups_w13, - ), - "Marlin": ( - num_experts, - num_groups_w13, - w13_num_shards * intermediate_size_per_partition, - ), - }, - "w13_zp": { - "Marlin": ( - num_experts, - num_groups_w13, - self._packed_dim(w13_num_shards * intermediate_size_per_partition), - ), - }, - "w2_weight": { - "Flashinfer": ( - num_experts, - hidden_size, - self._packed_dim(intermediate_size_per_partition), - ), - "Marlin": ( - num_experts, - self._packed_dim(intermediate_size_per_partition), - hidden_size, - ), - }, - "w2_scale": { - "Flashinfer": (num_experts, hidden_size, num_groups_w2), - "Marlin": (num_experts, num_groups_w2, hidden_size), - }, - "w2_zp": { - "Marlin": ( - num_experts, - num_groups_w2, - self._packed_dim(hidden_size), - ), - }, + w13_n = w13_num_shards * intermediate_size_per_partition + pf = self.packed_factor + shape_map = { + "w13_weight": (num_experts, w13_n, hidden_size // pf), + "w13_scale": (num_experts, w13_n, num_groups_w13), + "w13_zp": (num_experts, w13_n // pf, num_groups_w13), + "w2_weight": ( + num_experts, + hidden_size, + intermediate_size_per_partition // pf, + ), + "w2_scale": (num_experts, hidden_size, num_groups_w2), + "w2_zp": (num_experts, hidden_size // pf, num_groups_w2), } - backend_key = "Marlin" if self.is_transposed else "Flashinfer" - shape = shape_map[weight_name][backend_key] - assert shape[1] is not None and shape[2] is not None - return shape[0], shape[1], shape[2] + shape = shape_map[weight_name] + assert isinstance(shape, tuple) and all(isinstance(d, int) for d in shape) + return shape # type: ignore[return-value] def create_weights( self, @@ -225,11 +176,8 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod): params_dtype: torch.dtype, **extra_weight_attrs, ): - # Will transpose the loaded weight along the - # intermediate and hidden dim sizes. Will - # shard for TP along the transposed dims extra_weight_attrs.update( - {"is_transposed": self.is_transposed, "quant_method": self.strategy} + {"is_transposed": False, "quant_method": self.strategy} ) w13_weight = torch.nn.Parameter( @@ -440,12 +388,10 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod): replace_parameter(layer, "w13_weight_scale", w13_scales) replace_parameter(layer, "w2_weight_scale", w2_scales) - # CPU fused_experts_cpu and the RDNA3 HIP kernel require zero points - # even for symmetric quant (the oracle synthesizes them). + # CPU fused_experts_cpu requires zero points even for symmetric quant. # EMULATION bakes ZP into the dequantized bf16 weights — ZP is None. if ( - not self.symmetric - or self.wna16_backend in (WNA16MoEBackend.CPU, WNA16MoEBackend.RDNA3) + not self.symmetric or self.wna16_backend == WNA16MoEBackend.CPU ) and self.wna16_backend != WNA16MoEBackend.EMULATION: assert w13_qzeros is not None and w2_qzeros is not None replace_parameter(layer, "w13_weight_zero_point", w13_qzeros) diff --git a/vllm/model_executor/layers/quantization/moe_wna16.py b/vllm/model_executor/layers/quantization/moe_wna16.py index 6a27332b06..a8b97abc86 100644 --- a/vllm/model_executor/layers/quantization/moe_wna16.py +++ b/vllm/model_executor/layers/quantization/moe_wna16.py @@ -635,10 +635,14 @@ class MoeWNA16Method(FusedMoEMethodBase): tensor = loaded_weight.view( layer.moe_config.tp_size, -1, loaded_weight.size(1) )[tp_rank] + # qzeros are packed along the intermediate dim by + # bit8_pack_factor, so the w1/w3 boundary is in packed units + # (equals shard_size // 2 only for 4-bit). + shard_zp_size = shard_size // layer.quant_config.bit8_pack_factor if shard_id == "w1": - param.data[expert_id, : shard_size // 2] = tensor + param.data[expert_id, :shard_zp_size] = tensor else: - param.data[expert_id, shard_size // 2 :] = tensor + param.data[expert_id, shard_zp_size:] = tensor return True if return_success else None elif "w2_qzeros" in weight_name: param.data[expert_id] = loaded_weight.view( diff --git a/vllm/model_executor/layers/rotary_embedding/packed_qk_rope.py b/vllm/model_executor/layers/rotary_embedding/packed_qk_rope.py index e7d916e48f..71f73473c6 100644 --- a/vllm/model_executor/layers/rotary_embedding/packed_qk_rope.py +++ b/vllm/model_executor/layers/rotary_embedding/packed_qk_rope.py @@ -46,8 +46,10 @@ def packed_qk_rope_(xqkv: torch.Tensor, freqs_cis: torch.Tensor) -> None: freqs_cis: contiguous (seqlen, headdim // 2) complex64 rotary freqs, read through its interleaved re/im fp32 view. - Bitwise identical to ``ApplyRotaryEmb(enable_fp32_compute=True)`` applied - to Q and K separately, but in one kernel with ~5x less memory traffic. + Matches ``ApplyRotaryEmb(enable_fp32_compute=True)`` applied to Q and K + separately to within one ULP, but in one kernel with ~5x less memory + traffic. Both compute in fp32; the gap is which product the compiler + keeps exact inside the fused multiply-add, which differs per backend. Requires triton: callers must check HAS_TRITON and use the unfused path otherwise. Contract violations raise. diff --git a/vllm/model_executor/models/aria.py b/vllm/model_executor/models/aria.py index da91c9d9bf..f063f12fdb 100644 --- a/vllm/model_executor/models/aria.py +++ b/vllm/model_executor/models/aria.py @@ -526,6 +526,9 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal): logits = self.logits_processor(self.lm_head, hidden_states) return logits - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): + def load_weights( + self, + weights: Iterable[tuple[str, torch.Tensor]], + ) -> set[str]: loader = AutoWeightsLoader(self) - loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) diff --git a/vllm/model_executor/models/colqwen3.py b/vllm/model_executor/models/colqwen3.py index 0e92363001..e5cdcc773c 100644 --- a/vllm/model_executor/models/colqwen3.py +++ b/vllm/model_executor/models/colqwen3.py @@ -134,6 +134,8 @@ class ColQwen3Model(Qwen3VLForConditionalGeneration, SupportsLateInteraction): # Mark this as a pooling model so vLLM routes to pooler path is_pooling_model = True + # Mark as false to avoid collision with the checkpoint's custom code path + supports_mm_device_do_normalize = False # Override hf_to_vllm_mapper to handle ColQwen3 weight naming. # NOTE: WeightsMapper applies ALL matching prefix rules sequentially diff --git a/vllm/model_executor/models/diffusion_gemma.py b/vllm/model_executor/models/diffusion_gemma.py index 1ae1cc7116..8f26edf9e4 100644 --- a/vllm/model_executor/models/diffusion_gemma.py +++ b/vllm/model_executor/models/diffusion_gemma.py @@ -1493,7 +1493,7 @@ class DiffusionSampler: # [tile * W, vocab] copies, so a tile is also bounded by free memory. widths_np = states.canvas_width_np[decode_slots_np] order = np.argsort(widths_np, kind="stable") - free = current_platform.mem_get_info()[0] if num_decode > 0 else 0 + free = torch.accelerator.get_memory_info()[0] if num_decode > 0 else 0 run_start = 0 while run_start < num_decode: W = int(widths_np[order[run_start]]) diff --git a/vllm/model_executor/models/funaudiochat.py b/vllm/model_executor/models/funaudiochat.py index 8b40ddf0cd..8e64fe03e2 100644 --- a/vllm/model_executor/models/funaudiochat.py +++ b/vllm/model_executor/models/funaudiochat.py @@ -529,7 +529,7 @@ class FunAudioChatProcessingInfo(BaseProcessingInfo): return TokenizersBackend.from_pretrained( self.model_id, subfolder="speech_tokenizer", - revision=self.ctx.model_config.tokenizer_revision, + revision=self.ctx.model_config.revision, ) def get_feature_extractor(self) -> WhisperFeatureExtractor: diff --git a/vllm/model_executor/models/gemma4.py b/vllm/model_executor/models/gemma4.py index e3dfe65c2f..17b8251e9a 100644 --- a/vllm/model_executor/models/gemma4.py +++ b/vllm/model_executor/models/gemma4.py @@ -846,13 +846,22 @@ class Gemma4SelfDecoderLayers(nn.Module): # Shared references to modules owned by Gemma4Model — must be # inside this nn.Module so torch.compile captures them. self.embed_tokens = embed_tokens - self.normalizer = normalizer self.embed_tokens_per_layer = embed_tokens_per_layer - self.embed_scale_per_layer = embed_scale_per_layer self.per_layer_model_projection = per_layer_model_projection self.per_layer_projection_norm = per_layer_projection_norm - self.per_layer_input_scale = per_layer_input_scale - self.per_layer_projection_scale = per_layer_projection_scale + + # Register the shared scalars as buffers so they follow the model on + # .to(device); plain attributes stay on CPU. + self.register_buffer("normalizer", normalizer, persistent=False) + self.register_buffer( + "embed_scale_per_layer", embed_scale_per_layer, persistent=False + ) + self.register_buffer( + "per_layer_input_scale", per_layer_input_scale, persistent=False + ) + self.register_buffer( + "per_layer_projection_scale", per_layer_projection_scale, persistent=False + ) def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.embed_tokens(input_ids) * self.normalizer diff --git a/vllm/model_executor/models/interns1_pro.py b/vllm/model_executor/models/interns1_pro.py index e757241e94..629d0de3dc 100644 --- a/vllm/model_executor/models/interns1_pro.py +++ b/vllm/model_executor/models/interns1_pro.py @@ -45,6 +45,7 @@ from vllm.model_executor.layers.fused_moe import ( FusedMoEFactory, GateLinear, ) +from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -575,6 +576,7 @@ class InternS1ProForConditionalGeneration( self.visual = Qwen3_VisionTransformer( config.vision_config, norm_eps=getattr(config, "rms_norm_eps", 1e-6), + input_norm=build_mm_input_norm(self.model_config), prefix=maybe_prefix(prefix, "visual"), ) diff --git a/vllm/model_executor/models/interns2_mobius.py b/vllm/model_executor/models/interns2_mobius.py index c3fece062f..e186dc575b 100644 --- a/vllm/model_executor/models/interns2_mobius.py +++ b/vllm/model_executor/models/interns2_mobius.py @@ -21,6 +21,7 @@ from vllm.model_executor.layers.fused_moe import ( FusedMoEFactory, GateLinear, ) +from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import ( @@ -496,6 +497,7 @@ class InternS2MobiusForConditionalGeneration(InternS2PreviewForConditionalGenera config.vision_config, norm_eps=getattr(config, "rms_norm_eps", 1e-6), quant_config=quant_config, + input_norm=build_mm_input_norm(self.model_config), prefix=maybe_prefix(prefix, "visual"), ) diff --git a/vllm/model_executor/models/mistral3.py b/vllm/model_executor/models/mistral3.py index a70d8e1de5..593ba90ec0 100644 --- a/vllm/model_executor/models/mistral3.py +++ b/vllm/model_executor/models/mistral3.py @@ -3,7 +3,7 @@ import math from collections.abc import Iterable, Mapping, Sequence -from typing import Annotated, Literal +from typing import Annotated, Any, Literal, cast import torch import torch.nn as nn @@ -212,8 +212,14 @@ class Mistral3ProcessingInfo(BaseProcessingInfo): ) -> PixtralHFEncoderInfo: processor = self.get_hf_processor() size = processor.image_processor.size - merged_kwargs = self.ctx.get_merged_mm_kwargs(mm_processor_kwargs or {}) - if override_size := merged_kwargs.get("size"): + merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs( + mm_processor_kwargs or {} + ) + image_mm_kwargs = cast( + Mapping[str, Any], + merged_mm_kwargs.get("images_kwargs", {}), + ) + if override_size := image_mm_kwargs.get("size"): size = size | override_size image_size = size["longest_edge"] diff --git a/vllm/model_executor/models/qwen2_5_vl.py b/vllm/model_executor/models/qwen2_5_vl.py index 864a77f095..581ff77303 100644 --- a/vllm/model_executor/models/qwen2_5_vl.py +++ b/vllm/model_executor/models/qwen2_5_vl.py @@ -1976,7 +1976,8 @@ class Qwen2_5_VLForConditionalGeneration( for _ in range(max_batch_size) ] - # Create dummy pixel_values + # Create dummy pixel_values; uint8 when normalization is fused + # on-device. Contents are overwritten before every replay. patch_embed = self.visual.patch_embed in_channels = patch_embed.proj.in_channels patch_size = patch_embed.patch_size @@ -1985,8 +1986,11 @@ class Qwen2_5_VLForConditionalGeneration( flattened_patch_size = ( in_channels * temporal_patch_size * patch_size * patch_size ) - dummy_pixel_values = torch.randn( - total_patches, flattened_patch_size, device=device, dtype=dtype + dummy_pixel_values = torch.zeros( + total_patches, + flattened_patch_size, + device=device, + dtype=self.visual.input_norm.input_dtype or dtype, ) # Override max_seqlen with a safe upper bound for capture. diff --git a/vllm/model_executor/models/qwen2_vl.py b/vllm/model_executor/models/qwen2_vl.py index 3dae3ffd2a..7f0ed0e6a7 100644 --- a/vllm/model_executor/models/qwen2_vl.py +++ b/vllm/model_executor/models/qwen2_vl.py @@ -28,7 +28,7 @@ import math from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping, Sequence from functools import partial -from typing import Annotated, Any, Literal, TypeAlias +from typing import Annotated, Any, Literal, TypeAlias, cast import numpy as np import torch @@ -865,6 +865,118 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo): def get_supported_mm_limits(self) -> Mapping[str, int | None]: return {"image": None, "video": None} + @staticmethod + def _complete_mm_processor_size_aliases( + mm_kwargs: Mapping[str, object], + ) -> dict[str, object]: + """Complete missing Qwen-VL size aliases. + + Qwen-VL accepts ``size.shortest_edge`` / ``min_pixels`` and + ``size.longest_edge`` / ``max_pixels`` as equivalent controls. When + only one representation is provided with a non-``None`` value, add the + missing equivalent representation. If both are already present, leave + them unchanged even when their values conflict. + + Apply this independently to the flat kwargs and to each existing + ``images_kwargs`` / ``videos_kwargs`` mapping. Values are not propagated + between these mappings. + """ + + def add_missing_size_aliases( + kwargs: Mapping[str, object], + ) -> dict[str, object]: + """Add missing size aliases within a kwargs mapping.""" + out_kwargs = dict(kwargs) + # Copy `size` separately because `out_kwargs` is a shallow copy and + # size edges may be added below. + raw_size = cast( + Mapping[str, object] | None, + out_kwargs.get("size"), + ) + size_kwargs = {} if raw_size is None else dict(raw_size) + + for edge_key, pixel_key in ( + ("shortest_edge", "min_pixels"), + ("longest_edge", "max_pixels"), + ): + # Add the pixel alias from `size` when only the size edge is + # present. + if pixel_key not in out_kwargs: + if edge_key in size_kwargs and size_kwargs[edge_key] is not None: + out_kwargs[pixel_key] = size_kwargs[edge_key] + continue + + pixel_value = out_kwargs[pixel_key] + # HF treats `min_pixels` / `max_pixels` set to None as no size + # override, so do not propagate them to the equivalent size edge. + if pixel_value is None: + continue + + # Add the size edge when only the pixel alias is present. + if edge_key not in size_kwargs: + size_kwargs[edge_key] = pixel_value + out_kwargs["size"] = size_kwargs + + return out_kwargs + + prepared_mm_kwargs = add_missing_size_aliases(mm_kwargs) + # Complete aliases independently inside existing image/video kwargs; + # flat values are not copied into the nested mappings. + for nested_kwargs_key in ("images_kwargs", "videos_kwargs"): + nested_kwargs = prepared_mm_kwargs.get(nested_kwargs_key) + if isinstance(nested_kwargs, Mapping): + prepared_mm_kwargs[nested_kwargs_key] = add_missing_size_aliases( + nested_kwargs + ) + + return prepared_mm_kwargs + + def _merge_and_resolve_mm_processor_kwargs( + self, + mm_kwargs: Mapping[str, object], + ) -> dict[str, Any]: + """Merge configured and request Qwen-VL ``mm_processor_kwargs``. + + Request-side ``size.shortest_edge`` / ``min_pixels`` and + ``size.longest_edge`` / ``max_pixels`` aliases are completed before the + merge so request precedence is preserved across equivalent + representations. + """ + # Complete request-side Qwen-VL size aliases before merging with + # configured kwargs. + prepared_mm_kwargs = self._complete_mm_processor_size_aliases(mm_kwargs) + + return super()._merge_and_resolve_mm_processor_kwargs(prepared_mm_kwargs) + + @staticmethod + def _get_vision_size( + mm_kwargs: Mapping[str, object], + default_size: Mapping[str, int] | None = None, + ) -> dict[str, int]: + """Resolve Qwen-VL vision size kwargs with optional processor defaults. + + ``size`` overrides any processor defaults, then ``min_pixels`` and + ``max_pixels`` override the corresponding size edges. + """ + size = dict(default_size) if default_size is not None else {} + + override_size = cast( + Mapping[str, int] | None, + mm_kwargs.get("size"), + ) + if override_size is not None: + size.update(override_size) + + min_pixels = cast(int | None, mm_kwargs.get("min_pixels")) + if min_pixels is not None: + size["shortest_edge"] = min_pixels + + max_pixels = cast(int | None, mm_kwargs.get("max_pixels")) + if max_pixels is not None: + size["longest_edge"] = max_pixels + + return size + def get_mm_max_tokens_per_item( self, seq_len: int, @@ -883,7 +995,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo): do_resize: bool = True, image_processor: Qwen2VLImageProcessor, mm_kwargs: Mapping[str, object], - modality: str | None = None, + modality: Literal["image", "video"] | None = None, ) -> tuple[ImageSize, int]: hf_config = self.get_hf_config() vision_config = hf_config.vision_config @@ -891,14 +1003,16 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo): merge_size = vision_config.spatial_merge_size temporal_patch_size = vision_config.temporal_patch_size - mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality=modality) - size = image_processor.size - if override_size := mm_kwargs.get("size"): - size = size | override_size - if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None: - size = size | {"shortest_edge": override_min_pixels} - if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None: - size = size | {"longest_edge": override_max_pixels} + if modality is None: + modality = "image" + + merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs(mm_kwargs) + scope = "images_kwargs" if modality == "image" else "videos_kwargs" + scoped_mm_kwargs = merged_mm_kwargs.get(scope, {}) + size = self._get_vision_size( + scoped_mm_kwargs, + default_size=image_processor.size, + ) if do_resize: resized_height, resized_width = smart_resize( @@ -986,14 +1100,12 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo): if max_pixels is None: image_processor = self.get_image_processor() - mm_kwargs = self.ctx.get_merged_mm_kwargs({}, modality="image") - size = image_processor.size - if override_size := mm_kwargs.get("size"): - size = size | override_size - if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None: - size = size | {"shortest_edge": override_min_pixels} - if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None: - size = size | {"longest_edge": override_max_pixels} + merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs({}) + image_mm_kwargs = merged_mm_kwargs.get("images_kwargs", {}) + size = self._get_vision_size( + image_mm_kwargs, + default_size=image_processor.size, + ) max_pixels = size["longest_edge"] @@ -1616,7 +1728,8 @@ class Qwen2VLForConditionalGeneration( for _ in range(max_batch_size) ] - # Create dummy pixel_values. + # Create dummy pixel_values; uint8 when normalization is fused + # on-device. Contents are overwritten before every replay. patch_embed = self.visual.patch_embed in_channels = patch_embed.proj.in_channels patch_size = patch_embed.patch_size @@ -1625,8 +1738,11 @@ class Qwen2VLForConditionalGeneration( flattened_patch_size = ( in_channels * temporal_patch_size * patch_size * patch_size ) - dummy_pixel_values = torch.randn( - total_patches, flattened_patch_size, device=device, dtype=dtype + dummy_pixel_values = torch.zeros( + total_patches, + flattened_patch_size, + device=device, + dtype=self.visual.input_norm.input_dtype or dtype, ) # max_seqlen.item() gets baked into the CUDA graph at capture time. diff --git a/vllm/model_executor/models/qwen3_5.py b/vllm/model_executor/models/qwen3_5.py index 124d3bbbec..1ecf8a16ff 100644 --- a/vllm/model_executor/models/qwen3_5.py +++ b/vllm/model_executor/models/qwen3_5.py @@ -39,6 +39,7 @@ from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe.utils import ( is_model_fused_shared_expert_compatible, ) +from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm from vllm.model_executor.layers.layernorm import GemmaRMSNorm as Qwen3_5RMSNorm from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import ( @@ -519,6 +520,7 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid) config.vision_config, norm_eps=getattr(config, "rms_norm_eps", 1e-6), quant_config=quant_config, + input_norm=build_mm_input_norm(self.model_config), prefix=maybe_prefix(prefix, "visual"), ) @@ -741,6 +743,7 @@ class Qwen3_5MoeForConditionalGeneration( config.vision_config, norm_eps=getattr(config, "rms_norm_eps", 1e-6), quant_config=quant_config, + input_norm=build_mm_input_norm(self.model_config), prefix=maybe_prefix(prefix, "visual"), ) diff --git a/vllm/model_executor/models/qwen3_dflash.py b/vllm/model_executor/models/qwen3_dflash.py index 36a6c4c2c1..3161cf2261 100644 --- a/vllm/model_executor/models/qwen3_dflash.py +++ b/vllm/model_executor/models/qwen3_dflash.py @@ -34,7 +34,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( VocabParallelEmbedding, ) from vllm.multimodal.inputs import NestedTensors -from vllm.platforms import current_platform from vllm.transformers_utils.config import set_default_rope_theta from vllm.transformers_utils.repo_utils import get_hf_file_bytes from vllm.v1.attention.backend import AttentionType @@ -622,15 +621,6 @@ class DFlashQwen3Model(nn.Module): # --- Grouped RMSNorm K across all layers ([L, num_ctx, nkv, hd]) --- # The weight is selected per layer by the outermost (layer) index. all_k_normed = torch.empty_like(all_k) - if current_platform.is_xpu(): - for layer_idx in range(all_k.shape[0]): - ops.rms_norm( - all_k_normed[layer_idx], - all_k[layer_idx], - self._k_norm_weights[layer_idx], - self._rms_norm_eps, - ) - return all_k_normed ops.rms_norm( all_k_normed, all_k, diff --git a/vllm/model_executor/models/qwen3_vl.py b/vllm/model_executor/models/qwen3_vl.py index c9aa0d2066..6648949c73 100644 --- a/vllm/model_executor/models/qwen3_vl.py +++ b/vllm/model_executor/models/qwen3_vl.py @@ -27,7 +27,7 @@ from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping, Sequence from functools import lru_cache, partial from itertools import islice -from typing import Any +from typing import Any, Literal import numpy as np import torch @@ -67,6 +67,10 @@ from vllm.model_executor.layers.attention.mm_encoder_attention import ( MMEncoderAttention, ) from vllm.model_executor.layers.conv import Conv3dLayer +from vllm.model_executor.layers.fusion.mm_input_norm import ( + IdentityInputNorm, + build_mm_input_norm, +) from vllm.model_executor.layers.linear import ( ColumnParallelLinear, RowParallelLinear, @@ -566,6 +570,7 @@ class Qwen3_VisionTransformer(nn.Module): vision_config: Qwen3VLVisionConfig, norm_eps: float = 1e-6, quant_config: QuantizationConfig | None = None, + input_norm: nn.Module | None = None, prefix: str = "", ) -> None: super().__init__() @@ -605,6 +610,7 @@ class Qwen3_VisionTransformer(nn.Module): in_channels=vision_config.in_channels, hidden_size=self.hidden_size, ) + self.input_norm = input_norm if input_norm is not None else IdentityInputNorm() self.pos_embed = nn.Embedding(self.num_position_embeddings, self.hidden_size) @@ -858,7 +864,9 @@ class Qwen3_VisionTransformer(nn.Module): *, encoder_metadata: dict[str, torch.Tensor] | None = None, ) -> torch.Tensor: - hidden_states = x.to(device=self.device, dtype=self.dtype, non_blocking=True) + hidden_states = self.input_norm( + x.to(device=self.device, non_blocking=True), self.dtype + ) hidden_states = self.patch_embed(hidden_states) if encoder_metadata is None: @@ -936,6 +944,16 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo): allow_missing_mm_embeddings=self.allow_missing_mm_embeddings, ) + def get_supported_mm_processor_kwargs(self) -> dict[str, set[str]]: + supported = super().get_supported_mm_processor_kwargs() + + # vLLM has historically accepted `min_pixels` / `max_pixels` for + # Qwen3-VL video. Include them in the routing schema even though + # Qwen3VLVideoProcessor only accepts the equivalent `size`; they are + # folded into `size` before calling HF. + supported["videos_kwargs"].update({"min_pixels", "max_pixels"}) + return supported + def _get_vision_info( self, *, @@ -945,7 +963,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo): do_resize: bool = True, image_processor: Qwen2VLImageProcessor | Qwen3VLVideoProcessor, mm_kwargs: Mapping[str, object], - modality: str | None = None, + modality: Literal["image", "video"] | None = None, ) -> tuple[ImageSize, int]: is_video = isinstance(image_processor, Qwen3VLVideoProcessor) @@ -957,14 +975,14 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo): if modality is None: modality = "video" if is_video else "image" - mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality=modality) - size = image_processor.size - if override_size := mm_kwargs.get("size"): - size = size | override_size - if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None: - size = size | {"shortest_edge": override_min_pixels} - if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None: - size = size | {"longest_edge": override_max_pixels} + + merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs(mm_kwargs) + scope = "videos_kwargs" if modality == "video" else "images_kwargs" + scoped_mm_kwargs = merged_mm_kwargs.get(scope, {}) + size = self._get_vision_size( + scoped_mm_kwargs, + default_size=image_processor.size, + ) if do_resize: if is_video: @@ -1022,9 +1040,13 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo): ) -> int: video_processor = self.get_video_processor() - mm_kwargs = self.ctx.get_merged_mm_kwargs({}, modality="video") - video_size = mm_kwargs.get("size", video_processor.size) - temporal_patch_size = mm_kwargs.get( + merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs({}) + video_mm_kwargs = merged_mm_kwargs.get("videos_kwargs", {}) + video_size = self._get_vision_size( + video_mm_kwargs, + default_size=video_processor.size, + ) + temporal_patch_size = video_mm_kwargs.get( "temporal_patch_size", video_processor.temporal_patch_size ) @@ -1154,9 +1176,13 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]): video_processor = self.info.get_video_processor() - mm_kwargs = self.info.ctx.get_merged_mm_kwargs({}, modality="video") - video_size = mm_kwargs.get("size", video_processor.size) - temporal_patch_size = mm_kwargs.get( + merged_mm_kwargs = self.info._merge_and_resolve_mm_processor_kwargs({}) + video_mm_kwargs = merged_mm_kwargs.get("videos_kwargs", {}) + video_size = self.info._get_vision_size( + video_mm_kwargs, + default_size=video_processor.size, + ) + temporal_patch_size = video_mm_kwargs.get( "temporal_patch_size", video_processor.temporal_patch_size ) @@ -1172,13 +1198,13 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]): # over enough frames that the cap is not binding for the dummy. # This takes precedence over a shrinking num_frames override, which # would otherwise reintroduce the under-profiling. - if mm_kwargs.get("cap_pixels_per_frame"): - max_video_tokens = mm_kwargs.get( + if video_mm_kwargs.get("cap_pixels_per_frame"): + max_video_tokens = video_mm_kwargs.get( "max_video_tokens", getattr(video_processor, "max_video_tokens", 768), ) - patch_size = mm_kwargs.get("patch_size", video_processor.patch_size) - merge_size = mm_kwargs.get("merge_size", video_processor.merge_size) + patch_size = video_mm_kwargs.get("patch_size", video_processor.patch_size) + merge_size = video_mm_kwargs.get("merge_size", video_processor.merge_size) per_frame_cap = max_video_tokens * (patch_size * merge_size) ** 2 target_num_frames = max( target_num_frames, @@ -1340,6 +1366,47 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) if not hf_data: return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data) + # Merge and resolve the multimodal processor kwargs once so every HF + # call below starts from the same engine/request state and scoped form. + merged_mm_kwargs = self.info._merge_and_resolve_mm_processor_kwargs(hf_kwargs) + + if "images_kwargs" in merged_mm_kwargs: + image_scoped_kwargs = dict(merged_mm_kwargs["images_kwargs"]) + + # Qwen2VLImageProcessor requires both `size` edges. If a partial size + # override is passed, fill missing edges from the processor defaults + # before any HF call. + if image_scoped_kwargs.keys() & {"size", "min_pixels", "max_pixels"}: + default_image_size = self.info.get_image_processor().size + image_scoped_kwargs["size"] = self.info._get_vision_size( + image_scoped_kwargs, + default_size=default_image_size, + ) + + merged_mm_kwargs["images_kwargs"] = image_scoped_kwargs + + if "videos_kwargs" in merged_mm_kwargs: + video_scoped_kwargs = dict(merged_mm_kwargs["videos_kwargs"]) + + # Qwen3VLVideoProcessor requires both `size` edges. If a partial size + # override is passed, fill missing edges from the processor defaults + # before any HF call. + if video_scoped_kwargs.keys() & {"size", "min_pixels", "max_pixels"}: + default_video_size = self.info.get_video_processor().size + video_scoped_kwargs["size"] = self.info._get_vision_size( + video_scoped_kwargs, + default_size=default_video_size, + ) + + # Qwen3VLVideoProcessor does not accept vLLM's min/max pixel + # compatibility kwargs. They have already been folded into `size`, + # so remove them before any HF call, including calls that contain + # only text/images. + video_scoped_kwargs.pop("min_pixels", None) + video_scoped_kwargs.pop("max_pixels", None) + + merged_mm_kwargs["videos_kwargs"] = video_scoped_kwargs + # Separate video processing from image processing. Because the videos # are processed into several image patches video_input_ids_lst: list[list[int]] = [] @@ -1365,32 +1432,22 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) # used to calculate the timestamps. Make sure that # do_sample_frames in mm_kwargs is false for presampled videos. - # NOTE: a copy of is created to update do_sample_frames, - # otherwise mm_hash for the object will be incorrect. - video_mm_kwargs: dict[str, Any] = dict(hf_kwargs) - merged = self.info.ctx.get_merged_mm_kwargs(hf_kwargs, modality="video") - if merged.keys() & {"size", "min_pixels", "max_pixels"}: - video_size = dict(self.info.get_video_processor().size) - size_override = merged.get("size") - if size_override is not None: - video_size = video_size | size_override - min_pixels = merged.get("min_pixels") - if min_pixels is not None: - video_size["shortest_edge"] = min_pixels - max_pixels = merged.get("max_pixels") - if max_pixels is not None: - video_size["longest_edge"] = max_pixels - video_mm_kwargs["size"] = video_size - sampled_fps = video_mm_kwargs.get("fps") + # The root copy is shallow, so give each item its own video + # scope before modifying per-video values below. + video_mm_kwargs = dict(merged_mm_kwargs) + video_scoped_kwargs = dict(video_mm_kwargs.get("videos_kwargs", {})) + video_mm_kwargs["videos_kwargs"] = video_scoped_kwargs + + sampled_fps = video_scoped_kwargs.get("fps") if is_list_of(sampled_fps, float): - video_mm_kwargs["fps"] = sampled_fps[item_idx] - sampled_num_frames = video_mm_kwargs.get("num_frames") + video_scoped_kwargs["fps"] = sampled_fps[item_idx] + sampled_num_frames = video_scoped_kwargs.get("num_frames") if is_list_of(sampled_num_frames, int): - video_mm_kwargs["num_frames"] = sampled_num_frames[item_idx] - if "do_sample_frames" not in video_mm_kwargs: + video_scoped_kwargs["num_frames"] = sampled_num_frames[item_idx] + if "do_sample_frames" not in video_scoped_kwargs: # qwen_vl_utils already has "do_sample_frames" in # mm_kwargs, don't overwrite it. - video_mm_kwargs["do_sample_frames"] = metadata.get( + video_scoped_kwargs["do_sample_frames"] = metadata.get( "do_sample_frames", False ) @@ -1401,9 +1458,9 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) # Compute timestamps here where we have access to metadata timestamps = self.info._get_video_second_idx( metadata=metadata, - do_sample_frames=video_mm_kwargs["do_sample_frames"], - sampled_fps=video_mm_kwargs.get("fps"), - sampled_num_frames=video_mm_kwargs.get("num_frames"), + do_sample_frames=video_scoped_kwargs["do_sample_frames"], + sampled_fps=video_scoped_kwargs.get("fps"), + sampled_num_frames=video_scoped_kwargs.get("num_frames"), ) timestamps_per_video.append(timestamps) @@ -1415,8 +1472,11 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) # to prevent HF's BaseVideoProcessor.preprocess() from # filling in the class default (fps=2) via setdefault(), # which would conflict with num_frames (mutually exclusive). - if "num_frames" in video_mm_kwargs and "fps" not in video_mm_kwargs: - video_mm_kwargs["fps"] = None + if ( + "num_frames" in video_scoped_kwargs + and "fps" not in video_scoped_kwargs + ): + video_scoped_kwargs["fps"] = None video_outputs = self.info.ctx.call_hf_processor( self.info.get_hf_processor(**video_mm_kwargs), @@ -1477,11 +1537,14 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) else: video_outputs = dict() - # fps/num_frames are video-only kwargs already consumed by the loop; - # exclude them so the text/image processor call below never gets a list. - non_video_mm_kwargs = { - k: v for k, v in hf_kwargs.items() if k not in ("fps", "num_frames") - } + # The root copy is shallow, so copy the video scope before removing + # per-video kwargs to avoid mutating `merged_mm_kwargs`. + non_video_mm_kwargs = dict(merged_mm_kwargs) + if "videos_kwargs" in non_video_mm_kwargs: + non_video_scoped_kwargs = dict(non_video_mm_kwargs["videos_kwargs"]) + non_video_scoped_kwargs.pop("fps", None) + non_video_scoped_kwargs.pop("num_frames", None) + non_video_mm_kwargs["videos_kwargs"] = non_video_scoped_kwargs processed_data = self.info.ctx.call_hf_processor( self.info.get_hf_processor(**non_video_mm_kwargs), hf_data, @@ -1813,6 +1876,7 @@ class Qwen3VLForConditionalGeneration( supports_encoder_tp_data = True supports_tower_connector_lora = True + supports_mm_device_do_normalize = True supported_video_pruning_methods = ("evs", "vidcom2") @@ -1863,6 +1927,7 @@ class Qwen3VLForConditionalGeneration( config.vision_config, norm_eps=getattr(config, "rms_norm_eps", 1e-6), quant_config=quant_config, + input_norm=build_mm_input_norm(self.model_config), prefix=maybe_prefix(prefix, "visual"), ) @@ -2172,8 +2237,11 @@ class Qwen3VLForConditionalGeneration( flattened_patch_size = ( in_channels * temporal_patch_size * patch_size * patch_size ) - dummy_pixel_values = torch.randn( - total_patches, flattened_patch_size, device=device, dtype=dtype + dummy_pixel_values = torch.zeros( + total_patches, + flattened_patch_size, + device=device, + dtype=self.visual.input_norm.input_dtype or dtype, ) # Override max_seqlen with a safe upper bound for capture. @@ -2307,7 +2375,7 @@ class Qwen3VLForConditionalGeneration( if image_input["type"] == "image_embeds": image_embeds = image_input["image_embeds"].type(self.visual.dtype) else: - pixel_values = image_input["pixel_values"].type(self.visual.dtype) + pixel_values = image_input["pixel_values"] if self.use_data_parallel: return run_dp_sharded_mrope_vision_model( self.visual, pixel_values, grid_thw.tolist(), rope_type="rope_3d" @@ -2329,9 +2397,7 @@ class Qwen3VLForConditionalGeneration( if video_input["type"] == "video_embeds": video_embeds = video_input["video_embeds"].type(self.visual.dtype) else: - pixel_values_videos = video_input["pixel_values_videos"].type( - self.visual.dtype - ) + pixel_values_videos = video_input["pixel_values_videos"] if self.use_data_parallel: grid_thw_list = grid_thw.tolist() return run_dp_sharded_mrope_vision_model( diff --git a/vllm/model_executor/models/qwen3_vl_moe.py b/vllm/model_executor/models/qwen3_vl_moe.py index a345f577b1..4d0c7788a6 100644 --- a/vllm/model_executor/models/qwen3_vl_moe.py +++ b/vllm/model_executor/models/qwen3_vl_moe.py @@ -36,6 +36,7 @@ from vllm.compilation.decorators import support_torch_compile from vllm.config import VllmConfig from vllm.distributed import get_pp_group from vllm.logger import init_logger +from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead from vllm.multimodal import MULTIMODAL_REGISTRY @@ -254,6 +255,7 @@ class Qwen3VLMoeForConditionalGeneration( config.vision_config, norm_eps=getattr(config, "rms_norm_eps", 1e-6), quant_config=quant_config, + input_norm=build_mm_input_norm(self.model_config), prefix=maybe_prefix(prefix, "visual"), ) diff --git a/vllm/model_executor/models/vision.py b/vllm/model_executor/models/vision.py index b35db4de27..d16e631872 100644 --- a/vllm/model_executor/models/vision.py +++ b/vllm/model_executor/models/vision.py @@ -5,7 +5,7 @@ import itertools import math from abc import ABC, abstractmethod from collections.abc import Callable -from typing import Final, Generic, Literal, Protocol, TypeAlias, TypeVar +from typing import Generic, Literal, Protocol, TypeAlias, TypeVar import torch from transformers import PreTrainedConfig @@ -24,10 +24,12 @@ from vllm.v1.attention.backends.registry import AttentionBackendEnum logger = init_logger(__name__) _C = TypeVar("_C", bound=PreTrainedConfig) +_C_co = TypeVar("_C_co", bound=PreTrainedConfig, covariant=True) -class _RootConfig(Protocol[_C]): - vision_config: _C +class _RootConfig(Protocol[_C_co]): + @property + def vision_config(self) -> _C_co: ... class VisionEncoderInfo(ABC, Generic[_C]): @@ -60,7 +62,8 @@ class VisionEncoderInfo(ABC, Generic[_C]): class VisionLanguageConfig(Protocol): - vision_config: Final[PreTrainedConfig] + @property + def vision_config(self) -> PreTrainedConfig: ... def get_vision_encoder_info(hf_config: VisionLanguageConfig) -> VisionEncoderInfo: diff --git a/vllm/model_executor/models/voxtral.py b/vllm/model_executor/models/voxtral.py index 21b291344d..c65dc38298 100644 --- a/vllm/model_executor/models/voxtral.py +++ b/vllm/model_executor/models/voxtral.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import math -from collections.abc import Iterable, Mapping, Sequence +from collections.abc import Callable, Iterable, Mapping, Sequence from functools import partial from typing import cast @@ -317,7 +317,7 @@ class VoxtralForConditionalGeneration( # update quant config to so that ignored module and target module names # match the vLLM model names - if hasattr(vllm_config, "quant_config"): + if vllm_config.quant_config is not None: vllm_config.quant_config = self.maybe_update_quant_config( vllm_config.quant_config ) @@ -724,6 +724,7 @@ class VoxtralEncoderModel(nn.Module): self.config = cast(WhisperConfig, vllm_config.model_config.hf_config) self.dtype: torch.dtype = vllm_config.model_config.dtype self.is_causal = getattr(self.config, "is_causal", False) + WhisperEncoderCls: Callable[..., WhisperEncoder | WhisperCausalEncoder] if self.is_causal: WhisperEncoderCls = WhisperCausalEncoder else: @@ -830,7 +831,7 @@ class VoxtralEncoderModel(nn.Module): return results def load_weight(self, weight: tuple[str, torch.Tensor]) -> str: - stacked_params_mapping = [ + stacked_params_mapping: list[tuple[str, str, str | int]] = [ # (param_name, shard_name, shard_id) ("qkv_proj", "q_proj", "q"), ("qkv_proj", "k_proj", "k"), diff --git a/vllm/model_executor/models/voxtral_realtime.py b/vllm/model_executor/models/voxtral_realtime.py index 9fdc946df9..8531d7d18f 100644 --- a/vllm/model_executor/models/voxtral_realtime.py +++ b/vllm/model_executor/models/voxtral_realtime.py @@ -16,7 +16,6 @@ from mistral_common.tokens.tokenizers.audio import Audio, AudioConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig from vllm.config.speech_to_text import SpeechToTextParams -from vllm.engine.protocol import StreamingInput from vllm.inputs import PromptType, TokensPrompt from vllm.logger import init_logger from vllm.model_executor.models.interfaces import MultiModalEmbeddings, SupportsRealtime @@ -26,6 +25,7 @@ from vllm.model_executor.models.voxtral import ( VoxtralMultiModalProcessor, VoxtralProcessingInfo, ) +from vllm.model_executor.models.whisper_causal import WhisperCausalEncoder from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.parse import MultiModalDataItems from vllm.multimodal.processing import ProcessorInputs, TimingContext @@ -56,7 +56,7 @@ class VoxtralRealtimeMultiModalProcessor(VoxtralMultiModalProcessor): mm_items: MultiModalDataItems, mm_res: MultiModalProcessingResult, ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]: - mm_kwargs = mm_res.kwargs + mm_kwargs = mm_res.kwargs.require_data() # there are no placeholder audio tokens for streaming # so we need to build the place placeholder positions manually @@ -69,7 +69,9 @@ class VoxtralRealtimeMultiModalProcessor(VoxtralMultiModalProcessor): tokenizer = self.info.get_tokenizer() audio_config = tokenizer.instruct.audio_encoder.audio_config - num_audio_samples = audios[0]["audio_arrays"].data.shape[0] + audio_array = audios[0]["audio_arrays"].data + assert isinstance(audio_array, (torch.Tensor, np.ndarray)) + num_audio_samples = audio_array.shape[0] length = audio_config.num_audio_tokens(num_audio_samples) features_info = PlaceholderFeaturesInfo( @@ -162,7 +164,7 @@ class VoxtralRealtimeBuffer: for token in tokens: await self._token_queue.put(token) - async def get_input_stream(self) -> AsyncGenerator[StreamingInput]: + async def get_input_stream(self) -> AsyncGenerator[TokensPrompt]: for frame_size, num_tokens in self._generate_frame_size_and_num_tokens(): next_tokens = [await self._token_queue.get() for _ in range(num_tokens)] @@ -188,11 +190,9 @@ class VoxtralRealtimeBuffer: self._leftover = audio_array[stride:] - yield StreamingInput( - TokensPrompt( - prompt_token_ids=next_tokens, - multi_modal_data={"audio": (frame, None)}, - ) + yield TokensPrompt( + prompt_token_ids=next_tokens, + multi_modal_data={"audio": (frame, None)}, ) @@ -274,8 +274,8 @@ class VoxtralRealtimeGeneration(VoxtralForConditionalGeneration, SupportsRealtim token_task = asyncio.create_task(feed_tokens()) try: - async for streaming_input in buffer.get_input_stream(): - yield streaming_input.prompt + async for prompt in buffer.get_input_stream(): + yield prompt finally: audio_task.cancel() token_task.cancel() @@ -420,6 +420,7 @@ class VoxtralRealtimeGeneration(VoxtralForConditionalGeneration, SupportsRealtim seq_lens = [mel.shape[1] for mel in mel_features] # [total_num_20ms_frames, hidden_size] + assert isinstance(self.whisper_encoder.whisper_encoder, WhisperCausalEncoder) audio_embeddings = self.whisper_encoder.whisper_encoder.forward_conv( mel_features ) diff --git a/vllm/models/deepseek_v4/amd/mtp.py b/vllm/models/deepseek_v4/amd/mtp.py index 045a731321..48ab036563 100644 --- a/vllm/models/deepseek_v4/amd/mtp.py +++ b/vllm/models/deepseek_v4/amd/mtp.py @@ -240,13 +240,11 @@ class DeepSeekV4MultiTokenPredictor(nn.Module): current_step_idx, ) - def compute_logits( + def _pre_head( self, + mtp_layer: DeepSeekV4MultiTokenPredictorLayer, hidden_states: torch.Tensor, - spec_step_idx: int = 0, ) -> torch.Tensor: - current_step_idx = spec_step_idx % self.num_mtp_layers - mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)] # MTP forward returns the pre-hc_head residual (T, hc_mult * D); apply # hc_head here so logits are computed from the dense hidden state. hidden_states = hidden_states.view( @@ -260,14 +258,38 @@ class DeepSeekV4MultiTokenPredictor(nn.Module): mtp_layer.rms_norm_eps, mtp_layer.hc_eps, ) - hidden_states = _MTP_SHARED_HEAD_RMSNORM_KERNEL( + return _MTP_SHARED_HEAD_RMSNORM_KERNEL( hidden_states, mtp_layer.shared_head.norm.weight.data, mtp_layer.shared_head.norm.variance_epsilon, ) - logits = self.logits_processor(mtp_layer.shared_head.head, hidden_states) + + def compute_logits( + self, + hidden_states: torch.Tensor, + spec_step_idx: int = 0, + ) -> torch.Tensor: + current_step_idx = spec_step_idx % self.num_mtp_layers + mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)] + logits = self.logits_processor( + mtp_layer.shared_head.head, self._pre_head(mtp_layer, hidden_states) + ) return logits + def get_top_tokens( + self, + hidden_states: torch.Tensor, + spec_step_idx: int = 0, + ) -> torch.Tensor: + current_step_idx = spec_step_idx % self.num_mtp_layers + mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)] + # Vocab-parallel argmax for the greedy draft: per-rank head projection + # + local argmax + a [batch, 2*tp] (value, index) reduce, instead of + # all-gathering the full vocab logits. + return self.logits_processor.get_top_tokens( + mtp_layer.shared_head.head, self._pre_head(mtp_layer, hidden_states) + ) + class DeepSeekV4MTP(nn.Module): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): @@ -302,6 +324,15 @@ class DeepSeekV4MTP(nn.Module): ) -> torch.Tensor | None: return self.model.compute_logits(hidden_states, spec_step_idx) + def get_top_tokens( + self, + hidden_states: torch.Tensor, + spec_step_idx: int = 0, + ) -> torch.Tensor: + # Greedy-draft path used when use_local_argmax_reduction is enabled: + # vocab-parallel argmax, no full-vocab logits. + return self.model.get_top_tokens(hidden_states, spec_step_idx) + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: # Weight name remapping for checkpoint compatibility. # Maps checkpoint weight paths to model parameter paths. diff --git a/vllm/models/deepseek_v4/amd/rocm.py b/vllm/models/deepseek_v4/amd/rocm.py index 78dc10ba00..292901dfe9 100644 --- a/vllm/models/deepseek_v4/amd/rocm.py +++ b/vllm/models/deepseek_v4/amd/rocm.py @@ -35,6 +35,7 @@ from vllm.v1.attention.backends.mla.sparse_swa import ( DeepseekSparseSWAMetadataBuilder, ) from vllm.v1.attention.ops.rocm_aiter_mla_sparse import ( + build_prefill_topk_ragged_indices, build_ragged_indices_from_dense, rocm_inv_rope_einsum, rocm_inverse_rope_rows_, @@ -103,6 +104,22 @@ def _trust_dsv4_extra_cache_nan_free( ) +def _aiter_sparse_mla_enabled(has_kv_transfer: bool) -> bool: + from vllm._aiter_ops import rocm_aiter_ops + + if not rocm_aiter_ops.is_triton_sparse_mla_enabled(): + return False + # The aiter kernel does not scrub NaNs, so like the trust check above it + # needs a cache only the canonical writer has touched. + if has_kv_transfer: + logger.warning_once( + "VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA is ignored with a KV " + "connector; DeepSeek V4 keeps the Triton sparse attention." + ) + return False + return True + + def _build_indptr_from_lengths(lengths: torch.Tensor) -> torch.Tensor: lengths = lengths.to(dtype=torch.int32).contiguous() indptr = torch.zeros(lengths.shape[0] + 1, dtype=torch.int32, device=lengths.device) @@ -446,6 +463,9 @@ class DeepseekV4ROCMAiterMLASparseMetadata(DeepseekV4FlashMLAMetadata): class DeepseekV4ROCMAiterSparseSWAMetadata(DeepseekSparseSWAMetadata): decode_swa_ragged_indices: torch.Tensor | None = None decode_swa_ragged_indptr: torch.Tensor | None = None + # Built only for the aiter prefill, which reads the SWA cache in place. + prefill_swa_ragged_indices: torch.Tensor | None = None + prefill_swa_ragged_indptr: torch.Tensor | None = None class DeepseekV4ROCMAiterMLASparseMetadataBuilder(DeepseekV4SparseMLAMetadataBuilder): @@ -565,6 +585,9 @@ class DeepseekV4ROCMAiterSparseSWAMetadataBuilder(DeepseekSparseSWAMetadataBuild dtype=torch.int32, device=self.device, ) + self._use_aiter_sparse_mla = _aiter_sparse_mla_enabled( + self.vllm_config.kv_transfer_config is not None + ) def build( self, @@ -580,6 +603,23 @@ class DeepseekV4ROCMAiterSparseSWAMetadataBuilder(DeepseekSparseSWAMetadataBuild replay_start=replay_start, ) + prefill_ragged_indices = None + prefill_ragged_indptr = None + if ( + self._use_aiter_sparse_mla + and base.num_prefill_tokens > 0 + and base.prefill_swa_indices is not None + and base.prefill_swa_lens is not None + ): + # Paged-coordinate rows (image visibility included), so every + # layer's prefill shares one build. + prefill_ragged_indices, prefill_ragged_indptr = ( + build_ragged_indices_from_dense( + base.prefill_swa_indices.reshape(base.num_prefill_tokens, -1), + base.prefill_swa_lens, + ) + ) + ragged_indices = None ragged_indptr = None if ( @@ -606,6 +646,8 @@ class DeepseekV4ROCMAiterSparseSWAMetadataBuilder(DeepseekSparseSWAMetadataBuild **vars(base), decode_swa_ragged_indices=ragged_indices, decode_swa_ragged_indptr=ragged_indptr, + prefill_swa_ragged_indices=prefill_ragged_indices, + prefill_swa_ragged_indptr=prefill_ragged_indptr, ) @@ -623,6 +665,7 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention): """ROCm sparse MLA attention layer for DeepSeek V4.""" backend_cls = DeepseekV4ROCMAiterMLASparseBackend + _use_aiter_sparse_mla = False def __init__(self, *args, **kwargs): vllm_config = args[0] if args else kwargs["vllm_config"] @@ -648,23 +691,7 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention): else: # Disable indexer inner overlap. self.indexer.aux_stream = None - - def _enable_multi_stream_overlap(self) -> bool: - """ROCm multi-stream gates: streams and capture region. - - Dict metadata marks piecewise cudagraph, whose eager breaks rebuild - the attention inputs on the owning stream. Forking side streams - there would rely on runtime HIP event sync, which is unreliable in - this overlap on ROCm (event waits can hang), so multi-stream only - runs where the fork/join becomes static graph edges: inside capture, - or with non-dict metadata (full cudagraph or the profile run), which - has no eager breaks. Covers both the HCA and CSA forks. - """ - attn_metadata = get_forward_context().attn_metadata - return self.aux_stream_list is not None and ( - torch.cuda.is_current_stream_capturing() - or not isinstance(attn_metadata, dict) - ) + self._use_aiter_sparse_mla = _aiter_sparse_mla_enabled(self._has_kv_transfer) def _run_sequential_pipeline( self, @@ -709,7 +736,9 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention): device=hidden_states.device, ) - if self._enable_multi_stream_overlap(): + if current_platform.enable_multi_stream_overlap( + self.aux_stream_list, get_forward_context().attn_metadata + ): # The ROCm override consumes these sentinels inside the capture # boundary, moving the stream fan-out ahead of the projections. self._prepare_and_attn_fn( @@ -769,7 +798,9 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention): # Re-check: forward's gate ran inside a captured segment that # _prepare_and_attn_eager (MRV1) then broke, making this region eager. - if not self._enable_multi_stream_overlap(): + if not current_platform.enable_multi_stream_overlap( + self.aux_stream_list, get_forward_context().attn_metadata + ): self._run_sequential_pipeline(hidden_states, positions, o_padded) return @@ -1203,18 +1234,20 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention): if attn_metadata is None: # Warmup dummy run: no real metadata. Reserve the same bf16 # gather workspace _forward_prefill would; the dequantize / topk - # / sparse_fwd kernels are skipped this step. - swa_only = self.compress_ratio <= 1 - N = ( - 0 - if swa_only - else (self.max_model_len + self.compress_ratio - 1) - // self.compress_ratio - ) - M = N + self.window_size + self.max_num_batched_tokens - current_workspace_manager().get_simultaneous( - ((self.PREFILL_CHUNK_SIZE, M, q.shape[-1]), torch.bfloat16), - ) + # / sparse_fwd kernels are skipped this step. The aiter prefill + # reads the caches in place and never asks for it. + if not self._use_aiter_sparse_mla: + swa_only = self.compress_ratio <= 1 + N = ( + 0 + if swa_only + else (self.max_model_len + self.compress_ratio - 1) + // self.compress_ratio + ) + M = N + self.window_size + self.max_num_batched_tokens + current_workspace_manager().get_simultaneous( + ((self.PREFILL_CHUNK_SIZE, M, q.shape[-1]), torch.bfloat16), + ) output.zero_() return @@ -1325,6 +1358,22 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention): topk_ragged_indices = attn_metadata.c128a_decode_topk_ragged_indices topk_ragged_indptr = attn_metadata.c128a_decode_topk_ragged_indptr + if self._use_aiter_sparse_mla: + assert swa_metadata.decode_swa_ragged_indices is not None + assert swa_metadata.decode_swa_ragged_indptr is not None + self._aiter_sparse_mla( + q, + output, + self.swa_cache_layer.kv_cache, + swa_metadata.decode_swa_ragged_indices, + swa_metadata.decode_swa_ragged_indptr, + kv_cache, + topk_ragged_indices, + topk_ragged_indptr, + ) + # The aiter kernel leaves the inverse RoPE to the caller. + return 0 + return rocm_sparse_attn_decode( q=q, kv_cache=kv_cache, @@ -1364,6 +1413,17 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention): attn_metadata: DeepseekV4ROCMAiterMLASparseMetadata | None, swa_metadata: DeepseekV4ROCMAiterSparseSWAMetadata, ) -> None: + if self._use_aiter_sparse_mla: + self._forward_prefill_aiter( + q=q, + compressed_k_cache=compressed_k_cache, + swa_k_cache=swa_k_cache, + output=output, + attn_metadata=attn_metadata, + swa_metadata=swa_metadata, + ) + return + swa_only = attn_metadata is None num_prefills = swa_metadata.num_prefills @@ -1489,3 +1549,90 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention): attn_sink=self.attn_sink, output=output[query_start:query_end], ) + + def _forward_prefill_aiter( + self, + q: torch.Tensor, + compressed_k_cache: torch.Tensor | None, + swa_k_cache: torch.Tensor, + output: torch.Tensor, + attn_metadata: DeepseekV4ROCMAiterMLASparseMetadata | None, + swa_metadata: DeepseekV4ROCMAiterSparseSWAMetadata, + ) -> None: + """Prefill straight off the paged caches, with no bf16 gather.""" + num_decode_tokens = swa_metadata.num_decode_tokens + num_prefill_tokens = swa_metadata.num_prefill_tokens + assert swa_metadata.prefill_swa_ragged_indices is not None + assert swa_metadata.prefill_swa_ragged_indptr is not None + + topk_ragged_indices = None + topk_ragged_indptr = None + if attn_metadata is None: + compressed_k_cache = None + else: + assert compressed_k_cache is not None + assert swa_metadata.token_to_req_indices is not None + assert swa_metadata.query_start_loc is not None + assert swa_metadata.seq_lens is not None + assert swa_metadata.is_valid_token is not None + if self.compress_ratio == 4: + assert self.topk_indices_buffer is not None + topk_indices = self.topk_indices_buffer[ + num_decode_tokens : num_decode_tokens + num_prefill_tokens + ] + else: + topk_indices = attn_metadata.c128a_prefill_topk_indices + assert topk_indices is not None + topk_ragged_indices, topk_ragged_indptr = build_prefill_topk_ragged_indices( + topk_indices, + swa_metadata.token_to_req_indices, + swa_metadata.query_start_loc, + swa_metadata.seq_lens, + swa_metadata.is_valid_token, + attn_metadata.block_table, + block_size=attn_metadata.block_size // self.compress_ratio, + compress_ratio=self.compress_ratio, + num_compressed=(self.max_model_len + self.compress_ratio - 1) + // self.compress_ratio, + token_offset=num_decode_tokens, + num_rows=compressed_k_cache.shape[0] * compressed_k_cache.shape[1], + ) + + self._aiter_sparse_mla( + q[:num_prefill_tokens], + output[:num_prefill_tokens], + swa_k_cache, + swa_metadata.prefill_swa_ragged_indices, + swa_metadata.prefill_swa_ragged_indptr, + compressed_k_cache, + topk_ragged_indices, + topk_ragged_indptr, + ) + + def _aiter_sparse_mla( + self, + q: torch.Tensor, + output: torch.Tensor, + swa_k_cache: torch.Tensor, + swa_indices: torch.Tensor, + swa_indptr: torch.Tensor, + compressed_k_cache: torch.Tensor | None, + topk_indices: torch.Tensor | None, + topk_indptr: torch.Tensor | None, + ) -> None: + from vllm._aiter_ops import rocm_aiter_ops + + # Both fp8 caches are read in place: the SWA window as the main segment, + # the top-k compressed tokens as the extra one. + rocm_aiter_ops.triton_sparse_mla_fwd( + q, + swa_k_cache, + output, + self.scale, + swa_indptr, + swa_indices, + attn_sink=self.attn_sink[: q.shape[1]], + extra_kv_buffer=compressed_k_cache, + extra_kv_indptr=topk_indptr, + extra_kv_indices=topk_indices, + ) diff --git a/vllm/models/deepseek_v41/amd/model.py b/vllm/models/deepseek_v41/amd/model.py index 0cad301cb6..61c8d5de30 100644 --- a/vllm/models/deepseek_v41/amd/model.py +++ b/vllm/models/deepseek_v41/amd/model.py @@ -22,7 +22,11 @@ from vllm.model_executor.layers.fused_moe import ( ) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.logits_processor import LogitsProcessor -from vllm.model_executor.layers.mhc import MHCPostOp, MHCPreDelayedOp +from vllm.model_executor.layers.mhc import ( + HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM, + MHCPostOp, + MHCPreDelayedOp, +) from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, @@ -251,6 +255,10 @@ class DeepseekV4DecoderLayer(nn.Module): ) self.mhc_pre_delayed = MHCPreDelayedOp() self.mhc_post = MHCPostOp() + # Where aiter's fused seam kernel runs (gfx950), it folds the following + # attn_norm / ffn_norm into its collapse, so the separate norms are + # skipped for the seams it takes. + self.fuse_seam_norm = HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM @staticmethod def _hc_collapse(x: torch.Tensor, pre_mix: torch.Tensor) -> torch.Tensor: @@ -269,6 +277,11 @@ class DeepseekV4DecoderLayer(nn.Module): engram_hashes: torch.Tensor | None = None, engram_mask: torch.Tensor | None = None, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + # Layer 0's attention seam projects the 2-D embedding with the folded + # hc_attn_fn_broadcast instead of the 4-stream residual with hc_attn_fn. + # The fused kernel only takes the latter, so that seam keeps the + # separate attn_norm. + fuse_attn_norm = self.fuse_seam_norm and not (residual is None and x.dim() == 2) # The reference collapses each sublayer's input with the *previous* # sublayer's pre-mix: attention uses the pre-mix carried in (identity # for the first layer), the FFN uses this layer's attention pre-mix. @@ -303,6 +316,8 @@ class DeepseekV4DecoderLayer(nn.Module): self.hc_post_alpha, self.hc_sinkhorn_iters, pre_mix=pre_mix, + norm_weight=self.attn_norm.weight if self.fuse_seam_norm else None, + norm_eps=self.attn_norm.variance_epsilon, ) else: pre_args = ( @@ -328,7 +343,11 @@ class DeepseekV4DecoderLayer(nn.Module): engram_mask, ) residual, post_mix, res_mix, x, attn_pre = self.mhc_pre_delayed( - residual, *pre_args, pre_mix=pre_mix + residual, + *pre_args, + pre_mix=pre_mix, + norm_weight=self.attn_norm.weight if self.fuse_seam_norm else None, + norm_eps=self.attn_norm.variance_epsilon, ) else: ( @@ -344,8 +363,11 @@ class DeepseekV4DecoderLayer(nn.Module): sublayer_out=x, post_layer_mix=post_mix, comb_res_mix=res_mix, + norm_weight=self.attn_norm.weight if self.fuse_seam_norm else None, + norm_eps=self.attn_norm.variance_epsilon, ) - x = self.attn_norm(x) + if not fuse_attn_norm: + x = self.attn_norm(x) if self.use_sequence_parallel: x = sp_all_gather(x)[: positions.shape[0]] @@ -368,8 +390,11 @@ class DeepseekV4DecoderLayer(nn.Module): sublayer_out=x, post_layer_mix=post_mix, comb_res_mix=res_mix, + norm_weight=self.ffn_norm.weight if self.fuse_seam_norm else None, + norm_eps=self.ffn_norm.variance_epsilon, ) - x = self.ffn_norm(x) + if not self.fuse_seam_norm: + x = self.ffn_norm(x) x = self.ffn(x, input_ids) return x, residual, post_mix, res_mix, ffn_pre diff --git a/vllm/models/deepseek_v41/amd/rocm.py b/vllm/models/deepseek_v41/amd/rocm.py index 9de09bad8c..897b0d12db 100644 --- a/vllm/models/deepseek_v41/amd/rocm.py +++ b/vllm/models/deepseek_v41/amd/rocm.py @@ -38,6 +38,7 @@ from vllm.v1.attention.backends.mla.sparse_swa import ( DeepseekSparseSWAMetadata, ) from vllm.v1.attention.ops.rocm_aiter_mla_sparse import ( + build_prefill_topk_ragged_indices, build_ragged_indices_from_dense, rocm_inv_rope_einsum, rocm_inverse_rope_mxfp8_rows, @@ -64,6 +65,22 @@ def _trust_dsv4_extra_cache_nan_free( ) +def _aiter_sparse_mla_enabled(has_kv_transfer: bool) -> bool: + from vllm._aiter_ops import rocm_aiter_ops + + if not rocm_aiter_ops.is_triton_sparse_mla_enabled(): + return False + # The aiter kernel does not scrub NaNs, so like the trust check above it + # needs a cache only the canonical writer has touched. + if has_kv_transfer: + logger.warning_once( + "VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA is ignored with a KV " + "connector; DeepSeek V4 keeps the Triton sparse attention." + ) + return False + return True + + def _build_indptr_from_lengths(lengths: torch.Tensor) -> torch.Tensor: lengths = lengths.to(dtype=torch.int32).contiguous() indptr = torch.zeros(lengths.shape[0] + 1, dtype=torch.int32, device=lengths.device) @@ -408,6 +425,9 @@ _TopkRagged = tuple[torch.Tensor, torch.Tensor, torch.Tensor] class DeepseekV4ROCMAiterSparseSWAMetadata(DeepseekSparseSWAMetadata): decode_swa_ragged_indices: torch.Tensor | None = None decode_swa_ragged_indptr: torch.Tensor | None = None + # Built only for the aiter prefill, which reads the SWA cache in place. + prefill_swa_ragged_indices: torch.Tensor | None = None + prefill_swa_ragged_indptr: torch.Tensor | None = None class DeepseekV4ROCMAiterSparseSWAMetadataBuilder(DeepseekV41SparseSWAMetadataBuilder): @@ -432,6 +452,9 @@ class DeepseekV4ROCMAiterSparseSWAMetadataBuilder(DeepseekV41SparseSWAMetadataBu dtype=torch.int32, device=self.device, ) + self._use_aiter_sparse_mla = _aiter_sparse_mla_enabled( + self.vllm_config.kv_transfer_config is not None + ) def build( self, @@ -447,6 +470,22 @@ class DeepseekV4ROCMAiterSparseSWAMetadataBuilder(DeepseekV41SparseSWAMetadataBu replay_start=replay_start, ) + prefill_ragged_indices = None + prefill_ragged_indptr = None + if ( + self._use_aiter_sparse_mla + and base.num_prefill_tokens > 0 + and base.prefill_swa_indices is not None + and base.prefill_swa_lens is not None + ): + # Paged-coordinate rows, so every layer's prefill shares one build. + prefill_ragged_indices, prefill_ragged_indptr = ( + build_ragged_indices_from_dense( + base.prefill_swa_indices.reshape(base.num_prefill_tokens, -1), + base.prefill_swa_lens, + ) + ) + ragged_indices = None ragged_indptr = None if ( @@ -473,6 +512,8 @@ class DeepseekV4ROCMAiterSparseSWAMetadataBuilder(DeepseekV41SparseSWAMetadataBu **vars(base), decode_swa_ragged_indices=ragged_indices, decode_swa_ragged_indptr=ragged_indptr, + prefill_swa_ragged_indices=prefill_ragged_indices, + prefill_swa_ragged_indptr=prefill_ragged_indptr, ) @@ -497,6 +538,7 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): backend_cls = DeepseekV4ROCMAiterMLASparseBackend swa_backend_cls = DeepseekV41ROCMAiterSparseSWABackend + _use_aiter_sparse_mla = False def __init__(self, *args, **kwargs): vllm_config = args[0] if args else kwargs["vllm_config"] @@ -518,6 +560,9 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): # same thing. The source owns one cache per compress ratio, refreshed # when it runs; consumers below it read through. self._topk_ragged_cache: dict[int, _TopkRagged] = {} + self._prefill_topk_ragged_cache: dict[ + str, tuple[torch.Tensor, torch.Tensor] + ] = {} self._index_source_prefix: str | None = None if self.compress_ratio > 0: assert self.index_source_layer_id is not None @@ -530,6 +575,7 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): "this rank; PP splits inside a v4.1 index-sharing group " "are not supported." ) + self._use_aiter_sparse_mla = _aiter_sparse_mla_enabled(self._has_kv_transfer) if self.compressor is None and self.indexer is None: # Dense layers have nothing to overlap; keep the base serial path. self.aux_stream_list = None @@ -615,23 +661,6 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): self.aux_stream_list = aux_streams return super()._run_parallel_input_projections(hidden_states) - def _enable_multi_stream_overlap(self) -> bool: - """ROCm multi-stream gates: streams and capture region. - - Dict metadata marks piecewise cudagraph, whose eager breaks rebuild - the attention inputs on the owning stream. Forking side streams - there would rely on runtime HIP event sync, which is unreliable in - this overlap on ROCm (event waits can hang), so multi-stream only - runs where the fork/join becomes static graph edges: inside capture, - or with non-dict metadata (full cudagraph or the profile run), which - has no eager breaks. - """ - attn_metadata = get_forward_context().attn_metadata - return self.aux_stream_list is not None and ( - torch.cuda.is_current_stream_capturing() - or not isinstance(attn_metadata, dict) - ) - def forward( self, positions: torch.Tensor, @@ -641,7 +670,9 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): if ( self.indexer is None or self._prepare_and_attn_fn == self._prepare_and_attn_eager - or not self._enable_multi_stream_overlap() + or not current_platform.enable_multi_stream_overlap( + self.aux_stream_list, get_forward_context().attn_metadata + ) ): # Sequential fallback: no forks outside the capture region, where # HIP event sync is unreliable; MRV1 keeps the input prep in its @@ -930,9 +961,10 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): attn_metadata = forward_context.attn_metadata if attn_metadata is None: - # Warmup dummy run: no real metadata. Reserve the same bf16 - # gather workspace _forward_prefill would; the dequantize / topk - # / sparse_fwd kernels are skipped this step. + # Warmup dummy run: no real metadata. Reserve the workspace + # _forward_prefill would; the dequantize / topk / sparse_fwd kernels + # are skipped this step. The aiter prefill reads the caches in place, + # so it asks only for the bf16 rows of the MXFP8 output. swa_only = self.compress_ratio == 0 N = ( 0 @@ -941,9 +973,14 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): // self.compress_ratio ) M = N + self.window_size + self.max_num_batched_tokens - current_workspace_manager().get_simultaneous( - *self._prefill_workspace_shapes(M, self.max_num_batched_tokens, q) + shapes = self._prefill_workspace_shapes( + M, + self.max_num_batched_tokens, + q, + gather=not self._use_aiter_sparse_mla, ) + if shapes: + current_workspace_manager().get_simultaneous(*shapes) if mxfp8_out is None: output.zero_() else: @@ -1102,6 +1139,38 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): num_decode_tokens=num_decode_tokens, ) + if self._use_aiter_sparse_mla: + assert swa_metadata.decode_swa_ragged_indices is not None + assert swa_metadata.decode_swa_ragged_indptr is not None + out = output + if output_mxfp8 is not None: + # The aiter kernel has no MXFP8 epilogue: write bf16 rows, then + # rotate and quantize them as the decode reduce would have. + out = self._mxfp8_bf16_rows(q.shape[0], q) + assert out is not None + self._aiter_sparse_mla( + q, + out, + self.swa_cache_layer.kv_cache, + swa_metadata.decode_swa_ragged_indices, + swa_metadata.decode_swa_ragged_indptr, + kv_cache, + topk_ragged_indices, + topk_ragged_indptr, + ) + if output_mxfp8 is not None: + rocm_inverse_rope_mxfp8_rows( + out, + positions, + self.rotary_emb.cos_sin_cache, + self.rope_head_dim, + output_mxfp8[0], + output_mxfp8[1], + ) + return q.shape[0] + # The aiter kernel leaves the inverse RoPE to the caller. + return 0 + return rocm_sparse_attn_decode( q=q, kv_cache=kv_cache, @@ -1147,6 +1216,19 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): With ``mxfp8_out`` the rows go to a bf16 workspace first and are then inverse-RoPE'd and quantized into it in one pass. """ + if self._use_aiter_sparse_mla: + self._forward_prefill_aiter( + q=q, + positions=positions, + compressed_k_cache=compressed_k_cache, + swa_k_cache=swa_k_cache, + output=output, + attn_metadata=attn_metadata, + swa_metadata=swa_metadata, + mxfp8_out=mxfp8_out, + ) + return + swa_only = attn_metadata is None num_prefills = swa_metadata.num_prefills @@ -1264,15 +1346,17 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): ) def _prefill_workspace_shapes( - self, M: int, num_prefill_tokens: int, q: torch.Tensor + self, M: int, num_prefill_tokens: int, q: torch.Tensor, gather: bool = True ) -> list[tuple[tuple[int, ...], torch.dtype]]: """The prefill gather buffer, plus bf16 rows for the MXFP8 output. One ``get_simultaneous`` call: separate calls alias the same memory. + ``gather=False`` leaves out the gather buffer, which the aiter kernel, + reading the caches in place, never needs. """ - shapes: list[tuple[tuple[int, ...], torch.dtype]] = [ - ((self.PREFILL_CHUNK_SIZE, M, q.shape[-1]), torch.bfloat16) - ] + shapes: list[tuple[tuple[int, ...], torch.dtype]] = [] + if gather: + shapes.append(((self.PREFILL_CHUNK_SIZE, M, q.shape[-1]), torch.bfloat16)) if _ON_GFX950: shapes.append( ( @@ -1281,3 +1365,136 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): ) ) return shapes + + def _prefill_topk_ragged( + self, + swa_metadata: DeepseekV4ROCMAiterSparseSWAMetadata, + attn_metadata: DeepseekV4FlashMLAMetadata, + compressed_k_cache: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Prefill counterpart of ``_decode_topk_ragged``: layers that share an + index source and a compressed cache reuse one build per step.""" + assert swa_metadata.token_to_req_indices is not None + assert swa_metadata.query_start_loc is not None + assert swa_metadata.seq_lens is not None + assert swa_metadata.is_valid_token is not None + assert self.topk_indices_buffer is not None + assert self._index_source_prefix is not None + assert self.compressed_cache_prefix is not None + + source = self._static_forward_context[self._index_source_prefix] + if source is self: + source._prefill_topk_ragged_cache = {} + cached = source._prefill_topk_ragged_cache.get(self.compressed_cache_prefix) + if cached is not None: + return cached + + num_decode_tokens = swa_metadata.num_decode_tokens + num_prefill_tokens = swa_metadata.num_prefill_tokens + built = build_prefill_topk_ragged_indices( + self.topk_indices_buffer[ + num_decode_tokens : num_decode_tokens + num_prefill_tokens + ], + swa_metadata.token_to_req_indices, + swa_metadata.query_start_loc, + swa_metadata.seq_lens, + swa_metadata.is_valid_token, + attn_metadata.block_table, + block_size=attn_metadata.block_size // self.compress_ratio, + compress_ratio=self.compress_ratio, + num_compressed=(self.max_model_len + self.compress_ratio - 1) + // self.compress_ratio, + token_offset=num_decode_tokens, + num_rows=compressed_k_cache.shape[0] * compressed_k_cache.shape[1], + ) + source._prefill_topk_ragged_cache[self.compressed_cache_prefix] = built + return built + + def _forward_prefill_aiter( + self, + q: torch.Tensor, + positions: torch.Tensor, + compressed_k_cache: torch.Tensor | None, + swa_k_cache: torch.Tensor, + output: torch.Tensor | None, + attn_metadata: DeepseekV4FlashMLAMetadata | None, + swa_metadata: DeepseekV4ROCMAiterSparseSWAMetadata, + mxfp8_out: tuple[torch.Tensor, torch.Tensor] | None = None, + ) -> None: + """Prefill straight off the paged caches, with no bf16 gather. + + With ``mxfp8_out`` the kernel writes bf16 rows to the workspace, which + are then inverse-RoPE'd and quantized into it as ``_forward_prefill`` + does. + """ + num_prefill_tokens = swa_metadata.num_prefill_tokens + assert swa_metadata.prefill_swa_ragged_indices is not None + assert swa_metadata.prefill_swa_ragged_indptr is not None + + topk_indices = None + topk_indptr = None + if attn_metadata is None: + compressed_k_cache = None + else: + assert compressed_k_cache is not None + topk_indices, topk_indptr = self._prefill_topk_ragged( + swa_metadata, attn_metadata, compressed_k_cache + ) + + if mxfp8_out is not None: + output = self._mxfp8_bf16_rows(num_prefill_tokens, q) + assert output is not None + self._aiter_sparse_mla( + q[:num_prefill_tokens], + output[:num_prefill_tokens], + swa_k_cache, + swa_metadata.prefill_swa_ragged_indices, + swa_metadata.prefill_swa_ragged_indptr, + compressed_k_cache, + topk_indices, + topk_indptr, + ) + if mxfp8_out is not None: + rocm_inverse_rope_mxfp8_rows( + output[:num_prefill_tokens], + positions[:num_prefill_tokens], + self.rotary_emb.cos_sin_cache, + self.rope_head_dim, + mxfp8_out[0][:num_prefill_tokens], + mxfp8_out[1][:num_prefill_tokens], + ) + + def _mxfp8_bf16_rows(self, num_tokens: int, q: torch.Tensor) -> torch.Tensor: + """bf16 attention rows for the aiter kernel to write when the layer's + output is MXFP8, taken from the workspace the warmup reserved.""" + shapes = self._prefill_workspace_shapes(0, num_tokens, q, gather=False) + assert shapes, "an MXFP8 attention output is gfx950-only" + return current_workspace_manager().get_simultaneous(*shapes)[0] + + def _aiter_sparse_mla( + self, + q: torch.Tensor, + output: torch.Tensor, + swa_k_cache: torch.Tensor, + swa_indices: torch.Tensor, + swa_indptr: torch.Tensor, + compressed_k_cache: torch.Tensor | None, + topk_indices: torch.Tensor | None, + topk_indptr: torch.Tensor | None, + ) -> None: + from vllm._aiter_ops import rocm_aiter_ops + + # Both fp8 caches are read in place: the SWA window as the main segment, + # the top-k compressed tokens as the extra one. + rocm_aiter_ops.triton_sparse_mla_fwd( + q, + swa_k_cache, + output, + self.scale, + swa_indptr, + swa_indices, + attn_sink=self.attn_sink[: q.shape[1]], + extra_kv_buffer=compressed_k_cache, + extra_kv_indptr=topk_indptr, + extra_kv_indices=topk_indices, + ) diff --git a/vllm/models/kimi_k3/amd/latent_moe_runner.py b/vllm/models/kimi_k3/amd/latent_moe_runner.py index 96e71700cd..dbebfd4182 100644 --- a/vllm/models/kimi_k3/amd/latent_moe_runner.py +++ b/vllm/models/kimi_k3/amd/latent_moe_runner.py @@ -6,6 +6,7 @@ import torch from vllm.distributed import ( get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, tensor_model_parallel_all_reduce, ) from vllm.logger import init_logger @@ -33,7 +34,7 @@ class ROCmLatentMoERunner(MoERunner): transform = self.routed_output_transform up_proj = getattr(transform, "up_proj", None) - tp_size = self.moe_config.tp_size + tp_size = get_tensor_model_parallel_world_size() self._up_proj_shard_size = 0 self._tail_shardable = ( diff --git a/vllm/models/kimi_k3/nvidia/dspark_mla.py b/vllm/models/kimi_k3/nvidia/dspark_mla.py index 0baff884cc..37207d7bc9 100644 --- a/vllm/models/kimi_k3/nvidia/dspark_mla.py +++ b/vllm/models/kimi_k3/nvidia/dspark_mla.py @@ -376,7 +376,12 @@ class K3DSparkModel(nn.Module): self, cache_layers: list[MultiHeadLatentAttention], ) -> bool: - if not hasattr(self, "_layers_share_kv_block_layout"): + key = tuple(cl.kv_cache.data_ptr() for cl in cache_layers) + if getattr(self, "_kv_cache_ptrs_key", None) != key: + assert not torch.cuda.is_current_stream_capturing() + self._kv_cache_ptrs_key = key + if hasattr(self, "_context_cache_ptrs"): + del self._context_cache_ptrs ref_cache = cache_layers[0].kv_cache self._layers_share_kv_block_layout = all( cl.kv_cache.size(1) == ref_cache.size(1) diff --git a/vllm/models/minimax_m3/common/indexer.py b/vllm/models/minimax_m3/common/indexer.py index 2459d4ef85..4e8d6dbd3a 100644 --- a/vllm/models/minimax_m3/common/indexer.py +++ b/vllm/models/minimax_m3/common/indexer.py @@ -502,16 +502,21 @@ def select_indexer_impl_cls( On Blackwell (SM100) with ``topk_blocks == 16`` (the only width fmha_sm100's ``sparse_topk_select`` kernel supports), the fmha_sm100 score + top-k path is used for both bf16 and fp8 index caches. Everything else falls back to the - Triton indexer (bf16 only). + Triton indexer, with fp8 restricted to CUDA platforms that advertise fp8 + support. """ if indexer_kv_dtype in ("mxfp4", "nvfp4"): raise NotImplementedError( f"indexer_kv_dtype={indexer_kv_dtype!r} needs the (not-yet-added) " "CuteDSL indexer impl." ) - is_sm100 = ( - current_platform.is_cuda() and current_platform.is_device_capability_family(100) - ) + is_cuda = current_platform.is_cuda() + is_fp8 = indexer_kv_dtype in ("fp8", "fp8_e4m3") + if is_fp8 and not (is_cuda and current_platform.supports_fp8()): + raise NotImplementedError( + f"indexer_kv_dtype={indexer_kv_dtype!r} requires CUDA fp8 support." + ) + is_sm100 = is_cuda and current_platform.is_device_capability_family(100) use_msa = ( is_sm100 and topk_blocks == 16 @@ -530,7 +535,7 @@ def select_indexer_impl_cls( indexer_kv_dtype, ) return MiniMaxM3IndexerMSAImpl - if indexer_kv_dtype != "bf16": + if indexer_kv_dtype not in ("bf16", "fp8", "fp8_e4m3"): raise NotImplementedError( f"indexer_kv_dtype={indexer_kv_dtype!r} is not supported by the " "Triton indexer impl." diff --git a/vllm/models/minimax_m3/common/ops/index_topk.py b/vllm/models/minimax_m3/common/ops/index_topk.py index ed677fd871..5123dabe90 100644 --- a/vllm/models/minimax_m3/common/ops/index_topk.py +++ b/vllm/models/minimax_m3/common/ops/index_topk.py @@ -123,6 +123,10 @@ def _index_block_score_kernel( order=(1, 0), ) q = tl.load(q_ptrs, boundary_check=(0,), padding_option="zero") + # FP8 has no mixed-dtype dot with bf16/fp32. Leave bf16 and fp32 loads + # in their stored dtype so fp32 pipeline tests keep full precision. + if q.dtype.is_fp8(): + q = q.to(tl.bfloat16) q_start = prefix_len + pid_q * BLOCK_SIZE_Q off_q = tl.arange(0, BLOCK_SIZE_Q) + pid_q * BLOCK_SIZE_Q + prefix_len @@ -146,7 +150,9 @@ def _index_block_score_kernel( + off_k[None, :] * stride_ik_pos + off_d[:, None] * stride_ik_d, ) - qk = tl.dot(q, k) + if k.dtype.is_fp8(): + k = k.to(tl.bfloat16) + qk = tl.dot(q, k, out_dtype=tl.float32) # apply causal mask as needed if q_start < i + BLOCK_SIZE_K: qk = tl.where(off_q[:, None] >= pos[None, :], qk, float("-inf")) @@ -360,6 +366,8 @@ def _decode_index_score_kernel( mask=q_mask[None, :], other=0.0, ) # [D,HQ] + if q.dtype.is_fp8(): + q = q.to(tl.bfloat16) for blk in tl.range(chunk_start_block, chunk_end_block): page = tl.load(bt_row + blk).to(tl.int64) pos = blk * BLOCK_SIZE_K + off_k @@ -373,9 +381,11 @@ def _decode_index_score_kernel( + off_k[:, None] * stride_ik_pos + off_d * stride_ik_d, ) # [N,D] - # fp32 accumulation is required for the fp8 (e4m3) index cache: q/k are - # loaded in their stored dtype (bf16 or e4m3) and the MMA accumulates in - # fp32 so the per-block max score is exact for the fp8 indexer too. + # Upcast only FP8 cache loads. BF16 operands keep FP8 dot + # instructions out of the fallback path; FP32 accumulation + # preserves score accuracy. BF16/FP32 loads stay as stored. + if k.dtype.is_fp8(): + k = k.to(tl.bfloat16) kq = tl.dot(k, q, out_dtype=tl.float32) # [N,HQ] kq = tl.where(pos_mask & q_mask[None, :], kq, float("-inf")) score = tl.max(kq, axis=0) # [HQ] diff --git a/vllm/models/qwen4_exp/amd/model.py b/vllm/models/qwen4_exp/amd/model.py index b320967892..db94a2a7d6 100644 --- a/vllm/models/qwen4_exp/amd/model.py +++ b/vllm/models/qwen4_exp/amd/model.py @@ -14,6 +14,7 @@ from vllm.distributed import get_pp_group from vllm.model_executor.layers.fused_moe.utils import ( is_model_fused_shared_expert_compatible, ) +from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import ( QwenGatedDeltaNetAttention, @@ -897,6 +898,7 @@ class Qwen4ExpForConditionalGeneration( config.vision_config, norm_eps=config.text_config.rms_norm_eps, quant_config=quant_config, + input_norm=build_mm_input_norm(self.model_config), prefix=maybe_prefix(prefix, "visual"), ) diff --git a/vllm/models/qwen4_exp/amd/ple_layer.py b/vllm/models/qwen4_exp/amd/ple_layer.py index 0055821d56..11291ff385 100644 --- a/vllm/models/qwen4_exp/amd/ple_layer.py +++ b/vllm/models/qwen4_exp/amd/ple_layer.py @@ -672,14 +672,9 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase): num_decode_tokens: int, num_prefill_tokens: int, ) -> torch.Tensor: - # ``non_spec_query_start_loc`` covers the non-spec (decode + prefill) - # requests and equals ``query_start_loc`` when spec-decode is inactive. - non_spec_query_start_loc = metadata.non_spec_query_start_loc - if non_spec_query_start_loc is None: + query_start_loc_p = metadata.query_start_loc_p + if query_start_loc_p is None: raise ValueError("query_start_loc is required for prefill short-conv") - query_start_loc_p = ( - non_spec_query_start_loc[-num_prefills - 1 :] - num_decode_tokens - ) # The metadata builder guarantees that the prefill query offsets start # at 0 and end at num_prefill_tokens. Avoid reading those values here, # since doing so would force a device-to-host synchronization. diff --git a/vllm/models/qwen4_exp/common/qsa_cache.py b/vllm/models/qwen4_exp/common/qsa_cache.py index eb0758025f..13228185ec 100644 --- a/vllm/models/qwen4_exp/common/qsa_cache.py +++ b/vllm/models/qwen4_exp/common/qsa_cache.py @@ -825,15 +825,18 @@ class QSAKeyStateCache(_QSAStateCache): ) super().__init__(head_size=storage_head_size, **kwargs) - def bind_kv_cache(self, kv_cache: torch.Tensor) -> None: - super().bind_kv_cache(kv_cache) - qsa_cache = self.kv_cache - self.key_cache = qsa_cache[..., : self.key_head_size] - if self.cache_rope_positions: - position_tail = qsa_cache[..., self.rope_position_offset :] - self.rope_position_cache = position_tail.view(torch.int64) - else: - self.rope_position_cache = None + # Derived on access so `kv_cache` stays the only reference to the bound + # storage: clearing it (e.g. after CUDA graph memory profiling) must free + # the cache, or the freed block stays pinned in the allocator. + @property + def key_cache(self) -> torch.Tensor: + return self.kv_cache[..., : self.key_head_size] + + @property + def rope_position_cache(self) -> torch.Tensor | None: + if not self.cache_rope_positions: + return None + return self.kv_cache[..., self.rope_position_offset :].view(torch.int64) def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec: # Hold the open group's committed keys plus every row a speculative diff --git a/vllm/models/qwen4_exp/nvidia/hyperconnection.py b/vllm/models/qwen4_exp/nvidia/hyperconnection.py index acdba8c781..dcf8daa956 100644 --- a/vllm/models/qwen4_exp/nvidia/hyperconnection.py +++ b/vllm/models/qwen4_exp/nvidia/hyperconnection.py @@ -25,16 +25,19 @@ Typical usage inside a transformer decoder layer:: import torch from torch import nn +import vllm.envs as envs from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, ReplicatedLinear, ) from vllm.model_executor.models.utils import maybe_prefix +from vllm.platforms import current_platform from ..common.hyperconnection import ( GroupedGemmaRMSNorm, HyperConnectionConfig, ) +from .ops.cute_dsl.hc_down_silu import MAX_FUSED_M, hc_down_silu from .ops.hc import ( grouped_gemma_rmsnorm, hc_combine, @@ -58,9 +61,9 @@ class GatedResidual(nn.Module): injection. Weights: the norm owns the grouped GemmaRMSNorm affine; the projections - are vLLM Linear modules (merged replicated linear for down+inject), so - GEMM dispatch (e.g. the low-latency skinny GEMM) applies through the - standard quant_method mechanism. + are vLLM Linear modules (merged replicated linear for down+inject). + Eligible down+inject projections use the fused SiLU GEMM; other projections + use their Linear module's GEMM dispatch. """ def __init__( @@ -105,6 +108,12 @@ class GatedResidual(nn.Module): return_bias=False, disable_tp=True, ) + weight = self.input_mix_weight_down_block_inject.weight + self._use_hc_down_silu = ( + weight.shape[1] % 8 == 0 + and weight.dtype == torch.bfloat16 + and current_platform.has_device_capability(90) + ) else: self.input_mix_weight_down = ReplicatedLinear( self.hyper_hidden_size, @@ -125,6 +134,30 @@ class GatedResidual(nn.Module): return_bias=False, ) + def _down_and_inject( + self, xn: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor | None]: + """Down projection + SiLU; also returns injection logits if combined.""" + if not self.use_combine: + return hc_silu(self.input_mix_weight_down(xn), self.hc_count), None + + use_fused = ( + self._use_hc_down_silu + and not envs.VLLM_BATCH_INVARIANT + and 1 <= xn.shape[0] <= MAX_FUSED_M + ) + if use_fused: + return hc_down_silu( + xn, + self.input_mix_weight_down_block_inject.weight, + self.lora_rank, + self.hc_count, + ) + down_and_injection = self.input_mix_weight_down_block_inject(xn) + split_sizes = [self.lora_rank, self.hc_count, self.pad_size] + lora, injection, _ = down_and_injection.split(split_sizes, dim=-1) + return hc_silu(lora, self.hc_count), injection + def mix( self, hidden_states: torch.Tensor ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]: @@ -135,16 +168,7 @@ class GatedResidual(nn.Module): self.hc_count, ) - if self.use_combine: - # produce injection logits for combine - split_sizes = [self.lora_rank, self.hc_count, self.pad_size] - down_and_injection = self.input_mix_weight_down_block_inject(xn) - lora, injection, _ = down_and_injection.split(split_sizes, dim=-1) - else: - lora = self.input_mix_weight_down(xn) - injection = None - - lora = hc_silu(lora, self.hc_count) + lora, injection = self._down_and_inject(xn) gate = self.input_mix_weight_up(lora) # [M, D] block_input = hc_gate_mix(xn, gate, self.hc_count) @@ -172,16 +196,7 @@ class GatedResidual(nn.Module): self.hc_count, ) - if self.use_combine: - # produce injection logits for combine - split_sizes = [self.lora_rank, self.hc_count, self.pad_size] - down_and_injection = self.input_mix_weight_down_block_inject(xn) - lora, injection, _ = down_and_injection.split(split_sizes, dim=-1) - else: - lora = self.input_mix_weight_down(xn) - injection = None - - lora = hc_silu(lora, self.hc_count) + lora, injection = self._down_and_inject(xn) gate = self.input_mix_weight_up(lora) # [M, D] block_input = hc_gate_mix(xn, gate, self.hc_count) diff --git a/vllm/models/qwen4_exp/nvidia/model.py b/vllm/models/qwen4_exp/nvidia/model.py index 2e38339655..6f35bea130 100644 --- a/vllm/models/qwen4_exp/nvidia/model.py +++ b/vllm/models/qwen4_exp/nvidia/model.py @@ -13,6 +13,7 @@ from vllm.distributed import get_pp_group from vllm.model_executor.layers.fused_moe.utils import ( is_model_fused_shared_expert_compatible, ) +from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import ( QwenGatedDeltaNetAttention, @@ -78,6 +79,7 @@ from vllm.v1.kv_cache_interface import MambaSpec from ..config import ATTENTION_LAYER_TYPES, QSA_LAYER_TYPE, Qwen4ExpConfig from .hyperconnection import GatedResidual, HyperConnectionConfig from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm +from .ops.cute_dsl.hc_down_silu import request_hc_down_silu_warmup from .ple_layer import Qwen4ExpPLELayer from .qsa import Qwen4ExpQSAAttention @@ -686,6 +688,16 @@ class Qwen4ExpForCausalLM( ) self.set_moe_parameters(self.model.layers) enable_qwen4_exp_low_latency_gemm(self, self.model_config.dtype) + if self.model_config.dtype == torch.bfloat16: + # Precompile the fused HC down+SiLU kernels for every CUDA-graph + # capture size in the fused dispatch range, so no CuTe-DSL JIT + # happens during graph capture. + request_hc_down_silu_warmup( + vllm_config.compilation_config.cudagraph_capture_sizes or (), + self.config.hc_lowrank, + self.config.hc_count, + self.config.hidden_size * self.config.hc_count, + ) @staticmethod def get_model_state_cls(): @@ -928,6 +940,7 @@ class Qwen4ExpForConditionalGeneration( config.vision_config, norm_eps=config.text_config.rms_norm_eps, quant_config=quant_config, + input_norm=build_mm_input_norm(self.model_config), prefix=maybe_prefix(prefix, "visual"), ) diff --git a/vllm/models/qwen4_exp/nvidia/ops/cute_dsl/__init__.py b/vllm/models/qwen4_exp/nvidia/ops/cute_dsl/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/vllm/models/qwen4_exp/nvidia/ops/cute_dsl/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm/models/qwen4_exp/nvidia/ops/cute_dsl/_hc_down_silu_fma.py b/vllm/models/qwen4_exp/nvidia/ops/cute_dsl/_hc_down_silu_fma.py new file mode 100644 index 0000000000..d0df347566 --- /dev/null +++ b/vllm/models/qwen4_exp/nvidia/ops/cute_dsl/_hc_down_silu_fma.py @@ -0,0 +1,334 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Adapted from vllm/model_executor/kernels/linear/cute_dsl/_ll_bf16_dotprod.py. + +Fuses the Qwen4Exp mHC SiLU epilogue (SiLU(x/HC) on columns < rank, passthrough +elsewhere, with the production bf16 rounding boundary) into the FMA kernel. +Output is bf16 instead of fp32. +""" + +import cutlass +import cutlass.cute as cute +from cuda.bindings.driver import CUstream +from cutlass import const_expr + + +def _sigmoid_f32(x): + return cute.math.rcp(1.0 + cute.math.exp(-x), approx=True) + + +class HcDownSiluFma: + """BF16 GEMM kernel based on CTA-local FMA dot products, with the mHC SiLU + epilogue fused into the final store. + + Computes C[M, N] = A[M, K] @ B[N, K]^T for bf16 inputs, rounds the fp32 + accumulator to bf16 (production rounding boundary), then applies + SiLU(x/HC) to columns < ``rank`` and passes columns >= ``rank`` through. + Output is bf16 [M, N]. + + See the base class docstring in + vllm/model_executor/kernels/linear/cute_dsl/_ll_bf16_dotprod.py for the + GEMM structure; only the epilogue differs. The compile key is + ``(M, K, threadblock_size, rank, hc)``. + """ + + def __init__( + self, + k: int, + threadblock_size: int = 128, + main_vec_width: int = 8, + tail_vec_width: int = 4, + prefetch_pdl_weights: bool = False, + rank: int = 320, + hc: int = 4, + ): + self.threadblock_size = threadblock_size + self.main_vec_width = main_vec_width + self.tail_vec_width = tail_vec_width + self.prefetch_pdl_weights = prefetch_pdl_weights + self.rank = rank + self.hc = hc + self.num_warps = threadblock_size // cute.arch.WARP_SIZE + self._init_k_tiles(k) + self.main_prefetch_tiles = min(self.main_tiles, 8) + + def _vectorized_elems(self, k_extent: int, vec_width: int) -> int: + vector_tile = vec_width * self.threadblock_size + return (k_extent // vector_tile) * vector_tile + + def _init_k_tiles(self, k: int) -> None: + """Split K into vector loops, scalar rounds, and ragged tail.""" + self.k_main_elems = self._vectorized_elems(k, self.main_vec_width) + self.k_after_main = k - self.k_main_elems + self.k_tail_elems = self._vectorized_elems( + self.k_after_main, self.tail_vec_width + ) + self.k_done_all = self.k_main_elems + self.k_tail_elems + self.scalar_rem = k - self.k_done_all + self.ks_full = self.scalar_rem // self.threadblock_size + self.ks_part = self.scalar_rem % self.threadblock_size + self.k_scalar_full = self.ks_full * self.threadblock_size + self.k_part_offset = self.k_done_all + self.k_scalar_full + self.main_tiles = self.k_main_elems // ( + self.main_vec_width * self.threadblock_size + ) + self.tail_tiles = self.k_tail_elems // ( + self.tail_vec_width * self.threadblock_size + ) + + @cute.jit + def _vector_fma( + self, + acc: cute.Tensor, + tA: cute.Tensor, + tB: cute.Tensor, + M: cutlass.Constexpr, + num_tiles: cutlass.Constexpr, + align_bytes: cutlass.Constexpr, + ): + for tile in cutlass.range_constexpr(num_tiles): + bt = tB[None, tile] + br = cute.make_rmem_tensor_like(bt) + cute.autovec_copy(bt, br) + br_f32 = br.load().to(cutlass.Float32) + + for m in cutlass.range_constexpr(M): + at = tA[m, None, tile] + ar = cute.make_rmem_tensor_like(at) + cute.autovec_copy(at, ar) + vec_width: cutlass.Constexpr = cute.size(ar) + for v in cutlass.range_constexpr(vec_width): + acc[m] = acc[m] + ar[v].to(cutlass.Float32) * br_f32[v] + + @cute.jit + def _vector_fma_prefetched( + self, + acc: cute.Tensor, + tA: cute.Tensor, + tB: cute.Tensor, + rB: cute.Tensor, + M: cutlass.Constexpr, + num_tiles: cutlass.Constexpr, + prefetch_tiles: cutlass.Constexpr, + ): + for tile in cutlass.range_constexpr(num_tiles): + if const_expr(tile < prefetch_tiles): + br_f32 = rB[tile, None].load().to(cutlass.Float32) + else: + bt = tB[None, tile] + br = cute.make_rmem_tensor_like(bt) + cute.autovec_copy(bt, br) + br_f32 = br.load().to(cutlass.Float32) + for m in cutlass.range_constexpr(M): + at = tA[m, None, tile] + ar = cute.make_rmem_tensor_like(at) + cute.autovec_copy(at, ar) + vec_width: cutlass.Constexpr = cute.size(ar) + for v in cutlass.range_constexpr(vec_width): + acc[m] = acc[m] + ar[v].to(cutlass.Float32) * br_f32[v] + + def _make_thread_vector_slice( + self, + gA_vec: cute.Tensor, + gB_vec: cute.Tensor, + tidx: cutlass.Int32, + n_idx: cutlass.Int32, + threadblock_size: cutlass.Constexpr, + ): + # (M/N, K_TILE, K_LANE, K_VEC); tidx selects K_LANE. + tA = cute.logical_divide(gA_vec, (None, (None, threadblock_size))) + tB = cute.logical_divide(gB_vec, (None, (None, threadblock_size))) + return tA[None, (None, (tidx, None))], tB[n_idx, (None, (tidx, None))] + + def _make_k_slice( + self, + gX: cute.Tensor, + k_offset: cutlass.Constexpr, + k_extent: cutlass.Constexpr, + ): + k_layout_extent: cutlass.Constexpr = ( + 1 if const_expr(k_extent == 0) else k_extent + ) + + if const_expr(k_offset == 0): + return cute.local_tile( + gX, (cute.size(gX, mode=[0]), k_layout_extent), (0, 0) + ) + return cute.local_tile( + cute.domain_offset((0, k_offset), gX), + (cute.size(gX, mode=[0]), k_layout_extent), + (0, 0), + ) + + @cute.jit + def __call__( + self, + gA: cute.Tensor, + gB: cute.Tensor, + gC: cute.Tensor, + M: cutlass.Constexpr, + K_dim: cutlass.Constexpr, + N_dim: cutlass.Int32, + stream: CUstream, + ): + self.kernel( + gA, + gB, + gC, + M, + self.main_vec_width, + self.tail_vec_width, + self.threadblock_size, + self.num_warps, + self.k_main_elems, + self.k_tail_elems, + self.k_done_all, + self.ks_full, + self.ks_part, + self.k_scalar_full, + self.k_part_offset, + self.main_tiles, + self.tail_tiles, + self.rank, + self.hc, + ).launch( + grid=[N_dim, 1, 1], + block=[self.threadblock_size, 1, 1], + smem=M * 4 * self.num_warps, + stream=stream, + use_pdl=True, + min_blocks_per_mp=1, + ) + + @cute.kernel + def kernel( + self, + gA: cute.Tensor, + gB: cute.Tensor, + gC: cute.Tensor, + M: cutlass.Constexpr, + main_vec_width: cutlass.Constexpr, + tail_vec_width: cutlass.Constexpr, + threadblock_size: cutlass.Constexpr, + num_warps: cutlass.Constexpr, + k_main_elems: cutlass.Constexpr, + k_tail_elems: cutlass.Constexpr, + k_done_all: cutlass.Constexpr, + ks_full: cutlass.Constexpr, + ks_part: cutlass.Constexpr, + k_scalar_full: cutlass.Constexpr, + k_part_offset: cutlass.Constexpr, + main_tiles: cutlass.Constexpr, + tail_tiles: cutlass.Constexpr, + rank: cutlass.Constexpr, + hc: cutlass.Constexpr, + ): + tidx, _, _ = cute.arch.thread_idx() + n_idx, _, _ = cute.arch.block_idx() + wid = cute.arch.warp_idx() + + # One FP32 accumulator per token. + acc = cute.make_rmem_tensor((M,), cutlass.Float32) + acc.fill(0.0) + + if const_expr(k_main_elems > 0): + gA_main = self._make_k_slice(gA, 0, k_main_elems) + gB_main = self._make_k_slice(gB, 0, k_main_elems) + gA_vec = cute.logical_divide(gA_main, (None, main_vec_width)) + gB_vec = cute.logical_divide(gB_main, (None, main_vec_width)) + tA, tB = self._make_thread_vector_slice( + gA_vec, gB_vec, tidx, n_idx, threadblock_size + ) + if const_expr(self.prefetch_pdl_weights): + prefetch_tiles: cutlass.Constexpr = self.main_prefetch_tiles + prefetched_b = cute.make_rmem_tensor( + cute.make_layout( + (prefetch_tiles, main_vec_width), + stride=(main_vec_width, 1), + ), + cutlass.BFloat16, + ) + for tile in cutlass.range_constexpr(prefetch_tiles): + cute.autovec_copy(tB[None, tile], prefetched_b[tile, None]) + + cute.arch.griddepcontrol_wait() + + # 128-bit vectorized main loop + if const_expr(k_main_elems > 0): + if const_expr(self.prefetch_pdl_weights): + self._vector_fma_prefetched( + acc, + tA, + tB, + prefetched_b, + M, + main_tiles, + prefetch_tiles, + ) + else: + self._vector_fma(acc, tA, tB, M, main_tiles, 16) + + # 64-bit vectorized tail (K remainder after main loop) + if const_expr(k_tail_elems > 0): + gA_tail = self._make_k_slice(gA, k_main_elems, k_tail_elems) + gB_tail = self._make_k_slice(gB, k_main_elems, k_tail_elems) + gA_tail_vec = cute.logical_divide(gA_tail, (None, tail_vec_width)) + gB_tail_vec = cute.logical_divide(gB_tail, (None, tail_vec_width)) + tA_t, tB_t = self._make_thread_vector_slice( + gA_tail_vec, gB_tail_vec, tidx, n_idx, threadblock_size + ) + self._vector_fma(acc, tA_t, tB_t, M, tail_tiles, 8) + + # Full scalar rounds use CuTe width-1 tiles; KS_PART is the ragged tail. + if const_expr(ks_full > 0): + gA_scalar = self._make_k_slice(gA, k_done_all, k_scalar_full) + gB_scalar = self._make_k_slice(gB, k_done_all, k_scalar_full) + gA_scalar_vec = cute.logical_divide(gA_scalar, (None, 1)) + gB_scalar_vec = cute.logical_divide(gB_scalar, (None, 1)) + tA_s, tB_s = self._make_thread_vector_slice( + gA_scalar_vec, gB_scalar_vec, tidx, n_idx, threadblock_size + ) + self._vector_fma(acc, tA_s, tB_s, M, ks_full, 2) + + # Only threads below KS_PART load the ragged tail. + if const_expr(ks_part > 0): + gA_part = self._make_k_slice(gA, k_part_offset, ks_part) + gB_part = self._make_k_slice(gB, k_part_offset, ks_part) + if tidx < ks_part: + bv2 = gB_part[n_idx, tidx].to(cutlass.Float32) + for m in cutlass.range_constexpr(M): + acc[m] = acc[m] + gA_part[m, tidx].to(cutlass.Float32) * bv2 + + # Intra-warp shuffle reduction + for m in cutlass.range_constexpr(M): + acc[m] = cute.arch.warp_reduction_sum(acc[m]) + + # Cross-warp reduction via shared memory + smem_red_layout = cute.make_layout((M, num_warps), stride=(num_warps, 1)) + smem = cutlass.utils.SmemAllocator() + sm = smem.allocate_tensor(cutlass.Float32, smem_red_layout, byte_alignment=16) + with cute.arch.elect_one(): + for m in cutlass.range_constexpr(M): + sm[m, wid] = acc[m] + + # Final reduction, fused mHC SiLU epilogue, and output. + cute.arch.sync_threads() + if tidx == 0: + for m in cutlass.range_constexpr(M): + partials = sm[m, None].load() + total = cutlass.Float32( + partials.reduce( + cute.ReductionOp.ADD, + init_val=cutlass.Float32(0.0), + reduction_profile=0, + ) + ) + # Production rounding boundary: GEMM output is materialized as + # bf16 before _hc_silu_kernel reads it. + x_bf16 = total.to(cutlass.BFloat16) + if n_idx < rank: + z = x_bf16.to(cutlass.Float32) * (1.0 / hc) + gC[m, n_idx] = (z * _sigmoid_f32(z)).to(cutlass.BFloat16) + else: + gC[m, n_idx] = x_bf16 + cute.arch.griddepcontrol_launch_dependents() diff --git a/vllm/models/qwen4_exp/nvidia/ops/cute_dsl/_hc_down_silu_mma.py b/vllm/models/qwen4_exp/nvidia/ops/cute_dsl/_hc_down_silu_mma.py new file mode 100644 index 0000000000..57a3aab145 --- /dev/null +++ b/vllm/models/qwen4_exp/nvidia/ops/cute_dsl/_hc_down_silu_mma.py @@ -0,0 +1,536 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Adapted from vllm/model_executor/kernels/linear/cute_dsl/_ll_bf16_splitk.py. + +Fuses the Qwen4Exp mHC SiLU epilogue (SiLU(x/HC) on columns < rank, passthrough +elsewhere, with the production bf16 rounding boundary) into the split-K kernel, +applied after the cluster reduction where the final fp32 result materializes. +Output is bf16 instead of fp32. +""" + +import math + +import cutlass +import cutlass.cute as cute +from cuda.bindings.driver import CUstream +from cutlass._mlir import ir as _ir +from cutlass._mlir.dialects import llvm as _llvm +from cutlass.cutlass_dsl import dsl_user_op +from cutlass.pipeline import sm90 as pipeline + + +# Map a local smem address to the same peer CTA DSMEM offset. +@dsl_user_op +def set_block_rank(smem_ptr, peer_rank, *, loc=None, ip=None): + dsmem_ptr = cute.arch.map_dsmem_ptr(smem_ptr, peer_rank, loc=loc, ip=ip) + return cutlass.Int32(dsmem_ptr.toint(loc=loc, ip=ip)) + + +# Plain DSMEM store; mbarrier helpers do not model this reduction. +@dsl_user_op +def st_shared_remote_f32(remote_addr, val, *, loc=None, ip=None): + i32 = _ir.IntegerType.get_signless(32) + addr_ir = remote_addr.ir_value(loc=loc, ip=ip) + val_ir = val.ir_value(loc=loc, ip=ip) + _llvm.inline_asm( + i32, + [addr_ir, val_ir], + "st.shared::cluster.f32 [$0], $1; mov.u32 $2, 0;", + "r,f,=r", + has_side_effects=True, # keep the inline store ordered + loc=loc, + ip=ip, + ) + + +def _sigmoid_f32(x): + return cute.math.rcp(1.0 + cute.math.exp(-x), approx=True) + + +class HcDownSiluMma: + """BF16 GEMM kernel based on clustered split-K MMA, with the mHC SiLU + epilogue fused into the final store. + + Computes C[M, N] = A[M, K] @ B[N, K]^T for bf16 inputs, rounds the reduced + fp32 accumulator to bf16 (production rounding boundary), then applies + SiLU(x/HC) to columns < ``rank`` and passes columns >= ``rank`` through. + Output is bf16 [M, N]. + + See the base class docstring in + vllm/model_executor/kernels/linear/cute_dsl/_ll_bf16_splitk.py for the + GEMM structure; only the epilogue differs. The compile key is + ``(split_k, num_stages, tile_n, rank, hc)``. + """ + + def __init__( + self, + ab_dtype=cutlass.BFloat16, + acc_dtype=cutlass.Float32, + tile_n: int = 16, + tile_k: int = 256, + num_stages: int = 2, + num_dma_warps: int = 4, + split_k: int = 8, + rank: int = 320, + hc: int = 4, + ): + self.ab_dtype = ab_dtype + self.acc_dtype = acc_dtype + self.rank = rank + self.hc = hc + self.tile_m = 16 + self.tile_n = tile_n + self.tile_k = tile_k + self.copy_bits = 128 + self.num_stages = num_stages + self.split_k = split_k + self.mma_shape = (16, 8, 16) # mma.sync.aligned.m16n8k16 + self.atom_layout = (1, 1, 1) # one MMA atom per warp + self.num_dma_warps = num_dma_warps + self.num_mma_warps = 4 + self.num_dma_threads = self.num_dma_warps * cute.arch.WARP_SIZE + self.num_mma_threads = self.num_mma_warps * cute.arch.WARP_SIZE + self.num_threads = self.num_dma_threads + self.num_mma_threads + self.num_epilogue_elems = self.tile_m * self.tile_n + self.epilogue_elems_per_thread = self.num_epilogue_elems // self.num_mma_threads + + def _make_smem_layout_AB(self, dtype, copy_bits, smem_tiler): + """Build the staged swizzled SMEM layout for A or B tiles.""" + major_size = min(smem_tiler[1], 64) + # Match swizzle span to contiguous K bytes, capped by CuTe 3-bit swizzle. + swizzle_bits = int(math.log2(major_size * dtype.width // copy_bits)) + swizzle_bits = min(swizzle_bits, 3) + # Tile the swizzled atom across (M_or_N, K, stages). + layout_atom_outer = cute.make_layout((8, major_size), stride=(major_size, 1)) + layout_atom = cute.make_composed_layout( + cute.make_swizzle(swizzle_bits, 3, 3), 0, layout_atom_outer + ) + return cute.tile_to_shape(layout_atom, smem_tiler, (0, 1, 2)) + + def _make_gmem_tiled_copy(self, atom_copy, dtype, copy_bits, num_threads): + """Build the per-thread cp.async vector-copy layout.""" + # Lay threads across K so each lane issues one vector copy. + copy_elems = copy_bits // dtype.width + k_threads = cute.size(self.tile_k) // copy_elems # threads along K + thread_layout = cute.make_layout( + (num_threads // k_threads, k_threads), stride=(k_threads, 1) + ) + value_layout = cute.make_layout((1, copy_elems)) + return cute.make_tiled_copy_tv(atom_copy, thread_layout, value_layout) + + @cute.jit + def _fill_pred(self, pred_flat, coord_tensor, k_tile, dim_limit, K_total): + # Predicate one K tile and keep the stage-broadcast view in sync. + coord_ktile = coord_tensor[None, None, 0, k_tile] + num_vec = pred_flat.shape[0] + num_mn = pred_flat.shape[1] + for v in cutlass.range_constexpr(num_vec): + # pred_flat is (K_VEC, M/N) for one K_TILE. + for j in cutlass.range_constexpr(num_mn): + pred_flat[v, j] = cute.elem_less( + coord_ktile[(0, v), j], (dim_limit, K_total) + ) + + def _make_pred(self, tXcX, k_tile, dim_limit, K_total): + # pred_flat is (K_VEC, M/N); pred is (K_VEC, M/N, STAGE). + num_vec = tXcX.shape[0][1] + num_mn = cute.size(tXcX, mode=[1]) + pred_flat = cute.make_rmem_tensor( + cute.make_layout((num_vec, num_mn), stride=(num_mn, 1)), + cutlass.Boolean, + ) + pred = cute.make_tensor( + pred_flat.iterator, + cute.make_layout( + (num_vec, num_mn, cute.size(tXcX, mode=[2])), + stride=(num_mn, 1, 0), + ), + ) + return pred_flat, pred + + @cute.jit + def __call__( + self, + mA: cute.Tensor, + mB: cute.Tensor, + mC: cute.Tensor, + stream: CUstream, + ): + bM, bN, bK = self.tile_m, self.tile_n, self.tile_k + copy_bits: cutlass.Constexpr = self.copy_bits + sA_layout = self._make_smem_layout_AB( + mA.element_type, copy_bits, (bM, bK, self.num_stages) + ) + sB_layout = self._make_smem_layout_AB( + mB.element_type, copy_bits, (bN, bK, self.num_stages) + ) + + @cute.struct + class SharedStorage: + a: cute.struct.Align[ + cute.struct.MemRange[mA.element_type, cute.cosize(sA_layout)], 16 + ] + b: cute.struct.Align[ + cute.struct.MemRange[mB.element_type, cute.cosize(sB_layout)], 16 + ] + mbar: cute.struct.Align[ + cute.struct.MemRange[cutlass.Int64, self.num_stages * 2], 8 + ] + + atom_g2s = cute.make_copy_atom( + cute.nvgpu.cpasync.CopyG2SOp( + cache_mode=cute.nvgpu.cpasync.LoadCacheMode.GLOBAL + ), + mA.element_type, + num_bits_per_copy=copy_bits, + ) # cp.async GMEM -> SMEM, bypassing L1 + tiled_copy_A = self._make_gmem_tiled_copy( + atom_g2s, mA.element_type, copy_bits, self.num_dma_threads + ) + tiled_copy_B = self._make_gmem_tiled_copy( + atom_g2s, mB.element_type, copy_bits, self.num_dma_threads + ) + op = cute.nvgpu.warp.MmaF16BF16Op(self.ab_dtype, self.acc_dtype, self.mma_shape) + # Repeat the m16n8k16 atom along N to cover the CTA output tile. + perm_mnk = ( + self.atom_layout[0] * self.mma_shape[0], + self.atom_layout[1] * self.mma_shape[1] * (self.tile_n // 8), + self.atom_layout[2] * self.mma_shape[2], + ) + tiled_mma = cute.make_tiled_mma( + op, cute.make_layout(self.atom_layout), permutation_mnk=perm_mnk + ) + tiler_mn = (bM, bN) + grid_m, grid_n = cute.ceil_div(mC.shape, tiler_mn) + self.kernel( + mA, + mB, + mC, + sA_layout, + sB_layout, + tiled_copy_A, + tiled_copy_B, + tiled_mma, + SharedStorage, + self.rank, + self.hc, + ).launch( + grid=[ + cute.size(grid_m), + cute.size(grid_n), + self.split_k, + ], + block=[self.num_threads, 1, 1], + cluster=[ + 1, + 1, + self.split_k, + ], # split-K CTAs form one cluster + stream=stream, + use_pdl=True, + ) + + @cute.kernel + def kernel( + self, + mA, + mB, + mC, + sA_layout: cute.ComposedLayout, + sB_layout: cute.ComposedLayout, + tiled_copy_A: cute.TiledCopy, + tiled_copy_B: cute.TiledCopy, + tiled_mma: cute.TiledMma, + shared_storage: cutlass.Constexpr, + rank: cutlass.Constexpr, + hc: cutlass.Constexpr, + ): + bM, bN, bK = self.tile_m, self.tile_n, self.tile_k + num_stages = self.num_stages + tidx, _, _ = cute.arch.thread_idx() + bid_m, bid_n, bid_z = cute.arch.block_idx() + warp_idx = cute.arch.warp_idx() + lane_id = cute.arch.lane_idx() + num_dma_warps: cutlass.Constexpr = self.num_dma_warps + is_dma_warp = warp_idx < num_dma_warps + dma_tidx = tidx + mma_tidx = tidx - self.num_dma_threads + mma_warp_idx = warp_idx - num_dma_warps + + cta_tiler = (bM, bN, bK) + coord = (bid_m, bid_n, None) # all K tiles + # CTA-local tiles. + gA = cute.local_tile( + mA, tiler=cta_tiler, coord=coord, proj=(1, None, 1) + ) # skip N + gB = cute.local_tile( + mB, tiler=cta_tiler, coord=coord, proj=(None, 1, 1) + ) # skip M + gC = cute.local_tile( + mC, tiler=cta_tiler, coord=coord, proj=(1, 1, None) + ) # skip K + + mcA = cute.make_identity_tensor(mA.layout.shape) + mcB = cute.make_identity_tensor(mB.layout.shape) + mcC = cute.make_identity_tensor(mC.layout.shape) + # Coordinate modes: cA=(M,K,k_tile), cB=(N,K,k_tile), cC=(M,N). + cA = cute.local_tile(mcA, tiler=cta_tiler, coord=coord, proj=(1, None, 1)) + cB = cute.local_tile(mcB, tiler=cta_tiler, coord=coord, proj=(None, 1, 1)) + cC = cute.local_tile(mcC, tiler=cta_tiler, coord=coord, proj=(1, 1, None)) + + # 128-bit cp.async copies require 16-byte aligned GMEM views. + gA = cute.make_tensor(gA.iterator.align(16), gA.layout) + gB = cute.make_tensor(gB.iterator.align(16), gB.layout) + + smem = cutlass.utils.SmemAllocator() + storage_ptr = smem.allocate(shared_storage.size_in_bytes(), byte_alignment=16) # type: ignore[attr-defined] + storage = shared_storage(storage_ptr) # type: ignore[call-arg] + sA = storage.a.get_tensor(sA_layout) + sB = storage.b.get_tensor(sB_layout) + + # Pipeline cp.async producers into MMA consumers. + producer_group = pipeline.CooperativeGroup( + pipeline.Agent.Thread, self.num_dma_threads + ) + consumer_group = pipeline.CooperativeGroup( + pipeline.Agent.Thread, self.num_mma_threads + ) + mainloop_pipeline = pipeline.PipelineCpAsync.create( + barrier_storage=storage.mbar.data_ptr(), + num_stages=num_stages, + producer_group=producer_group, + consumer_group=consumer_group, + ) + + # Round-robin split-K: split z handles tiles z, z+split_k, ... + K_total = cute.size(mA, mode=[1]) + k_tile_count = cute.size(gA, mode=[2]) + k_start = bid_z + num_k_tiles = cute.ceil_div(k_tile_count - k_start, self.split_k) + + if is_dma_warp: + # DMA warps trade registers for copy throughput. + cute.arch.setmaxregister_decrease(40) + thr_A = tiled_copy_A.get_slice(dma_tidx) + thr_B = tiled_copy_B.get_slice(dma_tidx) + tAgA = thr_A.partition_S(gA) + tAsA = thr_A.partition_D(sA) + tBgB = thr_B.partition_S(gB) + tBsB = thr_B.partition_D(sB) + tAcA = thr_A.partition_S(cA) + tBcB = thr_B.partition_S(cB) + + # Build M/K and N/K predicates, broadcast across K-tile copies. + tApA_flat, tApA = self._make_pred(tAcA, k_start, mA.shape[0], K_total) + tBpB_flat, tBpB = self._make_pred(tBcB, k_start, mB.shape[0], K_total) + self._fill_pred(tApA_flat, tAcA, k_start, mA.shape[0], K_total) + self._fill_pred(tBpB_flat, tBcB, k_start, mB.shape[0], K_total) + + producer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Producer, num_stages + ) + + # Prime the first pipeline stage. + mainloop_pipeline.producer_acquire(producer_state) + cute.copy( + tiled_copy_B, + tBgB[None, None, None, k_start], + tBsB[None, None, None, producer_state.index], + pred=tBpB, + ) + cute.arch.griddepcontrol_wait() + cute.copy( + tiled_copy_A, + tAgA[None, None, None, k_start], + tAsA[None, None, None, producer_state.index], + pred=tApA, + ) + mainloop_pipeline.producer_commit(producer_state) + producer_state.advance() + + for k_tile in cutlass.range( + k_start + self.split_k, k_tile_count, self.split_k, unroll=1 + ): + self._fill_pred(tApA_flat, tAcA, k_tile, mA.shape[0], K_total) + self._fill_pred(tBpB_flat, tBcB, k_tile, mB.shape[0], K_total) + mainloop_pipeline.producer_acquire(producer_state) + cute.copy( + tiled_copy_A, + tAgA[None, None, None, k_tile], + tAsA[None, None, None, producer_state.index], + pred=tApA, + ) + cute.copy( + tiled_copy_B, + tBgB[None, None, None, k_tile], + tBsB[None, None, None, producer_state.index], + pred=tBpB, + ) + mainloop_pipeline.producer_commit(producer_state) + producer_state.advance() + + mainloop_pipeline.producer_tail(producer_state) + + else: + # MMA warps with k-phase interleaving + cute.arch.setmaxregister_increase(232) # large MMA fragments + + num_mma_warps: cutlass.Constexpr = self.num_mma_warps + + thr_mma = tiled_mma.get_slice(lane_id) + tCsA = thr_mma.partition_A(sA) + tCsB = thr_mma.partition_B(sB) + tCgC = thr_mma.partition_C(gC) + + tCrA = tiled_mma.make_fragment_A(tCsA[None, None, None, 0]) + tCrB = tiled_mma.make_fragment_B(tCsB[None, None, None, 0]) + tCrC = tiled_mma.make_fragment_C(tCgC) + tCrC.fill(0.0) + + # ldmatrix moves SMEM fragments into MMA registers. + atom_s2r_A = cute.make_copy_atom( + cute.nvgpu.warp.LdMatrix8x8x16bOp(False, 4), mA.element_type + ) # non-transposed, x4 + atom_s2r_B = cute.make_copy_atom( + cute.nvgpu.warp.LdMatrix8x8x16bOp(False, 4), mB.element_type + ) + + # SMEM -> register copy path. + tiled_s2r_A = cute.make_tiled_copy_A(atom_s2r_A, tiled_mma) + tiled_s2r_B = cute.make_tiled_copy_B(atom_s2r_B, tiled_mma) + thr_s2r_A = tiled_s2r_A.get_slice(lane_id) + thr_s2r_B = tiled_s2r_B.get_slice(lane_id) + tCsA_v = thr_s2r_A.partition_S(sA) # views, not copies + tCrA_v = thr_s2r_A.retile(tCrA) + tCsB_v = thr_s2r_B.partition_S(sB) + tCrB_v = thr_s2r_B.retile(tCrB) + # Split the MMA K-fragments across the MMA warps. + tCsA_warp_v = cute.logical_divide(tCsA_v, (None, None, num_mma_warps, None)) + tCsB_warp_v = cute.logical_divide(tCsB_v, (None, None, num_mma_warps, None)) + + num_k_blocks = cute.size(tCrA, mode=[2]) + k_blocks_per_warp: cutlass.Constexpr = num_k_blocks // num_mma_warps + + consumer_state = pipeline.make_pipeline_state( + pipeline.PipelineUserType.Consumer, num_stages + ) + + # Shape-dynamic split-K count, so this stays a runtime range. + for _ in cutlass.range(num_k_tiles, unroll_full=True): + mainloop_pipeline.consumer_wait(consumer_state) + for ki in cutlass.range_constexpr(k_blocks_per_warp): + cute.copy( + tiled_s2r_A, + tCsA_warp_v[ + None, None, (mma_warp_idx, ki), consumer_state.index + ], + tCrA_v[None, None, 0], + ) # ldmatrix + cute.copy( + tiled_s2r_B, + tCsB_warp_v[ + None, None, (mma_warp_idx, ki), consumer_state.index + ], + tCrB_v[None, None, 0], + ) + cute.gemm( + tiled_mma, tCrC, tCrA[None, None, 0], tCrB[None, None, 0], tCrC + ) # mma.sync + mainloop_pipeline.consumer_release(consumer_state) + consumer_state.advance() + + # Cluster reduction epilogue. + # Reduce per-warp accumulators within this CTA. + num_elems: cutlass.Constexpr = self.num_epilogue_elems + elems_per_thread: cutlass.Constexpr = self.epilogue_elems_per_thread + # Map MMA threads to linear CTA output elements. + epilogue_thread_layout = cute.make_layout( + (elems_per_thread, self.num_mma_threads), + stride=(self.num_mma_threads, 1), + ) + epilogue_slots = cute.make_tensor(0, epilogue_thread_layout) + epilogue_slot_coords = cute.make_identity_tensor((bN, bM)) + # Layout: (mma_warp, linear MN element). + smem_red = cute.make_tensor( + cute.arch.alloc_smem( + cutlass.Float32, num_elems * num_mma_warps, alignment=16 + ), + cute.make_layout((num_mma_warps, num_elems), stride=(num_elems, 1)), + ) + smem_warp = cute.make_tensor( + cute.domain_offset((mma_warp_idx, 0), smem_red).iterator, + cute.make_layout((bM, bN), stride=(bN, 1)), + ) + tCsC_partial = thr_mma.partition_C(smem_warp) + cute.autovec_copy(tCrC, tCsC_partial) + cute.arch.sync_threads() + + # Layout: (split-K rank, linear MN element). + partials = cute.make_tensor( + cute.arch.alloc_smem( + cutlass.Float32, num_elems * self.split_k, alignment=16 + ), + cute.make_layout((self.split_k, num_elems), stride=(num_elems, 1)), + ) + cta_rank = cute.arch.block_idx_in_cluster() + + for ei in cutlass.range_constexpr(elems_per_thread): + elem_idx = epilogue_slots[ei, mma_tidx] + local_coord = cute.select(epilogue_slot_coords[elem_idx], mode=[1, 0]) + total = cutlass.Float32(0.0) + if cute.elem_less(local_coord, gC.shape): + total = ( + smem_red[None, elem_idx] + .load() + .reduce( + cute.ReductionOp.ADD, + init_val=cutlass.Float32(0.0), + reduction_profile=0, + ) + ) + partials[cta_rank, elem_idx] = total + + cute.arch.sync_threads() + + # Broadcast this CTA's partials to peer DSMEM. + for ei in cutlass.range_constexpr(elems_per_thread): + elem_idx = epilogue_slots[ei, mma_tidx] + my_slot = cute.domain_offset((cta_rank, elem_idx), partials).iterator + my_val = partials[cta_rank, elem_idx] + for peer in cutlass.range_constexpr(self.split_k): + remote = set_block_rank(my_slot, cutlass.Int32(peer)) + st_shared_remote_f32(remote, my_val) + + cute.arch.cluster_arrive() + cute.arch.cluster_wait() # peer DSMEM stores are now visible + + if mma_tidx == 0: + cute.arch.griddepcontrol_launch_dependents() + cute.arch.sync_threads() + + # Reduce split-K partials, apply the fused mHC SiLU epilogue, and + # write the bf16 global output. + for ei in cutlass.range_constexpr(elems_per_thread): + elem_idx = epilogue_slots[ei, mma_tidx] + local_coord = cute.select(epilogue_slot_coords[elem_idx], mode=[1, 0]) + global_coord = cC[local_coord] + if cute.elem_less(global_coord, mC.shape): + acc = cutlass.Float32( + partials[None, elem_idx] + .load() + .reduce( + cute.ReductionOp.ADD, + init_val=cutlass.Float32(0.0), + reduction_profile=0, + ) + ) + # Production rounding boundary: the GEMM result is + # materialized as bf16 before _hc_silu_kernel reads it. + x_bf16 = acc.to(cutlass.BFloat16) + if global_coord[1] < rank: + z = x_bf16.to(cutlass.Float32) * (1.0 / hc) + gC[local_coord] = (z * _sigmoid_f32(z)).to(cutlass.BFloat16) + else: + gC[local_coord] = x_bf16 + + cute.arch.sync_threads() diff --git a/vllm/models/qwen4_exp/nvidia/ops/cute_dsl/hc_down_silu.py b/vllm/models/qwen4_exp/nvidia/ops/cute_dsl/hc_down_silu.py new file mode 100644 index 0000000000..2495d8c09f --- /dev/null +++ b/vllm/models/qwen4_exp/nvidia/ops/cute_dsl/hc_down_silu.py @@ -0,0 +1,224 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Adapted from vllm/model_executor/kernels/linear/cute_dsl/ll_bf16.py. + +Fused Qwen4Exp mHC down projection + SiLU epilogue, where +columns < lora_rank get ``silu(bf16(acc) / hc_count)``, the hc_count +injection-logit columns pass through as ``bf16(acc)``, and the pad columns +are never computed. Output is bf16 (ll_bf16's fp32-output bonus is given up +to preserve the production rounding boundary). + +The GEMM uses an FMA backend for M <= 4 and a split-K MMA backend beyond, +both with PDL. The model uses its Linear module above M=48. +""" + +import logging +from collections.abc import Iterable +from functools import lru_cache, partial +from typing import Any, Literal + +import torch + +from vllm.platforms import current_platform + +logger = logging.getLogger(__name__) + +# The fused kernel stops winning past M ~ 64 on Qwen3.8-Next-Flash. +MAX_FUSED_M = 48 + +# Split-K configs tuned on SM100f for Qwen3.8-Flash-Next (K=10240, N=324). +_SM100F_TUNED_SPLITK: dict[int, tuple[int, int, int]] = { + **{m: (6, 5, 8) for m in range(5, 17)}, + **{m: (6, 5, 16) for m in range(17, 33)}, + **{m: (6, 4, 16) for m in range(33, MAX_FUSED_M + 1)}, +} + +# FMA: (backend, M, K, threadblock_size); MMA: (backend, split_k, stages, tile_n). +CompileKey = tuple[Literal["fma", "mma"], int, int, int] + + +class HcDownSiluGemm: + """Dispatch/compile cache for the fused mHC down+SiLU GEMM.""" + + def __init__( + self, rank: int, hc: int, k: int, *, prefetch_pdl_weights: bool = False + ) -> None: + self.rank = rank + self.hc = hc + self.k = k + self._prefetch_pdl_weights = prefetch_pdl_weights + self._compiled: dict[CompileKey, Any] = {} + self._warmup_m: set[int] = set() + self._warmup_registered = False + + def dispatch(self, m: int) -> CompileKey: + if m <= 4 or self.k < 2048: + return ("fma", m, self.k, 128) + return ("mma", *_SM100F_TUNED_SPLITK[m]) + + def compile(self, compile_key: CompileKey) -> None: + if compile_key in self._compiled: + return + + import cutlass.cute as cute + from cutlass import BFloat16 + from quack.compile_utils import make_fake_tensor + + N = cute.sym_int() + gemm: Any + extra_args: tuple[int, ...] + if compile_key[0] == "mma": + from ._hc_down_silu_mma import HcDownSiluMma + + _, split_k, num_stages, tile_n = compile_key + M, K = cute.sym_int(), cute.sym_int() + gemm = HcDownSiluMma( + tile_n=tile_n, + num_stages=num_stages, + split_k=split_k, + rank=self.rank, + hc=self.hc, + ) + extra_args = () + options = "--enable-tvm-ffi" + else: + from ._hc_down_silu_fma import HcDownSiluFma + + _, M, K, threadblock_size = compile_key + gemm = HcDownSiluFma( + k=K, + threadblock_size=threadblock_size, + prefetch_pdl_weights=self._prefetch_pdl_weights, + rank=self.rank, + hc=self.hc, + ) + extra_args = (M, K, 1) # runtime N placeholder + options = "--enable-tvm-ffi --ptxas-options -maxrregcount=64" + + hidden_states = make_fake_tensor(BFloat16, (M, K), divisibility=8) + router_weight = make_fake_tensor(BFloat16, (N, K), divisibility=8) + output = make_fake_tensor(BFloat16, (M, N), divisibility=1) + self._compiled[compile_key] = cute.compile( + gemm, + hidden_states, + router_weight, + output, + *extra_args, + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + options=options, + ) + logger.debug("Compiled hc_down_silu: %s", compile_key) + + def request_warmup(self, m_values: Iterable[int]) -> None: + m_values = set(m_values) + if not m_values: + return + self._warmup_m.update(m_values) + if self._warmup_registered: + return + from vllm.model_executor.warmup.cutedsl_warmup import ( + register_cutedsl_warmup_provider, + ) + + register_cutedsl_warmup_provider(self) + self._warmup_registered = True + + def get_cutedsl_warmup_compile_units(self): + from vllm.model_executor.warmup.cutedsl_warmup import CuTeDSLCompileUnit + + return tuple( + CuTeDSLCompileUnit( + name="Qwen4Exp HC down+SiLU GEMM", + key=( + "qwen4-exp-hc-down-silu", + self.rank, + self.hc, + self.k, + self._prefetch_pdl_weights, + compile_key, + ), + compile=partial(self.compile, compile_key), + ) + for compile_key in dict.fromkeys( + self.dispatch(m) for m in sorted(self._warmup_m) + ) + ) + + def __call__( + self, + hidden_states: torch.Tensor, # [M, K] bf16 + router_weight: torch.Tensor, # [N, K] bf16 + ) -> torch.Tensor: # [M, N] bf16 + M = hidden_states.shape[0] + N = router_weight.shape[0] + n_compute = self.rank + self.hc + w_gemm = router_weight[:n_compute] + compile_key = self.dispatch(M) + self.compile(compile_key) + kernel = self._compiled[compile_key] + + output = torch.empty(M, N, dtype=torch.bfloat16, device=hidden_states.device) + out_gemm = output[:, :n_compute] + if compile_key[0] == "mma": + kernel(hidden_states, w_gemm, out_gemm) + else: + kernel(hidden_states, w_gemm, out_gemm, n_compute) + return output + + +@lru_cache +def _get_kernel( + rank: int, hc: int, k: int, prefetch_pdl_weights: bool = False +) -> HcDownSiluGemm: + return HcDownSiluGemm(rank, hc, k, prefetch_pdl_weights=prefetch_pdl_weights) + + +def hc_down_silu( + x: torch.Tensor, + weight: torch.Tensor, + rank: int, + hc: int, +) -> tuple[torch.Tensor, torch.Tensor]: + """Fused Qwen4Exp HC down projection + SiLU. + + Args: + x: Normalized hyper-hidden input, [M, K]. + weight: Merged down+inject weight, [N, K]. + rank: Number of low-rank output columns. + hc: Number of injection-logit output columns. + + Returns: + LoRA activations [M, rank] and injection logits [M, hc]. + + """ + kernel = _get_kernel(rank, hc, weight.shape[1], x.shape[0] == 1) + output = kernel(x, weight) + return output[:, :rank], output[:, rank : rank + hc] + + +def request_hc_down_silu_warmup( + m_values: Iterable[int], rank: int, hc: int, k: int +) -> None: + """Precompile the fused kernels for CUDA-graph capture token counts. + + Args: + m_values: Token counts the model may capture CUDA graphs for; values + outside the fused dispatch range are ignored. + rank: Number of low-rank output columns. + hc: Number of injection-logit output columns. + k: Input feature size. + + """ + if not current_platform.has_device_capability(90) or k % 8 != 0: + return + m_set = {int(m) for m in m_values if 1 <= m <= MAX_FUSED_M} + # M=1 dispatches to the weight-prefetching PDL variant. + _get_kernel(rank, hc, k, True).request_warmup(m_set & {1}) + _get_kernel(rank, hc, k).request_warmup(m_set - {1}) + + +__all__ = [ + "MAX_FUSED_M", + "hc_down_silu", + "request_hc_down_silu_warmup", +] diff --git a/vllm/models/qwen4_exp/nvidia/ple_layer.py b/vllm/models/qwen4_exp/nvidia/ple_layer.py index b04cdaad21..2aac4f0f58 100644 --- a/vllm/models/qwen4_exp/nvidia/ple_layer.py +++ b/vllm/models/qwen4_exp/nvidia/ple_layer.py @@ -292,10 +292,9 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase): token_indices=token_indices_d, ) - query_start_loc = metadata.non_spec_query_start_loc + query_start_loc = metadata.query_start_loc_p if query_start_loc is None: raise ValueError("query_start_loc is required for prefill short-conv") - query_start_loc = query_start_loc[-num_prefills - 1 :] - num_decode_tokens has_initial_states = metadata.has_initial_states_p if has_initial_states is None: raise ValueError( diff --git a/vllm/multimodal/processing/context.py b/vllm/multimodal/processing/context.py index db1cffefd6..46adcd7093 100644 --- a/vllm/multimodal/processing/context.py +++ b/vllm/multimodal/processing/context.py @@ -5,7 +5,7 @@ from collections.abc import Callable, Mapping from contextlib import contextmanager from dataclasses import dataclass, field from functools import cached_property -from typing import TYPE_CHECKING, Any, overload +from typing import TYPE_CHECKING, Any, get_type_hints, overload import torch from typing_extensions import TypeVar @@ -42,46 +42,129 @@ else: logger = init_logger(__name__) -# HuggingFace processors accept nested ``images_kwargs`` / ``videos_kwargs`` / -# ``audio_kwargs`` in addition to a shared flat namespace. vLLM's token-budget -# and dummy-input code only reads the flat keys, so scoped overrides have to -# be overlaid per modality. -_HF_MODALITY_PROCESSOR_KWARGS = { - "image": "images_kwargs", - "video": "videos_kwargs", - "audio": "audio_kwargs", -} +# HuggingFace processors accept a shared flat namespace in addition to the +# nested processor kwarg scopes ``text_kwargs`` / ``images_kwargs`` / +# ``videos_kwargs`` / ``audio_kwargs``. vLLM recognizes these scopes when +# resolving ``mm_processor_kwargs``. +_HF_PROCESSOR_KWARG_SCOPES = ( + "text_kwargs", + "images_kwargs", + "videos_kwargs", + "audio_kwargs", +) -def overlay_modality_mm_kwargs( - kwargs: Mapping[str, object], - modality: str | None, -) -> dict[str, Any]: - """Overlay HF-style nested processor kwargs onto the flat namespace. - - Overlay only the requested modality so a video ``size`` bump does not leak - into the image budget. Flat keys keep their current shared-namespace - behavior when no scoped dict is present. - - Args: - kwargs: Merged multi-modal processor kwargs. - modality: Target modality (``image``, ``video``, or ``audio``). - When ``None``, ``kwargs`` is copied without overlay. - - Returns: - A new dict with the modality-scoped keys overlaid when present. +def _merge_scoped_mm_processor_value( + flat_value: object, + scoped_value: object, +) -> object: + """Merge a flat value with an existing, more specific scoped value. + Mappings are merged recursively; otherwise the scoped value replaces the + flat value. """ - merged = dict(kwargs) - if modality is None: + if isinstance(flat_value, Mapping) and isinstance(scoped_value, Mapping): + # Merge recursively so scoped leaves override matching flat leaves + # without dropping unrelated flat entries. + merged = dict(flat_value) + for key, value in scoped_value.items(): + if key in merged: + merged[key] = _merge_scoped_mm_processor_value(merged[key], value) + else: + merged[key] = value return merged - scoped_key = _HF_MODALITY_PROCESSOR_KWARGS.get(modality) - if scoped_key is None: - return merged - scoped = merged.get(scoped_key) - if not isinstance(scoped, Mapping): - return merged - return merged | dict(scoped) + return scoped_value + + +def _resolve_mm_processor_kwargs( + kwargs: Mapping[str, object], + supported_mm_processor_kwargs: Mapping[str, set[str]] | None = None, +) -> dict[str, object]: + """Resolve flat processor kwargs into HuggingFace processor kwarg scopes. + + With ``supported_mm_processor_kwargs``, each scope lists the flat keys it + supports. Matching flat keys are routed to every scope that supports them, + existing scoped values take precedence, and keys not supported by any scope + remain flat. + + Without ``supported_mm_processor_kwargs``, flat keys already represented in + at least one existing mapping-valued scope are removed from the shared flat + namespace; all other flat keys are left unchanged because there is no + information to determine which scopes should receive them. + """ + resolved = dict(kwargs) + + # Without ``supported_mm_processor_kwargs``, remove flat entries already + # represented in at least one existing mapping-valued scope. There is no + # information to route the remaining flat entries. + if supported_mm_processor_kwargs is None: + # Collect all keys represented in existing mapping-valued scopes. + # Non-mapping scope values do not contain scoped kwargs and are left + # untouched. + represented_scoped_keys: set[str] = set() + for scoped_key in _HF_PROCESSOR_KWARG_SCOPES: + scoped_kwargs = resolved.get(scoped_key) + if isinstance(scoped_kwargs, Mapping): + represented_scoped_keys.update(scoped_kwargs) + + # Remove flat entries for keys already represented in a scope. + for key in represented_scoped_keys: + if key not in _HF_PROCESSOR_KWARG_SCOPES: + resolved.pop(key, None) + return resolved + + # With ``supported_mm_processor_kwargs``, collect the flat kwargs before + # routing each one to the processor kwarg scopes that support it. + flat_kwargs = { + key: value + for key, value in resolved.items() + if key not in _HF_PROCESSOR_KWARG_SCOPES + } + + for key, flat_value in flat_kwargs.items(): + # Find every processor kwarg scope that supports this flat key. If none + # supports it, keep the key in the shared flat namespace. + supported_scopes = [ + scoped_key + for scoped_key in _HF_PROCESSOR_KWARG_SCOPES + if key in supported_mm_processor_kwargs.get(scoped_key, set()) + ] + if not supported_scopes: + continue + + # Add the flat value to each supported destination scope. + for scoped_key in supported_scopes: + # Start from a copy of the existing scoped kwargs, or an empty scope + # if absent. + if scoped_key in resolved: + scoped_kwargs = resolved[scoped_key] + # A scope must be a mapping before keys can be added to it. Do + # not replace an explicit non-mapping value. + if not isinstance(scoped_kwargs, Mapping): + raise TypeError(f"`{scoped_key}` must be a mapping") + scoped_kwargs = dict(scoped_kwargs) + else: + scoped_kwargs = {} + + # Add the flat value to this scope. Existing scoped values take + # precedence. + if key in scoped_kwargs: + scoped_kwargs[key] = _merge_scoped_mm_processor_value( + flat_value, scoped_kwargs[key] + ) + else: + # Assign the flat value to this scope. For mappings, assign a + # copy so the same mapping object is not shared between the flat + # value and multiple scopes. + scoped_kwargs[key] = ( + dict(flat_value) if isinstance(flat_value, Mapping) else flat_value + ) + resolved[scoped_key] = scoped_kwargs + + # After routing to every destination, remove the flat copy. + resolved.pop(key) + + return resolved @dataclass @@ -294,19 +377,27 @@ class InputProcessingContext: self, kwargs: Mapping[str, object], *, - modality: str | None = None, + supported_mm_processor_kwargs: Mapping[str, set[str]] | None = None, ) -> dict[str, Any]: """Merge configured and request ``mm_processor_kwargs``. - When ``modality`` is set, HF-style nested - ``images_kwargs`` / ``videos_kwargs`` / ``audio_kwargs`` are overlaid - onto the flat namespace for vLLM-side reads (token budgets, dummy - inputs). Processor construction and HF ``__call__`` should omit - ``modality`` so the nested dicts still reach the HF processor. + When ``supported_mm_processor_kwargs`` is provided, flat keys are matched + against the keys supported by each HuggingFace processor kwarg scope. + Matching flat keys are routed to every scope that supports them and + removed from the shared flat namespace; keys not supported by any scope + remain flat. + + Without ``supported_mm_processor_kwargs``, flat keys already represented + in at least one existing mapping-valued scope are removed from the shared + flat namespace; all other flat keys are left unchanged because there is + no information to determine which scopes should receive them. """ mm_config = self.model_config.get_multimodal_config() merged = mm_config.merge_mm_processor_kwargs(kwargs) - return overlay_modality_mm_kwargs(merged, modality) + return _resolve_mm_processor_kwargs( + merged, + supported_mm_processor_kwargs=supported_mm_processor_kwargs, + ) def call_hf_processor( self, @@ -380,6 +471,49 @@ class BaseProcessingInfo: """ return self.ctx.get_hf_processor(**kwargs) + def get_supported_mm_processor_kwargs(self) -> dict[str, set[str]]: + """Return supported kwarg names for each HF processor kwargs scope.""" + processor = self.get_hf_processor() + processor_kwargs = get_type_hints(processor.valid_processor_kwargs) + + supported = { + "text_kwargs": set(processor_kwargs["text_kwargs"].__annotations__), + } + if "image" in self.supported_mm_limits: + supported["images_kwargs"] = set( + processor.image_processor.valid_kwargs.__annotations__ + ) + if "video" in self.supported_mm_limits: + supported["videos_kwargs"] = set( + processor.video_processor.valid_kwargs.__annotations__ + ) + if "audio" in self.supported_mm_limits: + supported["audio_kwargs"] = set( + processor_kwargs["audio_kwargs"].__annotations__ + ) + + return supported + + @cached_property + def supported_mm_processor_kwargs(self) -> dict[str, set[str]]: + """Supported kwarg names for each HF processor kwargs scope.""" + return self.get_supported_mm_processor_kwargs() + + def _merge_and_resolve_mm_processor_kwargs( + self, + mm_kwargs: Mapping[str, object], + ) -> dict[str, Any]: + """Merge configured and request ``mm_processor_kwargs``. + + Flat kwargs are routed into the HuggingFace processor kwarg scopes that + support them after the configured/request merge. When a routed flat value + conflicts with an existing scoped value, the scoped value takes precedence. + """ + return self.ctx.get_merged_mm_kwargs( + mm_kwargs, + supported_mm_processor_kwargs=self.supported_mm_processor_kwargs, + ) + def get_default_tok_params(self) -> TokenizeParams: """Construct the default parameters for tokenization.""" from vllm.renderers import TokenizeParams diff --git a/vllm/outputs.py b/vllm/outputs.py index e9a08f7c46..049d795f78 100644 --- a/vllm/outputs.py +++ b/vllm/outputs.py @@ -117,6 +117,10 @@ class RequestOutput: For encoder/decoder models, this is the decoder input prompt token ids. prompt_logprobs: The log probabilities to return per prompt token. + prompt_token_id_logprobs: Logprobs of prompt_logprob_token_ids, shaped + [max(prompt_len - 1 - prompt_logprob_start, 0), + len(prompt_logprob_token_ids)]; row i scores them as predictions of + prompt token prompt_logprob_start + i + 1. outputs: The output sequences of the request. finished: Whether the whole request is finished. metrics: Metrics associated with the request. @@ -148,6 +152,7 @@ class RequestOutput: num_cached_tokens: int | None = None, num_cache_creation_tokens: int | None = None, *, + prompt_token_id_logprobs: np.ndarray | None = None, kv_transfer_params: dict[str, Any] | None = None, ec_transfer_params: dict[str, Any] | None = None, # Forward compatibility, code that uses args added in new release can @@ -162,6 +167,7 @@ class RequestOutput: self.prompt = prompt self.prompt_token_ids = prompt_token_ids self.prompt_logprobs = prompt_logprobs + self.prompt_token_id_logprobs = prompt_token_id_logprobs self.outputs = outputs self.finished = finished self.metrics = metrics @@ -215,6 +221,7 @@ class RequestOutput: f"encoder_prompt={self.encoder_prompt!r}, " f"encoder_prompt_token_ids={self.encoder_prompt_token_ids}, " f"prompt_logprobs={self.prompt_logprobs}, " + f"prompt_token_id_logprobs={self.prompt_token_id_logprobs}, " f"outputs={self.outputs}, " f"finished={self.finished}, " f"metrics={self.metrics}, " diff --git a/vllm/parser/abstract_parser.py b/vllm/parser/abstract_parser.py index 20f22bd0e3..b15d86fd07 100644 --- a/vllm/parser/abstract_parser.py +++ b/vllm/parser/abstract_parser.py @@ -38,6 +38,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import ( from vllm.entrypoints.openai.responses.protocol import ResponsesRequest from vllm.exceptions import VLLMValidationError from vllm.logger import init_logger +from vllm.parser.engine.adapters import ParserEngineToolAdapter from vllm.parser.metrics import record_tool_parser_invocation from vllm.parser.utils import count_history_tool_calls from vllm.reasoning.abs_reasoning_parsers import ReasoningParser @@ -128,7 +129,15 @@ class Parser: tokenizer, *args, model_config=model_config, **kwargs ) if self.__class__.tool_parser_cls is not None: - self._tool_parser = self.__class__.tool_parser_cls(tokenizer, tools) + # Engine-based adapters take the same construction kwargs as + # the reasoning parser (e.g. chat_template_kwargs for per-request + # thinking toggles); legacy ToolParser classes take none. + if issubclass(self.__class__.tool_parser_cls, ParserEngineToolAdapter): + self._tool_parser = self.__class__.tool_parser_cls( + tokenizer, tools, model_config=model_config, **kwargs + ) + else: + self._tool_parser = self.__class__.tool_parser_cls(tokenizer, tools) self._engine_based = ( self._reasoning_parser is None diff --git a/vllm/parser/engine/registered_adapters.py b/vllm/parser/engine/registered_adapters.py index 7353a6016b..7ce123c787 100644 --- a/vllm/parser/engine/registered_adapters.py +++ b/vllm/parser/engine/registered_adapters.py @@ -20,6 +20,7 @@ from vllm.parser.kimi_k2 import KimiK2Parser from vllm.parser.minimax_m2 import MinimaxM2Parser from vllm.parser.mistral import MistralParser from vllm.parser.nemotron_v3 import NemotronV3Parser +from vllm.parser.plamo3 import Plamo3Parser from vllm.parser.qwen3 import Qwen3Parser from vllm.parser.seed_oss import SeedOssParser @@ -92,3 +93,8 @@ from vllm.parser.seed_oss import SeedOssParser MistralParserReasoningAdapter, MistralParserToolAdapter, ) = make_adapters(MistralParser) + +( + Plamo3ParserReasoningAdapter, + Plamo3ParserToolAdapter, +) = make_adapters(Plamo3Parser) diff --git a/vllm/parser/plamo3.py b/vllm/parser/plamo3.py new file mode 100644 index 0000000000..8f37cc9b56 --- /dev/null +++ b/vllm/parser/plamo3.py @@ -0,0 +1,318 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import functools +import json +from dataclasses import replace +from itertools import groupby +from typing import TYPE_CHECKING + +from vllm.entrypoints.generate.base.protocol import ( + DeltaFunctionCall, + DeltaMessage, + DeltaToolCall, +) +from vllm.parser.engine.events import EventType, SemanticEvent +from vllm.parser.engine.parser_engine import ParserEngine +from vllm.parser.engine.parser_engine_config import ( + ParserEngineConfig, + ParserState, + Transition, +) + +if TYPE_CHECKING: + from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionRequest, + ) + from vllm.entrypoints.openai.responses.protocol import ResponsesRequest + from vllm.tokenizers import TokenizerLike + from vllm.tool_parsers.abstract_tool_parser import Tool + +BEGIN_TOOL_REQUESTS = "<|plamo:begin_tool_requests:plamo|>" +END_TOOL_REQUESTS = "<|plamo:end_tool_requests:plamo|>" +BEGIN_TOOL_REQUEST = "<|plamo:begin_tool_request:plamo|>" +END_TOOL_REQUEST = "<|plamo:end_tool_request:plamo|>" +BEGIN_TOOL_NAME = "<|plamo:begin_tool_name:plamo|>" +END_TOOL_NAME = "<|plamo:end_tool_name:plamo|>" +BEGIN_TOOL_ARGUMENTS = ( + "<|plamo:begin_tool_arguments:plamo|><|plamo:constrain|>json<|plamo:msg|>" +) +END_TOOL_ARGUMENTS = "<|plamo:end_tool_arguments:plamo|>" +BEGIN_THINK = "<|plamo:begin_think:plamo|>" +END_THINK = "<|plamo:end_think:plamo|>" +EOT = "<|plamo:tag|>" + +# PLaMo3 control markers are split into multiple tokenizer tokens. These +# atomic pieces must not be treated as unrelated special tokens by the +# parser-engine drop-token handling. +PLAMO_MARKER_TOKENS = frozenset( + { + "<|plamo:begin_", + "<|plamo:end_", + ":plamo|>", + } +) + + +@functools.cache +def plamo3_config(thinking: bool = True) -> ParserEngineConfig: + if thinking: + reasoning_terminals = { + "THINK_START": BEGIN_THINK, + "THINK_END": END_THINK, + } + reasoning_transitions = { + (ParserState.REASONING, "THINK_START"): Transition( + ParserState.REASONING, + ), + (ParserState.CONTENT, "THINK_START"): Transition( + ParserState.REASONING, + (EventType.REASONING_START,), + ), + (ParserState.REASONING, "THINK_END"): Transition( + ParserState.CONTENT, + (EventType.REASONING_END,), + ), + (ParserState.CONTENT, "THINK_END"): Transition( + ParserState.CONTENT, + (), + ), + (ParserState.REASONING, "TOOL_REQUESTS_START"): Transition( + ParserState.TOOL_PREAMBLE, + (EventType.REASONING_END,), + ), + } + initial_state = ParserState.REASONING + else: + reasoning_terminals = {} + reasoning_transitions = {} + initial_state = ParserState.CONTENT + + return ParserEngineConfig( + name="plamo3", + initial_state=initial_state, + terminals={ + **reasoning_terminals, + "TOOL_REQUESTS_START": BEGIN_TOOL_REQUESTS, + "TOOL_REQUESTS_END": END_TOOL_REQUESTS, + "TOOL_REQUEST_START": BEGIN_TOOL_REQUEST, + "TOOL_REQUEST_END": END_TOOL_REQUEST, + "TOOL_NAME_START": BEGIN_TOOL_NAME, + "TOOL_NAME_END": END_TOOL_NAME, + "TOOL_ARGS_START": BEGIN_TOOL_ARGUMENTS, + "TOOL_ARGS_END": END_TOOL_ARGUMENTS, + "EOT": EOT, + }, + transitions={ + **reasoning_transitions, + (ParserState.CONTENT, "TOOL_REQUESTS_START"): Transition( + ParserState.TOOL_PREAMBLE, + ), + (ParserState.TOOL_PREAMBLE, "TOOL_REQUEST_START"): Transition( + ParserState.TOOL_PREAMBLE, + ), + (ParserState.TOOL_PREAMBLE, "TOOL_NAME_START"): Transition( + ParserState.TOOL_NAME, + (EventType.TOOL_CALL_START,), + ), + (ParserState.TOOL_NAME, "TOOL_NAME_END"): Transition( + ParserState.TOOL_PREAMBLE, + ), + (ParserState.TOOL_PREAMBLE, "TOOL_ARGS_START"): Transition( + ParserState.TOOL_ARGS, + ), + (ParserState.TOOL_ARGS, "TOOL_ARGS_END"): Transition( + ParserState.TOOL_BETWEEN, + ), + (ParserState.TOOL_BETWEEN, "TOOL_REQUEST_END"): Transition( + ParserState.TOOL_PREAMBLE, + (EventType.TOOL_CALL_END,), + ), + (ParserState.TOOL_PREAMBLE, "TOOL_REQUESTS_END"): Transition( + ParserState.CONTENT, + ), + # EOT is normally consumed by generation stop handling. Keep it + # in the grammar as well so text-only parsing and explicitly + # supplied stop tokens cannot leak it into user-visible output. + (ParserState.REASONING, "EOT"): Transition( + ParserState.CONTENT, + (EventType.REASONING_END,), + ), + (ParserState.CONTENT, "EOT"): Transition(ParserState.CONTENT), + (ParserState.TOOL_PREAMBLE, "EOT"): Transition(ParserState.CONTENT), + (ParserState.TOOL_NAME, "EOT"): Transition(ParserState.CONTENT), + (ParserState.TOOL_ARGS, "EOT"): Transition(ParserState.CONTENT), + (ParserState.TOOL_BETWEEN, "EOT"): Transition(ParserState.CONTENT), + }, + token_id_terminals={"EOT": EOT, **reasoning_terminals}, + drop_whitespace_only_content_before_tools=True, + stream_arg_deltas=True, + strip_content_whitespace_with_tools=False, + strip_trailing_reasoning_whitespace=False, + preserve_tokens=PLAMO_MARKER_TOKENS, + tool_args_json=True, + ) + + +class Plamo3Parser(ParserEngine): + """PLaMo3 reasoning and tool-call parser backed by ParserEngine.""" + + def __init__( + self, + tokenizer: TokenizerLike, + tools: list[Tool] | None = None, + **kwargs, + ) -> None: + # Configure PLaMo parsing and cache its multi-token reasoning markers. + chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {} + self.thinking_enabled = chat_kwargs.get("enable_thinking", True) + kwargs.setdefault( + "parser_engine_config", + plamo3_config(thinking=self.thinking_enabled), + ) + super().__init__(tokenizer, tools, **kwargs) + + self._reasoning_start_token_id_sequence: list[int] = list( + tokenizer.encode(BEGIN_THINK, add_special_tokens=False) + ) + self._reasoning_end_token_id_sequence: list[int] = list( + tokenizer.encode(END_THINK, add_special_tokens=False) + ) + # Token-ID sequences of every terminal literal, for token-level + # unfinished-marker detection. + self._terminal_token_id_sequences: dict[str, tuple[int, ...]] = { + literal: tuple(tokenizer.encode(literal, add_special_tokens=False)) + for literal in self.parser_engine_config.terminal_literals + } + + def is_reasoning_end(self, input_ids: list[int]) -> bool: + # Detect PLaMo reasoning boundaries that span multiple token IDs. + if not self.thinking_enabled: + return True + + start_ids = self._reasoning_start_token_id_sequence + end_ids = self._reasoning_end_token_id_sequence + for i in range(len(input_ids) - 1, -1, -1): + if input_ids[i : i + len(end_ids)] == end_ids: + return True + if input_ids[i : i + len(start_ids)] == start_ids: + return False + if input_ids[i] == self.vocab.get(EOT): + return False + return False + + def extract_content_ids(self, input_ids: list[int]) -> list[int]: + # Extract content after PLaMo's multi-token reasoning end marker. + if not self.thinking_enabled: + return input_ids + end_ids = self._reasoning_end_token_id_sequence + for i in range(len(input_ids) - len(end_ids), -1, -1): + if input_ids[i : i + len(end_ids)] == end_ids: + return input_ids[i + len(end_ids) :] + return input_ids + + def _strip_unfinished_marker(self, value: str | None) -> str | None: + if not value: + return value + marker_start = value.rfind("<|plamo:") + if marker_start < 0: + return value or None + suffix = value[marker_start:] + # Strip a trailing partial marker at the token-ID level. + suffix_ids = tuple( + self.model_tokenizer.encode(suffix, add_special_tokens=False) + ) + if any( + len(seq) > len(suffix_ids) and seq[: len(suffix_ids)] == suffix_ids + for seq in self._terminal_token_id_sequences.values() + ): + return value[:marker_start] or None + return value or None + + def _coalesce_finished_tool_events( + self, events: list[SemanticEvent] + ) -> list[SemanticEvent]: + combined = [] + for (kind, _), group in groupby( + events, key=lambda event: (event.type, event.tool_index) + ): + chunks = list(group) + if kind not in (EventType.TOOL_NAME, EventType.ARG_VALUE_CHUNK): + combined.extend(chunks) + continue + if kind == EventType.TOOL_NAME and chunks[0].tool_index < 0: + continue + value = "".join(chunk.value for chunk in chunks) + if kind == EventType.TOOL_NAME: + value = self._strip_unfinished_marker(value) or "" + if value: + combined.append(replace(chunks[0], value=value)) + return combined + + def _events_to_delta( + self, + events: list[SemanticEvent], + finished: bool = False, + ) -> DeltaMessage | None: + # Coalesce flushed tool fragments before stripping truncated markers. + if not finished: + return super()._events_to_delta(events) + + events = self._coalesce_finished_tool_events(events) + delta = super()._events_to_delta(events, finished=True) + if delta is None: + return None + + delta.reasoning = self._strip_unfinished_marker(delta.reasoning) + if not self.skip_tool_parsing: + delta.content = self._strip_unfinished_marker(delta.content) + # The flush emits the remaining (never-streamed) tool-call + # arguments; an empty remainder must match non-streaming + # extraction (_extract_args_json), which normalizes it to '{}'. + for tc in delta.tool_calls or []: + if tc.function is not None and not (tc.function.arguments or "").strip(): + tc.function.arguments = "{}" + if delta.reasoning is None and delta.content is None and not delta.tool_calls: + return None + return delta + + def _handle_arg_chunk( + self, + event: SemanticEvent, + deltas: list[DeltaToolCall], + ) -> None: + # Strip truncated marker suffixes and preserve the first argument delta. + idx = event.tool_index + slot = self._tool_slots[idx] + if (marker_pos := event.value.rfind("<|plamo:")) >= 0: + stripped = event.value[:marker_pos] + try: + json.loads(slot.args + stripped) + except ValueError: + pass + else: + event = replace(event, value=stripped) + name_sent_before = slot.name_sent + super()._handle_arg_chunk(event, deltas) + if event.value and not name_sent_before and slot.name_sent: + deltas.append( + DeltaToolCall( + index=idx, + function=DeltaFunctionCall(arguments=event.value), + ) + ) + + def _extract_args_json(self, raw_args: str, func_name: str) -> str: + # Return the raw arguments as JSON + # since PLaMo3 generates function names and arguments separately. + return raw_args.strip() or "{}" + + def extract_reasoning( + self, + model_output: str, + request: ChatCompletionRequest | ResponsesRequest, + ) -> tuple[str | None, str | None]: + reasoning, content = super().extract_reasoning(model_output, request) + return self._strip_unfinished_marker(reasoning), content diff --git a/vllm/platforms/interface.py b/vllm/platforms/interface.py index 752024e2b2..5b053c4c78 100644 --- a/vllm/platforms/interface.py +++ b/vllm/platforms/interface.py @@ -1120,6 +1120,16 @@ class Platform: """ return False + @classmethod + def enable_multi_stream_overlap( + cls, + aux_stream_list: list[torch.cuda.Stream] | None, + attn_metadata: object, + ) -> bool: + """Whether the current platform should enable multi-stream overlap + for the given aux streams and attention metadata.""" + return False + @classmethod def supports_mx(cls) -> bool: """Returns whether the current platform supports MX types.""" diff --git a/vllm/platforms/rocm.py b/vllm/platforms/rocm.py index 726673b29b..487cffa670 100644 --- a/vllm/platforms/rocm.py +++ b/vllm/platforms/rocm.py @@ -1106,6 +1106,27 @@ class RocmPlatform(Platform): def is_navi(cls) -> bool: return "gfx1" in _GCN_ARCH + @classmethod + def enable_multi_stream_overlap( + cls, + aux_stream_list: list[torch.cuda.Stream] | None, + attn_metadata: object, + ) -> bool: + """ROCm multi-stream gates: streams and capture region. + + Dict metadata marks piecewise cudagraph, whose eager breaks rebuild + the attention inputs on the owning stream. Forking side streams + there would rely on runtime HIP event sync, which is unreliable in + this overlap on ROCm (event waits can hang), so multi-stream only + runs where the fork/join becomes static graph edges: inside capture, + or with non-dict metadata (full cudagraph or the profile run), which + has no eager breaks. + """ + return aux_stream_list is not None and ( + torch.cuda.is_current_stream_capturing() + or not isinstance(attn_metadata, dict) + ) + @classmethod def get_static_graph_wrapper_cls(cls) -> str: return "vllm.compilation.cuda_graph.CUDAGraphWrapper" diff --git a/vllm/platforms/xpu.py b/vllm/platforms/xpu.py index d9277fb2c5..b3399bd53e 100644 --- a/vllm/platforms/xpu.py +++ b/vllm/platforms/xpu.py @@ -464,13 +464,16 @@ class XPUPlatform(Platform): ) cache_config.block_size = new_block_size logger.info( - "[XPU]Setting attention block size to %d tokens to ensure multiple of %d, " - "set mamba_page_size_padded to %d bytes accordingly, before was %d bytes.", + "[XPU]Setting attention block size to %d tokens to ensure multiple of %d.", new_block_size, kernel_block_size, - cache_config.mamba_page_size_padded, - original_mamba_page_size_padded, ) + if original_mamba_page_size_padded is not None: + logger.info( + "[XPU]Scaled mamba_page_size_padded from %d to %d bytes accordingly.", + original_mamba_page_size_padded, + cache_config.mamba_page_size_padded, + ) @classmethod def support_hybrid_kv_cache(cls) -> bool: diff --git a/vllm/reasoning/__init__.py b/vllm/reasoning/__init__.py index cad835ff1f..f1ab60b6b7 100644 --- a/vllm/reasoning/__init__.py +++ b/vllm/reasoning/__init__.py @@ -136,6 +136,10 @@ _REASONING_PARSERS_TO_REGISTER = { "olmo3_reasoning_parser", "Olmo3ReasoningParser", ), + "plamo3": ( + "plamo3_engine_reasoning_parser", + "Plamo3ParserReasoningAdapter", + ), "muse_glimmer": ( "muse_glimmer_reasoning_parser", "MuseGlimmerReasoningParser", diff --git a/vllm/reasoning/plamo3_engine_reasoning_parser.py b/vllm/reasoning/plamo3_engine_reasoning_parser.py new file mode 100644 index 0000000000..20037fe08f --- /dev/null +++ b/vllm/reasoning/plamo3_engine_reasoning_parser.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.parser.engine.registered_adapters import Plamo3ParserReasoningAdapter + +__all__ = ["Plamo3ParserReasoningAdapter"] diff --git a/vllm/renderers/hf.py b/vllm/renderers/hf.py index 4ef14aee58..11f562f74f 100644 --- a/vllm/renderers/hf.py +++ b/vllm/renderers/hf.py @@ -277,8 +277,12 @@ def resolve_chat_template( if tools is None: chat_template = _try_get_processor_chat_template( tokenizer, - revision=model_config.revision, - code_revision=model_config.code_revision, + revision=model_config.tokenizer_revision, + code_revision=( + model_config.code_revision + if model_config.tokenizer == model_config.model + else None + ), trust_remote_code=model_config.trust_remote_code, ) if chat_template is not None: diff --git a/vllm/renderers/online_renderer.py b/vllm/renderers/online_renderer.py index be0056a350..5108adfe81 100644 --- a/vllm/renderers/online_renderer.py +++ b/vllm/renderers/online_renderer.py @@ -107,11 +107,35 @@ def _reused_prompt_token_ids(request: Any) -> list[int] | None: Disaggregated serving carries the prefill stage's ids in ``kv_transfer_params`` so the decode stage can skip re-tokenizing. Removing the key keeps the id list out of the engine's sampling metadata. + + Returns None without checking the ids when ``echo`` is set, since echo + needs ``messages`` to be rendered. Otherwise raises VLLMValidationError if + the ids are malformed. """ kv = getattr(request, "kv_transfer_params", None) if not isinstance(kv, dict): return None - return kv.pop("prompt_token_ids", None) or None + ids = kv.pop("prompt_token_ids", None) + if ids is None: + return None + if getattr(request, "echo", False): + logger.debug( + "Ignoring kv_transfer_params['prompt_token_ids']: " + "echo is set, so messages are rendered instead." + ) + return None + # bool is an int subclass, hence the exact type check. + if ( + not isinstance(ids, list) + or not ids + or any(type(x) is not int or x < 0 for x in ids) + ): + raise VLLMValidationError( + "`kv_transfer_params['prompt_token_ids']` must be a non-empty list " + "of non-negative integers.", + parameter="kv_transfer_params.prompt_token_ids", + ) + return ids class OnlineRenderer: diff --git a/vllm/sampling_params.py b/vllm/sampling_params.py index 88a8978d69..15657997ab 100644 --- a/vllm/sampling_params.py +++ b/vllm/sampling_params.py @@ -289,6 +289,12 @@ class SamplingParams( prompt_logprobs: int | None = None """Number of log probabilities to return per prompt token. When set to -1, return all `vocab_size` log probabilities.""" + prompt_logprob_token_ids: list[int] | None = None + """Token IDs to score at each scored causal prompt row, where row i scores + them as predictions of prompt token i + 1. The last prompt row is excluded, + so the result has `max(prompt_len - 1 - prompt_logprob_start, 0)` rows.""" + prompt_logprob_start: int | None = None + """First causal prompt row to score; defaults to the first row.""" logprob_token_ids: list[int] | None = None """Specific token IDs to return logprobs for. More efficient than logprobs=-1 when you only need logprobs for a small set of tokens. @@ -398,6 +404,8 @@ class SamplingParams( min_tokens: int = 0, logprobs: int | None = None, prompt_logprobs: int | None = None, + prompt_logprob_token_ids: list[int] | None = None, + prompt_logprob_start: int | None = None, detokenize: bool = True, skip_special_tokens: bool = True, spaces_between_special_tokens: bool = True, @@ -464,6 +472,8 @@ class SamplingParams( min_tokens=min_tokens, logprobs=logprobs, prompt_logprobs=prompt_logprobs, + prompt_logprob_token_ids=prompt_logprob_token_ids, + prompt_logprob_start=prompt_logprob_start, logprob_token_ids=logprob_token_ids, detokenize=detokenize, skip_special_tokens=skip_special_tokens, @@ -550,7 +560,10 @@ class SamplingParams( # If prefix caching is enabled, # the output of prompt logprobs may less than n_prompt_tokens, # we need to skip reading cache at this request. - self.skip_reading_prefix_cache = self.prompt_logprobs is not None + self.skip_reading_prefix_cache = ( + self.prompt_logprobs is not None + or self.prompt_logprob_token_ids is not None + ) def _verify_args(self) -> None: _verify_num_sequences(self.n, "n") @@ -902,6 +915,56 @@ class SamplingParams( value=num_prompt_logprobs, ) + # Validate prompt_logprob_token_ids. + if self.prompt_logprob_token_ids is not None: + n = len(self.prompt_logprob_token_ids) + if n == 0: + raise VLLMValidationError( + "prompt_logprob_token_ids must not be empty.", + parameter="prompt_logprob_token_ids", + value=n, + ) + if n > max_logprobs: + raise VLLMValidationError( + f"Requested prompt_logprob_token_ids of length {n}, " + f"which is greater than max allowed: {max_logprobs}. " + f"Set max_logprobs (--max-logprobs) to at least {n}.", + parameter="prompt_logprob_token_ids", + value=n, + ) + vocab_size = model_config.get_vocab_size() + invalid_token_ids = [ + token_id + for token_id in self.prompt_logprob_token_ids + if token_id < 0 or token_id >= vocab_size + ] + if invalid_token_ids: + raise VLLMValidationError( + f"token_id(s) {invalid_token_ids} in " + f"prompt_logprob_token_ids contain out-of-vocab token ids. " + f"Vocabulary size: {vocab_size}", + parameter="prompt_logprob_token_ids", + value=invalid_token_ids, + ) + if len(set(self.prompt_logprob_token_ids)) != n: + raise VLLMValidationError( + "prompt_logprob_token_ids must not contain duplicates.", + parameter="prompt_logprob_token_ids", + value=self.prompt_logprob_token_ids, + ) + if self.prompt_logprob_start is not None and self.prompt_logprob_start < 0: + raise VLLMValidationError( + "prompt_logprob_start must be non-negative.", + parameter="prompt_logprob_start", + value=self.prompt_logprob_start, + ) + elif self.prompt_logprob_start is not None: + raise VLLMValidationError( + "prompt_logprob_start requires prompt_logprob_token_ids.", + parameter="prompt_logprob_start", + value=self.prompt_logprob_start, + ) + def _validate_stop_token_ids(self, model_config: ModelConfig) -> None: """Validate stop_token_ids are within vocabulary range.""" if not self.stop_token_ids: @@ -969,6 +1032,10 @@ class SamplingParams( raise ValueError( "trace_decode_token_ids is not supported with prompt_logprobs." ) + if self.prompt_logprob_token_ids is not None: + raise ValueError( + "trace_decode_token_ids is not supported with prompt_logprob_token_ids." + ) if speculative_config is not None: raise ValueError( "trace_decode_token_ids is not supported with speculative decoding." diff --git a/vllm/tool_parsers/__init__.py b/vllm/tool_parsers/__init__.py index 2515009d0e..90a724858a 100644 --- a/vllm/tool_parsers/__init__.py +++ b/vllm/tool_parsers/__init__.py @@ -166,6 +166,10 @@ _TOOL_PARSERS_TO_REGISTER = { "olmo3_tool_parser", "Olmo3PythonicToolParser", ), + "plamo3": ( + "plamo3_engine_tool_parser", + "Plamo3EngineToolParser", + ), "muse_glimmer": ( "muse_glimmer_tool_parser", "MuseGlimmerToolParser", diff --git a/vllm/tool_parsers/plamo3_engine_tool_parser.py b/vllm/tool_parsers/plamo3_engine_tool_parser.py new file mode 100644 index 0000000000..f64999742e --- /dev/null +++ b/vllm/tool_parsers/plamo3_engine_tool_parser.py @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.parser.engine.registered_adapters import Plamo3ParserToolAdapter + + +class Plamo3EngineToolParser(Plamo3ParserToolAdapter): # type: ignore[valid-type, misc] + structural_tag_model = "plamo3" + supports_required_and_named = False diff --git a/vllm/tool_parsers/structural_tag_registry.py b/vllm/tool_parsers/structural_tag_registry.py index f3e5c9ec98..fdb4e4f92a 100644 --- a/vllm/tool_parsers/structural_tag_registry.py +++ b/vllm/tool_parsers/structural_tag_registry.py @@ -73,7 +73,7 @@ XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset( "deepseek_v4_1", } ) -VLLM_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset({"hermes", "hy_v4", "kimi_k3"}) +VLLM_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset({"hermes", "hy_v4", "kimi_k3", "plamo3"}) SUPPORTED_STRUCTURAL_TAG_MODELS = ( XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS | VLLM_BUILTIN_STRUCTURAL_TAG_MODELS ) @@ -407,6 +407,70 @@ def get_minimax_structural_tag( return StructuralTag(format=suffix_tag) +_PLAMO3_BEGIN_TOOL_REQUESTS = "<|plamo:begin_tool_requests:plamo|>" +_PLAMO3_END_TOOL_REQUESTS = "<|plamo:end_tool_requests:plamo|>" +_PLAMO3_BEGIN_TOOL_REQUEST = "<|plamo:begin_tool_request:plamo|>" +_PLAMO3_END_TOOL_REQUEST = "<|plamo:end_tool_request:plamo|>" +_PLAMO3_BEGIN_TOOL_NAME = "<|plamo:begin_tool_name:plamo|>" +_PLAMO3_END_TOOL_NAME = "<|plamo:end_tool_name:plamo|>" +_PLAMO3_BEGIN_TOOL_ARGUMENTS = ( + "<|plamo:begin_tool_arguments:plamo|><|plamo:constrain|>json<|plamo:msg|>" +) +_PLAMO3_END_TOOL_ARGUMENTS = "<|plamo:end_tool_arguments:plamo|>" +_PLAMO3_EOT = "<|plamo:tag|>" + + +@register_vllm_structural_tag("plamo3") +def get_plamo3_structural_tag( + tools: list[FunctionToolParam], + builtin_tools: list[BuiltinToolParam], + tool_choice: SimplifiedToolChoice, + reasoning: bool, + token_suffix: str = "", +) -> StructuralTag: + """Build PLaMo3's explicit tool-request structural tag.""" + del builtin_tools, reasoning, token_suffix + + request_tags = [ + TagFormat( + begin=( + _PLAMO3_BEGIN_TOOL_REQUEST + + _PLAMO3_BEGIN_TOOL_NAME + + tool.function.name + + _PLAMO3_END_TOOL_NAME + + _PLAMO3_BEGIN_TOOL_ARGUMENTS + ), + content=JSONSchemaFormat( + json_schema=get_function_parameters(tool.function) + ), + end=_PLAMO3_END_TOOL_ARGUMENTS + _PLAMO3_END_TOOL_REQUEST, + ) + for tool in tools + ] + requests_tag = TagFormat( + begin=_PLAMO3_BEGIN_TOOL_REQUESTS, + content=TagsWithSeparatorFormat( + tags=request_tags, + separator="", + at_least_one=True, + stop_after_first=tool_choice == "forced", + ), + end=[ + _PLAMO3_END_TOOL_REQUESTS, + _PLAMO3_END_TOOL_REQUESTS + _PLAMO3_EOT, + ], + ) + + if tool_choice == "auto": + return StructuralTag( + format=TriggeredTagsFormat( + triggers=[_PLAMO3_BEGIN_TOOL_REQUESTS], + tags=[requests_tag], + ) + ) + return StructuralTag(format=requests_tag) + + # --------------------------------------------------------------------------- # Kimi K3 (XTML channel format) # --------------------------------------------------------------------------- diff --git a/vllm/utils/flashinfer.py b/vllm/utils/flashinfer.py index abc4d75500..44da20a3ed 100644 --- a/vllm/utils/flashinfer.py +++ b/vllm/utils/flashinfer.py @@ -79,6 +79,23 @@ FLASHINFER_CUBINS_REPOSITORY = os.environ.get( "https://edge.urm.nvidia.com/artifactory/sw-kernelinferencelibrary-public-generic-local/", # noqa: E501 ) +_DEFAULT_CUDA_HOME = "/usr/local/cuda" + + +def _flashinfer_nvcc_path() -> str | None: + """Return the nvcc FlashInfer's JIT would run, or None if it is missing. + + Mirrors ``flashinfer.jit.cpp_ext.get_cuda_path()`` without importing + FlashInfer, whose import initializes CUDA. + """ + cuda_home = os.environ.get("CUDA_HOME") or os.environ.get("CUDA_PATH") + if not cuda_home: + nvcc = shutil.which("nvcc") + cuda_home = ( + os.path.dirname(os.path.dirname(nvcc)) if nvcc else _DEFAULT_CUDA_HOME + ) + return shutil.which(os.path.join(cuda_home, "bin", "nvcc")) + @functools.cache def has_flashinfer_cubin() -> bool: @@ -99,12 +116,15 @@ def has_flashinfer() -> bool: if importlib.util.find_spec("flashinfer") is None: logger.debug_once("FlashInfer unavailable since package was not found") return False - # When not using flashinfer cubin, - # Also check if nvcc is available since it's required to JIT compile flashinfer - if not has_flashinfer_cubin() and shutil.which("nvcc") is None: - logger.debug_once( - "FlashInfer unavailable since nvcc was not found " - "and not using pre-downloaded cubins" + # FlashInfer's JIT runs nvcc and `ninja` (from PATH). + if not has_flashinfer_cubin() and ( + _flashinfer_nvcc_path() is None or shutil.which("ninja") is None + ): + logger.warning_once( + "FlashInfer kernels are disabled: flashinfer-cubin is not installed " + "and nvcc (CUDA_HOME, CUDA_PATH, PATH or /usr/local/cuda) or ninja " + "(PATH) is missing. Set CUDA_HOME to a CUDA toolkit and put ninja on " + "PATH, or run `flashinfer download-kernels`." ) return False return True diff --git a/vllm/utils/torch_utils.py b/vllm/utils/torch_utils.py index 849d0a38f5..b4a13419d5 100644 --- a/vllm/utils/torch_utils.py +++ b/vllm/utils/torch_utils.py @@ -912,10 +912,15 @@ def get_accelerator_view_from_cpu_tensor(cpu_tensor: torch.Tensor) -> torch.Tens if cpu_tensor.numel() == 0: return torch.empty(cpu_tensor.shape, dtype=cpu_tensor.dtype, device="xpu") if not cpu_tensor.is_pinned(): - contiguous_cpu = cpu_tensor.contiguous() - pinned = torch.empty_like(contiguous_cpu, pin_memory=True) - pinned.copy_(contiguous_cpu) - cpu_tensor = pinned + pinned = torch.empty_strided( + cpu_tensor.size(), + cpu_tensor.stride(), + dtype=cpu_tensor.dtype, + device="cpu", + pin_memory=True, + ) + pinned.copy_(cpu_tensor) + return torch.ops._C.get_xpu_view_from_cpu_tensor(pinned) return torch.ops._C.get_xpu_view_from_cpu_tensor(cpu_tensor) elif current_platform.is_cuda_alike(): return torch.ops._C.get_cuda_view_from_cpu_tensor(cpu_tensor) diff --git a/vllm/v1/attention/backend.py b/vllm/v1/attention/backend.py index 5d661634d7..319b9977f4 100644 --- a/vllm/v1/attention/backend.py +++ b/vllm/v1/attention/backend.py @@ -54,6 +54,9 @@ class MultipleOf: def __init__(self, base: int): self.base = base + def __repr__(self) -> str: + return f"MultipleOf({self.base})" + class AttentionBackend(ABC): """Abstract class for attention backends.""" diff --git a/vllm/v1/attention/backends/mamba_attn.py b/vllm/v1/attention/backends/mamba_attn.py index 63f98f1786..c831266109 100644 --- a/vllm/v1/attention/backends/mamba_attn.py +++ b/vllm/v1/attention/backends/mamba_attn.py @@ -97,6 +97,7 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): # Will be disabled if speculative decoding is used supports_update_block_table: bool = True + needs_causal_conv1d_metadata: bool = True def __init__( self, @@ -603,10 +604,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): if num_computed_tokens is None: num_computed_tokens = common_attn_metadata.compute_num_computed_tokens() - query_start_loc_p_cpu = ( - common_attn_metadata.query_start_loc_cpu[-num_prefills - 1 :] - - num_decode_tokens - ) query_start_loc_p = ( common_attn_metadata.query_start_loc[-num_prefills - 1 :] - num_decode_tokens @@ -615,12 +612,17 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): num_computed_tokens[num_reqs - num_prefills : num_reqs] > 0 ) - nums_dict, batch_ptr, token_chunk_offset_ptr = ( - compute_causal_conv1d_metadata( - query_start_loc_p_cpu, - device=common_attn_metadata.query_start_loc.device, + if self.needs_causal_conv1d_metadata: + query_start_loc_p_cpu = ( + common_attn_metadata.query_start_loc_cpu[-num_prefills - 1 :] + - num_decode_tokens + ) + nums_dict, batch_ptr, token_chunk_offset_ptr = ( + compute_causal_conv1d_metadata( + query_start_loc_p_cpu, + device=common_attn_metadata.query_start_loc.device, + ) ) - ) if self.vllm_config.cache_config.mamba_cache_mode == "all": assert num_computed_tokens is not None diff --git a/vllm/v1/attention/backends/mla/rocm_aiter_mla.py b/vllm/v1/attention/backends/mla/rocm_aiter_mla.py index 1ab4df0664..f48dedb1d6 100644 --- a/vllm/v1/attention/backends/mla/rocm_aiter_mla.py +++ b/vllm/v1/attention/backends/mla/rocm_aiter_mla.py @@ -3,11 +3,13 @@ import functools from dataclasses import dataclass +from enum import Enum from pathlib import Path from typing import TYPE_CHECKING, ClassVar, Final import torch +from vllm import envs from vllm._aiter_ops import rocm_aiter_ops from vllm.config import VllmConfig from vllm.config.cache import CacheDType @@ -239,6 +241,78 @@ def _segmented_dcp_verify_supported(dcp_world_size: int, cp_interleave: int) -> ) +# DCP decode routing +# +# With decode context parallelism each rank holds a round-robin shard of the KV +# cache (token t on rank t % dcp_world_size; requires interleave size 1) and +# attends over it with all DCP-gathered query heads. Partial outputs are merged +# across ranks by LSE. Each decode batch takes one route: +# +# * plain: qlen == 1 or a non-causal block. Every query sees every local token. +# * cprr: causal qlen >= _MIN_CPRR_QLEN, gfx950, speculative decoding. AITER's +# round-robin ASM decode rebuilds global positions from g_kv_indptr, cp_rank +# and cp_world_size and applies the causal window in-kernel. Kernels exist only +# at _NATIVE_CPRR_HEADS; other gathered head counts are padded up. +# * segmented: causal qlen > 1 when cprr is not selected or qlen is below +# _MIN_CPRR_QLEN. Each verify token becomes a single-query row whose KV length +# encodes its causal window. +# +# The plain kernel must never serve a causal qlen > 1 block: it applies +# causality on local indices, which is wrong over a round-robin shard. +_NATIVE_CPRR_HEADS: Final = (16, 32, 64, 128) +_MIN_CPRR_QLEN: Final = 3 + + +class _DCPDecodeRoute(Enum): + PLAIN = "plain" + SEGMENTED = "segmented" + CPRR = "cprr" + + +def _asm_dcp_verify_heads(decode_num_heads: int) -> int: + """Head count the cprr asm decode runs at, or 0 if it cannot serve this shape.""" + if decode_num_heads in _NATIVE_CPRR_HEADS: + return decode_num_heads + for nat in _NATIVE_CPRR_HEADS: + if nat >= decode_num_heads: + return nat + return 0 + + +def _select_dcp_decode_route( + supports_segmented: bool, + causal: bool, + max_qo_len: int, + asm_selected: bool, +) -> _DCPDecodeRoute: + """Select one DCP decode route for this batch; see DCP decode routing.""" + if not causal or max_qo_len <= 1: + return _DCPDecodeRoute.PLAIN + if asm_selected and max_qo_len >= _MIN_CPRR_QLEN: + return _DCPDecodeRoute.CPRR + if supports_segmented: + return _DCPDecodeRoute.SEGMENTED + raise RuntimeError( + "ROCM_AITER_MLA DCP multi-token verify requires either segmented " + "MLA or the round-robin asm decode." + ) + + +def _asm_dcp_verify_configured( + dcp_world_size: int, cp_interleave: int, multi_token_decode: bool +) -> bool: + """Whether asm cprr DCP verify is reachable in this process. + + Requires DCP with interleave size 1, speculative decoding, and gfx950 (the + only arch AITER builds cprr kernels for). + """ + if not (dcp_world_size > 1 and cp_interleave == 1 and multi_token_decode): + return False + from vllm.platforms.rocm import on_gfx950 + + return bool(on_gfx950()) + + def _aiter_mla_small_head_mode() -> str: """Small-head (<16) MLA decode kernel selection. @@ -410,8 +484,19 @@ class AiterMLADecodeMetadata(MLACommonDecodeMetadata): max_qo_len: int | None = None # Minimum KV length used by Gluon to choose a safe split count. min_kv_seq_len: int = 1 + # Exactly one DCP decode route is selected by the metadata builder. + dcp_route: _DCPDecodeRoute = _DCPDecodeRoute.PLAIN # Set exactly when this batch routes to segmented DCP verification. dcp_verify: AiterMLADCPVerifyMetadata | None = None + # cprr only: global per-request page indptr and this rank's position. + g_kv_indptr: torch.Tensor | None = None + cp_world_size: int = 1 + cp_rank: int = 0 + # Padded head count cprr runs at; 0 when this batch is not on cprr. + asm_decode_num_heads: int = 0 + # Must equal the split cap the reduce scratch was sized with, or the kernel + # writes past it and faults. + mla_num_kv_splits: int = 0 # Small-head decode uses Gluon (avoids padding to 16). use_gluon_decode: bool = False # Small-head non-DCP multi-token verify uses Gluon's native MTP entry. @@ -460,6 +545,9 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): # Set from the common metadata every build; a batch is causal unless the # drafter says otherwise. _decode_causal: bool = True + # A non-causal draft needs no cross-shard causal window: the ordinary + # mask0 decode returns per-row LSE for the DCP merge. + supports_non_causal_multi_token_dcp: ClassVar[bool] = True @staticmethod def _uniform_padded_mtp_qo_len( @@ -513,15 +601,48 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): parallel_config.decode_context_parallel_size, parallel_config.cp_kv_cache_interleave_size, ) + asm_dcp_verify_config = ( + _asm_dcp_verify_configured( + parallel_config.decode_context_parallel_size, + parallel_config.cp_kv_cache_interleave_size, + multi_token_decode=vllm_config.speculative_config is not None, + ) + and envs.VLLM_ROCM_AITER_MLA_DCP_VERIFY == "asm" + ) super().__init__( kv_cache_spec, layer_names, vllm_config, device, AiterMLAMetadata, - supports_dcp_with_varlen=supports_segmented_dcp_verify, + supports_dcp_with_varlen=( + supports_segmented_dcp_verify or asm_dcp_verify_config + ), ) + self._asm_dcp_verify = False + self._asm_dcp_verify_heads = 0 + # The head-count half of the route decision needs self.num_heads, so only + # the config half is known before super().__init__(). + if asm_dcp_verify_config: + self._asm_dcp_verify_heads = _asm_dcp_verify_heads( + self.num_heads * self.dcp_world_size + ) + if not self._asm_dcp_verify_heads: + raise ValueError( + "VLLM_ROCM_AITER_MLA_DCP_VERIFY=asm, but the round-robin asm " + f"decode has no kernel for {self.num_heads * self.dcp_world_size} " + f"DCP-gathered heads (native counts: {_NATIVE_CPRR_HEADS}). " + "Set VLLM_ROCM_AITER_MLA_DCP_VERIFY=segmented." + ) + self._asm_dcp_verify = True self._supports_segmented_dcp_verify = supports_segmented_dcp_verify + self._mla_max_split_per_batch = 0 + if self._asm_dcp_verify: + # Cap on KV splits per batch; at low batch the KV axis is the only + # parallelism. Buffer sizing and every runtime call must see this value. + self._mla_max_split_per_batch = torch.cuda.get_device_properties( + device + ).multi_processor_count self.compilation_config = vllm_config.compilation_config self.decode_attn_out_dtype = vllm_config.model_config.dtype @@ -576,6 +697,12 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): self._num_attention_heads = AiterMLAHelper.get_actual_mla_num_heads( self._decode_num_heads ) + # the cprr kernel exists only at the native head + # counts, so run (and size the persistent schedule) at the padded count. + # 96 is 16-aligned, so get_actual_mla_num_heads leaves it alone and the + # pad has to be applied here explicitly. + if self._asm_dcp_verify: + self._num_attention_heads = self._asm_dcp_verify_heads kv_cache_dtype_str = getattr(vllm_config.cache_config, "cache_dtype", "auto") if kv_cache_dtype_str in ("fp8", "fp8_e4m3", "fp8_e5m2"): kv_cache_dtype_str = "fp8" @@ -619,6 +746,13 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): kv_dtype, is_sparse=False, fast_mode=True, + # Must match max_split_per_batch in get_mla_metadata_v1; without it + # aiter sizes the reduce scratch from capacity (~9 GiB). + **( + dict(max_split_per_batch=self._mla_max_split_per_batch) + if self._asm_dcp_verify + else {} + ), ) self._mla_work_meta_data = torch.empty( work_meta_data_size, dtype=work_meta_data_type, device=device @@ -664,6 +798,14 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): device, ) + # Full cudagraphs need a stable address for the cprr global page indptr. + # Zero-initialized; element 0 is the indptr base and is never rewritten. + self._g_kv_indptr_buf: torch.Tensor | None = None + if self._asm_dcp_verify: + self._g_kv_indptr_buf = torch.zeros( + max_num_reqs + 1, dtype=torch.int32, device=device + ) + # Persistent buffers for segmented DCP verification. Captured graphs # require stable addresses while row lengths vary between replays. self._dcp_verify_buffers: AiterMLADCPVerifyMetadata | None = None @@ -683,8 +825,11 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): ) if self._supports_segmented_dcp_verify and self._mtp_decode_qlen > 1: - # A DCP rank's shard of the longest sequence bounds every verify - # row, and full graphs need that bound to be constant. + # Allocate even when CPRR is the preferred route: a later + # replay can fall below _MIN_CPRR_QLEN and needs these + # addresses. A DCP rank's shard of the longest sequence + # bounds every verify row, and full graphs need that bound + # to be constant. num_dcp_partitions = ( self.dcp_world_size * self.cp_kv_cache_interleave_size ) @@ -1063,6 +1208,12 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): pad_uniform_mtp = padded_mtp_qo_len > 0 seq_lens_for_kernel = seq_lens_device + # the global lengths need the same dummy-row + # treatment as the local ones below, but only for g_kv_indptr -- the + # metadata field keeps the true values, which is what the segmented + # route wants (it derives its own row lens and gives a dummy row + # length 0, correctly). + g_tot_seq_lens = dcp_tot_seq_lens_device num_kernel_reqs = num_reqs if pad_uniform_mtp: qo_lens_device = ( @@ -1074,6 +1225,19 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): seq_lens_for_kernel, seq_lens_for_kernel.new_full((), max_qo_len), ) + # A dummy row just got a LOCAL length of max_qo_len, so its GLOBAL + # length has to say the same thing or the cprr mask is degenerate: + # global_len 0 puts the row query positions at {-3,-2,-1} while its + # local shard sits at {rank, W+rank, 2W+rank}, so every entry is + # masked and the row softmaxes over nothing. max_qo_len * W is + # exactly the global length whose round-robin shard is max_qo_len + # on every rank. + if g_tot_seq_lens is not None: + g_tot_seq_lens = torch.where( + qo_lens_device > 0, + g_tot_seq_lens, + g_tot_seq_lens.new_full((), max_qo_len * self.dcp_world_size), + ) if self._graph_seq_lens is not None: self._graph_seq_lens[:num_kernel_reqs].copy_( @@ -1106,16 +1270,21 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): causal, self._kv_cache_bytes, ) - use_segmented_dcp_verify = ( - self._supports_segmented_dcp_verify and max_qo_len > 1 - ) + dcp_route = _DCPDecodeRoute.PLAIN + if self.dcp_world_size > 1: + dcp_route = _select_dcp_decode_route( + self._supports_segmented_dcp_verify, + causal, + int(max_qo_len), + self._asm_dcp_verify, + ) # Segmented DCP verify carries its own per-row subpage table, so the # flat per-token view is dead work for it. Leave the buffer alone and # hand the metadata None, so a future reader cannot pick up whatever # the previous batch left behind. paged_kv_indices = None - if not use_segmented_dcp_verify: + if dcp_route is not _DCPDecodeRoute.SEGMENTED: if self.compilation_config.cudagraph_mode.has_full_cudagraphs(): self.paged_kv_indices.fill_(-1) @@ -1169,6 +1338,19 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): else: qo_indptr = query_start_loc_device[: 1 + num_kernel_reqs] + g_kv_indptr = None + if dcp_route is _DCPDecodeRoute.CPRR and g_tot_seq_lens is not None: + assert self._g_kv_indptr_buf is not None + _ngk = g_tot_seq_lens.shape[0] + g_kv_indptr = self._g_kv_indptr_buf[: _ngk + 1] + g_kv_indptr[1:].copy_(g_tot_seq_lens.cumsum(dim=0, dtype=torch.int32)) + # A FULL cudagraph replays at its capture-time request count, so a + # smaller real batch leaves the tail holding the PREVIOUS step's + # cumsum. Flatten it the way paged_kv_indptr is flattened below, so + # padding rows read global_len == 0 and agree with their zero-length + # local shard instead of a stale global length. + self._g_kv_indptr_buf[_ngk + 1 :].fill_(g_kv_indptr[-1]) + has_persistent_metadata = False # Only the asm decode consumes the schedule, so gate on the routing # rather than on num_heads >= 16, which denies it to a padded rank @@ -1178,7 +1360,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): use_persistent_metadata = ( not use_gluon_decode and not use_gluon_verify - and not use_segmented_dcp_verify + and dcp_route is not _DCPDecodeRoute.SEGMENTED # A padded rank has no bf16 persistent kernel past qlen 4 where the # gfx950 fold is absent; the non-persistent entry covers it. fp8 # keeps the schedule -- its fold rejects non-persistent outright. @@ -1218,13 +1400,28 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): uni_qo_len = ( max_qo_len if pad_uniform_mtp or torch.all(qo_len == max_qo_len) else -1 ) + cprr_kwargs: dict = {} + is_causal = causal + if self._asm_dcp_verify: + cprr_kwargs = dict( + max_split_per_batch=self._mla_max_split_per_batch, + intra_batch_mode=False, + ) + if dcp_route is _DCPDecodeRoute.CPRR: + is_causal = False + cprr_kwargs["is_cp_round_robin"] = True + metadata_num_heads = ( + self._asm_dcp_verify_heads + if dcp_route is _DCPDecodeRoute.CPRR + else AiterMLAHelper.get_actual_mla_num_heads(self._decode_num_heads) + ) get_mla_metadata_v1( qo_indptr, paged_kv_indptr, paged_kv_last_page_len, - self._num_attention_heads, + metadata_num_heads, 1, - causal, + is_causal, self._mla_work_meta_data, self._mla_work_info_set, self._mla_work_indptr, @@ -1238,6 +1435,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): fast_mode=True, dtype_q=self._mla_q_dtype, dtype_kv=self._mla_kv_dtype, + **cprr_kwargs, ) has_persistent_metadata = True @@ -1261,7 +1459,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): min_kv_seq_len = int(per_req_len.min().item()) dcp_verify = None - if use_segmented_dcp_verify: + if dcp_route is _DCPDecodeRoute.SEGMENTED: assert dcp_tot_seq_lens_device is not None dcp_verify = self._build_dcp_verify_row_view( int(max_qo_len), @@ -1279,7 +1477,19 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): dcp_tot_seq_lens=dcp_tot_seq_lens_device, max_qo_len=max_qo_len, min_kv_seq_len=min_kv_seq_len, + dcp_route=dcp_route, dcp_verify=dcp_verify, + g_kv_indptr=g_kv_indptr, + cp_world_size=self.dcp_world_size, + cp_rank=self.dcp_rank, + asm_decode_num_heads=( + self._asm_dcp_verify_heads if dcp_route is _DCPDecodeRoute.CPRR else 0 + ), + mla_num_kv_splits=( + self._mla_max_split_per_batch + if dcp_route is _DCPDecodeRoute.CPRR + else 0 + ), use_gluon_decode=use_gluon_decode, use_gluon_verify=use_gluon_verify, attn_out_dtype=self.decode_attn_out_dtype, @@ -2062,8 +2272,10 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]): ) return o, None - verify = decode.dcp_verify - if verify is not None: + dcp_route = decode.dcp_route + if dcp_route is _DCPDecodeRoute.SEGMENTED: + verify = decode.dcp_verify + assert verify is not None if type(q) is tuple: q_nope, q_pe = q else: @@ -2085,11 +2297,6 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]): decode.attn_out_dtype, ) - if self.dcp_world_size > 1 and int(decode.max_qo_len) > 1: - raise RuntimeError( - "ROCM_AITER_MLA DCP multi-token verify requires segmented MLA." - ) - if type(q) is tuple: q = torch.cat(q, dim=-1) @@ -2102,8 +2309,19 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]): "ROCM_AITER_MLA decode expected the DCP-gathered query head count " f"{self._decode_num_heads}, got {q.shape[1]}" ) - mla_padded_q = AiterMLAHelper.get_mla_padded_q(self._decode_num_heads, q) - mla_num_heads = AiterMLAHelper.get_actual_mla_num_heads(self._decode_num_heads) + # cprr has kernels only at native head counts, so pad q up (e.g. 96 -> + # 128). Heads are independent in MLA; the extras are sliced off below. + asm_dcp_heads = decode.asm_decode_num_heads + if asm_dcp_heads > self._decode_num_heads: + mla_num_heads = asm_dcp_heads + mla_padded_q = AiterMLAHelper.get_mla_padded_q( + self._decode_num_heads, q, target_heads=mla_num_heads + ) + else: + mla_padded_q = AiterMLAHelper.get_mla_padded_q(self._decode_num_heads, q) + mla_num_heads = AiterMLAHelper.get_actual_mla_num_heads( + self._decode_num_heads + ) o = torch.empty( B, mla_num_heads, @@ -2137,6 +2355,16 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]): lse = None if self.dcp_world_size > 1: + if dcp_route is _DCPDecodeRoute.CPRR: + # cprr only: global positions for the in-kernel causal window. + assert decode.g_kv_indptr is not None + mla_kwargs["g_kv_indptr"] = decode.g_kv_indptr + mla_kwargs["cp_world_size"] = decode.cp_world_size + mla_kwargs["cp_rank"] = decode.cp_rank + # Must match the schedule get_mla_metadata_v1 was built with. + assert decode.mla_num_kv_splits > 0 + mla_kwargs["num_kv_splits"] = decode.mla_num_kv_splits + mla_kwargs["intra_batch_mode"] = False # The vLLM custom-op wrapper exposes only the in-place output and # drops aiter's final LSE, which the cross-shard merge needs, so go # through aiter's native entry point on the DCP path. @@ -2151,6 +2379,7 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]): decode.max_qo_len, sm_scale=self.scale, return_lse=True, + causal=attn_metadata.causal, **mla_kwargs, ) assert lse is not None, ( @@ -2172,7 +2401,13 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]): **mla_kwargs, ) - output = AiterMLAHelper.get_mla_unpadded_o(self._decode_num_heads, o) - if lse is not None: - lse = AiterMLAHelper.get_mla_unpadded_lse(self._decode_num_heads, lse) + if asm_dcp_heads > self._decode_num_heads: + # Keep a view: a post-kernel copy races the symm-mem a2a combine. + output = o[:, : self._decode_num_heads] + if lse is not None: + lse = lse[:, : self._decode_num_heads, ...] + else: + output = AiterMLAHelper.get_mla_unpadded_o(self._decode_num_heads, o) + if lse is not None: + lse = AiterMLAHelper.get_mla_unpadded_lse(self._decode_num_heads, lse) return output, lse diff --git a/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py b/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py index f6258759d0..c5a672e5d4 100644 --- a/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py +++ b/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py @@ -396,9 +396,11 @@ class ROCMAiterMLASparseMetadataBuilder( attention_context = vllm_config.compilation_config.static_forward_context # Sink decode must use AITER's nonpersistent path. In particular, # gfx942 has no persistent+LSE kernel, and its metadata heuristic - # terminates for HY-V4's TP1 H64 shape. + # terminates for HY-V4's TP1 H64 shape. The Triton sparse MLA kernel + # never reads the persistent metadata either. self._use_persistent_metadata = all( getattr(attention_context[name].impl, "sinks", None) is None + and not getattr(attention_context[name].impl, "use_aiter_sparse_mla", False) for name in layer_names ) # Bounds the KV-split heuristic (see `_sparse_decode_max_split`). @@ -716,6 +718,7 @@ class ROCMAiterMLASparseImpl( is_sparse = True supports_dense_mha_prefill = False supports_dcp = False + use_aiter_sparse_mla = False def __init__( self, @@ -765,6 +768,32 @@ class ROCMAiterMLASparseImpl( (self.q_concat_buffer,) = current_workspace_manager().get_simultaneous( (q_concat_shape, vllm_config.model_config.dtype), ) + self.qk_rope_head_dim: int = mla_args["qk_rope_head_dim"] + + if rocm_aiter_ops.is_triton_sparse_mla_enabled(): + reason = self._aiter_sparse_mla_unsupported_reason(vllm_config) + if reason is None: + self.use_aiter_sparse_mla = True + else: + logger.warning_once( + "VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA is set, but %s; " + "using the default sparse MLA path instead.", + reason, + ) + + def _aiter_sparse_mla_unsupported_reason( + self, vllm_config: VllmConfig + ) -> str | None: + model_dtype = vllm_config.model_config.dtype + if model_dtype != torch.bfloat16: + return f"the model dtype is {model_dtype}, not bfloat16" + cache_dtype = self.kv_cache_dtype + if cache_dtype not in ("auto", "bfloat16", "fp8", "fp8_e4m3"): + return f"kv-cache dtype {cache_dtype!r} is neither bfloat16 nor e4m3 fp8" + # Context parallelism merges each rank's partials with a per-token LSE. + if self.dcp_world_size > 1 or self.pcp_world_size > 1: + return "context parallelism is not supported" + return None def record_logical_topk_ready(self) -> None: # This impl shares the top-k indices buffer via SharedTopkIndicesBuffer @@ -975,6 +1004,43 @@ class ROCMAiterMLASparseImpl( return output, lse + def _forward_mla_aiter( + self, + layer: AttentionLayer, + q: torch.Tensor, # [sq, heads, d_qk], not head-padded + kv_c_and_k_pe_cache: torch.Tensor, + attn_metadata: ROCMAiterMLASparseMetadata, + ) -> torch.Tensor: + """_forward_mla on aiter's Triton sparse MLA kernel. + + Reads the same index stream, but needs no q head padding and no + persistent metadata, and its launch depends on shapes only, so it is + CUDA-graph capturable. + """ + num_actual_tokens = attn_metadata.num_actual_tokens + output = torch.empty( + [q.shape[0], self.num_heads, self.kv_lora_rank], + dtype=attn_metadata.attn_out_dtype, + device=q.device, + ) + rocm_aiter_ops.triton_sparse_mla_fwd( + q[:num_actual_tokens], + kv_c_and_k_pe_cache.view(-1, 1, 1, q.shape[-1]), + output[:num_actual_tokens], + self.scale, + attn_metadata.paged_kv_indptr, + attn_metadata.paged_kv_indices, + kv_lora_rank=self.kv_lora_rank, + qk_rope_head_dim=self.qk_rope_head_dim, + q_scale=layer._q_scale, + kv_scale=layer._k_scale, + attn_sink=self.sinks, + # triton_convert_req_index_to_global_index writes 0, never -1, + # for an invalid top-k entry, so no slot in the stream is negative. + has_invalid=False, + ) + return output + def forward_mqa( self, q: torch.Tensor | tuple[torch.Tensor, torch.Tensor], @@ -1026,6 +1092,11 @@ class ROCMAiterMLASparseImpl( original_q_shape = q.shape q, _ = ops.scaled_fp8_quant(q.view(q.shape[0], -1), layer._q_scale) q = q.view(original_q_shape) + if self.use_aiter_sparse_mla: + output = self._forward_mla_aiter( + layer, q, kv_c_and_k_pe_cache, attn_metadata + ) + return output, None mla_padded_q = AiterMLAHelper.get_mla_padded_q(self.num_heads, q) return self._forward_mla( layer, mla_padded_q, kv_c_and_k_pe_cache, attn_metadata diff --git a/vllm/v1/attention/backends/rocm_aiter_unified_attn.py b/vllm/v1/attention/backends/rocm_aiter_unified_attn.py index 6aa456e22c..d57b420cd7 100644 --- a/vllm/v1/attention/backends/rocm_aiter_unified_attn.py +++ b/vllm/v1/attention/backends/rocm_aiter_unified_attn.py @@ -83,7 +83,8 @@ class RocmAiterUnifiedAttentionBackend(RocmAttentionBackend): @classmethod def supports_mm_prefix(cls) -> bool: - return True + # Not implemented + return False @classmethod def supports_sink(cls) -> bool: diff --git a/vllm/v1/attention/backends/rocm_attn.py b/vllm/v1/attention/backends/rocm_attn.py index 76382aa75e..453f49b468 100644 --- a/vllm/v1/attention/backends/rocm_attn.py +++ b/vllm/v1/attention/backends/rocm_attn.py @@ -193,7 +193,8 @@ class RocmAttentionBackend(AttentionBackend): @classmethod def supports_mm_prefix(cls) -> bool: - return True + # Not implemented + return False @classmethod def supports_sink(cls) -> bool: diff --git a/vllm/v1/attention/backends/short_conv_attn.py b/vllm/v1/attention/backends/short_conv_attn.py index 5a0c6f7616..f2cc3dae99 100644 --- a/vllm/v1/attention/backends/short_conv_attn.py +++ b/vllm/v1/attention/backends/short_conv_attn.py @@ -18,7 +18,6 @@ from vllm.v1.attention.backends.mamba_attn import ( ) from vllm.v1.attention.backends.utils import ( NULL_BLOCK_ID, - compute_causal_conv1d_metadata, mamba_get_block_table_tensor, ) from vllm.v1.kv_cache_interface import MambaSpec @@ -108,6 +107,7 @@ class PleShortConvAttentionMetadataBuilder(ShortConvAttentionMetadataBuilder): _cudagraph_support = AttentionCGSupport.UNIFORM_BATCH reorder_batch_threshold: int = 1 supports_update_block_table = False + needs_causal_conv1d_metadata = False def __init__( self, @@ -281,18 +281,10 @@ class PleShortConvAttentionMetadataBuilder(ShortConvAttentionMetadataBuilder): spec_sequence_masks_cpu, device=query_start_loc.device ) - # For causal_conv1d (non-spec prefill Triton kernel metadata). - nums_dict = None - batch_ptr = None - token_chunk_offset_ptr = None has_initial_states_p = None has_initial_states_d = None num_computed_tokens_p = None - # Original request indices of the non-spec requests, ordered - # [decodes, prefills]. Used to gather per-request data consistently. - non_spec_req_idx_cpu: torch.Tensor | None = None - query_lens = torch.diff(query_start_loc) # Per-request classification by mask, NOT by position. With # spec-decode, the front decode group can contain both spec-decode # requests and plain non-spec single-token decodes. Spec requests are @@ -317,37 +309,32 @@ class PleShortConvAttentionMetadataBuilder(ShortConvAttentionMetadataBuilder): else 0 ) - # Original request indices grouped as - # [spec | non-spec decode | non-spec prefill]; each group keeps the - # original (already reordered) relative order via a stable nonzero. - spec_req_idx_cpu = spec_sequence_masks_cpu.nonzero(as_tuple=True)[0] - decode_req_idx_cpu = decode_mask_cpu.nonzero(as_tuple=True)[0] - prefill_req_idx_cpu = prefill_mask_cpu.nonzero(as_tuple=True)[0] - non_spec_req_idx_cpu = torch.cat((decode_req_idx_cpu, prefill_req_idx_cpu)) - spec_req_idx = async_tensor_h2d(spec_req_idx_cpu, device=query_start_loc.device) + assert num_accepted_tokens is not None non_spec_req_idx: torch.Tensor | None = None if num_decodes == 0 and num_prefills == 0: - # Pure speculative-decode batch: all real tokens are spec tokens. - spec_token_indx = torch.arange( - num_spec_decode_tokens, - dtype=torch.int32, - device=query_start_loc.device, - ) - non_spec_token_indx = torch.empty( - 0, dtype=torch.int32, device=query_start_loc.device - ) - spec_state_indices_tensor = block_table_tensor[spec_req_idx, 0] + # All real requests are speculative; zero-length padding is trailing. + spec_token_indx = None + non_spec_token_indx = None + spec_state_indices_tensor = block_table_tensor[:num_spec_decodes, 0] + num_accepted_tokens = num_accepted_tokens[:num_spec_decodes] non_spec_state_indices_tensor = None spec_query_start_loc = query_start_loc[: num_spec_decodes + 1] non_spec_query_start_loc = None - non_spec_query_start_loc_cpu = None else: # Mixed batch: build a per-token group key consistent with the # request grouping above (spec=0 | decode=1 | prefill=2) and a # stable sort, so tokens of each request stay contiguous and in # request order. This yields spec tokens first, then the non-spec # [decode, prefill] tokens. + query_lens = torch.diff(query_start_loc) + spec_req_idx_cpu = spec_sequence_masks_cpu.nonzero(as_tuple=True)[0] + decode_req_idx_cpu = decode_mask_cpu.nonzero(as_tuple=True)[0] + prefill_req_idx_cpu = prefill_mask_cpu.nonzero(as_tuple=True)[0] + non_spec_req_idx_cpu = torch.cat((decode_req_idx_cpu, prefill_req_idx_cpu)) + spec_req_idx = async_tensor_h2d( + spec_req_idx_cpu, device=query_start_loc.device + ) non_spec_req_idx = async_tensor_h2d( non_spec_req_idx_cpu, device=query_start_loc.device ) @@ -360,14 +347,17 @@ class PleShortConvAttentionMetadataBuilder(ShortConvAttentionMetadataBuilder): dtype=torch.int64, device=query_start_loc.device, ) - req_group[spec_req_idx] = 0 - req_group[decode_req_idx] = 1 - token_group = torch.repeat_interleave(req_group, query_lens) + req_group.index_fill_(0, spec_req_idx, 0) + req_group.index_fill_(0, decode_req_idx, 1) + token_group = torch.repeat_interleave( + req_group, query_lens, output_size=int(query_start_loc_cpu[-1]) + ) token_perm = torch.argsort(token_group, stable=True) spec_token_indx = token_perm[:num_spec_decode_tokens] non_spec_token_indx = token_perm[num_spec_decode_tokens:] spec_state_indices_tensor = block_table_tensor[spec_req_idx, 0] + num_accepted_tokens = num_accepted_tokens[spec_req_idx] non_spec_state_indices_tensor = block_table_tensor[non_spec_req_idx, 0] spec_query_start_loc = torch.zeros( num_spec_decodes + 1, @@ -385,22 +375,8 @@ class PleShortConvAttentionMetadataBuilder(ShortConvAttentionMetadataBuilder): dim=0, out=non_spec_query_start_loc[1:], ) - non_spec_query_start_loc_cpu = torch.zeros( - num_decodes + num_prefills + 1, dtype=torch.int32 - ) - torch.cumsum( - query_lens_cpu[non_spec_req_idx_cpu], - dim=0, - out=non_spec_query_start_loc_cpu[1:], - ) - assert num_accepted_tokens is not None - # Accepted-token counts must follow the same request order as the - # speculative state indices. - num_accepted_tokens = num_accepted_tokens[spec_req_idx] - - # Compute the conv-state slots for the non-spec decode/prefill split, - # plus the initial-state masks and Triton causal_conv1d metadata. + # Compute the conv-state slots for the non-spec decode/prefill split. if non_spec_state_indices_tensor is None: state_indices_tensor = block_table_tensor[:0, 0] else: @@ -431,20 +407,9 @@ class PleShortConvAttentionMetadataBuilder(ShortConvAttentionMetadataBuilder): ] has_initial_states_p = num_computed_tokens_p > 0 assert non_spec_query_start_loc is not None - assert non_spec_query_start_loc_cpu is not None query_start_loc_p = ( non_spec_query_start_loc[num_decodes:] - num_decode_tokens ) - query_start_loc_p_cpu = ( - non_spec_query_start_loc_cpu[num_decodes:] - num_decode_tokens - ) - if query_start_loc.device.type != "cpu": - nums_dict, batch_ptr, token_chunk_offset_ptr = ( - compute_causal_conv1d_metadata( - query_start_loc_p_cpu, - device=query_start_loc.device, - ) - ) # Prepare persistent tensors for CUDA graph capture and replay. # ``m.num_actual_tokens`` is already padded by the model runner. @@ -509,9 +474,6 @@ class PleShortConvAttentionMetadataBuilder(ShortConvAttentionMetadataBuilder): non_spec_token_indx=non_spec_token_indx, num_accepted_tokens=num_accepted_tokens, num_decode_draft_tokens_cpu=num_decode_draft_tokens_cpu, - nums_dict=nums_dict, - batch_ptr=batch_ptr, - token_chunk_offset_ptr=token_chunk_offset_ptr, query_start_loc_p=query_start_loc_p, query_start_loc_d=query_start_loc_d, state_indices_tensor_p=state_indices_tensor_p, diff --git a/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py b/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py index a3608a9c7e..e676576fb7 100644 --- a/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py +++ b/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py @@ -2017,7 +2017,48 @@ def _validate_dsv4_sparse_dims( @triton.jit -def _pack_dense_prefix_to_ragged_kernel( +def _count_valid_row_entries_kernel( + indices_ptr, + lengths_ptr, + counts_ptr, + indptr_ptr, + indices_stride0, + num_rows_limit, + num_queries, + row_width, + BLOCK_R: tl.constexpr, + BLOCK_W: tl.constexpr, +): + pid = tl.program_id(0) + # The scan fills indptr[1:], so write indptr[0] here and skip a zero fill. + if pid == 0: + tl.store(indptr_ptr, 0) + + rows = pid * BLOCK_R + tl.arange(0, BLOCK_R) + row_ok = rows < num_queries + row_len = tl.load(lengths_ptr + rows, mask=row_ok, other=0) + + acc = tl.zeros((BLOCK_R,), dtype=tl.int32) + for start in range(0, row_width, BLOCK_W): + cols = start + tl.arange(0, BLOCK_W) + in_row = cols[None, :] < row_width + vals = tl.load( + indices_ptr + rows[:, None] * indices_stride0 + cols[None, :], + mask=row_ok[:, None] & in_row, + other=-1, + ).to(tl.int32) + keep = ( + row_ok[:, None] & in_row & (cols[None, :] < row_len[:, None]) & (vals >= 0) + ) + if num_rows_limit >= 0: + keep = keep & (vals < num_rows_limit) + acc += tl.sum(keep.to(tl.int32), axis=1) + + tl.store(counts_ptr + rows, acc, mask=row_ok) + + +@triton.jit +def _compact_dense_row_to_ragged_kernel( indices_ptr, lengths_ptr, indptr_ptr, @@ -2025,28 +2066,25 @@ def _pack_dense_prefix_to_ragged_kernel( indices_stride0, num_rows_limit, row_width, - BLOCK_SIZE: tl.constexpr, + BLOCK_W: tl.constexpr, ): row_idx = tl.program_id(0) - block_idx = tl.program_id(1) - offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) - row_len = tl.load(lengths_ptr + row_idx) - if block_idx * BLOCK_SIZE >= row_len: - return - - mask = offsets < row_len - safe_offsets = tl.where(offsets < row_width, offsets, 0) - vals = tl.load( - indices_ptr + row_idx * indices_stride0 + safe_offsets, - mask=mask & (offsets < row_width), - other=-1, - ).to(tl.int32) - if num_rows_limit >= 0: - vals = tl.where((vals >= 0) & (vals < num_rows_limit), vals, -1) - out_start = tl.load(indptr_ptr + row_idx) - tl.store(out_ptr + out_start + offsets, vals, mask=mask) + + written = 0 + for start in range(0, row_width, BLOCK_W): + cols = start + tl.arange(0, BLOCK_W) + in_row = cols < row_width + vals = tl.load( + indices_ptr + row_idx * indices_stride0 + cols, mask=in_row, other=-1 + ).to(tl.int32) + keep = in_row & (cols < row_len) & (vals >= 0) + if num_rows_limit >= 0: + keep = keep & (vals < num_rows_limit) + rank = tl.cumsum(keep.to(tl.int32), axis=0) - 1 + tl.store(out_ptr + out_start + written + rank, vals, mask=keep) + written += tl.sum(keep.to(tl.int32), axis=0) def build_ragged_indices_from_dense( @@ -2061,10 +2099,40 @@ def build_ragged_indices_from_dense( ) max_width = indices.shape[1] if indices.ndim == 2 else 0 - lengths = lengths.clamp(min=0, max=max_width).contiguous() + # Both kernels bound the row by row_width and row_len, so no clamp needed. + lengths = lengths.contiguous() - indptr = torch.zeros(indices.shape[0] + 1, dtype=torch.int32, device=indices.device) - torch.cumsum(lengths, dim=0, out=indptr[1:]) + # Dense rows reserve slots they may not fill, leaving -1 anywhere in the + # first `lengths` entries. Keep only the valid ones, since the consumers + # index the KV pool without checking the sign. + num_queries = indices.shape[0] + # max_width is whatever the caller packed (align(topk + window, 128) for the + # V4.1 indexer) and nothing caps it, so loop the row rather than size the + # block by it. + block = min(triton.next_power_of_2(max_width), 1024) if max_width > 0 else 1 + counts = torch.empty(num_queries, dtype=torch.int32, device=indices.device) + indptr = torch.empty(num_queries + 1, dtype=torch.int32, device=indices.device) + if num_queries > 0 and max_width > 0: + # A program per row only gets row_width elements, which is launch + # bound for narrow rows. Tile rows to fill the block. + block_r = max(1, 1024 // block) + _count_valid_row_entries_kernel[(triton.cdiv(num_queries, block_r),)]( + indices, + lengths, + counts, + indptr, + indices.stride(0), + int(num_rows), + num_queries, + max_width, + BLOCK_R=block_r, + BLOCK_W=block, + ) + else: + counts.zero_() + indptr.zero_() + + torch.cumsum(counts, dim=0, out=indptr[1:]) if indices.numel() == 0: flat = torch.empty(0, dtype=torch.int32, device=indices.device) @@ -2075,10 +2143,7 @@ def build_ragged_indices_from_dense( device=indices.device, ) if flat.numel() > 0: - block_size = 128 - _pack_dense_prefix_to_ragged_kernel[ - (indices.shape[0], triton.cdiv(max_width, block_size)) - ]( + _compact_dense_row_to_ragged_kernel[(indices.shape[0],)]( indices, lengths, indptr, @@ -2086,7 +2151,7 @@ def build_ragged_indices_from_dense( indices.stride(0), int(num_rows), max_width, - BLOCK_SIZE=block_size, + BLOCK_W=block, ) return flat, indptr @@ -4241,3 +4306,99 @@ def rocm_sparse_attn_decode( if direct_out is None: output.copy_(attn_out.to(output.dtype)) return output.shape[0] if inv_rope_positions is not None else 0 + + +@triton.jit(do_not_specialize=["token_offset"]) +def _prefill_topk_global_slots_kernel( + out_ptr, + lens_ptr, + topk_ptr, + topk_stride, + token_to_req_ptr, + query_start_loc_ptr, + seq_lens_ptr, + is_valid_token_ptr, + block_table_ptr, + block_table_stride, + token_offset, + num_compressed, + TOPK: tl.constexpr, + COMPRESS_RATIO: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + BLOCK_W: tl.constexpr, +): + row = tl.program_id(0) + token = row + token_offset + req = tl.load(token_to_req_ptr + token) + query_start = tl.load(query_start_loc_ptr + req) + query_len = tl.load(query_start_loc_ptr + req + 1) - query_start + pos = tl.load(seq_lens_ptr + req) - query_len + token - query_start + # Only (pos + 1) // ratio compressed tokens exist yet; the rest of the row + # can hold stale indices. + row_len = tl.minimum((pos + 1) // COMPRESS_RATIO, TOPK) + row_len = tl.where(tl.load(is_valid_token_ptr + token), row_len, 0) + + for start in range(0, TOPK, BLOCK_W): + cols = start + tl.arange(0, BLOCK_W) + live = cols < row_len + local = tl.load(topk_ptr + row * topk_stride + cols, mask=live, other=-1) + valid = live & (local >= 0) & (local < num_compressed) + local = tl.where(valid, local, 0) + block = tl.load( + block_table_ptr + req * block_table_stride + local // BLOCK_SIZE, + mask=valid, + other=0, + ) + slot = tl.where(valid, block * BLOCK_SIZE + local % BLOCK_SIZE, -1) + tl.store(out_ptr + row * TOPK + cols, slot, mask=cols < TOPK) + tl.store(lens_ptr + row, row_len) + + +def build_prefill_topk_ragged_indices( + topk_indices: torch.Tensor, + token_to_req_indices: torch.Tensor, + query_start_loc: torch.Tensor, + seq_lens: torch.Tensor, + is_valid_token: torch.Tensor, + block_table: torch.Tensor, + block_size: int, + compress_ratio: int, + num_compressed: int, + token_offset: int, + num_rows: int = -1, +) -> tuple[torch.Tensor, torch.Tensor]: + """Map prefill top-k rows to a ragged stream of compressed-cache slots. + + ``topk_indices`` holds local compressed positions for the prefill tokens, + which sit at ``token_offset`` in the batch; ``token_to_req_indices``, + ``query_start_loc``, ``seq_lens`` and ``block_table`` are batch-wide. + ``block_size`` is the compressed cache's, i.e. already divided by the ratio. + """ + topk_indices = topk_indices.reshape(topk_indices.shape[0], -1) + num_tokens, width = topk_indices.shape + dense = torch.empty( + (num_tokens, width), dtype=torch.int32, device=topk_indices.device + ) + lens = torch.empty(num_tokens, dtype=torch.int32, device=topk_indices.device) + if num_tokens > 0 and width > 0: + _prefill_topk_global_slots_kernel[(num_tokens,)]( + dense, + lens, + topk_indices, + topk_indices.stride(0), + token_to_req_indices, + query_start_loc, + seq_lens, + is_valid_token, + block_table, + block_table.stride(0), + token_offset, + num_compressed, + TOPK=width, + COMPRESS_RATIO=compress_ratio, + BLOCK_SIZE=block_size, + BLOCK_W=min(triton.next_power_of_2(width), 1024), + ) + else: + lens.zero_() + return build_ragged_indices_from_dense(dense, lens, num_rows=num_rows) diff --git a/vllm/v1/attention/ops/vit_attn_wrappers.py b/vllm/v1/attention/ops/vit_attn_wrappers.py index dccc670ce5..36f7de393c 100644 --- a/vllm/v1/attention/ops/vit_attn_wrappers.py +++ b/vllm/v1/attention/ops/vit_attn_wrappers.py @@ -58,6 +58,13 @@ def flash_attn_maxseqlen_wrapper( max_seqlen = max_seqlen.item() q, k, v = (einops.rearrange(x, "b s ... -> (b s) ...") for x in [q, k, v]) + + if current_platform.is_cuda() and torch.cuda.is_current_stream_capturing(): + # workaround for encoder CUDA-graph replay paddings with NaNs edge case + kwargs["out"] = torch.zeros( + q.shape[0], q.shape[1], v.shape[-1], dtype=q.dtype, device=q.device + ) + output = flash_attn_varlen_func( q, k, diff --git a/vllm/v1/core/sched/request_queue.py b/vllm/v1/core/sched/request_queue.py index ce58e6e1c7..0a0277eb56 100644 --- a/vllm/v1/core/sched/request_queue.py +++ b/vllm/v1/core/sched/request_queue.py @@ -41,7 +41,7 @@ class RequestQueue(ABC): pass @abstractmethod - def prepend_requests(self, requests: "RequestQueue") -> None: + def prepend_requests(self, requests: Iterable[Request]) -> None: """Prepend all requests from another queue to the front of this queue.""" pass @@ -93,7 +93,7 @@ class FCFSRequestQueue(deque[Request], RequestQueue): """Prepend a request to the front of the queue.""" self.appendleft(request) - def prepend_requests(self, requests: RequestQueue) -> None: + def prepend_requests(self, requests: Iterable[Request]) -> None: """Prepend all requests from another queue to the front of this queue. @@ -163,7 +163,7 @@ class PriorityRequestQueue(RequestQueue): front. Requests are ordered by (priority, arrival_time).""" self.add_request(request) - def prepend_requests(self, requests: RequestQueue) -> None: + def prepend_requests(self, requests: Iterable[Request]) -> None: """Add all requests from another queue according to priority policy. Note: In a priority queue, there is no concept of prepending to the diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index ce832a18f6..e4af907154 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -210,8 +210,13 @@ class Scheduler(SchedulerInterface): ) from e # Priority queues for requests. self.waiting = create_request_queue(self.policy) - # requests skipped in waiting flow due async deps or constraints. - self.skipped_waiting = create_request_queue(self.policy) + # Waiting requests that hold KV blocks are always drained before + # self.waiting: a block-holder must never sit behind a request whose + # failed allocation would stop the scheduling scan. + self.kv_holding_waiting = create_request_queue(self.policy) + # Waiting requests that are deferred, i.e. were skipped by the scheduling + # scan or enqueued in a blocked status. + self.deferred_waiting: set[Request] = set() self.running: list[Request] = [] # The request IDs that are finished in between the previous and the @@ -606,9 +611,8 @@ class Scheduler(SchedulerInterface): # `long_prefill_token_threshold` exists to stop a long prefill from # starving other requests of the token budget. When it is the only # request there is nobody to starve, so let it use the whole budget. - num_eligible_reqs = ( - len(self.running) + len(self.waiting) + len(self.skipped_waiting) - ) + num_running, num_waiting = self.get_request_counts() + num_eligible_reqs = num_running + num_waiting long_prefill_token_threshold = ( self.scheduler_config.long_prefill_token_threshold if num_eligible_reqs > 1 @@ -867,10 +871,21 @@ class Scheduler(SchedulerInterface): # Next, schedule the WAITING requests. if not preempted_reqs and self._pause_state == PauseState.UNPAUSED: - step_skipped_waiting = create_request_queue(self.policy) + step_skipped_waiting: deque[Request] = deque() + step_skipped_kv_holding: deque[Request] = deque() - while (self.waiting or self.skipped_waiting) and token_budget > 0: - if input_budget <= draft_slots: + def skip_request(from_queue: RequestQueue) -> None: + request_to_skip = from_queue.pop_request() + self.deferred_waiting.add(request_to_skip) + if self._holds_kv_blocks(request_to_skip): + step_skipped_kv_holding.appendleft(request_to_skip) + else: + step_skipped_waiting.appendleft(request_to_skip) + + while token_budget > 0: + # Requests holding KV blocks are always drained first. + request_queue = self.kv_holding_waiting or self.waiting + if not request_queue or input_budget <= draft_slots: break # Paused streaming sessions (WAITING_FOR_STREAMING_REQ) are not # in `running` but still hold a model-runner request slot. @@ -878,38 +893,22 @@ class Scheduler(SchedulerInterface): if num_running >= self.max_num_active_reqs: break - request_queue = self._select_waiting_queue_for_scheduling() - assert request_queue is not None - request = request_queue.peek_request() request_id = request.request_id - # try to promote blocked statuses while traversing skipped queue. - if self._is_blocked_waiting_status( - request.status - ) and not self._try_promote_blocked_waiting_request(request): - if request.status == RequestStatus.WAITING_FOR_REMOTE_KVS: - logger.debug( - "%s is still in WAITING_FOR_REMOTE_KVS state.", - request_id, - ) - request_queue.pop_request() - step_skipped_waiting.prepend_request(request) + ready_to_schedule = self._handle_blocked_waiting_request(request) + if not ready_to_schedule: + skip_request(request_queue) continue - if ( - request.num_stale_output_tokens > 0 - and not request.drop_stale_output - ): + if request.num_stale_output_tokens and not request.drop_stale_output: # Deliverable stale output still in flight: resuming now # could resample a position that output later delivers. # It drains within the pipeline depth. - request_queue.pop_request() - step_skipped_waiting.prepend_request(request) + skip_request(request_queue) continue - # Check that adding the request still respects the max_loras - # constraint. + # Check that adding the request still respects the max_loras constraint. if ( self.lora_config and request.lora_request @@ -919,8 +918,7 @@ class Scheduler(SchedulerInterface): ) ): # Scheduling would exceed max_loras, skip. - request_queue.pop_request() - step_skipped_waiting.prepend_request(request) + skip_request(request_queue) continue num_external_computed_tokens = 0 @@ -959,8 +957,7 @@ class Scheduler(SchedulerInterface): # The request cannot be scheduled because # the KVConnector couldn't determine # the number of matched tokens. - request_queue.pop_request() - step_skipped_waiting.prepend_request(request) + skip_request(request_queue) continue if self.prefix_replay_tokens: @@ -1030,8 +1027,7 @@ class Scheduler(SchedulerInterface): # Skip request with pending mm encoding prefetches if self._ec_transfer_pending(request, num_computed_tokens): - request_queue.pop_request() - step_skipped_waiting.prepend_request(request) + skip_request(request_queue) continue # Track first scheduled prefill, not post-preemption repeat prefills @@ -1052,8 +1048,7 @@ class Scheduler(SchedulerInterface): num_computed_tokens = request.num_computed_tokens if self._ec_transfer_pending(request, num_computed_tokens): - request_queue.pop_request() - step_skipped_waiting.prepend_request(request) + skip_request(request_queue) continue encoder_inputs_to_schedule = None @@ -1260,12 +1255,10 @@ class Scheduler(SchedulerInterface): request, num_new_local_computed_tokens ) - request = request_queue.pop_request() if load_kv_async: # If loading async, allocate memory and put request # into the WAITING_FOR_REMOTE_KV state. request.status = RequestStatus.WAITING_FOR_REMOTE_KVS - step_skipped_waiting.prepend_request(request) # Set num_computed_tokens even though KVs are not yet loaded. # request.num_computed_tokens will not be used anywhere until # the request finished the KV transfer. @@ -1286,13 +1279,16 @@ class Scheduler(SchedulerInterface): # overwrite; the zeroing could race the write. self._skip_zero_block_ids.update( self.kv_cache_manager.get_zeroing_block_ids_in_range( - request.request_id, + request_id, num_new_local_computed_tokens, num_computed_tokens, ) ) + skip_request(request_queue) continue + request = request_queue.pop_request() + self.deferred_waiting.discard(request) self.running.append(request) if num_external_computed_tokens > 0: # load_kv_async is False here @@ -1343,13 +1339,16 @@ class Scheduler(SchedulerInterface): self.ec_connector.update_state_after_alloc(request, i) # re-queue requests skipped in this pass ahead of older skipped items. + if step_skipped_kv_holding: + self.kv_holding_waiting.prepend_requests(step_skipped_kv_holding) if step_skipped_waiting: - self.skipped_waiting.prepend_requests(step_skipped_waiting) + self.waiting.prepend_requests(step_skipped_waiting) # DP prefill balancing: on a step that admitted prefills (release), # record whether it was capacity-bound. if not defer_prefills: - self.prefill_capacity_bound = bool(self.waiting) + _, num_waiting = self.get_request_counts() + self.prefill_capacity_bound = num_waiting > len(self.deferred_waiting) # Check if the scheduling constraints are satisfied. total_num_scheduled_tokens = sum(num_scheduled_tokens.values()) @@ -1649,6 +1648,7 @@ class Scheduler(SchedulerInterface): session.num_prompt_tokens = len(session.prompt_token_ids) session.arrival_time = update.arrival_time session.sampling_params = update.sampling_params + session.max_tokens = update.max_tokens if session.status == RequestStatus.WAITING_FOR_STREAMING_REQ: self.num_waiting_for_streaming_input -= 1 session.status = RequestStatus.WAITING @@ -1972,6 +1972,9 @@ class Scheduler(SchedulerInterface): sampled_token_ids = model_runner_output.sampled_token_ids logprobs = model_runner_output.logprobs prompt_logprobs_dict = model_runner_output.prompt_logprobs_dict + prompt_token_id_logprobs_dict = ( + model_runner_output.prompt_token_id_logprobs_dict + ) num_scheduled_tokens = scheduler_output.num_scheduled_tokens pooler_outputs = model_runner_output.pooler_output num_nans_in_logits = model_runner_output.num_nans_in_logits @@ -2149,6 +2152,15 @@ class Scheduler(SchedulerInterface): self.kv_cache_manager.estimate_cached_tokens(request) ) + # Extract sample logprobs before stop handling can replace the + # sampling parameters for a streaming continuation. + if ( + request.sampling_params is not None + and request.sampling_params.num_logprobs is not None + and logprobs + ): + new_logprobs = logprobs.slice_request(req_index, len(new_token_ids)) + finish_reason = None if stopped: # Capture finish_reason BEFORE _handle_stopped_request, which may @@ -2174,14 +2186,6 @@ class Scheduler(SchedulerInterface): else: stopped_preempted_reqs.add(request) - # Extract sample logprobs if needed. - if ( - request.sampling_params is not None - and request.sampling_params.num_logprobs is not None - and logprobs - ): - new_logprobs = logprobs.slice_request(req_index, len(new_token_ids)) - if self.return_sampling_mask: sampling_masks = model_runner_output.sampling_masks if new_token_ids and sampling_masks is not None: @@ -2194,6 +2198,7 @@ class Scheduler(SchedulerInterface): # Get prompt logprobs for this request. prompt_logprobs_tensors = prompt_logprobs_dict.get(req_id) + prompt_token_id_logprobs = prompt_token_id_logprobs_dict.get(req_id) if should_emit_output: # Add EngineCoreOutput for this Request. outputs[request.client_index].append( @@ -2204,6 +2209,7 @@ class Scheduler(SchedulerInterface): new_logprobs=new_logprobs, new_sampling_mask=new_sampling_mask, new_prompt_logprobs_tensors=prompt_logprobs_tensors, + prompt_token_id_logprobs=prompt_token_id_logprobs, pooling_output=pooler_output, stop_reason=request.stop_reason, events=request.take_events(), @@ -2223,6 +2229,7 @@ class Scheduler(SchedulerInterface): else: # Invariant: EngineCore returns no partial prefill outputs. assert not prompt_logprobs_tensors + assert prompt_token_id_logprobs is None # Remove the stopped requests from the running and waiting queues. if stopped_running_reqs: @@ -2230,7 +2237,8 @@ class Scheduler(SchedulerInterface): if stopped_preempted_reqs: # This is a rare case and unlikely to impact performance. self.waiting.remove_requests(stopped_preempted_reqs) - self.skipped_waiting.remove_requests(stopped_preempted_reqs) + self.kv_holding_waiting.remove_requests(stopped_preempted_reqs) + self.deferred_waiting.difference_update(stopped_preempted_reqs) error_req_ids = set(self.grammar_compile_error_reqs) self.grammar_compile_error_reqs.clear() @@ -2374,29 +2382,42 @@ class Scheduler(SchedulerInterface): RequestStatus.WAITING_FOR_STREAMING_REQ, ) + @staticmethod + def _holds_kv_blocks(request: Request) -> bool: + # Whether a request currently has kv blocks allocated. + # While always the case when status is WAITING_FOR_REMOTE_KVS + # or WAITING_FOR_STREAMING_REQ, there are such cases where it's + # WAITING or PREEMPTED. + return ( + request.num_computed_tokens > 0 + # num_computed_tokens is reset to 0 if kv transfer fails for requests in + # WAITING_FOR_REMOTE_KVS state, before their allocated blocks are freed. + or request.status == RequestStatus.WAITING_FOR_REMOTE_KVS + ) + def _enqueue_waiting_request(self, request: Request) -> None: if self._is_blocked_waiting_status(request.status): - self.skipped_waiting.add_request(request) + self.deferred_waiting.add(request) + if self._holds_kv_blocks(request): + self.kv_holding_waiting.add_request(request) else: self.waiting.add_request(request) - def _select_waiting_queue_for_scheduling(self) -> RequestQueue | None: - if self.policy == SchedulingPolicy.FCFS: - return self.skipped_waiting or self.waiting or None - - # PRIORITY mode: compare queue heads when both queues are non-empty. - if self.waiting and self.skipped_waiting: - waiting_req = self.waiting.peek_request() - skipped_req = self.skipped_waiting.peek_request() - return self.waiting if waiting_req < skipped_req else self.skipped_waiting - - return self.waiting or self.skipped_waiting or None - def _handle_stopped_request(self, request: Request) -> bool: """Return True if finished (can be False for resumable requests).""" if not request.resumable: return True + # Drop the finished turn's in-flight work and resume from the + # materialized frontier. + safe_frontier = request.num_computed_tokens - request.num_output_placeholders + assert safe_frontier >= 0 + request.num_computed_tokens = safe_frontier + request.spec_token_ids = [] + request.drop_stale_output = True + request.num_stale_output_tokens = request.num_in_flight_tokens + request.num_output_placeholders = 0 + if request.streaming_queue: update = request.streaming_queue.popleft() if update is None: @@ -2527,7 +2548,7 @@ class Scheduler(SchedulerInterface): def get_request_counts(self) -> tuple[int, int]: """Returns (num_running_reqs, num_waiting_reqs).""" - return len(self.running), len(self.waiting) + len(self.skipped_waiting) + return len(self.running), len(self.waiting) + len(self.kv_holding_waiting) def get_kv_cache_usage(self) -> float: """Returns the fraction of the KV cache currently in use (0.0-1.0).""" @@ -2608,7 +2629,8 @@ class Scheduler(SchedulerInterface): self.running = remove_all(self.running, running_requests_to_remove) if waiting_requests_to_remove: self.waiting.remove_requests(waiting_requests_to_remove) - self.skipped_waiting.remove_requests(waiting_requests_to_remove) + self.kv_holding_waiting.remove_requests(waiting_requests_to_remove) + self.deferred_waiting.difference_update(waiting_requests_to_remove) # Second pass: set status and free requests for request in valid_requests: @@ -2716,14 +2738,11 @@ class Scheduler(SchedulerInterface): def get_num_unfinished_requests(self) -> int: if self._pause_state == PauseState.PAUSED_ALL: return 0 + num_running, num_waiting = self.get_request_counts() if self._pause_state == PauseState.PAUSED_NEW: - return len(self.running) - num_waiting = ( - len(self.waiting) - + len(self.skipped_waiting) - - self.num_waiting_for_streaming_input - ) - return num_waiting + len(self.running) + return num_running + num_waiting -= self.num_waiting_for_streaming_input + return num_waiting + num_running def has_finished_requests(self) -> bool: if self.finished_req_ids: @@ -2732,10 +2751,8 @@ class Scheduler(SchedulerInterface): return False # Finished requests waiting on delayed connector cleanup remain in # self.requests after they have been removed from scheduling queues. - num_in_queues = ( - len(self.waiting) + len(self.skipped_waiting) + len(self.running) - ) - return len(self.requests) > num_in_queues + num_running, num_waiting = self.get_request_counts() + return len(self.requests) > (num_running + num_waiting) def has_requests(self) -> bool: # Override the interface default to also keep the engine alive while a @@ -2867,10 +2884,12 @@ class Scheduler(SchedulerInterface): ec_connector_stats_payload = ( ec_connector_stats.data if ec_connector_stats else None ) + num_running, num_waiting = self.get_request_counts() + num_deferred = len(self.deferred_waiting) return SchedulerStats( - num_running_reqs=len(self.running), - num_waiting_reqs=len(self.waiting), - num_skipped_waiting_reqs=len(self.skipped_waiting), + num_running_reqs=num_running, + num_waiting_reqs=num_waiting - num_deferred, + num_skipped_waiting_reqs=num_deferred, kv_cache_usage=self.kv_cache_manager.usage, prefix_cache_stats=prefix_cache_stats, connector_prefix_cache_stats=connector_prefix_cache_stats, @@ -3076,39 +3095,39 @@ class Scheduler(SchedulerInterface): request.num_computed_tokens = request.num_tokens - 1 self.finished_recving_kv_req_ids.remove(request.request_id) - def _try_promote_blocked_waiting_request(self, request: Request) -> bool: - """Try to promote a blocked waiting request back to schedulable states.""" + def _handle_blocked_waiting_request(self, request: Request) -> bool: + """Returns True if scheduling of this request should proceed, False if + it should be skipped for this step.""" if request.status == RequestStatus.WAITING_FOR_REMOTE_KVS: - # finished_recving_kv_req_ids is populated during - # update_from_output(), based on worker-side connector signals - # in KVConnectorOutput.finished_recving - if request.request_id not in self.finished_recving_kv_req_ids: + # Check whether async loading requests are finished. + request_id = request.request_id + if request_id not in self.finished_recving_kv_req_ids: + logger.debug("%s is still in WAITING_FOR_REMOTE_KVS state.", request_id) return False + # Promote request to waiting or preempted. self._update_waiting_for_remote_kv(request) if request.num_preemptions: request.status = RequestStatus.PREEMPTED else: request.status = RequestStatus.WAITING - return True - if request.status == RequestStatus.WAITING_FOR_STRUCTURED_OUTPUT_GRAMMAR: - structured_output_req = request.structured_output_request - if not structured_output_req or structured_output_req.grammar is None: + elif request.status == RequestStatus.WAITING_FOR_STRUCTURED_OUTPUT_GRAMMAR: + # Check whether grammar compilation has finished. + struct_output_req = request.structured_output_request + if not struct_output_req or struct_output_req.grammar is None: return False - if isinstance(structured_output_req.grammar, Exception): + if isinstance(struct_output_req.grammar, Exception): self.grammar_compile_error_reqs.add(request.request_id) return False + # Grammar is ready, promote to waiting state. request.status = RequestStatus.WAITING - return True - if request.status == RequestStatus.WAITING_FOR_STREAMING_REQ: + elif request.status == RequestStatus.WAITING_FOR_STREAMING_REQ: + # Streaming input request waiting for more input. assert not request.streaming_queue return False - raise AssertionError( - "Unexpected blocked waiting status in promotion: " - f"{request.status.name} for request {request.request_id}" - ) + return True def _update_from_kv_xfer_finished(self, kv_connector_output: KVConnectorOutput): """KV Connector: update the scheduler state based on the output. @@ -3281,7 +3300,7 @@ class Scheduler(SchedulerInterface): # handle async KV loads (not cached yet, evict_blocks=False) async_load_reqs = ( req - for req in self.skipped_waiting + for req in self.kv_holding_waiting if req.status == RequestStatus.WAITING_FOR_REMOTE_KVS ) async_failed_req_ids, num_failed_tokens, _ = ( diff --git a/vllm/v1/engine/__init__.py b/vllm/v1/engine/__init__.py index e5bbb96a9c..effe8d2416 100644 --- a/vllm/v1/engine/__init__.py +++ b/vllm/v1/engine/__init__.py @@ -236,6 +236,9 @@ class EngineCoreOutput( # Appended last so `array_like` positional serialization stays compatible. spec_decode_metrics: RequestSpecDecodeMetrics | None = None + # [num_scored_rows, num_token_ids]; appended last for array_like compatibility. + prompt_token_id_logprobs: torch.Tensor | None = None + @property def finished(self) -> bool: return self.finish_reason is not None diff --git a/vllm/v1/engine/core_client.py b/vllm/v1/engine/core_client.py index 5d59bdeaea..32770a9072 100644 --- a/vllm/v1/engine/core_client.py +++ b/vllm/v1/engine/core_client.py @@ -1587,6 +1587,10 @@ class DPLBAsyncMPClient(DPAsyncMPClient): ) // client_count def get_core_engine_for_request(self, request: EngineCoreRequest) -> EngineIdentity: + if (engine := self.reqs_in_flight.get(request.request_id)) is not None: + # This is for streaming-input session affinity. + return engine + # Engines are in rank order. if (eng_index := request.data_parallel_rank) is None and ( eng_index := get_late_interaction_engine_index( @@ -1889,13 +1893,29 @@ class DPLBAsyncMPClient(DPAsyncMPClient): await asyncio.to_thread(self._coord_store.wait, ready_keys) logger.info("[Elastic EP] Successfully started new engines") + def _eep_commit_pause_mode(self) -> PauseMode: + from vllm.distributed.elastic_ep.elastic_execute import ( + can_reuse_fused_moe_kernel, + ) + + # MRV2 re-warms through the request pool, so running requests must + # finish first while new ones stay queued in the scheduler. + parallel_config = self.vllm_config.parallel_config + if self.vllm_config.use_v2_model_runner and not can_reuse_fused_moe_kernel( + parallel_config + ): + return "wait" + return "keep" + async def _commit_scale_up_elastic_ep(self, new_data_parallel_size: int) -> None: new_core_engines = [ rank.to_bytes(2, "little") for rank in range(len(self.core_engines), new_data_parallel_size) ] - await self.pause_scheduler_async(mode="keep", clear_cache=False) + await self.pause_scheduler_async( + mode=self._eep_commit_pause_mode(), clear_cache=False + ) wait_future = self._eep_wait_for_setup_switch_complete() finish_futures = [ asyncio.create_task( @@ -1967,7 +1987,8 @@ class DPLBAsyncMPClient(DPAsyncMPClient): for rank in range(new_data_parallel_size, cur_data_parallel_size): self._kv_event_sources.pop(rank, None) removed_dp_size = cur_data_parallel_size - new_data_parallel_size - pause_modes = ["keep"] * new_data_parallel_size + ["abort"] * removed_dp_size + eep_mode = self._eep_commit_pause_mode() + pause_modes = [eep_mode] * new_data_parallel_size + ["abort"] * removed_dp_size pause_futures = [ self._call_utility_async("pause_scheduler", mode, False, engine=engine) for mode, engine in zip(pause_modes, old_core_engines) diff --git a/vllm/v1/engine/input_processor.py b/vllm/v1/engine/input_processor.py index 369c31ca27..33d413ba22 100644 --- a/vllm/v1/engine/input_processor.py +++ b/vllm/v1/engine/input_processor.py @@ -131,6 +131,19 @@ class InputProcessor: self.structured_outputs_config, self.tokenizer, ) + if params.prompt_logprob_token_ids is not None: + if not self.vllm_config.use_v2_model_runner: + raise VLLMValidationError( + "prompt_logprob_token_ids requires the V2 model runner " + "(VLLM_USE_V2_MODEL_RUNNER=1).", + parameter="prompt_logprob_token_ids", + ) + if self.vllm_config.cache_config.kv_sharing_fast_prefill: + raise VLLMValidationError( + "prompt_logprob_token_ids is incorrect with " + "--kv-sharing-fast-prefill; disable it for scoring.", + parameter="prompt_logprob_token_ids", + ) self.validate_logits_processors_params(params) diff --git a/vllm/v1/engine/logprobs.py b/vllm/v1/engine/logprobs.py index ff483e128e..f8230e5af6 100644 --- a/vllm/v1/engine/logprobs.py +++ b/vllm/v1/engine/logprobs.py @@ -5,6 +5,8 @@ import itertools from collections.abc import Iterable from dataclasses import dataclass +import numpy as np + from vllm.logger import init_logger from vllm.logprobs import ( FlatLogprobs, @@ -38,6 +40,8 @@ class LogprobsProcessor: cumulative_logprob: float | None num_logprobs: int | None num_prompt_logprobs: int | None + # [num_scored_rows, num_token_ids], set once on the final prefill chunk. + prompt_token_id_logprobs: np.ndarray | None = None @classmethod def from_new_request( @@ -204,6 +208,12 @@ class LogprobsProcessor: self.prompt_logprobs = [] return plp + def pop_prompt_token_id_logprobs(self) -> np.ndarray | None: + """Pop and return the fixed-ID prompt scores.""" + scores = self.prompt_token_id_logprobs + self.prompt_token_id_logprobs = None + return scores + @staticmethod def _get_sampled_context_ids( logprobs_source: SampleLogprobs | PromptLogprobs | None, @@ -352,3 +362,5 @@ class LogprobsProcessor: self._update_sample_logprobs(output.new_logprobs) if output.new_prompt_logprobs_tensors is not None: self._update_prompt_logprobs(output.new_prompt_logprobs_tensors) + if output.prompt_token_id_logprobs is not None: + self.prompt_token_id_logprobs = output.prompt_token_id_logprobs.numpy() diff --git a/vllm/v1/engine/output_processor.py b/vllm/v1/engine/output_processor.py index b4f7338048..5ab5636f81 100644 --- a/vllm/v1/engine/output_processor.py +++ b/vllm/v1/engine/output_processor.py @@ -390,8 +390,12 @@ class RequestState: if self.output_kind == RequestOutputKind.DELTA: # Side effect: logprobs processor forgets prompt logprobs prompt_logprobs = self.logprobs_processor.pop_prompt_logprobs() + prompt_token_id_logprobs = ( + self.logprobs_processor.pop_prompt_token_id_logprobs() + ) else: prompt_logprobs = self.logprobs_processor.prompt_logprobs + prompt_token_id_logprobs = self.logprobs_processor.prompt_token_id_logprobs return RequestOutput( request_id=external_req_id, # request_id is what was provided externally @@ -399,6 +403,7 @@ class RequestState: prompt=self.prompt, prompt_token_ids=prompt_token_ids, prompt_logprobs=prompt_logprobs, + prompt_token_id_logprobs=prompt_token_id_logprobs, outputs=cast(list[CompletionOutput], outputs), finished=finished, kv_transfer_params=kv_transfer_params, diff --git a/vllm/v1/fault_tolerance/engine_core_sentinel.py b/vllm/v1/fault_tolerance/engine_core_sentinel.py index 1d82dd9f74..3ab493e249 100644 --- a/vllm/v1/fault_tolerance/engine_core_sentinel.py +++ b/vllm/v1/fault_tolerance/engine_core_sentinel.py @@ -9,7 +9,6 @@ from typing import TYPE_CHECKING import msgspec -from vllm.config import set_current_vllm_config from vllm.distributed import stateless_destroy_torch_distributed_process_group from vllm.distributed.utils import stateless_init_torch_distributed_process_group from vllm.logger import init_logger @@ -123,8 +122,7 @@ class EngineCoreSentinel: engine = self.engine executor = engine.model_executor - with set_current_vllm_config(engine.vllm_config): - ft_request.params.update(self._reinit_dp_group()) + ft_request.params.update(self._reinit_dp_group()) if hasattr(engine, "step_counter"): engine.step_counter = 0 @@ -165,6 +163,7 @@ class EngineCoreSentinel: parallel_config.data_parallel_size, backend="gloo", return_store=True, + timeout=parallel_config.cpu_distributed_timeout, ) ) return {"new_stateless_dp_group_ports": worker_ports} diff --git a/vllm/v1/metrics/stats.py b/vllm/v1/metrics/stats.py index 66088c7954..da88c9379a 100644 --- a/vllm/v1/metrics/stats.py +++ b/vllm/v1/metrics/stats.py @@ -188,8 +188,9 @@ class SchedulerStats: num_running_reqs: int = 0 - num_waiting_reqs: int = 0 # length of the "waiting" request queue - num_skipped_waiting_reqs: int = 0 # length of the "skipped waiting" queue + num_waiting_reqs: int = 0 # waiting requests not deferred + # waiting requests deferred by transient constraints or a blocked status + num_skipped_waiting_reqs: int = 0 # These are used for internal DP load-balancing. step_counter: int = 0 diff --git a/vllm/v1/outputs.py b/vllm/v1/outputs.py index c38eafa4a4..1f04c2ccf9 100644 --- a/vllm/v1/outputs.py +++ b/vllm/v1/outputs.py @@ -283,6 +283,9 @@ class ModelRunnerOutput: default_factory=dict ) + # req_id -> [num_scored_rows, num_token_ids] prompt_logprob_token_ids scores. + prompt_token_id_logprobs_dict: dict[str, torch.Tensor] = field(default_factory=dict) + # [num_reqs, hidden_size] pooler_output: list[torch.Tensor | None] | None = None diff --git a/vllm/v1/structured_output/backend_xgrammar.py b/vllm/v1/structured_output/backend_xgrammar.py index 5d2c2a4985..337b2285f0 100644 --- a/vllm/v1/structured_output/backend_xgrammar.py +++ b/vllm/v1/structured_output/backend_xgrammar.py @@ -38,6 +38,10 @@ class XgrammarBackend(StructuredOutputBackend): self.disable_any_whitespace = ( self.vllm_config.structured_outputs_config.disable_any_whitespace ) + model_config = self.vllm_config.model_config + is_plamo3 = ( + model_config is not None and model_config.hf_config.model_type == "plamo3" + ) if is_mistral_tokenizer(self.tokenizer): # NOTE: ideally, xgrammar should handle this accordingly. @@ -57,6 +61,10 @@ class XgrammarBackend(StructuredOutputBackend): stop_token_ids=stop_token_ids, add_prefix_space=True, ) + elif is_plamo3 and callable( + init_xgrammar := getattr(self.tokenizer, "init_xgrammar", None) + ): + tokenizer_info, _ = init_xgrammar() else: tokenizer_info = xgr.TokenizerInfo.from_huggingface( self.tokenizer, diff --git a/vllm/v1/worker/gpu/async_utils.py b/vllm/v1/worker/gpu/async_utils.py index 02592980a8..3c58433318 100644 --- a/vllm/v1/worker/gpu/async_utils.py +++ b/vllm/v1/worker/gpu/async_utils.py @@ -156,6 +156,10 @@ class AsyncOutput(AsyncModelRunnerOutput): k: v.to_cpu_nonblocking() if v is not None else None for k, v in self.model_runner_output.prompt_logprobs_dict.items() } + token_id_logprobs = self.model_runner_output.prompt_token_id_logprobs_dict + self.prompt_token_id_logprobs_dict = { + k: v.to("cpu", non_blocking=True) for k, v in token_id_logprobs.items() + } if self.pending_aux_output is not None: self.pending_aux_output.enqueue_cpu_copy( num_sampled=self.num_sampled_tokens_np, @@ -194,6 +198,9 @@ class AsyncOutput(AsyncModelRunnerOutput): if self.logprobs_tensors is not None: self.model_runner_output.logprobs = self.logprobs_tensors.tolists() self.model_runner_output.prompt_logprobs_dict = self.prompt_logprobs_dict + self.model_runner_output.prompt_token_id_logprobs_dict = ( + self.prompt_token_id_logprobs_dict + ) if self.pending_aux_output is not None: self.model_runner_output.aux_output_connector_output = ( self.pending_aux_output.process_output() diff --git a/vllm/v1/worker/gpu/block_table.py b/vllm/v1/worker/gpu/block_table.py index 8cf3ba98dd..c7665c46b8 100644 --- a/vllm/v1/worker/gpu/block_table.py +++ b/vllm/v1/worker/gpu/block_table.py @@ -15,6 +15,9 @@ from vllm.v1.worker.gpu.buffer_utils import ( class BlockTables: + # Set by the elastic EP warmup so KV writes land in the null block. + redirect_writes_to_null_block = False + def __init__( self, block_sizes: list[int], @@ -126,6 +129,8 @@ class BlockTables: bpk = self.blocks_per_kv_block[i] if bpk > 1: block_ids = [b * bpk + k for b in block_ids for k in range(bpk)] + if self.redirect_writes_to_null_block: + block_ids = [0] * len(block_ids) end = start + len(block_ids) row_capacity = self.block_tables[i].gpu.shape[1] if end > row_capacity: diff --git a/vllm/v1/worker/gpu/cudagraph_utils.py b/vllm/v1/worker/gpu/cudagraph_utils.py index fa1dc4a04b..19f3157cea 100644 --- a/vllm/v1/worker/gpu/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/cudagraph_utils.py @@ -161,7 +161,6 @@ class CudaGraphManager: # DBO supports FULL CUDA graphs only. self.ubatch_runner = ubatch_runner - self.dp_size = vllm_config.parallel_config.data_parallel_size self.tp_size = vllm_config.parallel_config.tensor_parallel_size self.is_first_pp_rank = get_pp_group().is_first_rank self.is_last_pp_rank = get_pp_group().is_last_rank @@ -385,6 +384,23 @@ class CudaGraphManager: self._candidates.setdefault(key, []).extend(matching) current_range_start = num_tokens + 1 + @property + def dp_size(self) -> int: + # Not cached: elastic EP rewrites parallel_config in place on scale. + return self.vllm_config.parallel_config.data_parallel_size + + def release_graphs(self) -> None: + """Drop the captured graphs so a later capture() can refill them. + + Elastic EP reallocates the MoE workspace when it grows, which leaves + every captured graph holding a stale data pointer. `_capture_descs` is + kept, so `needs_capture()` still reports the work to redo. + """ + self.graphs.clear() + self._graphs_captured = False + if self.breakable_cg_runner is not None: + BreakableCUDAGraphWrapper.clear_all_graphs() + def needs_capture(self) -> bool: return len(self._capture_descs) > 0 diff --git a/vllm/v1/worker/gpu/eplb_utils.py b/vllm/v1/worker/gpu/eplb_utils.py index d1bbd3338e..e5eb31ad28 100644 --- a/vllm/v1/worker/gpu/eplb_utils.py +++ b/vllm/v1/worker/gpu/eplb_utils.py @@ -1,9 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from collections.abc import Callable +from collections.abc import Callable, Iterator +from contextlib import contextmanager from functools import wraps -from typing import Any +from typing import TYPE_CHECKING, Any import torch import torch.nn as nn @@ -15,6 +16,9 @@ from vllm.model_executor.models.interfaces import ( get_mixture_of_experts_model, ) +if TYPE_CHECKING: + from vllm.v1.worker.gpu.model_runner import GPUModelRunner + logger = init_logger(__name__) @@ -37,6 +41,27 @@ def step_eplb_after(*, is_dummy: bool = False) -> Callable: return decorator +@contextmanager +def preserve_serving_state(model_runner: "GPUModelRunner") -> Iterator[None]: + """Keep the elastic EP warmup out of the request pool and the KV cache.""" + # After the drain only parked streaming sessions still hold a slot, and + # those are re-added from NewRequestData on their next chunk. + _remove_all_requests(model_runner) + model_runner.block_tables.redirect_writes_to_null_block = True + try: + yield + finally: + _remove_all_requests(model_runner) + model_runner.block_tables.redirect_writes_to_null_block = False + if model_runner.kv_block_zeroer is not None: + model_runner.kv_block_zeroer.zero_block_ids([0]) + + +def _remove_all_requests(model_runner: "GPUModelRunner") -> None: + for req_id in list(model_runner.req_states.req_id_to_index): + model_runner._remove_request(req_id) + + class EPLBController: def __init__(self, parallel_config: Any, device: torch.device): self.parallel_config = parallel_config @@ -82,10 +107,8 @@ class EPLBController: self._has_registered_models = True return True - def maybe_register_model( - self, model: nn.Module, model_config: Any, load_dummy_weights: bool - ) -> bool: - if not self.parallel_config.enable_eplb or load_dummy_weights: + def maybe_register_model(self, model: nn.Module, model_config: ModelConfig) -> bool: + if not self.parallel_config.enable_eplb: return False moe_model = get_mixture_of_experts_model(model) @@ -100,7 +123,13 @@ class EPLBController: self._has_registered_models = True return True - def maybe_start_async_loop(self, eplb_models_added: bool) -> None: + def maybe_start_async_loop( + self, eplb_models_added: bool, load_dummy_weights: bool = False + ) -> None: + # A scaling-up worker registers to join the communicator handshake, but + # its groups are not live yet, so _perform_eplb_reshuffle() starts this. + if load_dummy_weights: + return if eplb_models_added and self.state is not None and self.state.is_async: self.state.start_async_loop() @@ -128,18 +157,10 @@ class EPLBController: def setup_from_mapping( self, - model: nn.Module, - model_config: Any, + model_config: ModelConfig, expanded_physical_to_logical: torch.Tensor, ) -> None: - moe_model = get_mixture_of_experts_model(model) - assert moe_model is not None - - self.state = EplbState.from_mapping( - model=moe_model, - model_config=model_config, - device=self.device, - parallel_config=self.parallel_config, - expanded_physical_to_logical=expanded_physical_to_logical, - ) - self._has_registered_models = True + # Update in place: a fresh EplbState would build a second communicator + # over a group the other ranks have already left. + assert self.state is not None + self.state.update_mapping(model_config, expanded_physical_to_logical) diff --git a/vllm/v1/worker/gpu/input_batch.py b/vllm/v1/worker/gpu/input_batch.py index 75bbef5d8d..1fc3737837 100644 --- a/vllm/v1/worker/gpu/input_batch.py +++ b/vllm/v1/worker/gpu/input_batch.py @@ -81,6 +81,8 @@ class InputBatch: is_prefilling_np: np.ndarray # == np.any(is_prefilling_np) has_prefill: bool + # No prefills, or only prefill rows that run as decodes. + decode_graph_eligible: bool # [num_tokens_after_padding] input_ids: torch.Tensor @@ -113,6 +115,11 @@ class InputBatch: # [num_reqs] set only under PCP+DCP (see CommonAttentionMetadata). dcp_local_seq_lens_cpu_upper_bound: torch.Tensor | None = None + # [num_reqs] prefilling rows that schedule one new prompt token (excluding + # drafts) over existing context and so compute exactly like decodes. + # None if there are no prefills. + prefill_runs_as_decode_np: np.ndarray | None = None + @classmethod def make_dummy( cls, @@ -196,6 +203,7 @@ class InputBatch: num_computed_prefill_tokens_np=np.zeros(num_reqs, dtype=np.int32), is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_), has_prefill=False, + decode_graph_eligible=True, input_ids=input_ids, positions=positions, is_padding=is_padding, diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 9f25425ab6..800ae825d4 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -31,8 +31,9 @@ import torch.nn as nn import vllm.envs as envs from vllm.compilation.counter import compilation_counter from vllm.compilation.cuda_graph import CUDAGraphStat +from vllm.compilation.wrapper import compile_model_with_stock_torch from vllm.config import VllmConfig -from vllm.config.compilation import CUDAGraphMode +from vllm.config.compilation import CompilationMode, CUDAGraphMode from vllm.distributed.aux_output_connector.worker import ( AuxOutputWorkerConnector, get_aux_output_connector, @@ -234,10 +235,6 @@ class GPUModelRunner(LoRAModelRunnerMixin): # Persistent buffer for intermediate tensors (non-first PP ranks). self.intermediate_tensors: IntermediateTensors | None = None - # Data parallelism. - self.dp_size = self.parallel_config.data_parallel_size - self.dp_rank = self.parallel_config.data_parallel_rank - # Dual batch overlap. Created in initialize_kv_cache(), once everything # it runs the microbatched forward with exists. self.ubatch_runner: UBatchRunner | None = None @@ -494,7 +491,9 @@ class GPUModelRunner(LoRAModelRunnerMixin): ), ) self.prompt_logprobs_worker = PromptLogprobsWorker( - self.max_num_reqs, logprobs_mode=self.model_config.logprobs_mode + self.max_num_reqs, + self.device, + logprobs_mode=self.model_config.logprobs_mode, ) self.structured_outputs_worker = StructuredOutputsWorker( max_num_logits=self.max_num_reqs * self.decode_query_len, @@ -507,9 +506,9 @@ class GPUModelRunner(LoRAModelRunnerMixin): if self.is_pooling_model and self.is_last_pp_rank: self.pooling_runner = PoolingRunner(self.model, self.vllm_config) eplb_models_added |= self.eplb.maybe_register_model( - self.model, self.model_config, load_dummy_weights + self.model, self.model_config ) - self.eplb.maybe_start_async_loop(eplb_models_added) + self.eplb.maybe_start_async_loop(eplb_models_added, load_dummy_weights) if not self.is_first_pp_rank: # For non-first PP ranks, create intermediate tensors sized @@ -524,6 +523,19 @@ class GPUModelRunner(LoRAModelRunnerMixin): get_offloader().post_init() + if self.compilation_config.mode == CompilationMode.STOCK_TORCH_COMPILE: + compile_model_with_stock_torch(self.model, self.vllm_config) + + @property + def dp_size(self) -> int: + # Not cached: elastic EP rewrites parallel_config in place on reconfigure. + return self.parallel_config.data_parallel_size + + @property + def dp_rank(self) -> int: + # Not cached: elastic EP rewrites parallel_config in place on reconfigure. + return self.parallel_config.data_parallel_rank + def get_model(self) -> nn.Module: return self.model @@ -797,7 +809,11 @@ class GPUModelRunner(LoRAModelRunnerMixin): "skip_attn must only be True for initial memory profiling." ) - # Create a dummy scheduler output. + # Create a dummy scheduler output. Plain draft-model speculation adds + # one correction slot per request during prefill. The scheduler + # accounts for these slots, while dummy runs bypass the scheduler. + # Adjust the dummy token count only when the expanded draft batch + # would exceed the token budget. num_reqs = min(num_tokens, self.max_num_reqs) if uniform_decode: # HACK(lucas): for now since the worker is shared between MRV1 and MRV2, @@ -811,8 +827,15 @@ class GPUModelRunner(LoRAModelRunnerMixin): num_reqs = min( num_reqs, self.max_num_tokens // self.speculator.num_query_per_req ) + num_tokens = min( + num_tokens, + self.max_num_tokens + - num_reqs * self.speculator.num_extra_query_per_req, + ) + # Distribute the remainder evenly so no dummy request exceeds # ceil(num_tokens / num_reqs) <= max_model_len tokens. + num_tokens_per_request = [ num_tokens // num_reqs + (i >= num_reqs - num_tokens % num_reqs) for i in range(num_reqs) @@ -1229,7 +1252,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): if dummy_run: # Dummy batches are uniform by construction. return None, get_uniform_decode_token_count( - num_reqs, num_toks, max_query_len, has_prefill=False + num_reqs, num_toks, max_query_len, decode_graph_eligible=True ) draft_tokens = scheduler_output.scheduled_spec_decode_tokens @@ -1248,24 +1271,48 @@ class GPUModelRunner(LoRAModelRunnerMixin): idx_mapping_np ] is_prefilling_np = num_computed_prefill_tokens_np < prefill_len_np + has_prefill = bool(is_prefilling_np.any()) - if self.adaptive_verification is not None and draft_tokens: - num_toks = self.adaptive_verification.get_num_tokens( - num_tokens_per_req, draft_tokens + num_draft_tokens_np = None + if draft_tokens: + num_drafts_iter = (len(draft_tokens.get(req_id, ())) for req_id in req_ids) + num_draft_tokens_np = np.fromiter( + num_drafts_iter, dtype=np.int32, count=num_reqs + ) + if self.adaptive_verification is not None: + num_toks = self.adaptive_verification.get_num_tokens( + num_tokens_per_req, draft_tokens + ) + + prefill_runs_as_decode_np = None + decode_graph_eligible = not has_prefill + if has_prefill and self.pcp_manager is None: + # One new prompt token over existing context (possibly padded with + # placeholder drafts) runs as a decode. PCP partitions prefill rows. + num_new_tokens = num_scheduled_tokens + if num_draft_tokens_np is not None: + num_new_tokens = num_new_tokens - num_draft_tokens_np + one_token_ext = (num_new_tokens == 1) & (num_computed_prefill_tokens_np > 0) + prefill_runs_as_decode_np = is_prefilling_np & one_token_ext + decode_graph_eligible = bool( + (prefill_runs_as_decode_np | ~is_prefilling_np).all() ) batch_state = BatchReqState( req_ids=req_ids, num_scheduled_tokens=num_scheduled_tokens, num_tokens=num_toks, + num_draft_tokens_np=num_draft_tokens_np, idx_mapping_np=idx_mapping_np, prefill_len_np=prefill_len_np, num_computed_prefill_tokens_np=num_computed_prefill_tokens_np, is_prefilling_np=is_prefilling_np, - has_prefill=bool(is_prefilling_np.any()), + has_prefill=has_prefill, + prefill_runs_as_decode_np=prefill_runs_as_decode_np, + decode_graph_eligible=decode_graph_eligible, ) return batch_state, get_uniform_decode_token_count( - num_reqs, num_toks, max_query_len, batch_state.has_prefill + num_reqs, num_toks, max_query_len, decode_graph_eligible ) def prepare_inputs( @@ -1307,11 +1354,9 @@ class GPUModelRunner(LoRAModelRunnerMixin): num_reqs, dtype=torch.int32, device=self.device ) else: - num_draft_tokens_per_req = np.fromiter( - (len(draft_tokens.get(req_id, ())) for req_id in req_ids), - dtype=np.int32, - count=num_reqs, - ) + num_draft_tokens_per_req = batch_req_state.num_draft_tokens_np + if num_draft_tokens_per_req is None: + num_draft_tokens_per_req = np.zeros(num_reqs, dtype=np.int32) num_bonus_tokens = self.model_state.num_new_sampled_tokens_per_step total_num_draft_tokens = int(num_draft_tokens_per_req.sum()) total_num_logits = num_reqs * num_bonus_tokens + total_num_draft_tokens @@ -1450,6 +1495,8 @@ class GPUModelRunner(LoRAModelRunnerMixin): num_computed_prefill_tokens_np=batch_req_state.num_computed_prefill_tokens_np, is_prefilling_np=batch_req_state.is_prefilling_np, has_prefill=batch_req_state.has_prefill, + prefill_runs_as_decode_np=batch_req_state.prefill_runs_as_decode_np, + decode_graph_eligible=batch_req_state.decode_graph_eligible, input_ids=self.input_buffers.input_ids[:num_tokens_after_padding], positions=self.input_buffers.positions[:num_tokens_after_padding], is_padding=is_padding, @@ -2063,6 +2110,14 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.req_states.num_computed_tokens.gpu, self.req_states.prompt_len.np, ) + prompt_token_id_logprobs_dict = ( + self.prompt_logprobs_worker.compute_prompt_token_id_logprobs( + self.model.compute_logits, + hidden_states, + input_batch, + self.req_states.prompt_len.np, + ) + ) # Prepare the model runner output. model_runner_output = ModelRunnerOutput( @@ -2072,6 +2127,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): req_id_to_index={req_id: i for i, req_id in enumerate(input_batch.req_ids)}, sampled_token_ids=None, # type: ignore prompt_logprobs_dict=prompt_logprobs_dict, # type: ignore[arg-type] + prompt_token_id_logprobs_dict=prompt_token_id_logprobs_dict, cudagraph_stats=cudagraph_stats, ) pending_aux_output = None @@ -2291,9 +2347,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): def setup_eplb_from_mapping( self, expanded_physical_to_logical: torch.Tensor ) -> None: - self.eplb.setup_from_mapping( - self.model, self.model_config, expanded_physical_to_logical - ) + self.eplb.setup_from_mapping(self.model_config, expanded_physical_to_logical) ########### EPLB methods end ########### @@ -2323,11 +2377,14 @@ class BatchReqState(NamedTuple): # May be less than scheduler_output.total_num_scheduled_tokens: # adaptive verification trims the draft budget before running. num_tokens: int + num_draft_tokens_np: np.ndarray | None # [num_reqs], None if no drafts idx_mapping_np: np.ndarray # [num_reqs] prefill_len_np: np.ndarray # [num_reqs] num_computed_prefill_tokens_np: np.ndarray # [num_reqs] is_prefilling_np: np.ndarray # [num_reqs] has_prefill: bool + prefill_runs_as_decode_np: np.ndarray | None # [num_reqs] + decode_graph_eligible: bool def sort_batch_req_ids( diff --git a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py index 2b73b55bac..32924555b1 100644 --- a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py +++ b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py @@ -258,10 +258,13 @@ class MambaHybridModelState(DefaultModelState): else: max_seq_len = seq_lens_cpu_upper_bound[:num_reqs].max().item() + is_prefilling_np = input_batch.is_prefilling_np + if input_batch.prefill_runs_as_decode_np is not None: + # A prompt tail the scheduler padded with placeholder drafts must run + # as a spec-decode row: the prefill kernels can't roll them back. + is_prefilling_np = is_prefilling_np & ~input_batch.prefill_runs_as_decode_np is_prefilling = torch.zeros(num_reqs, dtype=torch.bool, device="cpu") - is_prefilling[: input_batch.num_reqs] = torch.from_numpy( - input_batch.is_prefilling_np - ) + is_prefilling[: input_batch.num_reqs] = torch.from_numpy(is_prefilling_np) # During CUDAGraph capture, num_decode_draft_tokens_cpu and num_accepted_tokens # are created by attn_metadata_builder.build_for_cudagraph_capture, so we only # compute them during actual (non-capture) forward execution. @@ -281,16 +284,7 @@ class MambaHybridModelState(DefaultModelState): # Test request state, not num_scheduled_tokens == draft_count+1: # adaptive rewrites num_scheduled_tokens to an even split, so that # equality rarely holds and would demote every verify row to decode. - # A one-token prompt tail over prior state that the scheduler padded - # with placeholder drafts is also a spec-decode row: the prefill - # kernels can't roll the placeholders back. - num_computed = input_batch.num_computed_prefill_tokens_np - is_prompt_tail = (num_computed > 0) & ( - input_batch.prefill_len_np - num_computed == 1 - ) - is_decode = (~input_batch.is_prefilling_np | is_prompt_tail) & ( - input_batch.num_scheduled_tokens > 0 - ) + is_decode = ~is_prefilling_np & (input_batch.num_scheduled_tokens > 0) spec_decode_mask = (num_draft_tokens_per_req > 0) & is_decode num_decode_draft_tokens_np[: input_batch.num_reqs] = np.where( spec_decode_mask, num_draft_tokens_per_req, -1 diff --git a/vllm/v1/worker/gpu/pcp_manager.py b/vllm/v1/worker/gpu/pcp_manager.py index de49fcffb9..11ee7522fc 100644 --- a/vllm/v1/worker/gpu/pcp_manager.py +++ b/vllm/v1/worker/gpu/pcp_manager.py @@ -595,6 +595,7 @@ class PCPManager: ) local_is_prefilling_np = np.zeros(num_reqs_after_padding, dtype=np.bool_) local_is_prefilling_np[:num_local_reqs] = real_local_is_prefilling_np + local_has_prefill = bool(local_is_prefilling_np.any()) seq_lens_cpu_upper_bound_np = np.zeros(num_reqs_after_padding, dtype=np.int32) seq_lens_cpu_upper_bound_np[:num_local_reqs] = ( local_start_pos_np + local_num_scheduled_tokens @@ -645,7 +646,9 @@ class PCPManager: prefill_len_np=local_prefill_len_np, num_computed_prefill_tokens_np=local_num_computed_prefill_tokens_np, is_prefilling_np=local_is_prefilling_np, - has_prefill=bool(local_is_prefilling_np.any()), + has_prefill=local_has_prefill, + decode_graph_eligible=not local_has_prefill, + prefill_runs_as_decode_np=None, input_ids=input_buffers.input_ids[:num_local_tokens_padded], positions=input_buffers.positions[:num_local_tokens_padded], is_padding=is_padding, diff --git a/vllm/v1/worker/gpu/sample/logprob.py b/vllm/v1/worker/gpu/sample/logprob.py index 384fcdb9fd..f78db95dea 100644 --- a/vllm/v1/worker/gpu/sample/logprob.py +++ b/vllm/v1/worker/gpu/sample/logprob.py @@ -19,6 +19,7 @@ def _topk_log_softmax_kernel( logits_ptr, logits_stride, topk_ids_ptr, + topk_ids_stride, topk, vocab_size, BLOCK_SIZE: tl.constexpr, @@ -49,7 +50,7 @@ def _topk_log_softmax_kernel( k_offset = j + tl.arange(0, TOPK_BLOCK_SIZE) k_mask = k_offset < topk topk_ids = tl.load( - topk_ids_ptr + req_idx * topk + k_offset, mask=k_mask, other=0 + topk_ids_ptr + req_idx * topk_ids_stride + k_offset, mask=k_mask, other=0 ) logits = tl.load(row_ptr + topk_ids, mask=k_mask) logits = logits.to(tl.float32) @@ -98,6 +99,7 @@ def compute_token_logprobs( logits, logits.stride(0), token_ids, + token_ids.stride(0), num_logprobs, vocab_size, BLOCK_SIZE=1024, # type: ignore diff --git a/vllm/v1/worker/gpu/sample/prompt_logprob.py b/vllm/v1/worker/gpu/sample/prompt_logprob.py index 50baf39ebb..24fdb6188e 100644 --- a/vllm/v1/worker/gpu/sample/prompt_logprob.py +++ b/vllm/v1/worker/gpu/sample/prompt_logprob.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from collections.abc import Callable +from dataclasses import dataclass import numpy as np import torch @@ -8,20 +9,41 @@ import torch from vllm.config.model import LogprobsMode from vllm.sampling_params import SamplingParams from vllm.triton_utils import tl, triton +from vllm.utils.torch_utils import async_tensor_h2d from vllm.v1.outputs import LogprobsTensors from vllm.v1.worker.gpu.input_batch import InputBatch -from vllm.v1.worker.gpu.sample.logprob import compute_topk_scores +from vllm.v1.worker.gpu.sample.logprob import ( + compute_token_logprobs, + compute_topk_scores, +) + +CHUNK_SIZE = 1024 + + +@dataclass +class _TokenIdScores: + token_ids: torch.Tensor # [num_ids], on device + start: int # first prompt row to score + scores: torch.Tensor | None = None # [num_rows, num_ids], filled per chunk class PromptLogprobsWorker: - def __init__(self, max_num_reqs: int, logprobs_mode: LogprobsMode = "raw_logprobs"): + def __init__( + self, + max_num_reqs: int, + device: torch.device, + logprobs_mode: LogprobsMode = "raw_logprobs", + ): self.max_num_reqs = max_num_reqs + self.device = device self.logprobs_mode = logprobs_mode self.uses_prompt_logprobs = np.zeros(self.max_num_reqs, dtype=bool) self.num_prompt_logprobs = np.zeros(self.max_num_reqs, dtype=np.int32) # req_idx -> list of in-progress LogprobsTensors self.in_progress_prompt_logprobs: dict[str, list[LogprobsTensors]] = {} + # req_id -> fixed-ID scoring state + self.token_id_scores: dict[str, _TokenIdScores] = {} def add_request(self, req_id: str, req_idx: int, sampling_params: SamplingParams): uses_prompt_logprobs = sampling_params.prompt_logprobs is not None @@ -29,9 +51,69 @@ class PromptLogprobsWorker: self.num_prompt_logprobs[req_idx] = sampling_params.prompt_logprobs or 0 if uses_prompt_logprobs: self.in_progress_prompt_logprobs[req_id] = [] + if sampling_params.prompt_logprob_token_ids is not None: + self.token_id_scores[req_id] = _TokenIdScores( + async_tensor_h2d( + sampling_params.prompt_logprob_token_ids, self.device, torch.int64 + ), + sampling_params.prompt_logprob_start or 0, + ) def remove_request(self, req_id: str) -> None: self.in_progress_prompt_logprobs.pop(req_id, None) + self.token_id_scores.pop(req_id, None) + + def compute_prompt_token_id_logprobs( + self, + logits_fn: Callable[[torch.Tensor], torch.Tensor], + hidden_states: torch.Tensor, + input_batch: InputBatch, + prompt_lens: np.ndarray, + ) -> dict[str, torch.Tensor]: + """Fill scores row (prompt row - req.start) per chunk; emit on the last.""" + if not self.token_id_scores: + return {} + logits_mode = self.logprobs_mode in ("raw_logits", "processed_logits") + out: dict[str, torch.Tensor] = {} + for i, req_id in enumerate(input_batch.req_ids): + req = self.token_id_scores.get(req_id) + if req is None: + continue + prompt_len = int(prompt_lens[input_batch.idx_mapping_np[i]]) + chunk_start = int(input_batch.num_computed_prefill_tokens_np[i]) + chunk_end = chunk_start + int(input_batch.num_scheduled_tokens[i]) + # Skip decode steps and, as compute_prompt_logprobs does, requests + # resumed after preemption: their scores were already emitted. + if chunk_start >= prompt_len or prompt_len < input_batch.prefill_len_np[i]: + continue + if req.scores is None: + if chunk_start > req.start: + # This prefill starts past the first scored row, so those + # rows will never be written; drop the request instead of + # emitting a buffer with unwritten rows. + self.token_id_scores.pop(req_id, None) + continue + req.scores = hidden_states.new_empty( + (max(prompt_len - 1 - req.start, 0), len(req.token_ids)), + dtype=torch.float32, + ) + # The last prompt row predicts the first decode token; skip it. + lo = max(req.start, chunk_start) + hi = min(chunk_end, prompt_len - 1) + base = int(input_batch.query_start_loc_np[i]) - chunk_start + for a in range(lo, hi, CHUNK_SIZE): + b = min(a + CHUNK_SIZE, hi) + logits = logits_fn(hidden_states[base + a : base + b]) + ids = req.token_ids.expand(b - a, -1) + req.scores[a - req.start : b - req.start] = ( + logits.gather(-1, ids) + if logits_mode + else compute_token_logprobs(logits, ids) + ) + if chunk_end >= prompt_len: + out[req_id] = req.scores + req.scores = None + return out def compute_prompt_logprobs( self, @@ -205,7 +287,6 @@ def compute_prompt_logprobs_with_chunking( ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: # Since materializing the full prompt logits can take too much memory, # we compute it in chunks. - CHUNK_SIZE = 1024 token_ids = [] scores = [] ranks = [] diff --git a/vllm/v1/worker/gpu/spec_decode/__init__.py b/vllm/v1/worker/gpu/spec_decode/__init__.py index ede733a755..0616f12dd5 100644 --- a/vllm/v1/worker/gpu/spec_decode/__init__.py +++ b/vllm/v1/worker/gpu/spec_decode/__init__.py @@ -60,5 +60,11 @@ def init_speculator(vllm_config: VllmConfig, device: torch.device): ) return EagleSpeculator(vllm_config, device) + elif speculative_config.uses_draft_model(): + from vllm.v1.worker.gpu.spec_decode.draft_model.speculator import ( + PlainDraftModelSpeculator, + ) + + return PlainDraftModelSpeculator(vllm_config, device) else: raise NotImplementedError(f"{speculative_config.method} is not supported yet.") diff --git a/vllm/v1/worker/gpu/spec_decode/acceptance_estimator.py b/vllm/v1/worker/gpu/spec_decode/acceptance_estimator.py index 46065d0546..f7a4a2f2c1 100644 --- a/vllm/v1/worker/gpu/spec_decode/acceptance_estimator.py +++ b/vllm/v1/worker/gpu/spec_decode/acceptance_estimator.py @@ -20,7 +20,7 @@ _INIT_SLOPE = 0.5 _INIT_BIAS = -0.2 -@triton.jit +@triton.jit(do_not_specialize=["num_reqs"]) def _accumulate_kernel( info_ptr, grad_ptr, @@ -160,7 +160,7 @@ def _refit_kernel( tl.store(counts_ptr + k, 0.0, mask=mask) -@triton.jit +@triton.jit(do_not_specialize=["num_tokens"]) def _local_max_sumexp_kernel( local_max_ptr, local_max_stride, @@ -216,7 +216,7 @@ def _local_max_sumexp_kernel( ) -@triton.jit +@triton.jit(do_not_specialize=["num_tokens"]) def _predict_kernel( features_ptr, features_stride, @@ -422,7 +422,8 @@ class OnlineAcceptanceEstimator: self.predictions.stride(0), self.counts, num_reqs, - BLOCK_R=triton.next_power_of_2(max(num_reqs, 1)), + BLOCK_R=max(256, triton.next_power_of_2(num_reqs)), + num_warps=1 if num_reqs <= 256 else 4, ) self._steps_since_refit += 1 diff --git a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py index ca0662076b..897e7af8de 100644 --- a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py @@ -282,7 +282,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): # the target model during FULL cudagraph. num_tokens, max_query_len, - input_batch.has_prefill, + input_batch.decode_graph_eligible, ) prefill_batch_desc, prefill_batch_sync = dispatch_cg_and_sync_dp( self.prefill_cudagraph_manager, diff --git a/vllm/v1/worker/gpu/spec_decode/draft_model/__init__.py b/vllm/v1/worker/gpu/spec_decode/draft_model/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/vllm/v1/worker/gpu/spec_decode/draft_model/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm/v1/worker/gpu/spec_decode/draft_model/speculator.py b/vllm/v1/worker/gpu/spec_decode/draft_model/speculator.py new file mode 100644 index 0000000000..05bc997f7a --- /dev/null +++ b/vllm/v1/worker/gpu/spec_decode/draft_model/speculator.py @@ -0,0 +1,527 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from typing import Any + +import torch +import torch.nn as nn + +from vllm.compilation.backends import set_model_tag +from vllm.config import VllmConfig, replace +from vllm.config.compilation import CUDAGraphMode +from vllm.forward_context import BatchDescriptor, set_forward_context +from vllm.logger import init_logger +from vllm.model_executor.model_loader import get_model +from vllm.triton_utils import tl, triton +from vllm.v1.kv_cache_interface import KVCacheConfig +from vllm.v1.spec_decode.utils import next_power_of_2 +from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer +from vllm.v1.worker.gpu.block_table import BlockTables +from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor +from vllm.v1.worker.gpu.dp_utils import DPSyncState +from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers +from vllm.v1.worker.gpu.model_states.interface import ModelState +from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator +from vllm.v1.worker.utils import AttentionGroup + +logger = init_logger(__name__) + + +class PlainDraftModelSpeculator(DraftModelSpeculator): + """Speculative decoding using a separate smaller draft LM. + + Unlike Eagle, the draft model runs fully independently of the target model. + Step 0 builds an expanded buffer (accepted + correction token + rejected + slots) via a Triton kernel; steps 1..k-1 are single-token decode steps. + """ + + # Plain draft model needs one extra slot per request for correction. + num_extra_query_per_req = 1 + + def __init__(self, vllm_config: VllmConfig, device: torch.device): + super().__init__(vllm_config, device) + + # draft_max_seq_len is read by the parent's attention metadata builder. + # Plain draft model doesn't do per-batch adjustment; cap at max. + self.draft_max_seq_len = self.max_model_len + + self.last_token_indices = torch.zeros( + self.max_num_reqs, dtype=torch.int64, device=device + ) + # GPU-side arange for decode query_start_loc initialisation. + self.arange_gpu = torch.arange( + self.max_num_reqs + 1, dtype=torch.int32, device=device + ) + # Scalar step counter used as column index into draft_logits. + self.current_draft_step = torch.tensor(0, dtype=torch.int64, device=device) + + _expanded_max = self.max_num_tokens + self.max_num_reqs + self.expanded_input_ids = torch.zeros( + _expanded_max, dtype=torch.int32, device=device + ) + self.expanded_positions = torch.zeros( + _expanded_max, dtype=torch.int64, device=device + ) + self.expanded_slot_mappings: torch.Tensor + self.supports_mm_inputs = False + + def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None: + pass # CUDA graph not yet supported for plain draft model speculator. + + def capture(self) -> None: + pass # CUDA graph not yet supported for plain draft model speculator. + + def set_attn( + self, + model_state: ModelState, + kv_cache_config: KVCacheConfig, + block_tables: BlockTables, + target_input_buffers: InputBuffers, + target_attn_groups: list[list[AttentionGroup]], + ) -> None: + super().set_attn( + model_state, + kv_cache_config, + block_tables, + target_input_buffers, + target_attn_groups, + ) + self.expanded_slot_mappings = torch.empty( + block_tables.num_kv_cache_groups, + self.max_num_tokens + self.max_num_reqs, + dtype=torch.int64, + device=self.device, + ) + + def load_draft_model( + self, + target_model: nn.Module, + target_attn_layer_names: set[str], + ) -> nn.Module: + spec = self.speculative_config + assert spec is not None + draft_vllm_config = replace( + self.vllm_config, + model_config=self.draft_model_config, + quant_config=None, + parallel_config=replace( + spec.draft_parallel_config, + rank=self.vllm_config.parallel_config.rank, + ), + ) + with set_model_tag("draft_model"): + return get_model( + vllm_config=draft_vllm_config, + prefix="draft_model", + ) + + @torch.inference_mode() + def _run_model( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + attn_metadata: dict[str, Any] | None, + slot_mappings: dict[str, torch.Tensor] | None, + num_tokens_across_dp: torch.Tensor | None, + ) -> torch.Tensor: + num_tokens = input_ids.shape[0] + batch_descriptor = BatchDescriptor(num_tokens=num_tokens) + with set_forward_context( + attn_metadata, + self.vllm_config, + num_tokens=num_tokens, + cudagraph_runtime_mode=CUDAGraphMode.NONE, + num_tokens_across_dp=num_tokens_across_dp, + slot_mapping=slot_mappings, + batch_descriptor=batch_descriptor, + ): + hidden_states = self.model( # type: ignore[misc] + input_ids=input_ids, + positions=positions, + ) + if isinstance(hidden_states, tuple): + hidden_states = hidden_states[0] + return hidden_states + + def _accepted_last_indices( + self, + input_batch: InputBatch, + num_rejected: torch.Tensor, + num_reqs: int, + ) -> torch.Tensor: + qsl = input_batch.query_start_loc + adjusted_lens = qsl[1 : num_reqs + 1] - qsl[:num_reqs] - num_rejected[:num_reqs] + return qsl[:num_reqs] + adjusted_lens - 1 + + def _prefill( + self, + input_batch: InputBatch, + num_rejected: torch.Tensor, + last_sampled: torch.Tensor, + num_reqs: int, + skip_attn: bool, + num_tokens_across_dp: torch.Tensor | None, + ) -> None: + if skip_attn: + src_tokens = input_batch.num_tokens + self.last_token_indices[:num_reqs] = self._accepted_last_indices( + input_batch, num_rejected, num_reqs + ) + hidden_states = self._run_model( + input_batch.input_ids[:src_tokens], + input_batch.positions[:src_tokens], + None, + None, + num_tokens_across_dp, + ) + positions_buf = input_batch.positions + else: + block_tables = self.block_tables + kv_cache_config = self.kv_cache_config + assert block_tables is not None + assert kv_cache_config is not None + + total_expanded = prepare_prefill_inputs( + input_buffers=self.input_buffers, + input_batch=input_batch, + last_sampled=last_sampled, + num_rejected=num_rejected, + expanded_input_ids=self.expanded_input_ids, + expanded_positions=self.expanded_positions, + last_token_indices=self.last_token_indices, + max_num_reqs=self.max_num_reqs, + max_model_len=self.max_model_len, + ) + + prefill_slot_mappings = block_tables.compute_slot_mappings( + input_batch.idx_mapping, + self.input_buffers.query_start_loc[: num_reqs + 1], + self.expanded_positions[:total_expanded], + total_expanded, + out=self.expanded_slot_mappings, + ) + qsl_np = input_batch.query_start_loc_np + query_start_loc_cpu_expanded = ( + torch.from_numpy(qsl_np[: num_reqs + 1]).int() + + torch.from_numpy(self.arange_np[: num_reqs + 1]).int() + ) + prefill_attn_md = self._build_attn_metadata( + batch_desc=BatchExecutionDescriptor( + cg_mode=CUDAGraphMode.NONE, + num_tokens=total_expanded, + num_reqs=num_reqs, + ), + num_reqs=num_reqs, + query_start_loc_np=query_start_loc_cpu_expanded.numpy(), + seq_lens_cpu_upper_bound=input_batch.seq_lens_cpu_upper_bound, + step=0, + slot_mappings=prefill_slot_mappings, + ) + prefill_slot_maps_by_layer = build_slot_mappings_by_layer( + prefill_slot_mappings, kv_cache_config + ) + hidden_states = self._run_model( + self.expanded_input_ids[:total_expanded], + self.expanded_positions[:total_expanded], + prefill_attn_md, + prefill_slot_maps_by_layer, + num_tokens_across_dp, + ) + positions_buf = self.expanded_positions + + last_indices = self.last_token_indices[:num_reqs] + self.current_draft_step.fill_(0) + self.draft_tokens[:num_reqs, 0] = self.sample_draft( + hidden_states[last_indices], + positions_buf[last_indices], + self.idx_mapping[:num_reqs], + self.temperature, + self.seeds, + self.current_draft_step, + self.draft_logits, + ) + + def _multi_step_decode( + self, + input_batch: InputBatch, + num_rejected: torch.Tensor, + num_reqs: int, + skip_attn: bool, + num_tokens_across_dp: torch.Tensor | None, + ) -> None: + if skip_attn: + last_positions = input_batch.positions[ + self._accepted_last_indices(input_batch, num_rejected, num_reqs) + ] + else: + last_positions = self.expanded_positions[self.last_token_indices[:num_reqs]] + + self.input_buffers.positions[:num_reqs].copy_(last_positions) + # The decode loop increments seq_lens BEFORE each forward. + # Initial seq_lens = target_seq_lens - num_rejected + 1. + # Step 1 forward uses target_seq_lens - num_rejected + 2. + # Step 2 forward uses target_seq_lens - num_rejected + 3. + self.input_buffers.seq_lens[:num_reqs].copy_( + torch.clamp( + input_batch.seq_lens[:num_reqs] - num_rejected[:num_reqs].int() + 1, + max=self.max_model_len, + ) + ) + self.input_buffers.query_start_loc[: num_reqs + 1].copy_( + self.arange_gpu[: num_reqs + 1] + ) + + idx_mapping = input_batch.idx_mapping + + for step in range(1, self.num_speculative_steps): + self.input_buffers.input_ids[:num_reqs].copy_( + self.draft_tokens[:num_reqs, step - 1].int() + ) + torch.clamp( + self.input_buffers.positions[:num_reqs] + 1, + max=self.max_model_len - 1, + out=self.input_buffers.positions[:num_reqs], + ) + # Increment seq_lens BEFORE the forward so the attention reads from + # the KV slot written by the previous step. + torch.clamp( + self.input_buffers.seq_lens[:num_reqs] + 1, + max=self.max_model_len, + out=self.input_buffers.seq_lens[:num_reqs], + ) + + decode_attn_md = None + step_slot_maps_by_layer = None + if not skip_attn: + block_tables = self.block_tables + kv_cache_config = self.kv_cache_config + assert block_tables is not None + assert kv_cache_config is not None + q_start = self.input_buffers.query_start_loc[: num_reqs + 1] + positions = self.input_buffers.positions[:num_reqs] + step_slot_mappings = block_tables.compute_slot_mappings( + idx_mapping, q_start, positions, num_reqs + ) + step_slot_maps_by_layer = build_slot_mappings_by_layer( + step_slot_mappings, kv_cache_config + ) + decode_attn_md = self._build_uniform_attn_metadata( + batch_desc=BatchExecutionDescriptor( + cg_mode=CUDAGraphMode.NONE, + num_tokens=num_reqs, + num_reqs=num_reqs, + uniform_token_count=1, + ), + num_reqs=num_reqs, + num_query_per_req=1, + seq_lens_cpu_upper_bound=input_batch.seq_lens_cpu_upper_bound, + # Include the correction token inserted during prefill. + step=step + 1, + ) + hidden_states = self._run_model( + self.input_buffers.input_ids[:num_reqs], + self.input_buffers.positions[:num_reqs], + decode_attn_md, + step_slot_maps_by_layer, + num_tokens_across_dp, + ) + + self.current_draft_step.fill_(step) + self.draft_tokens[:num_reqs, step] = self.sample_draft( + hidden_states[:num_reqs], + self.input_buffers.positions[:num_reqs], + self.idx_mapping[:num_reqs], + self.temperature, + self.seeds, + self.current_draft_step, + self.draft_logits, + ) + + @torch.inference_mode() + def propose( + self, + input_batch: InputBatch, + attn_metadata: dict[str, Any], + slot_mappings: dict[str, torch.Tensor], + last_hidden_states: torch.Tensor, # unused + aux_hidden_states: list[torch.Tensor] | None, # unused + num_sampled: torch.Tensor, + num_rejected: torch.Tensor, + last_sampled: torch.Tensor, + next_prefill_tokens: torch.Tensor, + temperature: torch.Tensor, + seeds: torch.Tensor, + dp_sync: DPSyncState | None = None, + dummy_run: bool = False, + skip_attn_for_dummy_run: bool = False, + mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None, + is_profile: bool = False, + ) -> torch.Tensor: + assert self.model is not None + + num_reqs = input_batch.num_reqs + skip_attn = dummy_run and skip_attn_for_dummy_run + num_tokens_across_dp = ( + dp_sync.num_tokens_across_dp if dp_sync is not None else None + ) + + # Copy per-request temperature/seeds/idx_mapping into pre-allocated + # buffers so sample_draft can read them without extra slicing. + self._copy_request_inputs( + num_reqs, + input_batch.idx_mapping, + temperature, + seeds, + ) + + self._prefill( + input_batch, + num_rejected, + last_sampled, + num_reqs, + skip_attn, + num_tokens_across_dp, + ) + + if self.num_speculative_steps == 1: + return self.draft_tokens[:num_reqs, :1] + + self._multi_step_decode( + input_batch, + num_rejected, + num_reqs, + skip_attn, + num_tokens_across_dp, + ) + + return self.draft_tokens[:num_reqs] + + +@triton.jit +def _prepare_prefill_inputs_kernel( + target_input_ids_ptr, # [src_tokens] int32 + target_positions_ptr, # [src_tokens] int64 + last_sampled_ptr, # [max_num_reqs] int32 + idx_mapping_ptr, # [num_reqs] int32 + out_input_ids_ptr, # [src_tokens + num_reqs] int32 + out_positions_ptr, # [src_tokens + num_reqs] int64 + last_token_indices_ptr, # [max_num_reqs] int64 + out_query_start_loc_ptr, # [max_num_reqs + 1] int32 + out_seq_lens_ptr, # [max_num_reqs] int32 + query_start_loc_ptr, # [num_reqs + 1] int32 + seq_lens_ptr, # [num_reqs] int32 + num_rejected_ptr, # [num_reqs] int32 + max_num_reqs, + max_model_len, + BLOCK_SIZE: tl.constexpr, +): + """Per-request step-0 input preparation for the plain draft-model speculator. + + Output layout for request i (out_start = query_start_loc[i] + i): + [out_start, out_start + num_valid) accepted tokens + [out_start + num_valid] correction token + (out_start + num_valid, out_start + total_out) rejected slots + + where num_valid = query_lens[i] - num_rejected[i] + total_out = query_lens[i] + 1 + """ + req_idx = tl.program_id(0) + num_reqs = tl.num_programs(0) + + req_state_idx = tl.load(idx_mapping_ptr + req_idx) + + q_start = tl.load(query_start_loc_ptr + req_idx) + q_next = tl.load(query_start_loc_ptr + req_idx + 1) + seq_len = tl.load(seq_lens_ptr + req_idx) + num_rejected = tl.load(num_rejected_ptr + req_idx) + + num_valid = q_next - q_start - num_rejected + correction_token = tl.load(last_sampled_ptr + req_state_idx).to(tl.int32) + start_pos = tl.load(target_positions_ptr + q_start) + out_start = q_start + req_idx + total_out = q_next - q_start + 1 + + for i in range(0, total_out, BLOCK_SIZE): + j = i + tl.arange(0, BLOCK_SIZE) + in_bounds = j < total_out + + is_valid = j < num_valid + is_correction = j == num_valid + src_idx = q_start + j + token_ids = tl.load(target_input_ids_ptr + src_idx, mask=is_valid, other=0) + positions = tl.minimum(start_pos + j, max_model_len - 1) + token_ids = tl.where(is_correction, correction_token, token_ids) + + out_idx = out_start + j + tl.store(out_input_ids_ptr + out_idx, token_ids, mask=in_bounds) + tl.store(out_positions_ptr + out_idx, positions, mask=in_bounds) + + tl.store(last_token_indices_ptr + req_idx, out_start + num_valid) + tl.store(out_query_start_loc_ptr + req_idx, out_start) + # seqlen_k for the expanded prefill attention: + # pre_existing = seq_len - query_len (draft KV before this round) + # expanded_query_len = query_len + 1 (adds correction token) + # seqlen_k = pre_existing + expanded_query_len = seq_len + 1 + # num_rejected does NOT change seqlen_k — the expanded query always has + # query_len+1 slots. Rejected positions write temporary KV entries that + # are overwritten with valid tokens in a later round. + new_seq_len = tl.minimum(seq_len + 1, max_model_len) + tl.store(out_seq_lens_ptr + req_idx, new_seq_len) + + if req_idx == num_reqs - 1: + total_expanded = out_start + total_out + tl.store(out_query_start_loc_ptr + num_reqs, total_expanded) + for i in range(num_reqs + 1, max_num_reqs + 1, BLOCK_SIZE): + block = i + tl.arange(0, BLOCK_SIZE) + mask = block <= max_num_reqs + tl.store(out_query_start_loc_ptr + block, total_expanded, mask=mask) + for i in range(num_reqs, max_num_reqs, BLOCK_SIZE): + block = i + tl.arange(0, BLOCK_SIZE) + mask = block < max_num_reqs + tl.store(out_seq_lens_ptr + block, 0, mask=mask) + tl.store(last_token_indices_ptr + block, 0, mask=mask) + + +def prepare_prefill_inputs( + input_buffers: InputBuffers, + input_batch: InputBatch, + last_sampled: torch.Tensor, # [max_num_reqs] int64 + num_rejected: torch.Tensor, # [num_reqs] int64 + expanded_input_ids: torch.Tensor, # [max_tokens + max_reqs] int32 + expanded_positions: torch.Tensor, # [max_tokens + max_reqs] int64 + last_token_indices: torch.Tensor, # [max_num_reqs] int64 + max_num_reqs: int, + max_model_len: int, +) -> int: + """Call _prepare_prefill_inputs_kernel and return total_expanded tokens. + + Side-effects (kernel writes): + - expanded_input_ids, expanded_positions + - last_token_indices + - input_buffers.query_start_loc (expanded: original[i] + i) + - input_buffers.seq_lens (= target_seq_lens - num_rejected + 1) + """ + num_reqs = input_batch.num_reqs + src_tokens = input_batch.num_tokens + qsl_np = input_batch.query_start_loc_np + query_lens = qsl_np[1 : num_reqs + 1] - qsl_np[:num_reqs] + max_total_out = int(query_lens.max()) + 2 # +1 correction, +1 alignment + BLOCK_SIZE = min(512, next_power_of_2(max_total_out)) + + _prepare_prefill_inputs_kernel[(num_reqs,)]( + input_batch.input_ids, + input_batch.positions, + last_sampled.int(), + input_batch.idx_mapping, + expanded_input_ids, + expanded_positions, + last_token_indices, + input_buffers.query_start_loc, + input_buffers.seq_lens, + input_batch.query_start_loc, + input_batch.seq_lens, + num_rejected.int(), + max_num_reqs, + max_model_len, + BLOCK_SIZE=BLOCK_SIZE, + ) + return src_tokens + num_reqs diff --git a/vllm/v1/worker/gpu/spec_decode/multi_module_mtp/speculator.py b/vllm/v1/worker/gpu/spec_decode/multi_module_mtp/speculator.py index fc0924e3a3..928498edf1 100644 --- a/vllm/v1/worker/gpu/spec_decode/multi_module_mtp/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/multi_module_mtp/speculator.py @@ -190,7 +190,7 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator): # When all requests are decoding (no true prefills), each has # num_speculative_steps + 1 tokens, enabling FULL graph replay. uniform_token_count = get_uniform_decode_token_count( - num_reqs, num_tokens, max_query_len, input_batch.has_prefill + num_reqs, num_tokens, max_query_len, input_batch.decode_graph_eligible ) batch_desc, batch_sync = dispatch_cg_and_sync_dp( self.cudagraph_manager, diff --git a/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py b/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py index 743d66010d..61eb3fb49a 100644 --- a/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py +++ b/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py @@ -74,6 +74,76 @@ def _flatten_sampled_kernel( tl.store(flat_sampled_ptr + start_idx + i, token_id) +@triton.jit +def _gather_draft_sampled_kernel( + # [num_logits] + draft_sampled_ptr, + # [num_logits] + pos_ptr, + # [num_tokens] + input_ids_ptr, + # [num_tokens] + positions_ptr, + # [num_logits] + logits_indices_ptr, + # [num_logits] + expanded_idx_mapping_ptr, + # [num_logits] + expanded_local_pos_ptr, + # [max_num_reqs] + prefill_len_ptr, + num_logits, + BLOCK_SIZE: tl.constexpr, +): + block = tl.program_id(0) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) + mask = block < num_logits + token_idx = tl.load(logits_indices_ptr + block, mask=mask) + token_id = tl.load(input_ids_ptr + token_idx, mask=mask) + pos = tl.load(positions_ptr + token_idx, mask=mask) + req_state_idx = tl.load(expanded_idx_mapping_ptr + block, mask=mask) + local_pos = tl.load(expanded_local_pos_ptr + block, mask=mask) + prefill_len = tl.load(prefill_len_ptr + req_state_idx, mask=mask) + # Drafts are only proposed after sampling past the prefill, so any draft + # slots in a step starting within the prefill are placeholders (e.g. the + # padded first step after a P/D remote KV load). + is_placeholder = (local_pos > 0) & (pos - local_pos < prefill_len) + token_id = tl.where(is_placeholder, -1, token_id) + tl.store(draft_sampled_ptr + block, token_id, mask=mask) + tl.store(pos_ptr + block, pos, mask=mask) + + +def gather_draft_sampled( + input_ids: torch.Tensor, + positions: torch.Tensor, + logits_indices: torch.Tensor, + expanded_idx_mapping: torch.Tensor, + expanded_local_pos: torch.Tensor, + prefill_len: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + """Gather the input token and position of each logits row. + + Draft rows of requests that have not yet sampled past their prefill are + set to -1 so that the rejection kernels reject them. + """ + num_logits = logits_indices.shape[0] + draft_sampled = input_ids.new_empty(num_logits) + pos = positions.new_empty(num_logits) + BLOCK_SIZE = 1024 + _gather_draft_sampled_kernel[(triton.cdiv(num_logits, BLOCK_SIZE),)]( + draft_sampled, + pos, + input_ids, + positions, + logits_indices, + expanded_idx_mapping, + expanded_local_pos, + prefill_len, + num_logits, + BLOCK_SIZE=BLOCK_SIZE, + ) + return draft_sampled, pos + + class RejectionSampler: def __init__( self, @@ -296,8 +366,14 @@ class RejectionSampler: # that num_nans is computed before applying penalties and temperature. num_nans = get_num_nans(logits) if self.sampler.compute_nans else None - draft_sampled = input_batch.input_ids[input_batch.logits_indices] - pos = input_batch.positions[input_batch.logits_indices] + draft_sampled, pos = gather_draft_sampled( + input_batch.input_ids, + input_batch.positions, + input_batch.logits_indices, + input_batch.expanded_idx_mapping, + input_batch.expanded_local_pos, + self.sampler.req_states.prefill_len.gpu, + ) max_num_logprobs = self.sampler.sampling_states.max_num_logprobs( input_batch.idx_mapping_np diff --git a/vllm/v1/worker/gpu/spec_decode/speculator.py b/vllm/v1/worker/gpu/spec_decode/speculator.py index 6f88febb66..c30b653196 100644 --- a/vllm/v1/worker/gpu/spec_decode/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/speculator.py @@ -58,6 +58,8 @@ def _target_feeds_hc_residual(vllm_config: VllmConfig) -> bool: class BaseSpeculator(ABC): num_query_per_req: int = 1 + # Extra query slots reserved per request outside the regular queries. + num_extra_query_per_req: int = 0 @abstractmethod def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None: @@ -302,6 +304,7 @@ class DraftModelSpeculator(BaseSpeculator): step: int, causal: bool | Mapping[int, bool] = True, dcp_local_seq_lens: torch.Tensor | None = None, + slot_mappings: torch.Tensor | None = None, ) -> dict[str, Any] | None: num_reqs_padded = batch_desc.num_reqs or num_reqs # A FULL graph replays a captured shape whose padded requests each hold @@ -323,7 +326,8 @@ class DraftModelSpeculator(BaseSpeculator): block_tables = [ x[:num_reqs_padded] for x in self.block_tables.input_block_tables ] - slot_mappings = self.block_tables.slot_mappings[:, :num_tokens] + if slot_mappings is None: + slot_mappings = self.block_tables.slot_mappings[:, :num_tokens] draft_seq_lens_cpu_upper_bound = torch.zeros( num_reqs_padded, dtype=torch.int32, device="cpu" ) diff --git a/vllm/v1/worker/gpu/ubatch_utils.py b/vllm/v1/worker/gpu/ubatch_utils.py index d6f24fcc69..6889103ed1 100644 --- a/vllm/v1/worker/gpu/ubatch_utils.py +++ b/vllm/v1/worker/gpu/ubatch_utils.py @@ -181,6 +181,11 @@ def _slice_input_batch( req_start:req_stop ], is_prefilling_np=input_batch.is_prefilling_np[req_start:req_stop], + prefill_runs_as_decode_np=( + None + if input_batch.prefill_runs_as_decode_np is None + else input_batch.prefill_runs_as_decode_np[req_start:req_stop] + ), input_ids=input_batch.input_ids[tok_start:tok_stop], positions=input_batch.positions[tok_start:tok_stop], is_padding=input_batch.is_padding[tok_start:tok_stop], diff --git a/vllm/v1/worker/gpu/warmup.py b/vllm/v1/worker/gpu/warmup.py index 56f7d438fc..2af97207ce 100644 --- a/vllm/v1/worker/gpu/warmup.py +++ b/vllm/v1/worker/gpu/warmup.py @@ -294,7 +294,11 @@ def _warmup_kernels( model_runner.scheduler_config.max_num_batched_tokens // max(prompt_len, decode_query_len), ) - if max_blocks_per_req > 0: + block_tables = getattr(model_runner, "block_tables", None) + null_blocks = block_tables is not None and ( + block_tables.redirect_writes_to_null_block + ) + if max_blocks_per_req > 0 and not null_blocks: # Reserve block 0 (null block) and ensure we have enough blocks. # Encoder-only models allocate no KV blocks, so this cap doesn't apply. num_reqs = min( @@ -450,4 +454,6 @@ def _warmup_kernels( cleanup_output.finished_req_ids = set(req_ids) worker_execute_model(cleanup_output) model_runner.kv_connector.set_disabled(False) + if model_runner.kv_block_zeroer is not None: + model_runner.kv_block_zeroer.zero_block_ids([0]) torch.accelerator.synchronize() diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 00e33b47c6..3dc0524bab 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -4833,6 +4833,23 @@ class GPUModelRunner( self.input_batch.num_tokens_no_spec[i] = pos + 1 self.input_batch.prev_req_id_to_index = prev_req_id_to_index + @contextmanager + def preserve_serving_state(self): + multi_block_table = self.input_batch.block_table + saved = [ + (bt.block_table.gpu.clone(), bt.block_table.cpu.clone()) + for bt in multi_block_table.block_tables + ] + multi_block_table.clear() + try: + yield + finally: + for bt, (saved_gpu, saved_cpu) in zip( + multi_block_table.block_tables, saved + ): + bt.block_table.gpu.copy_(saved_gpu) + bt.block_table.cpu.copy_(saved_cpu) + def take_draft_token_ids(self) -> DraftTokenIds | None: if not self.num_spec_tokens or not self._draft_token_req_ids: return None diff --git a/vllm/v1/worker/sentinel/gpu_worker_sentinel.py b/vllm/v1/worker/sentinel/gpu_worker_sentinel.py index 80050cf9d3..9d06a40990 100644 --- a/vllm/v1/worker/sentinel/gpu_worker_sentinel.py +++ b/vllm/v1/worker/sentinel/gpu_worker_sentinel.py @@ -65,6 +65,7 @@ class WorkerSentinel: self.dp_rank, self.dp_size, backend="gloo", + timeout=self.worker.parallel_config.cpu_distributed_timeout, ) def _clean_worker_state(self): diff --git a/vllm/v1/worker/utils.py b/vllm/v1/worker/utils.py index ff6d1a5a6f..a6ef84bbd3 100644 --- a/vllm/v1/worker/utils.py +++ b/vllm/v1/worker/utils.py @@ -383,7 +383,13 @@ def select_common_block_size( for size in sorted(candidates, reverse=True): if block_size_is_supported(backends, size): return size - raise ValueError(f"No common block size for {kv_manager_block_size}.") + raise ValueError( + f"No common block size for {kv_manager_block_size} (" + + "; ".join( + f"{b.get_name()}: {b.get_supported_kernel_block_sizes()}" for b in backends + ) + + ")." + ) def allocate_kv_cache( @@ -741,10 +747,12 @@ def is_uniform_query_len(num_reqs: int, num_tokens: int, max_query_len: int) -> def get_uniform_decode_token_count( - num_reqs: int, num_tokens: int, max_query_len: int, has_prefill: bool + num_reqs: int, num_tokens: int, max_query_len: int, decode_graph_eligible: bool ) -> int | None: """Per-request token count of a uniform decode batch, or None.""" - if not has_prefill and is_uniform_query_len(num_reqs, num_tokens, max_query_len): + if decode_graph_eligible and is_uniform_query_len( + num_reqs, num_tokens, max_query_len + ): return max_query_len return None diff --git a/vllm/version.py b/vllm/version.py index 63095f8bce..76884f4908 100644 --- a/vllm/version.py +++ b/vllm/version.py @@ -26,8 +26,6 @@ def _prev_minor_version_was(version_str): if __version_tuple__[0:2] == (0, 0): return True - # Note - this won't do the right thing when we release 1.0! - assert __version_tuple__[0] == 0 assert isinstance(__version_tuple__[1], int) return version_str == f"{__version_tuple__[0]}.{__version_tuple__[1] - 1}"