Merge remote-tracking branch 'origin/main' into sync/lesj/qwen4exp-bugfixes

Signed-off-by: lesj0610 <lesj0610@users.noreply.github.com>
This commit is contained in:
lesj0610
2026-09-20 09:27:57 +09:00
682 changed files with 26127 additions and 5580 deletions
+11 -5
View File
@@ -22,10 +22,14 @@ description: Write or review Triton kernels for vLLM, with practical guidance fo
a small, legible heuristic when workloads need different choices. Use
`triton.autotune` only when tuning is critical to performance, such as for a
matrix multiplication. Otherwise prioritize simple code and fast startup.
- Be careful to avoid unintended runtime JIT compilation. For example, put
unimportant runtime integer scalars in `do_not_specialize`, especially those
that may alternate between values such as 0 and 1, which can produce
different specialization keys.
- Avoid unintended runtime JIT compilation. Put unimportant runtime integer
scalars in `do_not_specialize`, especially values that alternate between 0
and 1. Pointer alignment also affects specialization: an offset tensor view
can trigger a new variant after warmup used an aligned allocation. For
scalar-only pointers whose alignment provides no useful optimization, use
`@triton.jit(do_not_specialize_on_alignment=["end_ptr"])`. Retain alignment
specialization when vectorized accesses benefit from it. See the
[Triton JIT options](https://triton-lang.org/main/python-api/generated/triton.jit.html).
- The Triton compiler does not guarantee safe ordering when a kernel writes to
a pointer and subsequently reads from the same pointer. This pattern must
have a `tl.debug_barrier()` between the write and read. The barrier
@@ -49,7 +53,9 @@ description: Write or review Triton kernels for vLLM, with practical guidance fo
## Validation
- Check correctness at boundary shapes and at sizes that exercise masks and
large offsets.
large offsets. Include offset tensor views, not only fresh allocations.
Warm aligned buffers, then exercise offset views with vLLM's
`--jit-monitor-mode error --jit-monitor-verbose` to catch missing variants.
- Choose accumulation and intermediate dtypes explicitly. Test numerically
difficult inputs, not only random, well-scaled tensors.
- Use `$kernel-microbenchmark` for benchmark construction, measurement, and
+1 -1
View File
@@ -98,7 +98,7 @@ steps:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'cd tests && pip install dpctl &&
pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py &&
pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py --ignore=v1/core/prefix_cache/test_hybrid_prefix_cache_hit_rate.py &&
pytest -v -s v1/engine --ignore=v1/engine/test_output_processor.py &&
pytest -v -s v1/sample --ignore=v1/sample/test_logprobs.py --ignore=v1/sample/test_logprobs_e2e.py -k "not test_topk_only and not test_topp_only and not test_topk_and_topp" &&
pytest -v -s v1/worker --ignore=v1/worker/test_gpu_model_runner.py --ignore=v1/worker/test_worker_memory_snapshot.py &&
+92
View File
@@ -316,6 +316,66 @@ steps:
- "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129"
- 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129" release-images'
- label: "Build release image - x86_64 - Rubin CUDA 13.4"
depends_on: ~
id: build-release-image-x86-rubin-cuda-13-4
if: build.env("NIGHTLY") == "1"
agents:
queue: cpu_queue_release
commands:
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
- |
DOCKER_BUILDKIT=1 docker buildx build --load \
$(bash .buildkite/scripts/docker-build-metadata-args.sh cu134) \
--platform linux/amd64 \
--build-arg max_jobs=16 \
--build-arg USE_SCCACHE=1 \
--build-arg GIT_REPO_CHECK=1 \
--build-arg CUDA_VERSION=13.4 \
--build-arg torch_cuda_arch_list="${CUDA_ARCH_X86}" \
--build-arg INSTALL_KV_CONNECTORS=true \
--build-arg INSTALL_RUBIN_PRERELEASE=true \
--build-arg TRITON_INSTALL_FROM_SOURCE_REPO=https://github.com/triton-lang/triton.git \
--build-arg TRITON_INSTALL_FROM_SOURCE_REVISION=3f6e41132b5edf639bfb872ad73d4688765e08b8 \
--build-arg RUN_WHEEL_CHECK=false \
--build-arg BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda13.4 \
--build-arg FINAL_BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:26.08-cuda13.4-devel-ubuntu24.04 \
--target vllm-openai \
--progress plain \
-f docker/Dockerfile .
- "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu134"
- 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu134" release-images'
- label: "Build release image - aarch64 - Rubin CUDA 13.4"
depends_on: ~
id: build-release-image-arm64-rubin-cuda-13-4
if: build.env("NIGHTLY") == "1"
agents:
queue: arm64_cpu_queue_release
commands:
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
- |
DOCKER_BUILDKIT=1 docker buildx build --load \
$(bash .buildkite/scripts/docker-build-metadata-args.sh cu134) \
--platform linux/arm64 \
--build-arg max_jobs=16 \
--build-arg USE_SCCACHE=1 \
--build-arg GIT_REPO_CHECK=1 \
--build-arg CUDA_VERSION=13.4 \
--build-arg torch_cuda_arch_list="${CUDA_ARCH_AARCH64}" \
--build-arg INSTALL_KV_CONNECTORS=true \
--build-arg INSTALL_RUBIN_PRERELEASE=true \
--build-arg TRITON_INSTALL_FROM_SOURCE_REPO=https://github.com/triton-lang/triton.git \
--build-arg TRITON_INSTALL_FROM_SOURCE_REVISION=3f6e41132b5edf639bfb872ad73d4688765e08b8 \
--build-arg RUN_WHEEL_CHECK=false \
--build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda13.4 \
--build-arg FINAL_BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:26.08-cuda13.4-devel-ubuntu24.04 \
--target vllm-openai \
--progress plain \
-f docker/Dockerfile .
- "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu134"
- 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu134" release-images'
- label: "Build release image - x86_64 - CUDA 13.0 - Ubuntu 24.04"
depends_on: ~
id: build-release-image-x86-ubuntu2404
@@ -513,6 +573,20 @@ steps:
- "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129"
- 'bash .buildkite/scripts/annotate-build-artifact.sh "Manifest: CUDA 12.9" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129" release-manifests'
- label: "Create multi-arch manifest - Rubin CUDA 13.4"
depends_on:
- build-release-image-x86-rubin-cuda-13-4
- build-release-image-arm64-rubin-cuda-13-4
id: create-multi-arch-manifest-rubin-cuda-13-4
if: build.env("NIGHTLY") == "1"
agents:
queue: small_cpu_queue_release
commands:
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
- "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu134 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-cu134 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-cu134 --amend"
- "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu134"
- 'bash .buildkite/scripts/annotate-build-artifact.sh "Manifest: Rubin CUDA 13.4" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu134" release-manifests'
- label: "Create multi-arch manifest - CUDA 13.0 - Ubuntu 24.04"
depends_on:
- build-release-image-x86-ubuntu2404
@@ -585,6 +659,24 @@ steps:
DOCKER_BUILDKIT: "1"
DOCKERHUB_USERNAME: "vllmbot"
- label: "Publish nightly multi-arch Rubin image to DockerHub - CUDA 13.4"
depends_on:
- create-multi-arch-manifest-rubin-cuda-13-4
if: build.env("NIGHTLY") == "1"
agents:
queue: small_cpu_queue_release
commands:
- "bash .buildkite/scripts/push-nightly-builds.sh cu134"
# Clean up old nightly builds (keep only last 14)
- "bash .buildkite/scripts/cleanup-nightly-builds.sh cu134-nightly-"
plugins:
- docker-login#v3.0.0:
username: vllmbot
password-env: DOCKERHUB_TOKEN
env:
DOCKER_BUILDKIT: "1"
DOCKERHUB_USERNAME: "vllmbot"
- label: "Publish nightly CPU multi-arch image to DockerHub"
depends_on:
- build-cpu-release-image-x86
+1 -1
View File
@@ -418,7 +418,7 @@ validate_ci_build_context_source() {
describe_ci_revision() {
git -C "$1" describe --tags --long --abbrev=10 \
--match '*[0-9]*' "$2" 2>/dev/null
--match 'v[0-9]*' "$2" 2>/dev/null
}
write_ci_git_archival_metadata() {
+12 -3
View File
@@ -10,11 +10,20 @@ fi
INPUT_FILE="$1"
# GNU sed uses "-i [-r]" while BSD/macOS sed needs "-i '' [-E]"; detect once.
if sed --version >/dev/null 2>&1; then
sedi() { sed -i "$@"; }
SED_ERE=-r
else
sedi() { sed -i '' "$@"; }
SED_ERE=-E
fi
# Strip timestamps
sed -i 's/^\[[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}T[0-9]\{2\}:[0-9]\{2\}:[0-9]\{2\}Z\] //' "$INPUT_FILE"
sedi 's/^\[[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}T[0-9]\{2\}:[0-9]\{2\}:[0-9]\{2\}Z\] //' "$INPUT_FILE"
# Strip Buildkite inline timestamp markers (ESC _bk;t=<ms> BEL)
sed -i 's/\x1B_bk;t=[0-9]*\x07//g' "$INPUT_FILE"
sedi 's/\x1B_bk;t=[0-9]*\x07//g' "$INPUT_FILE"
# Strip colorization
sed -i -r 's/\x1B\[[0-9;]*[mK]//g' "$INPUT_FILE"
sedi "$SED_ERE" 's/\x1B\[[0-9;]*[mK]//g' "$INPUT_FILE"
+2 -2
View File
@@ -56,8 +56,8 @@ get_all_tags() {
set -x
# Get both last_updated timestamp and tag name, separated by |
# Exclude nightly-dev tags from cleanup
local tags=$(echo "$response" | jq -r --arg prefix "$TAG_PREFIX" '.results[] | select(.name | startswith($prefix)) | select(.name | startswith("nightly-dev") | not) | "\(.last_updated)|\(.name)"')
# Exclude mutable architecture tags and nightly-dev tags from cleanup
local tags=$(echo "$response" | jq -r --arg prefix "$TAG_PREFIX" '.results[] | select(.name | startswith($prefix)) | select(.name != ($prefix + "x86_64") and .name != ($prefix + "aarch64")) | select(.name | startswith("nightly-dev") | not) | "\(.last_updated)|\(.name)"')
if [ -z "$tags" ]; then
break
@@ -1612,6 +1612,13 @@ fi
echo "Final commands: $commands"
# Match native CPU jobs even when the container can see AMD devices.
cpu_platform_env=()
if [[ "${VLLM_CI_EXPECTED_GPU_COUNT:-1}" == "0" \
&& "$commands" != *python_only_compile.sh* ]]; then
cpu_platform_env=(-e "VLLM_TARGET_DEVICE=cpu")
fi
standalone_merge_base_env=()
if [[ "$commands" == *python_only_compile.sh* ]]; then
# The ROCm test image often ships /vllm-workspace without .git. Resolve the
@@ -1769,6 +1776,7 @@ else
-e "VLLM_CACHE_ROOT=${CONTAINER_CACHE_ROOT}/vllm" \
-e "XDG_CACHE_HOME=${CONTAINER_CACHE_ROOT}/xdg" \
-e "PYTORCH_ROCM_ARCH=" \
"${cpu_platform_env[@]}" \
"${standalone_merge_base_env[@]}" \
--name "${container_name}" \
"${image_name}" \
+1197 -548
View File
File diff suppressed because it is too large Load Diff
+2 -2
View File
@@ -92,7 +92,7 @@ steps:
dind: false
device: mi355_1
working_dir: "/vllm-workspace/"
timeout_in_minutes: 25
timeout_in_minutes: 30
depends_on:
- image-build-amd
source_file_dependencies:
@@ -133,7 +133,7 @@ steps:
dind: false
device: mi355_1
working_dir: "/vllm-workspace/"
timeout_in_minutes: 30
timeout_in_minutes: 40
depends_on:
- image-build-amd
source_file_dependencies:
+3 -2
View File
@@ -23,7 +23,7 @@ steps:
dind: false
device: mi355_1
working_dir: "/vllm-workspace/tests"
timeout_in_minutes: 20
timeout_in_minutes: 25
depends_on:
- image-build-amd
source_file_dependencies:
@@ -49,8 +49,9 @@ steps:
mirror:
amd:
label: ":amd: (MI250) CUDAGraph"
dind: false
device: mi250_1
timeout_in_minutes: 45
timeout_in_minutes: 55
depends_on:
- image-build-amd
source_file_dependencies:
+11 -8
View File
@@ -16,9 +16,9 @@ steps:
- bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
mirror:
amd:
label: ":amd: (MI300) Distributed NixlConnector PD accuracy"
label: ":amd: (MI355) Distributed NixlConnector PD accuracy"
dind: false
device: mi300_4
device: mi355_4
timeout_in_minutes: 60
depends_on:
- image-build-amd
@@ -71,9 +71,9 @@ steps:
- DP_EP=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
mirror:
amd:
label: ":amd: (MI300) DP EP Distributed NixlConnector PD accuracy"
label: ":amd: (MI355) DP EP Distributed NixlConnector PD accuracy"
dind: false
device: mi300_4
device: mi355_4
timeout_in_minutes: 40
depends_on:
- image-build-amd
@@ -130,7 +130,7 @@ steps:
label: ":amd: (MI300) Hybrid SSM NixlConnector PD accuracy"
dind: false
device: mi300_4
timeout_in_minutes: 55
timeout_in_minutes: 60
depends_on:
- image-build-amd
source_file_dependencies:
@@ -221,9 +221,9 @@ steps:
- bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh
mirror:
amd:
label: ":amd: (MI300) NixlConnector PD + Spec Decode acceptance"
label: ":amd: (MI355) NixlConnector PD + Spec Decode acceptance"
dind: false
device: mi300_2
device: mi355_2
timeout_in_minutes: 45
depends_on:
- image-build-amd
@@ -287,7 +287,7 @@ steps:
PREFILL_BLOCK_SIZE: "256"
DECODE_BLOCK_SIZE: "256"
MODEL_NAMES: "deepseek-ai/DeepSeek-V4-Flash"
VLLM_ENGINE_READY_TIMEOUT_S: "900"
VLLM_ENGINE_READY_TIMEOUT_S: "1800"
VLLM_SERVE_EXTRA_ARGS: "--trust-remote-code,--kv-cache-dtype,fp8"
# The V4 indexer only supports block-outermost layouts; let it pick.
PREFILLER_KV_LAYOUT: ""
@@ -311,6 +311,9 @@ steps:
PREFILLER_TP_SIZE: "2"
DECODER_TP_SIZE: "2"
MODEL_NAMES: "moonshotai/Kimi-Linear-48B-A3B-Instruct"
# Keep TP=2 prefill and TP=1 decode above both hybrid page-size minima.
PREFILL_BLOCK_SIZE: "2048"
DECODE_BLOCK_SIZE: "2048"
VLLM_SERVE_EXTRA_ARGS: "--trust-remote-code,--mamba-cache-mode align,--max-model-len 32768"
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
@@ -40,7 +40,14 @@ steps:
- vllm/v1/worker/gpu_worker.py
- vllm/v1/worker/gpu/ec_connector.py
- vllm/v1/worker/gpu/model_runner.py
- vllm/v1/worker/mm_encoder_model_runner.py
- vllm/v1/worker/encoder_cudagraph.py
- vllm/v1/worker/encoder_cudagraph_defs.py
- vllm/v1/attention/ops/vit_attn_wrappers.py
- vllm/v1/worker/gpu/model_states/
- vllm/v1/worker/gpu/mm/
- tests/v1/cudagraph/test_encoder_cudagraph.py
- tests/models/multimodal/generation/test_vit_cudagraph.py
- examples/disaggregated/disaggregated_encoder/
- tests/v1/ec_connector/
- requirements/kv_connectors.txt
@@ -49,6 +56,9 @@ steps:
PYTHONUNBUFFERED: "1"
VLLM_HOST_IP: "127.0.0.1"
VLLM_USE_V2_MODEL_RUNNER: "1"
E_CUDAGRAPH_MM_ENCODER: "1"
# FA padding NaNs are tracked separately in #57136.
E_MM_ENCODER_ATTN_BACKEND: "TRITON_ATTN"
MOONCAKE_EC_PROTOCOL: "tcp"
USE_MM_PROMPTS: "1"
SKIP_BASELINE: "0"
+57 -2
View File
@@ -65,9 +65,9 @@ steps:
- pytest -v -s v1/e2e/spec_decode/test_sharded_sampling.py
mirror:
amd:
label: ":amd: (MI300) Distributed DP Basic"
label: ":amd: (MI355) Distributed DP Basic"
dind: false
device: mi300_2
device: mi355_2
timeout_in_minutes: 45
depends_on:
- image-build-amd
@@ -125,6 +125,9 @@ steps:
- tests/compile/test_wrapper.py
- tests/entrypoints/llm/test_collective_rpc.py
- vllm/platforms/rocm.py
commands:
- pytest -v -s entrypoints/llm/test_collective_rpc.py
- pytest -v -s ./compile/test_wrapper.py
- label: ":nvidia: (L4) Distributed Torchrun + Shutdown"
key: distributed-torchrun-shutdown-tests-2-gpus
@@ -296,6 +299,57 @@ steps:
- pytest -v -s tests/v1/distributed/test_dbo.py
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py
- pytest -v -s tests/distributed/test_packed_tensor.py
mirror:
amd:
label: ':amd: (MI355) Distributed Features'
dind: false
device: mi355_2
timeout_in_minutes: 90
working_dir: /vllm-workspace/
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/
- vllm/v1/distributed/
- vllm/model_executor/layers/fused_moe/
- vllm/v1/attention/backends/
- vllm/v1/attention/selector.py
- tests/v1/distributed/test_dbo.py
- tests/distributed/test_context_parallel.py
- examples/features/data_parallel/data_parallel_offline.py
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- csrc/custom_quickreduce.cu
- csrc/ops.h
- csrc/torch_bindings.cpp
- vllm/model_executor/layers/
- vllm/entrypoints/llm.py
- vllm/config/parallel.py
- vllm/v1/engine/
- vllm/v1/executor/
- vllm/v1/worker/
- vllm/_custom_ops.py
- vllm/envs.py
- tests/distributed/test_rocm_aiter_custom_ar.py
- tests/distributed/test_rocm_quick_reduce.py
- tests/distributed/test_quick_all_reduce.py
- tests/v1/e2e/general/test_rocm_aiter_custom_ar.py
- tests/utils.py
- examples/rl/rlhf_async_new_apis.py
- tests/distributed/test_weight_transfer.py
- tests/distributed/test_packed_tensor.py
commands:
- pytest -v -s tests/distributed/test_context_parallel.py
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_async_new_apis.py
- VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=deepep_high_throughput
- VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=allgather_reducescatter --disable-nccl-for-dp-synchronization
- pytest -v -s tests/v1/distributed/test_dbo.py
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py
- pytest -v -s tests/distributed/test_packed_tensor.py
- pytest -v -s tests/distributed/test_rocm_aiter_custom_ar.py
- pytest -v -s tests/v1/e2e/general/test_rocm_aiter_custom_ar.py
- pytest -v -s tests/distributed/test_rocm_quick_reduce.py
- pytest -v -s tests/distributed/test_quick_all_reduce.py
- label: ":nvidia: (B200) Distributed"
key: distributed-tests-2xb200
@@ -362,6 +416,7 @@ steps:
mirror:
amd:
label: ":amd: (MI250) Pipeline + Context Parallelism"
dind: false
device: mi250_4
timeout_in_minutes: 90
depends_on:
+1
View File
@@ -18,6 +18,7 @@ steps:
mirror:
amd:
label: ":amd: (MI250) Docker Build Metadata"
dind: false
device: mi250_1
no_gpu: true
timeout_in_minutes: 35
+1 -1
View File
@@ -57,7 +57,7 @@ steps:
dind: false
device: mi355_1
working_dir: "/vllm-workspace"
timeout_in_minutes: 20
timeout_in_minutes: 30
depends_on:
- image-build-amd
source_file_dependencies:
+8 -1
View File
@@ -52,10 +52,13 @@ steps:
mirror:
amd:
label: ":amd: (MI250) V1 Engine"
dind: false
device: mi250_1
timeout_in_minutes: 45
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/platforms/rocm.py
- label: ":nvidia: (H200 MIG 18GB) E2E Scheduling"
key: e2e-scheduling-1-gpu
@@ -69,15 +72,18 @@ steps:
mirror:
amd:
label: ":amd: (MI250) E2E Scheduling"
dind: false
device: mi250_1
timeout_in_minutes: 55
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/platforms/rocm.py
- label: ":nvidia: (H200 MIG 18GB) E2E Core"
device: h200_18gb
key: e2e-core-1-gpu
timeout_in_minutes: 50
timeout_in_minutes: 65
source_file_dependencies:
- vllm/v1/
- tests/v1/e2e/general/
@@ -86,6 +92,7 @@ steps:
mirror:
amd:
label: ":amd: (MI250) E2E Core"
dind: false
commands:
- pytest -v -s v1/e2e/general --ignore v1/e2e/general/test_async_scheduling.py
device: mi250_1
+22 -2
View File
@@ -21,9 +21,11 @@ steps:
label: ":amd: (MI355) Entrypoints Unit"
dind: false
device: mi355_1
timeout_in_minutes: 35
timeout_in_minutes: 50
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/platforms/rocm.py
- label: ":nvidia: (H200 MIG 35GB) Entrypoints Integration (LLM)"
device: h200_35gb
@@ -44,7 +46,7 @@ steps:
label: ":amd: (MI300) Entrypoints Integration (LLM)"
dind: false
device: mi300_1
timeout_in_minutes: 55
timeout_in_minutes: 90
depends_on:
- image-build-amd
@@ -176,6 +178,15 @@ steps:
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s entrypoints/speech_to_text
mirror:
amd:
label: ":amd: (MI355) Entrypoints Integration (Speech to Text)"
dind: false
device: mi355_1
timeout_in_minutes: 60
working_dir: "/vllm-workspace/tests"
depends_on:
- image-build-amd
- label: ":nvidia: (H200 MIG 35GB) Entrypoints Integration (Multimodal)"
device: h200_35gb
@@ -210,6 +221,15 @@ steps:
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s entrypoints/pooling
mirror:
amd:
label: ":amd: (MI355) Entrypoints Integration (Pooling)"
dind: false
device: mi355_1
timeout_in_minutes: 65
working_dir: "/vllm-workspace/tests"
depends_on:
- image-build-amd
- label: ":nvidia: (H200 MIG 18GB) OpenAI API Correctness"
key: openai-api-correctness
@@ -43,7 +43,7 @@ steps:
- label: ":nvidia: (H100) Elastic EP Scaling"
key: elastic-ep-scaling-test
timeout_in_minutes: 30
timeout_in_minutes: 40
device: h100
working_dir: "/vllm-workspace/tests"
num_devices: 4
+192 -4
View File
@@ -13,6 +13,22 @@ steps:
commands:
- pytest -v -s tests/ir
- pytest -v -s tests/kernels/ir
mirror:
amd:
label: ":amd: (MI355) vLLM IR"
dind: false
device: mi355_1
timeout_in_minutes: 50
working_dir: "/vllm-workspace/"
depends_on:
- image-build-amd
source_file_dependencies:
- tests/ir/
- tests/kernels/ir/
- vllm/ir
- vllm/kernels
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- label: ":nvidia: (H200 MIG 35GB) Core Operation Kernels Shard %N"
device: h200_35gb
@@ -26,6 +42,29 @@ steps:
commands:
- pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_concat_mla_q.py kernels/test_fused_qk_norm_rope_gate.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
parallelism: 3
mirror:
amd:
label: ":amd: (MI355) Core Operation Kernels Shard %N"
dind: false
device: mi355_1
timeout_in_minutes: 40
working_dir: "/vllm-workspace/tests"
depends_on:
- image-build-amd
source_file_dependencies:
- csrc/
- tests/kernels/core
- tests/kernels/test_top_k_per_row.py
- tests/kernels/test_concat_mla_q.py
- tests/kernels/test_rocm_fp32_router_gemm.py
- vllm/model_executor/layers/fused_moe/router/gate_linear.py
- vllm/model_executor/layers/fused_moe/router/rocm_fp32_router_gemm.py
- vllm/model_executor/layers/rotary_embedding/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- tests/kernels/test_fused_qk_norm_rope_gate.py
commands:
- pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_fused_qk_norm_rope_gate.py kernels/test_concat_mla_q.py kernels/test_top_k_per_row.py kernels/test_rocm_fp32_router_gemm.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- label: ":nvidia: (H100) MiniMax Reduce RMS Kernels"
key: kernels-minimax-reduce-rms-test-2-gpus
@@ -81,6 +120,7 @@ steps:
- csrc/fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu
- vllm/models/deepseek_v4/common/ops/
- vllm/models/deepseek_v4/nvidia/
- vllm/utils/deep_gemm.py
- tests/kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py
- tests/models/test_deepseek_v4_mega_moe.py
commands:
@@ -145,6 +185,7 @@ steps:
dind: false
device: mi300_1
timeout_in_minutes: 90
parallelism: 6
depends_on:
- image-build-amd
source_file_dependencies:
@@ -208,9 +249,6 @@ steps:
label: ":amd: (MI300) Quantization Kernels Shard %N"
dind: false
device: mi300_1
# Pin the AMD mirror at its current 2 shards so raising NVIDIA
# parallelism does not multiply AMD copies (needs ci-infra #473).
parallelism: 2
timeout_in_minutes: 120
source_file_dependencies:
- csrc/quantization/
@@ -222,6 +260,12 @@ steps:
- vllm/kernels/aiter_ops.py
- vllm/platforms/rocm.py
- vllm/model_executor/kernels/
- csrc/rocm/
- vllm/v1/attention/backends/rocm_aiter_fa.py
- tests/kernels/quant_utils.py
- tests/kernels/utils.py
- vllm/_custom_ops.py
- vllm/envs.py
depends_on:
- image-build-amd
@@ -258,6 +302,10 @@ steps:
- vllm/config
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- csrc/rocm/
- tests/kernels/utils.py
- vllm/model_executor/layers/quantization/mxfp4.py
- vllm/model_executor/layers/quantization/utils/quant_utils.py
depends_on:
- image-build-amd
@@ -268,16 +316,20 @@ steps:
source_file_dependencies:
- csrc/mamba/
- tests/kernels/mamba
- vllm/model_executor/layers/mamba/gdn
- vllm/model_executor/layers/mamba/ops
commands:
- pytest -v -s kernels/mamba
mirror:
amd:
label: ":amd: (MI250) Mamba Kernels"
dind: false
device: mi250_1
timeout_in_minutes: 55
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/platforms/rocm.py
- label: ":nvidia: (H100) DeepGEMM Kernels"
key: kernels-deepgemm-test-h100
@@ -317,6 +369,7 @@ steps:
- csrc/quantization/fp4/
- csrc/libtorch_stable/nvfp4_kv_cache_kernels.cu
- csrc/libtorch_stable/quantization/fp4/
- csrc/libtorch_stable/quantization/w8a8/cutlass/
- csrc/attention/mla/
- csrc/quantization/cutlass_w8a8/moe/
- vllm/model_executor/layers/fusion/
@@ -358,6 +411,7 @@ steps:
- tests/kernels/test_ll_bf16_gemm.py
- tests/kernels/test_top_k_per_row.py
- tests/kernels/quantization/test_silu_mul_nvfp4_quant.py
- tests/kernels/quantization/test_block_fp8.py
- tests/models/quantization/test_nvfp4.py
commands:
- nvidia-smi
@@ -392,6 +446,7 @@ steps:
pytest -v -s tests/kernels/attention/test_flashinfer.py -k 'not num_heads2'
pytest -v -s tests/kernels/attention/test_cutlass_mla_decode.py
pytest -v -s tests/kernels/quantization/test_cutlass_scaled_mm.py -k 'fp8'
pytest -v -s tests/kernels/quantization/test_block_fp8.py::test_w8a8_block_fp8_cutlass_matmul
pytest -v -s tests/kernels/quantization/test_nvfp4_quant.py
pytest -v -s tests/kernels/quantization/test_nvfp4_scaled_mm.py
pytest -v -s tests/kernels/quantization/test_mxfp4_qutlass.py
@@ -459,10 +514,14 @@ steps:
mirror:
amd:
label: ":amd: (MI250) Helion Kernels Shard %N"
dind: false
device: mi250_1
timeout_in_minutes: 45
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/platforms/rocm.py
- scripts/benchmark_helion_kernels.py
- label: ":nvidia: (H100) FP8 MoE Kernels"
@@ -491,6 +550,25 @@ steps:
commands:
- pytest -v -s kernels/moe/test_deepep_deepgemm_moe.py
- pytest -v -s kernels/moe/test_deepep_moe.py
mirror:
amd:
label: ":amd: (MI355) DeepEP FP8 MoE Kernels"
dind: false
device: mi355_2
timeout_in_minutes: 55
working_dir: "/vllm-workspace/tests"
depends_on:
- image-build-amd
source_file_dependencies:
- csrc/moe/
- csrc/quantization/w8a8/cutlass/moe/
- vllm/model_executor/layers/fused_moe/
- tests/kernels/moe/test_deepep_moe.py
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- vllm/envs.py
commands:
- pytest -v -s kernels/moe/test_deepep_moe.py
- label: ":nvidia: (B200) FP4 MoE Kernels"
key: kernels-fp4-moe-test-b200
@@ -520,7 +598,30 @@ steps:
- vllm/config
commands:
- pytest -v -s kernels/moe/test_moe_layer.py
mirror:
amd:
label: ":amd: (MI355) FusedMoE Layer Kernels"
dind: false
device: mi355_2
timeout_in_minutes: 35
working_dir: "/vllm-workspace/tests"
depends_on:
- image-build-amd
source_file_dependencies:
- csrc/moe/
- csrc/rocm/
- tests/kernels/moe
- vllm/model_executor/layers/fused_moe/
- vllm/model_executor/layers/quantization/
- vllm/distributed/
- vllm/config/
- vllm/forward_context.py
- vllm/v1/worker/workspace.py
- vllm/utils/import_utils.py
- vllm/utils/math_utils.py
- vllm/utils/torch_utils.py
- vllm/platforms/
- vllm/_aiter_ops.py
- label: ":nvidia: (B200) FusedMoE Layer Kernels"
key: kernels-fusedmoe-layer-test-2-b200s
@@ -538,3 +639,90 @@ steps:
commands:
- pytest -v -s kernels/moe/test_moe_layer.py
- pytest -v -s kernels/moe/test_deepep_v2_moe.py
- label: ":amd: (MI355) MLA Kernels"
key: amd-mla-kernels-mi355
dind: false
device: mi355_1
num_devices: 1
timeout_in_minutes: 25
working_dir: /vllm-workspace/
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/v1/attention/backends/mla/rocm_aiter_mla.py
- vllm/v1/attention/selector.py
- vllm/platforms/rocm.py
- vllm/_aiter_ops.py
- tests/kernels/attention/test_rocm_aiter_mla_decode.py
- tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py
- tests/kernels/attention/test_rocm_aiter_mla_fp8_support.py
- tests/kernels/attention/test_rocm_aiter_mla_op_registration.py
commands:
- amd-smi
- pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py
- pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode.py
- pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_fp8_support.py
- pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_op_registration.py
- label: ":amd: (MI355) FP8 MoE Kernels"
key: amd-fp8-moe-kernels-mi355
dind: false
device: mi355_1
num_devices: 1
timeout_in_minutes: 40
working_dir: /vllm-workspace/tests
depends_on:
- image-build-amd
source_file_dependencies:
- csrc/moe/
- vllm/model_executor/layers/fused_moe/
- tests/kernels/moe/test_deepep_moe.py
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- vllm/envs.py
- tests/kernels/moe/test_gpt_oss_triton_kernels.py
- tests/kernels/moe/test_modular_oai_triton_moe.py
- tests/kernels/moe/test_moe.py
- tests/kernels/moe/test_block_int8.py
- tests/kernels/moe/test_triton_moe_no_act_mul.py
- tests/kernels/moe/test_triton_moe_ptpc_fp8.py
commands:
- pytest -v -s kernels/moe/test_gpt_oss_triton_kernels.py
- pytest -v -s kernels/moe/test_modular_oai_triton_moe.py
- pytest -v -s kernels/moe/test_moe.py
- pytest -v -s kernels/moe/test_block_int8.py
- pytest -v -s kernels/moe/test_triton_moe_no_act_mul.py
- pytest -v -s kernels/moe/test_triton_moe_ptpc_fp8.py
- label: ":amd: (MI355) Kernels"
key: amd-kernels-mi355
dind: false
device: mi355_1
num_devices: 1
timeout_in_minutes: 25
working_dir: /vllm-workspace/
depends_on:
- image-build-amd
source_file_dependencies:
- csrc/quantization/fp4/
- csrc/attention/mla/
- csrc/quantization/cutlass_w8a8/moe/
- vllm/model_executor/layers/fused_moe/cutlass_moe.py
- vllm/v1/attention/backends/triton_attn.py
- vllm/v1/attention/backends/rocm_attn.py
- vllm/v1/attention/backends/rocm_aiter_fa.py
- vllm/v1/attention/backends/rocm_aiter_unified_attn.py
- vllm/v1/attention/backends/mla/aiter_triton_mla.py
- vllm/v1/attention/backends/mla/rocm_aiter_mla.py
- vllm/v1/attention/selector.py
- vllm/platforms/rocm.py
- vllm/_aiter_ops.py
- examples/basic/offline_inference/chat.py
- tests/kernels/attention/test_attention_selector.py
- tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py
commands:
- amd-smi
- python3 examples/basic/offline_inference/chat.py --attention-backend TRITON_ATTN
- pytest -v -s tests/kernels/attention/test_attention_selector.py
- pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py
+27 -3
View File
@@ -30,6 +30,27 @@ steps:
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- label: ":amd: (MI300) LM Eval Small Models Harness"
key: amd-lm-eval-small-models-harness
dind: false
device: mi300_1
timeout_in_minutes: 35
depends_on:
- image-build-amd
working_dir: /vllm-workspace/.buildkite/lm-eval-harness
source_file_dependencies:
- csrc/
- vllm/model_executor/layers/quantization
- vllm/model_executor/models/
- vllm/model_executor/model_loader/
- vllm/v1/attention/backends/
- vllm/v1/attention/selector.py
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- .buildkite/lm-eval-harness/
commands:
- pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-small-rocm.txt
- label: ":nvidia: (H200 MIG 35GB) LM Eval Watermarking"
device: h200_35gb
key: lm-eval-watermarking
@@ -100,7 +121,7 @@ steps:
label: ":amd: (MI355) LM Eval Small Models Distributed"
dind: false
device: mi355_2
timeout_in_minutes: 45
timeout_in_minutes: 55
depends_on:
- image-build-amd
source_file_dependencies:
@@ -275,7 +296,7 @@ steps:
- label: ":nvidia: (H200) LM Eval Large Models"
key: lm-eval-large-models-8xh200
timeout_in_minutes: 50
timeout_in_minutes: 120
device: h200
optional: true
num_devices: 8
@@ -286,13 +307,16 @@ steps:
label: ":amd: (MI300) LM Eval Large Models"
dind: false
device: mi300_8
timeout_in_minutes: 40
timeout_in_minutes: 65
depends_on:
- image-build-amd
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- export PYTORCH_ROCM_ARCH=gfx942 # Limit Quark compilation to save time
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx.txt
source_file_dependencies:
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- label: ":nvidia: (H100) MoE Refactor Integration TEMPORARY"
key: moe-refactor-integration-test-h100-temporary
+9 -9
View File
@@ -61,7 +61,7 @@ steps:
dind: false
device: mi300_4
working_dir: "/vllm-workspace/tests"
timeout_in_minutes: 25
timeout_in_minutes: 65
depends_on:
- image-build-amd
source_file_dependencies:
@@ -73,11 +73,11 @@ steps:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
# expandable_segments breaks custom all-reduce IPC on ROCm.
- case "$$BUILDKITE_PARALLEL_JOB" in 0|1|2|3) ;; *) echo "unexpected BUILDKITE_PARALLEL_JOB=$$BUILDKITE_PARALLEL_JOB" && exit 1;; esac
- test "$$BUILDKITE_PARALLEL_JOB" != "3" || pytest -v -s -x lora/test_chatglm3_tp.py
- test "$$BUILDKITE_PARALLEL_JOB" != "1" || pytest -v -s -x lora/test_llama_tp.py
- test "$$BUILDKITE_PARALLEL_JOB" != "3" || pytest -v -s -x lora/test_qwen3_with_multi_loras.py
- test "$$BUILDKITE_PARALLEL_JOB" != "2" || pytest -v -s -x lora/test_olmoe_tp.py
- test "$$BUILDKITE_PARALLEL_JOB" != "0" || pytest -v -s -x lora/test_gptoss_tp.py
- test "$$BUILDKITE_PARALLEL_JOB" != "1" || pytest -v -s -x lora/test_qwen35_densemodel_lora.py
- test "$$BUILDKITE_PARALLEL_JOB" != "2" || pytest -v -s -x lora/test_gemma4_tp.py
- test "$$BUILDKITE_PARALLEL_JOB" != "0" || pytest -v -s -x lora/test_sequence_classification.py::test_batched_loras_tp
- if [ "$$BUILDKITE_PARALLEL_JOB" = "3" ]; then pytest -v -s -x lora/test_chatglm3_tp.py; fi
- if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then pytest -v -s -x lora/test_llama_tp.py; fi
- if [ "$$BUILDKITE_PARALLEL_JOB" = "3" ]; then pytest -v -s -x lora/test_qwen3_with_multi_loras.py; fi
- if [ "$$BUILDKITE_PARALLEL_JOB" = "2" ]; then pytest -v -s -x lora/test_olmoe_tp.py; fi
- if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s -x lora/test_gptoss_tp.py; fi
- if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then pytest -v -s -x lora/test_qwen35_densemodel_lora.py; fi
- if [ "$$BUILDKITE_PARALLEL_JOB" = "2" ]; then pytest -v -s -x lora/test_gemma4_tp.py; fi
- if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s -x lora/test_sequence_classification.py::test_batched_loras_tp; fi
+37 -7
View File
@@ -99,7 +99,7 @@ steps:
label: ":amd: (MI300) V1 Core"
dind: false
device: mi300_1
timeout_in_minutes: 45
timeout_in_minutes: 60
depends_on:
- image-build-amd
@@ -119,7 +119,7 @@ steps:
label: ":amd: (MI300) V1 Executor + Worker"
dind: false
device: mi300_1
timeout_in_minutes: 45
timeout_in_minutes: 60
depends_on:
- image-build-amd
@@ -138,7 +138,7 @@ steps:
label: ":amd: (MI300) V1 KV Offload"
dind: false
device: mi300_1
timeout_in_minutes: 45
timeout_in_minutes: 60
depends_on:
- image-build-amd
@@ -158,7 +158,7 @@ steps:
label: ":amd: (MI300) V1 KV Connectors Shard %N"
dind: false
device: mi300_1
timeout_in_minutes: 45
timeout_in_minutes: 60
depends_on:
- image-build-amd
@@ -258,7 +258,7 @@ steps:
label: ":amd: (MI355) Extract Hidden States Integration"
dind: false
device: mi355_2
timeout_in_minutes: 20
timeout_in_minutes: 30
depends_on:
- image-build-amd
source_file_dependencies:
@@ -305,6 +305,15 @@ steps:
- pip install 'modelscope<1.38'
- pytest -v -s test_regression.py
working_dir: "/vllm-workspace/tests" # optional
mirror:
amd:
label: ":amd: (MI355) Regression"
dind: false
device: mi355_1
timeout_in_minutes: 25
depends_on:
- image-build-amd
working_dir: /vllm-workspace/tests
- label: ":nvidia: (H200 MIG 35GB) Examples"
device: h200_35gb
@@ -446,9 +455,15 @@ steps:
label: ":amd: (MI355) Async Engine, Inputs, Utils, Worker"
dind: false
device: mi355_1
timeout_in_minutes: 40
timeout_in_minutes: 50
depends_on:
- image-build-amd
commands:
- pytest -v -s detokenizer
- pytest -v -s -m 'not cpu_test' multimodal
# test_cache.py marks every test cpu_test, including this GPU regression.
- pytest -v -s multimodal/test_cache.py::test_sleep_wake_preserves_mm_cache_consistency
- pytest -v -s utils_
- label: ":computer: (CPU) Params, Env, Tokenizers, Parser"
key: cpu-params-env-tokenizers-parser
@@ -571,8 +586,9 @@ steps:
mirror:
amd:
label: ":amd: (MI250) Batch Invariance"
dind: false
device: mi250_1
timeout_in_minutes: 30
timeout_in_minutes: 35
depends_on:
- image-build-amd
source_file_dependencies:
@@ -623,3 +639,17 @@ steps:
commands:
- export VLLM_ALLOW_INSECURE_SERIALIZATION=1
- pytest -v -s v1/spec_decode/test_acceptance_length.py -m slow_test
mirror:
amd:
label: ":amd: (MI355) Acceptance Length (Large Models)"
dind: false
device: mi355_1
timeout_in_minutes: 45
depends_on:
- image-build-amd
working_dir: /vllm-workspace/tests
source_file_dependencies:
- vllm/v1/spec_decode/
- vllm/model_executor/models/mlp_speculator.py
- tests/v1/spec_decode/test_acceptance_length.py
- vllm/platforms/rocm.py
+22 -3
View File
@@ -19,7 +19,7 @@ steps:
label: ":amd: (MI355) Basic Models (Initialization)"
dind: false
device: mi355_1
timeout_in_minutes: 25
timeout_in_minutes: 30
depends_on:
- image-build-amd
@@ -37,6 +37,23 @@ steps:
# test.) Also run if model initialization test file is modified
- pytest -v -s models/test_initialization.py -k 'not test_can_initialize_small_subset' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
parallelism: 14
mirror:
amd:
label: ":amd: (MI355) Basic Models (Extra Initialization) Shard %N"
dind: false
device: mi355_1
timeout_in_minutes: 60
parallelism: 10
working_dir: "/vllm-workspace/tests"
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/model_executor/models/
- vllm/model_executor/layers/
- tests/models/test_initialization.py
- tests/models/registry.py
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- label: ":nvidia: (H200 MIG 35GB) Basic Models (Other)"
device: h200_35gb
@@ -153,15 +170,17 @@ steps:
- tests/models/test_adapters.py
- tests/models/test_qwen3_5_mtp_config.py
- tests/models/transformers/fusers/
- tests/model_executor/layers/test_activation.py
device: cpu-small
commands:
- pytest -v -s models/test_utils.py models/test_vision.py models/test_adapters.py models/test_qwen3_5_mtp_config.py models/transformers/fusers/
- pytest -v -s models/test_utils.py models/test_vision.py models/test_adapters.py models/test_qwen3_5_mtp_config.py models/transformers/fusers/ model_executor/layers/test_activation.py
mirror:
amd:
label: ":amd: (MI250) Basic Models Other"
dind: false
device: mi250_1
no_gpu: true
working_dir: "/vllm-workspace/tests"
timeout_in_minutes: 25
timeout_in_minutes: 35
depends_on:
- image-build-amd
@@ -38,7 +38,7 @@ steps:
label: ":amd: (MI300) Distributed Models Shard %N"
dind: false
device: mi300_2
timeout_in_minutes: 35
timeout_in_minutes: 80
depends_on:
- image-build-amd
source_file_dependencies:
@@ -63,7 +63,7 @@ steps:
- |
case "$$BUILDKITE_PARALLEL_JOB" in
0)
TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)' &&
HIP_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)'
;;
1)
@@ -72,8 +72,8 @@ steps:
VLLM_ROCM_USE_AITER=1 pytest -v -s models/test_hyv4_rocm.py -m 'distributed(num_gpus=2)'
;;
2)
pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py
pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)'
pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py &&
pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)' &&
VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)'
;;
*) exit 2 ;;
+7 -8
View File
@@ -19,7 +19,7 @@ steps:
label: ":amd: (MI300) Language Models (Standard)"
dind: false
device: mi300_1
timeout_in_minutes: 45
timeout_in_minutes: 60
depends_on:
- image-build-amd
@@ -82,10 +82,10 @@ steps:
parallelism: 2
mirror:
amd:
label: ":amd: (MI300) Language Models (Hybrid) Shard %N"
label: ":amd: (MI355) Language Models (Hybrid) Shard %N"
dind: false
device: mi300_1
timeout_in_minutes: 60
device: mi355_1
timeout_in_minutes: 65
depends_on:
- image-build-amd
commands:
@@ -133,7 +133,7 @@ steps:
label: ":amd: (MI355) Language Models (Extended Generation)"
dind: false
device: mi355_1
timeout_in_minutes: 70
timeout_in_minutes: 95
depends_on:
- image-build-amd
commands:
@@ -155,6 +155,7 @@ steps:
mirror:
amd:
label: ":amd: (MI250) Language Models (PPL)"
dind: false
device: mi250_1
timeout_in_minutes: 65
depends_on:
@@ -197,11 +198,8 @@ steps:
dind: false
device: mi300_1
timeout_in_minutes: 95
parallelism: 2
depends_on:
- image-build-amd
commands:
- pytest -v -s models/language/pooling -m 'not core_model' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
- label: ":nvidia: (H200 MIG 18GB) Language Models (MTEB)"
key: language-models-test-mteb
@@ -217,6 +215,7 @@ steps:
mirror:
amd:
label: ":amd: (MI250) Language Models (MTEB)"
dind: false
device: mi250_1
timeout_in_minutes: 60
depends_on:
+11 -6
View File
@@ -57,6 +57,7 @@ steps:
mirror:
amd:
label: ":amd: (MI250) Multimodal Models (Standard) 3: llava + qwen2_vl"
dind: false
device: mi250_1
timeout_in_minutes: 55
depends_on:
@@ -77,9 +78,9 @@ steps:
- cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model # Otherwise, mp_method="spawn" doesn't work
mirror:
amd:
label: ":amd: (MI300) Multimodal Models (Standard) 4: other + whisper"
label: ":amd: (MI355) Multimodal Models (Standard) 4: other + whisper"
dind: false
device: mi300_1
device: mi355_1
timeout_in_minutes: 70
depends_on:
- image-build-amd
@@ -101,29 +102,33 @@ steps:
mirror:
amd:
label: ":amd: (MI250) Multimodal Processor Shard %N"
dind: false
device: mi250_1
no_gpu: true
working_dir: "/vllm-workspace/tests"
timeout_in_minutes: 80
timeout_in_minutes: 90
parallelism: 6
depends_on:
- image-build-amd
- label: ":nvidia: (H200 MIG 18GB) Multimodal Processor"
- label: ":nvidia: (H200 MIG 18GB) Multimodal Processor Shard %N"
key: multi-modal-processor
timeout_in_minutes: 98
device: h200_18gb
parallelism: 4
source_file_dependencies:
- vllm/
- "!vllm/distributed/kv_transfer/"
- tests/models/multimodal
- tests/models/registry.py
commands:
- pytest -v -s models/multimodal/processing/test_tensor_schema.py
- pytest -v -s models/multimodal/processing/test_tensor_schema.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
mirror:
amd:
label: ":amd: (MI355) Multimodal Processor"
label: ":amd: (MI355) Multimodal Processor Shard %N"
dind: false
device: mi355_1
parallelism: 4
timeout_in_minutes: 115
depends_on:
- image-build-amd
+15
View File
@@ -53,6 +53,7 @@ steps:
mirror:
amd:
label: ":amd: (MI250) Plugin Integration"
dind: false
device: mi250_2
timeout_in_minutes: 50
depends_on:
@@ -75,6 +76,20 @@ steps:
commands:
- pip install "vllm-gguf-plugin >= 0.0.2"
- pytest -v -s plugins_tests/gguf
mirror:
amd:
label: ":amd: (MI355) GGUF Plugin"
dind: false
device: mi355_1
timeout_in_minutes: 45
working_dir: "/vllm-workspace/tests"
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/model_executor/layers/quantization
- tests/plugins_tests/test_gguf_plugin.py
- tests/plugins_tests/gguf
- vllm/platforms/rocm.py
- label: ":nvidia: (H200 MIG 18GB) BitsAndBytes Plugin"
key: bitsandbytes-plugin
+16
View File
@@ -122,6 +122,21 @@ steps:
- tests/compile/passes
commands:
- pytest -s -v compile/passes --ignore compile/passes/distributed
mirror:
amd:
label: ':amd: (MI300) PyTorch Compilation Passes'
dind: false
device: mi300_1
timeout_in_minutes: 80
working_dir: /vllm-workspace/tests
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/
- tests/compile/passes
- vllm/platforms/rocm.py
commands:
- pytest -s -v compile/passes --ignore compile/passes/distributed
- label: ":nvidia: (H200 MIG 35GB) PyTorch Fullgraph"
device: h200_35gb
@@ -154,6 +169,7 @@ steps:
- vllm/utils/
- vllm/v1/
- tests/compile
- csrc/
commands:
# Run smoke tests under fullgraph directory, except test_full_cudagraph.py
# as it is a heavy test that is covered in other steps.
+44
View File
@@ -20,6 +20,31 @@ steps:
# The SM90-only checkpoint currently contains a removed weight_chan_scale
# parameter. It was not exercised by the previous L4 job.
- VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py -k 'not test_compressed_tensors_w4a8_fp8' --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
mirror:
amd:
label: ":amd: (MI355) Quantization Shard %N"
dind: false
device: mi355_1
timeout_in_minutes: 115
working_dir: "/vllm-workspace/tests"
depends_on:
- image-build-amd
source_file_dependencies:
- csrc/
- vllm/model_executor/layers/quantization
- tests/quantization
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- vllm/model_executor/layers/fused_moe/oracle/unquantized.py
- vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py
- tests/rocm/test_moe_weight_replay.py
commands:
# Use the ROCm default instead of the NVIDIA runner override.
- unset VLLM_USE_V2_MODEL_RUNNER
- uv pip install --system torchao==0.17.0
- uv pip install --system conch-triton-kernels
- VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s rocm/test_moe_weight_replay.py; fi
- label: ":nvidia: (H200 MIG 35GB) Quantized Fusions"
device: h200_35gb
@@ -63,3 +88,22 @@ steps:
- tests/models/quantization
commands:
- pytest -v -s models/quantization
mirror:
amd:
label: ":amd: (MI355) Quantized Models"
dind: false
device: mi355_1
timeout_in_minutes: 80
working_dir: "/vllm-workspace/tests"
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/model_executor/layers/quantization
- tests/models/quantization
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- vllm/model_executor/model_loader/
commands:
# Use the ROCm default instead of the NVIDIA runner override.
- unset VLLM_USE_V2_MODEL_RUNNER
- pytest -v -s models/quantization
+16
View File
@@ -16,3 +16,19 @@ steps:
- setup.py
commands:
- bash /vllm-workspace/.buildkite/scripts/check-ray-compatibility.sh
mirror:
amd:
label: ":amd: (MI355) Ray Dependency Compatibility Check"
dind: false
device: mi355_1
timeout_in_minutes: 25
working_dir: "/"
depends_on:
- image-build-amd
source_file_dependencies:
- .buildkite/scripts/check-ray-compatibility.sh
- requirements/
- setup.py
- vllm/platforms/rocm.py
commands:
- bash /vllm-workspace/.buildkite/scripts/check-ray-compatibility.sh
+2 -2
View File
@@ -136,7 +136,7 @@ steps:
label: ":amd: (MI355) Rust Frontend Core Correctness"
dind: false
device: mi355_1
timeout_in_minutes: 20
timeout_in_minutes: 25
depends_on:
- image-build-amd
source_file_dependencies:
@@ -166,7 +166,7 @@ steps:
label: ":amd: (MI355) Rust Frontend Tool Use"
dind: false
device: mi355_1
timeout_in_minutes: 25
timeout_in_minutes: 30
depends_on:
- image-build-amd
source_file_dependencies:
@@ -17,6 +17,7 @@ steps:
mirror:
amd:
label: ":amd: (MI250) Rust Frontend Cargo Style + Clippy"
dind: false
device: mi250_1
no_gpu: true
working_dir: "/vllm-workspace"
@@ -40,6 +41,7 @@ steps:
mirror:
amd:
label: ":amd: (MI250) Rust Frontend Cargo"
dind: false
device: mi250_1
no_gpu: true
working_dir: "/vllm-workspace"
+1
View File
@@ -20,6 +20,7 @@ steps:
mirror:
amd:
label: ":amd: (MI250) Samplers"
dind: false
device: mi250_1
timeout_in_minutes: 40
depends_on:
+41 -7
View File
@@ -47,7 +47,7 @@ steps:
label: ":amd: (MI300) Spec Decode Eagle 1: DeepSeek + Qwen"
dind: false
device: mi300_1
timeout_in_minutes: 35
timeout_in_minutes: 60
depends_on:
- image-build-amd
source_file_dependencies:
@@ -74,7 +74,7 @@ steps:
label: ":amd: (MI300) Spec Decode Eagle 2: Llama 3 + Qwen VL + Other"
dind: false
device: mi300_1
timeout_in_minutes: 25
timeout_in_minutes: 60
depends_on:
- image-build-amd
source_file_dependencies:
@@ -174,7 +174,7 @@ steps:
- label: ":nvidia: (H200 MIG 18GB) Spec Decode Draft Model"
key: spec-decode-draft-model
timeout_in_minutes: 45
timeout_in_minutes: 60
device: h200_18gb
source_file_dependencies:
- vllm/v1/spec_decode/
@@ -187,7 +187,7 @@ steps:
label: ":amd: (MI300) Spec Decode Draft Model"
dind: false
device: mi300_1
timeout_in_minutes: 55
timeout_in_minutes: 60
depends_on:
- image-build-amd
source_file_dependencies:
@@ -233,6 +233,9 @@ steps:
timeout_in_minutes: 40
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- label: ":nvidia: (B200) Spec Decode DeepSeek MTP Parallel Load"
key: spec-decode-deepseek-mtp-parallel-load-2xb200-2xmi300
@@ -280,6 +283,8 @@ steps:
- vllm/v1/spec_decode/
- vllm/v1/worker/gpu/spec_decode/
- vllm/model_executor/models/qwen3_dflash.py
- vllm/model_executor/models/qwen3_dflash2.py
- vllm/model_executor/models/qwen3_5.py
- vllm/model_executor/models/laguna_dflash.py
- tests/v1/e2e/spec_decode/
commands:
@@ -290,6 +295,8 @@ steps:
dind: false
device: mi300_1
timeout_in_minutes: 120
commands:
- pytest -v -s v1/e2e/spec_decode/acceptance_rates/dflash/ -k "not dflash2"
depends_on:
- image-build-amd
source_file_dependencies:
@@ -362,6 +369,33 @@ steps:
# Each Gemma4 parametrization is an indivisible ~23m floor. Keep them in
# separate shards and put the remaining tests in shard 0, where new tests
# land by default.
- test "$$BUILDKITE_PARALLEL_JOB" != "0" || pytest -v -s v1/e2e/spec_decode/acceptance_rates/mtp_other/ --deselect "tests/v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[False]" --deselect "tests/v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[True]"
- test "$$BUILDKITE_PARALLEL_JOB" != "1" || pytest -v -s "v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[False]"
- test "$$BUILDKITE_PARALLEL_JOB" != "2" || pytest -v -s "v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[True]"
- if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s v1/e2e/spec_decode/acceptance_rates/mtp_other/ --deselect "tests/v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[False]" --deselect "tests/v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[True]"; fi
- if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then pytest -v -s "v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[False]"; fi
- if [ "$$BUILDKITE_PARALLEL_JOB" = "2" ]; then pytest -v -s "v1/e2e/spec_decode/acceptance_rates/mtp_other/test_mtp.py::test_gemma4_mtp_acceptance_lengths[True]"; fi
mirror:
amd:
label: ":amd: (MI355) Spec Decode AL MTP + Other Acceptance Nightly"
dind: false
device: mi355_1
timeout_in_minutes: 55
depends_on:
- image-build-amd
working_dir: /vllm-workspace/tests
source_file_dependencies:
- vllm/v1/spec_decode/
- vllm/v1/sample/
- vllm/v1/worker/gpu/
- vllm/v1/worker/gpu_model_runner.py
- vllm/v1/worker/gpu_worker.py
- vllm/model_executor/model_loader/
- vllm/model_executor/models/gemma4.py
- vllm/model_executor/models/gemma4_mm.py
- vllm/model_executor/models/gemma4_mtp.py
- vllm/model_executor/models/llama.py
- vllm/model_executor/models/llama_eagle3.py
- vllm/model_executor/models/medusa.py
- vllm/model_executor/models/registry.py
- tests/evals/gsm8k/
- tests/utils.py
- tests/v1/e2e/spec_decode/
- vllm/platforms/rocm.py
-38
View File
@@ -1,38 +0,0 @@
group: Weight Loading
depends_on:
- image-build
steps:
- label: ":nvidia: (L4) Weight Loading Multi-GPU"
key: weight-loading-multiple-gpu
timeout_in_minutes: 50
working_dir: "/vllm-workspace/tests"
device: l4
num_devices: 2
optional: true
source_file_dependencies:
- vllm/
- "!vllm/distributed/kv_transfer/"
- tests/weight_loading
commands:
- bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models.txt
mirror:
amd:
label: ":amd: (MI355) Weight Loading Multi-GPU"
dind: false
device: mi355_2
timeout_in_minutes: 40
depends_on:
- image-build-amd
commands:
- bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-amd.txt
# - label: Weight Loading Multiple GPU - Large Models # optional
# working_dir: "/vllm-workspace/tests"
# num_devices: 2
# device: a100
# optional: true
# source_file_dependencies:
# - vllm/
# - tests/weight_loading
# commands:
# - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-large.txt
-1
View File
@@ -110,7 +110,6 @@
/tests/entrypoints/llm/test_struct_output_generate.py @mgoin @aarnphm @arpera
/tests/v1/structured_output @mgoin @aarnphm @arpera
/tests/v1/core @WoosukKwon @robertgshaw2-redhat @njhill @ywang96 @alexm-redhat @heheda12345 @ApostaC @orozery @ivanium
/tests/weight_loading @mgoin @youkaichao @yewentao256
/tests/lora @jeejeelee
/tests/models/language/generation/test_hybrid.py @tdoublep @tomeras91
/tests/v1/kv_connector/nixl_integration @NickLucche
+11
View File
@@ -488,6 +488,7 @@ set(VLLM_EXT_SRC
"csrc/moe/dynamic_4bit_int_moe_cpu.cpp"
"csrc/cpu/cpu_fused_moe.cpp"
"csrc/cpu/cpu_attn.cpp"
"csrc/cpu/cpu_isa.cpp"
"csrc/cpu/torch_bindings.cpp")
if (CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64" AND VLLM_RVV_VLEN AND
@@ -522,6 +523,12 @@ if (POWER9_FOUND OR POWER10_FOUND OR POWER11_FOUND)
${VLLM_EXT_SRC})
endif()
if (POWER10_FOUND OR POWER11_FOUND)
set(VLLM_EXT_SRC
"csrc/cpu/cpu_fused_moe_int8.cpp"
${VLLM_EXT_SRC})
endif()
if(USE_ONEDNN)
set(VLLM_EXT_SRC
"csrc/cpu/dnnl_kernels.cpp"
@@ -540,11 +547,13 @@ if (ENABLE_X86_ISA)
"csrc/cpu/sgl-kernels/gemm.cpp"
"csrc/cpu/sgl-kernels/gemm_int8.cpp"
"csrc/cpu/sgl-kernels/gemm_fp8.cpp"
"csrc/cpu/sgl-kernels/gemm_fp8_w8a8.cpp"
"csrc/cpu/sgl-kernels/gemm_int4.cpp"
"csrc/cpu/sgl-kernels/moe.cpp"
"csrc/cpu/sgl-kernels/moe_int8.cpp"
"csrc/cpu/sgl-kernels/moe_int4.cpp"
"csrc/cpu/sgl-kernels/moe_fp8.cpp"
"csrc/cpu/sgl-kernels/moe_fp8_w8a8.cpp"
"csrc/cpu/sgl-kernels/bmm.cpp"
"csrc/cpu/sgl-kernels/decode.cpp"
"csrc/cpu/sgl-kernels/extend.cpp"
@@ -565,6 +574,7 @@ if (ENABLE_X86_ISA)
"csrc/cpu/utils.cpp"
"csrc/cpu/spec_decode_utils.cpp"
"csrc/cpu/cpu_attn.cpp"
"csrc/cpu/cpu_isa.cpp"
"csrc/cpu/dnnl_kernels.cpp"
"csrc/cpu/mamba_cpu.cpp"
"csrc/cpu/torch_bindings.cpp"
@@ -581,6 +591,7 @@ if (ENABLE_X86_ISA)
"csrc/cpu/utils.cpp"
"csrc/cpu/spec_decode_utils.cpp"
"csrc/cpu/cpu_attn.cpp"
"csrc/cpu/cpu_isa.cpp"
"csrc/cpu/mamba_cpu.cpp"
"csrc/cpu/dnnl_kernels.cpp"
"csrc/cpu/torch_bindings.cpp"
+2 -11
View File
@@ -1,16 +1,7 @@
#include "cpu_attn_dispatch_generated.h"
#if defined(__x86_64__) || defined(_M_X64)
#include <cpuid.h>
// Runtime check for AMX-FP8 (Diamond Rapids): CPUID leaf 7, subleaf 1, EAX[21].
static bool runtime_has_amx_fp8() {
unsigned int eax = 0, ebx = 0, ecx = 0, edx = 0;
if (__get_cpuid_count(7, 1, &eax, &ebx, &ecx, &edx)) {
return (eax >> 21) & 1u;
}
return false;
}
#endif // x86_64
// Runtime check for AMX-FP8, implemented in cpu_isa.cpp.
extern bool runtime_has_amx_fp8();
// Maps kv_cache_dtype string to Fp8KVCacheDataType enum.
// "auto" -> kAuto(0); "fp8"/"fp8_e4m3" -> kFp8E4M3; "fp8_e5m2" -> kFp8E5M2.
+16 -3
View File
@@ -26,10 +26,23 @@
#define NEON_DISPATCH(SCALAR_TYPE, ...) case cpu_utils::ISA::NEON:
#endif
#if defined(__powerpc64__)
#include "cpu/micro_gemm/cpu_micro_gemm_int8_vsx.hpp"
#define VSX_DISPATCH(SCALAR_TYPE, ...) \
case cpu_utils::ISA::VSX: { \
using gemm_t = \
cpu_micro_gemm::MicroGemmINT8<cpu_utils::ISA::VSX, SCALAR_TYPE>; \
return __VA_ARGS__(); \
}
#else
#define VSX_DISPATCH(SCALAR_TYPE, ...) case cpu_utils::ISA::VSX:
#endif
#define CPU_INT8_ISA_DISPATCH_IMPL(ISA_TYPE, SCALAR_TYPE, ...) \
[&] { \
switch (ISA_TYPE) { \
NEON_DISPATCH(SCALAR_TYPE, __VA_ARGS__) \
VSX_DISPATCH(SCALAR_TYPE, __VA_ARGS__) \
default: { \
TORCH_CHECK(false, "Invalid CPU ISA type."); \
} \
@@ -81,9 +94,9 @@ void fused_moe_int8_impl(
const int32_t thread_num = cpu_utils::get_max_threads();
const int32_t w13_input_buffer_size = cpu_utils::round_up<64>(
gemm_m_tile_size * input_size_13 * sizeof(int8_t));
const int32_t w2_input_buffer_size =
cpu_utils::round_up<64>(gemm_m_tile_size * input_size_2 * sizeof(int8_t));
(gemm_m_tile_size * input_size_13 + 64) * sizeof(int8_t));
const int32_t w2_input_buffer_size = cpu_utils::round_up<64>(
(gemm_m_tile_size * input_size_2 + 64) * sizeof(int8_t));
const int32_t w13_n_tile_size = [&]() {
const int64_t cache_size = cpu_utils::get_available_l2_size();
+20
View File
@@ -0,0 +1,20 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
// AMX-FP8 is reported at CPUID leaf 0x1E, subleaf 1, EAX[4].
#if defined(__x86_64__) || defined(_M_X64)
#include <cpuid.h>
bool runtime_has_amx_fp8() {
unsigned int eax = 0, ebx = 0, ecx = 0, edx = 0;
if (__get_cpuid_max(0, nullptr) < 0x1e) return false;
if (__get_cpuid_count(0x1e, 1, &eax, &ebx, &ecx, &edx)) {
return (eax >> 4) & 1u;
}
return false;
}
bool cpu_has_amx_fp8() { return runtime_has_amx_fp8(); }
#else
bool cpu_has_amx_fp8() { return false; }
#endif
+22 -9
View File
@@ -13,10 +13,10 @@ namespace vec_op {
struct fp8_e4m3_tag {};
struct fp8_e5m2_tag {};
#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__) \
AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__)
#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__)
// Note: FP16 (Half) is not supported on POWER VSX — no FP16Vec16 type.
#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))
@@ -416,6 +416,7 @@ struct FP32Vec8 : public Vec<FP32Vec8> {
tmp.val[0] = two_x;
tmp.val[1] = two_x;
FP32Vec8 temp_vec(tmp);
FP32Vec8 exp_vec = temp_vec.exp();
vector float e = temp_vec.exp().reg.val[0];
vector float num = vec_sub(e, one);
@@ -634,6 +635,16 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
reg.val[3] = vec_ctf(v.reg.val[3], 0);
}
explicit FP32Vec16(const float* __restrict__ base, const INT32Vec16& index) {
INT32Vec16::AliasReg idx;
idx.reg = index.reg;
AliasReg ar;
for (int i = 0; i < VEC_ELEM_NUM; ++i) {
ar.values[i] = base[idx.values[i]];
}
reg = ar.reg;
}
FP32Vec16 operator*(const FP32Vec16& b) const {
return FP32Vec16(f32x4x4_t({vec_mul(reg.val[0], b.reg.val[0]),
vec_mul(reg.val[1], b.reg.val[1]),
@@ -648,6 +659,13 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
vec_add(reg.val[3], b.reg.val[3])}));
}
FP32Vec16 operator-() const {
const __vector float zero = vec_splats(0.0f);
return FP32Vec16(
f32x4x4_t({vec_sub(zero, reg.val[0]), vec_sub(zero, reg.val[1]),
vec_sub(zero, reg.val[2]), vec_sub(zero, reg.val[3])}));
}
FP32Vec16 operator-(const FP32Vec16& b) const {
return FP32Vec16(f32x4x4_t({vec_sub(reg.val[0], b.reg.val[0]),
vec_sub(reg.val[1], b.reg.val[1]),
@@ -655,11 +673,6 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
vec_sub(reg.val[3], b.reg.val[3])}));
}
FP32Vec16 operator-() const {
return FP32Vec16(
f32x4x4_t({-reg.val[0], -reg.val[1], -reg.val[2], -reg.val[3]}));
}
FP32Vec16 operator/(const FP32Vec16& b) const {
return FP32Vec16(f32x4x4_t({vec_div(reg.val[0], b.reg.val[0]),
vec_div(reg.val[1], b.reg.val[1]),
@@ -0,0 +1,287 @@
#ifndef CPU_MICRO_GEMM_INT8_VSX_HPP
#define CPU_MICRO_GEMM_INT8_VSX_HPP
#include <algorithm>
#include <cstdint>
#include <cstring>
#include <type_traits>
#include <utility>
#include "cpu/micro_gemm/cpu_micro_gemm_impl.hpp"
#include <altivec.h>
#include <c10/util/BFloat16.h>
#include <c10/util/Exception.h>
namespace cpu_micro_gemm {
namespace vsx_int8 {
constexpr int32_t K = 4;
constexpr int32_t Mr = 4;
constexpr int32_t Nr = 32;
// ---------------------------------------------------------------------------
// 8-Accumulator MMA 4x32 micro-GEMM (POWER10 xvi8ger4pp).
//
// Computes 4 rows x 32 columns (128 outputs) using all 8 hardware accumulators.
// Operates on pre-biased uint8 weights (B + 128) and subtracts 128 * a_sums.
// Works for all m in [1, 4] with zero scalar loop overhead.
// ---------------------------------------------------------------------------
__attribute__((target("cpu=power10"))) FORCE_INLINE void gemm_mma_4x32(
const int8_t* __restrict__ a_packed, const uint8_t* __restrict__ b_panel,
const int32_t* __restrict__ a_sums, int32_t* __restrict__ c_ptr,
const int32_t m, const int32_t k_size, const int64_t ldc) {
TORCH_CHECK(m > 0 && m <= Mr);
TORCH_CHECK(k_size % K == 0);
__vector_quad acc0, acc1, acc2, acc3, acc4, acc5, acc6, acc7;
__builtin_mma_xxsetaccz(&acc0);
__builtin_mma_xxsetaccz(&acc1);
__builtin_mma_xxsetaccz(&acc2);
__builtin_mma_xxsetaccz(&acc3);
__builtin_mma_xxsetaccz(&acc4);
__builtin_mma_xxsetaccz(&acc5);
__builtin_mma_xxsetaccz(&acc6);
__builtin_mma_xxsetaccz(&acc7);
const int8_t* __restrict__ ap = a_packed;
const uint8_t* __restrict__ bp = b_panel;
#pragma GCC unroll 4
for (int32_t k = 0; k < k_size; k += 4) {
const __vector unsigned char avec =
vec_xl(0, reinterpret_cast<const unsigned char*>(ap));
__builtin_mma_xvi8ger4pp(&acc0, avec, vec_xl(0, bp));
__builtin_mma_xvi8ger4pp(&acc1, avec, vec_xl(16, bp));
__builtin_mma_xvi8ger4pp(&acc2, avec, vec_xl(32, bp));
__builtin_mma_xvi8ger4pp(&acc3, avec, vec_xl(48, bp));
__builtin_mma_xvi8ger4pp(&acc4, avec, vec_xl(64, bp));
__builtin_mma_xvi8ger4pp(&acc5, avec, vec_xl(80, bp));
__builtin_mma_xvi8ger4pp(&acc6, avec, vec_xl(96, bp));
__builtin_mma_xvi8ger4pp(&acc7, avec, vec_xl(112, bp));
ap += 16;
bp += 128;
}
__vector signed int res[4];
#define STORE_ACC(ACC, CG_IDX) \
{ \
__builtin_mma_disassemble_acc(res, &ACC); \
for (int r = 0; r < m; ++r) { \
const __vector signed int vcorr = vec_splats(128 * a_sums[r]); \
const __vector signed int val = vec_sub(res[r], vcorr); \
vec_xst(val, 0, \
reinterpret_cast<signed int*>(c_ptr + r * ldc + (CG_IDX) * 4)); \
} \
}
STORE_ACC(acc0, 0);
STORE_ACC(acc1, 1);
STORE_ACC(acc2, 2);
STORE_ACC(acc3, 3);
STORE_ACC(acc4, 4);
STORE_ACC(acc5, 5);
STORE_ACC(acc6, 6);
STORE_ACC(acc7, 7);
#undef STORE_ACC
}
} // namespace vsx_int8
// =============================================================================
// MicroGemmINT8 specialisation for VSX (POWER10+)
// =============================================================================
template <typename scalar_t>
class MicroGemmINT8<cpu_utils::ISA::VSX, scalar_t> {
public:
static constexpr int32_t K = vsx_int8::K; // 4
static constexpr int32_t Mr = vsx_int8::Mr; // 4
static constexpr int32_t Nr = vsx_int8::Nr; // 32
static constexpr int32_t NrGemv = Nr;
static constexpr int32_t MaxMSize = 8;
static constexpr int32_t NSize = Nr;
static constexpr int32_t WeightOCGroupSize = Nr;
static_assert(MaxMSize % Mr == 0);
// -------------------------------------------------------------------------
// quantize_row: scalar_t -> INT8 + per-token scale
// Single-pass 16-element vector conversion with aligned 16-byte stores.
// -------------------------------------------------------------------------
static FORCE_INLINE void quantize_row(const scalar_t* input, int8_t* output,
float& scale, const int32_t size) {
TORCH_CHECK_EQ(size % 16, 0);
const __vector signed short z = vec_splats((signed short)0);
auto load_lo_hi =
[&](int32_t i) -> std::pair<__vector float, __vector float> {
if constexpr (std::is_same_v<scalar_t, float>) {
const float* p = reinterpret_cast<const float*>(input) + i;
return {vec_xl(0, p), vec_xl(16, p)};
} else {
const signed short* p =
reinterpret_cast<const signed short*>(input + i);
const __vector signed short raw = (__vector signed short)vec_xl(0, p);
return {(__vector float)vec_mergeh(z, raw),
(__vector float)vec_mergel(z, raw)};
}
};
__vector float max_v = vec_splats(0.0f);
for (int32_t i = 0; i < size; i += 8) {
auto [lo, hi] = load_lo_hi(i);
max_v = vec_max(max_v, vec_abs(lo));
max_v = vec_max(max_v, vec_abs(hi));
}
__vector float t = vec_max(max_v, vec_sld(max_v, max_v, 8));
t = vec_max(t, vec_sld(t, t, 4));
const float abs_max = std::max(vec_extract(t, 0), 1.0e-7f);
scale = abs_max / 127.0f;
const __vector float inv_scale = vec_splats(127.0f / abs_max);
for (int32_t i = 0; i < size; i += 16) {
auto [lo0, hi0] = load_lo_hi(i);
auto [lo1, hi1] = load_lo_hi(i + 8);
const __vector signed int i0 =
vec_cts(vec_round(vec_mul(lo0, inv_scale)), 0);
const __vector signed int i1 =
vec_cts(vec_round(vec_mul(hi0, inv_scale)), 0);
const __vector signed int i2 =
vec_cts(vec_round(vec_mul(lo1, inv_scale)), 0);
const __vector signed int i3 =
vec_cts(vec_round(vec_mul(hi1, inv_scale)), 0);
const __vector signed short s0 = vec_packs(i0, i1);
const __vector signed short s1 = vec_packs(i2, i3);
const __vector signed char s8 = vec_packs(s0, s1);
vec_xst(s8, 0, reinterpret_cast<signed char*>(output + i));
}
}
// -------------------------------------------------------------------------
// dequantize_tile: INT32 accumulator -> FP32 scaled output
// -------------------------------------------------------------------------
static FORCE_INLINE void dequantize_tile(
int32_t* input, float* output, const float* __restrict__ input_scales,
const float* __restrict__ weight_scales, const int32_t m, const int32_t n,
const int32_t stride) {
TORCH_CHECK_EQ(n % 4, 0);
for (int32_t mi = 0; mi < m; ++mi) {
const __vector float a_scale = vec_splats(input_scales[mi]);
for (int32_t ni = 0; ni < n; ni += 4) {
const __vector signed int i32 = vec_xl(0, input + mi * stride + ni);
const __vector float wscale = vec_xl(0, weight_scales + ni);
vec_xst(vec_mul(vec_ctf(i32, 0), vec_mul(a_scale, wscale)), 0,
output + mi * stride + ni);
}
}
}
// -------------------------------------------------------------------------
// pack_input_from_rows: pack rows into 4-row MMA tiles and compute row sums
// Header: 64 bytes (contains int32 a_sums[0..7])
// Data: a_packed + 64 (contains 4x4 tiles of A)
// -------------------------------------------------------------------------
static void pack_input_from_rows(const int8_t* const* __restrict__ rows,
int8_t* __restrict__ a_packed,
const int32_t m, const int32_t k) {
TORCH_CHECK(m > 0 && m <= MaxMSize);
TORCH_CHECK_EQ(k % K, 0);
int32_t* a_sums = reinterpret_cast<int32_t*>(a_packed);
for (int32_t r = 0; r < 8; ++r) a_sums[r] = 0;
int8_t* a_data = a_packed + 64;
for (int32_t row_base = 0; row_base < m; row_base += Mr) {
const int32_t panel_m = std::min(Mr, m - row_base);
const int8_t* const* panel_rows = rows + row_base;
int32_t* panel_sums = a_sums + row_base;
int8_t* dst_panel = a_data + (row_base / Mr) * (k * Mr);
for (int32_t ki = 0; ki < k; ki += 4) {
int8_t* dst = dst_panel + (ki / 4) * 16;
for (int32_t r = 0; r < 4; ++r) {
if (r < panel_m) {
const int8_t* src = panel_rows[r] + ki;
dst[r * 4 + 0] = src[0];
dst[r * 4 + 1] = src[1];
dst[r * 4 + 2] = src[2];
dst[r * 4 + 3] = src[3];
panel_sums[r] +=
static_cast<int32_t>(src[0]) + src[1] + src[2] + src[3];
} else {
dst[r * 4 + 0] = 0;
dst[r * 4 + 1] = 0;
dst[r * 4 + 2] = 0;
dst[r * 4 + 3] = 0;
}
}
}
}
}
// -------------------------------------------------------------------------
// pack_weight: [output_size, input_size] INT8 -> packed layout
// Pre-biases with +128 in [N/32][K/4][8 col-groups][4 cols x 4 k] layout
// -------------------------------------------------------------------------
static void pack_weight(const int8_t* __restrict__ weight,
int8_t* __restrict__ packed_weight,
const int32_t output_size, const int32_t input_size) {
TORCH_CHECK_EQ(output_size % NSize, 0);
TORCH_CHECK_EQ(input_size % K, 0);
uint8_t* dst_base = reinterpret_cast<uint8_t*>(packed_weight);
for (int32_t o = 0; o < output_size; o += Nr) {
uint8_t* dst_panel = dst_base + o * input_size;
for (int32_t k = 0; k < input_size; k += 4) {
for (int32_t cg = 0; cg < 8; ++cg) {
uint8_t* dst = dst_panel + (k / 4) * 128 + cg * 16;
for (int32_t col = 0; col < 4; ++col) {
for (int32_t ki = 0; ki < 4; ++ki) {
const int8_t w = weight[(o + cg * 4 + col) * input_size + k + ki];
dst[col * 4 + ki] =
static_cast<uint8_t>(static_cast<int>(w) + 128);
}
}
}
}
}
}
// -------------------------------------------------------------------------
// gemm: 8-accumulator MMA GEMM over 32-column tiles
// -------------------------------------------------------------------------
void gemm(const int8_t* __restrict__ a_packed,
const int8_t* __restrict__ b_packed, int32_t* __restrict__ c,
const int32_t m, const int32_t k, const int64_t b_n_group_stride,
const int64_t ldc) const {
TORCH_CHECK(m > 0 && m <= MaxMSize);
TORCH_CHECK_EQ(k % K, 0);
(void)b_n_group_stride;
const int32_t* a_sums = reinterpret_cast<const int32_t*>(a_packed);
const int8_t* a_data = a_packed + 64;
const uint8_t* b_u8 = reinterpret_cast<const uint8_t*>(b_packed);
for (int32_t n_idx = 0; n_idx < NSize; n_idx += Nr) {
const uint8_t* b_panel = b_u8 + n_idx * k;
for (int32_t row_base = 0; row_base < m; row_base += Mr) {
const int32_t panel_m = std::min(Mr, m - row_base);
const int8_t* a_panel = a_data + (row_base / Mr) * (k * Mr);
const int32_t* panel_sums = a_sums + row_base;
vsx_int8::gemm_mma_4x32(a_panel, b_panel, panel_sums,
c + row_base * ldc + n_idx, panel_m, k, ldc);
}
}
}
};
} // namespace cpu_micro_gemm
#endif // CPU_MICRO_GEMM_INT8_VSX_HPP
+83
View File
@@ -12,11 +12,45 @@
#include <ATen/ATen.h>
#include <ATen/Dispatch.h>
#include <ATen/Parallel.h>
#include <cstring>
#if defined(_OPENMP)
#include <omp.h>
#endif
// Adapted from sglang: dispatch utilities for FP8 W8A8 kernels
#define AT_DISPATCH_OUT_TYPES(TYPE, NAME, ...) \
AT_DISPATCH_SWITCH( \
TYPE, \
NAME, \
AT_PRIVATE_CASE_TYPE_USING_HINT(at::ScalarType::Float, out_t, __VA_ARGS__) \
AT_PRIVATE_CASE_TYPE_USING_HINT(at::ScalarType::BFloat16, out_t, __VA_ARGS__) \
AT_PRIVATE_CASE_TYPE_USING_HINT(at::ScalarType::Half, out_t, __VA_ARGS__))
#define AT_DISPATCH_CASE_ENUM_NO_RETURN(VALUE, TYPE, HINT, ...) \
case VALUE: { \
constexpr TYPE HINT = VALUE; \
__VA_ARGS__; \
break; \
}
#define AT_DISPATCH_BOOL_NO_RETURN(VALUE, NAME, HINT, ...) \
[&]() { \
switch (VALUE) { \
AT_DISPATCH_CASE_ENUM_NO_RETURN(true, bool, HINT, __VA_ARGS__) \
AT_DISPATCH_CASE_ENUM_NO_RETURN(false, bool, HINT, __VA_ARGS__) \
} \
}()
#define AT_DISPATCH_QUANT_MODE_NO_RETURN(MODE, NAME, HINT, ...) \
[&]() { \
switch (MODE) { \
AT_DISPATCH_CASE_ENUM_NO_RETURN(PER_TENSOR, int, HINT, __VA_ARGS__) \
AT_DISPATCH_CASE_ENUM_NO_RETURN(PER_ROW, int, HINT, __VA_ARGS__) \
AT_DISPATCH_CASE_ENUM_NO_RETURN(PER_GROUP, int, HINT, __VA_ARGS__) \
} \
}()
namespace {
// dispatch bool
@@ -472,4 +506,53 @@ inline T* conditional_data_ptr(const std::optional<at::Tensor>& opt) {
return opt.has_value() ? opt.value().data_ptr<T>() : nullptr;
}
// Adapted from sglang: M blocking utilities for FP8 W8A8 kernels
inline int64_t get_m_block(int64_t M) {
if (M <= 48) {
return M;
} else if (M < 64) {
return 32;
} else if (M < 96) {
return 64;
} else {
return 128;
}
}
inline std::tuple<bool, int64_t, int64_t, int64_t> get_m_blocking(int64_t M) {
bool parallel_on_M = M > 128;
int64_t block_m = get_m_block(M);
int64_t Mc = (M + block_m - 1) / block_m;
int64_t Mc_parallel = parallel_on_M ? Mc : 1;
return std::make_tuple(parallel_on_M, block_m, Mc, Mc_parallel);
}
#if defined(CPU_CAPABILITY_AVX512)
template <typename T>
void zero_buffer(T* data, int64_t size) {
const int32_t vec_size = at::vec::Vectorized<T>::size();
auto zero_vec = at::vec::Vectorized<T>(0);
int64_t d = 0;
for (; d < size - (size % vec_size); d += vec_size) {
zero_vec.store(data + d);
}
if (d < size) {
zero_vec.store(data + d, size - d);
}
}
#else
template <typename T>
void zero_buffer(T* data, int64_t size) {
memset(data, 0, sizeof(T) * size);
}
#endif
#if defined(__x86_64__)
bool avx10_2_available() {
// __builtin_cpu_supports returns the masked feature bit rather than 0/1.
static const bool available = __builtin_cpu_supports("avx10.2");
return available;
}
#endif
} // anonymous namespace
+53 -1
View File
@@ -94,7 +94,7 @@ enum class CPUActMethod : int {
gelu_and_mul = 2,
};
enum class CPUQuantMethod : int64_t { BF16 = 0, INT8_W8A8 = 1, FP8_W8A16 = 2, INT4_W4A8 = 3, MXFP4 = 4 };
enum class CPUQuantMethod : int64_t { BF16 = 0, INT8_W8A8 = 1, FP8_W8A16 = 2, INT4_W4A8 = 3, MXFP4 = 4, FP8_W8A8 = 5 };
constexpr bool operator==(CPUQuantMethod a, int64_t b) {
return static_cast<int64_t>(a) == b;
@@ -395,3 +395,55 @@ void tinygemm_kernel(
bool brg,
int64_t block_size_K,
bool do_unpack = true);
// FP8 W8A8 tinygemm kernel (defined in gemm_fp8_w8a8.cpp)
void tinygemm_kernel(
float* C,
const at::Float8_e4m3fn* A,
const float* scales_a,
const at::Float8_e4m3fn* B,
const float* scales_b,
int64_t M,
int64_t K,
int64_t lda,
int64_t ldc,
int64_t ldsa,
float* ukernel_buf,
at::BFloat16* dqA_buf,
at::BFloat16* dqB_buf);
// FP8 W8A8 quantize activation (defined in gemm_fp8_w8a8.cpp)
std::tuple<at::Tensor, at::Tensor> quantize_fp8e4m3_vec(
const at::Tensor& t,
bool channelwise,
c10::optional<at::Tensor> scale_opt);
// FP8 W8A8 fused MoE kernel (defined in moe_fp8_w8a8.cpp)
template <typename scalar_t>
void fused_experts_fp8_a8_kernel_impl(
scalar_t* __restrict__ output,
scalar_t* __restrict__ ic0,
scalar_t* __restrict__ ic1,
scalar_t* __restrict__ ic2,
at::Float8_e4m3fn* __restrict__ A_tmp,
scalar_t* __restrict__ B_tmp,
float* __restrict__ C_tmp,
float* __restrict__ Ukernel_tmp,
const at::Float8_e4m3fn* __restrict__ input,
const at::Float8_e4m3fn* __restrict__ packed_w1,
const at::Float8_e4m3fn* __restrict__ packed_w2,
const float* __restrict__ As,
const float* __restrict__ w1s,
const float* __restrict__ w2s,
int64_t block_size_N,
int64_t block_size_K,
const float* __restrict__ topk_weights,
const int32_t* __restrict__ sorted_ids,
const int32_t* __restrict__ expert_ids,
const int32_t* __restrict__ offsets,
int64_t M,
int64_t N,
int64_t K,
int64_t E,
int64_t topk,
int64_t num_tokens_post_pad);
File diff suppressed because it is too large Load Diff
+93 -7
View File
@@ -836,6 +836,11 @@ static inline void check_moe_scales(
TORCH_CHECK(w2_scale.has_value(), "missing w2_scale for mxfp4.");
TORCH_CHECK(w1_scale.value().scalar_type() == at::kByte, "expect w1_scale to be uint8.");
TORCH_CHECK(w2_scale.value().scalar_type() == at::kByte, "expect w2_scale to be uint8.");
} else if constexpr (quant == CPUQuantMethod::FP8_W8A8) {
TORCH_CHECK(w1_scale.has_value(), "missing w1_scale for fp8 w8a8.");
TORCH_CHECK(w2_scale.has_value(), "missing w2_scale for fp8 w8a8.");
TORCH_CHECK(block_size.has_value(), "missing block_size for fp8 w8a8.");
TORCH_CHECK(block_size.value().size() == 2, "expect block_size.size() to be 2.");
}
}
@@ -850,9 +855,14 @@ static inline void check_moe_scales(
check_moe_scales<CPUQuantMethod::FP8_W8A16>(w1_scale, w2_scale, block_size);
} else if (moe_comp_method == CPUQuantMethod::MXFP4) {
check_moe_scales<CPUQuantMethod::MXFP4>(w1_scale, w2_scale, block_size);
} else if (moe_comp_method == CPUQuantMethod::FP8_W8A8) {
check_moe_scales<CPUQuantMethod::FP8_W8A8>(w1_scale, w2_scale, block_size);
}
}
// CHECK_MOE_SCALES_FP8: validate scale shapes for FP8_W8A16.
// w1s shape [E, 2N/block_N, K/block_K, block_N]: DIM0=1, DIM1=2
// w2s shape [E, K/block_N, N/block_K]: DIM0=1, DIM1=2
#define CHECK_MOE_SCALES_FP8(DIM0, DIM1) \
auto w1s = w1_scale.value(); \
auto w2s = w2_scale.value(); \
@@ -864,6 +874,17 @@ static inline void check_moe_scales(
TORCH_CHECK(w2s.size(DIM0) == div_up(K, block_size_N)); \
TORCH_CHECK(w2s.size(DIM1) == div_up(N, block_size_K))
// CHECK_MOE_SCALES_FP8_W8A8: extract scale info for FP8_W8A8 without shape
// validation, because w1s is pre-packed by float8_linear_prepack_cpu with
// shape [E, 2N/BLOCK_N, G, BLOCK_N] where BLOCK_N=32 != block_size_N(128).
// This matches the sglang upstream behavior (checks are commented out there).
#define CHECK_MOE_SCALES_FP8_W8A8(DIM0, DIM1) \
auto w1s = w1_scale.value(); \
auto w2s = w2_scale.value(); \
auto block_size_val = block_size.value(); \
int64_t block_size_N = block_size_val[0]; \
int64_t block_size_K = block_size_val[1]
// hidden_states: [M, K]
// w1: [E, 2N, K] or [E, 2N, K / 2] for uint8
// w2: [E, K, N] or [E, K, N / 2] for uint8
@@ -883,6 +904,7 @@ void fused_experts_cpu(
const std::optional<at::Tensor>& w2_scale,
const std::optional<at::Tensor>& w1_zero,
const std::optional<at::Tensor>& w2_zero,
const std::optional<at::Tensor>& a1_scale,
const std::optional<std::vector<int64_t>> block_size,
const std::optional<at::Tensor>& w1_bias,
const std::optional<at::Tensor>& w2_bias,
@@ -895,7 +917,10 @@ void fused_experts_cpu(
constexpr int64_t BLOCK_M = block_size_m();
constexpr int64_t BLOCK_N = block_size_n();
const auto st = hidden_states.scalar_type();
// FP8_W8A8 hidden_states input is FP8; scalar_t should be BF16 for output
const auto st = (moe_comp_method == CPUQuantMethod::FP8_W8A8)
? at::kBFloat16
: hidden_states.scalar_type();
// TODO: fused_topk_torch_native (CPU fallback for models like MiniMax)
// returns int64 topk_ids; fused_experts_cpu requires int32. Remove the typecast after topk kernel is provided
auto topk_ids_ = topk_ids.scalar_type() == at::kInt ? topk_ids : topk_ids.to(at::kInt);
@@ -908,6 +933,9 @@ void fused_experts_cpu(
if (moe_comp_method == CPUQuantMethod::INT4_W4A8 && is_vnni) {
CHECK_DIM(4, w1);
CHECK_DIM(4, w2);
} else if (moe_comp_method == CPUQuantMethod::FP8_W8A8) {
// w1 is prepackaged by float8_linear_prepack_cpu (5D), skip w1 dim check
CHECK_DIM(3, w2);
} else {
CHECK_DIM(3, w1);
CHECK_DIM(3, w2);
@@ -924,8 +952,9 @@ void fused_experts_cpu(
int64_t M = hidden_states.size(0);
int64_t K = hidden_states.size(1);
int64_t N = moe_comp_method == CPUQuantMethod::INT4_W4A8 ? w1_scale.value().size(1) * w1_scale.value().size(3) / 2
: w1.size(1) / 2;
int64_t N = moe_comp_method == CPUQuantMethod::INT4_W4A8
? w1_scale.value().size(1) * w1_scale.value().size(3) / 2
: (moe_comp_method == CPUQuantMethod::FP8_W8A8 ? w2.size(2) : w1.size(1) / 2);
int64_t E = w1.size(0);
int64_t topk = topk_weights_.size(1);
@@ -935,7 +964,10 @@ void fused_experts_cpu(
// check weight shapes
CHECK_EQ(w2.size(0), E);
if (!(moe_comp_method == CPUQuantMethod::INT4_W4A8)) {
if (moe_comp_method == CPUQuantMethod::FP8_W8A8) {
CHECK_EQ(w2.size(1), K);
CHECK_EQ(packed_w2.size(2), packed_N);
} else if (!(moe_comp_method == CPUQuantMethod::INT4_W4A8)) {
CHECK_EQ(w2.size(1), K);
CHECK_EQ(packed_w1.size(2), packed_K / (moe_comp_method == CPUQuantMethod::INT4_W4A8 ? 2 : 1));
CHECK_EQ(packed_w2.size(2), packed_N / (moe_comp_method == CPUQuantMethod::INT4_W4A8 ? 2 : 1));
@@ -993,22 +1025,35 @@ void fused_experts_cpu(
// 5. Aq_tmp : [M, K] or [M * topk, N]
// 6. As_tmp : [M * topk]
//
// for fp8 w8a16 and mxfp4:
// for fp8 w8a16, mxfp4, and fp8 w8a8:
// 7. intermediate_cache0 : [M * topk, 2N]
// 8. B_tmp : [T, MAX_CACHE_BLOCK_SIZE, BLOCK_N, std::max(K, N)]
//
// for fp8 w8a8 additionally:
// 9. Ukernel_tmp : [T, 2 * BLOCK_M * BLOCK_N] (float32 for FP8×FP8 brgemm)
//
int64_t buffer_size_nbytes =
M * topk * N * 2 + M * topk * K * 2 +
num_threads * BLOCK_M * K *
(moe_comp_method == CPUQuantMethod::INT8_W8A8 | moe_comp_method == CPUQuantMethod::INT4_W4A8 ? 1 : 2) +
(moe_comp_method == CPUQuantMethod::INT8_W8A8 | moe_comp_method == CPUQuantMethod::INT4_W4A8
| moe_comp_method == CPUQuantMethod::FP8_W8A8 ? 1 : 2) +
num_threads * 2 * BLOCK_M * BLOCK_N * sizeof(float);
if (moe_comp_method == CPUQuantMethod::INT8_W8A8) {
buffer_size_nbytes += std::max(M * K, M * topk * N) + M * topk * sizeof(float);
}
if (moe_comp_method == CPUQuantMethod::FP8_W8A16 || moe_comp_method == CPUQuantMethod::MXFP4) {
if (moe_comp_method == CPUQuantMethod::FP8_W8A16 || moe_comp_method == CPUQuantMethod::MXFP4
|| moe_comp_method == CPUQuantMethod::FP8_W8A8) {
buffer_size_nbytes += M * topk * 2 * N * 2 + num_threads * MAX_CACHE_BLOCK_SIZE * BLOCK_N * std::max(K, N) * 2;
}
if (moe_comp_method == CPUQuantMethod::FP8_W8A8) {
// Ukernel buffer: float32 workspace per thread for FP8×FP8 brgemm
buffer_size_nbytes += num_threads * 2 * BLOCK_M * BLOCK_N * sizeof(float);
#ifndef CPUBLAS_BRGEMM_F8F8F32
// BF16 fallback: also need dqA [T, BLOCK_M, BLOCK_K] + dqB [T, BLOCK_K, BLOCK_N]
buffer_size_nbytes += num_threads * (BLOCK_M * BLOCK_K + BLOCK_K * BLOCK_N) * 2;
#endif
}
if (moe_comp_method == CPUQuantMethod::INT4_W4A8) {
buffer_size_nbytes += M * topk * 2 * N * 2 + std::max(M * K, M * topk * N) + M * topk * sizeof(float) +
num_threads * 2 * get_4bit_block_k_size(K / w1_scale.value().size(2)) * BLOCK_N;
@@ -1181,6 +1226,47 @@ void fused_experts_cpu(
E,
topk,
num_tokens_post_pad);
} else if (moe_comp_method == CPUQuantMethod::FP8_W8A8) {
// Stage 1: FP8×FP8 GEMM (hidden_states is FP8 input)
// Stage 2: BF16×FP8 GEMM (W8A16 path for down-projection)
at::Float8_e4m3fn* __restrict__ A_tmp =
(at::Float8_e4m3fn*)((void*)(intermediate_cache2 + M * topk * K));
float* __restrict__ C_tmp = (float*)((void*)(A_tmp + num_threads * BLOCK_M * K));
scalar_t* __restrict__ intermediate_cache0 =
(scalar_t*)((void*)(C_tmp + num_threads * 2 * BLOCK_M * BLOCK_N));
scalar_t* __restrict__ B_tmp =
(scalar_t*)((void*)(intermediate_cache0 + M * topk * 2 * N));
float* __restrict__ Ukernel_tmp =
(float*)((void*)(B_tmp + num_threads * MAX_CACHE_BLOCK_SIZE * BLOCK_N * std::max(K, N)));
CHECK_MOE_SCALES_FP8_W8A8(1, 2);
fused_experts_fp8_a8_kernel_impl(
out_hidden_states.data_ptr<scalar_t>(),
intermediate_cache0,
intermediate_cache1,
intermediate_cache2,
A_tmp,
B_tmp,
C_tmp,
Ukernel_tmp,
hidden_states.data_ptr<at::Float8_e4m3fn>(),
packed_w1.data_ptr<at::Float8_e4m3fn>(),
packed_w2.data_ptr<at::Float8_e4m3fn>(),
a1_scale.has_value() ? a1_scale.value().data_ptr<float>() : nullptr,
w1s.data_ptr<float>(),
w2s.data_ptr<float>(),
block_size_N,
block_size_K,
topk_weights_.data_ptr<float>(),
sorted_ids,
expert_ids,
offsets,
M,
N,
K,
E,
topk,
num_tokens_post_pad);
} else {
scalar_t* __restrict__ A_tmp = intermediate_cache2 + M * topk * K;
float* __restrict__ C_tmp = (float*)((void*)(A_tmp + num_threads * BLOCK_M * K));
+432
View File
@@ -0,0 +1,432 @@
// Adapted from
// https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc/cpu
// clang-format off
#include <c10/util/Unroll.h>
#include "common.h"
#include "gemm.h"
#include "moe.h"
namespace {
// Byte-wise copy for FP8 elements (no remainder assumption)
inline void copy_stub2(
at::Float8_e4m3fn* __restrict__ out,
const at::Float8_e4m3fn* __restrict__ input,
int64_t size) {
for (int64_t d = 0; d < size; d++) {
out[d] = input[d];
}
}
#define PER_TENSOR 1
#define PER_ROW 2
#define PER_GROUP 3
// Store float32 GEMM result to output buffer with dequantization and dtype
// conversion. Activations are PER_ROW quantized; weights are PER_GROUP quantized.
// Uses AVX512 on capable platforms.
#if defined(CPU_CAPABILITY_AVX512)
template <typename out_dtype, int64_t N, int act_quant_mode, int wei_quant_mode>
inline void store_out(
const float* y_buf,
out_dtype* c_ptr,
int64_t M,
int64_t lda,
const float* scales_a,
const float* scales_b,
const float* bias) {
float a_scale = 1.0, b_scale = 1.0;
__m512 va_scale, vb_scale;
if constexpr (act_quant_mode == PER_TENSOR) {
a_scale = *scales_a;
}
if constexpr (wei_quant_mode == PER_TENSOR) {
b_scale = *scales_b;
vb_scale = _mm512_set1_ps(b_scale);
}
for (int i = 0; i < M; ++i) {
if constexpr (act_quant_mode == PER_ROW) {
a_scale = *(scales_a + i);
}
if constexpr (act_quant_mode != PER_GROUP) {
va_scale = _mm512_set1_ps(a_scale);
}
constexpr int N_UNROLL = N / 16;
c10::ForcedUnroll<N_UNROLL>{}([&](auto idx) {
constexpr int j = idx * 16;
__m512 y_vec = _mm512_loadu_ps(y_buf + i * N + j);
__m512 bias_vec = bias ? _mm512_loadu_ps(bias + j) : _mm512_setzero_ps();
if constexpr (act_quant_mode != PER_GROUP) {
y_vec = _mm512_mul_ps(y_vec, va_scale);
}
if constexpr (wei_quant_mode == PER_ROW) {
vb_scale = _mm512_loadu_ps(scales_b + j);
}
if constexpr (wei_quant_mode != PER_GROUP) {
y_vec = _mm512_mul_ps(y_vec, vb_scale);
}
y_vec = _mm512_add_ps(y_vec, bias_vec);
if constexpr (std::is_same<out_dtype, float>::value) {
_mm512_storeu_ps(c_ptr + i * lda + j, y_vec);
} else if constexpr (std::is_same<out_dtype, at::BFloat16>::value) {
__m256i y_bf16_vec = at::vec::cvtfp32_bf16(y_vec);
_mm256_storeu_si256(reinterpret_cast<__m256i*>(c_ptr + i * lda + j), y_bf16_vec);
} else if constexpr (std::is_same<out_dtype, at::Half>::value) {
__m256i y_fp16_vec = at::vec::cvtfp32_fp16(y_vec);
_mm256_storeu_si256(reinterpret_cast<__m256i*>(c_ptr + i * lda + j), y_fp16_vec);
} else {
TORCH_CHECK(false, "Unsupported output dtype");
}
});
constexpr int tail_start = N / 16 * 16;
for (int j = tail_start; j < N; ++j) {
if constexpr (wei_quant_mode == PER_ROW) {
b_scale = scales_b[j];
}
c_ptr[i * lda + j] = static_cast<out_dtype>(y_buf[i * N + j] * a_scale * b_scale);
}
} // for M
}
#else // no AVX512
template <typename out_dtype, int64_t N, int act_quant_mode, int wei_quant_mode>
inline void store_out(
const float* y_buf,
out_dtype* c_ptr,
int64_t M,
int64_t lda,
const float* scales_a,
const float* scales_b,
const float* bias) {
float a_scale = 1.0, b_scale = 1.0;
if constexpr (act_quant_mode == PER_TENSOR) {
a_scale = *scales_a;
}
if constexpr (wei_quant_mode == PER_TENSOR) {
b_scale = *scales_b;
}
for (int i = 0; i < M; ++i) {
if constexpr (act_quant_mode == PER_ROW) {
a_scale = *(scales_a + i);
}
for (int j = 0; j < N; ++j) {
if constexpr (wei_quant_mode == PER_ROW) {
b_scale = scales_b[j];
}
float val = y_buf[i * N + j] * a_scale * b_scale;
if (bias) val += bias[j];
c_ptr[i * lda + j] = static_cast<out_dtype>(val);
}
}
}
#endif // CPU_CAPABILITY_AVX512
} // anonymous namespace
// FP8 W8A8 fused MoE kernel.
//
// hidden_states: [M, K] FP8 E4M3, pre-quantized activation
// packed_w1: [E, 2*Nc, Kc, BLOCK_K, BLOCK_N] FP8 W8A8 prepack (float8_linear_prepack_cpu)
// packed_w2: [E, K, N_packed] FP8 W8A16 VNNI pack (convert_weight_packed)
// As: [M] float32 per-token activation scales
// w1s: [E, num_groups, 2N] float32 per-group weight scales for w1
// w2s: [E, ...] float32 per-group weight scales for w2
//
// Stage 1: hidden_states(FP8) @ w1(FP8) → ic0(BF16) [M*topk, 2N]
// Uses FP8×FP8 GEMM (tinygemm_kernel with FP8 inputs)
// Stage 1.5: ic1 = silu(ic0[:,:N]) * ic0[:,N:] [M*topk, N]
// Stage 2: ic1(BF16) @ w2(FP8) → ic2(BF16) [M*topk, K]
// Uses BF16×FP8 GEMM (W8A16 path, tinygemm_kernel<scalar_t>)
// Stage 3: out = weighted_sum(ic2, topk_weights) [M, K]
template <typename scalar_t>
void fused_experts_fp8_a8_kernel_impl(
scalar_t* __restrict__ output,
scalar_t* __restrict__ ic0,
scalar_t* __restrict__ ic1,
scalar_t* __restrict__ ic2,
at::Float8_e4m3fn* __restrict__ A_tmp,
scalar_t* __restrict__ B_tmp,
float* __restrict__ C_tmp,
float* __restrict__ Ukernel_tmp,
const at::Float8_e4m3fn* __restrict__ input,
const at::Float8_e4m3fn* __restrict__ packed_w1,
const at::Float8_e4m3fn* __restrict__ packed_w2,
const float* __restrict__ As,
const float* __restrict__ w1s,
const float* __restrict__ w2s,
int64_t block_size_N,
int64_t block_size_K,
const float* __restrict__ topk_weights,
const int32_t* __restrict__ sorted_ids,
const int32_t* __restrict__ expert_ids,
const int32_t* __restrict__ offsets,
int64_t M,
int64_t N,
int64_t K,
int64_t E,
int64_t topk,
int64_t num_tokens_post_pad) {
// 1. intermediate_cache1 : [M * topk, N]
// 2. intermediate_cache2 : [M * topk, K]
// 3. A_tmp : [T, BLOCK_M * K] FP8
// 4. C_tmp : [T, 2 * BLOCK_M * BLOCK_N] float32
// 5. intermediate_cache0 : [M * topk, 2N]
// 6. B_tmp : [T, MAX_CACHE_BLOCK_SIZE, BLOCK_N, max(K, N)] scalar_t
// 7. Ukernel_tmp: [T, 2 * BLOCK_M * BLOCK_N] float32
// 8. (non-FP8-brgemm only) dqA [T, BLOCK_M * BLOCK_K] + dqB [T, BLOCK_K * BLOCK_N] in BF16
// appended after Ukernel_tmp; see moe.cpp FP8_W8A8 buffer allocation
constexpr int64_t BLOCK_M = block_size_m();
constexpr int64_t BLOCK_N_VAL = block_size_n(); // = 32
const int64_t KB = K / BLOCK_K; // K blocks
int64_t B_tmp_size_per_thread = MAX_CACHE_BLOCK_SIZE * BLOCK_N_VAL * std::max(K, N);
// stage 1: intermediate_cache0 = hidden_states @ w1
const int64_t MB = div_up(num_tokens_post_pad, BLOCK_M);
const int64_t NB = div_up(N, BLOCK_N_VAL);
int64_t scale_size_N = div_up(2 * N, block_size_N);
int64_t scale_size_K = div_up(K, block_size_K);
int64_t blocks_n_per_group = block_size_N / BLOCK_N_VAL;
int64_t num_groups = div_up(K, block_size_K); // G
int64_t blocks_k_per_group = block_size_K / BLOCK_K;
const int64_t stride_e = 2 * N * K; // per-expert stride for w1 (total elements)
bool use_brgemm = true;
int64_t num_thread = at::get_num_threads();
// For the BF16 fallback path (no native FP8 brgemm), tinygemm_kernel needs temporary
// BF16 buffers to dequantize FP8 inputs before running AMX-BF16 brgemm.
// These are placed after the Ukernel_tmp region (see moe.cpp FP8_W8A8 buffer allocation).
#ifndef CPUBLAS_BRGEMM_F8F8F32
// dqA: [T, BLOCK_M, BLOCK_K] BF16
// dqB: [T, BLOCK_K, BLOCK_N] BF16
// Layout: immediately after Ukernel_tmp[num_thread * 2 * BLOCK_M * BLOCK_N_VAL]
at::BFloat16* __restrict__ dq_base = reinterpret_cast<at::BFloat16*>(
Ukernel_tmp + num_thread * 2 * BLOCK_M * BLOCK_N_VAL);
const int64_t dq_per_thread = BLOCK_M * BLOCK_K + BLOCK_K * BLOCK_N_VAL;
#endif
at::parallel_for(0, MB * NB, 1, [&](int64_t begin, int64_t end) {
int tid = get_thread_num();
// A_tmp: FP8 activation buffer for this thread [BLOCK_M, K]
at::Float8_e4m3fn* __restrict__ A = A_tmp + tid * BLOCK_M * K;
float* __restrict__ C0 = C_tmp + tid * 2 * BLOCK_M * BLOCK_N_VAL;
float* __restrict__ C1 = C0 + BLOCK_M * BLOCK_N_VAL;
alignas(64) float As_[BLOCK_M];
float* __restrict__ ukernel_buf_1 = Ukernel_tmp + tid * 2 * BLOCK_M * BLOCK_N_VAL;
#ifndef CPUBLAS_BRGEMM_F8F8F32
at::BFloat16* __restrict__ dqA_buf = dq_base + tid * dq_per_thread;
at::BFloat16* __restrict__ dqB_buf = dqA_buf + BLOCK_M * BLOCK_K;
#else
at::BFloat16* __restrict__ dqA_buf = nullptr;
at::BFloat16* __restrict__ dqB_buf = nullptr;
#endif
float* __restrict__ ukernel_buf_2 = ukernel_buf_1 + BLOCK_M * BLOCK_N_VAL;
auto ldsa = 1; // activation per-row quantized
for (int64_t i = begin; i < end; ++i) {
int64_t mb = i / NB;
int64_t nb = i % NB;
int64_t nb1 = nb + NB; // gate is first N columns, up is second N columns
int64_t n_size = std::min(N - nb * BLOCK_N_VAL, BLOCK_N_VAL);
// B (weight) for expert expert_id: gate+up fused [2N, K] prepackaged as FP8
int32_t expert_id = expert_ids[mb];
const at::Float8_e4m3fn* __restrict__ B = packed_w1 + expert_id * stride_e;
const float* __restrict__ Bs = w1s + expert_id * num_groups * (2 * N);
// Load activation tile for this M-block
const int32_t* A_ids = sorted_ids + mb * BLOCK_M;
int64_t m_size = offsets[mb + 1] - offsets[mb];
for (int64_t m = 0; m < m_size; ++m) {
int32_t index = A_ids[m] / topk;
copy_stub2(A + m * K, input + index * K, K);
As_[m] = As[index];
}
const int64_t offset = offsets[mb];
// Gate path (nb): w1[:N, :]
zero_buffer(C0, BLOCK_M * BLOCK_N_VAL);
for (int kci = 0; kci < KB; ++kci) {
// FP8×FP8 tinygemm: act_quant=PER_ROW(2), wei_quant=PER_GROUP(3)
tinygemm_kernel(
/* C */ C0,
/* A */ A + kci * BLOCK_K,
/* scales_a*/ As_ + kci / blocks_k_per_group,
/* B */ B + (nb * KB + kci) * BLOCK_K * BLOCK_N_VAL,
/* scales_b*/ Bs + nb * BLOCK_N_VAL * num_groups + kci / blocks_k_per_group * BLOCK_N_VAL,
/* M */ m_size,
/* K */ BLOCK_K,
/* lda */ K,
/* ldc */ BLOCK_N_VAL,
/* ldsa */ ldsa,
/* ukernel */ ukernel_buf_1,
/* dqA_buf */ dqA_buf,
/* dqB_buf */ dqB_buf);
}
store_out<scalar_t, BLOCK_N_VAL, PER_ROW, PER_GROUP>(
C0,
ic0 + offset * 2 * N + nb * BLOCK_N_VAL,
m_size,
2 * N, // lda
As_,
nullptr,
nullptr);
// Up path (nb1): w1[N:, :]
zero_buffer(C1, BLOCK_M * BLOCK_N_VAL);
for (int kci = 0; kci < KB; ++kci) {
tinygemm_kernel(
/* C */ C1,
/* A */ A + kci * BLOCK_K,
/* scales_a*/ As_ + kci / blocks_k_per_group,
/* B */ B + (nb1 * KB + kci) * BLOCK_K * BLOCK_N_VAL,
/* scales_b*/ Bs + nb1 * BLOCK_N_VAL * num_groups + kci / blocks_k_per_group * BLOCK_N_VAL,
/* M */ m_size,
/* K */ BLOCK_K,
/* lda */ K,
/* ldc */ BLOCK_N_VAL,
/* ldsa */ ldsa,
/* ukernel */ ukernel_buf_2,
/* dqA_buf */ dqA_buf,
/* dqB_buf */ dqB_buf);
}
store_out<scalar_t, BLOCK_N_VAL, PER_ROW, PER_GROUP>(
C1,
ic0 + offset * 2 * N + nb1 * BLOCK_N_VAL,
m_size,
2 * N, // lda
As_,
nullptr,
nullptr);
}
if (use_brgemm) {
at::native::cpublas::brgemm_release();
}
});
// stage 1.5: intermediate_cache1 = silu(ic0[:, :N]) * ic0[:, N:]
at::parallel_for(0, M * topk, 0, [&](int64_t begin, int64_t end) {
for (int64_t m = begin; m < end; ++m) {
silu_and_mul_stub(ic1 + m * N, ic0 + m * 2 * N, ic0 + m * 2 * N + N, N);
}
});
// stage 2: intermediate_cache2 = intermediate_cache1 @ w2
// w2: [E, K, N] FP8, VNNI packed; treated as W8A16 (BF16 act × FP8 weight)
const int64_t OC = K; // output channels (hidden dim)
const int64_t IC = N; // input channels (expert intermediate dim)
const int64_t MB2 = MB;
const int64_t NB2 = div_up(OC, BLOCK_N_VAL);
scale_size_N = div_up(K, block_size_N);
scale_size_K = div_up(N, block_size_K);
const int64_t stride_e2 = OC * IC; // per-expert stride for w2
const int64_t stride_oc = IC;
int64_t avg_M = std::max(int64_t(1), M * topk / E);
bool use_brgemm2 = can_use_brgemm<at::Float8_e4m3fn>(avg_M);
parallel_2d(MB2, NB2, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) {
int tid = get_thread_num();
alignas(64) scalar_t C[BLOCK_M * BLOCK_K];
loop_2d<at::Float8_e4m3fn>(mb0, mb1, nb0, nb1, BLOCK_N_VAL * IC, [&](int64_t mb, int64_t nb, int64_t nb_offset) {
int64_t m_size = offsets[mb + 1] - offsets[mb];
int64_t n_size = std::min(OC - nb * BLOCK_N_VAL, BLOCK_N_VAL);
// A from ic1: [M*topk, N] in sorted order (contiguous after stage 1.5)
const scalar_t* __restrict__ A2 = ic1 + offsets[mb] * N;
const int32_t* A_ids = sorted_ids + mb * BLOCK_M;
// B from w2: [E, K, N] VNNI packed, offset to expert + output block
int32_t expert_id = expert_ids[mb];
const at::Float8_e4m3fn* __restrict__ B2 =
packed_w2 + expert_id * stride_e2 + nb * BLOCK_N_VAL * stride_oc;
const float* __restrict__ Bs2 =
w2s + expert_id * scale_size_N * scale_size_K + (nb / blocks_n_per_group) * scale_size_K;
// Only unpack B on first visit or new expert
int32_t pre_expert_id = mb == 0 ? -1 : expert_ids[mb - 1];
bool do_unpack = (mb == mb0) || (expert_id != pre_expert_id);
tinygemm_kernel<scalar_t>(
/* A */ A2,
/* B */ B2,
/* C */ C,
/* Btmp */ B_tmp + tid * B_tmp_size_per_thread + nb_offset * BLOCK_N_VAL * IC,
/* Ctmp */ C_tmp + tid * 2 * BLOCK_M * BLOCK_N_VAL,
/* Bbias */ nullptr,
/* scale */ Bs2,
/* M */ m_size,
/* N */ n_size,
/* K */ IC,
/* lda */ IC,
/* ldb */ n_size,
/* ldc */ BLOCK_N_VAL,
/* brg */ use_brgemm2,
/* block_sz_K */ block_size_K,
/* do_unpack */ do_unpack);
// Scatter ic2 in original token order, weighting by topk_weights
for (int64_t m = 0; m < m_size; ++m) {
int32_t index = A_ids[m];
float weight = topk_weights[index];
copy_mul_stub(ic2 + index * K + nb * BLOCK_N_VAL, C + m * BLOCK_N_VAL, weight, n_size);
}
});
if (use_brgemm2) {
at::native::cpublas::brgemm_release();
}
});
// stage 3: output = sum(ic2, dim=topk) [M, K]
at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) {
for (int64_t m = begin; m < end; ++m) {
sum_stub(output + m * K, ic2 + m * topk * K, topk, K);
}
});
}
#define INSTANTIATE_MOE_FP8_A8_TEMPLATE(TYPE) \
template void fused_experts_fp8_a8_kernel_impl<TYPE>( \
TYPE* __restrict__ output, \
TYPE* __restrict__ ic0, \
TYPE* __restrict__ ic1, \
TYPE* __restrict__ ic2, \
at::Float8_e4m3fn* __restrict__ A_tmp, \
TYPE* __restrict__ B_tmp, \
float* __restrict__ C_tmp, \
float* __restrict__ Ukernel_tmp, \
const at::Float8_e4m3fn* __restrict__ input, \
const at::Float8_e4m3fn* __restrict__ packed_w1, \
const at::Float8_e4m3fn* __restrict__ packed_w2, \
const float* __restrict__ As, \
const float* __restrict__ w1s, \
const float* __restrict__ w2s, \
int64_t block_size_N, \
int64_t block_size_K, \
const float* __restrict__ topk_weights, \
const int32_t* __restrict__ sorted_ids, \
const int32_t* __restrict__ expert_ids, \
const int32_t* __restrict__ offsets, \
int64_t M, \
int64_t N, \
int64_t K, \
int64_t E, \
int64_t topk, \
int64_t num_tokens_post_pad)
INSTANTIATE_MOE_FP8_A8_TEMPLATE(at::BFloat16);
INSTANTIATE_MOE_FP8_A8_TEMPLATE(at::Half);
+40 -6
View File
@@ -73,6 +73,7 @@ void fused_experts_cpu(at::Tensor& out, at::Tensor& hidden_states,
const std::optional<at::Tensor>& w2_scale,
const std::optional<at::Tensor>& w1_zero,
const std::optional<at::Tensor>& w2_zero,
const std::optional<at::Tensor>& a1_scale,
const std::optional<std::vector<int64_t>> block_size,
const std::optional<at::Tensor>& w1_bias,
const std::optional<at::Tensor>& w2_bias,
@@ -91,6 +92,18 @@ at::Tensor fp8_scaled_mm_cpu(at::Tensor& mat1, at::Tensor& mat2,
const std::optional<at::Tensor>& bias,
at::ScalarType out_dtype, bool is_vnni);
// Adapted from sglang: FP8 W8A8 kernels
std::tuple<at::Tensor, at::Tensor> float8_linear_prepack_impl(
const at::Tensor& weight, const at::Tensor& scales);
at::Tensor fp8_scaled_mm_with_quant(const at::Tensor& act,
const std::optional<at::Tensor>& act_scales,
bool channelwise, const at::Tensor& weight,
const at::Tensor& weight_scales,
const std::optional<at::Tensor>& bias,
at::ScalarType output_dtype);
std::tuple<at::Tensor, at::Tensor> quantize_fp8e4m3_vec(
const at::Tensor& t, bool channelwise, c10::optional<at::Tensor> scale_opt);
// Adapted from sglang: MLA CPU kernels (AMX-only)
void decode_attention_cpu(at::Tensor& query, at::Tensor& k_buffer,
at::Tensor& v_buffer, at::Tensor& output,
@@ -281,6 +294,7 @@ void activation_lut_bf16(torch::Tensor& out, torch::Tensor& input,
const std::string& activation);
bool cpu_attn_has_isa(const std::string& isa);
bool cpu_has_amx_fp8();
torch::Tensor get_scheduler_metadata(
const int64_t num_req, const int64_t num_heads_q,
@@ -611,7 +625,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
"fused_experts_cpu(Tensor(a0!) out, Tensor hidden_states, Tensor w1, "
"Tensor w2, Tensor topk_weights, Tensor topk_ids, "
"int moe_comp_method, Tensor? w1_scale, Tensor? w2_scale, "
"Tensor? w1_zero, Tensor? w2_zero, int[]? block_size, "
"Tensor? w1_zero, Tensor? w2_zero, Tensor? a1_scale, int[]? block_size, "
"Tensor? w1_bias, Tensor? w2_bias, float? alpha, float? limit, "
"bool is_vnni) -> "
"()");
@@ -629,7 +643,23 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
"bool is_vnni) -> Tensor");
ops.impl("fp8_scaled_mm_cpu", torch::kCPU, &fp8_scaled_mm_cpu);
// Adapted from sglang: casual_conv1d kernels
// Adapted from sglang: FP8 W8A8 kernels
ops.def(
"float8_linear_prepack_cpu(Tensor weight, Tensor scales) -> (Tensor, "
"Tensor)");
ops.impl("float8_linear_prepack_cpu", torch::kCPU,
&float8_linear_prepack_impl);
ops.def(
"fp8_scaled_mm_with_quant(Tensor act, Tensor? act_scales, bool "
"channelwise, "
"Tensor weight, Tensor weight_scales, Tensor? bias, ScalarType "
"output_dtype) -> Tensor");
ops.impl("fp8_scaled_mm_with_quant", torch::kCPU, &fp8_scaled_mm_with_quant);
ops.def(
"quantize_fp8e4m3_vec(Tensor input, bool channelwise, Tensor? scale_opt) "
"-> (Tensor, Tensor)");
ops.impl("quantize_fp8e4m3_vec", torch::kCPU, &quantize_fp8e4m3_vec);
ops.def("causal_conv1d_weight_pack(Tensor weight) -> Tensor");
ops.impl("causal_conv1d_weight_pack", torch::kCPU,
&causal_conv1d_weight_pack);
@@ -832,6 +862,9 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
"-> (Tensor, Tensor)");
ops.impl("fused_gdn_gating_cpu", torch::kCPU, &fused_gdn_gating_cpu);
// Pure runtime CPU ISA capability check.
ops.def("cpu_has_amx_fp8() -> bool", &cpu_has_amx_fp8);
// CPU attention kernels
ops.def("cpu_attn_has_isa(str isa) -> bool", &cpu_attn_has_isa);
ops.def(
@@ -883,8 +916,10 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
"bool skip_weighted, "
"str act, str isa) -> ()");
ops.impl("cpu_fused_moe", torch::kCPU, &cpu_fused_moe);
#if defined(ARM_I8MM_SUPPORT) && defined(ARM_BF16_SUPPORT) && \
!defined(__APPLE__)
#if (defined(ARM_I8MM_SUPPORT) && defined(ARM_BF16_SUPPORT) && \
!defined(__APPLE__)) || \
defined(__powerpc64__)
ops.def(
"prepack_moe_weight_int8(Tensor weight, Tensor(a1!) packed_weight, "
"str isa) -> ()");
@@ -895,8 +930,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
"Tensor? w2_bias, Tensor topk_weights, Tensor topk_id, bool "
"skip_weighted, str act, str isa) -> ()");
ops.impl("cpu_fused_moe_int8", torch::kCPU, &cpu_fused_moe_int8);
#endif // #if defined(ARM_I8MM_SUPPORT) && defined(ARM_BF16_SUPPORT) &&
// !defined(__APPLE__)
#endif // ARM_I8MM+BF16 or __powerpc64__
ops.def(
"mla_decode_kvcache("
" Tensor! out, Tensor query, Tensor kv_cache,"
+5 -3
View File
@@ -19,7 +19,6 @@
#include <algorithm>
#include <cassert>
#include <cfloat>
#include <cstdlib>
#ifdef USE_ROCM
@@ -84,7 +83,9 @@ void swap_blocks(torch::stable::Tensor& src, torch::stable::Tensor& dst,
namespace {
// ROCm hipMemcpyBatchAsync faults for count > 8192 (MI350X/gfx950), so chunk
// at that ceiling.
#if defined(USE_ROCM)
constexpr int64_t kRocmDefaultMaxBatchDescriptors = 8192;
#endif
constexpr const char* kMaxBatchDescriptorsEnv =
"VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS";
@@ -587,8 +588,9 @@ __global__ void concat_and_cache_ds_mla_kernel(
max_abs = fmaxf(max_abs, VLLM_SHFL_XOR_SYNC_WIDTH(max_abs, offset, 16));
}
// Compute the scale for the tile
float tile_scale = fmaxf(max_abs / kFp8ScaleDivisor, FLT_MIN);
// Both SM90 and SM100 readers preserve power-of-two fp32 scales exactly.
float tile_scale = fmaxf(max_abs / kFp8ScaleDivisor, 1e-4f);
tile_scale = exp2f(ceilf(log2f(tile_scale)));
// The first lane of each half-warp writes the scale to kv_cache
if ((lane_idx == 0) || (lane_idx == 16)) {
@@ -35,8 +35,9 @@
*
* fp8_ds_mla (fused_kimi_k3_mla_key_concat_ds_mla_insert):
* - full key concat (bf16), and cache insert in DeepSeek's 656-byte
* block-scaled layout (NoPE fp8 in 4 tiles of 128 with per-tile dynamic
* scales, RoPE bf16), bit-compatible with concat_and_cache_ds_mla_kernel.
* block-scaled layout (NoPE fp8 in 4 tiles of 128 with per-tile
* power-of-two scales, RoPE bf16), bit-compatible with
* concat_and_cache_ds_mla_kernel.
*
* Both use Programmatic Dependent Launch (PDL) to overlap the tail of the
* producing GEMMs on sm_90+, and are structured after
@@ -68,7 +69,6 @@
#include "../quantization/w8a8/fp8/amd/quant_utils.cuh"
#endif
#include <cuda_runtime.h>
#include <cfloat>
#include <type_traits>
#ifdef USE_ROCM
@@ -111,9 +111,9 @@ constexpr float kFp8Max = 224.0f;
#else
constexpr float kFp8Max = 448.0f;
#endif
// Divisor for fp8_ds_mla per-tile dynamic scales (matches cache_kernels.cu).
// fp8_ds_mla 656B entry: [0,512) NoPE fp8 (4 tiles of 128), [512,528) 4 fp32
// tile scales, [528,656) RoPE 64 bf16.
// Divisor for fp8_ds_mla per-tile power-of-two scales (matches
// cache_kernels.cu). fp8_ds_mla 656B entry: [0,512) NoPE fp8 (4 tiles of 128),
// [512,528) 4 fp32 tile scales, [528,656) RoPE 64 bf16.
constexpr float kFp8ScaleDivisor = kFp8Max;
// Copy 8 source elements (one uint4 of bf16/fp16) to `dst`. FP8=false stores a
@@ -325,7 +325,7 @@ __device__ __forceinline__ void writeLatent576(
}
// Write [kv_c | k_pe] into the fp8_ds_mla 656B entry using one warp: NoPE 512
// as fp8 in 4 tiles of 128 (per-tile dynamic absmax scale, 4 fp32 scales at
// as fp8 in 4 tiles of 128 (per-tile power-of-two scale, 4 fp32 scales at
// [512,528)), RoPE 64 as bf16 at [528,656). Bit-compatible with
// concat_and_cache_ds_mla_kernel.
template <typename scalar_t, bool APPLY_ROPE = false>
@@ -349,7 +349,8 @@ __device__ __forceinline__ void writeDsMlaCache(
for (int offset = 4; offset > 0; offset /= 2) {
max_abs = fmaxf(max_abs, VLLM_SHFL_XOR_SYNC_WIDTH(max_abs, offset, 8));
}
float const tile_scale = fmaxf(max_abs / kFp8ScaleDivisor, FLT_MIN);
float tile_scale = fmaxf(max_abs / kFp8ScaleDivisor, 1e-4f);
tile_scale = exp2f(ceilf(log2f(tile_scale)));
if ((laneId & 7) == 0) {
reinterpret_cast<float*>(row)[kKvLoraRank / 4 + tile] = tile_scale;
}
@@ -598,7 +599,7 @@ __global__ void fusedKimiK3MLAKVConcatPackKernel(
// ds_mla variant: concat full key (bf16) + fp8_ds_mla latent cache insert
//
// Cache entry (656 bytes), matching concat_and_cache_ds_mla_kernel:
// [0, 512) NoPE 512 vals as fp8, 4 tiles of 128, each dynamically scaled
// [0, 512) NoPE 512 vals as fp8, 4 tiles of 128, power-of-two scaled
// [512, 528) 4 fp32 per-tile scales
// [528, 656) RoPE 64 vals as bf16 (unquantized)
// The cache slot uses one warp: lane L quantizes NoPE elems [L*16, L*16+16)
@@ -152,6 +152,29 @@ void cutlass_gemm_caller_blockwise(torch::stable::Tensor& out, torch::stable::Te
int32_t m = a.size(0), n = b.size(1), k = a.size(1);
// The SM100 blockwise SF copy requires the SFA M extent to be a multiple
// of 4 (4-row copy atom). For misaligned M on the non-swapAB path, pad only
// the per-token activation scales (column-major (m, k/128), 1/64 the size
// of the bf16 activation) and build layout_SFA from the padded extent.
int32_t m_sf = m;
std::optional<torch::stable::Tensor> a_scales_pad;
void const* a_scales_data = a_scales.data_ptr();
if constexpr (!swap_ab) {
if (m % 4 != 0) {
m_sf = (m + 3) & ~3;
int64_t num_groups = a_scales.numel() / m;
a_scales_pad = torch::stable::new_empty(
a_scales, {num_groups * m_sf},
torch::headeronly::ScalarType::Float);
cudaMemcpy2DAsync(a_scales_pad->data_ptr(), m_sf * sizeof(float),
a_scales.data_ptr(), m * sizeof(float),
m * sizeof(float), num_groups,
cudaMemcpyDeviceToDevice,
get_current_cuda_stream(a.get_device()));
a_scales_data = a_scales_pad->data_ptr();
}
}
StrideA a_stride;
StrideB b_stride;
StrideC c_stride;
@@ -162,16 +185,16 @@ void cutlass_gemm_caller_blockwise(torch::stable::Tensor& out, torch::stable::Te
c_stride =
cutlass::make_cute_packed_stride(StrideC{}, swap_ab ? cute::make_shape(n, m, 1) : cute::make_shape(m, n, 1));
LayoutSFA layout_SFA = swap_ab ?
LayoutSFA layout_SFA = swap_ab ?
ScaleConfig::tile_atom_to_shape_SFA(make_shape(n, m, k, 1)) :
ScaleConfig::tile_atom_to_shape_SFA(make_shape(m, n, k, 1));
ScaleConfig::tile_atom_to_shape_SFA(make_shape(m_sf, n, k, 1));
LayoutSFB layout_SFB = swap_ab ?
ScaleConfig::tile_atom_to_shape_SFB(make_shape(n, m, k, 1)) :
ScaleConfig::tile_atom_to_shape_SFB(make_shape(m, n, k, 1));
auto a_ptr = static_cast<ElementAB const*>(a.data_ptr());
auto b_ptr = static_cast<ElementAB const*>(b.data_ptr());
auto a_scales_ptr = static_cast<ElementBlockScale const*>(a_scales.data_ptr());
auto a_scales_ptr = static_cast<ElementBlockScale const*>(a_scales_data);
auto b_scales_ptr = static_cast<ElementBlockScale const*>(b_scales.data_ptr());
typename GemmKernel::MainloopArguments mainloop_args{};
@@ -212,7 +235,9 @@ void cutlass_gemm_blockwise_sm100_fp8_dispatch(torch::stable::Tensor& out,
constexpr int TILE_K = 128;
// TODO: better heuristics
bool swap_ab = (m < 16) || (m % 4 != 0);
// swapAB only wins for small M; misaligned M >= 65 is faster on the
// non-swapAB path with padded activation scales (measured on GB300).
bool swap_ab = (m <= 64);
bool use_tma_epilogue = (m * n) % 4 == 0;
if (!swap_ab) {
constexpr int TILE_N = 128;
+5
View File
@@ -231,10 +231,12 @@ RUN --mount=type=cache,target=/opt/uv/cache \
echo "Installing Rubin prerelease requirements..." \
&& /opt/venv/bin/python3 use_existing_torch.py --prefix \
&& uv pip install --python /opt/venv/bin/python3 \
--prerelease=allow --index-strategy unsafe-first-match \
-r /tmp/vllm-requirements/rubin-prerelease.txt \
&& uv pip install --python /opt/venv/bin/python3 \
-r requirements/cuda.txt \
&& uv pip install --python /opt/venv/bin/python3 \
--prerelease=allow --index-strategy unsafe-first-match \
-r /tmp/vllm-requirements/rubin-prerelease.txt; \
elif [ "${PYTORCH_NIGHTLY}" = "1" ]; then \
echo "Installing torch nightly..." \
@@ -941,11 +943,13 @@ RUN --mount=type=cache,target=/opt/uv/cache \
fi && \
if [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ]; then \
uv pip install --system \
--prerelease=allow --index-strategy unsafe-first-match \
-r /tmp/vllm-requirements/rubin-prerelease.txt \
&& sed -Ei '/^[[:space:]]*(torch|torchvision|torchaudio)[[:space:]]*==/Id' \
/tmp/requirements-cuda.txt \
&& uv pip install --system -r /tmp/requirements-cuda.txt \
&& uv pip install --system \
--prerelease=allow --index-strategy unsafe-first-match \
-r /tmp/vllm-requirements/rubin-prerelease.txt \
&& printf '%s\n' \
"/usr/local/lib/python${PYTHON_VERSION}/dist-packages/nvpl/lib" \
@@ -1262,6 +1266,7 @@ RUN --mount=type=cache,target=/opt/uv/cache \
if [ "${INSTALL_RUBIN_PRERELEASE}" = "true" ]; then \
echo "Installing Rubin prerelease requirements..." \
&& uv pip install --system \
--prerelease=allow --index-strategy unsafe-first-match \
-r /tmp/vllm-requirements/rubin-prerelease.txt; \
fi
+1 -1
View File
@@ -6,7 +6,7 @@ ARG TORCHAUDIO_VERSION=2.11.0+rocm10.0.0
ARG TRITON_VERSION=3.8.0+git4cff872c.rocm10.0.0
ARG FA_BRANCH="0e60e394"
ARG FA_REPO="https://github.com/Dao-AILab/flash-attention.git"
ARG AITER_BRANCH="v0.1.21.post1"
ARG AITER_BRANCH="v0.1.22.post1"
ARG AITER_REPO="https://github.com/ROCm/aiter.git"
ARG MORI_BRANCH="f95cd5e0ca7b730f92f9ffdfb358484d6965d9f7" # TheRock compatible v1.1.0
ARG MORI_REPO="https://github.com/ROCm/mori.git"
+2 -2
View File
@@ -9,13 +9,13 @@ ARG PYTORCH_AUDIO_BRANCH="v2.11.0"
ARG PYTORCH_AUDIO_REPO="https://github.com/pytorch/audio.git"
ARG FA_BRANCH="0e60e394"
ARG FA_REPO="https://github.com/Dao-AILab/flash-attention.git"
ARG AITER_BRANCH="v0.1.21.post2"
ARG AITER_BRANCH="v0.1.22.post1"
ARG AITER_REPO="https://github.com/ROCm/aiter.git"
ARG MORI_BRANCH="v1.1.0"
ARG MORI_REPO="https://github.com/ROCm/mori.git"
ARG ROCM_SYSTEMS_REPO="https://github.com/ROCm/rocm-systems.git"
ARG ROCPROFILER_SDK_COMMIT="2b22ab0195cc1461cd9abf3b969e9dd7c10af350"
ARG ROCM_RUNTIME_COMMIT="b539bf7eebfd99ad0a69668caa1f4037034d501f"
ARG ROCM_RUNTIME_COMMIT="af70edc91f1dc5d228fd4c076df99e65e49c4521"
# Sccache configuration (only used in release pipeline)
ARG USE_SCCACHE
-6
View File
@@ -169,8 +169,6 @@ ENV PKG_CONFIG_PATH="/opt/rh/gcc-toolset-14/root/usr/lib64/pkgconfig:/usr/local/
ENV PATH="${VIRTUAL_ENV:+${VIRTUAL_ENV}/bin}:/opt/rh/gcc-toolset-14/root/usr/bin:/usr/local/bin:$CARGO_HOME/bin:$RUSTUP_HOME/bin:$PATH"
ENV PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
ENV UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
# Force pure Python protobuf to avoid s390x C++ extension crashes
ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
COPY . /workspace/vllm
WORKDIR /workspace/vllm
@@ -214,10 +212,6 @@ RUN --mount=type=cache,target=/root/.cache/uv \
VLLM_TARGET_DEVICE=cpu VLLM_CPU_MOE_PREPACK=0 python setup.py bdist_wheel && \
uv pip install "$(echo dist/*.whl)[tensorizer]"
# Remove protobuf C++ extension that crashes on s390x
RUN rm -rf /opt/vllm/lib64/python${PYTHON_VERSION}/site-packages/google/_upb/*.so \
/opt/vllm/lib64/python${PYTHON_VERSION}/site-packages/google/protobuf/pyext/*.so 2>/dev/null || true
# setup non-root user for vllm
RUN umask 002 && \
/usr/sbin/useradd --uid 2000 --gid 0 vllm && \
+2 -2
View File
@@ -147,7 +147,7 @@ With standard decoding, each streamed output usually contains one token, so ITL
and TPOT are typically similar.
With speculative decoding, one streamed output can contain multiple tokens,
such as several accepted draft tokens within a single engine tstep. ITL records
such as several accepted draft tokens within a single engine step. ITL records
only the gaps between streamed outputs; it does not add zero-duration gaps for
tokens in the same output. TPOT instead amortizes the request's decoding time
over every output token.
@@ -451,7 +451,7 @@ vllm bench serve \
--num-prompts -1
```
Available categories include `[high_entropy, mixed, low_entropy]`, where high entropy data contains unstructued data such as creative writing while low entropy data contains more structured data such as coding, more details are in the dataset card.
Available categories include `[high_entropy, mixed, low_entropy]`, where high entropy data contains unstructured data such as creative writing while low entropy data contains more structured data such as coding, more details are in the dataset card.
#### BFCL (Tool-Calling) Benchmark
+1 -1
View File
@@ -36,7 +36,7 @@ When run, benchmark script generates results under **benchmark/results** folder,
- `ON_CPU`: set the value to '1' on Intel® Xeon® and Arm® Neoverse™ Processors. Default value is 0.
- `SERVING_JSON`: JSON file to use for the serving tests. Default value is empty string (use default file).
- `LATENCY_JSON`: JSON file to use for the latency tests. Default value is empty string (use default file).
- `THROUGHPUT_JSON`: JSON file to use for the throughout tests. Default value is empty string (use default file).
- `THROUGHPUT_JSON`: JSON file to use for the throughput tests. Default value is empty string (use default file).
- `REMOTE_HOST`: IP for the remote vLLM service to benchmark. Default value is empty string.
- `REMOTE_PORT`: Port for the remote vLLM service to benchmark. Default value is empty string.
- `PROMPTS_PER_CONCURRENCY`: Multiplier to compute `num_prompts` for serving tests (`num_prompts = max_concurrency × value`). Overrides JSON `num_prompts`. Default is NULL.
+1 -1
View File
@@ -50,7 +50,7 @@ To rebuild only the Rust frontend binary:
./build_rust.sh --debug # faster build for development
```
If you are developing vLLM's Python and CUDA/C++ code, install Pytorch first:
If you are developing vLLM's Python and CUDA/C++ code, install PyTorch first:
```bash
uv pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu129
+1 -1
View File
@@ -670,7 +670,7 @@ fall under the more general heading of "Observability".
vLLM has support for OpenTelemetry tracing:
- Added by <https://github.com/vllm-project/vllm/pull/4687> and reinstated by <https://github.com/vllm-project/vllm/pull/20372>
- Configured with `--oltp-traces-endpoint` and `--collect-detailed-traces`
- Configured with `--otlp-traces-endpoint` and `--collect-detailed-traces`
- [OpenTelemetry blog post](https://opentelemetry.io/blog/2024/llm-observability/)
- [User-facing docs](../../examples/observability/opentelemetry/README.md)
- [Blog post](https://medium.com/@ronen.schaffer/follow-the-trail-supercharging-vllm-with-opentelemetry-distributed-tracing-aa655229b46f)
+77 -276
View File
@@ -8,84 +8,40 @@ A "custom" logits processor is written by a user of vLLM and is loaded into vLLM
This document shows how to write, load and use a custom logits processor.
!!! note
This document covers the Model Runner V2 (MRV2) interface. The legacy V1
model runner (`VLLM_USE_V2_MODEL_RUNNER=0`) uses a different interface,
`vllm.v1.sample.logits_processor.LogitsProcessor`, which is not covered
here.
## Logits Processors Background
A logits processor adjusts the next-token probability distribution, usually with the intention of steering the model towards a desired type of behavior.
In vLLM, logits processors operate at batch granularity. During a given engine step, the logits processor consumes a `(num_requests) x (vocab_size)` tensor of raw logits output by the model. For all requests which enable the logits processor, the logits processor applies a transformation to the corresponding row of the logits tensor, while leaving other rows unmodified. The transformed logits tensor is then passed to softmax.
In vLLM, logits processors operate at batch granularity. During a given engine step, the logits processor consumes a `(num_logits_rows) x (vocab_size)` tensor of raw logits output by the model. Note that a logits row is not a request: rows are reordered every step, and under speculative decoding a request owns one row per draft token. The logits processor applies a transformation to the rows of the logits tensor, while leaving other rows unmodified. The transformed logits tensor is then used for sampling.
## Creating a Custom Logits Processor
Custom logits processors must subclass `vllm.v1.sample.logits_processor.LogitsProcessor` and define (at minimum) the following methods:
Custom logits processors must subclass `vllm.v1.worker.gpu.sample.logits_processor.LogitsProcessor` and define (at minimum) the following methods:
* `validate_params(cls, sampling_params: SamplingParams)`:
* Raise `ValueError` if `SamplingParams` has invalid arguments (especially custom arguments) used by logits processor.
* When request is sent to entrypoint, `validate_params()` will validate `SamplingParams` and refuse request with invalid arguments.
* **Note:** it's important to implement `validate_params()` to prevent invalid parameters for custom logits processor. Otherwise requests with invalid parameters can cause unexpected behaviour in custom logits processor.
* `__init__(self, vllm_config: VllmConfig, device: torch.device, is_pin_memory: bool)`
* `__init__(self, vllm_config: VllmConfig, req_states: LogitsProcRequestState)`:
* `vllm_config`: engine configuration data structure
* `device`: hardware accelerator device info
* `is_pin_memory`: flag indicating whether pin memory is available to support logits processor implementation
* `req_states`: a narrow, read-only view of the persistent batch, exposing the on-device token history (`all_token_ids`, `prompt_len`, `prefill_len`, `total_len`) plus `device`, `max_num_reqs` and `vocab_size`
* `apply(self, logits: torch.Tensor) -> torch.Tensor`:
* Consume a `(num_requests) x (vocab_size)` logits tensor (`logits`)
* `add_request(self, req_idx, sampling_params) -> bool`:
* Initialize per-slot state for a request entering the batch. Slots are recycled through a free list, so per-slot state must be fully (re)initialized here; there is no removal hook, since freed slots are never read
* Return whether this processor modifies logits for the request. The sampler skips the logits-processing pipeline for batches in which no request needs it
* `apply_staged_writes(self) -> None` (optional):
* Flush host-side writes staged by `add_request()` to the device; called once per step before the forward pass
* `validate_params(cls, sampling_params) -> None` (optional classmethod):
* Raise `ValueError` for invalid per-request arguments (especially custom arguments); runs at request admission, so invalid arguments fail the request instead of reaching the sampler
* `apply(self, logits: torch.Tensor, ctx: LogitsContext) -> torch.Tensor`:
* Consume a `(num_logits_rows) x (vocab_size)` logits tensor and the step's batch layout (`ctx`: row-to-slot mappings, `input_ids`, positions)
* Apply logits processor transformation at batch granularity
* Return a transformed `(num_requests) x (vocab_size)` logits tensor
* You can modify the input logits processors in-place or out-of-place; in-place is more memory-efficient
* `is_argmax_invariant(self) -> bool`:
* Return `True` if the logits processor is argmax invariant (never changes what is the highest-logit-value token ID for a given request), `False` if the logits processor may modify argmax
* `is_argmax_invariant()` is evaluated once at startup; if `True`, vLLM will skip applying this logits processor in a given step when all requests use greedy sampling
* `update_state(self, batch_update: Optional["BatchUpdate"]) -> None`:
* Consume a `BatchUpdate` data structure representing persistent batch state changes at the beginning of the current engine step
* Use the `BatchUpdate` members to update logits processor internal state
* **Note:** batch update data structure may be `None`, signaling no change to the batch constituents. In this case, the LogitsProcessor might still want to update its state based on the updated `output_token_ids` lists that it could have retained when they were added.
### How the vLLM engine builds the `BatchUpdate` data structure
!!! important
Some logits processors design changes are still in progress. We expect
that in the future you will not need to account for batch state changes
when implementing a logits processor, and the information in this section
will become irrelevant.
Logits processor `update_state()` implementations should assume the following model for how the model runner updates persistent batch state (expressed here in terms of the `BatchUpdate` abstraction):
1. Identify indices of requests which finished in the current engine step
2. Identify new requests introduced in the current step
3. Use Add operations to replace as many finished requests with new requests, in order of increasing index of the replaced request starting with the lowest index
4. Based on the relative number of new and finished requests:
1. If the numbers of new and finished requests are the same, proceed to next step
2. *If there are more new requests than finished requests:* apply Add operations to extend the batch with the remaining new requests which did not replace finished requests. Assign consecutive indices to these new requests, starting with `current_max_batch_index + 1`
3. *If there are fewer new requests than finished requests:*
* Apply Remove operations to finished requests which were not replaced with new requests. These removed request indices will necessarily be greater than the greatest index of the finished requests which were replaced in the previous step. The Removes may leave the batch in a non-contiguous state
* **"Condense" the batch to be contiguous:** starting with the lowest-index empty slot (which was caused by a Remove), apply a Unidirectional Move from the current highest non-empty slot in the batch to fill the empty slot. Proceed with additional Unidirectional Move operations in order of increasing empty slot destination index and decreasing non-empty slot source index until the batch is contiguous
* **Shrink the batch:** a side effect of condensing the batch is that empty slots resulting from Remove operations are grouped in a contiguous block at the end of the batch array. Thus, after condensing, update `BatchUpdate.batch_size` to reflect the number of non-empty slots
5. Reorder the batch for improved efficiency. Depending on the attention backend implementation and the current characteristics of the batch, zero or more Swap Move operations may be applied to reorder the batch
Notes:
* A logits processor `update_state()` method must process batch update operations in the following order: removes, adds, moves
* The index argument for Add operations refers to the index *at the time the Add occurred*, i.e. before any Move operations
* Example: if a request is Added at index 5 and then swapped with index 3, the Add operation in `BatchUpdate.added` will be associated with index 5 not 3
* In other words Move operations can be assumed to be applied after Adds and Removes
* Move operations can be assumed to be applied in the order in which they appear in `BatchUpdate.moved`
* If there are no new/finished requests and there is no batch reordering, then the batch update for the logits processors will be `None`
* Return a transformed logits tensor. You can modify the input logits in-place or out-of-place; in-place is more memory-efficient
### Passing Custom Argument to a Custom Logits Processor
@@ -93,7 +49,7 @@ Unlike built-in logits processors, custom logits processors may require configur
### Example Custom Logits Processor Implementation
The contrived example below implements a custom logits processor which consumes a `(num\_requests) \times (vocab\_size)` logits tensor and masks out all tokens except for one (`target_token`) with `float(-inf)`. The logits processor is disabled for any request that does not specify `target_token`. To determine whether the logits processor is enabled and which token to leave unmasked, the logits processor checks `SamplingParams.extra_args` for a `target_token` custom argument associated with each request:
The contrived example below implements a custom logits processor which masks out all tokens except for one (`target_token`) for the requests that enable it. The processor is disabled for any request that does not specify `target_token`; `add_request()` reports this via its return value, so the sampler can skip the logits-processing pipeline entirely when no request enables the processor. To determine whether the logits processor is enabled and which token to leave unmasked, the processor checks `SamplingParams.extra_args` for a `target_token` custom argument associated with each request:
??? code "Example custom logits processor definition"
@@ -101,193 +57,55 @@ The contrived example below implements a custom logits processor which consumes
import torch
from vllm.config import VllmConfig
from vllm.sampling_params import SamplingParams
from vllm.v1.sample.logits_processor import (BatchUpdate,
LogitsProcessor,
MoveDirectionality)
class DummyLogitsProcessor(LogitsProcessor):
"""Fake logit processor to support unit testing and examples"""
@classmethod
def validate_params(cls, params: SamplingParams):
target_token: int | None = params.extra_args and params.extra_args.get(
"target_token"
)
if target_token is not None and not isinstance(target_token, int):
raise ValueError(f"target_token value {target_token} is not int")
def __init__(self, vllm_config: "VllmConfig", device: torch.device,
is_pin_memory: bool):
self.req_info: dict[int, int] = {}
def is_argmax_invariant(self) -> bool:
"""Never impacts greedy sampling"""
return False
def update_state(self, batch_update: BatchUpdate | None):
if not batch_update:
return
# Process added requests.
for index, params, _, _ in batch_update.added:
assert params is not None
self.validate_params(params)
if params.extra_args and (target_token :=
params.extra_args.get("target_token")):
self.req_info[index] = target_token
else:
self.req_info.pop(index, None)
if self.req_info:
# Process removed requests.
for index in batch_update.removed:
self.req_info.pop(index, None)
# Process moved requests, unidirectional move (a->b) and swap
# (a<->b)
for adx, bdx, direct in batch_update.moved:
a_val = self.req_info.pop(adx, None)
b_val = self.req_info.pop(bdx, None)
if a_val is not None:
self.req_info[bdx] = a_val
if direct == MoveDirectionality.SWAP and b_val is not None:
self.req_info[adx] = b_val
def apply(self, logits: torch.Tensor) -> torch.Tensor:
if not self.req_info:
return logits
# Save target values before modification
cols = torch.tensor(
list(self.req_info.values()), dtype=torch.long, device=logits.device
)
rows = torch.tensor(
list(self.req_info.keys()), dtype=torch.long, device=logits.device
)
values_to_keep = logits[rows, cols].clone()
# Mask all but target tokens
logits[rows] = float('-inf')
logits[rows, cols] = values_to_keep
return logits
```
In the rest of this document, we will use `DummyLogitsProcessor` as an example of a custom logits processor.
The `DummyLogitsProcessor.update_state()` implementation maintains a "sparse" representation of the batched requests in the `self.req_info` dictionary: only those requests which specify a `target_token` value have a key in the dictionary. `update_state()` adjusts the stored request indices and `target_token` values (keys and values respectively in `self.req_info`) in response to Add, Remove and Move operations against the persistent batch.
### Wrapping an Existing Request-Level Logits Processor
Although the vLLM engine applies logits processors at batch granularity, some users may want to use vLLM with a "request-level" logits processor implementation - an implementation which operates on individual requests. This will be especially true if your logits processor was developed for vLLM version 0, which required it to be a `Callable` (as described [here][vllm.logits_process]) conforming to the following type annotation:
``` python
RequestLogitsProcessor = Union[
# (output token ids, logits tensor) -> logits tensor
Callable[[list[int], Tensor], Tensor],
# (prompt token ids, output token ids, logits tensor) -> logits tensor
Callable[[list[int], list[int], Tensor], Tensor],
]
```
While request-level logits processors are explicitly *not* supported in the vLLM engine, vLLM *does* provide a convenient process to wrap an existing `Callable` request-level logits processor and create a batch-level logits processor that is compatible with vLLM. The `Callable` must conform to the type annotation above; if your request-level logits processor has a different interface, then in order to wrap it, you may need to modify it or implement an additional wrapper layer to comply with the interface specification above.
You can wrap the request-level logits processor by subclassing `AdapterLogitsProcessor` as shown in the example below (in this example, `DummyPerReqLogitsProcessor` is a stand-in for your request-level logits processor which needs to be wrapped.):
* Override `AdapterLogitsProcessor.validate_params(cls,params)` to validate request's sampling parameters.
* Override `AdapterLogitsProcessor.is_argmax_invariant(self)` to accurately reflect whether your request-level logits processor may impact which token has the highest-value logit.
* Override `AdapterLogitsProcessor.new_req_logits_processor(self,params)` to create a new request-level logits processor instance from a `SamplingParams` instance:
??? code "Example of Wrapping a Request-Level Logits Processor"
``` python
...
from vllm.v1.sample.logits_processor import (
AdapterLogitsProcessor, # Wrapper base-class
RequestLogitsProcessor, # Request-level logitsproc type annotation
from vllm.v1.worker.gpu.sample.logits_processor import (
LogitsProcRequestState,
LogitsContext,
LogitsProcessor,
)
...
# Stand-in for your request-level logits processor:
class DummyPerReqLogitsProcessor:
"""The request-level logits processor masks out all logits except the
token id identified by `target_token`"""
class TargetTokenLogitsProcessor(LogitsProcessor):
"""Masks out all tokens except `target_token` (a per-request custom
argument); requests without it are left alone."""
def __init__(self, target_token: int) -> None:
"""Specify `target_token`"""
self.target_token = target_token
def __init__(self, vllm_config: "VllmConfig", req_states: LogitsProcRequestState):
# Per-slot target; -1 means disabled. Staged on the host and
# flushed to the device once per step in apply_staged_writes().
self.target_token = torch.full(
(req_states.max_num_reqs,), -1, dtype=torch.int64
)
self.target_token_dev = torch.full(
(req_states.max_num_reqs,), -1, dtype=torch.int64, device=req_states.device
)
def __call__(
self,
output_ids: list[int],
logits: torch.Tensor,
) -> torch.Tensor:
val_to_keep = logits[self.target_token].item()
logits[:] = float("-inf")
logits[self.target_token] = val_to_keep
def add_request(self, req_idx: int, sampling_params: SamplingParams) -> bool:
target = (sampling_params.extra_args or {}).get("target_token")
self.target_token[req_idx] = target if target is not None else -1
return target is not None
def apply_staged_writes(self) -> None:
self.target_token_dev.copy_(self.target_token, non_blocking=True)
def apply(self, logits: torch.Tensor, ctx: LogitsContext) -> torch.Tensor:
# ctx.expanded_idx_mapping maps each logits row to its request slot.
cols = self.target_token_dev[ctx.expanded_idx_mapping.long()]
rows = torch.nonzero(cols >= 0).squeeze(1)
if rows.numel() == 0:
return logits
kept = logits[rows, cols[rows]].clone()
logits[rows] = float("-inf")
logits[rows, cols[rows]] = kept
return logits
...
# Example of wrapping the request-level logits processor:
class WrappedPerReqLogitsProcessor(AdapterLogitsProcessor):
"""Example of wrapping a fake request-level logit processor to create a
batch-level logits processor"""
@classmethod
def validate_params(cls, params: SamplingParams):
target_token: Any | None = params.extra_args and params.extra_args.get(
"target_token"
)
if target_token is not None and not isinstance(target_token, int):
raise ValueError(
f"target_token value {target_token} is not int"
)
def is_argmax_invariant(self) -> bool:
return False
def new_req_logits_processor(
self,
params: SamplingParams,
) -> Optional[RequestLogitsProcessor]:
"""This method returns a new request-level logits processor, customized
to the `target_token` value associated with a particular request.
Returns None if the logits processor should not be applied to the
particular request. To use the logits processor the request must have
a "target_token" custom argument with an integer value.
Args:
params: per-request sampling params
Returns:
`Callable` request logits processor, or None
"""
target_token: Any | None = params.extra_args and params.extra_args.get(
"target_token"
)
if target_token is None:
return None
return DummyPerReqLogitsProcessor(target_token)
```
!!! note
Your `new_req_logits_processor()` override can return `None` to signal that the wrapped logits processor should not be applied to the request in question.
Once you have created a custom subclass (like `WrappedPerReqLogitsProcessor`) which wraps your request level logits processor, you can pass the custom subclass to vLLM via any of the methods described in the following section.
Per-request state is keyed by the request slot index, and slots are recycled through a free list. The example keeps a per-slot state where `-1` means disabled; note that `add_request()` overwrites the slot's entry unconditionally, which is what keeps recycled slots from leaking state between requests.
## Ways to Load Your Custom Logits Processor in vLLM
Logits processors are loaded at initialization. Critically, the set of loaded logits processors cannot be modified after the vLLM engine finishes loading, and new logits processors cannot be loaded on-demand for individual requests.
Loaded classes are validated against the Model Runner V2 interface; a class that does not subclass `vllm.v1.worker.gpu.sample.logits_processor.LogitsProcessor` is rejected with a clear error.
This section details different ways of making your logits processor visible to vLLM and triggering vLLM to load your logits processor.
### Method 1: Pass the Custom Logits Processor Fully-Qualified Class Name (FQCN) to vLLM at Initialization Time
@@ -314,7 +132,7 @@ See examples below:
# Pass in FQCN
llm = LLM(
model="facebook/opt-125m",
logits_processors=["your.module.path:DummyLogitsProcessor"],
logits_processors=["your.module.path:TargetTokenLogitsProcessor"],
)
```
@@ -323,14 +141,14 @@ See examples below:
``` python
# Pass in FQCN
engine_args = AsyncEngineArgs(model="facebook/opt-125m",
logits_processors=["your.module.path:DummyLogitsProcessor"])
logits_processors=["your.module.path:TargetTokenLogitsProcessor"])
async_llm = AsyncLLM.from_engine_args(engine_args)
```
??? code "Passing custom logits processor FQCN to vLLM server via CLI"
```bash
vllm serve facebook/opt-125m --logits_processors your.module.path:DummyLogitsProcessor
vllm serve facebook/opt-125m --logits_processors your.module.path:TargetTokenLogitsProcessor
```
### Method 2: Automatically Detect Custom Logits Processors Installed in Your Python Environment As Entry Points
@@ -345,7 +163,7 @@ Suppose that you have developed a Python package that holds your custom logits p
``` toml
[project.entry-points."vllm.logits_processors"]
dummy_logits_processor = "your.module.path:DummyLogitsProcessor"
target_token_logits_processor = "your.module.path:TargetTokenLogitsProcessor"
```
Once your package is installed, your custom logits processor will be loaded automatically whenever vLLM is initialized. You do *not* need to pass the custom logits processor to the `LLM` or `AsyncLLM` constructors or to the vLLM server explicitly at initialization time if your logits processor is exposed as an entry point.
@@ -361,26 +179,26 @@ You can pass one or more custom logits processor class objects to the `LLM` and
``` python
# Import custom logits processor
from some.module import DummyLogitsProcessor
from some.module import TargetTokenLogitsProcessor
# ...or...
# Define custom logits processor locally
from vllm.v1.sample.logits_processor import LogitsProcessor
from vllm.v1.worker.gpu.sample.logits_processor import LogitsProcessor
class DummyLogitsProcessor(LogitsProcessor):
# See DummyLogitsProcessor implementation above
class TargetTokenLogitsProcessor(LogitsProcessor):
# See TargetTokenLogitsProcessor implementation above
...
# Pass class object to LLM constructor
llm = LLM(
model="facebook/opt-125m",
logits_processors=[DummyLogitsProcessor],
logits_processors=[TargetTokenLogitsProcessor],
)
# Pass class object to AsyncLLM constructor
engine_args = AsyncEngineArgs(model="facebook/opt-125m",
logits_processors=[DummyLogitsProcessor])
logits_processors=[TargetTokenLogitsProcessor])
async_llm = AsyncLLM.from_engine_args(engine_args)
```
@@ -388,7 +206,7 @@ You can pass one or more custom logits processor class objects to the `LLM` and
The design of the custom logits processor determines whether the logits processor must be enabled/disabled for a given request, and what arguments must be provided to configure the logits processor.
The examples below show how a user would pass a custom argument (`target_token`) to `DummyLogitsProcessor` in order to (1) enable the logits processor for that particular request and (2) control the logits processor's behavior.
The examples below show how a user would pass a custom argument (`target_token`) to `TargetTokenLogitsProcessor` in order to (1) enable the logits processor for that particular request and (2) control the logits processor's behavior.
??? code "vLLM REST API: configure custom logits processor for a request"
@@ -419,7 +237,7 @@ The examples below show how a user would pass a custom argument (`target_token`)
??? code "Offline: configure custom logits processor for an `LLM` request"
``` python
outputs_logitproc = llm.generate("your prompt",
outputs_logitproc = llm.generate("your prompt",
SamplingParams(...,
extra_args={"target_token": 67}))
```
@@ -438,31 +256,14 @@ The examples below show how a user would pass a custom argument (`target_token`)
## Best Practices for Writing Custom Logits Processors
Once vLLM loads a logits processor during initialization, then vLLM will invoke `update_state()` and `apply()` against that logits processor in every engine step. Both methods operate on all requests which currently reside in the vLLM persistent batch. Thus, it is important to implement these methods efficiently.
* Write efficient `apply()` and `update_state()` implementations in light of the fact that logits processors operate at batch granularity
* For example, you may be able to use efficient vectorized operations to implement `apply()` or update internal state vectors in `update_state()`
* However, if you think that a logits processor may be used infrequently, it may be appropriate to use a "sparse" representation of request state i.e. the class can represent request configuration using a dictionary which only stores metadata about requests that enable the logits processor
* **Note:** wrapped request-level logits processors do not need to implement `apply()` and `update_state()`; the default `AdapterLogitsProcessor.update_state()` implementation maintains a sparse representation of request state, wherein requests for which `new_req_logits_processor()` returns `None` are not represented in the base-class state dictionary. The default implementation of `AdapterLogitsProcessor.apply()` applies the request-level logits processor to each row of input logits sequentially and assembles the output logits tensor. If the performance of this `AdapterLogitsProcessor` default implementation is insufficient, then avoid wrapping your request-level logits processor and instead re-implement it as a `LogitsProcessor` subclass with optimized `apply()` and `update_state()` implementations that operate at batch granularity
* Write an efficient `apply()` implementation in light of the fact that logits processors operate at batch granularity. For example, you may be able to use efficient vectorized operations to implement `apply()`, and to stage per-request state into contiguous tensors in `add_request()` instead of per-row Python loops
* It is up to the logits processor author to determine:
1. **The per-request attributes which configure the logits processor's behavior against that request.** Your custom logits processor's `update_state()` override determines how `SamplingParams` fields are mapped into logits processor state
1. **The per-request attributes which configure the logits processor's behavior against that request.** Your custom logits processor's `add_request()` override determines how `SamplingParams` fields are mapped into logits processor state
* **Note:** for wrapped request-level logits processors, `new_req_logits_processor()` determines how `SamplingParams` fields are used to initialize a request-level logits processor instance.
2. **The conditions under which the logits processor is or is not enabled on a per-request basis.** Unless your intention is for the custom logits processor to act on all requests all the time, you should write your logits processor in such a way that it is possible to disable the logits processor for a given request, i.e. by defaulting an argument to `None` or by passing in a specific do-nothing argument value i.e. `0.0`. Return `False` from `add_request()` for such requests so that the sampler can skip the logits-processing pipeline for batches in which no request needs it
2. **The conditions under which the logits processor is or is not enabled on a per-request basis.** Unless your intention is for the custom logits processor to act on all requests all the time, you should write your logits processor in such a way that it is possible to disable the logits processor for a given request, i.e. by defaulting an argument to `None` or by passing in a specific do-nothing argument value i.e. `0.0`. Try to save compute and memory for requests which disable the logits processor
* Stage host-side writes in `add_request()` and flush them once per step in `apply_staged_writes()`, instead of writing device tensors directly; this is the discipline every built-in sampler state follows
* **Note:** for wrapped per-request logits processors, the default `AdapterLogitsProcessor.update_state()` implementation ensures that the request-level logits processor is disabled when `new_req_logits_processor()` returns `None` for that request
3. **The conditions under which the logits processor is short-circuited at the batch level.** Even if you have defined a way to disable the custom logits processor at the request level, it may be difficult to translate this into compute savings i.e. if your `update_state()` and `apply()` implementations use efficient vectorized implementations that operate on the whole persistent batch in a single command. For example, you cannot skip an entire vectorized operation in `apply()` just because one request disabled the logits processor. To save compute in the edge-case where no running requests utilize the custom logits processor, we recommend designing `apply()` to return the unmodified input tensor if all requests have the logits processor disabled. Similarly, consider whether steps can be skipped in `update_state()` if no requests enable the logits processor
* Additionally, an easy way to save compute in `update_state()` is to exit early when the `batch_update` is `None`
* **Note:** for wrapped per-request logits processors, the `AdapterLogitsProcessor` base-class implements the above optimizations by default
* Ensure that the logits processor `update_state` method discards information about finished requests (i.e. requests which are replaced by an Add or which are subject to a Remove)
* **Note:** for wrapped per-request logits processors, the `AdapterLogitsProcessor` base-class handles this by default
* `is_argmax_invariant()` can be hard-coded to `True` or `False` if the logits processor has consistent behavior. However, the argmax invariance may also be determined programmatically (i.e. if your logits processor is user-customizable in some way that impacts whether the logits processor is argmax invariant). For this reason, `is_argmax_invariant()` is not a class method
* Since there is no removal hook, `add_request()` must fully (re)initialize the slot's state; do not rely on state left behind by the slot's previous occupant
+9 -6
View File
@@ -418,12 +418,13 @@ activity for the last reporting interval. Example output:
```text
KV Transfer metrics: Num successful transfers=4, Avg xfer time (ms)=1.381,
P90 xfer time (ms)=2.601, Avg post time (ms)=0.672, P90 post time (ms)=0.801,
Avg MB per transfer=2.25, Throughput (MB/s)=1629.549, Avg number of descriptors=72.0
Avg MB per transfer=2.25, Throughput (MB/s)=1629.549, Avg number of descriptors=72.0,
Num failed transfers=0, Num KV expired reqs=0
```
The table below describes each field. All timing values cover only the
successful transfers recorded in the current interval; failed transfers are
counted separately via Prometheus (see
successful transfers recorded in the current interval; failures are reported
as interval counts in the same line and via Prometheus (see
[Prometheus metrics](#prometheus-metrics) below).
| Metric | Unit | Description |
@@ -436,6 +437,8 @@ counted separately via Prometheus (see
| `Avg MB per transfer` | MB | Mean payload size per transfer, computed as `total bytes transferred / number of transfers`. Reflects the average KV cache footprint of a single request (sequence length × layers × head dimension × dtype bytes). |
| `Throughput (MB/s)` | MB/s | Effective bandwidth over the interval: `total MB transferred / total xfer time (s)` across all successful transfers. This is aggregate throughput, not per-request bandwidth. |
| `Avg number of descriptors` | count | Mean number of NIXL memory descriptors (scatter-gather segments) submitted per transfer. More descriptors indicate more fragmented or larger KV cache allocations; very high counts can increase descriptor-registration overhead. |
| `Num failed transfers` | count | Failed NIXL transfers, handshakes, and completion notifications (`send_notif`) in the interval, grouped because all are sporadic lower-transport-layer events. |
| `Num KV expired reqs` | count | Requests whose KV blocks expired before being read. Reported separately from the failure counts above because it is an autoscaler/lease-tuning signal rather than a transport issue. |
### Prometheus metrics
@@ -448,9 +451,9 @@ exported when NixlConnector is active:
| `vllm:nixl_post_time_seconds` | Histogram | Time to submit the transfer request to the RDMA backend (seconds). |
| `vllm:nixl_bytes_transferred` | Histogram | Bytes moved per transfer. |
| `vllm:nixl_num_descriptors` | Histogram | Descriptor count per transfer. |
| `vllm:nixl_num_failed_transfers` | Counter | Cumulative count of failed NIXL KV-block transfers. |
| `vllm:nixl_num_failed_notifications` | Counter | Cumulative count of failed completion notifications (`send_notif`). |
| `vllm:nixl_num_kv_expired_reqs` | Counter | Requests whose KV blocks expired on the prefiller before the decoder read them (tracked on the P instance). |
| `vllm:nixl_num_failed_transfers` | Counter | Cumulative count of failed NIXL KV-block transfers, including handshake and notification (`send_notif`) failures. These are grouped because all are sporadic lower-transport-layer events. |
| `vllm:nixl_num_failed_notifications` | Counter | Cumulative count of failed notifications (`send_notif`), retained for compatibility. These failures are also included in `vllm:nixl_num_failed_transfers`; do not sum the two counters. |
| `vllm:nixl_num_kv_expired_reqs` | Counter | Requests whose KV blocks expired on the prefiller before the decoder read them (tracked on the P instance). Kept separate from the failure counters above: KV expiry is an autoscaler/lease-tuning signal, not a transport issue. |
!!! tip
High `vllm:nixl_num_kv_expired_reqs` indicates that the prefiller's lease
+1 -1
View File
@@ -100,7 +100,7 @@ kv-cache and the quantization algorithm is AutoSmoothQuant.
!!! note
Note the quantization algorithm needs a JSON config file and the config file is located in
[Quark Pytorch examples](https://quark.docs.amd.com/latest/pytorch/pytorch_examples.html),
[Quark PyTorch examples](https://quark.docs.amd.com/latest/pytorch/pytorch_examples.html),
under the directory `examples/torch/language_modeling/llm_ptq/models`. For example,
AutoSmoothQuant config file for Llama is
`examples/torch/language_modeling/llm_ptq/models/llama/autosmoothquant_config.json`.
+21
View File
@@ -74,6 +74,18 @@ llm.wake_up(tags=["weights"])
llm.wake_up(tags=["kv_cache"])
```
#### Release only KV cache memory
`LLM.release_kv_cache_memory()` discards KV cache physical memory while keeping model weights resident. It requires a completed pause and all executor memory to be resident: full sleep, partial wake-up, and repeated release without restoring memory are rejected. Requests retained with `mode="keep"` are recomputed after wake-up.
Use the default `cumem` backend with managed allocations: `enable_cumem_allocator=True` on CUDA/ROCm (also enabled by `enable_sleep_mode=True`), or `enable_sleep_mode=True` on XPU. Other backends must implement selective discard; CPU and unmanaged allocations are unsupported.
```python
llm.sleep(level=0, mode="keep") # Wait for the pause to complete.
llm.release_kv_cache_memory()
llm.wake_up(tags=["kv_cache"]) # Reallocate KV cache and resume scheduling.
```
### Online Serving
To enable sleep mode in a vLLM server you need to initialize it with the flag `VLLM_SERVER_DEV_MODE=1` and pass `--enable-sleep-mode` to the vLLM server.
@@ -107,9 +119,18 @@ curl -X POST 'http://localhost:8000/collective_rpc' -H 'Content-Type: applicatio
curl -X POST 'http://localhost:8000/wake_up?tags=kv_cache'
```
To release only KV cache memory, wait for the level 0 pause to complete before calling the release endpoint. The same resident-memory and backend requirements as the Python API apply.
```bash
curl -X POST 'http://localhost:8000/sleep?level=0&mode=keep'
curl -X POST 'http://localhost:8000/release_kv_cache_memory'
curl -X POST 'http://localhost:8000/wake_up?tags=kv_cache'
```
#### HTTP endpoints
- `POST /sleep?level=1` — Put the model to sleep (`level=1`).
- `POST /release_kv_cache_memory` — Discard KV cache memory after a completed pause, while all executor memory is resident.
- `POST /wake_up` — Wake up the model. Supports optional `tags` query parameters for partial wake-up (e.g., `?tags=weights`).
- `POST /collective_rpc` — Perform a collective remote procedure call (RPC).
- `GET /is_sleeping` — Check if the model is sleeping.
@@ -96,7 +96,7 @@ The code used to request completions as a client remains unchanged:
},
gpu_memory_utilization=0.5,
)
outputs = llm.generate(prompts,sampling_params)
outputs = llm.generate(prompts, sampling_params)
for output in outputs:
prompt = output.prompt
+30 -10
View File
@@ -109,18 +109,22 @@ vLLM supports the `tool_choice='none'` option in the chat completion API. When t
## Constrained Decoding Behavior
Whether vLLM enforces the tool parameter schema during generation depends on the `tool_choice` mode and the per-tool `strict` field:
Structural-tag parsers resolve call obligation, grammar activation, and argument-schema enforcement separately. With structural-tag enforcement enabled:
| `tool_choice` value | Schema-constrained decoding | Behavior |
| `tool_choice` value | Call obligation | Structural-tag activation |
| --- | --- | --- |
| Named function | Yes (via structured outputs backend) | Arguments are guaranteed to be valid JSON conforming to the function's parameter schema. |
| `"required"` | Yes (via structured outputs backend) | Same as named function. The model must produce at least one tool call. |
| `"auto"` | Only when `strict: true` is set on at least one tool | Structural-tag parsers constrain tool-call arguments when a tool opts in with `strict: true`. Without it, the model generates freely and tool calls are extracted from raw text. |
| `"none"` | N/A | No tool calls are produced. |
| Named function | Call the selected function | Always |
| `"required"` | Produce at least one tool call | Always |
| `"auto"` | Tool calls are optional | When at least one tool sets `strict: true`, or `--tool-strict-level` is `function` or `parameter` |
| `"none"` | Tool calling is disabled | Disabled |
When a structural tag applies, each tool's declared parameter schema is enforced only when that tool sets `strict: true` or the server uses `--tool-strict-level parameter`. Tools with omitted or false `strict` receive broad argument-syntax constraints at levels `auto` and `function`, including for required and named calls.
For parsers using schema-derived JSON constraints, required and named calls continue to enforce the declared parameter schemas.
### Strict Mode
For `tool_choice="required"` or named function calling, structural-tag constraints are always applied regardless of the `strict` field. For `tool_choice="auto"`, setting `strict: true` on at least one tool opts in to structural-tag constraints; without it, the model generates freely and tool calls are extracted from raw text. The `strict` field is supported across all three API surfaces: Chat Completion, Responses, and Anthropic Messages.
Structural-tag constraints have two layers. The tool-call *envelope* (markup and function name) is constrained whenever a structural tag applies: always for `tool_choice="required"` and named function calling, and for `tool_choice="auto"` once at least one tool sets `strict: true` or the server raises the floor. The *argument schema* of an individual tool is pinned only when that tool sets `strict: true` (or under `--tool-strict-level parameter`); a tool that omits `strict` keeps its arguments unconstrained, even when another tool in the same request is strict. The `strict` field is supported across all three API surfaces: Chat Completion, Responses, and Anthropic Messages.
For best compatibility with strict schema enforcement, define tool parameter schemas in the OpenAI strict-schema style:
@@ -134,6 +138,22 @@ vLLM also provides a global toggle via the `VLLM_ENFORCE_STRICT_TOOL_CALLING` en
VLLM_ENFORCE_STRICT_TOOL_CALLING=false vllm serve ...
```
### Server-Side Strictness Floor
Most OpenAI-compatible clients and agent frameworks never set `strict` on their tools, so with `tool_choice="auto"` the model generates tool calls without any grammar and malformed markup can leak into the response. The `--tool-strict-level` option lets the server operator raise the floor for every request that carries tools, independently of what the client declares:
| Value | Behavior |
| --- | --- |
| `auto` (default) | Follow the request's tool choice and per-tool strictness. Required/named choices activate structural tags; `tool_choice="auto"` activates them when at least one tool sets `strict: true`. |
| `function` | Constrain the tool-call envelope (markup and the function name) for every request with tools, leaving argument contents free unless the client marked the tool `strict: true`. |
| `parameter` | Additionally pin argument schemas for every tool, as if every tool had `strict: true`. |
```bash
vllm serve ... --tool-strict-level function
```
The floor never relaxes a constraint the request would already receive: tools the client marked `strict: true` keep their schemas at every level. With `tool_choice="auto"`, the grammar does not force a tool call; a plain text response stays valid. `VLLM_ENFORCE_STRICT_TOOL_CALLING=false` disables structural tags entirely and takes precedence over this option.
## Automatic Function Calling
To enable this feature, you should set the following flags:
@@ -152,7 +172,7 @@ from HuggingFace; and you can find an example of this in a `tokenizer_config.jso
If your favorite tool-calling model is not supported, please feel free to contribute a parser & tool use chat template!
!!! note
With `tool_choice="auto"`, schema-level constraint requires both `VLLM_ENFORCE_STRICT_TOOL_CALLING=true` (the default) and at least one tool with `strict: true`. When these conditions are met and the selected parser supports structural tags, vLLM constrains tool-call arguments. Otherwise, vLLM extracts tool calls from raw text, so arguments may occasionally be malformed or violate the function's parameter schema.
With `tool_choice="auto"`, structural-tag constraints require both `VLLM_ENFORCE_STRICT_TOOL_CALLING=true` (the default) and at least one tool with `strict: true`, or a server-side floor set via `--tool-strict-level`. When these conditions are met and the selected parser supports structural tags, vLLM constrains the tool-call envelope and pins the argument schema of each tool that sets `strict: true` (or of every tool under `--tool-strict-level parameter`). Otherwise, vLLM extracts tool calls from raw text, so arguments may occasionally be malformed or violate the function's parameter schema.
### Hermes Models (`hermes`)
@@ -162,8 +182,8 @@ All Nous Research Hermes-series models newer than Hermes 2 Pro should be support
* `NousResearch/Hermes-2-Theta-*`
* `NousResearch/Hermes-3-*`
_Note that the Hermes 2 **Theta** models are known to have degraded tool call quality and capabilities due to the merge
step in their creation_.
*Note that the Hermes 2 **Theta** models are known to have degraded tool call quality and capabilities due to the merge
step in their creation*.
Flags: `--tool-call-parser hermes`
+2 -2
View File
@@ -150,7 +150,7 @@ VLLM_USE_PRECOMPILED=1 VLLM_PRECOMPILED_WHEEL_VARIANT=cpu VLLM_TARGET_DEVICE=cpu
- `VLLM_CPU_KVCACHE_SPACE`: specify the KV Cache size (e.g, `VLLM_CPU_KVCACHE_SPACE=40` means 40 GiB space for KV cache), larger setting will allow vLLM to run more requests in parallel. This parameter should be set based on the hardware configuration and memory management pattern of users. Default value is `0`.
- `VLLM_CPU_OMP_THREADS_BIND`: specify the CPU cores dedicated to the OpenMP threads, can be set as CPU id lists, `auto` (by default), or `nobind` (to disable binding to individual CPU cores and to inherit user-defined OpenMP variables). For example, `VLLM_CPU_OMP_THREADS_BIND=0-31` means there will be 32 OpenMP threads bound on 0-31 CPU cores. `VLLM_CPU_OMP_THREADS_BIND=0-31|32-63` means there will be 2 tensor parallel processes, 32 OpenMP threads of rank0 are bound on 0-31 CPU cores, and the OpenMP threads of rank1 are bound on 32-63 CPU cores. By setting to `auto`, the OpenMP threads of each rank are bound to the CPU cores in each NUMA node respectively. If set to `nobind`, the number of OpenMP threads is determined by the standard `OMP_NUM_THREADS` environment variable.
- `VLLM_CPU_NUM_OF_RESERVED_CPU`: specify the number of CPU cores which are not dedicated to the OpenMP threads for each rank. The variable only takes effect when VLLM_CPU_OMP_THREADS_BIND is set to `auto`. Default value is `None`. If the value is not set and use `auto` thread binding, no CPU will be reserved for `world_size == 1`, 1 CPU per rank will be reserved for `world_size > 1`.
- `VLLM_CPU_NUM_OF_RESERVED_CPU`: specify the total number of CPU cores which are not dedicated to OpenMP threads. The variable only takes effect when `VLLM_CPU_OMP_THREADS_BIND` is set to `auto`. By default, x86, ARM, and RISC-V reserve `local_world_size` CPUs in total, while PowerPC and S390X retain a one-CPU default. When KV transfer is enabled, one additional CPU is reserved per local rank. Reserved CPUs are distributed across the local ranks.
- `CPU_VISIBLE_MEMORY_NODES`: specify visible NUMA memory nodes for vLLM CPU workers, similar to ```CUDA_VISIBLE_DEVICES```. The variable only takes effect when VLLM_CPU_OMP_THREADS_BIND is set to `auto`. The variable provides more control for the auto thread-binding feature, such as masking nodes and changing nodes binding sequence.
- `VLLM_ZENTORCH_WEIGHT_PREPACK` (AMD Zen only): when `ZenCpuPlatform` is active, eagerly prepack linear weights into ZenDNN's blocked layout at model load time, eliminating per-inference layout conversion overhead. Default is `1` (enabled). See [AMD Zen optimizations](#amd-zen-optimizations).
@@ -253,7 +253,7 @@ See [AMD Zen optimizations](#amd-zen-optimizations) for detection rules, support
### How to decide `VLLM_CPU_OMP_THREADS_BIND`?
- Default `auto` thread-binding is recommended for most cases. Ideally, each OpenMP thread will be bound to a dedicated physical core respectively, threads of each rank will be bound to the same NUMA node respectively, and 1 CPU per rank will be reserved for other vLLM components when `world_size > 1`. If you have any performance problems or unexpected binding behaviours, please try to bind threads as following.
- Default `auto` thread-binding is recommended for most cases. Ideally, each OpenMP thread will be bound to a dedicated physical core respectively, threads of each rank will be bound to the same NUMA node respectively, and the default reserved CPUs will be kept for other vLLM components. On x86, ARM, and RISC-V, `local_world_size` CPUs are reserved in total, plus one additional CPU per local rank when KV transfer is enabled. If you have any performance problems or unexpected binding behaviours, please try to bind threads as following.
- On a hyper-threading enabled platform with 16 logical CPU cores / 8 physical CPU cores:
@@ -138,19 +138,6 @@ VLLM_TARGET_DEVICE=cpu VLLM_CPU_MOE_PREPACK=0 python setup.py bdist_wheel && \
The Docker image (`Dockerfile.s390x`) already includes TCMalloc and sets
`LD_PRELOAD` automatically.
!!! warning "Protobuf workaround for s390x"
The C++ protobuf extension crashes on s390x. After installation, set the
following environment variable and remove the C++ extensions:
```bash
export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
# Remove C++ protobuf extensions that crash on s390x
SITE_PKGS=$(python -c "import site; print(site.getsitepackages()[0])")
rm -rf "$SITE_PKGS/google/_upb/"*.so \
"$SITE_PKGS/google/protobuf/pyext/"*.so 2>/dev/null || true
```
--8<-- [end:build-wheel-from-source]
--8<-- [start:pre-built-images]
@@ -33,7 +33,7 @@ uv pip install "https://github.com/vllm-project/vllm/releases/download/v${VLLM_V
pip install "https://github.com/vllm-project/vllm/releases/download/v${VLLM_VERSION}/vllm-${VLLM_VERSION}+cpu-cp38-abi3-manylinux_2_34_x86_64.whl" --extra-index-url https://download.pytorch.org/whl/cpu
```
!!! warning "set `LD_PRELOAD`"
Before use vLLM CPU installed via wheels, make Intel OpenMP is added to `LD_PRELOAD`:
Before use vLLM CPU installed via wheels, make sure Intel OpenMP is added to `LD_PRELOAD`:
```bash
# manually find the path
sudo find / -iname *libiomp5.so
+1 -1
View File
@@ -2,7 +2,7 @@
This document lists the current committers of the vLLM project and the core areas they maintain.
Committers have write access to the vLLM repository and are responsible for reviewing and merging PRs.
You can also refer to the [CODEOWNERS](https://github.com/vllm-project/vllm/blob/main/.github/CODEOWNERS) file for concrete file-level ownership and reviewers. Both this documents and the CODEOWNERS file are living documents and they complement each other.
You can also refer to the [CODEOWNERS](https://github.com/vllm-project/vllm/blob/main/.github/CODEOWNERS) file for concrete file-level ownership and reviewers. Both these documents and the CODEOWNERS file are living documents and they complement each other.
## Active Committers
+18 -3
View File
@@ -6,7 +6,7 @@ This closes the loop for a token-in / token-out engine in disaggregated serving:
- **GPU less post processing**: Detokenization, reasoning parsing, and tool call parsing run on the same GPU less frontend that hosts `/render`
- **Parser parity**: The derenderer reuses vLLM's tool and reasoning parsers, so a disaggregated deployment produces the same `content`/`reasoning`/ `tool_calls` split as a standard `vllm serve` server
- **Non-streaming**: The endpoints expect a complete `GenerateResponse` with all token IDs present and perform one-shot parsing. Streaming derender would require a separate endpoint design and is not currently supported but is in the pipeline
- **Streaming and one-shot parsing**: Non-streaming calls send a complete `GenerateResponse` with all token IDs present and get one-shot parsing. Both endpoints also accept `stream: true`, taking one `GenerateStreamResponse` delta plus a client carried `stream_state` and returning `{chunk, stream_state}`. The chat endpoint's streaming path supports reasoning and tool call parsing while emitting the same `reasoning`/`content`/`tool_calls` deltas the generate streaming path would for the same token IDs
Both endpoints are hosted by the GPU less rendering server started with [`vllm launch render`](../../cli/launch/render.md), alongside the `/render`
endpoints.
@@ -21,7 +21,7 @@ endpoints.
└─────────────── request + prompt_tokens ──┘
```
The derender step needs more than the engine's `token_ids`. It also consumes the original `chat_request`/`completion_request` and `prompt_tokens` carried over from the render step (see [Request format](#request-format)) so the tool and reasoning parsers have the context they need.
The derender step needs more than the engine's `token_ids`. It also consumes the original `chat_request`/`completion_request` and `prompt_tokens` carried over from the render step (see [Request format](#request-format)) so the tool and reasoning parsers have the context they need. The chat streaming path additionally needs `prompt_token_ids` (see [Streaming cost](#streaming-cost)) when a parser is configured.
## API Reference
@@ -52,6 +52,21 @@ Each request wraps the engine's `GenerateResponse`(s) together with the caller m
Oversized payloads are rejected with a `400` before any `tokenizer.decode()` or parser runs.
## Streaming cost
Streaming derender threads a client carried `stream_state` across per-chunk calls instead of keeping session state on the server.
Plain detokenization (no parser configured) carries only a small, bounded incremental decode window in `stream_state` independent of generation length.
When a tool or reasoning parser is configured, parser internal state (buffered markup, reasoning/tool phase) can't be serialized. `stream_state` therefore instead carries the full `output_token_ids` seen so far plus `output_chunk_lens`, the token count of each chunk they arrived in. Each chunk rebuilds a fresh parser and replays that history through `parse_delta`, one call per original chunk, before processing the new tokens for real. For the parser path only, this means:
- **Transport**: `output_token_ids` and `output_chunk_lens` round-trip in full in both directions on every call. `output_chunk_lens` has one entry per chunk, which is one per token without speculative decoding. This means O(n) bytes per chunk, O(n²) bytes over a full generation. Bounded by `max_model_len`. `prompt_token_ids` is sent in full on every call too and it isn't trimmed as `output_token_ids` grows. This means that for most of a stream it dominates the per chunk payload. A 100k token prompt with 1k tokens of output means `prompt_token_ids` is ~99% of the request body on every chunk.
- **Compute**: replay is O(n) `parse_delta` calls per chunk (O(n²) per generation). `parse_delta` itself is O(n) for parsers that re-scan accumulated text (e.g. Hermes tool-call JSON, DeepSeek-R1 reasoning). The per-generation cost is O(n³) character work, not O(n²). This is a deliberately minimal first implementation with no caching layer.
- The parser path also requires `prompt_token_ids` so `parse_delta` can settle whether the prompt left reasoning open or not. Since parser state can't be carried across calls, it re-scans the full prompt once per chunk.
- Replay runs off the event loop on the renderer's executor (`renderer_num_workers`, default `1`). Size it for the expected number of concurrent parser configured streams.
`output_token_ids` and `prompt_token_ids` are both bounded by `max_model_len` but callers streaming long reasoning traces through a parser configured model should expect materially more state transport and CPU cost than the plain detokenization path.
## Example
The example below drives the full `render → generate → derender` round trip for a chat request against a GPU less render server (`/render`, `/derender`) and a token-in / token-out engine (`/inference/v1/generate`).
@@ -102,4 +117,4 @@ with httpx.Client(timeout=60.0) as client:
print(response["choices"][0]["message"]["content"])
```
Passing `chat_request` lets the derenderer run the configured tool and reasoning parsers. This means `response["choices"][0]["message"]` carries the same `content` / `reasoning` / `tool_calls` split a `vllm serve` server would produce. Omit `chat_request` for plain detokenization only.
Passing `chat_request` lets the derenderer run the configured tool and reasoning parsers. This means `response["choices"][0]["message"]` carries the same `content` / `reasoning` / `tool_calls` split a `vllm serve` server would produce. `chat_request` can only be omitted for a model with no tool or reasoning parser configured. A parser configured model rejects a missing `chat_request` with a 400 rather than silently falling back to plain detokenization.
+1 -1
View File
@@ -233,7 +233,7 @@ Two implementations ship with vLLM:
| `RayVLLMWeightSyncClient(handle)` | One or more `AsyncLLM`/`LLM` Ray actors. Accepts a list and fans each call out to every handle, blocking on all of them, so a multi-actor (e.g. multi-DP) deployment is driven as one unit |
| `HTTPVLLMWeightSyncClient(base_url, timeout=300)` | A vLLM server over the RLHF HTTP routes |
Custom weight sync clients can be implement like so:
Custom weight sync clients can be implemented like so:
```python
class MyFrameworkWeightSyncClient:
@@ -106,6 +106,22 @@ If you enable prefill instance (`--prefill-servers-urls` not disabled), you will
| `--decode-servers-urls` | Comma-separated list of decode endpoints. Non-stream and stream paths both round-robin over this list. |
| `--host`, `--port` | Bind address for the proxy itself (defaults: `0.0.0.0:8000`). |
The proxy batches images from the same user request assigned to the same encoder.
Set the proxy environment variable `ENCODER_MAX_BATCH_SIZE` to limit the number
of images per encoder subrequest. It defaults to `0` (unlimited); `1` sends each
image separately. Audio and video remain separate subrequests.
For encoders with a smaller `--limit-mm-per-prompt` image limit than P/PD, set
`ENCODER_MAX_BATCH_SIZE` no higher than the smallest encoder image limit.
For example, for encoders configured with `--limit-mm-per-prompt '{"image": 2}'`:
```bash
ENCODER_MAX_BATCH_SIZE=2 python disagg_epd_proxy.py \
--encode-servers-urls "http://e1:8001,http://e2:8002" \
--prefill-servers-urls disable \
--decode-servers-urls "http://pd1:8003"
```
Example usage:
For E + PD setup:
@@ -10,8 +10,8 @@ clusters:
For MM input we:
1. Extract *every* image/audio/video item.
2. Fire N concurrent requests to the encoder cluster
(one request per item, with **all text removed**).
2. Send concurrent encoder requests with all text removed,
grouping images assigned to the same encoder.
3. Wait for all of them to succeed.
4. Forward the *original* request to a decode server.
"""
@@ -99,6 +99,11 @@ def validate_ec_consumer_routing(
# Diagnostic switch: forward the original request to the decoder so the
# only difference from the rewrite path is the rewrite itself.
NO_REWRITE = False
# Maximum images per encoder subrequest; 0 leaves batches unlimited.
ENCODER_MAX_BATCH_SIZE = int(os.getenv("ENCODER_MAX_BATCH_SIZE", "0"))
if ENCODER_MAX_BATCH_SIZE < 0:
raise ValueError("ENCODER_MAX_BATCH_SIZE must be non-negative")
# Decode-side retries for a retryable internal error (`finish_reason="error"`,
# e.g. an encoder embedding the connector could not deliver). Re-issuing runs
# the encode again, which produces a fresh transfer.
@@ -222,7 +227,7 @@ async def fanout_encoder_primer(
req_id: str,
consumer_zmq: str | None = None,
) -> tuple[dict[int, dict], dict[str, Any]]:
"""1. Build one request *per MM item* with all text removed.
"""1. Group images by encoder, retaining per-item round-robin assignment.
2. Send them concurrently to the encode cluster.
3. Raise if any of them fails.
@@ -248,6 +253,7 @@ async def fanout_encoder_primer(
item_uuids: dict[int, str] = {}
item_transfer_ids: dict[int, str] = {}
item_meta: dict[int, dict] = {}
transfer_items: dict[int, dict[str, str]] = {}
ec_params: dict[str, Any] = {}
# Round-robin over encode servers to distribute load a bit. The cursor
@@ -259,29 +265,45 @@ async def fanout_encoder_primer(
e_urls, encoder_rr_idx, len(mm_items)
)
groups: list[tuple[str, list[int]]] = []
image_groups: dict[str, list[int]] = {}
for idx, (item, target_url) in enumerate(zip(mm_items, url_cycle)):
if item["type"] == "image_url":
indices = image_groups.get(target_url)
if indices is None or (
ENCODER_MAX_BATCH_SIZE and len(indices) >= ENCODER_MAX_BATCH_SIZE
):
indices = []
image_groups[target_url] = indices
groups.append((target_url, indices))
indices.append(idx)
else:
groups.append((target_url, [idx]))
for target_url, indices in groups:
# Derive a *child* request id: <parent>:<index>:<random-short>
child_req_id = f"{req_id}:{idx}:{uuid.uuid4().hex[:6]}"
child_req_id = f"{req_id}:{indices[0]}:{uuid.uuid4().hex[:6]}"
headers = {"x-request-id": child_req_id, "Content-Type": "application/json"}
# With --no-rewrite the decoder still receives the raw image and derives
# the cache key by hashing it, so the encoder must do the same -- passing
# a uuid here would make the two disagree and silently defeat the EC
# transfer, leaving the decoder to encode the image itself.
item_uuid = None if NO_REWRITE else content_uuid(item)
if item_uuid is not None:
item_uuids[idx] = item_uuid
transfer_id = uuid.uuid4().hex
item_transfer_ids[idx] = transfer_id
content = []
for idx in indices:
item = mm_items[idx]
item_uuid = None if NO_REWRITE else content_uuid(item)
if item_uuid is not None:
item_uuids[idx] = item_uuid
item_transfer_ids[idx] = uuid.uuid4().hex
content.append(item if item_uuid is None else {**item, "uuid": item_uuid})
encoder_req = {
"model": orig_request.get("model"),
"messages": [
{
"role": "user",
"content": [
item if item_uuid is None else {**item, "uuid": item_uuid}
],
"content": content,
},
],
# No max_tokens cap: the encoder instance never samples, it finishes
@@ -297,17 +319,13 @@ async def fanout_encoder_primer(
if key in orig_request:
encoder_req[key] = orig_request[key]
if consumer_zmq is not None:
# No mm_hash here on purpose. The encoder's own
# `mm_features[i].identifier` is derived from the uuid *and* the
# engine's media_io_kwargs / mm_processor_kwargs, so this proxy
# cannot know it before the encoder runs. Sending the bare uuid
# would fail the connector's hash match and make the producer
# invent its own transfer id, which the consumer could then never
# cancel. Omitting it lets the connector match by position, which
# is exact: one encoder request carries exactly one item.
# The engine may rehash UUIDs with processing options. Match by
# position instead; batches contain only images in input order.
encoder_req["ec_transfer_params"] = {
"consumer_zmq": consumer_zmq,
"ec_items": [{"transfer_id": transfer_id}],
"ec_items": [
{"transfer_id": item_transfer_ids[idx]} for idx in indices
],
}
tasks.append(
encode_session.post(
@@ -320,7 +338,8 @@ async def fanout_encoder_primer(
results = await asyncio.gather(*tasks, return_exceptions=True)
# Fail fast if any sub-request failed
for idx, r in enumerate(results):
for (_, indices), r in zip(groups, results):
idx = indices[0]
if isinstance(r, Exception):
logger.error(
"[%s] Encoder request #%d raised exception: %s",
@@ -357,18 +376,45 @@ async def fanout_encoder_primer(
logger.warning("[%s] Could not read encoder metadata #%d", req_id, idx)
params = {}
if params:
# One encoder request carries exactly one item, so there is a
# single reported entry. Do not key it by this proxy's uuid: when
# media_io_kwargs or mm_processor_kwargs are set the engine
# re-hashes the uuid together with them, so the encoder's own
# `mm_features[i].identifier` is a derived value this proxy cannot
# predict. Fall back to the sole entry, and carry the key the
# encoder actually used through as `ec_mm_hash`.
ec_mm_hash = item_uuids.get(idx)
reported = params.get(ec_mm_hash)
if reported is None and len(params) == 1:
((ec_mm_hash, reported),) = params.items()
if reported:
by_index = {}
for mm_hash, reported in params.items():
if len(indices) == 1:
break
for local_index in reported.get("item_indices", []):
if type(local_index) is not int or not 0 <= local_index < len(
indices
):
raise HTTPException(502, "Invalid encoder item index")
if local_index in by_index:
raise HTTPException(502, "Duplicate encoder item index")
by_index[local_index] = (mm_hash, reported)
for local_index, idx in enumerate(indices):
matched = by_index.get(local_index)
if matched is None:
# Compatibility with encoders without position metadata.
mm_hash = item_uuids.get(idx)
if mm_hash in params:
matched = (mm_hash, params[mm_hash])
elif len(indices) == 1 and len(params) == 1:
matched = next(iter(params.items()))
elif (
mm_items[idx]["type"] == "video_url"
and (orig_request.get("mm_processor_kwargs") or {}).get(
"use_audio_in_video"
)
and len(params) == 2
and consumer_zmq is None
and all(
entry.keys() <= {"metadata", "item_indices"}
for entry in params.values()
)
):
# Keep raw video for its audio/video features when no transfer
# handles need to be forwarded.
continue
else:
raise HTTPException(502, "Encoder metadata cannot be matched")
ec_mm_hash, reported = matched
metadata = reported.get("metadata") or {}
if metadata:
item_meta[idx] = {
@@ -384,12 +430,16 @@ async def fanout_encoder_primer(
# looks it up by mm_hash on the request, so carry it through.
ec_params[ec_mm_hash] = reported
if NO_REWRITE and consumer_zmq is not None:
ec_params.setdefault("ec_items", []).append(
{"mm_hash": ec_mm_hash, "transfer_id": item_transfer_ids[idx]}
)
transfer_items[idx] = {
"mm_hash": ec_mm_hash,
"transfer_id": item_transfer_ids[idx],
}
if transfer_items:
ec_params["ec_items"] = [transfer_items[idx] for idx in sorted(transfer_items)]
logger.info(
"[%s] All %d encoder requests completed successfully", req_id, len(mm_items)
"[%s] All %d encoder requests completed successfully", req_id, len(groups)
)
return item_meta, ec_params
+3 -2
View File
@@ -11,7 +11,8 @@ transformers >= 5.10.4
huggingface_hub >= 1.31.0 # Required for the `huggingface_hub.utils.httpx` re-export
tokenizers >= 0.21.1 # Required for fast incremental detokenization.
safetensors >= 0.6.2 # MXFP4/MXFP6 dtype support (F8_E8M0, F4) added in 0.6.0: https://github.com/huggingface/safetensors/pull/611
protobuf >= 5.29.6, !=6.30.*, !=6.31.*, !=6.32.*, !=6.33.0.*, !=6.33.1.*, !=6.33.2.*, !=6.33.3.*, !=6.33.4.* # Required by LlamaTokenizer, gRPC. CVE-2026-0994
protobuf >= 5.29.6, !=6.30.*, !=6.31.*, !=6.32.*, !=6.33.0.*, !=6.33.1.*, !=6.33.2.*, !=6.33.3.*, !=6.33.4.*; platform_machine != "s390x"
protobuf == 7.36.1; platform_machine == "s390x"
fastapi[standard] >= 0.133.0, < 0.137.0 # First version supporting Starlette 1.0; < 0.137.0 avoids route-tree change that breaks model-hosting-container-standards handler overrides.
starlette >= 1.0.1 # CVE-2026-48710: Host header injection in < 1.0.1
aiohttp >= 3.13.3
@@ -25,7 +26,7 @@ lm-format-enforcer == 0.11.3
llguidance >= 1.7.0, < 1.8.0; platform_machine == "x86_64" or platform_machine == "arm64" or platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "s390x"
outlines_core == 0.2.14
lark == 1.2.2
xgrammar >= 0.2.1, < 1.0.0; platform_machine == "x86_64" or platform_machine == "aarch64" or platform_machine == "arm64" or platform_machine == "s390x" or platform_machine == "ppc64le"
xgrammar == 0.2.7; platform_machine == "x86_64" or platform_machine == "aarch64" or platform_machine == "arm64" or platform_machine == "s390x" or platform_machine == "ppc64le"
typing_extensions >= 4.10
filelock >= 3.16.1 # need to contain https://github.com/tox-dev/filelock/pull/317
partial-json-parser # used for parsing partial JSON outputs
+2 -2
View File
@@ -43,7 +43,7 @@ anyio==4.14.1
# sse-starlette
# starlette
# watchfiles
apache-tvm-ffi==0.1.9
apache-tvm-ffi==0.1.14.post0
# via xgrammar
arctic-inference==0.1.1
# via -r requirements/test/cuda.in
@@ -1298,7 +1298,7 @@ word2number==1.1
# via lm-eval
wrapt==1.17.2
# via smart-open
xgrammar==0.2.3
xgrammar==0.2.7
# via -r requirements/test/../common.txt
xxhash==3.5.0
# via
+1 -1
View File
@@ -1409,7 +1409,7 @@ word2number==1.1
# via lm-eval
wrapt==1.17.2
# via smart-open
xgrammar==0.2.3
xgrammar==0.2.7
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
+1 -1
View File
@@ -1360,7 +1360,7 @@ word2number==1.1
# via lm-eval
wrapt==2.1.2
# via smart-open
xgrammar==0.2.1
xgrammar==0.2.7
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
+1 -1
View File
@@ -1197,7 +1197,7 @@ werkzeug==3.1.8
# via schemathesis
word2number==1.1
# via lm-eval
xgrammar==0.2.3
xgrammar==0.2.7
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
+3 -3
View File
@@ -5941,7 +5941,7 @@ dependencies = [
[[package]]
name = "vllm-proto"
version = "0.3.0"
version = "0.4.0"
dependencies = [
"prost",
"prost-types",
@@ -6596,9 +6596,9 @@ checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9"
[[package]]
name = "xgrammar-structural-tag"
version = "0.2.0+xgrammar.0.2.4.dd729e7"
version = "0.3.0+xgrammar.0.2.7.82505d0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4d24c842efc3c24e9756aa426d530cbdac0980e49af223cb384e276e981ca0a"
checksum = "3864005575c2ab76057f789898f59fb9fa1378d07c5ceabc24764e012d0e90bb"
dependencies = [
"auto_impl",
"serde",
+1 -1
View File
@@ -153,7 +153,7 @@ vllm-text = { path = "src/text" }
vllm-tokenizer = { path = "src/tokenizer" }
vllm-tracing = { path = "src/tracing" }
winnow = { version = "1.0.2", features = ["simd"] }
xgrammar-structural-tag = "0.2.0"
xgrammar-structural-tag = "0.3.0"
zeromq = { version = "0.6.0", default-features = false, features = [
"tokio-runtime",
"all-transport",
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "vllm-proto"
version = "0.3.0"
version = "0.4.0"
edition.workspace = true
license.workspace = true
description = "Protobuf messages and gRPC bindings for the vLLM Rust frontend"
+2
View File
@@ -160,6 +160,8 @@ message SequenceOutput {
// Only present in final output for this sequence
optional FinishInfo finish_info = 8;
// Only present in final output when sampling-mask capture is enabled.
repeated TokenIds sampling_mask = 10;
}
// Prompt info, returned in the first response
+2
View File
@@ -134,6 +134,7 @@ impl ChatBackend for HfChatBackend {
self.tokenizer.clone(),
options.tool_call_parser,
options.reasoning_parser,
options.tool_strict_level,
)?))
}
}
@@ -343,6 +344,7 @@ mod tests {
let error = match backend.new_chat_output_processor(
&mut request,
NewChatOutputProcessorOptions {
tool_strict_level: crate::ToolStrictLevel::Auto,
tool_call_parser: &ParserSelection::Explicit("json".to_string()),
reasoning_parser: &ParserSelection::Auto,
},
+2 -1
View File
@@ -13,7 +13,7 @@ use crate::multimodal::{MmLimitPerPrompt, MultimodalModelInfo};
use crate::output::DynChatOutputProcessor;
use crate::renderer::DynChatRenderer;
use crate::request::ChatRequest;
use crate::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection};
use crate::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection, ToolStrictLevel};
pub mod hf;
@@ -21,6 +21,7 @@ pub mod hf;
pub struct NewChatOutputProcessorOptions<'a> {
pub tool_call_parser: &'a ParserSelection,
pub reasoning_parser: &'a ParserSelection,
pub tool_strict_level: ToolStrictLevel,
}
/// Minimal prompt-processing backend needed by `vllm-chat`.
+18 -1
View File
@@ -30,7 +30,7 @@ pub use parser::reasoning::{
ReasoningDelta, ReasoningError, ReasoningParser, ReasoningParserFactory,
};
pub use parser::tool::{ToolParser, ToolParserError, ToolParserFactory};
pub use parser::{ParserSelection, validate_parser_overrides};
pub use parser::{ParserSelection, ToolStrictLevel, validate_parser_overrides};
pub use reasoning::EffortValue;
pub use renderer::hf::ChatTemplateContentFormatOption;
pub use renderer::{
@@ -74,6 +74,8 @@ pub struct ChatRequestProcessor {
tool_call_parser: ParserSelection,
/// Reasoning parser selection used when preparing generation requests.
reasoning_parser: ParserSelection,
/// Server-side floor for tool-call structural tags.
tool_strict_level: ToolStrictLevel,
}
impl ChatRequestProcessor {
@@ -85,6 +87,7 @@ impl ChatRequestProcessor {
model_dtype: Some(model_dtype),
tool_call_parser: ParserSelection::Auto,
reasoning_parser: ParserSelection::Auto,
tool_strict_level: ToolStrictLevel::Auto,
}
}
@@ -95,6 +98,7 @@ impl ChatRequestProcessor {
model_dtype: None,
tool_call_parser: ParserSelection::Auto,
reasoning_parser: ParserSelection::Auto,
tool_strict_level: ToolStrictLevel::Auto,
}
}
@@ -109,6 +113,12 @@ impl ChatRequestProcessor {
self
}
/// Configure the server-side floor for tool-call structural tags.
pub fn with_tool_strict_level(mut self, tool_strict_level: ToolStrictLevel) -> Self {
self.tool_strict_level = tool_strict_level;
self
}
async fn finalize_rendered_prompt(
&self,
request: &ChatRequest,
@@ -207,6 +217,7 @@ impl ChatRequestProcessor {
NewChatOutputProcessorOptions {
tool_call_parser: &self.tool_call_parser,
reasoning_parser: &self.reasoning_parser,
tool_strict_level: self.tool_strict_level,
},
)?;
let text_request = self.prepare_text_request(request).await?;
@@ -255,6 +266,12 @@ impl ChatLlm {
self
}
/// Set the server-side floor for tool-call structural tags.
pub fn with_tool_strict_level(mut self, tool_strict_level: ToolStrictLevel) -> Self {
self.processor.tool_strict_level = tool_strict_level;
self
}
/// Override the effective model dtype used for multimodal tensor encoding.
pub fn with_model_dtype(mut self, model_dtype: ModelDtype) -> Self {
self.processor.model_dtype = Some(model_dtype);
+12 -3
View File
@@ -18,10 +18,10 @@ use self::unified::unified_event_stream;
use super::structured::structured_chat_event_stream;
use crate::error::Result;
use crate::output::{ChatOutputProcessor, DynChatEventStream, DynDecodedTextEventStream};
use crate::parser::ParserSelection;
use crate::parser::reasoning::{ReasoningParser, ReasoningParserFactory};
use crate::parser::tool::{ToolParser, ToolParserFactory};
use crate::parser::unified::UnifiedParserFactory;
use crate::parser::{ParserSelection, ToolStrictLevel};
use crate::request::{ChatRequest, ChatTool};
use crate::{Error, Result as ChatResult};
@@ -49,6 +49,7 @@ impl DefaultChatOutputProcessor {
tokenizer: DynTokenizer,
tool_call_parser: &ParserSelection,
reasoning_parser: &ParserSelection,
tool_strict_level: ToolStrictLevel,
) -> ChatResult<Self> {
let parser = if let Some(parser) = Self::resolve_optional_unified_parser(
request.tools(),
@@ -73,7 +74,11 @@ impl DefaultChatOutputProcessor {
Box::new(CombinedParser::new(reasoning_parser, tool_parser)) as Box<dyn UnifiedParser>
};
apply_structural_tag_constraint(request, parser.structural_tag_builder())?;
apply_structural_tag_constraint(
request,
parser.structural_tag_builder(),
tool_strict_level,
)?;
if parser.preserve_special_tokens() {
request.decode_options.skip_special_tokens = false;
@@ -191,7 +196,7 @@ mod tests {
use vllm_tokenizer::test_utils::TestTokenizer;
use super::DefaultChatOutputProcessor;
use crate::parser::ParserSelection;
use crate::parser::{ParserSelection, ToolStrictLevel};
use crate::request::ChatRequest;
fn tokenizer() -> Arc<TestTokenizer> {
@@ -213,6 +218,7 @@ mod tests {
tokenizer(),
&selection,
&selection,
ToolStrictLevel::Auto,
)
.unwrap();
}
@@ -227,6 +233,7 @@ mod tests {
tokenizer(),
&ParserSelection::Auto,
&ParserSelection::Auto,
ToolStrictLevel::Auto,
)
.unwrap();
}
@@ -244,6 +251,7 @@ mod tests {
tokenizer(),
tool,
reasoning,
ToolStrictLevel::Auto,
)
.unwrap();
}
@@ -258,6 +266,7 @@ mod tests {
tokenizer(),
&ParserSelection::Auto,
&ParserSelection::Explicit("gemma4".to_string()),
ToolStrictLevel::Auto,
) {
Ok(_) => panic!("expected mixed Gemma4 parser selection to fail"),
Err(error) => error,
@@ -14,19 +14,21 @@ use xgrammar_structural_tag::{
build_structural_tag,
};
use crate::parser::ToolStrictLevel;
use crate::request::{ChatRequest, ChatToolChoice};
use crate::{Error, Result as ChatResult};
/// Apply structural tag constraints to the request based on the tool parser's structural tag
/// support and the request's tool choice.
/// support, the request's tool choice, and the server-side strictness floor.
pub(super) fn apply_structural_tag_constraint(
request: &mut ChatRequest,
builder: Option<&dyn StructuralTagBuilder>,
strict_level: ToolStrictLevel,
) -> ChatResult<()> {
let Some(builder) = builder else {
return Ok(());
};
let Some(tool_choice) = structural_tag_tool_choice(request) else {
let Some(tool_choice) = structural_tag_tool_choice(request, strict_level) else {
return Ok(());
};
@@ -34,11 +36,14 @@ pub(super) fn apply_structural_tag_constraint(
.tools()
.iter()
.map(|tool| {
// A tool without an explicit `strict` is non-strict: its call envelope is
// constrained, but its arguments stay free unless the server level is `parameter`.
let strict = strict_level >= ToolStrictLevel::Parameter || tool.strict == Some(true);
ToolParam::Function(FunctionToolParam::new(FunctionDefinition {
name: tool.name.clone(),
description: tool.description.clone(),
parameters: Some(tool.parameters.clone()),
strict: tool.strict,
strict: Some(strict),
}))
})
.collect::<Vec<_>>();
@@ -66,17 +71,21 @@ pub(super) fn apply_structural_tag_constraint(
/// Resolve the tool choice used for [`xgrammar_structural_tag`] based on the request.
///
/// Returns `None` if no structural tag constraints should be applied.
fn structural_tag_tool_choice(request: &ChatRequest) -> Option<StructuralTagToolChoice> {
fn structural_tag_tool_choice(
request: &ChatRequest,
strict_level: ToolStrictLevel,
) -> Option<StructuralTagToolChoice> {
if request.tools().is_empty() {
return None;
}
match request.tool_choice() {
// For `Auto`, only apply the structural tag if there's at least one strict tool.
ChatToolChoice::Auto if request.tools().iter().any(|tool| tool.strict == Some(true)) => {
Some(StructuralTagToolChoice::auto())
}
ChatToolChoice::Auto | ChatToolChoice::None => None,
ChatToolChoice::None => None,
// For `Auto`, apply the structural tag only when the server raises the floor or at
// least one tool opts in with `strict: true`.
ChatToolChoice::Auto => (strict_level >= ToolStrictLevel::Function
|| request.tools().iter().any(|tool| tool.strict == Some(true)))
.then(StructuralTagToolChoice::auto),
ChatToolChoice::Required => Some(StructuralTagToolChoice::required()),
ChatToolChoice::Function { name } => Some(StructuralTagToolChoice::function(name.clone())),
@@ -148,8 +157,12 @@ mod tests {
let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", Some(true))]);
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
.expect("structural tag should build");
apply_structural_tag_constraint(
&mut request,
parser.structural_tag_builder(),
ToolStrictLevel::Auto,
)
.expect("structural tag should build");
let tag = structural_tag_value(&request);
assert_eq!(tag["type"], "structural_tag");
@@ -172,8 +185,12 @@ mod tests {
};
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
.expect("structural tag should build");
apply_structural_tag_constraint(
&mut request,
parser.structural_tag_builder(),
ToolStrictLevel::Auto,
)
.expect("structural tag should build");
let tag = structural_tag_value(&request);
assert!(tag.to_string().contains("lookup"));
@@ -199,8 +216,12 @@ mod tests {
};
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
.expect("structural tag should build");
apply_structural_tag_constraint(
&mut request,
parser.structural_tag_builder(),
ToolStrictLevel::Auto,
)
.expect("structural tag should build");
let tag = structural_tag_value(&request).to_string();
assert!(tag.contains("search"));
@@ -212,8 +233,12 @@ mod tests {
let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", None)]);
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
.expect("structural tag decision should succeed");
apply_structural_tag_constraint(
&mut request,
parser.structural_tag_builder(),
ToolStrictLevel::Auto,
)
.expect("structural tag decision should succeed");
assert!(request.sampling_params.structured_outputs.is_none());
}
@@ -227,8 +252,12 @@ mod tests {
});
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
.expect("structural tag should build");
apply_structural_tag_constraint(
&mut request,
parser.structural_tag_builder(),
ToolStrictLevel::Auto,
)
.expect("structural tag should build");
let params = structured_outputs(&request);
assert!(params.constraint.is_structural_tag());
@@ -242,8 +271,12 @@ mod tests {
let mut request = request(ChatToolChoice::Required, vec![chat_tool("search", None)]);
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
.expect("structural tag should build");
apply_structural_tag_constraint(
&mut request,
parser.structural_tag_builder(),
ToolStrictLevel::Auto,
)
.expect("structural tag should build");
let tag = structural_tag_value(&request);
assert_eq!(tag["type"], "structural_tag");
@@ -259,8 +292,12 @@ mod tests {
});
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
.expect("structural tag should build");
apply_structural_tag_constraint(
&mut request,
parser.structural_tag_builder(),
ToolStrictLevel::Auto,
)
.expect("structural tag should build");
let params = structured_outputs(&request);
assert!(params.constraint.is_structural_tag());
@@ -279,8 +316,12 @@ mod tests {
);
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
.expect("structural tag should build");
apply_structural_tag_constraint(
&mut request,
parser.structural_tag_builder(),
ToolStrictLevel::Auto,
)
.expect("structural tag should build");
let tag = structural_tag_value(&request).to_string();
assert!(tag.contains("lookup"));
@@ -292,8 +333,12 @@ mod tests {
let mut request = request(ChatToolChoice::None, vec![chat_tool("search", Some(true))]);
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
.expect("structural tag decision should succeed");
apply_structural_tag_constraint(
&mut request,
parser.structural_tag_builder(),
ToolStrictLevel::Auto,
)
.expect("structural tag decision should succeed");
assert!(request.sampling_params.structured_outputs.is_none());
}
@@ -307,10 +352,127 @@ mod tests {
});
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
.expect("structural tag decision should succeed");
apply_structural_tag_constraint(
&mut request,
parser.structural_tag_builder(),
ToolStrictLevel::Auto,
)
.expect("structural tag decision should succeed");
let params = structured_outputs(&request);
assert!(params.constraint.is_json_object());
}
fn build(tool_choice: ChatToolChoice, tools: Vec<Tool>, level: ToolStrictLevel) -> ChatRequest {
let mut request = request(tool_choice, tools);
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder(), level)
.expect("structural tag decision should succeed");
request
}
fn tag_string(request: &ChatRequest) -> String {
structural_tag_value(request).to_string()
}
#[test]
fn function_level_constrains_auto_without_strict_tools() {
let request = build(
ChatToolChoice::Auto,
vec![chat_tool("search", None)],
ToolStrictLevel::Function,
);
let tag = structural_tag_value(&request);
assert_eq!(tag["type"], "structural_tag");
assert!(tag.to_string().contains("search"));
}
#[test]
fn absent_strict_is_non_strict() {
let unset = build(
ChatToolChoice::Required,
vec![chat_tool("search", None)],
ToolStrictLevel::Auto,
);
let non_strict = build(
ChatToolChoice::Required,
vec![chat_tool("search", Some(false))],
ToolStrictLevel::Auto,
);
let strict = build(
ChatToolChoice::Required,
vec![chat_tool("search", Some(true))],
ToolStrictLevel::Auto,
);
assert_eq!(tag_string(&unset), tag_string(&non_strict));
assert_ne!(tag_string(&unset), tag_string(&strict));
}
#[test]
fn strict_tool_does_not_pin_its_neighbour() {
let mixed = build(
ChatToolChoice::Auto,
vec![chat_tool("weather", Some(true)), chat_tool("search", None)],
ToolStrictLevel::Auto,
);
let explicit = build(
ChatToolChoice::Auto,
vec![
chat_tool("weather", Some(true)),
chat_tool("search", Some(false)),
],
ToolStrictLevel::Auto,
);
assert_eq!(tag_string(&mixed), tag_string(&explicit));
}
#[test]
fn parameter_level_pins_argument_schemas() {
let pinned = build(
ChatToolChoice::Auto,
vec![chat_tool("search", None)],
ToolStrictLevel::Parameter,
);
let strict = build(
ChatToolChoice::Auto,
vec![chat_tool("search", Some(true))],
ToolStrictLevel::Auto,
);
let envelope_only = build(
ChatToolChoice::Auto,
vec![chat_tool("search", None)],
ToolStrictLevel::Function,
);
assert_eq!(tag_string(&pinned), tag_string(&strict));
assert_ne!(tag_string(&pinned), tag_string(&envelope_only));
}
#[test]
fn tool_strict_level_parses_case_insensitively() {
assert_eq!("AUTO".parse::<ToolStrictLevel>(), Ok(ToolStrictLevel::Auto));
assert_eq!(
"PARAMETER".parse::<ToolStrictLevel>(),
Ok(ToolStrictLevel::Parameter)
);
assert_eq!(
"function".parse::<ToolStrictLevel>(),
Ok(ToolStrictLevel::Function)
);
assert_eq!(ToolStrictLevel::default(), ToolStrictLevel::Auto);
assert_eq!(
serde_json::to_value(ToolStrictLevel::Auto).unwrap(),
json!("auto")
);
assert_eq!(
serde_json::from_value::<ToolStrictLevel>(json!("auto")).unwrap(),
ToolStrictLevel::Auto
);
assert!("off".parse::<ToolStrictLevel>().is_err());
assert!("strict".parse::<ToolStrictLevel>().is_err());
}
}
@@ -438,6 +438,7 @@ mod tests {
finish_reason: crate::FinishReason::Stop(None),
kv_transfer_params: None,
ec_transfer_params: None,
sampling_mask: None,
})),
}
}
@@ -58,6 +58,7 @@ fn finished() -> Finished {
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
ec_transfer_params: None,
sampling_mask: None,
}
}
+60
View File
@@ -57,6 +57,66 @@ impl ParserSelection {
}
}
/// Server-side floor for tool-call structural tags (`--tool-strict-level`).
#[derive(
Debug,
Clone,
Copy,
PartialEq,
Eq,
PartialOrd,
Ord,
Default,
DeserializeFromStr,
SerializeDisplay,
)]
pub enum ToolStrictLevel {
/// Derive tool constraints from the request's tool choice and per-tool strictness.
#[default]
Auto,
/// Constrain the tool-call envelope for every request with tools.
Function,
/// Additionally pin argument schemas, as if every tool were `strict: true`.
Parameter,
}
impl ToolStrictLevel {
pub const AUTO_LITERAL: &str = "auto";
pub const FUNCTION_LITERAL: &str = "function";
pub const PARAMETER_LITERAL: &str = "parameter";
}
impl FromStr for ToolStrictLevel {
type Err = String;
fn from_str(value: &str) -> Result<Self, Self::Err> {
if value.eq_ignore_ascii_case(Self::AUTO_LITERAL) {
Ok(Self::Auto)
} else if value.eq_ignore_ascii_case(Self::FUNCTION_LITERAL) {
Ok(Self::Function)
} else if value.eq_ignore_ascii_case(Self::PARAMETER_LITERAL) {
Ok(Self::Parameter)
} else {
Err(format!(
"unknown tool strict level {value:?}; expected one of {}, {}, {}",
Self::AUTO_LITERAL,
Self::FUNCTION_LITERAL,
Self::PARAMETER_LITERAL
))
}
}
}
impl fmt::Display for ToolStrictLevel {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(match self {
Self::Auto => Self::AUTO_LITERAL,
Self::Function => Self::FUNCTION_LITERAL,
Self::Parameter => Self::PARAMETER_LITERAL,
})
}
}
/// Validate explicit parser override names without starting request processing.
pub fn validate_parser_overrides(
tool_call_parser: &ParserSelection,
+1
View File
@@ -227,6 +227,7 @@ impl ChatBackend for FakeChatBackend {
self.tokenizer(),
options.tool_call_parser,
options.reasoning_parser,
options.tool_strict_level,
)?))
}
}
+3
View File
@@ -717,6 +717,7 @@ async fn parse_completion(
let processor = backends.chat_backend.new_chat_output_processor(
&mut request,
NewChatOutputProcessorOptions {
tool_strict_level: vllm_chat::ToolStrictLevel::Auto,
tool_call_parser: &case.tool_call_parser,
reasoning_parser: &case.reasoning_parser,
},
@@ -805,6 +806,7 @@ fn decoded_completion_stream(
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
ec_transfer_params: None,
sampling_mask: None,
})),
}
});
@@ -816,6 +818,7 @@ fn decoded_completion_stream(
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
ec_transfer_params: None,
sampling_mask: None,
});
events.push(DecodedTextEvent::TextDelta {
decoded: vllm_text::DecodedText::unattributed(chunk.delta),
+16
View File
@@ -24,6 +24,7 @@ use serde_with::{DefaultOnNull, OneOrMany, serde_as};
use thiserror_ext::AsReport as _;
use uuid::Uuid;
use vllm_chat::GenerationConfigMode;
use vllm_chat::ToolStrictLevel;
use vllm_chat::multimodal::MmLimitPerPrompt;
use vllm_engine_core_client::TransportMode;
use vllm_managed_engine::ManagedEngineConfig;
@@ -134,6 +135,10 @@ pub struct RenderArgs {
/// `none` to disable parsing.
#[arg(long, default_value_t)]
reasoning_parser: ParserSelection,
/// Server-side floor for structural-tag based tool calling: `auto`,
/// `function`, or `parameter`.
#[arg(long, default_value_t)]
tool_strict_level: ToolStrictLevel,
/// Select the native chat renderer implementation.
#[arg(long = "tokenizer-mode", default_value_t)]
renderer: RendererSelection,
@@ -171,6 +176,7 @@ impl RenderArgs {
port: self.port,
tool_call_parser: self.tool_call_parser,
reasoning_parser: self.reasoning_parser,
tool_strict_level: self.tool_strict_level,
renderer: self.renderer,
chat_template: self.chat_template,
default_chat_template_kwargs: self.default_chat_template_kwargs.unwrap_or_default(),
@@ -241,6 +247,14 @@ pub struct SharedRuntimeArgs {
#[arg(long, default_value_t)]
#[serde(default = "default_py_bootstrap_parser_selection")]
pub reasoning_parser: ParserSelection,
/// Server-side floor for structural-tag based tool calling, applied on
/// top of the per-tool `strict` field: `auto` follows the request's
/// tool choice and per-tool strictness,
/// `function` constrains the tool-call envelope for every request with
/// tools, `parameter` additionally pins argument schemas.
#[arg(long, default_value_t)]
#[serde(default)]
pub tool_strict_level: ToolStrictLevel,
/// Select the chat renderer implementation.
#[arg(long = "tokenizer-mode", default_value_t)]
#[serde(default, rename = "tokenizer_mode")]
@@ -517,6 +531,7 @@ impl SharedRuntimeArgs {
listener_mode: HttpListenerMode::InheritedFd { fd: listen_fd },
tool_call_parser: self.tool_call_parser,
reasoning_parser: self.reasoning_parser,
tool_strict_level: self.tool_strict_level,
renderer: self.renderer,
language_model_only: self.language_model_only,
chat_template: self.chat_template,
@@ -574,6 +589,7 @@ impl SharedRuntimeArgs {
listener_mode,
tool_call_parser: self.tool_call_parser,
reasoning_parser: self.reasoning_parser,
tool_strict_level: self.tool_strict_level,
renderer: self.renderer,
language_model_only: self.language_model_only,
chat_template: self.chat_template,
+6
View File
@@ -184,6 +184,7 @@ fn serve_args_forward_python_flags_with_separator() {
engine_ready_timeout_secs: 600,
tool_call_parser: Auto,
reasoning_parser: Auto,
tool_strict_level: Auto,
renderer: Auto,
language_model_only: false,
max_logprobs: None,
@@ -1023,6 +1024,7 @@ fn frontend_args_accept_json() {
engine_ready_timeout_secs: 600,
tool_call_parser: None,
reasoning_parser: None,
tool_strict_level: Auto,
renderer: Auto,
language_model_only: false,
max_logprobs: None,
@@ -1693,6 +1695,7 @@ fn serve_args_accept_handshake_aliases() {
engine_ready_timeout_secs: 600,
tool_call_parser: Auto,
reasoning_parser: Auto,
tool_strict_level: Auto,
renderer: Auto,
language_model_only: false,
max_logprobs: None,
@@ -1850,6 +1853,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
},
tool_call_parser: Auto,
reasoning_parser: Auto,
tool_strict_level: Auto,
renderer: Auto,
language_model_only: false,
chat_template: None,
@@ -1942,6 +1946,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
},
tool_call_parser: Auto,
reasoning_parser: Auto,
tool_strict_level: Auto,
renderer: Auto,
language_model_only: false,
chat_template: None,
@@ -2055,6 +2060,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
},
tool_call_parser: None,
reasoning_parser: None,
tool_strict_level: Auto,
renderer: Auto,
language_model_only: false,
chat_template: None,
+13 -4
View File
@@ -911,13 +911,22 @@ impl EngineCoreClient {
Ok(())
}
/// Wake the engine from sleep, optionally limiting the wake-up to specific
/// tags.
pub async fn wake_up(&self, tags: Option<Vec<String>>) -> Result<()> {
self.call_utility::<(), _>("wake_up", (tags,)).await?;
/// Release KV cache memory while keeping model weights resident.
pub async fn release_kv_cache_memory(&self) -> Result<()> {
self.call_utility::<(), _>("release_kv_cache_memory", ()).await?;
Ok(())
}
/// Wake the engine from sleep, optionally limiting the wake-up to specific
/// tags, and return whether every engine is fully awake.
pub async fn wake_up(&self, tags: Option<Vec<String>>) -> Result<bool> {
Ok(self
.call_utility::<bool, _>("wake_up", (tags,))
.await?
.into_iter()
.all(|fully_awake| fully_awake))
}
/// Pause the scheduler so generation can be halted
pub async fn pause_scheduler(&self, mode: PauseMode, clear_cache: bool) -> Result<()> {
self.call_utility::<(), _>("pause_scheduler", (mode, clear_cache)).await?;
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
mod array;
pub(crate) mod array;
#[cfg(test)]
mod tests;
mod wire;
@@ -50,7 +50,7 @@ pub(super) fn decode_array2_u32(
})
}
pub(super) fn decode_array1_u32(
pub(crate) fn decode_array1_u32(
value: WireNdArray,
field: &str,
frames: &[Bytes],
@@ -30,6 +30,7 @@ pub mod multimodal;
pub mod output;
pub mod request;
pub mod sampling;
pub mod sampling_mask;
mod serde_utils;
pub mod stats;
pub mod structured_outputs;
@@ -15,6 +15,7 @@ use super::serde_utils::AllowTrailingFields;
use super::utility::UtilityOutput;
use crate::error::{Error, Result, ext_value_decode};
use crate::protocol::logprobs::MaybeWireLogprobs;
use crate::protocol::sampling_mask::MaybeWireSamplingMask;
use crate::protocol::stats::{PrefillStats, SchedulerStats};
use crate::protocol::{OpaqueValue, decode_msgpack};
@@ -127,7 +128,7 @@ pub struct EngineCoreOutput {
#[serde(default)]
pub mm_cache_miss_hashes: Option<Vec<String>>,
#[serde(default)]
pub new_sampling_mask: Option<OpaqueValue>,
pub new_sampling_mask: Option<MaybeWireSamplingMask>,
/// Per-request speculative-decoding acceptance metrics, set on the final
/// output when `--per-request-spec-decode-metrics` is enabled. Opaque here;
/// the Rust frontend does not yet surface it in responses.
@@ -150,6 +151,9 @@ impl EngineCoreOutput {
self.new_prompt_logprobs_tensors = (self.new_prompt_logprobs_tensors.take())
.map(|value| value.resolve(frames, "new_prompt_logprobs_tensors"))
.transpose()?;
self.new_sampling_mask = (self.new_sampling_mask.take())
.map(|value| value.resolve(frames, "new_sampling_mask"))
.transpose()?;
Ok(())
}
}
@@ -0,0 +1,237 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::ops::{Deref, DerefMut};
use bytes::Bytes;
use enum_as_inner::EnumAsInner;
use serde::{Deserialize, Deserializer, Serialize};
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
use crate::error::{Error, Result, bail_ext_value_decode};
use crate::protocol::logprobs::array::decode_array1_u32;
use crate::protocol::tensor::WireNdArray;
/// Decoded sampling-mask support sets, one row per generated token position.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct SamplingMask {
pub rows: Vec<Vec<u32>>,
}
/// Python `SamplingMaskLists` tuple before ndarray raw views are resolved.
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple)]
pub struct WireSamplingMask {
token_ids: WireNdArray,
#[serde(default)]
offsets: Option<WireNdArray>,
#[serde(default)]
cu_num_generated_tokens: Option<Vec<usize>>,
}
/// Sampling-mask field while it transitions from Python wire data to rows.
#[derive(Debug, Clone, PartialEq, EnumAsInner)]
pub enum MaybeWireSamplingMask {
Wire(Box<WireSamplingMask>),
Direct(SamplingMask),
}
impl Deref for MaybeWireSamplingMask {
type Target = SamplingMask;
fn deref(&self) -> &Self::Target {
match self {
Self::Wire(_) => panic!("sampling mask is still in wire format"),
Self::Direct(value) => value,
}
}
}
impl DerefMut for MaybeWireSamplingMask {
fn deref_mut(&mut self) -> &mut Self::Target {
match self {
Self::Wire(_) => panic!("sampling mask is still in wire format"),
Self::Direct(value) => value,
}
}
}
impl<'de> Deserialize<'de> for MaybeWireSamplingMask {
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
where
D: Deserializer<'de>,
{
WireSamplingMask::deserialize(deserializer).map(|value| Self::Wire(Box::new(value)))
}
}
impl Serialize for MaybeWireSamplingMask {
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
match self {
Self::Wire(value) => value.serialize(serializer),
Self::Direct(value) => WireSamplingMask::from_direct(value)
.map_err(serde::ser::Error::custom)?
.serialize(serializer),
}
}
}
impl MaybeWireSamplingMask {
pub(super) fn resolve(self, frames: &[Bytes], field_prefix: &str) -> Result<Self> {
match self {
Self::Direct(value) => Ok(Self::Direct(value)),
Self::Wire(value) => value.resolve(frames, field_prefix).map(Self::Direct),
}
}
}
impl WireSamplingMask {
fn from_direct(value: &SamplingMask) -> std::result::Result<Self, String> {
let [row] = value.rows.as_slice() else {
return Err(format!(
"per-request engine-core sampling mask must contain exactly one row, got {}",
value.rows.len()
));
};
let token_ids = row.iter().map(|&token_id| i64::from(token_id)).collect::<Vec<_>>();
Ok(Self {
token_ids: WireNdArray::from_i64(vec![token_ids.len()], token_ids)?,
offsets: None,
cu_num_generated_tokens: None,
})
}
fn resolve(self, frames: &[Bytes], field_prefix: &str) -> Result<SamplingMask> {
if let Some(indices) = self.cu_num_generated_tokens {
bail_ext_value_decode!(
"{field_prefix}.cu_num_generated_tokens: expected None for per-request engine-core sampling-mask payload, got {indices:?}"
);
}
let token_ids =
decode_array1_u32(self.token_ids, &format!("{field_prefix}.token_ids"), frames)?;
let Some(offsets) = self.offsets else {
return Ok(SamplingMask {
rows: vec![token_ids],
});
};
let offsets = decode_array1_u32(offsets, &format!("{field_prefix}.offsets"), frames)?;
if offsets.first().copied() != Some(0) {
bail_ext_value_decode!("{field_prefix}.offsets: first offset must be zero");
}
if offsets.windows(2).any(|pair| pair[0] > pair[1]) {
bail_ext_value_decode!("{field_prefix}.offsets: offsets must be nondecreasing");
}
if offsets.last().copied().map(|value| value as usize) != Some(token_ids.len()) {
bail_ext_value_decode!(
"{field_prefix}.offsets: final offset {:?} does not match token_ids length {}",
offsets.last(),
token_ids.len()
);
}
let rows = offsets
.windows(2)
.map(|pair| token_ids[pair[0] as usize..pair[1] as usize].to_vec())
.collect();
Ok(SamplingMask { rows })
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::protocol::dtype::{NumpyDtype, TensorDtype};
use crate::protocol::output::{
EngineCoreOutput, RequestBatchOutputs, decode_engine_core_outputs,
};
use crate::protocol::tensor::WireArrayData;
use crate::protocol::{decode_msgpack, encode_msgpack};
#[test]
fn decodes_single_position_without_offsets() {
let wire = WireSamplingMask {
token_ids: WireNdArray::from_i64(vec![3], vec![2, 12, 16]).unwrap(),
offsets: None,
cu_num_generated_tokens: None,
};
let encoded = encode_msgpack(&wire).unwrap();
let decoded: MaybeWireSamplingMask = decode_msgpack(&encoded).unwrap();
let decoded = decoded.resolve(&[], "new_sampling_mask").unwrap();
assert_eq!(decoded.into_direct().unwrap().rows, vec![vec![2, 12, 16]]);
}
#[test]
fn decodes_csr_rows_and_rejects_bad_terminal_offset() {
let wire = WireSamplingMask {
token_ids: WireNdArray::from_i64(vec![4], vec![2, 12, 16, 18]).unwrap(),
offsets: Some(WireNdArray::from_i64(vec![3], vec![0, 2, 4]).unwrap()),
cu_num_generated_tokens: None,
};
let decoded = MaybeWireSamplingMask::Wire(Box::new(wire))
.resolve(&[], "new_sampling_mask")
.unwrap();
assert_eq!(
decoded.into_direct().unwrap().rows,
vec![vec![2, 12], vec![16, 18]]
);
let malformed = WireSamplingMask {
token_ids: WireNdArray::from_i64(vec![2], vec![2, 12]).unwrap(),
offsets: Some(WireNdArray::from_i64(vec![2], vec![0, 1]).unwrap()),
cu_num_generated_tokens: None,
};
let error = MaybeWireSamplingMask::Wire(Box::new(malformed))
.resolve(&[], "new_sampling_mask")
.unwrap_err();
assert!(error.to_string().contains("final offset"));
}
#[test]
fn decodes_sampling_mask_from_multipart_aux_frames() {
let output = EngineCoreOutput {
request_id: "req-multipart-mask".to_string(),
new_token_ids: vec![16, 18],
new_sampling_mask: Some(MaybeWireSamplingMask::Wire(Box::new(WireSamplingMask {
token_ids: WireNdArray {
dtype: NumpyDtype::little(TensorDtype::I32),
shape: vec![4],
data: WireArrayData::AuxIndex(1),
},
offsets: Some(WireNdArray {
dtype: NumpyDtype::little(TensorDtype::I64),
shape: vec![3],
data: WireArrayData::AuxIndex(2),
}),
cu_num_generated_tokens: None,
}))),
..Default::default()
};
let primary = encode_msgpack(&crate::protocol::output::EngineCoreOutputs::from(
RequestBatchOutputs {
outputs: vec![output],
..Default::default()
},
))
.unwrap();
let token_ids =
[2_i32, 12, 16, 18].into_iter().flat_map(i32::to_le_bytes).collect::<Vec<_>>();
let offsets = [0_i64, 2, 4].into_iter().flat_map(i64::to_le_bytes).collect::<Vec<_>>();
let decoded = decode_engine_core_outputs(&[
Bytes::from(primary),
Bytes::from(token_ids),
Bytes::from(offsets),
])
.unwrap();
let mask = decoded.as_request_batch().unwrap().outputs[0]
.new_sampling_mask
.as_ref()
.unwrap();
assert_eq!(mask.rows, vec![vec![2, 12], vec![16, 18]]);
}
}

Some files were not shown because too many files have changed in this diff Show More