Merge remote-tracking branch 'origin/main' into sync/lesj/qwen4exp-bugfixes

Signed-off-by: lesj0610 <lesj0610@users.noreply.github.com>
This commit is contained in:
lesj0610
2026-09-16 20:33:18 +09:00
117 changed files with 6209 additions and 995 deletions
+63 -15
View File
@@ -540,6 +540,26 @@ prepare_ci_build_context() {
echo "Using canonical CI Docker context: ${ROCM_BUILD_CONTEXT_ROOT}"
}
configure_custom_rocm_stages() {
using_custom_rocm_dockerfiles || return 0
local context_root="${ROCM_BUILD_CONTEXT_ROOT:-.}"
local dockerfile="${context_root}/${CI_BASE_DOCKERFILE}"
local stages=""
(cd "${context_root}" && \
validate_rocm_dockerfile "${CI_BASE_DOCKERFILE}" \
ci_base test export_vllm export_test_smoke csrc-build rust-build) \
|| return $?
# Hash all stages for custom layouts, including additional helper stages.
stages=$(rocm_dockerfile_stages "${dockerfile}" | paste -sd ' ')
CI_BASE_DOCKERFILE_STAGES="${CI_BASE_DOCKERFILE_STAGES:-${stages}}"
ROCM_CSRC_DOCKERFILE_STAGES="${ROCM_CSRC_DOCKERFILE_STAGES:-${stages}}"
ROCM_RUST_DOCKERFILE_STAGES="${ROCM_RUST_DOCKERFILE_STAGES:-${stages}}"
if [[ " ${stages} " != *" build_rocshmem "* ]]; then
unset ROCSHMEM_BRANCH ROCSHMEM_CACHE_KEY DEEPEP_CACHE_KEY
fi
}
compose_dependency_cache_key() {
local prefix="$1"
local material="$2"
@@ -569,11 +589,11 @@ hash_dockerfile_stages() {
}
emit = 0
}
$1 == "FROM" {
toupper($1) == "FROM" {
stage = ""
for (idx = 1; idx <= NF; idx++) {
if (tolower($idx) == "as" && idx < NF) {
stage = $(idx + 1)
stage = tolower($(idx + 1))
}
}
emit = (stage in wanted)
@@ -613,11 +633,11 @@ discover_dockerfile_stage_args() {
}
{
line = $0
if ($1 == "FROM") {
if (toupper($1) == "FROM") {
stage = ""
for (idx = 1; idx <= NF; idx++) {
if (tolower($idx) == "as" && idx < NF) {
stage = $(idx + 1)
stage = tolower($(idx + 1))
}
}
emit = (stage in wanted)
@@ -630,7 +650,7 @@ discover_dockerfile_stage_args() {
for (idx = 1; idx <= line_count; idx++) {
line = lines[idx]
arg_name = line
sub(/^[[:space:]]*ARG[[:space:]]+/, "", arg_name)
sub(/^[[:space:]]*[Aa][Rr][Gg][[:space:]]+/, "", arg_name)
if (arg_name != line) {
sub(/[=[:space:]].*/, "", arg_name)
if (arg_name ~ /^[A-Za-z_][A-Za-z0-9_]*$/) {
@@ -716,7 +736,7 @@ extract_dockerfile_arg_default() {
if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" && "${dockerfile}" != /* ]]; then
physical_dockerfile="${ROCM_BUILD_CONTEXT_ROOT}/${dockerfile}"
fi
sed -n -E "s/^[[:space:]]*ARG[[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
sed -n -E "s/^[[:space:]]*[Aa][Rr][Gg][[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
"${physical_dockerfile}" | head -1
}
@@ -1000,7 +1020,21 @@ create_and_bootstrap_builder() {
}
init_config() {
# shellcheck source=.buildkite/scripts/rocm/build-config.sh
source "$(dirname "${BASH_SOURCE[0]}")/rocm/build-config.sh"
configure_rocm_build
TARGET="${1:-test-ci}"
if using_custom_rocm_dockerfiles; then
if [[ -z "${BASE_IMAGE:-}" ]]; then
echo "Custom ROCm builds require BASE_IMAGE from the selected base handoff" >&2
return 1
fi
if ! is_ci_base_target && [[ -z "${CI_BASE_IMAGE:-}" ]]; then
echo "Custom ROCm builds require CI_BASE_IMAGE from the selected ci_base handoff" >&2
return 1
fi
fi
BAKE_TARGETS=("${TARGET}")
DEPENDENCY_CACHE_TARGETS=()
CI_HCL_SOURCE="${CI_HCL_SOURCE:-${CI_HCL_FILE:-${DEFAULT_CI_HCL_SOURCE}}}"
@@ -1010,10 +1044,12 @@ init_config() {
PYTORCH_ROCM_ARCH="${PYTORCH_ROCM_ARCH:-gfx90a;gfx942;gfx950}"
CI_BASE_CONTENT_FILES="${CI_BASE_CONTENT_FILES:-${DEFAULT_CI_BASE_CONTENT_FILES}}"
CI_BASE_DOCKERFILE="${CI_BASE_DOCKERFILE:-${DEFAULT_CI_BASE_DOCKERFILE}}"
CI_BASE_DOCKERFILE_STAGES="${CI_BASE_DOCKERFILE_STAGES:-${DEFAULT_CI_BASE_DOCKERFILE_STAGES}}"
if ! using_custom_rocm_dockerfiles; then
CI_BASE_DOCKERFILE_STAGES="${CI_BASE_DOCKERFILE_STAGES:-${DEFAULT_CI_BASE_DOCKERFILE_STAGES}}"
fi
CI_BASE_METADATA_VERSION="${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}"
CI_BASE_IMAGE_TAG="${CI_BASE_IMAGE_TAG:-rocm/vllm-dev:ci_base}"
export PYTORCH_ROCM_ARCH
export PYTORCH_ROCM_ARCH CI_BASE_DOCKERFILE
SCRIPT_TMP_DIR=$(mktemp -d -t ci-bake-rocm.XXXXXX)
CI_HCL_PATH="${SCRIPT_TMP_DIR}/ci.hcl"
@@ -1042,6 +1078,11 @@ print_header() {
}
validate_inputs() {
if using_custom_rocm_dockerfiles; then
validate_rocm_dockerfile "${ROCM_BASE_DOCKERFILE}" || return $?
validate_rocm_dockerfile "${CI_BASE_DOCKERFILE}" \
ci_base test export_vllm export_test_smoke csrc-build rust-build || return $?
fi
if [[ ! -f "${VLLM_BAKE_FILE}" ]]; then
echo "Error: vLLM bake file not found at ${VLLM_BAKE_FILE}"
echo "Make sure you're running from the vLLM repository root"
@@ -1069,6 +1110,11 @@ load_ci_hcl() {
init_bake_files() {
BAKE_FILES=(-f "${VLLM_BAKE_FILE}" -f "${CI_HCL_PATH}")
if using_custom_rocm_dockerfiles; then
# Custom stacks reuse local cache without replacing the
# standard ROCm registry caches for the same commit or branch.
BAKE_FILES+=(--set '*.cache-to=')
fi
}
compute_ci_base_hash_if_needed() {
@@ -1804,7 +1850,7 @@ compute_rocm_csrc_content_hash() {
local -a content_args=()
bake_dir=$(dirname "${VLLM_BAKE_FILE}")
dockerfile_rocm="${bake_dir}/Dockerfile.rocm"
dockerfile_rocm="${CI_BASE_DOCKERFILE:-${bake_dir}/Dockerfile.rocm}"
read -r -a content_paths <<< "${content_files}"
mapfile -t content_args < <(
get_content_arg_names "${dockerfile_rocm}" "${stages}" "${ROCM_CSRC_CONTENT_ARGS:-}"
@@ -1853,7 +1899,7 @@ compute_rocm_rust_content_hash() {
local -a content_args=()
bake_dir=$(dirname "${VLLM_BAKE_FILE}")
dockerfile_rocm="${bake_dir}/Dockerfile.rocm"
dockerfile_rocm="${CI_BASE_DOCKERFILE:-${bake_dir}/Dockerfile.rocm}"
read -r -a content_paths <<< "${content_files}"
mapfile -t content_args < <(
get_content_arg_names \
@@ -1943,7 +1989,7 @@ write_rocm_build_arg_override() {
local arg_value=""
bake_dir=$(dirname "${VLLM_BAKE_FILE}")
dockerfile_rocm="${bake_dir}/Dockerfile.rocm"
dockerfile_rocm="${CI_BASE_DOCKERFILE:-${bake_dir}/Dockerfile.rocm}"
mapfile -t arg_names < <(
{
get_content_arg_names \
@@ -2259,7 +2305,7 @@ extract_dependency_pins() {
local val=""
bake_dir=$(dirname "${VLLM_BAKE_FILE}")
dockerfile_rocm="${bake_dir}/Dockerfile.rocm"
dockerfile_rocm="${CI_BASE_DOCKERFILE:-${bake_dir}/Dockerfile.rocm}"
physical_dockerfile="${dockerfile_rocm}"
if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" && "${dockerfile_rocm}" != /* ]]; then
physical_dockerfile="${ROCM_BUILD_CONTEXT_ROOT}/${dockerfile_rocm}"
@@ -2275,12 +2321,12 @@ extract_dependency_pins() {
fi
val=$(
sed -n -E "s/^[[:space:]]*ARG[[:space:]]+${var}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
sed -n -E "s/^[[:space:]]*[Aa][Rr][Gg][[:space:]]+${var}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
"${physical_dockerfile}" | head -1
)
if [[ -n "${val}" ]]; then
export "${var}=${val}"
echo "Extracted ${var}=${val} from Dockerfile.rocm"
echo "Extracted ${var}=${val} from ${dockerfile_rocm}"
fi
done
}
@@ -2297,7 +2343,7 @@ compute_dependency_cache_keys() {
local deepep_material=""
bake_dir=$(dirname "${VLLM_BAKE_FILE}")
dockerfile_rocm="${bake_dir}/Dockerfile.rocm"
dockerfile_rocm="${CI_BASE_DOCKERFILE:-${bake_dir}/Dockerfile.rocm}"
nixl_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "NIXL_BRANCH")
ucx_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "UCX_BRANCH")
rocshmem_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "ROCSHMEM_BRANCH")
@@ -2687,6 +2733,7 @@ main() {
init_bake_files
if is_ci_base_target; then
prepare_ci_build_context
configure_custom_rocm_stages
fi
compute_ci_base_hash_if_needed
configure_ci_base_image_refs
@@ -2697,6 +2744,7 @@ main() {
# version metadata only after that lookup sees the available tag history.
if ! is_ci_base_target; then
prepare_ci_build_context
configure_custom_rocm_stages
fi
extract_dependency_pins
write_rocm_build_arg_override
@@ -34,6 +34,9 @@
###############################################################################
set -o pipefail
# shellcheck source=.buildkite/scripts/rocm/build-config.sh
source "$(dirname "${BASH_SOURCE[0]}")/../rocm/build-config.sh" || exit $?
: "${BUILDKIT_PROGRESS:=plain}"
: "${TERM:=xterm-256color}"
: "${FORCE_COLOR:=1}"
@@ -106,6 +109,10 @@ clear_ci_orchestration_env() {
VLLM_CI_REQUIRE_WORKSPACE_MOUNT \
VLLM_TEST_COMMANDS \
VLLM_CI_BRANCH \
CI_ROCM_DOCKERFILE_BASE \
CI_ROCM_DOCKERFILE \
ROCM_BASE_DOCKERFILE \
CI_BASE_DOCKERFILE \
VLLM_CI_BASE_IMAGE \
VLLM_CI_FALLBACK_IMAGE \
VLLM_CI_DOCKER_DISABLED \
@@ -225,6 +232,9 @@ prepare_artifact_image() {
metadata_file=$(find "${artifact_work_dir}" -name "ci-base-image.txt" -type f | head -1)
if [[ -n "${metadata_file}" && -s "${metadata_file}" ]]; then
base_image=$(tr -d '[:space:]' < "${metadata_file}")
elif using_custom_rocm_dockerfiles; then
echo "Custom ROCm ci_base metadata is missing; using the full CI image"
return 1
fi
echo "--- Preparing local ROCm test image"
+15 -6
View File
@@ -83,14 +83,23 @@ rm -f "$build_log"
# Run the image, setting --shm-size=4g for tensor parallel. Default to
# HF_HUB_OFFLINE so a warm ~/.cache/huggingface doesn't hit the network;
# retry once online if the cache is missing something. vllm's get_config()
# wraps the raw huggingface_hub offline-mode errors in a generic ValueError
# (see transformers_utils/config.py), so match that message too or the
# fallback never triggers for a config-lookup cache miss.
OFFLINE_RETRY_PATTERN='huggingface_hub\.errors\.(LocalEntryNotFoundError|OfflineModeIsEnabled)|Invalid repository ID or local directory specified'
# retry once online if the cache is missing something. vllm wraps offline
# cache-miss errors in ways that don't preserve the raw huggingface_hub
# exception name: get_config() re-raises as a generic ValueError
# (transformers_utils/config.py), and default_loader.py raises its own
# RuntimeError when a snapshot dir has a config but no weight files. Match
# those messages too or the fallback never triggers for those cache misses.
#
# Also mount ~/.cache/vllm: multimodal test fixtures (e.g. VideoAsset, via
# vllm/assets/video.py) download into VLLM_ASSETS_CACHE (~/.cache/vllm/assets
# by default), a directory distinct from the HF hub cache above. Without a
# persistent mount there, those assets never survive past one container's
# lifetime, so every run re-fetches them from the network on the "offline"
# attempt and unconditionally falls back to the online retry.
OFFLINE_RETRY_PATTERN='huggingface_hub\.errors\.(LocalEntryNotFoundError|OfflineModeIsEnabled)|Invalid repository ID or local directory specified|Cannot find any model weights with'
run_test() {
local hf_offline=$1
docker run --rm --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true -e HF_TOKEN -e VLLM_CPU_KVCACHE_SPACE=16 -e VLLM_CPU_CI_ENV=1 -e VLLM_CPU_SIM_MULTI_NUMA=1 -e VLLM_CPU_ATTN_SPLIT_KV=0 -e HF_HUB_OFFLINE="$hf_offline" -e HF_DATASETS_OFFLINE="$hf_offline" --shm-size=4g "$IMAGE_NAME" \
docker run --rm --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" -v ~/.cache/huggingface:/root/.cache/huggingface -v ~/.cache/vllm:/root/.cache/vllm --privileged=true -e HF_TOKEN -e VLLM_CPU_KVCACHE_SPACE=16 -e VLLM_CPU_CI_ENV=1 -e VLLM_CPU_SIM_MULTI_NUMA=1 -e VLLM_CPU_ATTN_SPLIT_KV=0 -e HF_HUB_OFFLINE="$hf_offline" -e HF_DATASETS_OFFLINE="$hf_offline" --shm-size=4g "$IMAGE_NAME" \
timeout "$TIMEOUT_VAL" bash -c "set -euox pipefail; echo \"--- Print packages\"; pip list; echo \"--- Running tests\"; ${TEST_COMMAND}"
}
+85
View File
@@ -0,0 +1,85 @@
#!/usr/bin/env bash
# Shared Dockerfile selection for the normal AMD CI build chain.
# CI_ROCM_DOCKERFILE_BASE and CI_ROCM_DOCKERFILE accept repository-relative paths.
# For Rock, set CI_ROCM_DOCKERFILE_BASE=docker/Dockerfile.rock_base and
# CI_ROCM_DOCKERFILE=docker/Dockerfile.rock.
# Custom recipes must preserve the CI targets and artifact layout. Additional
# COPY inputs also need the corresponding *_CONTENT_FILES overrides.
rocm_base_dockerfile() {
local default="docker/Dockerfile.rocm_base"
printf '%s\n' "${CI_ROCM_DOCKERFILE_BASE:-${ROCM_BASE_DOCKERFILE:-${default}}}"
}
rocm_ci_dockerfile() {
local default="docker/Dockerfile.rocm"
printf '%s\n' "${CI_ROCM_DOCKERFILE:-${CI_BASE_DOCKERFILE:-${default}}}"
}
using_custom_rocm_dockerfiles() {
[[ "$(rocm_base_dockerfile)" != "docker/Dockerfile.rocm_base" \
|| "$(rocm_ci_dockerfile)" != "docker/Dockerfile.rocm" ]]
}
rocm_dockerfile_stages() {
awk '
toupper($1) == "FROM" {
for (idx = 2; idx < NF; idx++) {
if (tolower($idx) == "as") print tolower($(idx + 1))
}
}
' "$1"
}
validate_rocm_dockerfile() {
local dockerfile="$1"
local stage=""
local stages=""
shift
case "${dockerfile}" in
/*|../*|*/../*|*/..|*[[:space:]]*)
echo "ROCm Dockerfile must be a repository-relative path without '..' or whitespace: ${dockerfile}" >&2
return 2
;;
esac
if [[ ! -f "${dockerfile}" ]]; then
echo "ROCm Dockerfile not found: ${dockerfile}" >&2
return 2
fi
if (($#)) && ! awk '
toupper($1) == "FROM" {
named = 0
for (idx = 2; idx < NF; idx++) {
if (tolower($idx) == "as") named = 1
}
if (!named) exit 1
}
' "${dockerfile}"; then
echo "ROCm Dockerfile must name every build stage: ${dockerfile}" >&2
return 2
fi
stages=$(rocm_dockerfile_stages "${dockerfile}")
for stage in "$@"; do
if ! grep -Fxq -- "${stage}" <<< "${stages}"; then
echo "ROCm Dockerfile ${dockerfile} is missing required stage: ${stage}" >&2
return 2
fi
done
}
configure_rocm_build() {
ROCM_BASE_DOCKERFILE=$(rocm_base_dockerfile)
CI_BASE_DOCKERFILE=$(rocm_ci_dockerfile)
export ROCM_BASE_DOCKERFILE CI_BASE_DOCKERFILE
using_custom_rocm_dockerfiles || return 0
# Custom stacks may supply dependencies without separate build stages.
export ROCM_DEP_CACHE_EXPORT_MODE=never
# Preserve the standard runtime tags, which are unique to each build,
# while preventing experimental builds from promoting stable images.
export ROCM_BASE_PUSH_STABLE_TAG=0
export CI_BASE_PUSH_STABLE_TAG=0
echo "Custom ROCm Dockerfiles: ${ROCM_BASE_DOCKERFILE} and ${CI_BASE_DOCKERFILE}"
}
+25 -4
View File
@@ -4,6 +4,10 @@
set -euo pipefail
# shellcheck source=.buildkite/scripts/rocm/build-config.sh
source "$(dirname "${BASH_SOURCE[0]}")/build-config.sh"
configure_rocm_build
DOCKERFILE="${ROCM_BASE_DOCKERFILE:-docker/Dockerfile.rocm_base}"
BASE_REPO="${ROCM_BASE_IMAGE_REPO:-rocm/vllm-dev}"
CACHE_REPO="${ROCM_BASE_CACHE_REPO:-${DOCKERHUB_CACHE_REPO:-rocm/vllm-ci-cache}}"
@@ -12,7 +16,12 @@ DEFAULT_ROCM_BASE_METADATA_VERSION="2"
DEFAULT_ROCM_BASE_CONTENT_FILES="${DOCKERFILE}"
ROCM_BASE_LAYER_CACHE_REF=""
ROCM_BASE_TRUSTED_LAYER_CACHE_REF="${CACHE_REPO}:rocm-base-main"
BASE_CACHE_PREFIX="rocm-base"
if using_custom_rocm_dockerfiles; then
BASE_CACHE_PREFIX="rocm-base-$(printf '%s\n%s\n' \
"${ROCM_BASE_DOCKERFILE}" "${CI_BASE_DOCKERFILE}" | sha256sum | cut -c1-12)"
fi
ROCM_BASE_TRUSTED_LAYER_CACHE_REF="${CACHE_REPO}:${BASE_CACHE_PREFIX}-main"
ROCM_BASE_STABLE_TAG_UPDATED=0
declare -a ROCM_BASE_CACHE_ARGS=()
@@ -120,7 +129,7 @@ configure_rocm_base_layer_cache() {
fi
scope=$(rocm_base_layer_cache_scope)
ROCM_BASE_LAYER_CACHE_REF="${CACHE_REPO}:rocm-base-${scope}"
ROCM_BASE_LAYER_CACHE_REF="${CACHE_REPO}:${BASE_CACHE_PREFIX}-${scope}"
ROCM_BASE_CACHE_ARGS+=(
--cache-from "type=registry,ref=${ROCM_BASE_LAYER_CACHE_REF}"
)
@@ -139,7 +148,7 @@ configure_rocm_base_layer_cache() {
extract_arg_default() {
local arg_name="$1"
sed -n -E "s/^[[:space:]]*ARG[[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
sed -n -E "s/^[[:space:]]*[Aa][Rr][Gg][[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
"${DOCKERFILE}" | head -1
}
@@ -419,11 +428,16 @@ build_base_image() {
read -r -a content_paths <<< "${content_files}"
content_files_hash="$(compute_content_hash "${content_paths[@]}")"
base_hash=$(compute_base_content_hash "${use_sccache}" "${base_image_digest}")
rocm_version="$(rocm_version_from_base_image "${base_image_arg}")"
rocm_version="$(extract_arg_default ROCM_SDK_VERSION)"
rocm_version="${rocm_version:-$(rocm_version_from_base_image "${base_image_arg}")}"
triton_arg="$(extract_arg_default TRITON_BRANCH)"
pytorch_arg="$(extract_arg_default PYTORCH_BRANCH)"
pytorch_vision_arg="$(extract_arg_default PYTORCH_VISION_BRANCH)"
pytorch_audio_arg="$(extract_arg_default PYTORCH_AUDIO_BRANCH)"
triton_arg="${triton_arg:-$(extract_arg_default TRITON_VERSION)}"
pytorch_arg="${pytorch_arg:-$(extract_arg_default TORCH_VERSION)}"
pytorch_vision_arg="${pytorch_vision_arg:-$(extract_arg_default TORCHVISION_VERSION)}"
pytorch_audio_arg="${pytorch_audio_arg:-$(extract_arg_default TORCHAUDIO_VERSION)}"
fa_arg="$(extract_arg_default FA_BRANCH)"
aiter_arg="$(extract_arg_default AITER_BRANCH)"
mori_arg="$(extract_arg_default MORI_BRANCH)"
@@ -569,6 +583,13 @@ build_base_image() {
}
main() {
if using_custom_rocm_dockerfiles; then
if [[ "${ROCM_BASE_REFRESH_SKIP:-0}" == "1" ]]; then
echo "Custom ROCm Dockerfiles require ROCM_BASE_REFRESH_SKIP=0" >&2
return 2
fi
validate_rocm_dockerfile "${DOCKERFILE}" || return $?
fi
metadata_set "rocm-base-refresh" "0"
if [[ "${ROCM_BASE_REFRESH_SKIP:-0}" == "1" ]]; then
@@ -0,0 +1,272 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""FlashMLA mega attention vs the split-KV decode pipeline (DeepSeek V4.1).
Times the full set of per-layer ops each path runs in a decode step, captured
in a CUDA graph -- eager timing is dominated by launch overhead and flatters
whichever path launches fewer kernels.
mega: fused_qnorm_rope_kv_rope_quant_insert (Q pad + KV insert)
+ mega kernel
split-KV: fused_qnorm_rope_kv_rope_quant_insert (Q RoPE + pad + KV insert)
+ flash_mla_with_kvcache + fused_inv_rope_fp8_quant
Both arms fuse their Q preparation into the KV insert, so that op is charged
to both; the mega arm's Q side is only a zero-pad (nothing at all when the
shard is already at the kernel head count) because its kernel does the Q RoPE,
the inverse RoPE and the FP8 cast.
``--local-heads`` is the variable that decides the outcome: the mega kernel
absorbs work proportional to the live head count while paying for the padded
one, so it wins at 64 live heads and is a wash at 16 (TP4 on a 64-head model).
Both arms use the same KV records so the comparison is kernel-for-kernel:
a V4.1 fp8 (528 B) sliding-window cache and a compressed cache that is either
V4.1 fp8 or V4.1 NVFP4 (288 B) -- FlashMLA's SM100 sparse decode reads the
NVFP4 compressed record in both the fused and unfused kernels.
Run: .venv/bin/python benchmarks/kernels/benchmark_dsv41_mega_attn.py
"""
import argparse
import torch
import vllm.v1.attention.ops.flashmla as fm
from vllm.models.deepseek_v4.common.ops.fused_inv_rope_fp8_quant import (
fused_inv_rope_fp8_quant,
)
from vllm.models.deepseek_v41.common.ops import quantize_and_insert_k_cache
from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import (
rope_quant_insert,
)
from vllm.models.deepseek_v41.common.ops.fused_layout import permute_q_to_fused
from vllm.models.deepseek_v41.nvidia.flash_mla_mega_attn import (
alloc_mega_attn_output,
is_flashmla_mega_attn_supported,
)
from vllm.utils.math_utils import round_up
HEAD_DIM, ROPE_DIM = 512, 64
V41_BYTES, V41_FP4_BYTES = 528, 288
SWA_BLOCK, COMPRESSED_BLOCK = 32, 128
def make_cos_sin_cache(max_pos, device):
inv_freq = 1.0 / (
10000 ** (torch.arange(0, ROPE_DIM, 2, device=device).float() / ROPE_DIM)
)
freqs = torch.outer(torch.arange(max_pos, device=device).float(), inv_freq)
return torch.cat([freqs.cos(), freqs.sin()], -1)
def empty_paged_cache(num_rows, block_size, bytes_per_token, device):
"""A zeroed paged cache view [num_blocks, block_size, bytes_per_token]."""
num_blocks = (num_rows + block_size - 1) // block_size + 1
page = round_up(block_size * bytes_per_token, 512)
backing = torch.zeros(num_blocks, page, dtype=torch.uint8, device=device)
return backing, backing.as_strided(
(num_blocks, block_size, bytes_per_token), (page, bytes_per_token, 1)
)
def fill_cache(view, bytes_per_token, cos_sin, device):
n = view.shape[0] * view.shape[1]
rows = torch.randn(n, HEAD_DIM, device=device, dtype=torch.bfloat16)
slots = torch.arange(n, dtype=torch.int64, device=device)
if bytes_per_token == V41_FP4_BYTES:
# A page-aligned cache has more slots than the cos_sin table has rows;
# wrap so the insert never indexes past it.
positions = slots % cos_sin.shape[0]
rope_quant_insert(rows, positions, cos_sin, view, slots, 1)
else:
quantize_and_insert_k_cache(
rows,
view.reshape(view.shape[0], -1),
slots,
block_size=view.shape[1],
bytes_per_token=bytes_per_token,
)
def time_graph(fn, iters=100, warmup=5):
"""Capture ``fn`` in a CUDA graph and time its replay."""
s = torch.cuda.Stream()
s.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(s):
for _ in range(3):
fn()
torch.cuda.current_stream().wait_stream(s)
g = torch.cuda.CUDAGraph()
with torch.cuda.graph(g):
fn()
for _ in range(warmup):
g.replay()
torch.accelerator.synchronize()
start = torch.cuda.Event(enable_timing=True)
end = torch.cuda.Event(enable_timing=True)
start.record()
for _ in range(iters):
g.replay()
end.record()
torch.accelerator.synchronize()
return start.elapsed_time(end) * 1000 / iters
def bench_decode(s_q, device, topk_extra, extra_bytes, local_heads):
padded_heads, scale = 64, HEAD_DIM**-0.5
n_wv_group = padded_heads // 8
cos_sin = make_cos_sin_cache(65536, device)
positions = torch.randint(0, 65536, (s_q,), device=device, dtype=torch.int64)
pos32 = positions.to(torch.int32)
# Per-step inputs: the wq_b output (local heads) and this step's KV row.
q_local = torch.randn(
s_q, local_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
)
q_fused = permute_q_to_fused(q_local)
kv = torch.randn(s_q, HEAD_DIM, device=device, dtype=torch.bfloat16)
n_swa = s_q * 128 + 128
swa_backing, swa = empty_paged_cache(n_swa, SWA_BLOCK, V41_BYTES, device)
fill_cache(swa, V41_BYTES, cos_sin, device)
_, extra = empty_paged_cache(65536, COMPRESSED_BLOCK, extra_bytes, device)
fill_cache(extra, extra_bytes, cos_sin, device)
slot_mapping = torch.arange(s_q, dtype=torch.int64, device=device)
swa_idx = torch.randint(0, n_swa, (s_q, 128), device=device, dtype=torch.int32)
swa_len = torch.full((s_q,), 128, device=device, dtype=torch.int32)
ex_idx = torch.randint(
0, 65536, (s_q, topk_extra), device=device, dtype=torch.int32
)
ex_len = torch.full((s_q,), topk_extra, device=device, dtype=torch.int32)
sink = torch.zeros(padded_heads, device=device)
sched = fm.FlashMLASchedMeta()
out = alloc_mega_attn_output(s_q, n_wv_group, device)
swa4 = swa.unsqueeze(-2)
extra4 = extra.unsqueeze(-2)
swa_2d = swa_backing
def mega():
# One launch zero-pads the fused-layout Q and inserts the SWA KV.
q_pad = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
q_fused,
kv,
swa_2d,
slot_mapping,
positions,
cos_sin,
0 if padded_heads == local_heads else padded_heads,
1e-6,
SWA_BLOCK,
False, # apply_q_norm
True, # kv_mxfp8
False, # apply_q_rope: the mega kernel rotates Q itself
True, # is_q_interleaved
)
torch.ops._flashmla_C.fused_norm_rope_attn_rope_cast_decode(
q_fused if padded_heads == local_heads else q_pad,
swa4,
swa_idx,
scale,
HEAD_DIM,
sink,
swa_len,
extra4,
ex_idx,
ex_len,
False,
0.0,
pos32,
False,
64,
cos_sin,
n_wv_group,
32,
True,
True,
True,
out.data,
out.scale,
)
def split_kv():
q_pad = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
q_local,
kv,
swa_2d,
slot_mapping,
positions,
cos_sin,
padded_heads,
1e-6,
SWA_BLOCK,
False,
True,
)
o = fm.flash_mla_with_kvcache(
q=q_pad.unsqueeze(1),
k_cache=swa4,
block_table=None,
head_dim_v=HEAD_DIM,
tile_scheduler_metadata=sched,
cache_seqlens=None,
is_fp8_kvcache=True,
indices=swa_idx.view(s_q, 1, -1),
topk_length=swa_len,
softmax_scale=scale,
attn_sink=sink,
extra_k_cache=extra4,
extra_indices_in_kvcache=ex_idx.view(s_q, 1, -1),
extra_topk_length=ex_len,
)[0]
fused_inv_rope_fp8_quant(
o.squeeze(1)[:, :local_heads],
positions,
cos_sin,
n_groups=local_heads // 8,
heads_per_group=8,
quant_group_size=32,
tma_aligned_scales=True,
)
return time_graph(mega), time_graph(split_kv)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--decode-s-q",
nargs="+",
type=int,
default=[1, 2, 4, 8, 16, 32, 64, 128, 256, 512],
)
parser.add_argument("--topk-extra", type=int, default=512)
parser.add_argument("--local-heads", type=int, default=16)
parser.add_argument(
"--extra-bytes",
type=int,
choices=[V41_BYTES, V41_FP4_BYTES],
default=V41_BYTES,
help="compressed-cache record: 528 (V4.1 fp8) or 288 (V4.1 NVFP4)",
)
args = parser.parse_args()
ok, reason = is_flashmla_mega_attn_supported()
if not ok:
raise SystemExit(reason)
device = torch.device("cuda")
torch.manual_seed(0)
print(
f"decode, cudagraph (topk_swa=128, topk_extra={args.topk_extra}, "
f"local_heads={args.local_heads}, extra={args.extra_bytes}B)"
)
print(f"{'s_q':>6} {'mega_us':>9} {'split_kv_us':>12} {'speedup':>8}")
for s_q in args.decode_s_q:
m, s = bench_decode(
s_q, device, args.topk_extra, args.extra_bytes, args.local_heads
)
print(f"{s_q:>6d} {m:9.1f} {s:12.1f} {s / m:7.2f}x")
if __name__ == "__main__":
main()
@@ -3,7 +3,10 @@
* SPDX-FileCopyrightText: Copyright contributors to the vLLM project
*
* Horizontally-fused DeepseekV4-MLA kernel:
* - Q side: optional per-head RMSNorm + GPT-J RoPE on last ROPE_DIM
* - Q side: optional per-head RMSNorm (apply_q_norm) and GPT-J RoPE on the
* last ROPE_DIM (apply_q_rope), plus the zero-pad to
* q_head_padded, in the head-major layout or (is_q_interleaved)
* FlashMLA's mega-attention chunk-interleaved one
* - KV side: GPT-J RoPE on last ROPE_DIM + UE8M0 FP8 quant on NoPE + paged
* cache insert
*
@@ -174,6 +177,30 @@ __device__ __forceinline__ float warpSum(float val) {
return val;
}
// Offset of the 16 elements a lane owns inside a Q tensor of `num_heads`
// heads. The default layout is head-major: a head's 512 dims are contiguous.
// `Q_INTERLEAVED` is FlashMLA's mega-attention layout, 32 chunks of
// [num_heads, 16] per token, which changes the address only -- lane L still
// owns dims [16L, 16L+16) of its head either way, so `dim_base` stays the
// true dim offset and the norm and RoPE below are equally valid in both.
// What it does change is where the padding heads land: at the tail of every
// chunk rather than after the live heads.
template <bool Q_INTERLEAVED>
__device__ __forceinline__ int64_t qElemOffset(int const tokenIdx,
int const slotIdx,
int const laneId,
int const dim_base,
int const num_heads) {
if constexpr (Q_INTERLEAVED) {
return static_cast<int64_t>(tokenIdx) * num_heads * kHeadDim +
static_cast<int64_t>(laneId) * num_heads * kElemsPerLane +
static_cast<int64_t>(slotIdx) * kElemsPerLane;
} else {
return (static_cast<int64_t>(tokenIdx) * num_heads + slotIdx) * kHeadDim +
dim_base;
}
}
// ────────────────────────────────────────────────────────────────────────────
// Per-slot inner pipeline
// ────────────────────────────────────────────────────────────────────────────
@@ -192,10 +219,16 @@ __device__ __forceinline__ float warpSum(float val) {
// + paged-cache
// insert)
//
// `kv_mxfp8` is grid-uniform and only steers the KV branch, so it stays a
// runtime argument rather than doubling every (head count, q-norm)
// instantiation of a kernel whose hot path is the Q branch.
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM>
// `kv_mxfp8` and `apply_q_rope` are grid-uniform, so they stay runtime
// arguments rather than multiplying every (head count, q-norm, q-layout)
// instantiation of a kernel whose hot path is the Q branch. `Q_INTERLEAVED`
// cannot: it picks the address arithmetic, which has to fold.
//
// The three Q knobs are independent. The mega-attention layer uses
// Q_INTERLEAVED with both norm and RoPE off, because its attention kernel
// applies those itself -- not because the layout forbids them.
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM,
bool Q_INTERLEAVED = false>
__device__ __forceinline__ void processDeepseekV4Slot(
uint4 v0, uint4 v1, int const tokenIdx, int const slotIdx,
int const dim_base, int const laneId, int const num_heads_q,
@@ -203,7 +236,7 @@ __device__ __forceinline__ void processDeepseekV4Slot(
uint8_t* __restrict__ k_cache, int64_t const* __restrict__ slot_mapping,
int64_t const* __restrict__ position_ids,
float const* __restrict__ cos_sin_cache, int const cache_block_size,
int const kv_block_stride, bool const kv_mxfp8) {
int const kv_block_stride, bool const kv_mxfp8, bool const apply_q_rope) {
using Converter = vllm::_typeConvert<scalar_t_in>;
bool const isKV = (slotIdx == kNumHeadsQPadded);
bool const isPadQ = !isKV && (slotIdx >= num_heads_q);
@@ -213,10 +246,8 @@ __device__ __forceinline__ void processDeepseekV4Slot(
// zero literal is correct. Matches the live-Q branch's vectorized store.
if (isPadQ) {
scalar_t_in* dst =
q_out +
(static_cast<int64_t>(tokenIdx) * kNumHeadsQPadded + slotIdx) *
kHeadDim +
dim_base;
q_out + qElemOffset<Q_INTERLEAVED>(tokenIdx, slotIdx, laneId, dim_base,
kNumHeadsQPadded);
uint4 const zero4 = {0u, 0u, 0u, 0u};
*reinterpret_cast<uint4*>(dst) = zero4;
*reinterpret_cast<uint4*>(dst + 8) = zero4;
@@ -264,7 +295,9 @@ __device__ __forceinline__ void processDeepseekV4Slot(
// ── GPT-J RoPE on dims [NOPE_DIM, HEAD_DIM) ─────────────────────────────
// All math in fp32. cos_sin_cache is loaded as fp32 (its native storage).
bool const is_rope_lane = dim_base >= kNopeDim;
// KV is always rotated; Q only when the caller has not already had it
// rotated elsewhere (the mega-attention kernel does its own).
bool const is_rope_lane = dim_base >= kNopeDim && (isKV || apply_q_rope);
if (is_rope_lane) {
int64_t const pos = position_ids[tokenIdx];
constexpr int kHalfRope = kRopeDim / 2;
@@ -314,10 +347,8 @@ __device__ __forceinline__ void processDeepseekV4Slot(
make_float2(elements[8 + 2 * i], elements[8 + 2 * i + 1]));
}
scalar_t_in* dst =
q_out +
(static_cast<int64_t>(tokenIdx) * kNumHeadsQPadded + slotIdx) *
kHeadDim +
dim_base;
q_out + qElemOffset<Q_INTERLEAVED>(tokenIdx, slotIdx, laneId, dim_base,
kNumHeadsQPadded);
*reinterpret_cast<uint4*>(dst) = out0;
*reinterpret_cast<uint4*>(dst + 8) = out1;
} else {
@@ -432,14 +463,17 @@ __device__ __forceinline__ void processDeepseekV4Slot(
// `kNumHeadsQPadded` is a template parameter (compile-time constant) so the
// divisions in the grid math and the KV-sentinel comparison fold to fast
// constant operations. The launch wrapper dispatches the runtime value to
// the matching instantiation.
// the matching instantiation. `kNumHeadsQPadded == 0` leaves one slot per
// token and is the KV-insert-only instantiation, for a Q that is already in
// the shape its attention kernel wants.
//
// With DP padding, q/kv/position_ids can have more rows than slot_mapping.
// The live-Q and pad-Q branches cover all `num_tokens_full` rows (downstream
// attention uses them). The KV branch only inserts the first
// `num_tokens_insert` tokens (= slot_mapping length) into the paged cache.
//
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM>
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM,
bool Q_INTERLEAVED>
__global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel(
scalar_t_in const* __restrict__ q_in, // [N, num_heads_q, 512]
scalar_t_in* __restrict__ q_out, // [N, kNumHeadsQPadded, 512]
@@ -454,7 +488,8 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel(
int const num_heads_q, // live Q heads (input layout)
int const cache_block_size, // tokens per paged-cache block
int const kv_block_stride, // bytes per paged-cache block
bool const kv_mxfp8) { // V4.1 all-dims MXFP8 record
bool const kv_mxfp8, // V4.1 all-dims MXFP8 record
bool const apply_q_rope) { // rotate Q too, not just KV
#if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM)
// BF16 _typeConvert specialization is unavailable on pre-Ampere. The
// DeepseekV4 kernel only runs with bf16 inputs in practice, so compile a
@@ -497,20 +532,18 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel(
if (isKV) {
src_ptr = kv_in + static_cast<int64_t>(tokenIdx) * kHeadDim + dim_base;
} else {
int64_t const q_row_offset =
(static_cast<int64_t>(tokenIdx) * num_heads_q + slotIdx) *
kHeadDim +
dim_base;
src_ptr = q_in + q_row_offset;
src_ptr = q_in + qElemOffset<Q_INTERLEAVED>(tokenIdx, slotIdx, laneId,
dim_base, num_heads_q);
}
v0 = *reinterpret_cast<uint4 const*>(src_ptr);
v1 = *reinterpret_cast<uint4 const*>(src_ptr + 8);
}
processDeepseekV4Slot<scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>(
processDeepseekV4Slot<scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM,
Q_INTERLEAVED>(
v0, v1, tokenIdx, slotIdx, dim_base, laneId, num_heads_q, eps, q_out,
k_cache, slot_mapping, position_ids, cos_sin_cache, cache_block_size,
kv_block_stride, kv_mxfp8);
kv_block_stride, kv_mxfp8, apply_q_rope);
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
cudaTriggerProgrammaticLaunchCompletion();
@@ -530,7 +563,8 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel(
// Q branch (optional RMSNorm + RoPE, in place) head_slot == num_heads_q
// KV branch (RoPE + UE8M0 quant + insert)
//
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM>
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM,
bool Q_INTERLEAVED>
__global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid(
scalar_t_in const* __restrict__ q_in, scalar_t_in* __restrict__ q_out,
scalar_t_in const* __restrict__ kv_in, uint8_t* __restrict__ k_cache,
@@ -539,7 +573,7 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid(
float const* __restrict__ cos_sin_cache, float const eps,
int const num_tokens_full, int const num_tokens_insert,
int const num_heads_q, int const cache_block_size,
int const kv_block_stride, bool const kv_mxfp8) {
int const kv_block_stride, bool const kv_mxfp8, bool const apply_q_rope) {
#if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM)
if constexpr (std::is_same_v<scalar_t_in, c10::BFloat16>) {
return;
@@ -569,11 +603,8 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid(
if (s == kNumHeadsQPadded) {
src = kv_in + static_cast<int64_t>(tokenIdx) * kHeadDim + dim_base;
} else {
src = q_in +
(static_cast<int64_t>(tokenIdx) * num_heads_q +
static_cast<int64_t>(s)) *
kHeadDim +
dim_base;
src = q_in + qElemOffset<Q_INTERLEAVED>(tokenIdx, s, laneId, dim_base,
num_heads_q);
}
va = *reinterpret_cast<uint4 const*>(src);
vb = *reinterpret_cast<uint4 const*>(src + 8);
@@ -594,10 +625,12 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid(
load_slot(next_slot, v0_next, v1_next);
}
processDeepseekV4Slot<scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>(
processDeepseekV4Slot<scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM,
Q_INTERLEAVED>(
v0_curr, v1_curr, tokenIdx, curr_slot, dim_base, laneId,
num_heads_q, eps, q_out, k_cache, slot_mapping, position_ids,
cos_sin_cache, cache_block_size, kv_block_stride, kv_mxfp8);
cos_sin_cache, cache_block_size, kv_block_stride, kv_mxfp8,
apply_q_rope);
// ── Buffer rotation: hand the prefetched LDGs to the next iter.
v0_curr = v0_next;
@@ -617,14 +650,15 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid(
// ────────────────────────────────────────────────────────────────────────────
// Launch wrapper
// ────────────────────────────────────────────────────────────────────────────
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM>
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM,
bool Q_INTERLEAVED>
static void launchFusedDeepseekV4Templated(
scalar_t_in const* q_in, scalar_t_in* q_out, scalar_t_in const* kv_in,
uint8_t* k_cache, int64_t const* slot_mapping, int64_t const* position_ids,
float const* cos_sin_cache, float const eps, int const num_tokens_full,
int const num_tokens_insert, int const num_heads_q,
int const cache_block_size, int const kv_block_stride, bool const kv_mxfp8,
cudaStream_t stream) {
bool const apply_q_rope, cudaStream_t stream) {
constexpr int kBlockSize = 256;
constexpr int kWarpsPerBlock = kBlockSize / 32;
int64_t const total_warps =
@@ -663,37 +697,40 @@ static void launchFusedDeepseekV4Templated(
// grid instead. Only reachable above NUM_TOKEN_CUTOFF tokens in a single
// insert, where it is worth ~2x.
if (kNumHeadsQPadded == 0 || num_tokens_full < NUM_TOKEN_CUTOFF) {
cudaLaunchKernelEx(&config,
fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel<
scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>,
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids,
cos_sin_cache, eps, num_tokens_full, num_tokens_insert,
num_heads_q, cache_block_size, kv_block_stride,
kv_mxfp8);
cudaLaunchKernelEx(
&config,
fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel<
scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM, Q_INTERLEAVED>,
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, cos_sin_cache,
eps, num_tokens_full, num_tokens_insert, num_heads_q, cache_block_size,
kv_block_stride, kv_mxfp8, apply_q_rope);
} else {
config.gridDim = dim3(num_tokens_full);
cudaLaunchKernelEx(
&config,
fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid<
scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>,
scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM, Q_INTERLEAVED>,
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, cos_sin_cache,
eps, num_tokens_full, num_tokens_insert, num_heads_q, cache_block_size,
kv_block_stride, kv_mxfp8);
kv_block_stride, kv_mxfp8, apply_q_rope);
}
#else
// ROCm: use standard kernel launch syntax (no PDL/stream serialization)
// clang-format off
fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel<
scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>
scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM, Q_INTERLEAVED>
<<<grid, kBlockSize, 0, stream>>>(
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids,
cos_sin_cache, eps, num_tokens_full, num_tokens_insert, num_heads_q,
cache_block_size, kv_block_stride, kv_mxfp8);
cache_block_size, kv_block_stride, kv_mxfp8, apply_q_rope);
#endif
}
// Runtime dispatch into one of the precompiled `kNumHeadsQPadded`
// instantiations. Supported padded head counts: 8, 16, 32, 64, 128.
// instantiations. Supported padded head counts: 8, 16, 32, 64, 128, plus 0
// for a KV-only launch that writes no Q at all. `apply_q_norm` and
// `is_q_interleaved` are template parameters, `apply_q_rope` is not (see
// processDeepseekV4Slot), so each head count carries four instantiations.
template <typename scalar_t_in>
void launchFusedDeepseekV4QNormRopeKVRopeQuantInsert(
scalar_t_in const* q_in, scalar_t_in* q_out, scalar_t_in const* kv_in,
@@ -702,22 +739,32 @@ void launchFusedDeepseekV4QNormRopeKVRopeQuantInsert(
int const num_tokens_full, int const num_tokens_insert,
int const num_heads_q, int const num_heads_q_padded,
int const cache_block_size, int const kv_block_stride,
bool const apply_q_norm, bool const kv_mxfp8, cudaStream_t stream) {
#define DISPATCH(N) \
case N: \
if (apply_q_norm) { \
launchFusedDeepseekV4Templated<scalar_t_in, N, true>( \
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, \
cos_sin_cache, eps, num_tokens_full, num_tokens_insert, \
num_heads_q, cache_block_size, kv_block_stride, kv_mxfp8, stream); \
} else { \
launchFusedDeepseekV4Templated<scalar_t_in, N, false>( \
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, \
cos_sin_cache, eps, num_tokens_full, num_tokens_insert, \
num_heads_q, cache_block_size, kv_block_stride, kv_mxfp8, stream); \
} \
bool const apply_q_norm, bool const kv_mxfp8, bool const apply_q_rope,
bool const is_q_interleaved, cudaStream_t stream) {
#define LAUNCH(N, NORM, INTERLEAVED) \
launchFusedDeepseekV4Templated<scalar_t_in, N, NORM, INTERLEAVED>( \
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, cos_sin_cache, \
eps, num_tokens_full, num_tokens_insert, num_heads_q, cache_block_size, \
kv_block_stride, kv_mxfp8, apply_q_rope, stream)
#define DISPATCH(N) \
case N: \
if (is_q_interleaved) { \
if (apply_q_norm) { \
LAUNCH(N, true, true); \
} else { \
LAUNCH(N, false, true); \
} \
} else if (apply_q_norm) { \
LAUNCH(N, true, false); \
} else { \
LAUNCH(N, false, false); \
} \
return;
if (num_heads_q_padded == 0) {
LAUNCH(0, false, false);
return;
}
switch (num_heads_q_padded) {
DISPATCH(8)
DISPATCH(16)
@@ -729,9 +776,10 @@ void launchFusedDeepseekV4QNormRopeKVRopeQuantInsert(
"fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert: "
"unsupported num_heads_q_padded=",
num_heads_q_padded,
" (compiled instantiations: 8, 16, 32, 64, 128).");
" (compiled instantiations: 0, 8, 16, 32, 64, 128).");
}
#undef DISPATCH
#undef LAUNCH
}
// ────────────────────────────────────────────────────────────────────────────
@@ -1059,10 +1107,15 @@ void fused_deepseek_v4_kv_rope_insert(
// Zero query heads schedules only the existing KV branch, preserving its
// RoPE rounding and quantization without allocating any query tensors.
if (packed) {
vllm::deepseek_v4_fused_ops::launchFusedDeepseekV4Templated<scalar_t, 0, false>(
// With zero query heads no Q slot is scheduled, so neither Q knob is
// reachable: Q_INTERLEAVED picks addressing nothing uses, and apply_q_rope
// gates a branch only a Q slot takes (KV is rotated unconditionally).
vllm::deepseek_v4_fused_ops::launchFusedDeepseekV4Templated<scalar_t, 0,
false, false>(
nullptr, nullptr, input, cache, slots, positions, cos_sin, 0.0f,
num_tokens, num_tokens, 0, static_cast<int>(cache_block_size),
static_cast<int>(k_cache.stride(0)), kv_mxfp8, stream);
static_cast<int>(k_cache.stride(0)), kv_mxfp8, /*apply_q_rope=*/true,
stream);
} else if (fp8) {
vllm::deepseek_v4_fused_ops::launchFullCacheKernel<scalar_t, false, true>(
nullptr, nullptr, 0, 0, input, cache, slots, positions, cos_sin,
@@ -1085,8 +1138,9 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
torch::stable::Tensor const& slot_mapping, // [N] int64
torch::stable::Tensor const& position_ids, // [N] int64
torch::stable::Tensor const& cos_sin_cache, // [max_pos, rope_dim] bf16
int64_t q_head_padded, // padded Q head count for output
double eps, int64_t cache_block_size, bool apply_q_norm, bool kv_mxfp8) {
int64_t q_head_padded, // padded Q head count for output, 0 for no Q
double eps, int64_t cache_block_size, bool apply_q_norm, bool kv_mxfp8,
bool apply_q_rope, bool is_q_interleaved) {
STD_TORCH_CHECK(q_in.device().is_cuda() && q_in.is_contiguous(),
"q_in must be contiguous CUDA");
STD_TORCH_CHECK(kv.device().is_cuda() && kv.is_contiguous(),
@@ -1106,8 +1160,11 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
STD_TORCH_CHECK(kv.dim() == 2 && kv.size(1) == 512, "kv shape [N, 512]");
STD_TORCH_CHECK(q_in.scalar_type() == kv.scalar_type(),
"q_in and kv dtype must match");
STD_TORCH_CHECK(q_head_padded >= q_in.size(1),
"q_head_padded must be >= q_in.size(1) (num_heads_q)");
// q_head_padded == 0 asks for the KV insert alone: Q is left untouched and
// an empty tensor comes back, for a caller whose Q is already in the shape
// its attention kernel reads.
STD_TORCH_CHECK(q_head_padded == 0 || q_head_padded >= q_in.size(1),
"q_head_padded must be 0 or >= q_in.size(1) (num_heads_q)");
STD_TORCH_CHECK(k_cache.scalar_type() == torch::headeronly::ScalarType::Byte,
"k_cache must be uint8");
STD_TORCH_CHECK(cos_sin_cache.dim() == 2 && cos_sin_cache.size(1) == 64,
@@ -1138,8 +1195,12 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
// Allocate the padded q output. The kernel writes every element (live
// region gets optional RMSNorm+RoPE; pad region gets zeros), so `empty` is
// safe.
auto q_out = torch::stable::new_empty(
q_in, {q_in.size(0), q_head_padded, q_in.size(2)}, q_in.scalar_type());
auto q_out =
q_head_padded == 0
? torch::stable::new_empty(q_in, {int64_t{0}}, q_in.scalar_type())
: torch::stable::new_empty(
q_in, {q_in.size(0), q_head_padded, q_in.size(2)},
q_in.scalar_type());
VLLM_STABLE_DISPATCH_HALF_TYPES(
q_in.scalar_type(), "fused_deepseek_v4_qnorm_rope_kv_insert", [&] {
@@ -1155,7 +1216,8 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
cos_sin_cache.const_data_ptr<float>(), static_cast<float>(eps),
num_tokens_full, num_tokens_insert, num_heads_q,
num_heads_q_padded, cache_block_size_i, kv_block_stride,
apply_q_norm, kv_mxfp8, stream);
apply_q_norm, kv_mxfp8, apply_q_rope, is_q_interleaved,
stream);
});
return q_out;
}
+2 -1
View File
@@ -271,7 +271,8 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
torch::stable::Tensor& k_cache, torch::stable::Tensor const& slot_mapping,
torch::stable::Tensor const& position_ids,
torch::stable::Tensor const& cos_sin_cache, int64_t q_head_padded,
double eps, int64_t cache_block_size, bool apply_q_norm, bool kv_mxfp8);
double eps, int64_t cache_block_size, bool apply_q_norm, bool kv_mxfp8,
bool apply_q_rope, bool is_q_interleaved);
void fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert(
torch::stable::Tensor& q, torch::stable::Tensor const& kv,
+8 -1
View File
@@ -425,6 +425,12 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
"bool is_neox, Tensor position_ids, "
"int forced_token_heads_per_warp=-1) -> ()");
// q_head_padded is the padded Q head count of the returned tensor, or 0 to
// do the KV insert alone and return an empty tensor. The Q knobs are
// independent: apply_q_norm and apply_q_rope each drop that step for Q
// alone (KV is always rotated), and is_q_interleaved reads and writes Q in
// FlashMLA's mega-attention chunk-interleaved layout, which moves the
// padding heads to the tail of every head-dim chunk.
ops.def(
"fused_deepseek_v4_kv_rope_insert("
"Tensor kv, Tensor! k_cache, Tensor slot_mapping, Tensor position_ids, "
@@ -436,7 +442,8 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
"Tensor q_in, Tensor kv, Tensor! k_cache, "
"Tensor slot_mapping, Tensor position_ids, Tensor cos_sin_cache, "
"int q_head_padded, float eps, int cache_block_size, "
"bool apply_q_norm=True, bool kv_mxfp8=False) -> Tensor");
"bool apply_q_norm=True, bool kv_mxfp8=False, bool apply_q_rope=True, "
"bool is_q_interleaved=False) -> Tensor");
// FlashInfer V4 full-cache variants: write Q in place (bf16) or to a separate
// FP8 tensor, and KV into a contiguous 512-wide token-strided cache.
+5 -1
View File
@@ -33,6 +33,10 @@ variable "CI_BASE_CONTENT_HASH" {
default = ""
}
variable "CI_BASE_DOCKERFILE" {
default = "docker/Dockerfile.rocm"
}
# REMOTE_VLLM=0: use local source via Docker build context (ONBUILD COPY ./ vllm/)
# REMOTE_VLLM=1: clone from GitHub at VLLM_BRANCH (standalone builds without local source)
variable "REMOTE_VLLM" {
@@ -74,7 +78,7 @@ group "default" {
}
target "_common-rocm" {
dockerfile = "docker/Dockerfile.rocm"
dockerfile = CI_BASE_DOCKERFILE
context = "."
args = {
max_jobs = MAX_JOBS
@@ -20,6 +20,15 @@ Both approaches are under active development.
Due to the auto-regressive nature of decoding, every decoding step needs to compute a small amount of query tokens w.r.t. a large number of key/value tokens stored in the paged KV cache. The core of decode context parallel is how to shard the KV cache across GPUs.
`engine_client.vllm_config.cache_config.effective_attention_block_size` reports the
initialized full-attention block size in tokens, including DCP: a physical block of
16 tokens with DCP=4 represents 64 tokens. The same value is available through
gRPC `Control.GetServerInfo.effective_attention_block_size`. Physical block-size
fields retain their existing meaning. The new value is unavailable (`None` in
Python, absent in gRPC) if there is no common full-attention size, the scheduler
does not expose it, or an older engine omits it. gRPC also omits the value when
engine replicas disagree. Partial cache events carry their own actual size.
For a model with `H` kv-heads, a request with `T` tokens in the context needs to store `H * T` key/value tensors in the KV cache.
1. If one GPU can hold them all, and the performance is good enough, then no parallelization is needed.
+1 -1
View File
@@ -147,7 +147,7 @@ vllm-llm = { path = "src/llm" }
vllm-managed-engine = { path = "src/managed-engine" }
vllm-metrics = { path = "src/metrics" }
vllm-parser = { path = "src/parser" }
vllm-proto = { version = "0.2.0", path = "proto" }
vllm-proto = { path = "proto" }
vllm-server = { path = "src/server" }
vllm-text = { path = "src/text" }
vllm-tokenizer = { path = "src/tokenizer" }
+4 -2
View File
@@ -2,6 +2,9 @@
This directory is the canonical source for vLLM's gRPC schema.
See [context parallel deployment](../../docs/serving/context_parallel_deployment.md)
for effective attention block-size metadata in Python and `Control.GetServerInfo`.
Schema updates are no longer published to the Buf Schema Registry. Rust consumers
should use `vllm-proto` from crates.io; consumers in other languages can generate
bindings from the `.proto` files in this directory. Buf still builds and lints
@@ -34,8 +37,7 @@ On pull requests and releases, `cargo-semver-checks` compares the crate with its
latest published version. Include any required version bump in the protocol
change PR. This check becomes available after the first manual publication.
1. Update the crate version and the `vllm-proto` workspace dependency together,
and update `rust/Cargo.lock`.
1. Update the crate version in `rust/proto/Cargo.toml` and update `rust/Cargo.lock`.
2. Run `cargo publish --manifest-path rust/proto/Cargo.toml --locked --dry-run`
and the frontend gRPC tests. Record the tested vLLM releases or revisions in
the release notes; matching crate versions alone do not establish runtime compatibility.
+2
View File
@@ -50,6 +50,8 @@ message ServerInfo {
uint64 max_batched_tokens = 9;
uint32 max_loras = 10;
RlCapabilities rl_capabilities = 11;
// Full-attention block size in tokens after initialization; absent if unavailable.
optional uint64 effective_attention_block_size = 14;
}
message RlCapabilities {
@@ -510,6 +510,15 @@ impl EngineCoreClient {
self.engines.iter().map(|engine| engine.ready_response.num_gpu_blocks).sum()
}
/// Return the effective attention block size if all engines report the same value.
pub fn effective_attention_block_size(&self) -> Option<u64> {
let size = self.ready_response().effective_attention_block_size?;
self.engines
.iter()
.all(|engine| engine.ready_response.effective_attention_block_size == Some(size))
.then_some(size)
}
/// Return the minimum engine-reported `max_model_len` across all engines.
///
/// This is the auto-fitted value after KV cache profiling and may differ
@@ -72,6 +72,7 @@ pub fn default_ready_response() -> EngineCoreReadyResponse {
weight_transfer_backend: None,
enable_sleep_mode: false,
supports_draft_weight_updates: false,
effective_attention_block_size: None,
}
}
@@ -104,6 +104,9 @@ pub struct EngineCoreReadyResponse {
/// Whether the engine has a speculative draft model that can be updated.
#[serde(default)]
pub supports_draft_weight_updates: bool,
/// Full-attention block size in tokens after initialization, or unavailable.
#[serde(default)]
pub effective_attention_block_size: Option<u64>,
}
/// Frontend-owned ZMQ addresses that are sent to the engine during startup
@@ -2846,6 +2846,11 @@ fn python_msgpack_fixtures_match_rust_encoding() {
let ready_response: EngineCoreReadyResponse =
rmp_serde::from_slice(&hex::decode(ready_response_hex).unwrap()).unwrap();
let mut legacy_ready = serde_json::to_value(&ready_response).unwrap();
legacy_ready.as_object_mut().unwrap().remove("effective_attention_block_size");
let legacy_ready: EngineCoreReadyResponse =
rmp_serde::from_slice(&rmp_serde::to_vec_named(&legacy_ready).unwrap()).unwrap();
assert!(legacy_ready.effective_attention_block_size.is_none());
assert!(ready_response.supports_lora);
assert_eq!(ready_response.max_loras, 8);
assert_eq!(
@@ -2854,6 +2859,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
);
assert!(ready_response.enable_sleep_mode);
assert!(ready_response.supports_draft_weight_updates);
assert_eq!(ready_response.effective_attention_block_size, Some(64));
let kv_events_config = ready_response.kv_events_config.expect("KV events config should decode");
assert!(kv_events_config.enable_kv_cache_events);
assert_eq!(kv_events_config.publisher, "zmq");
@@ -423,6 +423,7 @@ class EngineCoreReadyResponse:
weight_transfer_backend: str | None = None
enable_sleep_mode: bool = False
supports_draft_weight_updates: bool = False
effective_attention_block_size: int | None = None
ready_response = EngineCoreReadyResponse(
@@ -446,6 +447,7 @@ ready_response = EngineCoreReadyResponse(
weight_transfer_backend="nccl",
enable_sleep_mode=True,
supports_draft_weight_updates=True,
effective_attention_block_size=64,
kv_events_config=KVEventsConfig(
enable_kv_cache_events=True,
publisher="zmq",
+1
View File
@@ -195,6 +195,7 @@ impl pb::control_server::Control for ControlServiceImpl {
max_batched_tokens: ready.max_num_batched_tokens,
max_loras: ready.max_loras,
rl_capabilities: Some(self.rl_capabilities()),
effective_attention_block_size: self.client().effective_attention_block_size(),
}))
}
+12 -1
View File
@@ -1911,8 +1911,16 @@ async fn control_abort_resolves_external_id_and_empty_is_noop() {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn control_reports_server_and_model_info() {
let mut ready = default_ready_response();
ready.effective_attention_block_size = Some(64);
let (generate_service, control_service, engine_health, _engine_task) =
setup_grpc_service(b"engine-grpc-info", default_stream_output_specs()).await;
setup_grpc_service_with_engine_script(
b"engine-grpc-info".to_vec(),
ready,
Arc::new(FakeTextBackend),
|_, _| boxed_test_future(async {}),
)
.await;
let (channel, server_task) = start_grpc_test_server(
generate_service,
control_service,
@@ -1928,6 +1936,7 @@ async fn control_reports_server_and_model_info() {
.expect("get server info")
.into_inner();
assert_eq!(server.engine_version, "test-vllm-version");
assert_eq!(server.effective_attention_block_size, Some(64));
assert_eq!(server.api_version, "vllm");
assert_eq!(server.instance_id, "test-instance");
assert_eq!(server.max_model_len, DEFAULT_MOCK_MAX_MODEL_LEN as u32);
@@ -2195,6 +2204,7 @@ async fn control_aggregates_multi_engine_capacity() {
ready_0.weight_transfer_backend = Some("nccl".to_string());
ready_0.enable_sleep_mode = true;
ready_0.supports_draft_weight_updates = true;
ready_0.effective_attention_block_size = Some(64);
let mut ready_1 = default_ready_response();
ready_1.max_model_len = 4_096;
@@ -2248,6 +2258,7 @@ async fn control_aggregates_multi_engine_capacity() {
.into_inner();
assert_eq!(server.max_model_len, 4_096);
assert_eq!(server.total_kv_blocks, 30);
assert!(server.effective_attention_block_size.is_none());
let rl = server.rl_capabilities.expect("RL capabilities");
assert!(!rl.weight_transfer_enabled);
assert!(rl.weight_transfer_backend.is_empty());
+3
View File
@@ -215,6 +215,9 @@ def test_cache_config_hash_ignores_kv_cache_sizing_knobs():
base_hash = CacheConfig().compute_hash()
assert CacheConfig(kv_cache_memory_bytes=1 << 30).compute_hash() == base_hash
assert CacheConfig(gpu_memory_utilization=0.5).compute_hash() == base_hash
config = CacheConfig()
config.effective_attention_block_size = 64
assert config.compute_hash() == base_hash
def test_scheduler_config_hash_includes_max_num_seqs():
+32 -3
View File
@@ -694,14 +694,43 @@ def test_check_shm_free_space_raises_when_insufficient(tmp_path):
def test_check_shm_free_space_passes_when_sufficient(tmp_path):
with mock.patch.object(
shm_broadcast.shutil, "disk_usage", return_value=_fake_disk_usage(512 << 20)
with (
mock.patch.object(
shm_broadcast.shutil,
"disk_usage",
return_value=_fake_disk_usage(512 << 20),
),
mock.patch.object(shm_broadcast, "check_cgroup_memory_available"),
):
check_shm_free_space(240 << 20, shm_path=str(tmp_path))
def test_check_shm_free_space_skipped_when_path_missing(tmp_path):
check_shm_free_space(1 << 60, shm_path=str(tmp_path / "does-not-exist"))
with mock.patch.object(shm_broadcast, "check_cgroup_memory_available"):
check_shm_free_space(1 << 60, shm_path=str(tmp_path / "does-not-exist"))
def test_check_shm_free_space_checks_cgroup(tmp_path):
with (
mock.patch.object(
shm_broadcast.shutil,
"disk_usage",
return_value=_fake_disk_usage(512 << 20),
),
mock.patch.object(
shm_broadcast, "check_cgroup_memory_available"
) as check_cgroup,
):
check_shm_free_space(
240 << 20,
shm_path=str(tmp_path),
allocation_name="SHM mmap",
)
check_cgroup.assert_called_once_with(
240 << 20,
"SHM mmap",
)
def test_shm_ring_buffer_creation_checks_free_space():
@@ -948,7 +948,7 @@ def test_structured_output_batched_with_non_structured_outputs_requests(
prompts = [structured_outputs_prompt, non_structured_outputs_prompt]
sampling_params = [
SamplingParams(
temperature=1.0,
temperature=0,
max_tokens=400,
structured_outputs=StructuredOutputsParams(json=sample_json_schema),
),
@@ -45,7 +45,8 @@ async def test_include_reasoning_true_non_streaming(client: openai.AsyncOpenAI):
response = await client.chat.completions.create(
model=MODEL_NAME,
messages=MESSAGES,
max_tokens=200,
max_tokens=512,
temperature=0,
extra_body={"include_reasoning": True},
)
@@ -63,7 +64,8 @@ async def test_include_reasoning_false_non_streaming(client: openai.AsyncOpenAI)
response = await client.chat.completions.create(
model=MODEL_NAME,
messages=MESSAGES,
max_tokens=200,
max_tokens=512,
temperature=0,
extra_body={"include_reasoning": False},
)
@@ -99,7 +101,8 @@ async def test_include_reasoning_true_streaming(client: openai.AsyncOpenAI):
stream = await client.chat.completions.create(
model=MODEL_NAME,
messages=MESSAGES,
max_tokens=200,
max_tokens=512,
temperature=0,
stream=True,
extra_body={"include_reasoning": True},
)
@@ -130,7 +133,8 @@ async def test_include_reasoning_false_streaming(client: openai.AsyncOpenAI):
stream = await client.chat.completions.create(
model=MODEL_NAME,
messages=MESSAGES,
max_tokens=200,
max_tokens=512,
temperature=0,
stream=True,
extra_body={"include_reasoning": False},
)
@@ -353,7 +353,7 @@ async def test_parallel_no_streaming(client: openai.AsyncOpenAI, model_name: str
prompt = "What is an LLM?"
n = 3
max_tokens = 50 # we want some to finish earlier than others
max_tokens = 50
# High temperature to maximize chance of unique completions.
completion = await client.completions.create(
@@ -371,16 +371,12 @@ async def test_parallel_no_streaming(client: openai.AsyncOpenAI, model_name: str
num_completions = len(completion.choices)
assert num_completions == n, f"Num completions {num_completions} but expected {n}."
completion_repeats: dict[str, int] = {}
output_token_lengths = set()
for idx, choice in enumerate(completion.choices):
# Assert correct completion index & some finish reason.
assert choice.index == idx, f"Index {choice.index} but expected {idx}."
assert choice.finish_reason is not None, "None finish_reason is invalid."
text = choice.text
completion_repeats[text] = completion_repeats.get(text, 0) + 1
output_token_lengths.add(len(choice.logprobs.tokens))
# Assert subrequests finished at different times
assert len(output_token_lengths) > 1
# Assert `n` unique completions
num_unique = len(completion_repeats)
if num_unique != n:
@@ -403,7 +399,7 @@ async def test_parallel_streaming(client: openai.AsyncOpenAI, model_name: str):
prompt = "What is an LLM?"
n = 3
max_tokens = 50 # we want some to finish earlier than others
max_tokens = 50
stream = await client.completions.create(
model=model_name,
@@ -427,19 +423,15 @@ async def test_parallel_streaming(client: openai.AsyncOpenAI, model_name: str):
f"Expected {n} completions with valid indices and finish_reason."
)
completion_repeats: dict[str, int] = {}
chunk_lengths = set()
for chunk in chunks:
chunk_len = len(chunk)
# Assert correct number of completion tokens
chunk_lengths.add(chunk_len)
assert chunk_len <= max_tokens, (
f"max_tokens={max_tokens} but chunk len is {chunk_len}."
)
text = "".join(chunk)
completion_repeats[text] = completion_repeats.get(text, 0) + 1
print(text)
# Assert subrequests finished at different times
assert len(chunk_lengths) > 1
# Assert `n` unique completions
num_unique = len(completion_repeats)
if num_unique != n:
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from contextlib import AsyncExitStack
from contextlib import AsyncExitStack, asynccontextmanager
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
@@ -41,6 +41,7 @@ from vllm.entrypoints.mcp.tool_server import ToolServer
from vllm.entrypoints.openai.responses.context import (
ConversationContext,
HarmonyContext,
ParsableContext,
SimpleContext,
)
from vllm.entrypoints.openai.responses.protocol import (
@@ -551,6 +552,87 @@ def test_responses_render_result_rejects_non_single_prompt(engine_inputs):
class TestInitializeToolSessions:
"""Test class for _initialize_tool_sessions method"""
@pytest.fixture(params=[ParsableContext, HarmonyContext])
def tool_context(self, request):
context = request.param.__new__(request.param)
context.available_tools = ["browser", "python"]
context._tool_sessions = {}
context.called_tools = set()
return context
@pytest.mark.asyncio
@pytest.mark.parametrize(
("available_tools", "called_tools"),
[
([], []),
(["browser"], ["browser"]),
(["browser", "python"], []),
(["browser", "python"], ["browser"]),
(["browser", "python"], ["python"]),
(["browser", "python"], ["browser", "python"]),
],
ids=["empty", "single", "unused", "first", "last", "both"],
)
async def test_mcp_sessions_cleaned_once_before_close(
self, tool_context, available_tools, called_tools
):
events = []
sessions = {}
@asynccontextmanager
async def new_session(name, *_args):
session = MagicMock()
session.call_tool = AsyncMock(
side_effect=lambda *_: events.append(("cleanup", name))
)
sessions[name] = session
try:
yield session
finally:
events.append(("close", name))
tool_server = MagicMock(spec=ToolServer)
tool_server.new_session.side_effect = new_session
tool_context.available_tools = available_tools
async with AsyncExitStack() as stack:
await tool_context.init_tool_sessions(tool_server, stack, "req", {})
# Reinitializing existing sessions must not duplicate cleanup either.
await tool_context.init_tool_sessions(tool_server, stack, "req", {})
tool_context.called_tools.update(called_tools)
for name, session in sessions.items():
if name in called_tools:
session.call_tool.assert_awaited_once_with("cleanup_session", {})
assert events.index(("cleanup", name)) < events.index(("close", name))
else:
session.call_tool.assert_not_awaited()
assert [name for event, name in events if event == "close"] == list(
reversed(available_tools)
)
@pytest.mark.asyncio
async def test_mcp_partial_initialization_closes_opened_session(self, tool_context):
closed = []
session = MagicMock(call_tool=AsyncMock())
@asynccontextmanager
async def new_session(name, *_args):
if name == "python":
raise RuntimeError("session initialization failed")
try:
yield session
finally:
closed.append(name)
tool_server = MagicMock(spec=ToolServer)
tool_server.new_session.side_effect = new_session
with pytest.raises(RuntimeError, match="session initialization failed"):
async with AsyncExitStack() as stack:
await tool_context.init_tool_sessions(tool_server, stack, "req", {})
assert closed == ["browser"]
session.call_tool.assert_not_awaited()
@pytest_asyncio.fixture
async def serving_responses_instance(self):
"""Create a real OpenAIServingResponses instance for testing"""
@@ -17,6 +17,88 @@ from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
def assert_rope_close_rocm(
actual: torch.Tensor,
ref: torch.Tensor,
rope_input: torch.Tensor,
cos_sin_cache: torch.Tensor,
positions: torch.Tensor,
is_neox: bool,
*,
max_fail_fraction: float | None = None,
atol: float | None = None,
rtol: float | None = None,
) -> None:
"""assert_close for RoPE fused-vs-reference on ROCm, with a per-element
product-ULP allowance for catastrophic-cancellation elements.
The fused and reference kernels may contract the RoPE multiply-add into an
FMA with either product pre-rounded to the working dtype. The two valid
orderings differ by at most 2 ULPs of the product at that element, which
cancellation (x*cos ~= -y*sin) can amplify past the default tolerance.
Tier 1: |actual - ref| <= atol + rtol * |ref| (default tolerance).
Tier 2: elements failing tier 1 must each satisfy
|actual_e - ref_e| <= 2 * eps * |product_e|, where product_e is the larger
of the two products combined at that element, and their fraction must be
<= max_fail_fraction (1e-4 for fp16; 1e-2 for bf16, whose coarser eps
puts 1-ULP divergences past the default tolerance far more often).
"""
if atol is None:
atol = get_default_atol(ref)
if rtol is None:
rtol = get_default_rtol(ref)
if max_fail_fraction is None:
max_fail_fraction = 1e-2 if ref.dtype == torch.bfloat16 else 1e-4
eps = torch.finfo(ref.dtype).eps
# per-element product magnitude, in the (seq, heads, rot_dim) layout
seq_len, rot_dim = rope_input.shape[0], rope_input.shape[-1]
half = rot_dim // 2
inp = rope_input.reshape(seq_len, -1, rot_dim).to(torch.float32)
cs = cos_sin_cache[positions].to(torch.float32) # (seq_len, 2*half)
cos = cs[:, :half].unsqueeze(1) # broadcast over heads
sin = cs[:, half:].unsqueeze(1)
x, y = inp[..., :half], inp[..., half:]
if is_neox:
# out = [x*cos - y*sin, y*cos + x*sin]
prod = torch.cat(
[
torch.maximum((x * cos).abs(), (y * sin).abs()),
torch.maximum((y * cos).abs(), (x * sin).abs()),
],
dim=-1,
)
else:
# out[2i] = x[2i]*cos[i] - x[2i+1]*sin[i], interleaved
xe, xo = inp[..., 0::2], inp[..., 1::2]
prod = torch.stack(
[
torch.maximum((xe * cos).abs(), (xo * sin).abs()),
torch.maximum((xo * cos).abs(), (xe * sin).abs()),
],
dim=-1,
).flatten(-2)
prod = prod.reshape(rope_input.shape)
diff = (actual.to(torch.float32) - ref.to(torch.float32)).abs()
tol = atol + rtol * ref.to(torch.float32).abs()
fail = diff > tol
n_fail = int(fail.sum())
if n_fail == 0:
return
assert n_fail <= max_fail_fraction * diff.numel(), (
f"{n_fail} elements beyond default tolerance "
f"({n_fail / diff.numel():.2%} > {max_fail_fraction:.2%}); "
f"max diff {diff.max().item()}"
)
excess = diff[fail] - 2 * eps * prod[fail]
assert (excess <= 0).all(), (
f"{int((excess > 0).sum())} cancellation elements exceed 2 product-ULPs; "
f"worst excess {excess.max().item()}"
)
@pytest.fixture
def default_vllm_config(monkeypatch):
"""Enable the AITER triton rope on ROCm for fp16-consistent numerics.
@@ -99,6 +181,8 @@ def test_concat_and_cache_mla_rope_fused(
k_pe = torch.flatten(key[..., :qk_rope_head_dim], start_dim=1).to(device=device)
kv_c = torch.flatten(key[..., qk_rope_head_dim:], start_dim=1).to(device=device)
k_pe_orig = k_pe.clone()
query_orig = query.clone()
if current_platform.is_rocm():
# We use forward_hip for the same numerics as the fused custom kernel on ROCm
@@ -181,12 +265,9 @@ def test_concat_and_cache_mla_rope_fused(
kv_cache_scale,
)
# On ROCm the AITER Triton rope diverges by ~1 ULP from the fused kernel,
# which the tight CUDA tolerances don't cover for the low-precision paths:
# - bf16: up to ~1 bf16 ULP (0.0156-0.03125 at values ~2-3), so relax the
# kv-cache atol (bounded ~0.02) and the query atol (~0.04).
# - neox-style fp8: one e4m3 ULP (~12.5%), so relax rtol.
# Other paths use the CUDA defaults.
# On ROCm the AITER Triton rope diverges by up to 2 product-ULPs from the
# fused kernel, which the default tolerance misses at the rare
# catastrophic-cancellation elements. fp8 keeps its calibrated tolerance.
rocm_neox = current_platform.is_rocm() and is_neox_style
rocm_bf16 = current_platform.is_rocm() and dtype == torch.bfloat16
if kv_cache_dtype == "fp8":
@@ -207,16 +288,40 @@ def test_concat_and_cache_mla_rope_fused(
atol=0.004 if rocm_bf16 else 0.001,
rtol=0.15 if rocm_neox or rocm_bf16 else 0.1,
)
elif rocm_bf16:
torch.testing.assert_close(kv_cache, ref_kv_cache, atol=0.02, rtol=1e-3)
elif rocm_neox:
torch.testing.assert_close(kv_cache, ref_kv_cache, atol=1e-3, rtol=1e-3)
elif current_platform.is_rocm() and dtype in (torch.half, torch.bfloat16):
# gather the rope region of the cache back to token order
block_idx = slot_mapping // block_size
block_offset = slot_mapping % block_size
fused_k_rope = kv_cache[block_idx, block_offset, kv_lora_rank:]
ref_k_rope_gathered = ref_kv_cache[block_idx, block_offset, kv_lora_rank:]
assert_rope_close_rocm(
fused_k_rope,
ref_k_rope_gathered,
k_pe_orig,
rope.cos_sin_cache,
positions,
is_neox_style,
)
# the nope region is a byte-exact copy
torch.testing.assert_close(
kv_cache[..., :kv_lora_rank], ref_kv_cache[..., :kv_lora_rank]
)
else:
torch.testing.assert_close(kv_cache, ref_kv_cache)
torch.testing.assert_close(
query,
ref_q_pe,
atol=0.04 if rocm_bf16 else get_default_atol(query),
rtol=get_default_rtol(query),
)
if current_platform.is_rocm() and dtype in (torch.half, torch.bfloat16):
assert_rope_close_rocm(
query,
ref_q_pe,
query_orig,
rope.cos_sin_cache,
positions,
is_neox_style,
)
else:
torch.testing.assert_close(
query,
ref_q_pe,
atol=get_default_atol(query),
rtol=get_default_rtol(query),
)
@@ -0,0 +1,60 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the AITER norm support predicates.
The AITER norm wrappers flatten >2-D activations with ``Tensor.reshape``, which
silently copies when the flattened shape is not expressible with the input's
strides. The support predicates must reject those inputs so dispatch falls
through to a provider that handles arbitrary strides.
"""
import pytest
import torch
from vllm._aiter_ops import is_aiter_found_and_supported
from vllm.kernels.aiter_ops import flatten_to_2d_is_free
pytestmark = pytest.mark.skipif(
not is_aiter_found_and_supported(),
reason="Only test on ROCm with AITER installed and supported",
)
def _qkv_slice_by_head(num_tokens, num_q_heads, num_kv_heads, head_dim):
"""q viewed per-head from a fused QKV projection, as Qwen3-style QK-norm does."""
q_size, kv_size = num_q_heads * head_dim, num_kv_heads * head_dim
qkv = torch.empty(num_tokens, q_size + 2 * kv_size)
q = qkv.split([q_size, kv_size, kv_size], dim=-1)[0]
return q.view(*q.shape[:-1], num_q_heads, head_dim)
@pytest.mark.parametrize(
"x",
[
torch.empty(8, 16),
torch.empty(16),
torch.empty(2, 4, 16),
torch.empty(2, 1, 16),
# A contiguous tensor stays flattenable after a leading-dim slice.
torch.empty(8, 4, 16)[2:6],
],
)
def test_flattenable(x):
assert flatten_to_2d_is_free(x)
assert x.reshape(-1, x.shape[-1]).data_ptr() == x.data_ptr()
@pytest.mark.parametrize(
"x",
[
_qkv_slice_by_head(8, 32, 8, 128),
# Non-unit last-dim stride.
torch.empty(4, 8, 32).transpose(-1, -2),
# Leading dims cannot be merged: a slice along the middle dim.
torch.empty(4, 8, 32)[:, :4],
],
)
def test_not_flattenable(x):
assert not flatten_to_2d_is_free(x)
# reshape has to copy, which is exactly what the predicate guards against.
assert x.reshape(-1, x.shape[-1]).data_ptr() != x.data_ptr()
@@ -9,6 +9,15 @@ import pytest
import torch
from vllm import _custom_ops as ops
from vllm.model_executor.kernels.linear.scaled_mm.cpu import (
CPUFp8PerTensorScaledMMLinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.ScaledMMLinearKernel import (
FP8ScaledMMLinearLayerConfig,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8StaticTensorSym,
)
from vllm.platforms import current_platform
if not current_platform.is_cpu():
@@ -17,6 +26,9 @@ if not current_platform.is_cpu():
if not ops._supports_cpu_fp8_w8a16:
pytest.skip("fp8_scaled_mm_cpu op not available", allow_module_level=True)
if not torch.cpu._is_amx_tile_supported():
pytest.skip("requires AMX tile support", allow_module_level=True)
BLOCK_SIZE = [128, 128]
@@ -160,3 +172,104 @@ def test_cpu_fp8_scaled_mm(M: int, N: int, K: int, use_bias: bool):
assert kernel_out.dtype == out_dtype
torch.testing.assert_close(kernel_out, ref_out, rtol=0.02, atol=0.01)
def quantize_weight_per_tensor_fp8(
weight: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Quantize weight [N, K] to FP8 with a single per-tensor scale."""
fp8_max = torch.finfo(torch.float8_e4m3fn).max
scale = weight.abs().amax() / fp8_max
scale = torch.where(scale == 0, torch.ones_like(scale), scale)
q = (weight / scale).clamp(-fp8_max, fp8_max).to(torch.float8_e4m3fn)
return q, scale
def ref_fp8_per_tensor_scaled_mm(
x: torch.Tensor,
fp8_weight: torch.Tensor,
scale: torch.Tensor,
bias: torch.Tensor | None,
out_dtype: torch.dtype,
) -> torch.Tensor:
w_dq = fp8_weight.float() * scale
out = torch.mm(x.float(), w_dq.t())
if bias is not None:
out = out + bias.float()
return out.to(out_dtype)
NK_SIZES_PER_TENSOR = [
(32, 64),
(5120, 5120),
]
@pytest.mark.parametrize("M", [1, 64])
@pytest.mark.parametrize("N,K", NK_SIZES_PER_TENSOR)
@pytest.mark.parametrize("use_bias", [False, True])
def test_cpu_fp8_per_tensor_scaled_mm_kernel(
M: int, N: int, K: int, use_bias: bool, default_vllm_config
):
"""CPUFp8PerTensorScaledMMLinearKernel correctness against float reference.
Exercises the full kernel class (not just the raw op), including the
weight-orientation fixup: Fp8LinearMethod stores `layer.weight` as
[K, N] (torch._scaled_mm convention) before calling
process_weights_after_loading, so the kernel must transpose back to
[N, K] before VNNI-packing.
"""
torch.manual_seed(0)
out_dtype = torch.bfloat16
x = torch.randn(M, K, dtype=out_dtype) / (K**0.5)
w_f32 = torch.randn(N, K, dtype=torch.float32) / (K**0.5)
fp8_weight, scale = quantize_weight_per_tensor_fp8(w_f32)
bias = torch.randn(N, dtype=torch.float32) * 0.1 if use_bias else None
ref_out = ref_fp8_per_tensor_scaled_mm(x, fp8_weight, scale, bias, out_dtype)
config = FP8ScaledMMLinearLayerConfig(
weight_quant_key=kFp8StaticTensorSym,
activation_quant_key=kFp8StaticTensorSym,
weight_shape=(N, K),
input_dtype=out_dtype,
out_dtype=out_dtype,
)
kernel = CPUFp8PerTensorScaledMMLinearKernel(
config,
layer_param_names=["weight", "weight_scale", "input_scale", "input_scale_ub"],
)
layer = torch.nn.Module()
# Fp8LinearMethod stores the weight transposed to [K, N] before calling
# process_weights_after_loading.
layer.register_parameter(
"weight", torch.nn.Parameter(fp8_weight.t().contiguous(), requires_grad=False)
)
layer.register_parameter(
"weight_scale", torch.nn.Parameter(scale.clone(), requires_grad=False)
)
kernel.process_weights_after_loading(layer)
kernel_out = kernel.apply_weights(layer, x, bias)
assert kernel_out.dtype == out_dtype
torch.testing.assert_close(kernel_out, ref_out, rtol=0.02, atol=0.01)
@pytest.mark.parametrize("n", [16, 48])
def test_cpu_fp8_per_tensor_kernel_rejects_non_multiple_of_32_n(n: int):
"""The AMX tinygemm kernel tiles N in chunks of 32 and cannot handle a
remainder tile smaller than that, so can_implement must reject any N
that isn't a multiple of 32 rather than let it crash the process.
"""
config = FP8ScaledMMLinearLayerConfig(
weight_quant_key=kFp8StaticTensorSym,
activation_quant_key=kFp8StaticTensorSym,
weight_shape=(n, 64),
input_dtype=torch.bfloat16,
out_dtype=torch.bfloat16,
)
supported, _ = CPUFp8PerTensorScaledMMLinearKernel.can_implement(config)
assert not supported
+133 -6
View File
@@ -331,6 +331,19 @@ def test_v41_fused_save_compress_and_insert(
torch.testing.assert_close(cache_backing, expected_cache, rtol=0, atol=0)
def _rotate_rope_tail(
latent_row: torch.Tensor, cos_sin_row: torch.Tensor
) -> torch.Tensor:
"""GPT-J RoPE of one latent row's last 64 dims, in fp32."""
row = latent_row.float()
c, s = cos_sin_row.chunk(2)
rotated = row.clone()
even, odd = row[448::2], row[449::2]
rotated[448::2] = even * c - odd * s
rotated[449::2] = odd * c + even * s
return rotated
def _mxfp8_record_reference(
latent_row: torch.Tensor, cos_sin_row: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
@@ -339,12 +352,7 @@ def _mxfp8_record_reference(
Mirrors FlashMLA's ``KVCacheLayout.V41_FP8Sparse`` quantizer: rotate the
RoPE tail first, then scale every 32-dim tile, RoPE tiles included.
"""
row = latent_row.float()
c, s = cos_sin_row.chunk(2)
rotated = row.clone()
even, odd = row[448::2], row[449::2]
rotated[448::2] = even * c - odd * s
rotated[449::2] = odd * c + even * s
rotated = _rotate_rope_tail(latent_row, cos_sin_row)
quantized, scales = _ue8m0_reference(rotated, 32, 448.0)
return quantized.view(torch.uint8), (scales.log2() + 127).to(torch.uint8)
@@ -461,6 +469,125 @@ def test_v41_mxfp8_cache_round_trip():
assert (error <= tolerance.repeat_interleave(32, dim=-1)).all()
# e2m1 magnitudes indexed by the low 3 bits of a code; bit 3 is the sign.
_E2M1_MAGNITUDES = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]
def _decode_nvfp4_row(packed: torch.Tensor, scales: torch.Tensor) -> torch.Tensor:
"""Unpack one 256-byte e2m1 row and apply its 32 e4m3 tile scales."""
mags = torch.tensor(_E2M1_MAGNITUDES, device=packed.device)
codes = torch.empty(512, dtype=torch.uint8, device=packed.device)
codes[0::2] = packed & 0xF # even element in the low nibble
codes[1::2] = packed >> 4
vals = mags[(codes & 7).long()] * torch.where(codes >= 8, -1.0, 1.0)
return (
vals.view(32, 16) * scales.view(torch.float8_e4m3fn).float().view(32, 1)
).flatten()
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
@pytest.mark.parametrize("compress_ratio", [1, 2])
def test_v41_rope_insert_nvfp4_record(compress_ratio: int):
"""The 288-byte V4.1 NVFP4 record: 256 B of e2m1 pairs then 32 e4m3 scales.
Scales are byte-exact against the reference (``amax / 6`` clamped to the
e4m3 range); values are checked as round-to-nearest on the e2m1 grid, whose
coarsest step is 2 between 4 and 6, so half a step is ``1.0 * scale``.
"""
from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import (
rope_quant_insert,
)
torch.manual_seed(13)
device = "cuda"
cache_block = 64
cache_stride = math.ceil(cache_block * 288 / 512) * 512
num_tokens = 12
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
latent = torch.randn(num_tokens, 512, dtype=torch.bfloat16, device=device)
angles = torch.randn(64, 32, device=device)
cos_sin = torch.cat((angles.cos(), angles.sin()), dim=-1)
cache_backing = torch.full((2, cache_stride), 165, dtype=torch.uint8, device=device)
cache = cache_backing.as_strided((2, cache_block, 288), (cache_stride, 288, 1))
slots = torch.arange(num_tokens, dtype=torch.int64, device=device)
slots[3] = -1 # covers the negative-slot skip
rope_quant_insert(latent, positions, cos_sin, cache, slots, compress_ratio)
untouched = torch.full_like(cache_backing, 165)
for t in range(num_tokens):
slot = slots[t].item()
pos = positions[t].item()
if slot < 0 or (pos + 1) % compress_ratio:
continue
page, row = divmod(slot, cache_block)
cs = cos_sin[pos // compress_ratio * compress_ratio]
rotated = _rotate_rope_tail(latent[t], cs)
amax = rotated.view(32, 16).abs().amax(-1)
scale = (amax / 6.0).clamp(2.0**-9, 448.0).to(torch.float8_e4m3fn)
got_scales = cache_backing[page, cache_block * 256 + row * 32 :][:32]
torch.testing.assert_close(got_scales, scale.view(torch.uint8), rtol=0, atol=0)
decoded = _decode_nvfp4_row(
cache_backing[page, row * 256 : (row + 1) * 256], got_scales
)
step = scale.float().repeat_interleave(16)
assert ((decoded - rotated).abs() <= step).all()
untouched[page, row * 256 : (row + 1) * 256] = cache_backing[
page, row * 256 : (row + 1) * 256
]
untouched[page, cache_block * 256 + row * 32 :][:32] = got_scales
# Rows the kernel must not have touched, page padding included.
torch.testing.assert_close(cache_backing, untouched, rtol=0, atol=0)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
def test_v41_nvfp4_gather_matches_insert():
"""The NVFP4 gather dequantizes exactly what the insert kernel wrote."""
from vllm.models.deepseek_v41.common.ops import dequantize_and_gather_k_cache
from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import (
rope_quant_insert,
)
torch.manual_seed(17)
device = "cuda"
block_size = 64
num_tokens = 70
num_blocks = 4
page_bytes = math.ceil(block_size * 288 / 512) * 512
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
latent = torch.randn(num_tokens, 512, dtype=torch.bfloat16, device=device)
angles = torch.randn(128, 32, device=device)
cos_sin = torch.cat((angles.cos(), angles.sin()), dim=-1)
backing = torch.zeros(num_blocks, page_bytes, dtype=torch.uint8, device=device)
cache = backing.as_strided((num_blocks, block_size, 288), (page_bytes, 288, 1))
slots = torch.arange(num_tokens, dtype=torch.int64, device=device)
rope_quant_insert(latent, positions, cos_sin, cache, slots, 1)
out = torch.zeros(1, num_tokens, 512, dtype=torch.bfloat16, device=device)
dequantize_and_gather_k_cache(
out,
cache,
seq_lens=torch.tensor([num_tokens], dtype=torch.int32, device=device),
gather_lens=None,
block_table=torch.arange(num_blocks, dtype=torch.int32, device=device).view(
1, -1
),
block_size=block_size,
offset=0,
)
for t in (0, 1, block_size, num_tokens - 1):
page, row = divmod(t, block_size)
expected = _decode_nvfp4_row(
backing[page, row * 256 : (row + 1) * 256],
backing[page, block_size * 256 + row * 32 :][:32],
)
torch.testing.assert_close(out[0, t].float(), expected, rtol=0, atol=0)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
@pytest.mark.parametrize("compress_ratio", [1, 2])
@pytest.mark.parametrize("store_fp8", [False, True])
@@ -0,0 +1,71 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Layout plumbing for DeepSeek V4.1 mega attention.
The kernel's Q and O layouts are produced by permuting ``wq_b`` rows and
``wo_a`` columns once at load, so what has to hold is that the permuted GEMMs
agree with a reference that permutes the activation instead.
"""
import torch
from vllm.models.deepseek_v41.common.ops.fused_layout import (
WV_GROUP_SIZE,
o_fused_chunk_permutation,
o_fused_permutation,
permute_q_to_fused,
permute_wo_a_,
permute_wq_b_,
q_fused_permutation,
)
HEAD_DIM = 512
def test_permuted_wq_b_gemm_matches_permuted_activation():
"""Permuting wq_b's rows makes the Q GEMM emit the fused layout directly."""
torch.manual_seed(0)
num_heads, q_lora_rank, num_tokens = 16, 64, 5
weight = torch.randn(num_heads * HEAD_DIM, q_lora_rank)
# One scale row per weight row, as an MXFP8 wq_b shard carries.
scale = torch.randint(
0, 256, (num_heads * HEAD_DIM, q_lora_rank // 32), dtype=torch.uint8
)
qr = torch.randn(num_tokens, q_lora_rank)
orig_weight, orig_scale = weight.clone(), scale.clone()
standard = (qr @ weight.T).view(num_tokens, num_heads, HEAD_DIM)
expected = permute_q_to_fused(standard)
permute_wq_b_(weight, scale, num_heads)
fused = (qr @ weight.T).view(num_tokens, num_heads, HEAD_DIM)
torch.testing.assert_close(fused, expected)
# The scale has to follow its row, or dequant reads another row's scale.
perm = q_fused_permutation(num_heads, HEAD_DIM)
torch.testing.assert_close(weight, orig_weight[perm])
torch.testing.assert_close(scale, orig_scale[perm])
def test_permuted_wo_a_consumes_fused_output():
"""Permuting wo_a's input columns lets it read the kernel's O layout."""
torch.manual_seed(1)
out_features, num_tokens = 32, 4
in_features = WV_GROUP_SIZE * HEAD_DIM
weight = torch.randn(out_features, in_features)
scale = torch.randint(0, 256, (out_features, in_features // 32), dtype=torch.uint8)
standard_o = torch.randn(num_tokens, in_features)
orig_weight, orig_scale = weight.clone(), scale.clone()
expected = standard_o @ weight.T
# The kernel emits the same values in the fused chunk order.
perm = o_fused_permutation(WV_GROUP_SIZE, HEAD_DIM)
fused_o = standard_o[:, perm]
permute_wo_a_(weight, scale, WV_GROUP_SIZE)
torch.testing.assert_close(weight, orig_weight[:, perm])
torch.testing.assert_close(
scale, orig_scale[:, o_fused_chunk_permutation(WV_GROUP_SIZE, HEAD_DIM)]
)
# Unlike wq_b, this permutation sits inside the 4096-term reduction, so the
# summation order changes and the result differs in the last few ulps.
torch.testing.assert_close(fused_o @ weight.T, expected, rtol=1e-3, atol=1e-3)
@@ -522,6 +522,189 @@ def test_kv_path_mxfp8_record_matches_reference(num_tokens: int, block_size: int
)
# ── Test 2d: mega-attention Q layout (is_q_interleaved) ──────────────────────
Q_CHUNK = 16
NUM_Q_CHUNKS = HEAD_DIM // Q_CHUNK # 32
@pytest.mark.parametrize("num_tokens", [1, 17, 2048])
@pytest.mark.parametrize("n_heads,q_head_padded", [(16, 64), (64, 128), (64, 0)])
def test_q_interleaved_pads_without_rope(
num_tokens: int, n_heads: int, q_head_padded: int
):
"""``is_q_interleaved`` with no norm or RoPE only zero-pads Q.
The mega-attention kernel norms and rotates Q itself, so every live
element must survive bit-exact while the padding heads -- which interleave
with the live ones inside each of the 32 head-dim chunks -- come back
zero. ``q_head_padded=0`` drops the Q pass entirely and returns nothing,
for a shard already at the kernel's head count.
"""
torch.manual_seed(5)
device = "cuda"
dtype = torch.bfloat16
block_size = 64
q = torch.randn(num_tokens, n_heads, HEAD_DIM, dtype=dtype, device=device)
kv = torch.randn(num_tokens, HEAD_DIM, dtype=dtype, device=device)
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
cos_sin_cache = make_cos_sin_cache(4096, ROPE_DIM, torch.float32, device)
num_blocks = (num_tokens + block_size - 1) // block_size + 1
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
k_cache = torch.zeros(
num_blocks, block_size * V41_HEAD_BYTES, dtype=torch.uint8, device=device
)
q_out = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
q,
kv,
k_cache,
slot_mapping,
positions,
cos_sin_cache,
q_head_padded,
1e-6,
block_size,
False, # apply_q_norm
True, # kv_mxfp8
False, # apply_q_rope
True, # is_q_interleaved
)
# The KV half is unaffected by the Q mode: it must still be inserted.
assert k_cache.any()
if q_head_padded == 0:
assert q_out.numel() == 0
return
assert q_out.shape == (num_tokens, q_head_padded, HEAD_DIM)
got = q_out.view(num_tokens, NUM_Q_CHUNKS, q_head_padded, Q_CHUNK)
want = q.view(num_tokens, NUM_Q_CHUNKS, n_heads, Q_CHUNK)
torch.testing.assert_close(got[:, :, :n_heads], want, rtol=0, atol=0)
assert (got[:, :, n_heads:] == 0).all()
@pytest.mark.parametrize("n_heads", [8, 16, 32])
def test_q_interleaved_survives_graph_replay(n_heads: int):
"""A captured fused insert refills its padded Q buffer on every replay.
The op allocates ``q_out`` itself, so the tensor a capture hands back has
to keep receiving the new padding when the graph is replayed over changed
input -- that is what lets the layer prepare Q inside the captured region
rather than in ``forward_mqa``.
"""
torch.manual_seed(6)
device = "cuda"
dtype = torch.bfloat16
block_size, num_tokens, q_head_padded = 64, 7, 64
q = torch.randn(num_tokens, n_heads, HEAD_DIM, dtype=dtype, device=device)
kv = torch.randn(num_tokens, HEAD_DIM, dtype=dtype, device=device)
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
cos_sin_cache = make_cos_sin_cache(4096, ROPE_DIM, torch.float32, device)
num_blocks = (num_tokens + block_size - 1) // block_size + 1
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
k_cache = torch.zeros(
num_blocks, block_size * V41_HEAD_BYTES, dtype=torch.uint8, device=device
)
def prepare():
return torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
q,
kv,
k_cache,
slot_mapping,
positions,
cos_sin_cache,
q_head_padded,
1e-6,
block_size,
False, # apply_q_norm
True, # kv_mxfp8
False, # apply_q_rope
True, # is_q_interleaved
)
# Capture takes its warmup on a side stream.
side = torch.cuda.Stream()
side.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(side):
for _ in range(3):
prepare()
torch.cuda.current_stream().wait_stream(side)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
q_out = prepare()
for factor in (0.25, -0.75):
q.copy_(torch.randn_like(q) * factor)
graph.replay()
torch.accelerator.synchronize()
got = q_out.view(num_tokens, NUM_Q_CHUNKS, q_head_padded, Q_CHUNK)
want = q.view(num_tokens, NUM_Q_CHUNKS, n_heads, Q_CHUNK)
torch.testing.assert_close(got[:, :, :n_heads], want, rtol=0, atol=0)
assert (got[:, :, n_heads:] == 0).all()
@pytest.mark.parametrize("n_heads", [16, 64])
@pytest.mark.parametrize("apply_q_norm", [False, True])
@pytest.mark.parametrize("apply_q_rope", [False, True])
def test_q_interleaved_is_orthogonal_to_norm_and_rope(
n_heads: int, apply_q_norm: bool, apply_q_rope: bool
):
"""The Q layout composes with whatever norm and RoPE it is given.
A lane owns the same 512-dim slice of its head in either layout -- only
the address differs -- so the head-major result permuted into the
interleaved layout must equal running the interleaved path on a permuted
input, bit for bit, for every norm/RoPE combination.
"""
from vllm.models.deepseek_v41.common.ops.fused_layout import permute_q_to_fused
torch.manual_seed(7)
device = "cuda"
dtype = torch.bfloat16
block_size, num_tokens, q_head_padded = 64, 5, 64
q = torch.randn(num_tokens, n_heads, HEAD_DIM, dtype=dtype, device=device)
kv = torch.randn(num_tokens, HEAD_DIM, dtype=dtype, device=device)
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
cos_sin_cache = make_cos_sin_cache(4096, ROPE_DIM, torch.float32, device)
num_blocks = (num_tokens + block_size - 1) // block_size + 1
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
def run(q_in, is_q_interleaved):
cache = torch.zeros(
num_blocks, block_size * V41_HEAD_BYTES, dtype=torch.uint8, device=device
)
out = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
q_in,
kv,
cache,
slot_mapping,
positions,
cos_sin_cache,
q_head_padded,
1e-6,
block_size,
apply_q_norm,
True, # kv_mxfp8
apply_q_rope,
is_q_interleaved,
)
return out, cache
head_major, cache_major = run(q, False)
interleaved, cache_inter = run(permute_q_to_fused(q), True)
torch.testing.assert_close(
interleaved, permute_q_to_fused(head_major), rtol=0, atol=0
)
# The Q layout must not leak into the KV half.
torch.testing.assert_close(cache_inter, cache_major, rtol=0, atol=0)
# ── Test 2b: DP padding (slot_mapping shorter than q/kv) ─────────────────────
+80 -18
View File
@@ -367,17 +367,21 @@ def test_deepseek_v4_mega_moe_padding_preserves_weights(monkeypatch, intermediat
[
pytest.param(128, 512, 128, True, False, id="aligned-block128"),
pytest.param(128, 512, 128, True, True, id="aligned-block128-mxfp8"),
pytest.param(5120, 2304, 32, False, False, id="deepseek-v41-flash"),
pytest.param(5120, 2304, 32, True, False, id="deepseek-v41-flash"),
pytest.param(128, 2304, 128, True, False, id="padded-block128"),
pytest.param(5120, 2304, 32, True, True, id="deepseek-v41-flash-mxfp8"),
pytest.param(128, 2304, 128, False, False, id="padded-packed-scale-fallback"),
],
)
def test_deepseek_v4_mega_moe_finalizes_native_shared_expert_weights(
monkeypatch, hidden_size, intermediate_size, block_size, fused, mxfp8
):
"""V4.1-Flash's routed padding must preserve the serial shared-expert fallback."""
"""Shared fusion preserves checkpoint channels and zeros padded channels."""
class FakeDeepGemm:
transformed_dims: list[tuple[int, int]] = []
scale_inputs: list[tuple[int, ...]] = []
scale_values: list[torch.Tensor] = []
@staticmethod
def get_symm_buffer_for_mega_moe(*args, num_shared_experts=0, **kwargs):
@@ -402,6 +406,7 @@ def test_deepseek_v4_mega_moe_finalizes_native_shared_expert_weights(
@classmethod
def transform_sf_into_required_layout(cls, sf, mn, k, *args, **kwargs):
cls.scale_inputs.append(tuple(sf.shape))
cls.scale_values.append(sf)
return torch.empty((sf.shape[0], mn, k // 128), dtype=torch.int32)
@classmethod
@@ -429,7 +434,10 @@ def test_deepseek_v4_mega_moe_finalizes_native_shared_expert_weights(
def fp8_parameter(*shape):
return torch.nn.Parameter(
torch.empty(*shape, dtype=torch.float8_e4m3fn), requires_grad=False
torch.empty(*shape, dtype=torch.uint8)
.random_(1, 119)
.view(torch.float8_e4m3fn),
requires_grad=False,
)
def scale_parameter(*shape, dtype=torch.int32):
@@ -460,21 +468,34 @@ def test_deepseek_v4_mega_moe_finalizes_native_shared_expert_weights(
linear.weight_block_size = (1, 32)
linear.weight_scale = torch.nn.Parameter(
linear.weight_scale_inv.view(torch.uint8)
.repeat_interleave(128, dim=0)
.repeat_interleave(4, dim=1),
.repeat_interleave(block_size, dim=0)
.repeat_interleave(block_size // 32, dim=1),
requires_grad=False,
)
del linear.weight_scale_inv
originals = []
for linear in (shared_experts.gate_up_proj, shared_experts.down_proj):
scale = linear.weight_scale if mxfp8 else linear.weight_scale_inv
if not fused:
# Already packed scales cannot be padded as checkpoint UE8M0 bytes.
linear.weight_scale_inv = scale_parameter(
linear.weight.shape[0], linear.weight.shape[1] // 128
)
continue
scale.data.view(torch.uint8).random_(124, 130)
block_m, block_k = linear.weight_block_size
scale_1x32 = (
experts._ue8m0_uint8_to_float(scale.view(torch.uint8))
.repeat_interleave(block_m, dim=0)
.repeat_interleave(block_k // 32, dim=1)
)
originals.append((linear.weight.view(torch.uint8).clone(), scale_1x32))
monkeypatch.setattr("vllm.utils.deep_gemm._import_deep_gemm", lambda: FakeDeepGemm)
original_gate_up_ptr = shared_experts.gate_up_proj.weight.data_ptr()
original_down_ptr = shared_experts.down_proj.weight.data_ptr()
experts.finalize_weights(shared_experts)
assert FakeDeepGemm.scale_inputs[-2:] == [
(1, 2 * intermediate_size, hidden_size // 32),
(1, hidden_size, intermediate_size // 32),
]
assert experts.has_fused_shared_experts is fused
if not fused:
assert experts.intermediate_size == 2560
@@ -490,15 +511,56 @@ def test_deepseek_v4_mega_moe_finalizes_native_shared_expert_weights(
return
assert FakeDeepGemm.transformed_dims == [(3, 3), (2, 2)]
assert shared_experts.gate_up_proj.weight.data_ptr() != original_gate_up_ptr
assert (
experts._transformed_shared_l1_weights[0].data_ptr()
== shared_experts.gate_up_proj.weight.data_ptr()
)
assert (
experts._transformed_shared_l2_weights[0].data_ptr()
== shared_experts.down_proj.weight.data_ptr()
)
padded_size = experts.intermediate_size
assert FakeDeepGemm.scale_inputs[-2:] == [
(1, 2 * padded_size, hidden_size // 32),
(1, hidden_size, padded_size // 32),
]
for actual, original, fill in (
(
experts._transformed_shared_l1_weights[0].view(torch.uint8),
originals[0][0],
0,
),
(FakeDeepGemm.scale_values[-2][0], originals[0][1], 1),
):
actual = actual.unflatten(0, (2, padded_size))
assert torch.equal(
actual[:, :intermediate_size], original.unflatten(0, (2, intermediate_size))
)
assert torch.all(actual[:, intermediate_size:] == fill)
for actual, original, fill in (
(
experts._transformed_shared_l2_weights[0].view(torch.uint8),
originals[1][0],
0,
),
(FakeDeepGemm.scale_values[-1][0], originals[1][1], 1),
):
assert torch.equal(actual[:, : original.shape[1]], original)
assert torch.all(actual[:, original.shape[1] :] == fill)
if padded_size != intermediate_size:
# Generic linear post-load hooks still receive checkpoint-shaped weights.
for linear, (weight, _) in zip(
(shared_experts.gate_up_proj, shared_experts.down_proj), originals
):
assert torch.equal(linear.weight.view(torch.uint8), weight)
assert shared_experts.gate_up_proj.weight.data_ptr() == original_gate_up_ptr
assert shared_experts.down_proj.weight.data_ptr() == original_down_ptr
else:
assert shared_experts.gate_up_proj.weight.data_ptr() != original_gate_up_ptr
assert (
experts._transformed_shared_l1_weights[0].data_ptr()
== shared_experts.gate_up_proj.weight.data_ptr()
)
assert (
experts._transformed_shared_l2_weights[0].data_ptr()
== shared_experts.down_proj.weight.data_ptr()
)
transformed_l1 = experts._transformed_shared_l1_weights
experts.finalize_weights(shared_experts)
assert experts._transformed_shared_l1_weights is transformed_l1
@pytest.mark.parametrize("fused", [False, True])
+30 -1
View File
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import contextlib
import hashlib
import struct
import uuid
from io import BytesIO
from pathlib import Path
@@ -8,7 +10,7 @@ from pathlib import Path
import numpy as np
import pytest
import torch
from PIL import Image, ImageDraw
from PIL import Image, ImageDraw, ImageOps
from vllm.config.multimodal import MMHasherAlgorithm
from vllm.multimodal.hasher import MultiModalHasher
@@ -199,6 +201,33 @@ def test_hash_image_exif_id():
)
def test_hash_image_malformed_exif():
# Test that images with malformed EXIF headers (e.g. invalid TIFF header)
# do not raise an unhandled exception during hashing and fall back to image data.
buf = BytesIO()
Image.new("RGB", (64, 48)).save(buf, "JPEG")
jpg = buf.getvalue()
rest = jpg[2:]
rest = rest[2 + struct.unpack(">H", rest[2:4])[0] :]
payload = b"Exif\x00\x00XXXX\x00\x00\x00\x08" + bytes(32)
data = b"\xff\xd8\xff\xe1" + struct.pack(">H", len(payload) + 2) + payload + rest
image = Image.open(BytesIO(data))
with contextlib.suppress(Exception):
image = ImageOps.exif_transpose(image)
image.load()
hasher = MultiModalHasher
# Should hash without raising SyntaxError or any other exception
hash_val = hasher.hash_kwargs("blake3", image=image)
assert isinstance(hash_val, str) and len(hash_val) > 0
# Also verify MediaWithBytes wrapping the image with malformed EXIF
media_item = MediaWithBytes(image, data)
hash_media = hasher.hash_kwargs("blake3", image=media_item)
assert isinstance(hash_media, str) and len(hash_media) > 0
def _rgba_png_bytes() -> bytes:
image = Image.new("RGBA", (8, 8), (255, 0, 0, 128))
buf = BytesIO()
+108
View File
@@ -0,0 +1,108 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Reasoning-boundary tests for the GLM-4.7 parser.
GLM can transition straight from reasoning into a tool call without ever
emitting ``</think>``, so ``<tool_call>`` is an implicit reasoning
terminator. These cover that path plus the multi-turn prompts where an
earlier turn's markers must not be read as the current turn's state.
"""
import pytest
from tests.parser.engine.conftest import make_mock_tokenizer
from vllm.parser.glm47_moe import (
THINK_END,
THINK_START,
TOOL_CALL_END,
TOOL_CALL_START,
Glm47MoeParser,
)
THINK_S, THINK_E = 1000, 1001
TOOL_S, TOOL_E = 1002, 1003
ASSISTANT, OBSERVATION, USER = 1004, 1005, 1006
TEXT = 42 # stand-in for an ordinary reasoning/content token
VOCAB = {
THINK_START: THINK_S,
THINK_END: THINK_E,
TOOL_CALL_START: TOOL_S,
TOOL_CALL_END: TOOL_E,
"<|assistant|>": ASSISTANT,
"<|observation|>": OBSERVATION,
"<|user|>": USER,
}
@pytest.fixture
def parser():
return Glm47MoeParser(make_mock_tokenizer(VOCAB))
@pytest.fixture
def no_thinking_parser():
return Glm47MoeParser(
make_mock_tokenizer(VOCAB),
chat_template_kwargs={"thinking": False},
)
class TestIsReasoningEnd:
def test_open_reasoning(self, parser):
assert not parser.is_reasoning_end([THINK_S, TEXT])
def test_think_end(self, parser):
assert parser.is_reasoning_end([THINK_S, TEXT, THINK_E, TEXT])
def test_tool_call_without_think_end(self, parser):
"""Reasoning -> tool call with no ``</think>`` still ends reasoning."""
assert parser.is_reasoning_end([THINK_S, TEXT, TOOL_S, TEXT])
def test_previous_turn_tool_call_ignored(self, parser):
"""A finished tool call from an earlier turn says nothing about the
turn currently being generated."""
prompt = [THINK_S, TEXT, TOOL_S, TEXT, TOOL_E, OBSERVATION, TEXT, ASSISTANT]
assert not parser.is_reasoning_end(prompt)
def test_previous_turn_think_end_ignored(self, parser):
prompt = [THINK_S, TEXT, THINK_E, TEXT, USER, TEXT, ASSISTANT]
assert not parser.is_reasoning_end(prompt)
def test_empty_input(self, parser):
assert not parser.is_reasoning_end([])
def test_thinking_disabled(self, no_thinking_parser):
assert no_thinking_parser.is_reasoning_end([THINK_S, TEXT])
class TestExtractContentIds:
def test_think_end_wins_over_later_tool_call(self, parser):
"""Content between ``</think>`` and a tool call must survive."""
ids = [THINK_S, TEXT, THINK_E, 20, 21, TOOL_S, 30, TOOL_E]
assert parser.extract_content_ids(ids) == [20, 21, TOOL_S, 30, TOOL_E]
def test_every_tool_call_after_think_end_kept(self, parser):
ids = [THINK_E, 20, TOOL_S, 30, TOOL_E, 21, TOOL_S, 31, TOOL_E]
assert parser.extract_content_ids(ids) == ids[1:]
def test_falls_back_to_tool_call(self, parser):
"""Without ``</think>``, content starts at the opener itself so the
tool parser still receives a well-formed call."""
ids = [THINK_S, TEXT, TOOL_S, 30, TOOL_E]
assert parser.extract_content_ids(ids) == [TOOL_S, 30, TOOL_E]
def test_falls_back_to_first_tool_call_of_turn(self, parser):
ids = [THINK_S, TEXT, TOOL_S, 30, TOOL_E, TOOL_S, 31, TOOL_E]
assert parser.extract_content_ids(ids) == ids[2:]
def test_previous_turn_tool_call_ignored(self, parser):
ids = [TOOL_S, 30, TOOL_E, OBSERVATION, TEXT, ASSISTANT, THINK_S, TEXT]
assert parser.extract_content_ids(ids) == ids
def test_no_markers_returns_input_ids(self, parser):
assert parser.extract_content_ids([20, 21]) == [20, 21]
def test_thinking_disabled(self, no_thinking_parser):
ids = [THINK_S, TEXT, TOOL_S]
assert no_thinking_parser.extract_content_ids(ids) == ids
@@ -0,0 +1,163 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""`prompt_embeds` payloads that are not tensor files must be client errors.
`prompt_embeds` is a base64 blob the caller writes, and every other way of
getting it wrong -- wrong rank, wrong hidden_size, wrong dtype, not a tensor at
all -- already answers 400. A payload that `torch.load` cannot read at all was
the exception: torch has no single error type for that (the zip reader raises
`RuntimeError`, the legacy pickle path raises an unpickling error, `KeyError` or
`IndexError`, an empty payload raises `EOFError`), none of them is a
`ValueError`, and the entrypoints' fallback therefore mapped them to 500.
The assertions go through `create_error_response`, which is what actually turns
an exception into a status code, so they pin the HTTP answer rather than the
exception class alone.
"""
import io
import zipfile
from unittest.mock import patch
import pybase64 as base64
import pytest
import torch
from vllm.entrypoints.serve.exception_handling.error_response import (
create_error_response,
)
from vllm.exceptions import VLLMValidationError
from vllm.renderers.embed_utils import (
_MAX_EMBED_ERROR_REASON_CHARS,
safe_load_prompt_embeds,
)
@pytest.fixture
def model_config():
from vllm.config import ModelConfig
return ModelConfig(
model="facebook/opt-125m",
tokenizer="facebook/opt-125m",
tokenizer_mode="auto",
trust_remote_code=False,
dtype="float32",
seed=0,
enable_prompt_embeds=True,
)
def _encode(raw: bytes) -> bytes:
return base64.b64encode(raw)
def _zip_without_a_tensor() -> bytes:
buffer = io.BytesIO()
with zipfile.ZipFile(buffer, "w") as archive:
archive.writestr("junk.txt", "hello")
return buffer.getvalue()
def _saved_tensor(tensor: torch.Tensor) -> bytes:
buffer = io.BytesIO()
torch.save(tensor, buffer)
return buffer.getvalue()
UNPARSABLE_PAYLOADS = [
pytest.param(b"", id="empty"),
pytest.param(b"\x80", id="single-byte"),
pytest.param(b"\x00\x01\x02\x03not-a-tensor", id="random-bytes"),
pytest.param(b"hello world this is not a torch file", id="ascii-text"),
pytest.param(b"PK\x03\x04" + b"\x00" * 20, id="truncated-zip"),
pytest.param(_zip_without_a_tensor(), id="zip-without-a-tensor"),
# pickle.dumps(42): a valid pickle stream that is not a torch checkpoint.
# Spelled out rather than built, so this file does not import pickle.
pytest.param(b"\x80\x04K*.", id="legacy-pickle-of-an-int"),
]
@pytest.mark.parametrize("payload", UNPARSABLE_PAYLOADS)
def test_unreadable_prompt_embeds_payload_is_a_client_error(model_config, payload):
with pytest.raises(VLLMValidationError) as excinfo:
safe_load_prompt_embeds(model_config, _encode(payload))
assert excinfo.value.parameter == "prompt_embeds"
assert int(create_error_response(excinfo.value).error.code) == 400
@pytest.mark.parametrize("payload", UNPARSABLE_PAYLOADS)
def test_the_error_body_stays_bounded(model_config, payload):
"""torch's reason is built from the caller's bytes, so it must be truncated.
An unpickler that reports the offending global, for instance, quotes a name
the caller chose; without a bound a large one would be reflected whole.
"""
padding = b"A" * 100_000
with pytest.raises(VLLMValidationError) as excinfo:
safe_load_prompt_embeds(model_config, _encode(payload + padding))
assert len(str(excinfo.value)) < 500
def test_a_long_reason_is_truncated_and_says_how_much_was_left_out(model_config):
"""The cap is pinned against a reason of known length, not a real payload.
Which of torch's messages happen to exceed the cap is torch's business and
changes between releases, so the reason is injected here: that keeps the
assertion about *this* code's truncation rule rather than about whichever
error text the installed torch produces.
"""
reason = "E" * (_MAX_EMBED_ERROR_REASON_CHARS + 37)
with (
patch(
"vllm.renderers.embed_utils.torch.load", side_effect=RuntimeError(reason)
),
pytest.raises(VLLMValidationError) as excinfo,
):
safe_load_prompt_embeds(model_config, _encode(b"ignored"))
message = str(excinfo.value)
assert "E" * _MAX_EMBED_ERROR_REASON_CHARS in message
assert "E" * (_MAX_EMBED_ERROR_REASON_CHARS + 1) not in message
assert "... (37 more characters truncated)" in message
def test_a_short_reason_is_passed_through_whole(model_config):
"""A reason that fits is not annotated -- the count only appears when it is
telling the reader something."""
reason = "invalid magic number; corrupt file?"
assert len(reason) <= _MAX_EMBED_ERROR_REASON_CHARS
with (
patch(
"vllm.renderers.embed_utils.torch.load", side_effect=RuntimeError(reason)
),
pytest.raises(VLLMValidationError) as excinfo,
):
safe_load_prompt_embeds(model_config, _encode(b"ignored"))
message = str(excinfo.value)
assert reason in message
assert "truncated" not in message
def test_a_well_formed_payload_still_loads(model_config):
"""Positive control: the healthy path is untouched."""
hidden_size = model_config.get_hidden_size()
tensor = torch.zeros(3, hidden_size, dtype=torch.float32)
loaded = safe_load_prompt_embeds(model_config, _encode(_saved_tensor(tensor)))
assert loaded.shape == (3, hidden_size)
assert loaded.dtype == model_config.dtype
def test_a_payload_that_is_not_base64_is_still_a_client_error(model_config):
"""Positive control: the base64 stage already answered 400 and still does."""
with pytest.raises(ValueError) as excinfo:
safe_load_prompt_embeds(model_config, b"!!!!not base64!!!!")
assert int(create_error_response(excinfo.value).error.code) == 400
@@ -11,6 +11,7 @@ import pytest
import torch
from vllm.config import ModelConfig
from vllm.exceptions import VLLMValidationError
from vllm.multimodal.media import AudioEmbeddingMediaIO, ImageEmbeddingMediaIO
from vllm.renderers.embed_utils import safe_load_prompt_embeds
from vllm.utils.sparse_utils import check_sparse_tensor_invariants_threadsafe
@@ -54,7 +55,7 @@ class TestNegativeControlWithoutRace:
def test_malicious_sparse_rejected_by_prompt_loader(self, model_config):
encoded = _encode_tensor(_create_malicious_sparse_tensor())
with pytest.raises((RuntimeError, ValueError)):
with pytest.raises((RuntimeError, ValueError, VLLMValidationError)):
safe_load_prompt_embeds(model_config, encoded)
def test_malicious_sparse_rejected_by_image_loader(self):
@@ -147,7 +148,7 @@ class TestConcurrentRaceProtection:
try:
safe_load_prompt_embeds(model_config, malicious_encoded)
bypassed.append(True)
except (RuntimeError, ValueError):
except (RuntimeError, ValueError, VLLMValidationError):
rejected.append(True)
with ThreadPoolExecutor(max_workers=num_workers) as pool:
@@ -192,7 +193,7 @@ class TestGlobalFlagRestoration:
initial = torch.sparse.check_sparse_tensor_invariants.is_enabled()
malicious_encoded = _encode_tensor(_create_malicious_sparse_tensor())
with pytest.raises((RuntimeError, ValueError)):
with pytest.raises((RuntimeError, ValueError, VLLMValidationError)):
safe_load_prompt_embeds(model_config, malicious_encoded)
assert torch.sparse.check_sparse_tensor_invariants.is_enabled() == initial
@@ -203,7 +204,7 @@ class TestGlobalFlagRestoration:
malicious_encoded = _encode_tensor(_create_malicious_sparse_tensor())
def attempt_load(_):
with contextlib.suppress(RuntimeError, ValueError):
with contextlib.suppress(RuntimeError, ValueError, VLLMValidationError):
safe_load_prompt_embeds(model_config, malicious_encoded)
with ThreadPoolExecutor(max_workers=4) as pool:
@@ -244,7 +245,7 @@ class TestCrossLoaderLockSharing:
# serialization — one finishes before the other starts), AND invalid
# tensors are still rejected.
malicious = _encode_tensor(_create_malicious_sparse_tensor())
with pytest.raises((RuntimeError, ValueError)):
with pytest.raises((RuntimeError, ValueError, VLLMValidationError)):
safe_load_prompt_embeds(model_config, malicious)
with pytest.raises((RuntimeError, ValueError)):
io_handler.load_base64("", malicious.decode("utf-8"))
@@ -117,8 +117,10 @@ class TestPromptEmbedsValidation:
malicious_tensor = _create_malicious_sparse_tensor()
encoded = _encode_tensor(malicious_tensor)
# Should raise RuntimeError due to invalid sparse tensor
with pytest.raises((RuntimeError, ValueError)) as exc_info:
# The invariant check raises RuntimeError from inside torch.load;
# safe_load_prompt_embeds now reports that as a client error so the
# caller gets a 400 rather than the 500 a bare RuntimeError produced.
with pytest.raises(VLLMValidationError) as exc_info:
safe_load_prompt_embeds(model_config, encoded)
# Error should indicate sparse tensor validation failure
@@ -137,7 +139,7 @@ class TestPromptEmbedsValidation:
)
encoded = _encode_tensor(malicious_tensor)
with pytest.raises((RuntimeError, ValueError)):
with pytest.raises(VLLMValidationError):
safe_load_prompt_embeds(model_config, encoded)
def test_negative_indices_rejected(self, model_config):
@@ -152,7 +154,7 @@ class TestPromptEmbedsValidation:
)
encoded = _encode_tensor(malicious_tensor)
with pytest.raises((RuntimeError, ValueError)):
with pytest.raises(VLLMValidationError):
safe_load_prompt_embeds(model_config, encoded)
def test_hidden_size_mismatch_rejected(self, model_config):
@@ -379,8 +381,8 @@ class TestSparseTensorValidationIntegration:
# Step 1-2: Attacker creates malicious payload
attack_payload = _encode_tensor(_create_malicious_sparse_tensor())
# Step 3-4: Server processes and should reject
with pytest.raises((RuntimeError, ValueError)):
# Step 3-4: Server processes and should reject, as a client error
with pytest.raises(VLLMValidationError):
safe_load_prompt_embeds(model_config, attack_payload)
def test_attack_scenario_chat_api_image(self):
+50 -2
View File
@@ -65,7 +65,7 @@ def test_cpu_target_selects_cpu_platform_from_non_cpu_wheel(
rocm_plugin.assert_not_called()
def test_platform_detection_logs_zentorch_import_failure(caplog):
def test_broken_zentorch_falls_back_to_cpu_platform(caplog):
original_import = builtins.__import__
def import_with_broken_zentorch(name, *args, **kwargs):
@@ -88,6 +88,54 @@ def test_platform_detection_logs_zentorch_import_failure(caplog):
):
platform = resolve_current_platform_cls_qualname()
assert platform == "vllm.platforms.cpu.CpuPlatform"
assert "zentorch failed to import" in caplog.text
assert "OSError: incompatible shared library" in caplog.text
def test_zentorch_failure_other_than_oserror_is_not_recovered(caplog):
original_import = builtins.__import__
def import_with_broken_zentorch(name, *args, **kwargs):
if name == "zentorch":
raise RuntimeError("unknown zentorch failure")
return original_import(name, *args, **kwargs)
with (
patch("vllm.platforms.envs.VLLM_TARGET_DEVICE", "cuda"),
patch.dict(
"vllm.platforms.builtin_platform_plugins",
{"cpu": cpu_platform_plugin},
clear=True,
),
patch("vllm.platforms.load_plugins_by_group", return_value={}),
patch("vllm.platforms.vllm_version_matches_substr", return_value=True),
patch("vllm.platforms._is_amd_zen_cpu", return_value=True),
patch.object(builtins, "__import__", side_effect=import_with_broken_zentorch),
caplog.at_level(logging.DEBUG, logger="vllm.platforms"),
):
platform = resolve_current_platform_cls_qualname()
assert platform == "vllm.platforms.interface.UnspecifiedPlatform"
assert "RuntimeError: unknown zentorch failure" in caplog.text
def test_platform_plugin_failure_is_logged(caplog):
def failing_plugin():
raise RuntimeError("plugin exploded")
with (
patch("vllm.platforms.envs.VLLM_TARGET_DEVICE", "cuda"),
patch.dict(
"vllm.platforms.builtin_platform_plugins",
{"cpu": failing_plugin},
clear=True,
),
patch("vllm.platforms.load_plugins_by_group", return_value={}),
caplog.at_level(logging.DEBUG, logger="vllm.platforms"),
):
platform = resolve_current_platform_cls_qualname()
assert platform == "vllm.platforms.interface.UnspecifiedPlatform"
assert "Platform plugin cpu failed during detection" in caplog.text
assert "OSError: incompatible shared library" in caplog.text
assert "RuntimeError: plugin exploded" in caplog.text
+189
View File
@@ -0,0 +1,189 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Cgroup memory readers and SHM allocation preflight checks."""
import sys
from io import StringIO
from unittest import mock
import pytest
from vllm.utils import cpu_resource_utils as cru
from vllm.utils.mem_constants import GiB_bytes
_V2_LIMIT_PATH = "/sys/fs/cgroup/memory.max"
_V2_USAGE_PATH = "/sys/fs/cgroup/memory.current"
_V1_LIMIT_PATH = "/sys/fs/cgroup/memory/memory.limit_in_bytes"
_V1_USAGE_PATH = "/sys/fs/cgroup/memory/memory.usage_in_bytes"
@pytest.fixture(autouse=True)
def _clear_cgroup_cache():
cru.get_cgroup_memory_limit.cache_clear()
yield
cru.get_cgroup_memory_limit.cache_clear()
def _stub_files(monkeypatch, files: dict):
"""Stub ``open()`` for a fixed set of paths; ``None`` -> OSError."""
real_open = open
def fake_open(path, *args, **kwargs):
if path in files:
content = files[path]
if content is None:
raise OSError(f"no such file: {path}")
return StringIO(content)
return real_open(path, *args, **kwargs)
monkeypatch.setattr("builtins.open", fake_open)
def test_cgroup_v2_limit_and_usage(monkeypatch):
monkeypatch.setattr(sys, "platform", "linux")
_stub_files(
monkeypatch,
{
_V2_LIMIT_PATH: f"{20 * GiB_bytes}\n",
_V2_USAGE_PATH: f"{5 * GiB_bytes}\n",
},
)
assert cru.get_cgroup_memory_limit() == 20 * GiB_bytes
assert cru.get_cgroup_memory_usage() == 5 * GiB_bytes
def test_cgroup_v2_unlimited_falls_back_to_v1(monkeypatch):
monkeypatch.setattr(sys, "platform", "linux")
_stub_files(
monkeypatch,
{
_V2_LIMIT_PATH: "max\n",
_V1_LIMIT_PATH: f"{8 * GiB_bytes}\n",
_V1_USAGE_PATH: f"{1 * GiB_bytes}\n",
},
)
assert cru.get_cgroup_memory_limit() == 8 * GiB_bytes
assert cru.get_cgroup_memory_usage() == 1 * GiB_bytes
def test_cgroup_v1_limit_and_usage(monkeypatch):
monkeypatch.setattr(sys, "platform", "linux")
_stub_files(
monkeypatch,
{
_V2_LIMIT_PATH: None,
_V1_LIMIT_PATH: f"{8 * GiB_bytes}\n",
_V1_USAGE_PATH: f"{2 * GiB_bytes}\n",
},
)
assert cru.get_cgroup_memory_limit() == 8 * GiB_bytes
assert cru.get_cgroup_memory_usage() == 2 * GiB_bytes
def test_cgroup_v1_unlimited_sentinel_is_ignored(monkeypatch):
"""An unlimited cgroup v1 sentinel must not be treated as a real limit."""
monkeypatch.setattr(sys, "platform", "linux")
_stub_files(
monkeypatch,
{_V2_LIMIT_PATH: None, _V1_LIMIT_PATH: f"{(1 << 63) - 1}\n"},
)
assert cru.get_cgroup_memory_limit() is None
assert cru.get_cgroup_memory_usage() is None
def test_cgroup_no_limit_files_present(monkeypatch):
monkeypatch.setattr(sys, "platform", "linux")
_stub_files(monkeypatch, {_V2_LIMIT_PATH: None, _V1_LIMIT_PATH: None})
assert cru.get_cgroup_memory_limit() is None
assert cru.get_cgroup_memory_usage() is None
def test_cgroup_skipped_on_non_linux(monkeypatch):
monkeypatch.setattr(sys, "platform", "darwin")
_stub_files(monkeypatch, {_V2_LIMIT_PATH: f"{1 * GiB_bytes}\n"})
assert cru.get_cgroup_memory_limit() is None
assert cru.get_cgroup_memory_usage() is None
def test_cgroup_limit_is_cached_until_explicitly_cleared(monkeypatch):
monkeypatch.setattr(sys, "platform", "linux")
files = {
_V2_LIMIT_PATH: f"{1 * GiB_bytes}\n",
_V2_USAGE_PATH: f"{512 << 20}\n",
}
_stub_files(monkeypatch, files)
assert cru.get_cgroup_memory_limit() == 1 * GiB_bytes
files[_V2_LIMIT_PATH] = f"{2 * GiB_bytes}\n"
assert cru.get_cgroup_memory_limit() == 1 * GiB_bytes
cru.get_cgroup_memory_limit.cache_clear()
assert cru.get_cgroup_memory_limit() == 2 * GiB_bytes
def test_cgroup_usage_is_read_without_cache(monkeypatch):
monkeypatch.setattr(sys, "platform", "linux")
files = {
_V2_LIMIT_PATH: f"{1 * GiB_bytes}\n",
_V2_USAGE_PATH: f"{512 << 20}\n",
}
_stub_files(monkeypatch, files)
assert cru.get_cgroup_memory_usage() == 512 << 20
files[_V2_USAGE_PATH] = f"{768 << 20}\n"
assert cru.get_cgroup_memory_usage() == 768 << 20
def test_check_cgroup_memory_available_warns_on_low_headroom(monkeypatch):
monkeypatch.setattr(cru, "get_cgroup_memory_limit", lambda: 1 << 30)
monkeypatch.setattr(cru, "get_cgroup_memory_usage", lambda: 512 << 20)
with (
mock.patch.object(cru.logger, "debug") as log_debug,
mock.patch.object(cru.logger, "warning") as log_warning,
):
cru.check_cgroup_memory_available(600 << 20, "mmap")
log_debug.assert_not_called()
log_warning.assert_called_once()
assert "current headroom is below" in log_warning.call_args.args[-1]
def test_check_cgroup_memory_available_logs_success_at_debug(monkeypatch):
monkeypatch.setattr(cru, "get_cgroup_memory_limit", lambda: 1 << 30)
monkeypatch.setattr(cru, "get_cgroup_memory_usage", lambda: 512 << 20)
with (
mock.patch.object(cru.logger, "debug") as log_debug,
mock.patch.object(cru.logger, "warning") as log_warning,
):
cru.check_cgroup_memory_available(256 << 20, "mmap")
log_debug.assert_called_once()
log_warning.assert_not_called()
@pytest.mark.parametrize(
("limit", "usage"),
[(None, 512 << 20), (1 << 30, None)],
)
def test_check_cgroup_memory_available_skips_without_snapshot(
monkeypatch, limit, usage
):
monkeypatch.setattr(cru, "get_cgroup_memory_limit", lambda: limit)
monkeypatch.setattr(cru, "get_cgroup_memory_usage", lambda: usage)
with (
mock.patch.object(cru.logger, "debug") as log_debug,
mock.patch.object(cru.logger, "warning") as log_warning,
):
cru.check_cgroup_memory_available(1 << 60, "mmap")
log_debug.assert_not_called()
log_warning.assert_not_called()
@@ -20,10 +20,10 @@ kernel is documented to replicate.
import torch
# Bootstrap the glm5next package before entering the indexer module: its
# kpool_compress import runs glm5next/__init__, which pulls model ->
# attention -> back into sparse_attn_indexer_kpool (attention.py imports the
# class at module scope). Production always enters via attention.py first.
# Bootstrap the glm5next package before importing kpool_compress directly: its
# import runs glm5next/__init__, which pulls model -> attention -> the indexer
# dispatcher (sparse_indexer) -> back into the nvidia/amd sparse_indexer
# modules. Production always enters via attention.py first.
# isort: off
import vllm.models.glm5next # noqa: F401
from vllm.models.glm5next.nvidia.ops.kpool_compress import ( # noqa: E402
@@ -32,13 +32,17 @@ from vllm.models.glm5next.nvidia.ops.kpool_compress import ( # noqa: E402
)
# isort: on
import vllm.model_executor.layers.sparse_attn_indexer_kpool as indexer_mod
from vllm.model_executor.layers.sparse_attn_indexer_kpool import (
from vllm.models.glm5next.common.sparse_indexer import (
_decode_topk_seq_lens,
_fill_short_decode_causal_indices,
)
from vllm.platforms import current_platform
if current_platform.is_rocm():
from vllm.models.glm5next.amd import sparse_indexer as indexer_mod
else:
from vllm.models.glm5next.nvidia import sparse_indexer as indexer_mod
KPOOL = 4
TOPK_TOKENS = 16
SELECT_K = TOPK_TOKENS // KPOOL
+38
View File
@@ -279,6 +279,44 @@ def make_request(
)
@pytest.mark.parametrize("dcp", [1, 4])
def test_effective_attention_block_size_matches_events(dcp):
from vllm.distributed.kv_events import BlockStored
from vllm.v1.engine.core import EngineCore
config = KVCacheConfig(
num_blocks=32,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(["attention"], new_kv_cache_spec()),
],
)
manager = KVCacheManager(
generate_scheduler_kv_cache_config([config]),
max_model_len=256,
scheduler_block_size=16 * dcp,
hash_block_size=16 * dcp,
dcp_world_size=dcp,
enable_kv_cache_events=True,
)
core = EngineCore.__new__(EngineCore)
core.vllm_config = SimpleNamespace(cache_config=CacheConfig(block_size=16))
core.scheduler = SimpleNamespace(kv_cache_manager=manager)
core._initialize_effective_attention_block_size()
block_size = core.vllm_config.cache_config.effective_attention_block_size
assert block_size == 16 * dcp
request = make_request(
"block-size", list(range(64)), block_size=16 * dcp, hash_fn=sha256
)
assert manager.allocate_slots(request, 64) is not None
assert [
event.block_size
for event in manager.take_events()
if isinstance(event, BlockStored)
] == [block_size]
def new_kv_cache_spec(
block_size=16,
num_kv_heads=2,
@@ -179,6 +179,63 @@ def test_mamba_retirement_bounds_prefill_states(block_size, in_flight_chunks):
assert pool.get_num_free_blocks() == initial_free
@pytest.mark.parametrize("block_size", [896, 1536])
@pytest.mark.parametrize("num_speculative_blocks", [0, 1, 4])
@pytest.mark.parametrize("prompt_tokens", [25121, 704547])
def test_mamba_checkpoint_admission_matches_allocation(
block_size, num_speculative_blocks, prompt_tokens
):
"""Checkpoint admission must match the subsequent physical allocation."""
spec = MambaSpec(
block_size=block_size,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
num_speculative_blocks=num_speculative_blocks,
num_prefill_checkpoint_blocks=1,
prefill_checkpoint_alignment=64,
)
pool = BlockPool(
num_gpu_blocks=2048,
enable_caching=True,
hash_block_size=128,
)
manager = MambaManager(
spec,
block_pool=pool,
enable_caching=True,
kv_cache_group_id=0,
scheduler_block_size=block_size,
)
request_id = "prefill"
computed_tokens = 23040
def estimate(num_tokens, total_computed_tokens, apply_admission_cap):
return manager.get_num_blocks_to_allocate(
request_id=request_id,
num_tokens=num_tokens,
new_computed_blocks=[],
total_computed_tokens=total_computed_tokens,
num_local_computed_tokens=total_computed_tokens,
num_tokens_main_model=num_tokens,
apply_admission_cap=apply_admission_cap,
)
estimate(computed_tokens, 0, False)
manager.allocate_new_blocks(request_id, computed_tokens, computed_tokens)
assert request_id in manager._allocated_block_reqs
admission_estimate = estimate(prompt_tokens, computed_tokens, True)
allocation_estimate = estimate(prompt_tokens, computed_tokens, False)
assert request_id in manager._checkpoints
free_before = pool.get_num_free_blocks()
manager.allocate_new_blocks(request_id, prompt_tokens, prompt_tokens)
allocated = free_before - pool.get_num_free_blocks()
assert admission_estimate == allocation_estimate == allocated
def get_sliding_window_manager(
sliding_window_spec,
block_pool,
+1 -1
View File
@@ -131,7 +131,7 @@ SMOKE_CONFIGS = [
id="dflash",
),
pytest.param(
"Qwen/Qwen3.5-0.8B-Base",
"Qwen/Qwen3.5-0.8B",
{
"method": "mtp",
"num_speculative_tokens": 3,
+28 -3
View File
@@ -158,7 +158,10 @@ def test_mp_client_uses_env_timeout(monkeypatch: pytest.MonkeyPatch):
local_engines_only=False,
enable_elastic_ep=False,
)
vllm_config = SimpleNamespace(parallel_config=parallel_config)
vllm_config = SimpleNamespace(
parallel_config=parallel_config,
cache_config=SimpleNamespace(effective_attention_block_size=None),
)
client = core_client_mod.MPClient(
asyncio_mode=False,
@@ -368,10 +371,15 @@ def test_dplb_finished_requests_release_inflight():
assert req.request_id not in client.reqs_in_flight
def test_apply_ready_response_syncs_block_size():
@pytest.mark.parametrize(
("effective_size", "other_size"),
[(None, None), (4224, 4224), (4224, 1056), (4224, None)],
)
def test_apply_ready_response_syncs_block_size(effective_size, other_size):
import msgspec
client = object.__new__(MPClient)
client._effective_attention_block_sizes = set()
client.vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
model_config=SimpleNamespace(max_model_len=8192),
@@ -399,14 +407,31 @@ def test_apply_ready_response_syncs_block_size():
max_loras=0,
)
)
client._apply_ready_response(payload)
fields = msgspec.msgpack.decode(payload)
if effective_size is None:
del fields["effective_attention_block_size"]
else:
fields["effective_attention_block_size"] = effective_size
client._apply_ready_response(msgspec.msgpack.encode(fields))
assert client.vllm_config.cache_config.block_size == 1056
cache_config = client.vllm_config.cache_config
assert cache_config.effective_attention_block_size == effective_size
fields["effective_attention_block_size"] = other_size
client._apply_ready_response(msgspec.msgpack.encode(fields))
assert cache_config.effective_attention_block_size == (
effective_size if effective_size == other_size else None
)
client._apply_ready_response(b"")
assert cache_config.effective_attention_block_size is None
def test_apply_ready_response_syncs_mamba_block_size():
import msgspec
client = object.__new__(MPClient)
client._effective_attention_block_sizes = set()
client.vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
model_config=SimpleNamespace(max_model_len=8192),
@@ -79,20 +79,31 @@ from vllm.v1.outputs import KVConnectorOutput, ModelRunnerOutput
from vllm.v1.request import RequestStatus
@pytest.mark.parametrize("boundary", [3840, 3904])
@pytest.mark.parametrize(
"window,extra_retained", [(128, 0), (128, 3), (256, 1), (256, 3)]
)
@pytest.mark.parametrize("boundary", [3840, 3904, 3968])
@pytest.mark.parametrize("eagle", [False, True])
@pytest.mark.parametrize("left_state", ["present", "missing", "pending", "loading"])
def test_swa_offload_window_covers_unaligned_hit(boundary, eagle, left_state):
def test_swa_offload_window_covers_unaligned_hit(
boundary, eagle, left_state, window, extra_retained
):
"""A smaller SWA group can move the hit inside another group's CPU chunk."""
groups = []
for i, (block_size, window) in enumerate([(256, None), (64, 128), (8, 128)]):
for i, (block_size, group_window) in enumerate(
[(256, None), (64, window), (8, 128)]
):
kwargs = dict(
block_size=block_size, num_kv_heads=1, head_size=1, dtype=torch.float32
)
kv_spec = (
FullAttentionSpec(**kwargs)
if window is None
else SlidingWindowSpec(**kwargs, sliding_window=window)
if group_window is None
else SlidingWindowSpec(
**kwargs,
sliding_window=group_window,
extra_retained_tokens=extra_retained if i == 1 else 0,
)
)
groups.append(
KVCacheGroupSpec([f"layer{i}"], kv_spec, is_eagle_group=eagle and i == 1)
@@ -153,7 +164,9 @@ def test_swa_offload_window_covers_unaligned_hit(boundary, eagle, left_state):
sched._chunks_being_loaded.add(left_key)
external, load_async = sched.get_num_new_matched_tokens(request, 0)
if boundary == 3904 and left_state != "present":
first_retained_block = (boundary - window + 1 - extra_retained) // 64
needs_left = first_retained_block // 4 < end - 1
if needs_left and left_state != "present":
assert external == (0 if left_state == "missing" else None)
assert not load_async
return
@@ -177,8 +190,8 @@ def test_swa_offload_window_covers_unaligned_hit(boundary, eagle, left_state):
sched.update_state_after_alloc(request, blocks, external)
assert len(sched._jobs) == 1
job = next(iter(sched._jobs.values()))
if boundary == 3904:
assert {swa[14], swa[15]} <= job.keys
if needs_left:
assert {left_key, swa[end - 1]} <= job.keys
manager.complete_load(job.keys, state.req_context)
@@ -245,6 +258,9 @@ def test_partial_tail_store_uses_attention_and_recurrent_cow_sources():
jobs = scheduler._build_partial_tail_store_jobs(output)
assert len(jobs) == 1
offered_keys = scheduler.manager.prepare_store.call_args.args[0]
req_context = scheduler._req_status["req"].req_context
assert all(req_context.get_offload_key_position(key) == 28 for key in offered_keys)
[job_id] = jobs
src_spec = jobs[job_id].src_spec
assert isinstance(src_spec, GPULoadStoreSpec)
@@ -367,6 +383,13 @@ def test_normal_store_excludes_align_mode_mamba_sources():
req_status.group_states[0].block_ids[:] = [11]
req_status.group_states[1].block_ids[:] = [99]
req_status.update_offload_keys()
for group_config, group_state in zip(
scheduler.config.kv_group_configs, req_status.group_states
):
assert (
req_status.req_context.get_offload_key_position(group_state.offload_keys[0])
== group_config.tokens_per_chunk
)
scheduler.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
)
@@ -908,7 +931,6 @@ def test_offloading_connector(request_runner, async_scheduling: bool):
runner.run(decoded_tokens=[0] * (tokens_per_chunk + 1))
# 1 more block (+ token for kicking off offloading)
# now check touch was called with all 6 blocks
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
)
@@ -916,9 +938,6 @@ def test_offloading_connector(request_runner, async_scheduling: bool):
decoded_tokens=[0] * (tokens_per_chunk + 1),
expected_stored=(15, 16, 17),
)
runner.manager.touch.assert_called()
block_hashes1 = list(runner.manager.touch.call_args.args[0])
assert len(block_hashes1) == 6
# terminate request
runner.run(decoded_tokens=[EOS_TOKEN_ID])
@@ -926,13 +945,7 @@ def test_offloading_connector(request_runner, async_scheduling: bool):
# create a new request differing only on the last token
runner.new_request(token_ids=[0] * (tokens_per_chunk * 6 - 1) + [1])
runner.run(decoded_tokens=[0])
runner.manager.touch.assert_called()
block_hashes2 = list(runner.manager.touch.call_args.args[0])
assert len(block_hashes2) == 6
# verify hashes are the same, except for the last block
assert block_hashes1[:5] == block_hashes2[:5]
assert block_hashes1[5] != block_hashes2[5]
runner.manager.touch.assert_not_called()
# terminate request
runner.run(
@@ -1340,14 +1353,7 @@ def test_two_groups_full_and_sliding_window(request_runner, async_scheduling: bo
generate_store_output(keys)
)
runner.run(decoded_tokens=[0])
# _touch called from get_num_new_matched_tokens (2 groups) and
# _get_reqs_to_store (2 groups) → 4 touch calls total.
touch_calls = runner.manager.touch.call_args_list
assert len(touch_calls) == 4
assert len(touch_calls[0].args[0]) == 3
assert len(touch_calls[1].args[0]) == 3
assert len(touch_calls[2].args[0]) == 3
assert len(touch_calls[3].args[0]) == 3
runner.manager.touch.assert_not_called()
# store 3 more block
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
@@ -1358,9 +1364,7 @@ def test_two_groups_full_and_sliding_window(request_runner, async_scheduling: bo
expected_stored=(0, 1, 2, 3, 4, 5),
)
# touch called from _get_reqs_to_store * 3 blocks, once for each group
touch_calls = runner.manager.touch.call_args_list
assert len(touch_calls) == 6
runner.manager.touch.assert_not_called()
# EOS lands in the last slot of the 7th block (offset 6). No forward pass
# writes that slot, so the finishing step declines the block.
@@ -1379,13 +1383,7 @@ def test_two_groups_full_and_sliding_window(request_runner, async_scheduling: bo
expected_loaded=((0, 0), (0, 1), (0, 2), (1, 1), (1, 2)),
)
# 2 touch calls from get_num_new_matched_tokens (2 groups)
touch_calls = runner.manager.touch.call_args_list
assert len(touch_calls) == 2
# full attention group touched all 3 blocks
assert len(touch_calls[0].args[0]) == 3
# sliding window group touched just the last 2 blocks
assert len(touch_calls[1].args[0]) == 2
runner.manager.touch.assert_not_called()
# 3 blocks are hit on GPU [0, 1, 2]
# 1 block loaded [3,]
@@ -1451,15 +1449,7 @@ def test_two_groups_different_block_sizes(request_runner, async_scheduling: bool
generate_store_output(keys)
)
runner.run(decoded_tokens=[0])
# _touch called from get_num_new_matched_tokens (2 groups) and
# _get_reqs_to_store (2 groups) → 4 touch calls total.
# Group 0 has 2 offload keys, group 1 has 1.
touch_calls = runner.manager.touch.call_args_list
assert len(touch_calls) == 4
assert len(touch_calls[0].args[0]) == 2
assert len(touch_calls[1].args[0]) == 1
assert len(touch_calls[2].args[0]) == 2
assert len(touch_calls[3].args[0]) == 1
runner.manager.touch.assert_not_called()
# Get to 31 tokens
# No further blocks offloaded
@@ -1469,11 +1459,7 @@ def test_two_groups_different_block_sizes(request_runner, async_scheduling: bool
# Group 0 blocks: [0, 1], ending_token_offset = 24
# Group 1 blocks: [0, 1], ending_token_offset = 32
runner.run(decoded_tokens=[0])
# _get_reqs_to_store touch: only group 1 has a new block to store
touch_calls = runner.manager.touch.call_args_list
assert len(touch_calls) == 2
assert len(touch_calls[0].args[0]) == 2
assert len(touch_calls[1].args[0]) == 2
runner.manager.touch.assert_not_called()
# Get to 35 tokens
# No further blocks offloaded
@@ -1483,11 +1469,7 @@ def test_two_groups_different_block_sizes(request_runner, async_scheduling: bool
# Group 0 blocks: [0, 1, 2], ending_token_offset = 36
# Group 1 blocks: [0, 1], ending_token_offset = 32
runner.run(decoded_tokens=[0])
# _get_reqs_to_store touch: only group 0 has a new block to store
touch_calls = runner.manager.touch.call_args_list
assert len(touch_calls) == 2
assert len(touch_calls[0].args[0]) == 3
assert len(touch_calls[1].args[0]) == 2
runner.manager.touch.assert_not_called()
# Get to 47 tokens
# No further blocks offloaded
@@ -1497,11 +1479,7 @@ def test_two_groups_different_block_sizes(request_runner, async_scheduling: bool
# Group 0 blocks: [0, 1, 2, 3], ending_token_offset = 4
# Group 1 blocks: [0, 1, 2], ending_token_offset = 48
runner.run(decoded_tokens=[0])
# _get_reqs_to_store touch: both groups have a new block, each with 1 key
touch_calls = runner.manager.touch.call_args_list
assert len(touch_calls) == 2
assert len(touch_calls[0].args[0]) == 4
assert len(touch_calls[1].args[0]) == 3
runner.manager.touch.assert_not_called()
runner.run(decoded_tokens=[0], expected_stored=((0, 3), (1, 2)))
@@ -2629,27 +2607,14 @@ def test_async_preempt_readmit_before_transfer_output_is_deferred(request_runner
assert req_status.transfer_jobs == pending_store_jobs
@pytest.mark.parametrize("extra_retained", [0, 1, 3])
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_swa_alignment_skip(request_runner, async_scheduling: bool):
"""SWA blocks unreachable by the load path are skipped during store.
def test_swa_alignment_skip(request_runner, async_scheduling: bool, extra_retained):
"""Store the window a cold consumer must restore at each FA boundary.
Simulates a DeepSeek V4-like hybrid architecture where SWA groups have
much smaller block sizes than the full-attention (MLA) group, causing
most SWA blocks to be unreachable by the alignment-based load path.
Setup:
- Group 0: full attention (MLA-like), block_size=16
- Group 1: SWA, block_size=4, sliding_window=8
alignment_tokens = 16, so 16 / 4 = 4 SWA blocks per alignment segment.
sliding_window_size_in_chunks = ceil(8 / 4) = 2.
Within each segment of 4 SWA blocks, only the trailing 2 are stored.
With 32 tokens (2 full-attn blocks, 8 SWA blocks):
- Group 0 stores: blocks 0, 1 (all full-attn blocks)
- Group 1 stores: blocks 2, 3, 6, 7 (skip 0,1,4,5)
For real DeepSeek V4 (100K tokens), this reduces SWA stores by ~78%.
With FA blocks of 16 tokens and SWA blocks of 4 tokens, a window of 8
needs the last two blocks per segment. Retaining three extra tokens for
MTP requires one more block; retaining just one token does not.
"""
full_attn_block_size = 16
swa_block_size = 4
@@ -2674,6 +2639,7 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool):
head_size=1,
dtype=torch.float32,
sliding_window=sliding_window,
extra_retained_tokens=extra_retained,
),
),
]
@@ -2691,16 +2657,17 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool):
# Group 0: full attention
assert kv_group_configs[0].sliding_window_size_in_chunks is None
assert kv_group_configs[0].tokens_per_chunk == full_attn_block_size
# Group 1: SWA with sliding_window_size of 2 chunks
assert kv_group_configs[1].sliding_window_size_in_chunks == 2
# Group 1: SWA, including any additional retained blocks
assert kv_group_configs[1].sliding_window_size_in_chunks == (
3 if extra_retained > 1 else 2
)
assert kv_group_configs[1].tokens_per_chunk == swa_block_size
# alignment_tokens = full_attn_block_size = 16
assert runner.connector_scheduler.config.alignment_tokens == full_attn_block_size
# Send 32 tokens = 2 full-attn blocks (block_size=16) = 8 SWA blocks
# (block_size=4). Decode 1 token to kick off processing (stores are
# deferred to next step).
num_tokens = 32
# Keep a partial last block so decoding does not reuse the producer's
# freed blocks before the harness records its deferred stores.
num_tokens = 33
runner.new_request(token_ids=[0] * num_tokens)
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
@@ -2713,12 +2680,7 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool):
)
runner.run(
decoded_tokens=[EOS_TOKEN_ID],
# Group 0 (full attn, block_size=16): 2 offloaded chunks
# -> GPU blocks (0, 0) and (0, 1)
# Group 1 (SWA, block_size=4): 8 offloaded chunks, skip first 2
# per segment of 4:
# Segment 0 (blocks 0-3): skip 0,1 -> store (1, 2), (1, 3)
# Segment 1 (blocks 4-7): skip 4,5 -> store (1, 6), (1, 7)
# Retain each segment's tail, including MTP history when needed.
expected_stored=(
(0, 0),
(0, 1),
@@ -2726,27 +2688,26 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool):
(1, 3),
(1, 6),
(1, 7),
),
)
+ (((1, 1), (1, 5)) if extra_retained > 1 else ()),
)
# Verify that loads still work correctly for the stored SWA blocks.
runner.scheduler.reset_prefix_cache()
runner.new_request(token_ids=[0] * num_tokens + [1])
runner.manager.lookup.return_value = LookupResult.HIT
runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: 2
runner.manager.lookup.side_effect = lambda key, ctx: (
LookupResult.HIT if (key, 0) in runner.offloaded else LookupResult.MISS
)
runner.run(
decoded_tokens=[EOS_TOKEN_ID],
# Group 0: full prefix lookup hits 2 offloaded chunks
# -> loads GPU blocks (0, 0), (0, 1)
# Group 1: sliding window lookup finds trailing 2 from last segment
# (blocks 6, 7 which were stored)
# -> loads GPU blocks (1, 6), (1, 7)
# The consumer restores the full FA prefix and the final SWA tail.
expected_loaded=(
(0, 0),
(0, 1),
(1, 6),
(1, 7),
),
)
+ (((1, 5),) if extra_retained > 1 else ()),
)
+428 -8
View File
@@ -23,6 +23,7 @@ from vllm.v1.kv_offload.cpu.common import (
)
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy
from vllm.v1.kv_offload.cpu.policies.lru import LRUCachePolicy
def make_req_context(
@@ -430,31 +431,34 @@ def test_cpu_manager():
assert cpu_manager.lookup(to_key(0), _EMPTY_REQ_CTX) is LookupResult.MISS
# prepare load [2, 3]
prepare_load_output = cpu_manager.prepare_load(to_keys([2, 3]), _EMPTY_REQ_CTX)
load_ctx = make_req_context("load-2-3")
prepare_load_output = cpu_manager.prepare_load(to_keys([2, 3]), load_ctx)
verify_load_output(prepare_load_output, [1, 2])
# prepare store with no space ([2, 3] is being loaded)
assert cpu_manager.prepare_store(to_keys([6, 7, 8]), _EMPTY_REQ_CTX) is None
# complete load [2, 3]. Load changes the eviction list, making 2, 3 recent.
cpu_manager.complete_load(to_keys([2, 3]), _EMPTY_REQ_CTX)
cpu_manager.complete_load(to_keys([2, 3]), load_ctx)
cpu_manager.on_request_finished(load_ctx)
# prepare store [6, 7, 8] -> evicts [4, 5, 2] (oldest)
# prepare store [6, 7, 8] -> evicts [4, 5, 3] (oldest). Within
# the accessed prefix [2, 3], the tail is less valuable than the head.
prepare_store_output = cpu_manager.prepare_store(to_keys([6, 7, 8]), _EMPTY_REQ_CTX)
verify_store_output(
prepare_store_output,
ExpectedPrepareStoreOutput(
keys_to_store=[6, 7, 8],
store_chunk_ids=[1, 0, 3],
evicted_keys=[4, 5, 2],
store_chunk_ids=[2, 0, 3],
evicted_keys=[4, 5, 3],
),
)
# complete store [6, 7, 8]
cpu_manager.complete_store(to_keys([6, 7, 8]), _EMPTY_REQ_CTX)
# touch [3, 6, 7] (move to end of LRU order)
cpu_manager.touch(to_keys([3, 6, 7]), _EMPTY_REQ_CTX)
# touch [2, 6, 7] (move to end of LRU order)
cpu_manager.touch(to_keys([2, 6, 7]), _EMPTY_REQ_CTX)
# prepare store [7, 9] -> evicts [8] (oldest following previous touch)
prepare_store_output = cpu_manager.prepare_store(to_keys([9]), _EMPTY_REQ_CTX)
@@ -477,7 +481,7 @@ def test_cpu_manager():
verify_events(
cpu_manager.take_events(),
expected_stores=({3, 4, 5}, {6, 7, 8}),
expected_evictions=({4, 5, 2}, {8}),
expected_evictions=({4, 5, 3}, {8}),
)
@@ -518,6 +522,65 @@ def test_prepare_load_preserves_key_order():
manager.complete_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) # order irrelevant
def test_lru_batch_eviction_failure_is_atomic():
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
policy = manager._policy
assert isinstance(policy, LRUCachePolicy)
keys = to_keys([1, 2, 3, 4])
assert manager.prepare_store(keys, _EMPTY_REQ_CTX) is not None
manager.complete_store(keys, _EMPTY_REQ_CTX)
protected = set(keys[1:])
assert policy.evict(2, protected) is None
assert all(manager.lookup(key, _EMPTY_REQ_CTX) is LookupResult.HIT for key in keys)
evicted = policy.evict(1, protected)
assert evicted is not None
assert [key for key, _ in evicted] == [keys[0]]
def test_lru_repeated_pin_cycles_compact_lazy_heap_entries():
manager = make_cpu_manager(num_chunks=1, cache_policy="lru")
policy = manager._policy
assert isinstance(policy, LRUCachePolicy)
key = to_key(1)
assert manager.prepare_store([key], _EMPTY_REQ_CTX) is not None
manager.complete_store([key], _EMPTY_REQ_CTX)
for _ in range(128):
manager.prepare_load([key], _EMPTY_REQ_CTX)
manager.complete_load([key], _EMPTY_REQ_CTX)
assert len(policy._heap) < 64
evicted = policy.evict(1, set())
assert evicted is not None
assert [evicted_key for evicted_key, _ in evicted] == [key]
def test_reset_discards_stale_request_access_classification():
manager = make_cpu_manager(num_chunks=1, cache_policy="arc")
policy = manager._policy
assert isinstance(policy, ARCCachePolicy)
key = to_key(1)
resumed_ctx = make_req_context("resumed")
assert manager.prepare_store([key], resumed_ctx) is not None
manager.complete_store([key], resumed_ctx)
manager.reset_cache()
replacement_ctx = make_req_context("replacement")
assert manager.prepare_store([key], replacement_ctx) is not None
manager.complete_store([key], replacement_ctx)
manager.on_request_finished(replacement_ctx)
assert key in policy.t1
manager.prepare_load([key], resumed_ctx)
manager.complete_load([key], resumed_ctx)
manager.on_request_finished(resumed_ctx)
assert key in policy.t2
class TestARCPolicy:
"""Unit tests for CPUOffloadingManager with ARC eviction policy."""
@@ -1042,6 +1105,48 @@ def test_filter_reused_manager_oversized_offer_makes_progress():
assert stored_keys == set(keys)
def test_store_threshold_does_not_hide_ready_reuse():
manager = make_cpu_manager(
num_chunks=2,
cache_policy="lru",
store_threshold=2,
max_tracker_size=2,
)
head, tail, noise_1, noise_2, replacement = to_keys(list(range(5)))
seed_ctx = make_req_context("seed")
output = manager.prepare_store([head, tail], seed_ctx)
assert output is not None
assert not output.keys_to_store
output = manager.prepare_store([head, tail], seed_ctx)
assert output is not None
assert output.keys_to_store == [head, tail]
manager.complete_store([head, tail], seed_ctx)
manager.on_request_finished(seed_ctx)
# Age both resident keys out of the admission tracker without storing the
# noise keys. Reusing the tail recreates its counter at one, below the
# threshold, but must still refresh its cache recency.
output = manager.prepare_store([noise_1, noise_2], make_req_context("noise"))
assert output is not None
assert not output.keys_to_store
reuse_ctx = make_req_context("reuse")
skipped_before_reuse = manager.stores_skipped_in_current_batch
output = manager.prepare_store([tail], reuse_ctx)
assert output is not None
assert not output.keys_to_store
assert manager.stores_skipped_in_current_batch == skipped_before_reuse
manager.on_request_finished(reuse_ctx)
replacement_ctx = make_req_context("replacement")
output = manager.prepare_store([replacement], replacement_ctx)
assert output is not None
assert not output.keys_to_store
output = manager.prepare_store([replacement], replacement_ctx)
assert output is not None
assert output.evicted_keys == [head]
def test_evictable_cache_chunk_count():
"""
Verifies _num_evictable_cache_chunks is maintained correctly through the
@@ -1156,3 +1261,318 @@ def test_touch_forwards_req_context_to_policy(monkeypatch):
assert len(received) == 1
assert received[0][0] == keys
assert received[0][1] is ctx
@pytest.mark.parametrize("finish_before_completion", [False, True])
def test_request_finish_orders_lru_prefix_independent_of_store_completion(
finish_before_completion: bool,
):
"""A transfer's completion order must not make the prefix head LRU."""
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
prefix = to_keys([1, 2, 3, 4])
ctx = make_req_context("prefix")
output = manager.prepare_store(prefix, ctx)
assert output is not None
if finish_before_completion:
manager.on_request_finished(ctx)
manager.complete_store(set(prefix), ctx)
if not finish_before_completion:
manager.on_request_finished(ctx)
output = manager.prepare_store(to_keys([5]), make_req_context("evict"))
assert output is not None
assert output.evicted_keys == to_keys([4])
@pytest.mark.parametrize("finish_before_completion", [False, True])
def test_request_finish_orders_lru_prefix_independent_of_load_completion(
finish_before_completion: bool,
):
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
prefix = to_keys([1, 2, 3, 4])
seed_ctx = make_req_context("seed")
output = manager.prepare_store(prefix, seed_ctx)
assert output is not None
manager.complete_store(prefix, seed_ctx)
reuse_ctx = make_req_context("reuse")
manager.prepare_load(prefix, reuse_ctx)
if finish_before_completion:
manager.on_request_finished(reuse_ctx)
manager.complete_load(set(prefix), reuse_ctx)
if not finish_before_completion:
manager.on_request_finished(reuse_ctx)
output = manager.prepare_store(to_keys([5]), make_req_context("evict"))
assert output is not None
assert output.evicted_keys == to_keys([4])
def test_late_old_completion_does_not_override_newer_request_recency():
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
old_keys = to_keys([1, 2])
new_keys = to_keys([3, 4])
old_ctx = make_req_context("old")
assert manager.prepare_store(old_keys, old_ctx) is not None
manager.on_request_finished(old_ctx)
new_ctx = make_req_context("new")
assert manager.prepare_store(new_keys, new_ctx) is not None
manager.complete_store(new_keys, new_ctx)
manager.on_request_finished(new_ctx)
# Physical completion is late, but the logical access is still older.
manager.complete_store(old_keys, old_ctx)
output = manager.prepare_store(to_keys([5]), make_req_context("evict"))
assert output is not None
assert output.evicted_keys == [old_keys[-1]]
def test_arc_counts_reuse_once_and_keeps_insertions_in_t1():
manager = make_cpu_manager(num_chunks=4, cache_policy="arc")
policy = manager._policy
assert isinstance(policy, ARCCachePolicy)
prefix = to_keys([1, 2, 3, 4])
seed_ctx = make_req_context("seed")
assert manager.prepare_store(prefix, seed_ctx) is not None
manager.complete_store(prefix, seed_ctx)
# Repeated store offers from the same request must not become ARC hits.
assert manager.prepare_store(prefix, seed_ctx) is not None
# Nor should an internal read used to cascade the new chunks to a tier.
manager.prepare_load(prefix, seed_ctx)
manager.complete_load(prefix, seed_ctx)
manager.on_request_finished(seed_ctx)
assert list(policy.t1) == list(reversed(prefix))
assert not policy.t2
reuse_ctx = make_req_context("reuse")
manager.prepare_load(prefix, reuse_ctx)
manager.complete_load(prefix, reuse_ctx)
# A duplicate internal pin, as used by tiering cascades, is deduplicated.
manager.prepare_load(prefix, reuse_ctx)
manager.complete_load(prefix, reuse_ctx)
manager.on_request_finished(reuse_ctx)
assert not policy.t1
assert list(policy.t2) == list(reversed(prefix))
def test_arc_reuse_wins_if_same_request_later_reinserts_key():
manager = make_cpu_manager(num_chunks=1, cache_policy="arc")
policy = manager._policy
assert isinstance(policy, ARCCachePolicy)
key_1, key_2 = to_keys([1, 2])
seed_ctx = make_req_context("seed")
assert manager.prepare_store([key_1], seed_ctx) is not None
manager.complete_store([key_1], seed_ctx)
manager.on_request_finished(seed_ctx)
reuse_ctx = make_req_context("reuse-and-reinsert")
manager.prepare_load([key_1], reuse_ctx)
manager.complete_load([key_1], reuse_ctx)
other_ctx = make_req_context("evict-reused-key")
assert manager.prepare_store([key_2], other_ctx) is not None
manager.complete_store([key_2], other_ctx)
manager.on_request_finished(other_ctx)
assert manager.prepare_store([key_1], reuse_ctx) is not None
manager.complete_store([key_1], reuse_ctx)
manager.on_request_finished(reuse_ctx)
assert key_1 not in policy.t1
assert list(policy.t2) == [key_1]
def test_request_finish_forwards_grouped_deduplicated_accesses(monkeypatch):
manager = make_cpu_manager(num_chunks=5)
existing = make_offload_key(b"existing", 0)
group_0_new = make_offload_key(b"group-0-new", 0)
group_1_head = make_offload_key(b"group-1-head", 1)
group_1_tail = make_offload_key(b"group-1-tail", 1)
seed_ctx = make_req_context("seed")
assert manager.prepare_store([existing], seed_ctx) is not None
manager.complete_store([existing], seed_ctx)
ctx = make_req_context("grouped")
# Group 1 is observed first, but policy finalization follows group index.
offered = [group_1_head, existing, group_0_new]
assert manager.prepare_store(offered, ctx) is not None
# Later offers extend a group's prefix and may repeat earlier keys.
assert (
manager.prepare_store([group_0_new, group_1_head, group_1_tail], ctx)
is not None
)
received = []
def on_request_finished(key_groups, insertion_only_keys, reused_keys, req_context):
received.append((key_groups, insertion_only_keys, reused_keys, req_context))
monkeypatch.setattr(manager._policy, "on_request_finished", on_request_finished)
manager.on_request_finished(ctx)
manager.on_request_finished(ctx)
assert received == [
(
((existing, group_0_new), (group_1_head, group_1_tail)),
{group_0_new, group_1_head, group_1_tail},
{existing},
ctx,
)
]
def test_failed_store_records_access_once_and_uses_store_miss_hook(monkeypatch):
manager = make_cpu_manager(num_chunks=1)
resident, candidate = to_keys([1, 2])
seed_ctx = make_req_context("seed")
assert manager.prepare_store([resident], seed_ctx) is not None
manager.complete_store([resident], seed_ctx)
pin_ctx = make_req_context("pin")
manager.prepare_load([resident], pin_ctx)
record_calls = 0
original_record = manager._record_request_cache_access
def record_access(*args, **kwargs):
nonlocal record_calls
record_calls += 1
return original_record(*args, **kwargs)
store_miss_calls = []
def on_store_miss(keys, req_context):
store_miss_calls.append((list(keys), req_context))
finalized = []
def on_request_finished(key_groups, insertion_only_keys, reused_keys, req_context):
finalized.append((key_groups, insertion_only_keys, reused_keys, req_context))
monkeypatch.setattr(manager, "_record_request_cache_access", record_access)
monkeypatch.setattr(manager._policy, "on_store_miss", on_store_miss)
monkeypatch.setattr(manager._policy, "on_request_finished", on_request_finished)
ctx = make_req_context("failed-store")
assert manager.prepare_store([candidate, resident], ctx) is None
assert record_calls == 1
assert store_miss_calls == [([candidate], ctx)]
manager.on_request_finished(ctx)
assert finalized == [
(((candidate, resident),), set(), {resident}, ctx),
]
manager.complete_load([resident], pin_ctx)
def test_arc_ghost_hit_adapts_once_per_request_before_insertion():
manager = make_cpu_manager(num_chunks=2, cache_policy="arc")
policy = manager._policy
assert isinstance(policy, ARCCachePolicy)
keys = to_keys([1, 2, 3])
seed_ctx = make_req_context("seed")
assert manager.prepare_store(keys[:2], seed_ctx) is not None
manager.complete_store(keys[:2], seed_ctx)
manager.on_request_finished(seed_ctx)
evict_ctx = make_req_context("create-ghost")
output = manager.prepare_store([keys[2]], evict_ctx)
assert output is not None
[ghost_key] = output.evicted_keys
manager.complete_store([keys[2]], evict_ctx)
manager.on_request_finished(evict_ctx)
assert ghost_key in policy.b1
resident_keys = [key for key in keys if key != ghost_key]
pin_ctx = make_req_context("pin-residents")
manager.prepare_load(resident_keys, pin_ctx)
ghost_ctx = make_req_context("ghost-hit")
target_before = policy.target_t1_size
assert manager.prepare_store([ghost_key], ghost_ctx) is None
target_after_first_offer = policy.target_t1_size
assert target_after_first_offer > target_before
assert manager.prepare_store([ghost_key], ghost_ctx) is None
assert policy.target_t1_size == target_after_first_offer
manager.complete_load(resident_keys, pin_ctx)
output = manager.prepare_store([ghost_key], ghost_ctx)
assert output is not None
assert policy.target_t1_size == target_after_first_offer
manager.complete_store([ghost_key], ghost_ctx)
manager.on_request_finished(ghost_ctx)
assert ghost_key in policy.t1
assert ghost_key not in policy.t2
def test_arc_pending_chunk_is_not_a_frequency_hit():
manager = make_cpu_manager(num_chunks=2, cache_policy="arc")
policy = manager._policy
assert isinstance(policy, ARCCachePolicy)
key = to_key(1)
writer_ctx = make_req_context("writer")
assert manager.prepare_store([key], writer_ctx) is not None
observer_ctx = make_req_context("pending-observer")
output = manager.prepare_store([key], observer_ctx)
assert output is not None and not output.keys_to_store
manager.on_request_finished(observer_ctx)
chunk = policy.get(key)
assert chunk is not None and not chunk.is_ready
assert key in policy.t1
assert key not in policy.t2
def test_request_key_positions_override_store_observation_order():
manager = make_cpu_manager(num_chunks=2, cache_policy="lru")
head, tail = to_keys([1, 2])
ctx = make_req_context("out-of-order-store")
ctx.set_offload_key_position(head, 16)
ctx.set_offload_key_position(tail, 32)
# A backfill or partial-tail path may offer a later key first.
assert manager.prepare_store([tail], ctx) is not None
assert manager.prepare_store([head], ctx) is not None
manager.on_request_finished(ctx)
manager.complete_store({head, tail}, ctx)
output = manager.prepare_store([to_key(3)], make_req_context("evict-tail"))
assert output is not None
assert output.evicted_keys == [tail]
def test_request_key_positions_order_tails_across_kv_groups():
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
group_0_head = make_offload_key(b"group-0-head", 0)
group_0_tail = make_offload_key(b"group-0-tail", 0)
group_1_head = make_offload_key(b"group-1-head", 1)
group_1_tail = make_offload_key(b"group-1-tail", 1)
keys = [group_0_head, group_0_tail, group_1_head, group_1_tail]
ctx = make_req_context("hybrid-groups")
for key in (group_0_head, group_1_head):
ctx.set_offload_key_position(key, 16)
for key in (group_0_tail, group_1_tail):
ctx.set_offload_key_position(key, 32)
assert manager.prepare_store(keys, ctx) is not None
manager.complete_store(keys, ctx)
manager.on_request_finished(ctx)
output = manager.prepare_store(to_keys([10, 11]), make_req_context("evict"))
assert output is not None
assert set(output.evicted_keys) == {group_0_tail, group_1_tail}
@@ -906,12 +906,13 @@ def test_insufficient_space_raises_clear_error(monkeypatch):
monkeypatch.setattr(region.os, "open", mock_open)
monkeypatch.setattr(region.os, "unlink", mock_unlink)
monkeypatch.setattr(region.os, "close", mock_close)
mock_check = MagicMock(
side_effect=RuntimeError("Insufficient space in /dev/shm: 30 GB required.")
)
monkeypatch.setattr(
region,
"check_shm_free_space",
lambda *a, **kw: (_ for _ in ()).throw(
RuntimeError("Insufficient space in /dev/shm: 30 GB required.")
),
mock_check,
)
with pytest.raises(RuntimeError, match="Insufficient space"):
@@ -925,6 +926,10 @@ def test_insufficient_space_raises_clear_error(monkeypatch):
mock_unlink.assert_called_once_with(mmap_path)
mock_close.assert_called_once_with(9999)
mock_check.assert_called_once_with(
4 * PAGE_SIZE,
allocation_name="CPU KV offload shared region in /dev/shm",
)
def test_ftruncate_failure_cleans_up_creator(monkeypatch):
@@ -40,6 +40,7 @@ from vllm.v1.kv_offload.base import (
TierMatcher,
make_offload_key,
)
from vllm.v1.kv_offload.cpu.policies.lru import LRUCachePolicy
from vllm.v1.kv_offload.tiering.base import (
JobResult,
SecondaryTierManager,
@@ -753,14 +754,64 @@ class TestTieringOffloadingManager:
# Touch chunks
self.manager.touch(chunks, _CTX)
# Verify touch was called on primary tier (check LRU order)
primary_keys = list(self.primary_tier._policy.evictable_chunks.keys())
assert primary_keys[-3:] == list(reversed(chunks))
# Verify touch was called on the primary tier (head is most recent).
policy = self.primary_tier._policy
assert isinstance(policy, LRUCachePolicy)
ranks = policy._ranks
assert ranks[chunks[2]] < ranks[chunks[1]] < ranks[chunks[0]]
# Verify touch was propagated to all secondary tiers
self.secondary_tier1.touch.assert_called_once_with(chunks, _CTX)
self.secondary_tier2.touch.assert_called_once_with(chunks, _CTX)
def test_request_order_survives_late_cascade_completion(self, manager_setup):
"""Cascade completion must not replace request order with I/O order."""
chunks = to_keys(range(5))
self._start_request()
output = self.manager.prepare_store(chunks, _CTX)
assert output is not None
self.manager.complete_store(chunks, _CTX, success=True)
# Finalize while both secondary-tier cascades still pin primary chunks.
self.manager.on_request_finished(_CTX)
self._simulate_on_schedule_end()
self._simulate_on_schedule_end()
evict_ctx = ReqContext(req_id="evict")
self._start_request(evict_ctx)
output = self.manager.prepare_store(to_keys([5]), evict_ctx)
assert output is not None
assert output.evicted_keys == [chunks[-1]]
def test_late_old_store_submission_does_not_override_newer_request_order(
self, manager_setup
):
"""Cascade submission after finish is not a new primary access."""
old_blocks = to_keys(range(2))
old_ctx = ReqContext(req_id="old")
self._start_request(old_ctx)
assert self.manager.prepare_store(old_blocks, old_ctx) is not None
self.manager.on_request_finished(old_ctx)
new_blocks = to_keys(range(2, 5))
new_ctx = ReqContext(req_id="new")
self._start_request(new_ctx)
assert self.manager.prepare_store(new_blocks, new_ctx) is not None
self.manager.complete_store(new_blocks, new_ctx, success=True)
self.manager.on_request_finished(new_ctx)
# The old GPU->primary write lands after the newer request finished.
# Its cascade pins must not turn it into the more recent request.
self.manager.complete_store(old_blocks, old_ctx, success=True)
self._simulate_on_schedule_end()
self._simulate_on_schedule_end()
evict_ctx = ReqContext(req_id="evict")
self._start_request(evict_ctx)
output = self.manager.prepare_store(to_keys([5]), evict_ctx)
assert output is not None
assert output.evicted_keys == [old_blocks[-1]]
def test_failed_store_no_cascade(self, manager_setup):
"""Test that failed GPU→primary store doesn't cascade."""
chunks = to_keys(range(3))
@@ -873,10 +924,10 @@ class TestTieringOffloadingManager:
job_metadata = self.secondary_tier1.submit_store.call_args.args[0]
assert job_metadata.req_context is ctx
def test_on_request_finished_delays_secondary_until_store_submitted(
def test_on_request_finished_delays_only_secondary_until_store_submitted(
self, manager_setup
):
"""Manager hook is eager; secondary hooks wait for cascade submission."""
"""Primary order commits immediately; secondary cleanup waits."""
chunks = to_keys(range(2))
ctx = ReqContext(req_id="req_delayed_secondary")
calls: list[tuple[str, str]] = []
+105 -3
View File
@@ -21,7 +21,12 @@ pytest.importorskip("triton")
if not torch.cuda.is_available():
pytest.skip("CUDA required for Gumbel sampler tests", allow_module_level=True)
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
from vllm.triton_utils import tl, triton
from vllm.v1.worker.gpu.sample.gumbel import (
_uniform64_from_random53,
gumbel_sample,
murmur3_hash32,
)
DEVICE = "cuda"
VOCAB_SIZE = 200_000
@@ -33,6 +38,103 @@ HEAD_LOG_GAP = 18.0
Z_TOLERANCE = 10.0
@triton.jit
def _murmur3_known_answer_kernel(
seeds_ptr,
positions_ptr,
offsets_ptr,
hashes_ptr,
domain_hashes_ptr,
numel: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
idx = tl.arange(0, BLOCK_SIZE)
mask = idx < numel
seed = tl.load(seeds_ptr + idx, mask=mask)
pos = tl.load(positions_ptr + idx, mask=mask)
offset = tl.load(offsets_ptr + idx, mask=mask)
hash32 = murmur3_hash32(seed, pos, offset).to(tl.int64)
domain_hash32 = murmur3_hash32(seed, pos, offset, domain=0x9E3779B9).to(tl.int64)
tl.store(hashes_ptr + idx, hash32, mask=mask)
tl.store(domain_hashes_ptr + idx, domain_hash32, mask=mask)
@triton.jit
def _uniform64_endpoint_kernel(
random53_ptr,
uniform_ptr,
numel: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
idx = tl.arange(0, BLOCK_SIZE)
mask = idx < numel
random53 = tl.load(random53_ptr + idx, mask=mask).to(tl.uint64)
tl.store(uniform_ptr + idx, _uniform64_from_random53(random53), mask=mask)
def test_murmur3_known_answers():
seeds = torch.tensor(
[0, 1, -1, 0x123456789ABCDEF, -(1 << 63), (1 << 63) - 1],
dtype=torch.int64,
device=DEVICE,
)
positions = torch.tensor(
[0, 2, 17, 123456789, 1 << 30, (1 << 30) + 7],
dtype=torch.int64,
device=DEVICE,
)
offsets = torch.tensor(
[0, 3, 1023, 199999, 4096, 0xFFFFFFFF],
dtype=torch.int64,
device=DEVICE,
)
hashes = torch.empty_like(seeds)
domain_hashes = torch.empty_like(seeds)
_murmur3_known_answer_kernel[(1,)](
seeds,
positions,
offsets,
hashes,
domain_hashes,
numel=seeds.numel(),
BLOCK_SIZE=8,
)
assert hashes.tolist() == [
0x8134CDF8,
0x23B53F91,
0xC8C13C14,
0x5DDE974F,
0xF1A9355B,
0xD952F9AA,
]
assert domain_hashes.tolist() == [
0xD28AD962,
0xE3AAF818,
0xD223C51C,
0x0547D2FE,
0x3AED160F,
0x49E8EA7F,
]
def test_uniform64_excludes_endpoints():
random53 = torch.tensor([0, (1 << 53) - 1], dtype=torch.int64, device=DEVICE)
uniform = torch.empty(2, dtype=torch.float64, device=DEVICE)
_uniform64_endpoint_kernel[(1,)](
random53,
uniform,
numel=random53.numel(),
BLOCK_SIZE=2,
)
assert torch.isfinite(uniform).all()
assert ((uniform > 0.0) & (uniform < 1.0)).all()
assert uniform.tolist() == [2.0**-54, 1.0 - 2.0**-53]
def _make_heavy_tailed_counts(seed: int = 1234) -> torch.Tensor:
"""Non-negative int64 counts of shape [VOCAB_SIZE]; target prob = counts/N."""
gen = torch.Generator(device=DEVICE).manual_seed(seed)
@@ -180,7 +282,7 @@ def test_full_vocab_distribution_fidelity():
def test_drafting_uses_a_separate_noise_stream():
"""is_drafting salts the Philox offset: same inputs, different draws.
"""is_drafting salts the RNG position: same inputs, different draws.
The draft proposal and the residual resample after a rejection must be
independent. They key noise by the same (seed, pos), so only the salt keeps
@@ -189,7 +291,7 @@ def test_drafting_uses_a_separate_noise_stream():
tests/v1/spec_decode/test_rejection_sampler_utils.py for the distributional
consequence.
Relocating the offset must not distort the draw either, so both streams are
Salting the position must not distort the draw either, so both streams are
checked against the target's far-tail mass, which sits HEAD_LOG_GAP logits
below the head -- the regime where fp32 Gumbel precision matters.
"""
+8 -2
View File
@@ -1482,8 +1482,14 @@ def cutlass_w4a8_moe_mm(
Cumulative token offsets
problem_sizes:
Per-expert (M, N, K) GEMM sizes used by the grouped GEMM launcher.
a/b/c/group_scale_strides:
Strides describing the memory layout of the input tensors.
a_strides:
Strides describing the memory layout of a_tensors.
b_strides:
Strides describing the memory layout of b_tensors.
c_strides:
Strides describing the memory layout of out_tensors.
group_scale_strides:
Strides describing the memory layout of b_group_scales.
maybe_schedule:
Optional override to choose a specific kernel or epilogue schedule.
+3
View File
@@ -211,6 +211,8 @@ class CacheConfig:
"""The number of blocks to allocate for CPU memory."""
# Set after KV cache initialization.
effective_attention_block_size: int | None = field(default=None, init=False)
"""Full-attention block size in tokens, including DCP, or None if unavailable."""
kv_cache_size_tokens: int | None = field(default=None, init=False)
"""Per-DP-engine KV cache capacity in tokens (group-aware). Uses
group-aware capacity since num_gpu_blocks * block_size can be wrong
@@ -277,6 +279,7 @@ class CacheConfig:
# Post-init/derived counters
"num_gpu_blocks",
"num_cpu_blocks",
"effective_attention_block_size",
"kv_cache_size_tokens",
"kv_cache_max_concurrency",
# WIP feature toggle not impacting compiled graph shape
@@ -34,6 +34,7 @@ import vllm.envs as envs
from vllm.distributed.utils import StatelessProcessGroup, sched_yield
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.cpu_resource_utils import check_cgroup_memory_available
from vllm.utils.network_utils import (
get_ip,
get_open_zmq_inproc_path,
@@ -224,26 +225,37 @@ class SpinCondition:
SHM_PATH = "/dev/shm"
def check_shm_free_space(required_bytes: int, shm_path: str = SHM_PATH) -> None:
"""Raise if ``shm_path`` cannot fit a ``required_bytes`` shared segment.
def check_shm_free_space(
required_bytes: int,
shm_path: str = SHM_PATH,
*,
allocation_name: str = "shared-memory allocation",
) -> None:
"""Raise if SHM cannot fit a shared segment and log cgroup headroom.
Args:
required_bytes: Size of the shared-memory segment to be created.
shm_path: Mount point backing POSIX shared memory; skipped if absent.
shm_path: Mount point backing POSIX shared memory; its filesystem
check is skipped if absent.
allocation_name: Human-readable name used in errors and logs.
Raises:
RuntimeError: If ``required_bytes`` exceeds the free space.
RuntimeError: If the SHM filesystem has insufficient space.
"""
if not os.path.isdir(shm_path):
return
free_bytes = shutil.disk_usage(shm_path).free
if required_bytes <= free_bytes:
return
mib = 1 << 20
raise RuntimeError(
f"Insufficient space in {shm_path}: {required_bytes / mib:.0f} MiB "
f"required, {free_bytes / mib:.0f} MiB free. Increase {shm_path} "
"(e.g. --shm-size or --ipc=host)."
if os.path.isdir(shm_path):
free_bytes = shutil.disk_usage(shm_path).free
if required_bytes > free_bytes:
mib = 1 << 20
raise RuntimeError(
f"Insufficient space in {shm_path} for {allocation_name}: "
f"{required_bytes / mib:.0f} MiB required, "
f"{free_bytes / mib:.0f} MiB free. Increase {shm_path} "
"(e.g. --shm-size or --ipc=host)."
)
check_cgroup_memory_available(
required_bytes,
allocation_name,
)
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from collections.abc import Iterable, Sequence
from dataclasses import dataclass, field
from dataclasses import dataclass, field, replace
from itertools import chain, islice
from typing import Any, NamedTuple
@@ -114,7 +114,10 @@ class GroupOffloadConfig(NamedTuple):
window = max(
window,
cdiv(
self.kv_cache_spec.sliding_window - 1 + right_padding,
self.kv_cache_spec.sliding_window
- 1
+ self.kv_cache_spec.extra_retained_tokens
+ right_padding,
self.tokens_per_chunk,
),
)
@@ -126,7 +129,13 @@ def get_sliding_window_size_in_chunks(
) -> int | None:
if isinstance(kv_cache_spec, SlidingWindowSpec):
assert kv_cache_spec.sliding_window > 0
return cdiv(kv_cache_spec.sliding_window, tokens_per_chunk)
return max(
cdiv(kv_cache_spec.sliding_window, tokens_per_chunk),
cdiv(
kv_cache_spec.sliding_window - 1 + kv_cache_spec.extra_retained_tokens,
tokens_per_chunk,
),
)
if isinstance(kv_cache_spec, ChunkedLocalAttentionSpec):
# Attention never reaches back past one chunk
@@ -418,8 +427,11 @@ class RequestOffloadState:
None,
group_config.hashes_per_chunk,
):
group_state.offload_keys.append(
make_offload_key(req_block_hash, group_config.group_idx)
key = make_offload_key(req_block_hash, group_config.group_idx)
group_state.offload_keys.append(key)
self.req_context.set_offload_key_position(
key,
len(group_state.offload_keys) * group_config.tokens_per_chunk,
)
def update_block_id_groups(
@@ -733,37 +745,6 @@ class OffloadingConnectorScheduler:
)
return None if defer_lookup or pending_in_window else consecutive_hits
def _touch(self, req_status: RequestOffloadState):
for group_config, group_state in zip(
self.config.kv_group_configs, req_status.group_states
):
if group_config.sliding_window_size_in_chunks is None:
self.manager.touch(group_state.offload_keys, req_status.req_context)
else:
# Keep only chunks needed to hit the original request, plus
# decoded chunks.
chunks_to_skip = max(
0,
group_state.num_hit_chunks
- group_config.sliding_window_size_in_chunks,
)
self.manager.touch(
group_state.offload_keys[chunks_to_skip:],
req_status.req_context,
)
if req_status.partial_tail_boundary is not None:
self.manager.touch(
tuple(
self._make_boundary_key(
req_status.req,
group.group_idx,
req_status.partial_tail_boundary,
)
for group in self.config.kv_group_configs
),
req_status.req_context,
)
def _lookup_complete_chunks(
self,
req_status: RequestOffloadState,
@@ -968,10 +949,16 @@ class OffloadingConnectorScheduler:
return num_hit_tokens
def _make_boundary_key(
self, request: Request, group_idx: int, boundary_tokens: int
self,
request: Request,
group_idx: int,
boundary_tokens: int,
req_context: ReqContext,
) -> OffloadKey:
hash_idx = boundary_tokens // self.config.tokens_per_hash - 1
return make_offload_key(request.block_hashes[hash_idx], group_idx)
key = make_offload_key(request.block_hashes[hash_idx], group_idx)
req_context.set_offload_key_position(key, boundary_tokens)
return key
def _lookup(
self,
@@ -1006,7 +993,10 @@ class OffloadingConnectorScheduler:
boundary_keys = []
for group_config in self.config.kv_group_configs:
key = self._make_boundary_key(
req_status.req, group_config.group_idx, boundary
req_status.req,
group_config.group_idx,
boundary,
req_status.req_context,
)
boundary_keys.append(key)
result = self.manager.lookup(key, req_status.req_context)
@@ -1099,8 +1089,6 @@ class OffloadingConnectorScheduler:
self._maybe_observe_lookup_async_delay(req_status)
req_status.update_num_hit_chunks(num_computed_tokens + (num_hit_tokens or 0))
self._touch(req_status)
return num_hit_tokens, bool(num_hit_tokens)
def update_state_after_alloc(
@@ -1174,7 +1162,10 @@ class OffloadingConnectorScheduler:
if partial_tail_boundary is not None:
keys_to_load.append(
self._make_boundary_key(
request, group_config.group_idx, partial_tail_boundary
request,
group_config.group_idx,
partial_tail_boundary,
req_status.req_context,
)
)
@@ -1298,7 +1289,9 @@ class OffloadingConnectorScheduler:
):
continue
key = self._make_boundary_key(req, group_idx, boundary)
key = self._make_boundary_key(
req, group_idx, boundary, req_status.req_context
)
store_output = self.manager.prepare_store([key], req_status.req_context)
if store_output is None:
self._connector_stats.increase_counter(
@@ -1393,7 +1386,9 @@ class OffloadingConnectorScheduler:
):
continue
keys = [
self._make_boundary_key(req, group.group_idx, boundary)
self._make_boundary_key(
req, group.group_idx, boundary, req_status.req_context
)
for group in self.config.kv_group_configs
]
block_ids = [
@@ -1461,11 +1456,24 @@ class OffloadingConnectorScheduler:
) -> list[bool] | None:
"""Build the block mask for a range of candidate offload chunks."""
blocks_per_chunk = self.config.blocks_per_chunk
kv_cache_spec = group_config.kv_cache_spec
if isinstance(kv_cache_spec, SlidingWindowSpec) and (
kv_cache_spec.extra_retained_tokens
):
# Offload restores all allocated history, including MTP re-prefill
# tokens. Widen only the store mask, not the model's attention window.
kv_cache_spec = replace(
kv_cache_spec,
sliding_window=(
kv_cache_spec.sliding_window + kv_cache_spec.extra_retained_tokens
),
extra_retained_tokens=0,
)
return group_config.manager_cls.reachable_block_mask(
start_block=start_chunk_idx * blocks_per_chunk,
end_block=end_chunk_idx * blocks_per_chunk,
alignment_tokens=self.config.alignment_tokens,
kv_cache_spec=group_config.kv_cache_spec,
kv_cache_spec=kv_cache_spec,
use_eagle=group_config.is_eagle_group,
retention_interval=self.config.retention_interval,
reachable_boundaries=reachable_boundaries,
@@ -1667,8 +1675,6 @@ class OffloadingConnectorScheduler:
req_status.advance_stored_idx(num_offloadable_tokens)
continue
self._touch(req_status)
keys_to_store = set(store_output.keys_to_store)
group_sizes: list[int] = []
+1 -1
View File
@@ -2051,7 +2051,7 @@ class EngineArgs:
device_config = DeviceConfig(device=cast(Device, current_platform.device_type))
envs.validate_environ(self.fail_on_environ_validation)
current_platform.validate_environ(self.fail_on_environ_validation)
# Check if the model is a speculator and override model/tokenizer/config
# BEFORE creating ModelConfig, so the config is created with the target model
+19 -5
View File
@@ -46,21 +46,35 @@ def _interactive_cli(args: argparse.Namespace) -> tuple[str, OpenAI]:
def _print_chat_stream(stream, stats: bool = False) -> str:
output = ""
start = time.perf_counter()
output: str = ""
in_reasoning: bool = False
start: float = time.perf_counter()
ttft: float | None = None
completion_tokens = 0
completion_tokens: int = 0
for chunk in stream:
if chunk.usage is not None:
completion_tokens = chunk.usage.completion_tokens
if not chunk.choices:
continue
delta = chunk.choices[0].delta
reasoning = getattr(delta, "reasoning", None) or getattr(
delta, "reasoning_content", None
)
if ttft is None and (reasoning or delta.content):
ttft = time.perf_counter() - start
if reasoning:
if not in_reasoning:
print("<think>", flush=True)
in_reasoning = True
print(reasoning, end="", flush=True)
if delta.content:
if ttft is None:
ttft = time.perf_counter() - start
if in_reasoning:
print("\n</think>", flush=True)
in_reasoning = False
output += delta.content
print(delta.content, end="", flush=True)
if in_reasoning:
print("\n</think>", end="", flush=True)
print()
if stats:
_print_metrics(start, ttft, completion_tokens)
+7 -1
View File
@@ -565,6 +565,7 @@ class ParsableContext(ConversationContext):
mcp_tools: dict[str, Mcp],
):
if tool_server:
initialized_session = False
for tool_name in self.available_tools:
if tool_name in self._tool_sessions:
continue
@@ -577,6 +578,8 @@ class ParsableContext(ConversationContext):
tool_server.new_session(tool_name, request_id, headers)
)
self._tool_sessions[tool_name] = tool_session
initialized_session = True
if initialized_session:
exit_stack.push_async_exit(self.cleanup_session)
async def cleanup_session(self, *args, **kwargs) -> None:
@@ -863,6 +866,7 @@ class HarmonyContext(ConversationContext):
mcp_tools: dict[str, Mcp],
):
if tool_server:
initialized_session = False
for tool_name in self.available_tools:
if tool_name not in self._tool_sessions:
tool_type = _map_tool_name_to_tool_type(tool_name)
@@ -873,7 +877,9 @@ class HarmonyContext(ConversationContext):
tool_server.new_session(tool_name, request_id, headers)
)
self._tool_sessions[tool_name] = tool_session
exit_stack.push_async_exit(self.cleanup_session)
initialized_session = True
if initialized_session:
exit_stack.push_async_exit(self.cleanup_session)
async def call_container_tool(
self, tool_session: Union["ClientSession", Tool], last_msg: Message
+28 -2
View File
@@ -1,8 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import logging
import model_hosting_container_standards.sagemaker as sagemaker_standards
from fastapi import APIRouter, Depends, FastAPI, Request
from fastapi.responses import JSONResponse, Response
@@ -22,6 +21,12 @@ router = APIRouter()
def attach_router(app: FastAPI):
"""Attach the LoRA adapter load/unload endpoints to the API server.
Does nothing when dynamic LoRA updating is disabled. Handler levels are
snapshotted and restored because importing
model_hosting_container_standards may reconfigure root logging.
"""
if not envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
"""If LoRA dynamic loading & unloading is not enabled, do nothing."""
return
@@ -30,6 +35,23 @@ def attach_router(app: FastAPI):
"This should ONLY be used for local development!"
)
snapshot = [
(h, h.level)
for lg in (logging.getLogger(), logging.getLogger("vllm"))
for h in lg.handlers
]
try:
_attach_router(app)
finally:
for handler, level in snapshot:
handler.setLevel(level)
def _attach_router(app: FastAPI):
"""Register the LoRA adapter load/unload routes on the app."""
import model_hosting_container_standards.sagemaker as sagemaker_standards
@sagemaker_standards.register_load_adapter_handler(
request_shape={
"lora_name": "body.name",
@@ -40,6 +62,8 @@ def attach_router(app: FastAPI):
)
@router.post("/v1/load_lora_adapter", dependencies=[Depends(validate_json_request)])
async def load_lora_adapter(request: LoadLoRAAdapterRequest, raw_request: Request):
"""Handle POST /v1/load_lora_adapter: load a LoRA adapter into the
serving engine."""
handler: OpenAIServingModels = models(raw_request)
response = await handler.load_lora_adapter(request)
if isinstance(response, ErrorResponse):
@@ -60,6 +84,8 @@ def attach_router(app: FastAPI):
async def unload_lora_adapter(
request: UnloadLoRAAdapterRequest, raw_request: Request
):
"""Handle POST /v1/unload_lora_adapter: unload a LoRA adapter from
the serving engine."""
handler: OpenAIServingModels = models(raw_request)
response = await handler.unload_lora_adapter(request)
if isinstance(response, ErrorResponse):
+59 -2
View File
@@ -1,11 +1,11 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import logging
from collections.abc import Awaitable, Callable
from http import HTTPStatus
from typing import Any
import model_hosting_container_standards.sagemaker as sagemaker_standards
import pydantic
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
from fastapi.responses import JSONResponse, Response
@@ -27,11 +27,55 @@ GetHandlerFn = Callable[[Request], BaseServing | None]
EndpointFn = Callable[[RequestType, Request], Awaitable[Any]]
def _snapshot_handler_levels() -> list[tuple[logging.Handler, int]]:
"""Snapshot handler levels for loggers that may be affected by third-party
logging configuration side effects (e.g. model_hosting_container_standards
calling configure_root_logger() at import time)."""
loggers = [logging.getLogger(), logging.getLogger("vllm")]
seen: set[int] = set()
snapshot: list[tuple[logging.Handler, int]] = []
for logger in loggers:
for handler in logger.handlers:
if id(handler) not in seen:
seen.add(id(handler))
snapshot.append((handler, handler.level))
return snapshot
def _restore_handler_levels(
snapshot: list[tuple[logging.Handler, int]],
) -> None:
"""Restore handler levels from a snapshot."""
for handler, level in snapshot:
handler.setLevel(level)
def attach_router(
app: FastAPI,
supported_tasks: tuple["SupportedTask", ...],
model_config: ModelConfig | None = None,
):
"""Attach the SageMaker hosting endpoints to the API server.
Handler levels are snapshotted and restored because importing
model_hosting_container_standards may reconfigure root logging.
"""
snapshot = _snapshot_handler_levels()
try:
_attach_router(app, supported_tasks, model_config)
finally:
_restore_handler_levels(snapshot)
def _attach_router(
app: FastAPI,
supported_tasks: tuple["SupportedTask", ...],
model_config: ModelConfig | None = None,
):
"""Register the SageMaker hosting routes (/ping, /invocations) on the
app."""
import model_hosting_container_standards.sagemaker as sagemaker_standards
router = APIRouter()
# NOTE: Construct the TypeAdapters only once
@@ -99,4 +143,17 @@ def attach_router(
def sagemaker_standards_bootstrap(app: FastAPI) -> FastAPI:
return sagemaker_standards.bootstrap(app)
"""Bootstrap the app with the SageMaker hosting standards.
Handler levels are restored right after the import because importing
model_hosting_container_standards may reconfigure root logging, and
bootstrap must run with the original levels.
"""
snapshot = _snapshot_handler_levels()
try:
import model_hosting_container_standards.sagemaker as sagemaker_standards
app = sagemaker_standards.bootstrap(app)
finally:
_restore_handler_levels(snapshot)
return app
-9
View File
@@ -2249,15 +2249,6 @@ def is_set(name: str):
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
def validate_environ(hard_fail: bool) -> None:
for env in os.environ:
if env.startswith("VLLM_") and env not in environment_variables:
if hard_fail:
raise ValueError(f"Unknown vLLM environment variable detected: {env}")
else:
logger.warning("Unknown vLLM environment variable detected: %s", env)
def compile_factors() -> dict[str, object]:
"""Return env vars used for torch.compile cache keys.
+30 -2
View File
@@ -34,13 +34,38 @@ direct_register_aiter_op = functools.partial(
AITER_SUPPORTED = is_aiter_found()
"""Most kernels in this file are supported if AITER is installed."""
def flatten_to_2d_is_free(x: Tensor) -> bool:
"""Whether ``x.reshape(-1, x.shape[-1])`` is a view with unit-stride rows.
The AITER norm kernels only take dense 2D inputs, so the wrappers below
flatten the leading dims with ``Tensor.reshape``, which silently falls back
to ``contiguous()`` when the flattened shape is not expressible with the
existing strides, adding a whole-tensor device copy that costs more than
the kernel saves. A strided last dim flattens without a copy but not into a
dense buffer, so it is rejected too.
"""
if x.dim() <= 2:
return True
if x.stride(-1) != 1:
return False
expected_stride = x.size(-1)
for i in range(x.dim() - 2, -1, -1):
if x.size(i) != 1 and x.stride(i) != expected_stride:
return False
expected_stride *= x.size(i)
return True
rms_no_var_16bit_only = (
lambda x, weight, epsilon, variance_size=None: variance_size is None
and x.dtype in (torch.float16, torch.bfloat16)
and (weight is None or weight.dtype == x.dtype)
and flatten_to_2d_is_free(x)
)
"""AITER rms_norm only supports float16 and bfloat16 acts, no var_size override,
and requires weight dtype to match x dtype."""
requires weight dtype to match x dtype, and requires flattening the activation
to 2D to be free."""
@ir.ops.rms_norm.register_impl(
@@ -79,10 +104,13 @@ rms_add_no_var_16bit_only = (
lambda x, x_residual, weight, epsilon, variance_size=None: variance_size is None
and x.dtype in (torch.float16, torch.bfloat16)
and (weight is None or weight.dtype == x.dtype)
and flatten_to_2d_is_free(x)
and flatten_to_2d_is_free(x_residual)
)
"""
AITER fused_add_rms_norm only supports 16-bit activations and no var_size override.
Requires weight dtype to match x dtype.
Requires weight dtype to match x dtype, and flattening both the activation and the
residual to 2D to be free.
"""
@@ -190,9 +190,11 @@ def candidate_blocks_to_sparse_indices(
candidate_blocks: [rows, K] int32 request-local candidate block ids
(-1 padded), in units of ``candidate_block_size`` positions. K
must be a power of two (the in-kernel sort); rows may be strided.
row_ks/row_ke: [rows] int32 per-row K range; bounds are in the same
row_ks: [rows] int32 per-row K range start; bounds are in the same
(packed-workspace) coordinates the sparse kernel iterates over.
Pass zeros for the paged path, whose blocks are context-relative.
row_ke: [rows] int32 per-row K range end, in the same coordinates as
``row_ks``.
candidate_block_size: Positions per candidate block.
sparse_block_kv: Positions per sparse block (8 or 16).
out: Optional ``(sparse_indices, end)`` buffers to write into,
@@ -322,12 +324,16 @@ def sparse_mqa_logits_prefill_chunk(
k_scale: [total_kv] int32 packed UE8M0 K scales.
weights: [rows, H] bf16 per-head weights; the sparse kernels take
bf16 and do not fold the Q scale in.
cu_seqlen_ks/cu_seqlen_ke: [rows] int32 per-token K bounds in the
packed workspace.
cu_seqlen_ks: [rows] int32 per-token K start bounds in the packed
workspace.
cu_seqlen_ke: [rows] int32 per-token K end bounds in the packed
workspace.
candidate_blocks: [rows, K] int32 request-local candidate block ids.
topk_indices: [rows, topk_tokens] output buffer.
sparse_indices/end/col_indices: Caller-owned scratch, see
`candidate_blocks_to_sparse_indices` and `sparse_topk_remap`.
sparse_indices: Caller-owned scratch, see
`candidate_blocks_to_sparse_indices`.
end: Caller-owned scratch, see `candidate_blocks_to_sparse_indices`.
col_indices: Caller-owned scratch, see `sparse_topk_remap`.
kernel_metadata: DeepGEMM schedule from a previous call with the same
candidates and bounds (i.e. another indexer layer in the same
step). When given, the candidate expansion is skipped and
@@ -413,7 +419,9 @@ def sparse_mqa_logits_paged_decode(
candidate_blocks: [rows, K] int32 candidate block ids.
topk_indices: [rows, topk_tokens] output buffer.
row_ks: [rows] int32 zeros (paged blocks are context-relative).
sparse_indices/end/col_indices: Caller-owned scratch.
sparse_indices: Caller-owned scratch.
end: Caller-owned scratch.
col_indices: Caller-owned scratch.
kernel_metadata: See `sparse_mqa_logits_prefill_chunk`.
Returns:
@@ -187,6 +187,7 @@ from vllm.model_executor.kernels.linear.scaled_mm.b12x import (
)
from vllm.model_executor.kernels.linear.scaled_mm.cpu import (
CPUFp8BlockScaledMMKernel,
CPUFp8PerTensorScaledMMLinearKernel,
CPUInt8ScaledMMLinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.cutlass import (
@@ -438,6 +439,7 @@ _POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] =
ChannelWiseTorchFP8ScaledMMLinearKernel,
],
PlatformEnum.CPU: [
CPUFp8PerTensorScaledMMLinearKernel,
PerTensorTorchFP8ScaledMMLinearKernel,
ChannelWiseTorchFP8ScaledMMLinearKernel,
],
@@ -9,6 +9,7 @@ from vllm.model_executor.kernels.linear.scaled_mm.BlockScaledMMLinearKernel impo
)
from vllm.model_executor.kernels.linear.scaled_mm.cpu import (
CPUFp8BlockScaledMMKernel,
CPUFp8PerTensorScaledMMLinearKernel,
CPUInt8ScaledMMLinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.cutlass import (
@@ -67,5 +68,6 @@ __all__ = [
"ZentorchInt8ScaledMMLinearKernel",
"Fp8BlockScaledMMLinearKernel",
"CPUFp8BlockScaledMMKernel",
"CPUFp8PerTensorScaledMMLinearKernel",
"XPUFp8BlockScaledMMKernel",
]
@@ -21,10 +21,14 @@ from .BlockScaledMMLinearKernel import (
FP8ScaledMMLinearLayerConfig,
)
from .ScaledMMLinearKernel import (
FP8ScaledMMLinearKernel,
Int8ScaledMMLinearKernel,
Int8ScaledMMLinearLayerConfig,
)
# BLOCK_K in csrc/cpu/sgl-kernels/gemm.h — the AMX kernel's fixed K-block size.
_BLOCK_SIZE_K = 128
class CPUInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel):
@classmethod
@@ -330,3 +334,99 @@ class CPUFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
raise NotImplementedError(
"CPUFp8BlockScaledMMKernel overrides apply_weights directly."
)
class CPUFp8PerTensorScaledMMLinearKernel(FP8ScaledMMLinearKernel):
"""FP8 W8A16 per-tensor-scaled GEMM via AMX BRGEMM on CPU.
Reuses the block-scaled AMX kernel (fp8_scaled_mm_cpu) with a single
synthetic block spanning the whole weight, so activations stay BF16/FP32
— no FP8 activation quantization, unlike PerTensorTorchFP8ScaledMMLinearKernel.
"""
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_cpu():
return False, "requires CPU platform."
if not torch.cpu._is_amx_tile_supported():
return False, "requires AMX tile support (Sapphire Rapids or newer)."
if not ops._supports_cpu_fp8_w8a16:
return False, "fp8_scaled_mm_cpu op not available."
return True, None
@classmethod
def can_implement(cls, c: FP8ScaledMMLinearLayerConfig) -> tuple[bool, str | None]:
if not c.weight_quant_key.scale.group_shape.is_per_tensor():
return False, "requires a per-tensor weight scale."
if not c.activation_quant_key.scale.group_shape.is_per_tensor():
return False, "requires a per-tensor activation scale."
if c.out_dtype not in (torch.bfloat16, torch.float32):
return False, "Only bfloat16/float32 output dtype supported."
n = c.weight_shape[0]
if n % 32 != 0:
# The AMX tinygemm kernel tiles N in chunks of 32 and cannot
# handle a remainder tile smaller than that.
return False, f"requires weight output dim (N={n}) to be a multiple of 32."
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
w_name, w_s_name, _, _ = self.layer_param_names
# Fp8LinearMethod transposes weight to (K, N) for torch._scaled_mm-style
# kernels; convert_weight_packed/fp8_scaled_mm_cpu expect (N, K), as
# produced by CUDA nn.Linear-style checkpoints.
weight = getattr(layer, w_name).t().contiguous()
n, k = weight.shape
packed_weight = torch.ops._C.convert_weight_packed(weight)
replace_parameter(
layer, w_name, torch.nn.Parameter(packed_weight, requires_grad=False)
)
# Synthesize a single-block "block scale" tensor from the scalar
# per-tensor weight scale so fp8_scaled_mm_cpu can be reused as-is.
weight_scale = getattr(layer, w_s_name)
self._block_size_n = -(-n // 32) * 32 # round up to a multiple of 32
num_k_blocks = -(-k // _BLOCK_SIZE_K)
block_scale = weight_scale.reshape(()).expand(1, num_k_blocks).contiguous()
replace_parameter(
layer, w_s_name, torch.nn.Parameter(block_scale, requires_grad=False)
)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
w_name, w_s_name, _, _ = self.layer_param_names
weight = getattr(layer, w_name)
weight_scale = getattr(layer, w_s_name)
x_2d = x.reshape(-1, x.shape[-1]) if x.dim() > 2 else x
out = torch.ops._C.fp8_scaled_mm_cpu(
x_2d,
weight,
weight_scale,
[self._block_size_n, _BLOCK_SIZE_K],
bias,
x.dtype,
True, # is_vnni (weight already prepacked)
)
return out.reshape(x.shape[:-1] + (out.size(-1),)) if x.dim() > 2 else out
def apply_scaled_mm(
self,
*,
A: torch.Tensor,
B: torch.Tensor,
out_dtype: torch.dtype,
As: torch.Tensor,
Bs: torch.Tensor,
bias: torch.Tensor | None,
output_shape: list,
) -> torch.Tensor:
raise NotImplementedError(
"CPUFp8PerTensorScaledMMLinearKernel overrides apply_weights directly."
)
@@ -615,14 +615,15 @@ class RoutedExperts(PluggableLayer):
# compressed-tensors checkpoints with packed weights are stored flipped
# TODO (mgoin): check self.quant_method.quant_config.quant_format
# against known CompressionFormat enum values that have this quality
if quant_method_name in (
"CompressedTensorsWNA16MoEMethod",
"CompressedTensorsW4A16FlydslMoEMethod",
if (
quant_method_name
in (
"CompressedTensorsWNA16MoEMethod",
"CompressedTensorsW4A16FlydslMoEMethod",
)
and is_transposed
):
if is_transposed:
loaded_weight = loaded_weight.t().contiguous()
else:
loaded_weight = loaded_weight
loaded_weight = loaded_weight.t().contiguous()
if shard_id not in ("w1", "w2", "w3"):
raise ValueError(f"shard_id must be ['w1','w2','w3'] but got {shard_id}.")
@@ -727,11 +727,15 @@ def _get_available_ram_bytes() -> int:
host_available = psutil.virtual_memory().available
from vllm.utils.cpu_resource_utils import get_cgroup_memory_limit
from vllm.utils.cpu_resource_utils import (
get_cgroup_memory_limit,
get_cgroup_memory_usage,
)
cgroup_limit, cgroup_usage = get_cgroup_memory_limit()
cgroup_limit = get_cgroup_memory_limit()
if cgroup_limit is None:
return host_available
cgroup_usage = get_cgroup_memory_usage()
cgroup_available = (
cgroup_limit if cgroup_usage is None else max(0, cgroup_limit - cgroup_usage)
)
+1 -3
View File
@@ -780,9 +780,7 @@ class DeepCLIPVisionEmbeddings(CLIPVisionEmbeddings):
self, pixel_values: torch.Tensor, patch_embeds: torch.Tensor | None = None
) -> torch.Tensor:
batch_size = pixel_values.shape[0]
if patch_embeds is not None:
patch_embeds = patch_embeds
else:
if patch_embeds is None:
patch_embeds = self.patch_embedding(pixel_values)
patch_embeds = patch_embeds.flatten(2).transpose(1, 2)
@@ -964,7 +964,6 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
assert isinstance(va_item, (list, tuple)) and len(va_item) == 2
vid, audio_src = va_item
va_item = VideoAudioInput(video=vid, audio=audio_src)
vid = vid
# Convert video frames to (TCHW, timestamps) if needed
if (
isinstance(vid, tuple)
@@ -253,6 +253,7 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessing
hf_processor_mm_kwargs = dict(
**hf_processor_mm_kwargs,
sampling_rate=feature_extractor.sampling_rate,
truncation=True,
)
return mm_data, hf_processor_mm_kwargs
-1
View File
@@ -173,7 +173,6 @@ class Step3p5Attention(nn.Module):
rope_scaling = None
if sliding_window is not None and enable_sliding_window:
sliding_window = sliding_window
if swa_num_attention_heads is not None:
num_heads = swa_num_attention_heads
self.total_num_heads = swa_num_attention_heads
+48 -5
View File
@@ -204,6 +204,7 @@ class DeepseekV4MegaMoEExperts(nn.Module):
self.top_k = top_k
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.unpadded_intermediate_size = intermediate_size
self.num_shared_experts = num_shared_experts
self.max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens
@@ -386,6 +387,17 @@ class DeepseekV4MegaMoEExperts(nn.Module):
# the generic linear post-load hook replaces the raw checkpoint scales
# with its 128x128 DeepGEMM layout.
checkpoint_scale_dtypes = (torch.float8_e8m0fnu, torch.uint8)
unpadded_size = self.unpadded_intermediate_size * self.num_shared_experts
padding = self.intermediate_size * self.num_shared_experts - unpadded_size
pad_weights = (
padding > 0
and gate_up_weight.dtype == torch.float8_e4m3fn
and down_weight.dtype == torch.float8_e4m3fn
and gate_up_weight.shape == (2 * unpadded_size, self.hidden_size)
and down_weight.shape == (self.hidden_size, unpadded_size)
and gate_up_scale.dtype in checkpoint_scale_dtypes
and down_scale.dtype in checkpoint_scale_dtypes
)
if (
gate_up_scale.dtype in checkpoint_scale_dtypes
and down_scale.dtype in checkpoint_scale_dtypes
@@ -396,6 +408,7 @@ class DeepseekV4MegaMoEExperts(nn.Module):
gate_up_scale,
gate_up_weight.shape[0],
gate_up_weight.shape[1],
padding=(padding, 0) if pad_weights else (0, 0),
)
down_scale = self._prepare_shared_expert_scale(
deep_gemm,
@@ -403,12 +416,27 @@ class DeepseekV4MegaMoEExperts(nn.Module):
down_scale,
down_weight.shape[0],
down_weight.shape[1],
padding=(0, padding) if pad_weights else (0, 0),
)
if gate_up_scale is None or down_scale is None:
self.num_shared_experts = 0
return
if pad_weights:
# Pad gate/up separately so the SwiGLU split stays at the midpoint.
gate_up_weight = (
torch.nn.functional.pad(
gate_up_weight.view(torch.uint8).unflatten(0, (2, unpadded_size)),
(0, 0, 0, padding),
)
.flatten(0, 1)
.view(gate_up_weight.dtype)
)
down_weight = torch.nn.functional.pad(
down_weight.view(torch.uint8), (0, padding)
).view(down_weight.dtype)
shared_intermediate_size = self.intermediate_size * self.num_shared_experts
expected_gate_up_shape = (
2 * shared_intermediate_size,
@@ -449,11 +477,11 @@ class DeepseekV4MegaMoEExperts(nn.Module):
# released instead of adding roughly 0.7 GiB per rank on DSV4-Flash.
# The generic linear post-load hook may still repack the serial scales,
# but this shared MLP is never called after native fusion is enabled.
gate_up.weight.data = transformed_l1[0]
self._transformed_shared_l1_weights = (
gate_up.weight.data,
transformed_l1[1],
)
# Padded weights need separate storage: the generic linear post-load
# hooks still expect the original checkpoint weight/scale shapes.
if not pad_weights:
gate_up.weight.data = transformed_l1[0]
self._transformed_shared_l1_weights = transformed_l1
self._transformed_shared_l2_weights = transformed_l2
def _prepare_shared_expert_scale(
@@ -463,6 +491,8 @@ class DeepseekV4MegaMoEExperts(nn.Module):
scale: torch.Tensor,
mn: int,
k: int,
*,
padding: tuple[int, int] = (0, 0),
) -> torch.Tensor | None:
block_size = getattr(linear, "weight_block_size", None)
if block_size is None or len(block_size) != 2:
@@ -497,6 +527,19 @@ class DeepseekV4MegaMoEExperts(nn.Module):
.repeat_interleave(block_k // 32, dim=1)[:mn, : k // 32]
.contiguous()
)
pad_m, pad_k = padding
if pad_m:
scale_1x32 = torch.nn.functional.pad(
scale_1x32.unflatten(0, (2, mn // 2)),
(0, 0, 0, pad_m),
value=1.0,
).flatten(0, 1)
mn += 2 * pad_m
if pad_k:
scale_1x32 = torch.nn.functional.pad(
scale_1x32, (0, pad_k // 32), value=1.0
)
k += pad_k
# The grouped API is used with a singleton dimension to request the
# MN-major, TMA-aligned packed-UE8M0 strides, then squeezed back to the
# 2D layout required for a shared expert.
+2 -1
View File
@@ -636,7 +636,8 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
transpose_scale=False,
)
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
o = attn_out[:, : self.n_local_heads, :]
# ROCm BF16 reference wo_a path (inverse RoPE + einsum) + wo_b.
z = rocm_inv_rope_einsum(
self.rotary_emb,
+150 -54
View File
@@ -44,6 +44,7 @@ from vllm.config import (
VllmConfig,
get_current_vllm_config,
)
from vllm.config.cache import CacheDType
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
@@ -127,34 +128,40 @@ def _use_v41_mxfp8_kv_record() -> bool:
def _resolve_dsv4_kv_cache_dtype(
use_fp8_ds_mla_layout: bool,
kv_cache_dtype: str,
kv_cache_dtype: CacheDType,
cache_config: CacheConfig | None,
) -> tuple[str, torch.dtype]:
packed_kv_cache_dtype: CacheDType = "fp8_ds_mla",
) -> tuple[CacheDType, torch.dtype]:
"""Map ``(layout, --kv-cache-dtype)`` to ``(cache_dtype_str, torch_dtype)``.
Both layouts are paged; they differ in the per-token block format. The
``fp8_ds_mla`` format is UE8M0 block-scaled fp8 packed as ``uint8`` (the
canonical ``fp8_ds_mla`` string is written back onto ``cache_config`` so the
page-size specs pick the 576B per-token slot). Plain-row backends store each
token's KV row in its element dtype: bf16 or per-tensor FP8 E4M3.
packed formats are ``uint8``-backed: ``fp8_ds_mla`` is UE8M0 block-scaled
fp8 throughout, ``nvfp4_ds_mla`` keeps that sliding-window record and
stores the compressed cache as NVFP4. An unspecific ``--kv-cache-dtype``
(``auto`` / ``fp8``) resolves to ``packed_kv_cache_dtype``, the record this
layer's kernel prefers, and the canonical string is written back onto
``cache_config`` so the page-size specs pick the right per-token slot.
Plain-row backends store each token's KV row in its element dtype: bf16 or
per-tensor FP8 E4M3.
"""
if use_fp8_ds_mla_layout:
# fp8_ds_mla block format: UE8M0 block-scaled fp8 packed as uint8.
if kv_cache_dtype == "auto":
kv_cache_dtype = "fp8"
if not kv_cache_dtype.startswith("fp8"):
if kv_cache_dtype in ("auto", "fp8"):
kv_cache_dtype = packed_kv_cache_dtype
elif not kv_cache_dtype.endswith("_ds_mla"):
raise ValueError(
"DeepseekV4 fp8_ds_mla layout only supports fp8 "
f"kv-cache, got {kv_cache_dtype}. Please set "
"`--kv-cache-dtype fp8` or select a backend that supports "
"bfloat16 KV cache."
"DeepseekV4 packed KV layouts only support fp8 kv-cache, got "
f"{kv_cache_dtype}. Please set `--kv-cache-dtype fp8` or "
"select a backend that supports bfloat16 KV cache."
)
if kv_cache_dtype != "fp8_ds_mla":
if cache_config is not None:
cache_config.cache_dtype = "fp8_ds_mla"
kv_cache_dtype = "fp8_ds_mla"
logger.info_once("Using DeepSeek's fp8_ds_mla KV cache format.")
if cache_config is not None and cache_config.cache_dtype != kv_cache_dtype:
cache_config.cache_dtype = kv_cache_dtype
logger.info_once("Using DeepSeek's %s KV cache format.", kv_cache_dtype)
return kv_cache_dtype, torch.uint8
if kv_cache_dtype.endswith("_ds_mla"):
raise ValueError(
f"{kv_cache_dtype} is a packed FlashMLA DeepSeek V4.1 KV cache "
"format; the selected backend stores plain KV rows."
)
# Plain bf16 / per-tensor fp8 KV row (FlashInfer).
if kv_cache_dtype.startswith("fp8"):
@@ -189,6 +196,28 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
# path to pre-reserve that workspace.
PREFILL_CHUNK_SIZE: ClassVar[int] = 4
# ---- attention-interface contract, declared by the platform subclass ----
@property
def accepts_unnormed_unroped_query(self) -> bool:
"""Whether ``forward_mqa``'s ``q`` is the raw ``wq_b`` output.
True when the attention kernel applies the Q norm and RoPE itself, and
reads Q in its own chunk-interleaved layout, so the layer only
zero-pads Q to ``padded_heads`` and inserts KV before calling it.
"""
return False
@property
def packed_kv_cache_dtype(self) -> CacheDType:
"""The packed KV record this layer's kernel prefers.
What an unspecific ``--kv-cache-dtype`` (``auto`` / ``fp8``) resolves
to. Mega attention overrides it: its kernel is the one that can read
an NVFP4 compressed cache.
"""
return "fp8_ds_mla"
@classmethod
@abstractmethod
def get_padded_num_q_heads(cls, num_heads: int) -> int:
@@ -213,8 +242,19 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
raise NotImplementedError
@abstractmethod
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
"""Inverse-RoPE + wo_a + wo_b output projection (platform-specific)."""
def _o_proj(
self,
attn_out: "torch.Tensor | QuantizedActivation",
positions: torch.Tensor,
) -> torch.Tensor:
"""Project whatever ``_alloc_attn_out`` produced through wo_a and wo_b.
Takes the buffer whole, so each layer owns the shape it allocated: the
bf16 layers slice off their padding heads and apply the inverse RoPE,
while a layer whose attention kernel already did the inverse RoPE and
the FP8 cast gets a QuantizedActivation and has only wo_a and wo_b
left.
"""
raise NotImplementedError
def _uses_fp8_ds_mla_layout(self) -> bool:
@@ -459,11 +499,27 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
# Resolve the kv-cache dtype from this backend's block format. The same
# resolution drives the SWA cache tensor dtype below.
self.kv_cache_dtype, self.kv_cache_torch_dtype = _resolve_dsv4_kv_cache_dtype(
self._uses_fp8_ds_mla_layout(), cache_config.cache_dtype, cache_config
self._uses_fp8_ds_mla_layout(),
cache_config.cache_dtype,
cache_config,
self.packed_kv_cache_dtype,
)
self.kv_mxfp8 = _use_v41_mxfp8_kv_record()
self.kv_bytes_per_token = 528 if self.kv_mxfp8 else 584
self.swa_bytes_per_token = 528 if self.kv_mxfp8 else 584
# nvfp4_ds_mla keeps the MXFP8 sliding-window record and stores the
# compressed cache as NVFP4 (256 B of e2m1 pairs + 32 e4m3 scales).
self.compressed_bytes_per_token = (
288 if self.kv_cache_dtype == "nvfp4_ds_mla" else self.swa_bytes_per_token
)
# One alignment for every page in the block: the block stride is their
# sum, and 512 satisfies both TMA strides in play (512 for the V4.1
# fp8 record, 256 for NVFP4).
self.kv_page_alignment = 512 if self.kv_mxfp8 else 576
if self.kv_cache_dtype == "nvfp4_ds_mla" and not self.kv_mxfp8:
raise ValueError(
"nvfp4_ds_mla needs the V4.1 KV records, which FlashMLA "
"decodes only on SM100."
)
self.swa_cache_layer = DeepseekV4SWACache(
head_dim=self.head_dim,
@@ -473,7 +529,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
cache_config=cache_config,
backend_cls=self.swa_backend_cls,
block_size=32,
packed_bytes_per_token=self.kv_bytes_per_token,
packed_bytes_per_token=self.swa_bytes_per_token,
packed_page_alignment=self.kv_page_alignment,
)
@@ -553,8 +609,12 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
block_size=self.swa_cache_layer.block_size,
)
# Every backend that gathers a chunk's KV through
# combine_topk_swa_indices needs its Triton kernel warmed, mega
# attention included -- it calls it from _forward_prefill_mega.
if self.backend_cls.get_name() in (
"FLASHMLA_SPARSE_DSV41",
"FLASHMLA_MEGA_ATTN_DSV41",
"ROCM_FLASHMLA_SPARSE_DSV4",
):
from vllm.models.deepseek_v41.common.ops.cache_utils import (
@@ -569,14 +629,10 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
hidden_states: torch.Tensor,
llama_4_scaling: torch.Tensor | None = None,
) -> torch.Tensor:
# Pre-allocate attention output with FlashMLA-padded head count.
# The op writes into `o_padded`; we slice to n_local_heads after.
num_tokens = hidden_states.shape[0]
o_padded = torch.empty(
(num_tokens, self.padded_heads, self.head_dim),
dtype=hidden_states.dtype,
device=hidden_states.device,
)
# The eager attention region writes into a caller-owned buffer
# (breakable_cudagraph needs in-place outputs); its shape and how it is
# projected afterwards follow the interface contract above.
attn_out = self._alloc_attn_out(hidden_states.shape[0], hidden_states)
# Keep the attention input preparation in the captured graph. Only the
# sparse indexer and MLA attention run in the eager break below.
@@ -593,12 +649,19 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
kv_score,
indexer_weights,
positions,
o_padded,
attn_out,
)
o = o_padded[:, : self.n_local_heads, :]
return self._o_proj(attn_out, positions)
# Inverse-RoPE + wo_a + wo_b output projection (platform-specific).
return self._o_proj(o, positions)
def _alloc_attn_out(
self, num_tokens: int, hidden_states: torch.Tensor
) -> "torch.Tensor | QuantizedActivation":
"""The buffer ``forward_mqa`` fills, per the interface contract."""
return torch.empty(
(num_tokens, self.padded_heads, self.head_dim),
dtype=hidden_states.dtype,
device=hidden_states.device,
)
@cached_property
def _can_fuse_query_quant(self) -> bool:
@@ -652,7 +715,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
kv_score: torch.Tensor,
indexer_weights: torch.Tensor,
positions: torch.Tensor,
o_padded: torch.Tensor,
attn_out: "torch.Tensor | QuantizedActivation",
) -> None:
"""Wide eager region: the whole of ``_prepare_and_attn`` runs eagerly.
@@ -667,7 +730,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
kv_score,
indexer_weights,
positions,
o_padded,
attn_out,
)
def _prepare_and_attn(
@@ -679,7 +742,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
kv_score: torch.Tensor,
indexer_weights: torch.Tensor,
positions: torch.Tensor,
o_padded: torch.Tensor,
attn_out: "torch.Tensor | QuantizedActivation",
) -> None:
"""Attention input preparation followed by the sparse indexer and MLA.
@@ -751,7 +814,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
q,
kv,
positions,
o_padded,
attn_out,
)
def _fused_wqa_wkv_gemm(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -858,16 +921,28 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
dict[str, AttentionMetadata] | list[dict[str, AttentionMetadata]] | None
),
) -> torch.Tensor:
"""Ready ``q`` for the attention kernel and publish this step's KV.
One launch does both. With ``accepts_unnormed_unroped_query`` the
attention kernel norms and rotates Q itself and reads it in its own
chunk-interleaved layout, so the Q half of the launch is a zero-pad to
``padded_heads`` -- and nothing at all once the shard is that wide.
"""
if not isinstance(attn_metadata, dict):
# Profile run: kernel doesn't fire; produce a padded tensor so
# downstream FlashMLA gets the right shape.
if self.n_local_heads < self.padded_heads:
return F.pad(
q,
(0, 0, 0, self.padded_heads - self.n_local_heads),
value=0.0,
)
return q
if self.n_local_heads >= self.padded_heads:
return q
if self.accepts_unnormed_unroped_query:
# Padding heads sit at the tail of every head-dim chunk in
# that layout, so no head-major pad of `q` reproduces it --
# and nothing reads it on a profile run.
return q.new_zeros((q.shape[0], self.padded_heads, q.shape[2]))
return F.pad(
q,
(0, 0, 0, self.padded_heads - self.n_local_heads),
value=0.0,
)
swa_metadata = cast(
"DeepseekSparseSWAMetadata | None",
@@ -886,23 +961,38 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
if cache_dtype == torch.uint8:
# fp8_ds_mla UE8M0 paged path. Horizontally fused:
# Q side: GPT-J RoPE, zero-filling the padding head slots; the
# kernel allocates and returns the padded q tensor.
# kernel allocates and returns the padded q tensor. An
# interleaved Q skips the RoPE its attention kernel owns
# and keeps only the pad, which q_head_padded=0 drops
# too once the shard is already padded_heads wide.
# KV side: GPT-J RoPE + UE8M0 FP8 quant + paged cache insert.
swa_kv_cache_2d = swa_kv_cache.view(swa_kv_cache.shape[0], -1)
return torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
pad_to = (
0
if self.accepts_unnormed_unroped_query
and self.n_local_heads == self.padded_heads
else self.padded_heads
)
q_padded = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
q,
kv,
swa_kv_cache_2d,
swa_metadata.slot_mapping,
positions,
cos_sin_cache,
self.padded_heads,
pad_to,
self.eps,
swa_metadata.block_size,
False,
False, # apply_q_norm: qr is normed before wq_b
self.kv_mxfp8,
not self.accepts_unnormed_unroped_query, # apply_q_rope
self.accepts_unnormed_unroped_query, # is_q_interleaved
)
return q if pad_to == 0 else q_padded
assert not self.accepts_unnormed_unroped_query, (
"the chunk-interleaved Q layout only pairs with a packed KV record"
)
# Plain-row path: the [num_blocks, block_size, 512] cache stores the KV
# row in its element dtype (no Q padding). bf16 rewrites q in place;
# per-tensor fp8 writes a separately-allocated fp8 q and quantizes the
@@ -959,7 +1049,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
# fp8_ds_mla is a UE8M0 block-scaled uint8 layout whose page rounds up
# to the decode kernel's TMA stride; plain bf16 / per-tensor fp8 rows
# use natural element-size pages.
uses_fp8_ds_mla_layout = self.kv_cache_dtype == "fp8_ds_mla"
uses_fp8_ds_mla_layout = self.kv_cache_dtype in ("fp8_ds_mla", "nvfp4_ds_mla")
return MLAAttentionSpec(
block_size=vllm_config.cache_config.block_size,
num_kv_heads=1,
@@ -972,7 +1062,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
# Packed record width; head_size stays semantic (512).
state_content_bytes=(
self.kv_bytes_per_token if uses_fp8_ds_mla_layout else None
self.compressed_bytes_per_token if uses_fp8_ds_mla_layout else None
),
)
@@ -1016,7 +1106,13 @@ class DeepseekV4IndexerCache(torch.nn.Module, AttentionLayerBase):
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
# head_dim already carries the fp8 scale padding
# tokens_per_state=1 for V3.2, >1 for DeepseekV4; same cache layout.
uses_fp8_ds_mla_layout = vllm_config.cache_config.cache_dtype == "fp8_ds_mla"
# nvfp4_ds_mla is packed too: its compressed record is NVFP4 but the
# sliding-window record stays the V4.1 MXFP8 one, so the indexer page
# takes the same alignment either way.
uses_fp8_ds_mla_layout = vllm_config.cache_config.cache_dtype in (
"fp8_ds_mla",
"nvfp4_ds_mla",
)
page_alignment = (
576 if uses_fp8_ds_mla_layout and not _use_v41_mxfp8_kv_record() else 512
)
@@ -43,10 +43,16 @@ from vllm.utils.math_utils import next_power_of_2
# dims each plus a pad byte.
# V4.1 (528 B): all 512 dims as fp8 e4m3 (RoPE included), then 16 UE8M0
# scales of 32 dims each. FlashMLA's ``ModelType::V41``.
# V4.1 NVFP4 (288 B): 512 e2m1 values packed two per byte (even element in
# the low nibble), then 32 e4m3 scales of 16 dims each.
# FlashMLA's ``ModelType::V41_FP4``, compressed cache only.
V4_BYTES_PER_TOKEN = 584
V41_BYTES_PER_TOKEN = 528
V41_QUANT_BLOCK = 32
V41_NUM_SCALES = 512 // V41_QUANT_BLOCK # 16
V41_NVFP4_BYTES_PER_TOKEN = 288
V41_NVFP4_QUANT_BLOCK = 16
V41_NVFP4_NUM_SCALES = 512 // V41_NVFP4_QUANT_BLOCK # 32
@triton.jit
@@ -496,6 +502,73 @@ def _dequantize_and_gather_k_mxfp8_kernel(
tl.store(output_row_ptr + d, dequant.to(tl.bfloat16))
@triton.jit
def _dequantize_and_gather_k_nvfp4_kernel(
out_ptr,
out_stride0,
out_stride1,
k_cache_ptr,
seq_lens_ptr,
block_table_ptr,
offset,
gather_lens_ptr,
max_blocks_per_seq: tl.constexpr,
head_dim: tl.constexpr, # 512
scale_dim: tl.constexpr, # 32
quant_block: tl.constexpr, # 16
cache_block_size: tl.constexpr,
block_stride: tl.constexpr,
):
"""Gather and dequantize V4.1 NVFP4 rows into a bf16 workspace."""
batch_idx = tl.program_id(0)
worker_id = tl.program_id(1)
num_workers = tl.num_programs(1)
seq_len = tl.load(seq_lens_ptr + batch_idx)
if gather_lens_ptr is not None: # noqa: SIM108
gather_len = tl.load(gather_lens_ptr + batch_idx)
else:
gather_len = seq_len
start_pos = seq_len - gather_len
packed_bytes: tl.constexpr = head_dim // 2
d = tl.arange(0, head_dim)
for i in range(worker_id, gather_len, num_workers):
pos = start_pos + i
block_in_seq = pos // cache_block_size
pos_in_block = pos % cache_block_size
physical_block_idx = tl.load(
block_table_ptr + batch_idx * max_blocks_per_seq + block_in_seq
)
page = k_cache_ptr + physical_block_idx.to(tl.int64) * block_stride
packed = tl.load(
page + pos_in_block * packed_bytes + tl.arange(0, packed_bytes)
)
# Even element in the low nibble, odd in the high nibble.
codes = tl.interleave((packed & 0xF).to(tl.int32), (packed >> 4).to(tl.int32))
# e2m1 magnitudes: 0, 0.5, 1, 1.5, 2, 3, 4, 6.
mag_code = codes & 7
e = (mag_code >> 1).to(tl.float32)
m = (mag_code & 1).to(tl.float32)
mag = tl.where(mag_code < 2, m * 0.5, (1.0 + m * 0.5) * tl.exp2(e - 1.0))
vals = tl.where(codes >= 8, -mag, mag)
sf = tl.load(
page
+ cache_block_size * packed_bytes
+ pos_in_block * scale_dim
+ tl.arange(0, scale_dim)
)
scale = sf.to(tl.float8e4nv, bitcast=True).to(tl.float32)
tiles = tl.reshape(vals, (scale_dim, quant_block))
dequant = tl.reshape(tiles * tl.reshape(scale, (scale_dim, 1)), (head_dim,))
output_row_ptr = out_ptr + batch_idx * out_stride0 + (offset + i) * out_stride1
tl.store(output_row_ptr + d, dequant.to(tl.bfloat16))
def dequantize_and_gather_k_cache_triton(
# [num_reqs, max_num_tokens, head_size]
out: torch.Tensor,
@@ -513,6 +586,25 @@ def dequantize_and_gather_k_cache_triton(
) -> None:
num_reqs = seq_lens.shape[0]
NUM_WORKERS = 128
if k_cache.shape[-1] == V41_NVFP4_BYTES_PER_TOKEN:
_dequantize_and_gather_k_nvfp4_kernel[(num_reqs, NUM_WORKERS)](
out,
out.stride(0),
out.stride(1),
k_cache,
seq_lens,
block_table,
offset,
gather_lens,
max_blocks_per_seq=block_table.shape[-1],
head_dim=512,
scale_dim=V41_NVFP4_NUM_SCALES,
quant_block=V41_NVFP4_QUANT_BLOCK,
cache_block_size=block_size,
block_stride=k_cache.stride(0),
)
return
if k_cache.shape[-1] == V41_BYTES_PER_TOKEN:
_dequantize_and_gather_k_mxfp8_kernel[(num_reqs, NUM_WORKERS)](
out,
@@ -581,14 +673,16 @@ def dequantize_and_gather_k_cache(
) -> None:
"""Dequantize and gather a paged DSv4 K cache.
The record is read off ``k_cache.shape[-1]``; see the module header.
The record is read off ``k_cache.shape[-1]``; see the module header. Only
the fp8 records have a CuteDSL gather, so NVFP4 always takes the Triton
path.
``use_fnuz`` MUST match the encoder of the specific cache being read:
``False`` for ``compressed_k_cache`` (Triton encoder is OCP everywhere),
``current_platform.is_fp8_fnuz()`` for ``swa_k_cache`` (C++ encoder
writes FNUZ on gfx942 and OCP on gfx950).
"""
if has_cutedsl():
if has_cutedsl() and k_cache.shape[-1] != V41_NVFP4_BYTES_PER_TOKEN:
# lazily import, otherwise some tests fail due to CUDA driver init failure.
from vllm.models.deepseek_v4.nvidia.ops.dequant_gather_k_cutedsl import (
_DEQUANT_GATHER_K_CACHE_CUTEDSL_KERNEL,
@@ -4,6 +4,7 @@
import torch
from vllm.models.deepseek_v4.common.ops.fused_indexer_q import _fp32x2_to_fp4x2
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
@@ -232,13 +233,14 @@ def rope_quant_insert(
The BF16 latent supplies both NoPE quantization and RoPE input. It is read
only for valid slots at group boundaries. The cache dtype selects the
layout: ``uint8`` is the fp8_ds_mla paged layout, whose record the
per-token byte width names -- 584 B for V4 (576 value bytes and eight
segregated UE8M0 scale bytes, including one zero padding scale) or 528 B
for V4.1 (512 MXFP8 value bytes covering the RoPE dims too, then 16 UE8M0
scales of 32 dims each). ``bfloat16`` and ``float8_e4m3fn`` are the plain
[448 NoPE | 64 RoPE] rows read by FlashInfer, the latter scaled by the
per-tensor ``fp8_scale``.
layout: ``uint8`` is a paged FlashMLA layout, whose record the per-token
byte width names -- 584 B for V4 (576 value bytes and eight segregated
UE8M0 scale bytes, including one zero padding scale), 528 B for V4.1
(512 MXFP8 value bytes covering the RoPE dims too, then 16 UE8M0 scales of
32 dims each), or 288 B for V4.1 NVFP4 (256 bytes of e2m1 pairs then 32
e4m3 scales of 16 dims each), which only the compressed cache uses.
``bfloat16`` and ``float8_e4m3fn`` are the plain [448 NoPE | 64 RoPE] rows
read by FlashInfer, the latter scaled by the per-tensor ``fp8_scale``.
"""
assert compress_ratio in (1, 2)
assert latent.shape[1] == 512 and latent.dtype == torch.bfloat16
@@ -249,14 +251,14 @@ def rope_quant_insert(
return
launch_kwargs = {"launch_pdl": False} if current_platform.is_cuda() else {}
if kv_cache.dtype == torch.uint8:
assert kv_cache.shape[-1] in (584, 528), (
kernel = {
584: _rope_quant_insert_kernel,
528: _rope_quant_insert_mxfp8_kernel,
288: _rope_quant_insert_nvfp4_kernel,
}.get(kv_cache.shape[-1])
assert kernel is not None, (
f"unsupported paged KV record width {kv_cache.shape[-1]}"
)
kernel = (
_rope_quant_insert_mxfp8_kernel
if kv_cache.shape[-1] == 528
else _rope_quant_insert_kernel
)
kernel[(num_tokens,)](
latent,
positions,
@@ -402,6 +404,62 @@ def _rope_quant_insert_mxfp8_kernel(
tl.store(scales + tl.arange(0, 16), encoded.to(tl.uint8))
@triton.jit
def _rope_quant_insert_nvfp4_kernel(
latent,
positions,
cos_sin,
cache,
cache_slots,
COS_STRIDE: tl.constexpr,
CACHE_STRIDE: tl.constexpr,
CACHE_BLOCK: tl.constexpr,
COMPRESS_RATIO: tl.constexpr,
SANITIZE_CACHE_NANS: tl.constexpr,
):
"""V4.1 NVFP4 record: RoPE, then e2m1 with one e4m3 scale per 16 dims.
The scale is ``amax / 6`` (6 is e2m1's largest magnitude) clamped to the
e4m3 range, with no per-tensor scale on top.
"""
t = tl.program_id(0)
slot = tl.load(cache_slots + t)
if slot < 0:
return
position = tl.load(positions + t)
if (position + 1) % COMPRESS_RATIO != 0:
return
d = tl.arange(0, 512)
normed = tl.load(latent + t.to(tl.int64) * 512 + d).to(tl.float32)
# NoPE pairs load (cos, sin) = (1, 0), so the rotation is the identity there.
even, odd = tl.split(tl.reshape(normed, (256, 2)))
pair = tl.arange(0, 256) - 224
cs = cos_sin + (position // COMPRESS_RATIO * COMPRESS_RATIO) * COS_STRIDE
c = tl.load(cs + tl.maximum(pair, 0), pair >= 0, other=1.0).to(tl.float32)
s = tl.load(cs + 32 + tl.maximum(pair, 0), pair >= 0, other=0.0).to(tl.float32)
rotated = tl.interleave(even * c - odd * s, odd * c + even * s)
if SANITIZE_CACHE_NANS:
rotated = tl.where(rotated == rotated, rotated, 0.0)
tiles = tl.reshape(rotated, (32, 16))
amax = tl.max(tl.abs(tiles), 1)
# 2**-9 is the smallest normal e4m3 magnitude; 448 the largest.
scale = tl.clamp(amax * (1.0 / 6.0), 0.001953125, 448.0).to(tl.float8e4nv)
# Round-to-nearest division: Triton's default div.full misplaces values
# that land exactly on an e2m1 tie.
scaled = tl.math.div_rn(tiles, tl.reshape(scale.to(tl.float32), (32, 1)))
lo, hi = tl.split(tl.reshape(tl.reshape(scaled, (512,)), (256, 2)))
page = cache + (slot // CACHE_BLOCK).to(tl.int64) * CACHE_STRIDE
tl.store(
page + (slot % CACHE_BLOCK) * 256 + tl.arange(0, 256),
_fp32x2_to_fp4x2(lo, hi),
)
scales = page + CACHE_BLOCK * 256 + (slot % CACHE_BLOCK) * 32
tl.store(scales + tl.arange(0, 32), scale.to(tl.uint8, bitcast=True))
@triton.jit
def _rope_plain_insert_kernel(
latent,
@@ -0,0 +1,86 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Weight permutations for FlashMLA's mega-attention kernel.
The kernel reads Q with 16-element head-dim chunks interleaved across heads and
writes O with 32-element chunks interleaved across the 8 heads of a ``wo_a``
group. Every permutation here satisfies ``fused = standard[perm]``, and is
applied once to ``wq_b`` rows and ``wo_a`` columns at load time so the
surrounding GEMMs produce and consume the kernel's layouts directly -- no
per-step shuffle.
"""
import torch
HEAD_DIM = 512
Q_CHUNK = 16
O_CHUNK = 32
WV_GROUP_SIZE = 8
def q_fused_permutation(num_heads: int, head_dim: int = HEAD_DIM) -> torch.Tensor:
"""``fused[(d // 16) * (H * 16) + h * 16 + d % 16] = standard[h * D + d]``."""
h = torch.arange(num_heads).view(num_heads, 1)
d = torch.arange(head_dim).view(1, head_dim)
fused_index = (d // Q_CHUNK) * (num_heads * Q_CHUNK) + h * Q_CHUNK + d % Q_CHUNK
perm = torch.empty(num_heads * head_dim, dtype=torch.long)
perm[fused_index.reshape(-1)] = torch.arange(num_heads * head_dim)
return perm
def o_fused_permutation(
heads_per_group: int = WV_GROUP_SIZE, head_dim: int = HEAD_DIM
) -> torch.Tensor:
"""``fused[(c * G + h) * 32 + j] = standard[h * D + c * 32 + j]`` per group."""
h = torch.arange(heads_per_group).view(-1, 1, 1)
c = torch.arange(head_dim // O_CHUNK).view(1, -1, 1)
j = torch.arange(O_CHUNK).view(1, 1, -1)
fused_index = (c * heads_per_group + h) * O_CHUNK + j
perm = torch.empty(heads_per_group * head_dim, dtype=torch.long)
perm[fused_index.reshape(-1)] = torch.arange(heads_per_group * head_dim)
return perm
def o_fused_chunk_permutation(
heads_per_group: int = WV_GROUP_SIZE, head_dim: int = HEAD_DIM
) -> torch.Tensor:
"""Per-32-element-chunk form of :func:`o_fused_permutation`, for scales."""
return o_fused_permutation(heads_per_group, head_dim)[::O_CHUNK] // O_CHUNK
def permute_q_to_fused(q: torch.Tensor) -> torch.Tensor:
"""``[N, H, D]`` standard layout -> the same shape in the fused layout."""
n, h, d = q.shape
perm = q_fused_permutation(h, d).to(q.device)
return q.reshape(n, h * d)[:, perm].view(n, h, d)
def _bytes_view(t: torch.Tensor) -> torch.Tensor:
"""Byte view of a 1-byte-element tensor, so fp8/ue8m0 can be gathered."""
return t.view(torch.uint8) if t.element_size() == 1 else t
def permute_wq_b_(
weight: torch.Tensor, weight_scale: torch.Tensor, num_local_heads: int
) -> None:
"""Permute an MXFP8 ``wq_b`` shard's rows and per-row scales, in place."""
head_dim = weight.shape[0] // num_local_heads
perm = q_fused_permutation(num_local_heads, head_dim).to(weight.device)
for t in (weight, weight_scale):
b = _bytes_view(t)
b.copy_(b[perm])
def permute_wo_a_(
weight: torch.Tensor,
weight_scale: torch.Tensor,
heads_per_group: int = WV_GROUP_SIZE,
) -> None:
"""Permute an MXFP8 ``wo_a`` shard's input columns and scales, in place."""
head_dim = weight.shape[1] // heads_per_group
perm = o_fused_permutation(heads_per_group, head_dim).to(weight.device)
w = _bytes_view(weight)
w.copy_(w[:, perm])
chunk_perm = o_fused_chunk_permutation(heads_per_group, head_dim)
s = _bytes_view(weight_scale)
s.copy_(s[:, chunk_perm.to(weight.device)])
+14
View File
@@ -477,8 +477,22 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
for layer in self.model.layers:
layer.ffn.finalize_mega_moe_weights()
def _finalize_attn(self) -> None:
"""Run the attention backend's post-load weight step on the draft layers.
They are ordinary DeepseekV4DecoderLayers, so they take the engine-wide
attention backend and owe it whatever the target model owes it -- mega
attention permutes wq_b / wo_a here and refuses to run without it.
Idempotent, like the target's.
"""
for layer in self.model.layers:
finalize = getattr(layer.attn, "finalize_loaded_weights", None)
if finalize is not None:
finalize()
def process_weights_after_loading(self) -> None:
self._finalize_moe()
self._finalize_attn()
def _remap_dspark_name(self, name: str) -> str | None:
"""Map a checkpoint ``mtp.{i}.*`` name to this model's parameter path.
@@ -0,0 +1,497 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DeepSeek V4.1 attention on FlashMLA's mega-attention kernel (SM100).
One kernel does Q RoPE, sparse attention, the inverse RoPE of the output and
its FP8 cast, and writes straight into the buffer ``wo_a`` consumes. So the
layer declares ``accepts_unnormed_unroped_query`` -- the kernel RoPEs Q itself,
leaving the fused KV insert only the zero-pad to the kernel's head count -- and
its ``_alloc_attn_out`` / ``_o_proj`` pair speaks QuantizedActivation instead
of bf16, since the output needs no rotation or quantization here.
``wq_b`` rows and ``wo_a`` columns are permuted once at load so the surrounding
GEMMs speak the kernel's chunk-interleaved layouts directly. A step's prefill
and decode segments write disjoint token ranges of one output buffer pair, so
a single ``wo_a`` einsum covers the whole step.
"""
from dataclasses import replace
from typing import TYPE_CHECKING, cast
import torch
from vllm.config import VllmConfig
from vllm.config.cache import CacheDType
from vllm.forward_context import get_forward_context
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Dynamic
from vllm.models.deepseek_v41.common.ops import (
combine_topk_swa_indices,
compute_global_topk_indices_and_lens,
dequantize_and_gather_k_cache,
)
from vllm.models.deepseek_v41.common.ops.fused_layout import (
WV_GROUP_SIZE,
permute_wo_a_,
permute_wq_b_,
)
from vllm.models.deepseek_v41.nvidia.flashmla import DeepseekV4FlashMLAAttention
from vllm.models.deepseek_v41.sparse_mla import (
DeepseekV4FlashMLAMetadata,
FlashMLAMegaAttnBackend,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import fp8_einsum, get_tma_aligned_size
from vllm.utils.math_utils import round_up
from vllm.v1.attention.ops.flashmla import is_flashmla_sparse_supported
from vllm.v1.worker.workspace import current_workspace_manager
if TYPE_CHECKING:
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
_HEAD_DIM_V = 512
_ROPE_DIM = 64
# The kernel always emits per-32 scales, whatever num_per_channels says.
_QUANT_GROUP = 32
# Fixed kernel conventions: GPT-J (non-neox) RoPE, TMA-aligned col-major
# scales, round-to-nearest, packed ue8m0 -- i.e. what DeepGEMM's fp8_einsum
# consumes. Q norm is off: the V4.1 layer norms qr before wq_b.
_ROPE_ARGS = (False, _ROPE_DIM)
_SF_ARGS = (_QUANT_GROUP, True, True, True)
# Decode always goes through the mega kernel: there is no batch size at which
# falling back to the split-KV decode pays, so there is no minimum-token
# threshold and no fallback path to maintain. What decides whether this layer
# is worth selecting at all is TP, not the batch size.
#
# Measured on one GB300 with benchmarks/kernels/benchmark_dsv41_mega_attn.py,
# CUDA graphs on, counting every per-step op each path runs (64-head model,
# topk_swa=128, topk_extra=512, NVFP4 compressed cache), as mega / split-KV
# microseconds:
#
# s_q 1 32 128 256 512
# TP1 (64 live heads) 29/29 31/31 31/40 53/72 96/140
# TP2 (32 live) 29/29 31/30 35/35 61/63 110/119
# TP4 (16 live) 29/29 31/29 34/34 59/58 109/109
# TP8 (8 live) 29/29 31/29 33/33 59/57 107/104
#
# The kernel absorbs the inverse RoPE and the FP8 cast, whose cost scales with
# *live* heads, while its own cost scales with *padded* heads -- always 64,
# since FlashMLA's FP8 decode takes only h_q in {64, 128}. So it wins at TP1
# (up to 1.45x, and TP1 is also where the fused insert skips the Q pad
# entirely) and from TP2 down the two cancel to within a few percent, with a
# fixed ~2 us penalty at small s_q where the padded-head cost dominates.
# Measure before assuming a win at high TP -- and measure under CUDA graphs:
# in eager mode launch overhead swamps both paths and overstates the fused one
# several-fold.
def is_flashmla_mega_attn_supported() -> tuple[bool, str | None]:
"""Whether FlashMLA's fused mega-attention kernels are usable here."""
is_available, maybe_reason = is_flashmla_sparse_supported()
if not is_available:
return False, maybe_reason
if not current_platform.is_device_capability_family(100):
return False, "FlashMLA mega attention requires sm_10x GPUs."
if not hasattr(torch.ops._flashmla_C, "fused_norm_rope_attn_rope_cast_decode"):
return False, "vllm._flashmla_C was built without the mega attention ops."
if not hasattr(torch.ops._flashmla_C, "permute_q_b_proj"):
return False, "vllm._flashmla_C predates the mega attention weight permutes."
return True, None
def alloc_mega_attn_output(
num_tokens: int,
n_wv_group: int,
device: torch.device,
) -> QuantizedActivation:
"""Allocate the output buffer pair the mega-attention kernels write into.
One pair per forward step: the prefill and decode segments fill disjoint
token ranges, so a single ``fp8_einsum`` over all ``num_tokens`` can
consume the result instead of one call per segment.
The scale buffer is MN-major -- stride 1 along tokens, head-dim stride
``ceil4(num_tokens)`` -- which is both what the kernel requires of a
caller-provided buffer and what DeepGEMM expects for this N.
"""
d = WV_GROUP_SIZE * _HEAD_DIM_V
data = torch.empty(
(num_tokens, n_wv_group, d), dtype=torch.float8_e4m3fn, device=device
)
aligned = get_tma_aligned_size(num_tokens, torch.int32.itemsize)
scale = torch.empty(
(n_wv_group, d // (_QUANT_GROUP * 4), aligned),
dtype=torch.int32,
device=device,
).permute(2, 0, 1)[:num_tokens]
return QuantizedActivation(
data=data,
scale=scale,
orig_dtype=torch.bfloat16,
orig_shape=data.shape,
quant_key=kMxfp8Dynamic,
)
def _token_slice(out: QuantizedActivation, start: int, end: int) -> QuantizedActivation:
"""The ``[start, end)`` token slice of a mega-attention output buffer."""
data = out.data[start:end]
return replace(out, data=data, scale=out.scale[start:end], orig_shape=data.shape)
class DeepseekV4MegaAttnAttention(DeepseekV4FlashMLAAttention):
"""FlashMLA mega-attention layer for DeepSeek V4.1 (SM100)."""
backend_cls = FlashMLAMegaAttnBackend
@classmethod
def is_available_for(cls, vllm_config: VllmConfig) -> bool:
"""Whether this layer can serve the configured model on this device.
The kernel is SM100-only and wants ``WV_GROUP_SIZE`` heads per ``wo_a``
group, which stops holding once TP divides the head count far enough
(TP16 on a 64-head model leaves 4). ``__init__`` raises on both, so the
default-backend selector asks here first rather than turning an
unsupported topology into a startup crash.
"""
if not is_flashmla_mega_attn_supported()[0]:
return False
config = vllm_config.model_config.hf_config
config = getattr(config, "text_config", config)
n_heads = getattr(config, "num_attention_heads", 0) or 0
n_groups = getattr(config, "o_groups", 0) or 0
tp_size = vllm_config.parallel_config.tensor_parallel_size
if n_heads % tp_size or n_groups % tp_size:
return False
n_local_heads, n_local_groups = n_heads // tp_size, n_groups // tp_size
if not n_local_groups or n_local_heads % WV_GROUP_SIZE:
return False
return n_local_heads // n_local_groups == WV_GROUP_SIZE
def __init__(self, vllm_config: VllmConfig, *args, **kwargs) -> None:
super().__init__(vllm_config, *args, **kwargs)
if self.n_local_heads % WV_GROUP_SIZE:
raise ValueError(
f"{self.prefix}: mega attention needs the local head count "
f"({self.n_local_heads}) to be a multiple of {WV_GROUP_SIZE}."
)
if self.n_local_heads // self.n_local_groups != WV_GROUP_SIZE:
raise ValueError(
f"{self.prefix}: mega attention needs {WV_GROUP_SIZE} heads per "
"wo_a group."
)
self.n_wv_group = self.padded_heads // WV_GROUP_SIZE
self._fused_layouts_ready = False
# ---- interface contract ------------------------------------------------
@property
def accepts_unnormed_unroped_query(self) -> bool:
return True
@property
def packed_kv_cache_dtype(self) -> CacheDType:
# This kernel is the only one that reads an NVFP4 compressed cache, so
# it is what an unspecific --kv-cache-dtype resolves to here.
return "nvfp4_ds_mla"
def _alloc_attn_out(
self, num_tokens: int, hidden_states: torch.Tensor
) -> QuantizedActivation:
return alloc_mega_attn_output(num_tokens, self.n_wv_group, hidden_states.device)
def _o_proj(
self, attn_out: QuantizedActivation, positions: torch.Tensor
) -> torch.Tensor:
"""Grouped ``wo_a`` then ``wo_b`` over the kernel's quantized output.
The inverse RoPE and the FP8 cast happened inside the attention
kernel, and ``wo_a`` is permuted for its output layout, so there is
nothing to rotate or quantize here and ``positions`` is unused. The
scale already is DeepGEMM's packed-ue8m0 MN-major layout, so the
einsum consumes the kernel's output with no repacking.
"""
groups = self.n_local_groups
z = torch.empty(
(attn_out.data.shape[0], groups, self.o_lora_rank),
dtype=torch.bfloat16,
device=attn_out.data.device,
)
fp8_einsum(
"bhr,hdr->bhd",
(attn_out.data[:, :groups], attn_out.scale[:, :groups]),
(self.wo_a.weight, self.wo_a.weight_scale),
z,
recipe=self._einsum_recipe,
)
return self.wo_b(z.flatten(1))
# ---- weights -----------------------------------------------------------
def finalize_loaded_weights(self) -> None:
"""Permute wq_b rows / wo_a columns into the kernel's layouts.
Idempotent: a second post-load pass must not permute twice.
"""
if self._fused_layouts_ready:
return
permute_wq_b_(
self.wq_b.weight.data, self.wq_b.weight_scale.data, self.n_local_heads
)
permute_wo_a_(
self.wo_a.weight.data,
self.wo_a.weight_scale.data,
self.n_local_heads // self.n_local_groups,
)
self._fused_layouts_ready = True
# ---- forward -----------------------------------------------------------
def forward_mqa(
self,
q: torch.Tensor,
kv: torch.Tensor,
positions: torch.Tensor,
output: QuantizedActivation,
) -> None:
if not self._fused_layouts_ready:
raise RuntimeError(
f"{self.prefix}: wq_b / wo_a were never permuted for the mega "
"attention kernel; refusing to run with mismatched layouts."
)
attn_metadata = get_forward_context().attn_metadata
if attn_metadata is None:
# Warmup dummy run: reserve the prefill workspace, produce zeros.
self._reserve_prefill_workspace(q)
output.data.zero_()
output.scale.zero_()
return
assert isinstance(attn_metadata, dict)
# q was zero-padded to the kernel's head count by the fused KV insert.
assert q.shape[1] == self.padded_heads
flashmla_metadata = cast(
DeepseekV4FlashMLAMetadata | None,
attn_metadata.get(self.compressed_cache_prefix)
if self.compressed_cache_prefix is not None
else None,
)
swa_metadata = cast(
"DeepseekSparseSWAMetadata", attn_metadata[self.swa_cache_layer.prefix]
)
# The kernel takes int32 RoPE positions.
positions_int32 = positions.to(torch.int32)
num_decode_tokens = swa_metadata.num_decode_tokens
if swa_metadata.num_prefills > 0:
self._forward_prefill_mega(
q[num_decode_tokens:],
positions_int32[num_decode_tokens:],
flashmla_metadata,
swa_metadata,
output,
num_decode_tokens,
)
if swa_metadata.num_decodes > 0:
self._forward_decode_mega(
q[:num_decode_tokens],
positions_int32[:num_decode_tokens],
flashmla_metadata,
swa_metadata,
_token_slice(output, 0, num_decode_tokens),
)
def _reserve_prefill_workspace(self, q: torch.Tensor) -> None:
swa_only = self.compress_ratio == 0
n = 0 if swa_only else -(-self.max_model_len // self.compress_ratio)
m = n + self.window_size + self.max_num_batched_tokens
if swa_only:
top_k = 0
else:
assert self.topk_indices_buffer is not None
top_k = self.topk_indices_buffer.shape[-1]
combined_topk = round_up(top_k + self.window_size + self.max_image_tokens, 128)
current_workspace_manager().get_simultaneous(
((self.PREFILL_CHUNK_SIZE, m, q.shape[-1]), torch.bfloat16),
((self.max_num_batched_tokens, combined_topk), torch.int32),
((self.max_num_batched_tokens,), torch.int32),
)
def _decode_compressed_kv_and_topk(
self,
flashmla_metadata: DeepseekV4FlashMLAMetadata | None,
swa_metadata: "DeepseekSparseSWAMetadata",
) -> tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]:
"""The compressed cache and its ``[n, topk]`` slot indices / lengths."""
if self.compress_ratio == 0:
return None, None, None
assert flashmla_metadata is not None
assert swa_metadata.is_valid_token is not None
assert self.topk_indices_buffer is not None
num_decode_tokens = swa_metadata.num_decode_tokens
indices, lens = compute_global_topk_indices_and_lens(
self.topk_indices_buffer[:num_decode_tokens],
swa_metadata.token_to_req_indices,
flashmla_metadata.block_table[: swa_metadata.num_decodes],
flashmla_metadata.block_size // self.compress_ratio,
swa_metadata.is_valid_token[:num_decode_tokens],
)
return self._compressed_kv_cache().unsqueeze(-2), indices, lens
def _forward_decode_mega(
self,
q: torch.Tensor,
positions_int32: torch.Tensor,
flashmla_metadata: DeepseekV4FlashMLAMetadata | None,
swa_metadata: "DeepseekSparseSWAMetadata",
out: QuantizedActivation,
) -> None:
"""Mega attention over the paged quantized caches, in place into ``out``.
The caches are ``[num_blocks, page, 1, bytes]``, the record taken from
``bytes`` (528 V4.1 fp8, 288 V4.1 NVFP4; NVFP4 only as the compressed
cache beside an fp8 SWA one), and the indices ``[s_q, topk]`` int32
slot ids (``block * page + offset``, ``-1`` invalid).
"""
extra_cache, extra_idx, extra_len = self._decode_compressed_kv_and_topk(
flashmla_metadata, swa_metadata
)
assert swa_metadata.decode_swa_indices is not None
torch.ops._flashmla_C.fused_norm_rope_attn_rope_cast_decode(
q,
self.swa_cache_layer.kv_cache.unsqueeze(-2),
swa_metadata.decode_swa_indices.view(q.shape[0], -1),
self.scale,
_HEAD_DIM_V,
self.attn_sink,
swa_metadata.decode_swa_lens,
extra_cache,
extra_idx,
extra_len,
False, # enable_q_norm
0.0, # rms_norm_eps
positions_int32,
*_ROPE_ARGS,
self.rotary_emb.cos_sin_cache,
self.n_wv_group,
*_SF_ARGS,
out.data,
out.scale,
)
def _forward_prefill_mega(
self,
q: torch.Tensor,
positions_int32: torch.Tensor,
flashmla_metadata: DeepseekV4FlashMLAMetadata | None,
swa_metadata: "DeepseekSparseSWAMetadata",
out: QuantizedActivation,
token_base: int,
) -> None:
swa_only = self.compress_ratio == 0
num_decodes = swa_metadata.num_decodes
num_decode_tokens = swa_metadata.num_decode_tokens
seq_lens = swa_metadata.prefill_seq_lens
gather_lens = swa_metadata.prefill_gather_lens
query_start_loc_cpu = swa_metadata.query_start_loc_cpu
query_start_loc = swa_metadata.query_start_loc
assert seq_lens is not None and gather_lens is not None
assert query_start_loc_cpu is not None and query_start_loc is not None
prefill_token_base = query_start_loc_cpu[num_decodes]
assert self.topk_indices_buffer is not None
topk_indices = self.topk_indices_buffer[num_decode_tokens:][
: swa_metadata.num_prefill_tokens
]
top_k = 0 if swa_only else topk_indices.shape[-1]
chunk_plan = swa_metadata.get_prefill_chunk_plan(
compress_ratio=self.compress_ratio,
prefill_chunk_size=self.PREFILL_CHUNK_SIZE,
has_compressed=not swa_only,
)
assert chunk_plan, "prefill chunk plan must be non-empty when num_prefills > 0"
workspace_manager = current_workspace_manager()
combined_topk = round_up(top_k + self.window_size + self.max_image_tokens, 128)
for chunk_start, chunk_end, chunk_n, chunk_m in chunk_plan:
chunk_size = chunk_end - chunk_start
kv_ws, idx_ws, lens_ws = workspace_manager.get_simultaneous(
((chunk_size, chunk_m, q.shape[-1]), torch.bfloat16),
((self.max_num_batched_tokens, combined_topk), torch.int32),
((self.max_num_batched_tokens,), torch.int32),
)
if not swa_only:
assert flashmla_metadata is not None
dequantize_and_gather_k_cache(
kv_ws[:chunk_size],
self._compressed_kv_cache(),
seq_lens=seq_lens[chunk_start:chunk_end] // self.compress_ratio,
gather_lens=None,
block_table=flashmla_metadata.block_table[num_decodes:][
chunk_start:chunk_end
],
block_size=flashmla_metadata.block_size // self.compress_ratio,
offset=0,
)
dequantize_and_gather_k_cache(
kv_ws[:chunk_size],
self.swa_cache_layer.kv_cache,
seq_lens=seq_lens[chunk_start:chunk_end],
gather_lens=gather_lens[chunk_start:chunk_end],
block_table=swa_metadata.block_table[num_decodes:][
chunk_start:chunk_end
],
block_size=swa_metadata.block_size,
offset=chunk_n,
)
first, last = num_decodes + chunk_start, num_decodes + chunk_end
qs = int(query_start_loc_cpu[first] - prefill_token_base)
qe = int(query_start_loc_cpu[last] - prefill_token_base)
combined_indices, combined_lens = combine_topk_swa_indices(
topk_indices[qs:qe],
query_start_loc[first : last + 1],
seq_lens[chunk_start:chunk_end],
gather_lens[chunk_start:chunk_end],
self.window_size,
self.compress_ratio,
top_k,
chunk_m,
chunk_n,
out=(idx_ws[: qe - qs], lens_ws[: qe - qs]),
left_visible=(
swa_metadata.prefill_left_visible[
num_decode_tokens + qs : num_decode_tokens + qe
]
if swa_metadata.prefill_left_visible is not None
else None
),
right_visible=(
swa_metadata.prefill_right_visible[
num_decode_tokens + qs : num_decode_tokens + qe
]
if swa_metadata.prefill_right_visible is not None
else None
),
max_image_tokens=self.max_image_tokens,
)
chunk_out = _token_slice(out, token_base + qs, token_base + qe)
# Mega attention over the gathered non-paged bf16 KV (RoPE already
# applied), in place into this chunk's token range. `indices`
# entries outside [0, s_kv) skip.
torch.ops._flashmla_C.fused_norm_rope_attn_rope_cast_fwd(
q[qs:qe],
kv_ws.view(-1, 1, q.shape[-1]),
combined_indices.unsqueeze(1),
self.scale,
_HEAD_DIM_V,
self.attn_sink,
combined_lens,
False, # enable_q_norm
0.0, # rms_norm_eps
positions_int32[qs:qe],
*_ROPE_ARGS,
self.rotary_emb.cos_sin_cache,
self.n_wv_group,
*_SF_ARGS,
chunk_out.data,
chunk_out.scale,
)
@@ -201,7 +201,8 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention):
def get_padded_num_q_heads(cls, num_heads: int) -> int:
return _pad_to_supported_q_heads(num_heads)
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
o = attn_out[:, : self.n_local_heads, :]
return deep_gemm_fp8_o_proj(
o,
positions,
@@ -562,7 +563,8 @@ class DeepseekV4FlashInferSM120Attention(DeepseekV4Attention):
def get_padded_num_q_heads(cls, num_heads: int) -> int:
return _pad_to_supported_q_heads(num_heads)
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
o = attn_out[:, : self.n_local_heads, :]
return deep_gemm_fp8_o_proj(
o,
positions,
+2 -1
View File
@@ -58,7 +58,8 @@ class DeepseekV4FlashMLAAttention(DeepseekV4Attention):
self._o_proj_block_size
)
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
o = attn_out[:, : self.n_local_heads, :]
return deep_gemm_fp8_o_proj(
o,
positions,
+28 -2
View File
@@ -64,6 +64,9 @@ from vllm.models.deepseek_v4.nvidia.model import (
make_deepseek_v4_expert_params_mapping,
)
from vllm.models.deepseek_v41.attention import DeepseekV4Attention
from vllm.models.deepseek_v41.nvidia.flash_mla_mega_attn import (
DeepseekV4MegaAttnAttention,
)
from vllm.models.deepseek_v41.nvidia.flashinfer_sparse import (
DeepseekV4FlashInferMLAAttention,
DeepseekV4FlashInferSM120Attention,
@@ -119,8 +122,9 @@ def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]:
The generic CUDA backend selector does not instantiate DSv4 layers directly,
so map generic sparse-MLA choices to the DSv4-specialized attention class.
Without an explicit backend, SM12 defaults to FlashInfer while the other
CUDA arches keep the FlashMLA path.
Without an explicit backend: SM12 takes FlashInfer, SM100 takes mega
attention where the topology allows it, and everything else keeps the
FlashMLA path.
"""
backend = vllm_config.attention_config.backend
device_capability = current_platform.get_device_capability()
@@ -140,6 +144,8 @@ def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]:
if device_capability is not None and device_capability.major == 12:
return DeepseekV4FlashInferSM120Attention
return DeepseekV4FlashInferMLAAttention
if backend is AttentionBackendEnum.FLASHMLA_MEGA_ATTN_DSV41:
return DeepseekV4MegaAttnAttention
if backend in (
AttentionBackendEnum.FLASHMLA_SPARSE,
AttentionBackendEnum.FLASHMLA_SPARSE_DSV4,
@@ -149,6 +155,14 @@ def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]:
if device_capability is not None and device_capability.major == 12:
return DeepseekV4FlashInferSM120Attention
# Mega attention is the SM100 default: it fuses Q RoPE, sparse attention,
# the output's inverse RoPE and its FP8 cast into one launch, and brings
# the 288 B NVFP4 compressed record -- the format the reference
# implementation itself stores. It declines topologies it cannot serve
# (non-SM100, TP that leaves fewer than WV_GROUP_SIZE heads per wo_a
# group, a build without the kernel), which then fall through to FlashMLA.
if DeepseekV4MegaAttnAttention.is_available_for(vllm_config):
return DeepseekV4MegaAttnAttention
return DeepseekV4FlashMLAAttention
@@ -936,6 +950,17 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
for layer in islice(self.layers, self.start_layer, self.end_layer):
layer.ffn.finalize_mega_moe_weights()
def finalize_mega_attn_weights(self) -> None:
"""Permute wq_b / wo_a into FlashMLA's mega-attention layouts.
A no-op for every other attention layer, and idempotent, so a second
post-load pass cannot permute twice.
"""
for layer in islice(self.layers, self.start_layer, self.end_layer):
finalize = getattr(layer.attn, "finalize_loaded_weights", None)
if finalize is not None:
finalize()
def finalize_mhc_broadcast_weights(self) -> None:
if not get_pp_group().is_first_rank or self.start_layer >= self.end_layer:
return
@@ -1190,6 +1215,7 @@ class DeepseekV41LLMForCausalLM(
def process_weights_after_loading(self) -> None:
self.model.finalize_mega_moe_weights()
self.model.finalize_mhc_broadcast_weights()
self.model.finalize_mega_attn_weights()
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
return self.model.get_expert_mapping()
+51
View File
@@ -229,3 +229,54 @@ class DeepseekV4FlashMLABackend(DeepseekV4SparseMLABackend):
@staticmethod
def get_builder_cls() -> type[DeepseekV4FlashMLAMetadataBuilder]:
return DeepseekV4FlashMLAMetadataBuilder
class FlashMLAMegaAttnBackend(DeepseekV4FlashMLABackend):
"""FlashMLA's mega-attention kernel: Q RoPE + sparse attention + inverse
RoPE + FP8 cast of the output, in one launch.
Same metadata and KV-cache geometry as ``FLASHMLA_SPARSE_DSV41`` -- what
differs is the attention layer's interface contract (it takes an unnormed,
unroped Q and returns an already-inverse-RoPE'd, quantized output) and the
extra ``nvfp4_ds_mla`` compressed-cache record only this kernel can read.
SM100 only; the kernel has no SM90 instantiation.
"""
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
"auto",
"fp8_ds_mla",
"fp8", # alias for fp8_ds_mla
"nvfp4_ds_mla", # V4.1 fp8 SWA cache + NVFP4 compressed cache
]
@staticmethod
def get_name() -> str:
return "FLASHMLA_MEGA_ATTN_DSV41"
@staticmethod
def get_supported_kernel_block_sizes() -> list[int | MultipleOf]:
return [128]
@classmethod
def supports_compute_capability(cls, capability: DeviceCapability) -> bool:
return capability.major == 10
@classmethod
def supports_combination(
cls,
head_size: int,
dtype: torch.dtype,
kv_cache_dtype: CacheDType | None,
block_size: int | None,
use_mla: bool,
has_sink: bool,
use_sparse: bool,
use_mm_prefix: bool,
device_capability: DeviceCapability,
) -> str | None:
# Imported here: the layer module imports this backend class.
from vllm.models.deepseek_v41.nvidia.flash_mla_mega_attn import (
is_flashmla_mega_attn_supported,
)
return is_flashmla_mega_attn_supported()[1]
+729
View File
@@ -0,0 +1,729 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Custom Sparse Attention Indexer layers."""
import torch
import vllm.envs as envs
from vllm import _custom_ops # noqa: F401 # registers the torch.ops._C kernels
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import get_current_vllm_config_or_none
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.custom_op import CustomOp
from vllm.models.glm5next.amd.ops import kpool_compress as kpool_ops
from vllm.models.glm5next.common.sparse_indexer import (
RADIX_TOPK_WORKSPACE_SIZE,
_build_decode_scatter_indices,
_decode_topk_seq_lens,
_fill_causal_indices,
_fill_short_decode_causal_indices,
_gather_workspace_shapes,
_scatter_decode_tokens_by_request,
kv_cache_as_quant_view,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import (
LayerNameType,
_encode_layer_name,
_resolve_layer_name,
)
from vllm.v1.attention.backends.mla.indexer import (
DeepseekV32IndexerMetadata,
)
from vllm.v1.attention.ops.common import pack_seq_triton, unpack_seq_triton
from vllm.v1.worker.workspace import current_workspace_manager
logger = init_logger(__name__)
# kpool write helper: form pools from the current token batch and compress them
# into the index K cache via the fused Triton kernel.
def _kpool_compress_insert(
k: torch.Tensor,
gate_score: torch.Tensor,
ape: torch.Tensor,
kv_cache: torch.Tensor,
slot_mapping: torch.Tensor,
kpool: int,
head_dim: int,
round_scale: bool,
) -> None:
"""Pool ``kpool`` consecutive tokens into one fp8 K and write at pool slots.
``slot_mapping`` is pool-granular (compress_ratio == kpool on the spec):
only the *last* token of each complete pool carries a valid (>=0) slot;
intra-pool tokens are -1. Every position is treated as a pool-completion
candidate and non-completions are masked off inside the kernel. Compacting
the valid rows first costs two device syncs on the eager prefill path and
buys nothing numerically. Assumes pool-aligned chunk starts.
"""
n = slot_mapping.shape[0]
# No pool can complete in a batch smaller than one pool; also keeps the
# clamped gather indices below in bounds.
if n < kpool:
return
pos = torch.arange(n, device=k.device)
valid = slot_mapping >= 0
# Drop pools whose start falls before the batch (leading padding); their
# gate/k data is undefined anyway.
write_mask = valid & (pos >= kpool - 1)
offs = torch.arange(kpool, device=k.device)
idx = (pos - (kpool - 1)).clamp_min(0)[:, None] + offs[None, :]
kpool_ops.kpool_compress_and_write_cache(
kv_cache,
k[idx], # [n, kpool, head_dim]
gate_score[idx],
ape,
slot_mapping.to(torch.int64),
pool_size=kpool,
head_dim=head_dim,
write_mask=write_mask,
round_scale=round_scale,
write_cache=True,
return_compressed=False,
)
@eager_break_during_capture
def sparse_attn_indexer_kpool(
hidden_states: torch.Tensor,
k_cache_prefix: LayerNameType,
kv_cache: torch.Tensor,
q_quant: torch.Tensor,
q_scale: torch.Tensor | None,
k: torch.Tensor,
weights: torch.Tensor,
quant_block_size: int,
scale_fmt: str | None,
topk_tokens: int,
head_dim: int,
max_model_len: int,
total_seq_lens: int,
topk_indices_buffer: torch.Tensor,
skip_k_cache_insert: bool,
use_fp4_cache: bool = False,
# kpool params (Plan-A: gate is consumed at write time and read back at
# topk time to softmax-weight the pool).
gate_score: torch.Tensor | None = None,
compress_ape: torch.Tensor | None = None,
index_kpool: int = 1,
positions: torch.Tensor | None = None,
# Paged tail cache (in-progress pool's raw K + gate score), replacing the
# transient _DECODE_TAIL ring. tail_prefix resolves attn_metadata[tail_prefix]
# for the tail group's token-granular slot_mapping. None on the dummy/profiling
# path and when the tail cache is disabled.
tail_kv_cache: torch.Tensor | None = None,
tail_prefix: str | None = None,
) -> torch.Tensor:
# careful! this will be None in dummy run
attn_metadata = get_forward_context().attn_metadata
fp8_dtype = current_platform.fp8_dtype()
k_cache_prefix = _resolve_layer_name(k_cache_prefix)
# assert isinstance(attn_metadata, dict)
if not isinstance(attn_metadata, dict):
# Reserve workspace for indexer during profiling run
values_spec, scales_spec = _gather_workspace_shapes(
total_seq_lens, head_dim, fp8_dtype, use_fp4_cache
)
current_workspace_manager().get_simultaneous(
values_spec,
scales_spec,
((RADIX_TOPK_WORKSPACE_SIZE,), torch.uint8),
)
# Reserve profiler-visible memory for the worst-case decode logits,
# whose shape is [B * next_n, max_model_len]. This profiling branch
# returns before invoking the logits kernel itself.
cfg = get_current_vllm_config_or_none()
worst_decode_tokens = 0
if cfg is not None:
sched = cfg.scheduler_config
num_spec = (
cfg.speculative_config.num_speculative_tokens
if cfg.speculative_config is not None
else 0
)
worst_decode_tokens = min(
sched.max_num_seqs * (num_spec + 1),
sched.max_num_batched_tokens,
)
# float32 logits -> 4 bytes/element; uint8 sentinel so elems == bytes.
decode_logits_elems = worst_decode_tokens * max_model_len * 4
prefill_cap_elems = envs.VLLM_SPARSE_INDEXER_MAX_LOGITS_MB * 1024 * 1024
max_logits_elems = max(decode_logits_elems, prefill_cap_elems)
_ = torch.empty(
max_logits_elems, dtype=torch.uint8, device=hidden_states.device
)
return topk_indices_buffer
attn_metadata_narrowed = attn_metadata[k_cache_prefix]
assert isinstance(attn_metadata_narrowed, DeepseekV32IndexerMetadata)
slot_mapping = attn_metadata_narrowed.slot_mapping
has_decode = attn_metadata_narrowed.num_decodes > 0
has_prefill = attn_metadata_narrowed.num_prefills > 0
num_decode_tokens = attn_metadata_narrowed.num_decode_tokens
# q_scale is required iff the FP4 cache path is enabled; the FP8 path
# folds the Q scale into `weights` inside fused_indexer_q_rope_quant.
if use_fp4_cache:
assert q_scale is not None, "use_fp4_cache=True requires q_scale"
else:
assert q_scale is None, "q_scale must be None when use_fp4_cache=False"
# During speculative decoding, k may be padded to the CUDA graph batch
# size while slot_mapping only covers actual tokens. Truncate k to avoid
# out-of-bounds reads in the kernel.
num_tokens = slot_mapping.shape[0]
if k is not None:
k = k[:num_tokens]
if not skip_k_cache_insert:
assert not use_fp4_cache, "Unfused FP4 Insert is not supported yet"
if index_kpool > 1 and gate_score is not None and compress_ape is not None:
# kpool prefill write: pool kpool consecutive prefill tokens via
# softmax(gate+ape)-weighted sum -> Hadamard -> fp8 -> pool slots.
# Decode tokens (the first num_decode_tokens in the batch) cannot be
# pooled here — their pool's earlier tokens are not in this batch —
# so they are deferred to the tail-buffer kernel in has_decode.
# compress_ratio == index_kpool makes slot_mapping pool-granular.
n_prefill = num_tokens - num_decode_tokens
if n_prefill > 0:
# decode tokens are batched first; prefill tokens follow.
prefill_slice = slice(num_decode_tokens, num_tokens)
_kpool_compress_insert(
k[prefill_slice],
gate_score[prefill_slice],
compress_ape,
kv_cache,
slot_mapping[prefill_slice],
index_kpool,
head_dim,
round_scale=(scale_fmt is not None),
)
# Persist each request's incomplete prefill pool so decode can
# finish it, including after PD transfer. Tail slots use
# ``pos % kpool`` within the request's tail block. Processing
# only the batch's trailing tokens would miss all but the last
# request in a multi-request prefill.
if tail_kv_cache is not None and tail_prefix is not None:
tail_meta = attn_metadata.get(_resolve_layer_name(tail_prefix))
if tail_meta is not None:
assert isinstance(tail_meta, DeepseekV32IndexerMetadata)
kpool_ops.kpool_seed_tail_cache(
tail_kv_cache,
k[prefill_slice],
gate_score[prefill_slice],
tail_meta.slot_mapping[prefill_slice],
index_kpool,
head_dim,
)
else:
# standard: per-token fp8 quant + scatter (all tokens).
assert scale_fmt is not None
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
indexer_k_quant_and_cache_triton,
)
indexer_k_quant_and_cache_triton(
k,
kv_cache,
slot_mapping,
quant_block_size,
scale_fmt,
)
topk_indices_buffer[: hidden_states.shape[0]] = -1
if has_prefill:
prefill_metadata = attn_metadata_narrowed.prefill
assert prefill_metadata is not None
# Short sequences select every pool, so skip sparse scoring and fill
# the top-k buffer with all causal token indices. The index-K cache was
# already written above.
n_prefill_sf = num_tokens - num_decode_tokens
# Host-side short-prefill predicate: max_prefill_seq_len is computed
# in the metadata builder (exact for prefill rows) and equals
# positions[prefill_slice].max() + 1, so this replaces a
# positions.max().item() device sync per layer. -1 (unknown metadata)
# falls back to the device-side check.
if prefill_metadata.max_prefill_seq_len >= 0:
short_prefill = (
n_prefill_sf > 0
and positions is not None
and prefill_metadata.max_prefill_seq_len <= topk_tokens
)
else:
short_prefill = (
n_prefill_sf > 0
and positions is not None
and int(positions[num_decode_tokens:num_tokens].max().item()) + 1
<= topk_tokens
)
if short_prefill:
# short_prefill is only True when positions is not None (above),
# but narrow explicitly for the indexer below.
assert positions is not None
_pos = positions[num_decode_tokens:num_tokens].to(torch.int32)
_buf = topk_indices_buffer[num_decode_tokens:num_tokens]
_fill_causal_indices(_buf, _pos)
# Get the full shared workspace buffers once (will allocate on first use).
# Layout switches between FP8 (head_dim bytes + 4-byte fp32 scale) and
# MXFP4 (head_dim/2 bytes packed + head_dim/MXFP4_BLOCK_SIZE ue8m0
# scales) based on use_fp4_cache.
workspace_manager = current_workspace_manager()
values_spec, scales_spec = _gather_workspace_shapes(
total_seq_lens, head_dim, fp8_dtype, use_fp4_cache
)
k_quant_full, k_scale_full = workspace_manager.get_simultaneous(
values_spec,
scales_spec,
)
for chunk in prefill_metadata.chunks if not short_prefill else ():
k_quant = k_quant_full[: chunk.total_seq_lens]
k_scale = k_scale_full[: chunk.total_seq_lens]
if not chunk.skip_kv_gather:
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
cp_gather_indexer_k_quant_cache_triton,
)
cp_gather_indexer_k_quant_cache_triton(
kv_cache,
k_quant,
k_scale,
chunk.block_table,
chunk.cu_seq_lens,
token_to_seq=chunk.token_to_seq,
)
q_slice = q_quant[chunk.token_start : chunk.token_end]
q_scale_slice = (
q_scale[chunk.token_start : chunk.token_end]
if q_scale is not None
else None
)
# DeepGEMM scalar-type tags (zero-copy): MXFP4 values → int8
# (kPackedFP4), scales → int32 squeezed to 1-D kv_sf / 2-D q_sf.
if use_fp4_cache:
q_slice_cast = q_slice.view(torch.int8)
k_quant_cast = k_quant.view(torch.int8)
k_scale_cast = k_scale.view(torch.int32).squeeze(-1)
else:
q_slice_cast = q_slice
k_quant_cast = k_quant
k_scale_cast = k_scale.view(torch.float32).squeeze(-1)
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
rocm_fp8_mqa_logits,
)
assert q_scale_slice is None
logits = rocm_fp8_mqa_logits(
q_slice_cast,
(k_quant_cast, k_scale_cast),
weights[chunk.token_start : chunk.token_end],
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
)
num_rows = logits.shape[0]
# kpool: logits are pool-granular (compress_ratio == index_kpool),
# so topk selects pools. We pick topk_tokens // kpool pools then
# expand each pool back to its kpool constituent tokens.
select_k = topk_tokens // index_kpool if index_kpool > 1 else topk_tokens
if index_kpool > 1:
pool_topk = torch.empty(
(num_rows, select_k), dtype=torch.int32, device=logits.device
)
topk_dst = pool_topk
else:
topk_dst = topk_indices_buffer[
chunk.token_start : chunk.token_end, :topk_tokens
]
torch.ops._C.top_k_per_row_prefill(
logits,
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
topk_dst,
num_rows,
logits.stride(0),
logits.stride(1),
select_k,
)
if index_kpool > 1:
pool_ids = pool_topk.to(torch.int64)
if positions is not None:
# Fused expand-pools + append-tail into one Triton kernel
# (replaces ~25 elementwise ops). seq_len is token-granular
# (pos+1); the kernel derives pool_len internally.
q_seq = (
positions[chunk.token_start : chunk.token_end].to(torch.int32)
+ 1
)
expanded = kpool_ops.expand_pools_and_append_tail(
pool_ids, q_seq, index_kpool
)
else:
valid = pool_ids >= 0
expanded = kpool_ops.expand_pools_to_tokens(
pool_ids, valid, topk_tokens, index_kpool
)
topk_indices_buffer[
chunk.token_start : chunk.token_end, : expanded.shape[-1]
] = expanded
if has_decode:
decode_metadata = attn_metadata_narrowed.decode
assert decode_metadata is not None
kv_cache_raw = kv_cache # raw [num_blocks, block_size, head_dim+4] for writes
kv_cache = kv_cache_as_quant_view(kv_cache, head_dim, use_fp4_cache)
# Update the tail before reading logits; completed pools are compressed
# into the slot supplied by slot_mapping.
# Spec verification groups tokens by request and preserves position
# order so each token is stashed before the next completes its pool.
# Positions must remain token-granular because the kernel derives the
# pool phase and tail index from ``pos % kpool``.
if (
index_kpool > 1
and gate_score is not None
and compress_ape is not None
and positions is not None
and not skip_k_cache_insert
):
num_requests = attn_metadata_narrowed.num_decodes
# Kpool writes must recover the original request grouping after the
# indexer's flattened decode path. Host metadata avoids a CUDA graph
# sync when choosing the uniform or padded layout.
per_req_lens = decode_metadata.per_req_decode_lens
if per_req_lens is not None:
use_uniform = (
decode_metadata.decode_is_uniform
and num_decode_tokens
== num_requests * decode_metadata.write_max_decode_len
)
group_lens = per_req_lens
lmax = decode_metadata.write_max_decode_len
else:
# Legacy metadata without per-request lens: fall back to the
# host-side requires_padding flag. Unreached now (per-request
# lens is always populated for decode), kept defensive.
use_uniform = not decode_metadata.requires_padding
group_lens = decode_metadata.decode_lens
lmax = int(decode_metadata.decode_lens.max().item())
if not use_uniform:
# Non-uniform decode_lens (mixed plain-decode + spec-verify, or
# a variable MTP-verify batch): scatter actual tokens into a
# padded [B, lmax] layout. int32 tensors can't go through
# pack_seq_triton (float/uint8 only). The scatter indices are
# shared by all five scatters below (and the tail slot one).
scatter_idx = _build_decode_scatter_indices(
group_lens, num_requests, num_decode_tokens
)
dec_k = _scatter_decode_tokens_by_request(
k[:num_decode_tokens], 0, num_requests, lmax, scatter_idx
)
dec_gate = _scatter_decode_tokens_by_request(
gate_score[:num_decode_tokens],
0,
num_requests,
lmax,
scatter_idx,
)
dec_slot = _scatter_decode_tokens_by_request(
slot_mapping[:num_decode_tokens],
-1,
num_requests,
lmax,
scatter_idx,
)
dec_pos = _scatter_decode_tokens_by_request(
positions[:num_decode_tokens].to(torch.int32),
-1,
num_requests,
lmax,
scatter_idx,
)
else:
next_n = num_decode_tokens // num_requests
shape2 = (num_requests, next_n)
dec_k = k[:num_decode_tokens].view(*shape2, head_dim)
dec_gate = gate_score[:num_decode_tokens].view(*shape2, head_dim)
dec_slot = slot_mapping[:num_decode_tokens].view(shape2)
dec_pos = positions[:num_decode_tokens].to(torch.int32).view(shape2)
tail_meta = (
attn_metadata.get(_resolve_layer_name(tail_prefix))
if tail_prefix is not None
else None
)
# Paged tail cache replaces the transient _DECODE_TAIL ring. Group
# the tail group's token-granular slot_mapping per-request, mirroring
# dec_slot / dec_pos, so the kernel gets each request's current-token
# tail slot (block * kpool + pos % kpool).
if tail_meta is not None:
assert isinstance(tail_meta, DeepseekV32IndexerMetadata)
if tail_meta is None or tail_kv_cache is None:
dec_tail_slot = None
elif not use_uniform:
dec_tail_slot = _scatter_decode_tokens_by_request(
tail_meta.slot_mapping[:num_decode_tokens],
-1,
num_requests,
lmax,
scatter_idx,
)
else:
dec_tail_slot = tail_meta.slot_mapping[:num_decode_tokens].view(shape2)
# The compress kernel writes the raw fp8 cache (not the quant view);
# pass the underlying kv_cache, not kv_cache_quant_view.
if dec_tail_slot is not None:
# Single batched launch over [num_requests, next_n] replaces the
# per-token sequential loop. The kernel iterates each request's
# tokens in position order internally, preserving the
# pool-completion read-after-stash dependency that the loop
# provided. Inputs are already grouped per request (uniform:
# view; non-uniform: _scatter_decode_tokens_by_request padded to
# [B, lmax]) — no per-token .contiguous() copies needed.
kpool_ops.kpool_decode_update_and_maybe_write_cache_batched(
kv_cache_raw,
tail_kv_cache,
dec_tail_slot,
dec_k,
dec_gate,
compress_ape,
dec_slot,
dec_pos,
index_kpool,
head_dim,
round_scale=(scale_fmt is not None),
)
if current_platform.is_cuda_alike() and _fill_short_decode_causal_indices(
topk_indices_buffer,
positions,
num_decode_tokens,
attn_metadata_narrowed.max_seq_len,
topk_tokens,
):
return topk_indices_buffer
decode_lens = decode_metadata.decode_lens
if decode_metadata.requires_padding:
# Padding also covers short chunked prefills classified as decode.
# MXFP4 uses zero-byte padding so padded slots dequantize to zero.
if q_scale is not None:
padded_q_quant_decode_tokens = pack_seq_triton(
q_quant[:num_decode_tokens], decode_lens, pad_value=0
)
padded_q_scale = pack_seq_triton(
q_scale[:num_decode_tokens], decode_lens, pad_value=0
)
else:
padded_q_quant_decode_tokens = pack_seq_triton(
q_quant[:num_decode_tokens], decode_lens
)
padded_q_scale = None
padded_weights = pack_seq_triton(
weights[:num_decode_tokens], decode_lens, pad_value=0
).reshape(-1, *weights.shape[1:])
else:
padded_q_quant_decode_tokens = q_quant[:num_decode_tokens].reshape(
decode_lens.shape[0], -1, *q_quant.shape[1:]
)
if q_scale is not None:
padded_q_scale = q_scale[:num_decode_tokens].reshape(
decode_lens.shape[0], -1, *q_scale.shape[1:]
)
else:
padded_q_scale = None
padded_weights = weights[:num_decode_tokens]
# TODO: move and optimize below logic with triton kernels
batch_size = padded_q_quant_decode_tokens.shape[0]
next_n = padded_q_quant_decode_tokens.shape[1]
num_padded_tokens = batch_size * next_n
seq_lens = decode_metadata.seq_lens[:batch_size]
# seq_lens is always 2D: (B, next_n) for native spec decode, (B, 1)
# otherwise. deep_gemm fp8_fp4_paged_mqa_logits requires 2D context_lens;
# the downstream topk kernels accept both 1D and 2D.
padded_q_quant_cast = (
padded_q_quant_decode_tokens.view(torch.int8)
if use_fp4_cache
else padded_q_quant_decode_tokens
)
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
rocm_fp8_paged_mqa_logits,
)
assert padded_q_scale is None
logits = rocm_fp8_paged_mqa_logits(
padded_q_quant_cast,
kv_cache,
padded_weights[:num_padded_tokens],
seq_lens,
decode_metadata.block_table,
decode_metadata.schedule_metadata,
max_model_len=max_model_len,
)
num_rows = logits.shape[0]
# kpool: logits are pool-granular -> select topk_tokens//kpool pools,
# then expand each pool back to its kpool tokens.
select_k = topk_tokens // index_kpool if index_kpool > 1 else topk_tokens
if index_kpool > 1:
pool_topk = torch.empty(
(num_rows, select_k), dtype=torch.int32, device=logits.device
)
topk_dst = pool_topk
else:
topk_dst = topk_indices_buffer[:num_padded_tokens, :topk_tokens]
torch.ops._C.top_k_per_row_decode(
logits,
next_n,
seq_lens,
topk_dst,
num_rows,
logits.stride(0),
logits.stride(1),
select_k,
)
# Resolve to token-level indices in the output buffer.
if index_kpool > 1:
pool_ids = pool_topk.to(torch.int64)
n = pool_topk.shape[0]
# Decode seq_lens are pool-granular; recover token lengths from
# positions using the padded [B, next_n] row layout when needed.
if positions is not None:
dec_seq = _decode_topk_seq_lens(
positions,
decode_lens,
num_decode_tokens,
batch_size,
next_n,
decode_metadata.requires_padding,
)
else:
dec_seq = decode_metadata.seq_lens[:n]
if dec_seq.ndim == 2:
dec_seq = dec_seq[:, -1]
dec_seq = dec_seq.to(torch.int32)
out = kpool_ops.expand_pools_and_append_tail(pool_ids, dec_seq, index_kpool)
else:
out = topk_dst
if decode_metadata.requires_padding:
# Drop padded query rows introduced by the next_n padding above.
out = unpack_seq_triton(
out.reshape(batch_size, -1, out.shape[-1]), decode_lens
)
topk_indices_buffer[: out.shape[0], : out.shape[-1]] = out
return topk_indices_buffer
@CustomOp.register("sparse_attn_indexer_kpool")
class SparseAttnIndexerKpool(CustomOp):
"""Sparse Attention Indexer Custom Op Layer. This layer is extracted as a
separate custom op since it involves heavy custom kernels like `mqa_logits`,
`paged_mqa_logits` and `top_k_per_row`, etc. Those kernels maybe requires
specific memory layout or implementation for different hardware backends to
achieve optimal performance.
For now, the default native path will use CUDA backend path. Other platform
may requires add the corresponding Custom Op name `sparse_attn_indexer` to
`custom_ops` in `CompilationConfig` to enable the platform specific path.
"""
def __init__(
self,
k_cache,
quant_block_size: int,
scale_fmt: str,
topk_tokens: int,
head_dim: int,
max_model_len: int,
max_total_seq_len: int,
topk_indices_buffer: torch.Tensor,
skip_k_cache_insert: bool = False,
use_fp4_cache: bool = False,
tail_cache=None,
):
super().__init__()
self.k_cache = k_cache
self.tail_cache = tail_cache
self.quant_block_size = quant_block_size
self.scale_fmt = scale_fmt
self.topk_tokens = topk_tokens
self.head_dim = head_dim
self.max_model_len = max_model_len
self.max_total_seq_len = max_total_seq_len
self.topk_indices_buffer = topk_indices_buffer
self.skip_k_cache_insert = skip_k_cache_insert
self.use_fp4_cache = use_fp4_cache
def forward_native(
self,
hidden_states: torch.Tensor,
q_quant: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
k: torch.Tensor,
weights: torch.Tensor,
*,
gate_score: torch.Tensor | None = None,
compress_ape: torch.Tensor | None = None,
index_kpool: int = 1,
positions: torch.Tensor | None = None,
):
assert not self.use_fp4_cache, "AMD platform doesn't support fp4 cache yet"
assert isinstance(q_quant, torch.Tensor), (
"AMD sparse_attn_indexer expects a single FP8 q_quant tensor"
)
if not rocm_aiter_ops.is_enabled():
raise RuntimeError(
"Sparse attention indexer ROCm path is only supported on AITER. "
"Please enable aiter with VLLM_ROCM_USE_AITER=1"
)
if index_kpool <= 1:
return torch.ops.vllm.rocm_aiter_sparse_attn_indexer(
hidden_states,
_encode_layer_name(self.k_cache.prefix),
self.k_cache.kv_cache,
q_quant,
k,
weights,
self.quant_block_size,
self.scale_fmt,
self.topk_tokens,
self.head_dim,
self.max_model_len,
self.max_total_seq_len,
self.topk_indices_buffer,
skip_k_cache_insert=self.skip_k_cache_insert,
)
return sparse_attn_indexer_kpool(
hidden_states,
self.k_cache.prefix,
self.k_cache.kv_cache,
q_quant,
None,
k,
weights,
self.quant_block_size,
self.scale_fmt,
self.topk_tokens,
self.head_dim,
self.max_model_len,
self.max_total_seq_len,
self.topk_indices_buffer,
self.skip_k_cache_insert,
self.use_fp4_cache,
gate_score,
compress_ape,
index_kpool,
positions,
self.tail_cache.kv_cache if self.tail_cache is not None else None,
self.tail_cache.prefix if self.tail_cache is not None else None,
)
+1 -1
View File
@@ -23,7 +23,6 @@ from vllm.model_executor.layers.linear import (
from vllm.model_executor.layers.mla import MLAModules, MultiHeadLatentAttentionWrapper
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding, get_rope
from vllm.model_executor.layers.sparse_attn_indexer_kpool import SparseAttnIndexerKpool
from vllm.model_executor.models.deepseek_v2 import (
DeepSeekV2FusedQkvAProjLinear,
DeepseekV32IndexerCache,
@@ -31,6 +30,7 @@ from vllm.model_executor.models.deepseek_v2 import (
)
from vllm.model_executor.utils import maybe_disable_graph_partition
from vllm.models.glm5next.nvidia.ops.kpool_compress import fwht128_quant_fp8
from vllm.models.glm5next.sparse_indexer import SparseAttnIndexerKpool
from vllm.platforms import current_platform
from vllm.transformers_utils.configs.glm5_next import Glm5NextConfig
from vllm.utils.deep_gemm import PAGED_MQA_PAGE_SIZES
@@ -0,0 +1,170 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared helpers for the glm5next sparse attention indexer (kpool) layers."""
import torch
RADIX_TOPK_WORKSPACE_SIZE = 1024 * 1024
# MXFP4 layout: 2 values packed per byte, ue8m0 (1-byte) scale per block of 32.
MXFP4_BLOCK_SIZE = 32
def _build_decode_scatter_indices(
decode_lens: torch.Tensor,
num_requests: int,
n: int,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Per-token (request id, intra-request index) for a non-uniform decode
batch, with ``n == decode_lens.sum()`` as a host int (avoids a
device sync and keeps both repeat_interleaves sync-free).
Shared by every ``_scatter_decode_tokens_by_request`` call in a step:
building it per call would repeat the same repeat_interleave/cumsum chain
up to 5x per layer on the eager decode break.
"""
device = decode_lens.device
dl = decode_lens.to(torch.int64)
req_id = torch.repeat_interleave(
torch.arange(num_requests, device=device, dtype=torch.int64),
dl,
output_size=n,
)
req_starts = torch.cumsum(
torch.cat([torch.zeros(1, device=device, dtype=torch.int64), dl[:-1]]),
dim=0,
)
# Broadcast the per-request start offsets to per-token (length n ==
# dl.sum()) so each token's intra-request index subtracts its own
# request's start.
starts = torch.repeat_interleave(req_starts, dl, output_size=n)
intra = torch.arange(n, device=device, dtype=torch.int64) - starts
return req_id, intra
def _scatter_decode_tokens_by_request(
tokens: torch.Tensor,
pad_value,
num_requests: int,
lmax: int,
scatter_indices: tuple[torch.Tensor, torch.Tensor],
) -> torch.Tensor:
"""Group ``[N, ...]`` decode tokens into a padded ``[num_requests, lmax, ...]``
layout: request ``r``'s tokens at row ``r`` in order; short requests padded.
Unlike ``pack_seq_triton`` this is dtype-agnostic (needed for the int32
slot/pos tensors) — it scatters with the shared per-step indices from
``_build_decode_scatter_indices``. Used only for the non-uniform
(``requires_padding``) decode batch; uniform batches use a zero-copy
reshape.
"""
req_id, intra = scatter_indices
out = torch.full(
(num_requests, lmax, *tokens.shape[1:]),
pad_value,
dtype=tokens.dtype,
device=tokens.device,
)
out[req_id, intra] = tokens
return out
def _decode_topk_seq_lens(
positions: torch.Tensor,
decode_lens: torch.Tensor,
num_decode_tokens: int,
batch_size: int,
next_n: int,
requires_padding: bool,
) -> torch.Tensor:
"""Token-granular seq_len (pos + 1) per pool-topk row, layout-aware.
``pool_topk`` (and the logits it comes from) follow the padded
``[batch_size, next_n]`` grid whenever ``requires_padding`` is set, so row
``(b, t)`` corresponds to flat decode token ``offset_b + t`` -- NOT
``b * next_n + t``. Slicing flat ``positions[: batch_size * next_n]``
(the uniform-layout shortcut) misaligns every row after the first
non-uniform request and, past the decode region, reads prefill tokens'
positions; ``expand_pools_and_append_tail`` then anchors the tail at
another request's length, dropping the row's real tail tokens or emitting
indices past its sequence (out-of-bounds block-table reads). Padded rows
get 0 (empty tail); they are dropped by ``unpack_seq_triton`` anyway.
"""
n = batch_size * next_n
if not requires_padding:
return positions[:n].to(torch.int32) + 1
scatter_idx = _build_decode_scatter_indices(
decode_lens, batch_size, num_decode_tokens
)
padded = _scatter_decode_tokens_by_request(
positions[:num_decode_tokens].to(torch.int32),
-1,
batch_size,
next_n,
scatter_idx,
)
return padded.reshape(n) + 1 # pad rows: -1 + 1 = 0 -> empty tail
def _fill_causal_indices(rows: torch.Tensor, positions: torch.Tensor) -> None:
causal_range = torch.arange(rows.shape[1], device=rows.device, dtype=torch.int32)
positions = positions.to(torch.int32)
rows[:] = causal_range[None, :]
rows[causal_range[None, :] > positions[:, None]] = -1
def _fill_short_decode_causal_indices(
topk_indices_buffer: torch.Tensor,
positions: torch.Tensor | None,
num_decode_tokens: int,
max_seq_len: int,
topk_tokens: int,
) -> bool:
"""Fill exact causal rows when sparse decode would select every token."""
if positions is None or positions.numel() == 0 or max_seq_len > topk_tokens:
return False
_fill_causal_indices(
topk_indices_buffer[:num_decode_tokens], positions[:num_decode_tokens]
)
return True
def _gather_workspace_shapes(
total_seq_lens: int,
head_dim: int,
fp8_dtype: torch.dtype,
use_fp4_cache: bool,
) -> tuple[tuple[tuple[int, int], torch.dtype], tuple[tuple[int, int], torch.dtype]]:
"""Return ((values_shape, values_dtype), (scales_shape, scales_dtype)) for
the K-gather workspace. FP8 path: (T, head_dim) fp8 + (T, 4) uint8 fp32
scales. MXFP4 path: (T, head_dim // 2) uint8 packed mxfp4 +
(T, head_dim // MXFP4_BLOCK_SIZE) uint8 ue8m0 scales."""
if use_fp4_cache:
return (
((total_seq_lens, head_dim // 2), torch.uint8),
((total_seq_lens, head_dim // MXFP4_BLOCK_SIZE), torch.uint8),
)
return (
((total_seq_lens, head_dim), fp8_dtype),
((total_seq_lens, 4), torch.uint8),
)
def kv_cache_as_quant_view(
kv_cache: torch.Tensor,
head_dim: int,
use_fp4_cache: bool,
) -> torch.Tensor:
"""4D ``[num_blocks, block_size, 1, head_width]`` view expected by
DeepGEMM, from the 3D indexer kv-cache allocation."""
if use_fp4_cache:
assert kv_cache.ndim == 3 and kv_cache.dtype == torch.uint8
num_blocks, block_size, _ = kv_cache.shape
page_bytes = int(kv_cache.stride(0))
fp4_bytes = head_dim // 2 + head_dim // MXFP4_BLOCK_SIZE
return torch.as_strided(
kv_cache,
size=(num_blocks, block_size, 1, fp4_bytes),
stride=(page_bytes, fp4_bytes, fp4_bytes, 1),
)
return kv_cache.unsqueeze(-2)
@@ -2,30 +2,29 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Custom Sparse Attention Indexer layers."""
from typing import TYPE_CHECKING
import torch
import vllm.envs as envs
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import get_current_vllm_config_or_none
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.custom_op import CustomOp
from vllm.models.glm5next.common.sparse_indexer import (
RADIX_TOPK_WORKSPACE_SIZE,
_build_decode_scatter_indices,
_decode_topk_seq_lens,
_fill_causal_indices,
_fill_short_decode_causal_indices,
_gather_workspace_shapes,
_scatter_decode_tokens_by_request,
kv_cache_as_quant_view,
)
from vllm.models.glm5next.nvidia.ops import kpool_compress as kpool_ops
from vllm.platforms import current_platform
if TYPE_CHECKING:
from vllm.models.glm5next.nvidia.ops import kpool_compress as kpool_ops
elif current_platform.is_rocm():
from vllm.models.glm5next.amd.ops import kpool_compress as kpool_ops
else:
from vllm.models.glm5next.nvidia.ops import kpool_compress as kpool_ops
from vllm.utils.deep_gemm import has_deep_gemm
from vllm.utils.torch_utils import (
LayerNameType,
_encode_layer_name,
_resolve_layer_name,
)
from vllm.v1.attention.backends.mla.indexer import (
@@ -36,16 +35,9 @@ from vllm.v1.worker.workspace import current_workspace_manager
if current_platform.is_cuda_alike():
from vllm import _custom_ops as ops
elif current_platform.is_xpu():
from vllm._xpu_ops import xpu_ops
logger = init_logger(__name__)
RADIX_TOPK_WORKSPACE_SIZE = 1024 * 1024
# MXFP4 layout: 2 values packed per byte, ue8m0 (1-byte) scale per block of 32.
MXFP4_BLOCK_SIZE = 32
# kpool write helper: form pools from the current token batch and compress them
# into the index K cache via the fused Triton kernel.
@@ -96,166 +88,6 @@ def _kpool_compress_insert(
)
def _build_decode_scatter_indices(
decode_lens: torch.Tensor,
num_requests: int,
n: int,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Per-token (request id, intra-request index) for a non-uniform decode
batch, with ``n == decode_lens.sum()`` as a host int (avoids a
device sync and keeps both repeat_interleaves sync-free).
Shared by every ``_scatter_decode_tokens_by_request`` call in a step:
building it per call would repeat the same repeat_interleave/cumsum chain
up to 5x per layer on the eager decode break.
"""
device = decode_lens.device
dl = decode_lens.to(torch.int64)
req_id = torch.repeat_interleave(
torch.arange(num_requests, device=device, dtype=torch.int64),
dl,
output_size=n,
)
req_starts = torch.cumsum(
torch.cat([torch.zeros(1, device=device, dtype=torch.int64), dl[:-1]]),
dim=0,
)
# Broadcast the per-request start offsets to per-token (length n ==
# dl.sum()) so each token's intra-request index subtracts its own
# request's start.
starts = torch.repeat_interleave(req_starts, dl, output_size=n)
intra = torch.arange(n, device=device, dtype=torch.int64) - starts
return req_id, intra
def _scatter_decode_tokens_by_request(
tokens: torch.Tensor,
pad_value,
num_requests: int,
lmax: int,
scatter_indices: tuple[torch.Tensor, torch.Tensor],
) -> torch.Tensor:
"""Group ``[N, ...]`` decode tokens into a padded ``[num_requests, lmax, ...]``
layout: request ``r``'s tokens at row ``r`` in order; short requests padded.
Unlike ``pack_seq_triton`` this is dtype-agnostic (needed for the int32
slot/pos tensors) — it scatters with the shared per-step indices from
``_build_decode_scatter_indices``. Used only for the non-uniform
(``requires_padding``) decode batch; uniform batches use a zero-copy
reshape.
"""
req_id, intra = scatter_indices
out = torch.full(
(num_requests, lmax, *tokens.shape[1:]),
pad_value,
dtype=tokens.dtype,
device=tokens.device,
)
out[req_id, intra] = tokens
return out
def _decode_topk_seq_lens(
positions: torch.Tensor,
decode_lens: torch.Tensor,
num_decode_tokens: int,
batch_size: int,
next_n: int,
requires_padding: bool,
) -> torch.Tensor:
"""Token-granular seq_len (pos + 1) per pool-topk row, layout-aware.
``pool_topk`` (and the logits it comes from) follow the padded
``[batch_size, next_n]`` grid whenever ``requires_padding`` is set, so row
``(b, t)`` corresponds to flat decode token ``offset_b + t`` -- NOT
``b * next_n + t``. Slicing flat ``positions[: batch_size * next_n]``
(the uniform-layout shortcut) misaligns every row after the first
non-uniform request and, past the decode region, reads prefill tokens'
positions; ``expand_pools_and_append_tail`` then anchors the tail at
another request's length, dropping the row's real tail tokens or emitting
indices past its sequence (out-of-bounds block-table reads). Padded rows
get 0 (empty tail); they are dropped by ``unpack_seq_triton`` anyway.
"""
n = batch_size * next_n
if not requires_padding:
return positions[:n].to(torch.int32) + 1
scatter_idx = _build_decode_scatter_indices(
decode_lens, batch_size, num_decode_tokens
)
padded = _scatter_decode_tokens_by_request(
positions[:num_decode_tokens].to(torch.int32),
-1,
batch_size,
next_n,
scatter_idx,
)
return padded.reshape(n) + 1 # pad rows: -1 + 1 = 0 -> empty tail
def _fill_causal_indices(rows: torch.Tensor, positions: torch.Tensor) -> None:
causal_range = torch.arange(rows.shape[1], device=rows.device, dtype=torch.int32)
positions = positions.to(torch.int32)
rows[:] = causal_range[None, :]
rows[causal_range[None, :] > positions[:, None]] = -1
def _fill_short_decode_causal_indices(
topk_indices_buffer: torch.Tensor,
positions: torch.Tensor | None,
num_decode_tokens: int,
max_seq_len: int,
topk_tokens: int,
) -> bool:
"""Fill exact causal rows when sparse decode would select every token."""
if positions is None or positions.numel() == 0 or max_seq_len > topk_tokens:
return False
_fill_causal_indices(
topk_indices_buffer[:num_decode_tokens], positions[:num_decode_tokens]
)
return True
def _gather_workspace_shapes(
total_seq_lens: int,
head_dim: int,
fp8_dtype: torch.dtype,
use_fp4_cache: bool,
) -> tuple[tuple[tuple[int, int], torch.dtype], tuple[tuple[int, int], torch.dtype]]:
"""Return ((values_shape, values_dtype), (scales_shape, scales_dtype)) for
the K-gather workspace. FP8 path: (T, head_dim) fp8 + (T, 4) uint8 fp32
scales. MXFP4 path: (T, head_dim // 2) uint8 packed mxfp4 +
(T, head_dim // MXFP4_BLOCK_SIZE) uint8 ue8m0 scales."""
if use_fp4_cache:
return (
((total_seq_lens, head_dim // 2), torch.uint8),
((total_seq_lens, head_dim // MXFP4_BLOCK_SIZE), torch.uint8),
)
return (
((total_seq_lens, head_dim), fp8_dtype),
((total_seq_lens, 4), torch.uint8),
)
def kv_cache_as_quant_view(
kv_cache: torch.Tensor,
head_dim: int,
use_fp4_cache: bool,
) -> torch.Tensor:
"""4D ``[num_blocks, block_size, 1, head_width]`` view expected by
DeepGEMM, from the 3D indexer kv-cache allocation."""
if use_fp4_cache:
assert kv_cache.ndim == 3 and kv_cache.dtype == torch.uint8
num_blocks, block_size, _ = kv_cache.shape
page_bytes = int(kv_cache.stride(0))
fp4_bytes = head_dim // 2 + head_dim // MXFP4_BLOCK_SIZE
return torch.as_strided(
kv_cache,
size=(num_blocks, block_size, 1, fp4_bytes),
stride=(page_bytes, fp4_bytes, fp4_bytes, 1),
)
return kv_cache.unsqueeze(-2)
@eager_break_during_capture
def sparse_attn_indexer_kpool(
hidden_states: torch.Tensor,
@@ -393,26 +225,13 @@ def sparse_attn_indexer_kpool(
else:
# standard: per-token fp8 quant + scatter (all tokens).
assert scale_fmt is not None
if current_platform.is_rocm():
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
indexer_k_quant_and_cache_triton,
)
indexer_k_quant_and_cache_triton(
k,
kv_cache,
slot_mapping,
quant_block_size,
scale_fmt,
)
else:
ops.indexer_k_quant_and_cache(
k,
kv_cache,
slot_mapping,
quant_block_size,
scale_fmt,
)
ops.indexer_k_quant_and_cache(
k,
kv_cache,
slot_mapping,
quant_block_size,
scale_fmt,
)
topk_indices_buffer[: hidden_states.shape[0]] = -1
if has_prefill:
@@ -466,27 +285,13 @@ def sparse_attn_indexer_kpool(
k_scale = k_scale_full[: chunk.total_seq_lens]
if not chunk.skip_kv_gather:
if current_platform.is_rocm():
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
cp_gather_indexer_k_quant_cache_triton,
)
cp_gather_indexer_k_quant_cache_triton(
kv_cache,
k_quant,
k_scale,
chunk.block_table,
chunk.cu_seq_lens,
token_to_seq=chunk.token_to_seq,
)
else:
ops.cp_gather_indexer_k_quant_cache(
kv_cache,
k_quant,
k_scale,
chunk.block_table,
chunk.cu_seq_lens,
)
ops.cp_gather_indexer_k_quant_cache(
kv_cache,
k_quant,
k_scale,
chunk.block_table,
chunk.cu_seq_lens,
)
q_slice = q_quant[chunk.token_start : chunk.token_end]
q_scale_slice = (
@@ -504,30 +309,16 @@ def sparse_attn_indexer_kpool(
q_slice_cast = q_slice
k_quant_cast = k_quant
k_scale_cast = k_scale.view(torch.float32).squeeze(-1)
if current_platform.is_rocm():
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
rocm_fp8_mqa_logits,
)
from vllm.utils.deep_gemm import fp8_fp4_mqa_logits
assert q_scale_slice is None
logits = rocm_fp8_mqa_logits(
q_slice_cast,
(k_quant_cast, k_scale_cast),
weights[chunk.token_start : chunk.token_end],
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
)
else:
from vllm.utils.deep_gemm import fp8_fp4_mqa_logits
logits = fp8_fp4_mqa_logits(
(q_slice_cast, q_scale_slice),
(k_quant_cast, k_scale_cast),
weights[chunk.token_start : chunk.token_end],
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
clean_logits=False,
)
logits = fp8_fp4_mqa_logits(
(q_slice_cast, q_scale_slice),
(k_quant_cast, k_scale_cast),
weights[chunk.token_start : chunk.token_end],
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
clean_logits=False,
)
num_rows = logits.shape[0]
# kpool: logits are pool-granular (compress_ratio == index_kpool),
@@ -544,28 +335,16 @@ def sparse_attn_indexer_kpool(
chunk.token_start : chunk.token_end, :topk_tokens
]
if current_platform.is_xpu():
xpu_ops.top_k_per_row_prefill( # type: ignore[attr-defined]
logits,
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
topk_dst,
num_rows,
logits.stride(0),
logits.stride(1),
select_k,
)
else:
torch.ops._C.top_k_per_row_prefill(
logits,
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
topk_dst,
num_rows,
logits.stride(0),
logits.stride(1),
select_k,
)
torch.ops._C.top_k_per_row_prefill(
logits,
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
topk_dst,
num_rows,
logits.stride(0),
logits.stride(1),
select_k,
)
if index_kpool > 1:
pool_ids = pool_topk.to(torch.int64)
@@ -765,34 +544,18 @@ def sparse_attn_indexer_kpool(
if use_fp4_cache
else padded_q_quant_decode_tokens
)
if current_platform.is_rocm():
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
rocm_fp8_paged_mqa_logits,
)
from vllm.utils.deep_gemm import fp8_fp4_paged_mqa_logits
assert padded_q_scale is None
logits = rocm_fp8_paged_mqa_logits(
padded_q_quant_cast,
kv_cache,
padded_weights[:num_padded_tokens],
seq_lens,
decode_metadata.block_table,
decode_metadata.schedule_metadata,
max_model_len=max_model_len,
)
else:
from vllm.utils.deep_gemm import fp8_fp4_paged_mqa_logits
logits = fp8_fp4_paged_mqa_logits(
(padded_q_quant_cast, padded_q_scale),
kv_cache,
padded_weights[:num_padded_tokens],
seq_lens,
decode_metadata.block_table,
decode_metadata.schedule_metadata,
max_model_len=max_model_len,
clean_logits=False,
)
logits = fp8_fp4_paged_mqa_logits(
(padded_q_quant_cast, padded_q_scale),
kv_cache,
padded_weights[:num_padded_tokens],
seq_lens,
decode_metadata.block_table,
decode_metadata.schedule_metadata,
max_model_len=max_model_len,
clean_logits=False,
)
num_rows = logits.shape[0]
# kpool: logits are pool-granular -> select topk_tokens//kpool pools,
# then expand each pool back to its kpool tokens.
@@ -819,28 +582,16 @@ def sparse_attn_indexer_kpool(
attn_metadata_narrowed.max_seq_len,
)
else:
if current_platform.is_xpu():
xpu_ops.top_k_per_row_decode( # type: ignore[attr-defined]
logits,
next_n,
seq_lens,
topk_dst,
num_rows,
logits.stride(0),
logits.stride(1),
select_k,
)
else:
torch.ops._C.top_k_per_row_decode(
logits,
next_n,
seq_lens,
topk_dst,
num_rows,
logits.stride(0),
logits.stride(1),
select_k,
)
torch.ops._C.top_k_per_row_decode(
logits,
next_n,
seq_lens,
topk_dst,
num_rows,
logits.stride(0),
logits.stride(1),
select_k,
)
# Resolve to token-level indices in the output buffer.
if index_kpool > 1:
@@ -942,33 +693,16 @@ class SparseAttnIndexerKpool(CustomOp):
index_kpool: int = 1,
positions: torch.Tensor | None = None,
):
if current_platform.is_cuda() or current_platform.is_xpu():
return self.forward_cuda(
hidden_states,
q_quant,
k,
weights,
gate_score=gate_score,
compress_ape=compress_ape,
index_kpool=index_kpool,
positions=positions,
)
elif current_platform.is_rocm():
return self.forward_hip(
hidden_states,
q_quant,
k,
weights,
gate_score=gate_score,
compress_ape=compress_ape,
index_kpool=index_kpool,
positions=positions,
)
else:
raise NotImplementedError(
"SparseAttnIndexer native forward is only implemented for "
"CUDA, ROCm and XPU platforms."
)
return self.forward_cuda(
hidden_states,
q_quant,
k,
weights,
gate_score=gate_score,
compress_ape=compress_ape,
index_kpool=index_kpool,
positions=positions,
)
def forward_cuda(
self,
@@ -1012,52 +746,3 @@ class SparseAttnIndexerKpool(CustomOp):
self.tail_cache.kv_cache if self.tail_cache is not None else None,
self.tail_cache.prefix if self.tail_cache is not None else None,
)
def forward_hip(
self,
hidden_states: torch.Tensor,
q_quant: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
k: torch.Tensor,
weights: torch.Tensor,
*,
gate_score: torch.Tensor | None = None,
compress_ape: torch.Tensor | None = None,
index_kpool: int = 1,
positions: torch.Tensor | None = None,
):
assert not self.use_fp4_cache, "AMD platform doesn't support fp4 cache yet"
assert isinstance(q_quant, torch.Tensor), (
"AMD sparse_attn_indexer expects a single FP8 q_quant tensor"
)
if rocm_aiter_ops.is_enabled():
if index_kpool <= 1:
return torch.ops.vllm.rocm_aiter_sparse_attn_indexer(
hidden_states,
_encode_layer_name(self.k_cache.prefix),
self.k_cache.kv_cache,
q_quant,
k,
weights,
self.quant_block_size,
self.scale_fmt,
self.topk_tokens,
self.head_dim,
self.max_model_len,
self.max_total_seq_len,
self.topk_indices_buffer,
skip_k_cache_insert=self.skip_k_cache_insert,
)
return self.forward_cuda(
hidden_states,
q_quant,
k,
weights,
gate_score=gate_score,
compress_ape=compress_ape,
index_kpool=index_kpool,
positions=positions,
)
raise RuntimeError(
"Sparse attention indexer ROCm path is only supported on AITER. "
"Please enable aiter with VLLM_ROCM_USE_AITER=1"
)
+22
View File
@@ -0,0 +1,22 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Sparse attention indexer (kpool) — hardware-isolated entry point.
The implementation lives under ``nvidia/`` and ``amd/`` (with shared helpers
in ``common/``); this module picks the right one for the current platform.
"""
from vllm.platforms import current_platform
if current_platform.is_rocm():
from .amd.sparse_indexer import (
SparseAttnIndexerKpool,
)
else:
from .nvidia.sparse_indexer import (
SparseAttnIndexerKpool,
)
__all__ = [
"SparseAttnIndexerKpool",
]
+19 -10
View File
@@ -48,6 +48,19 @@ def _get_hasher_factory(
raise ValueError(f"Unsupported hash algorithm: {algorithm}")
def _get_image_id_bytes(image: Image.Image) -> bytes | None:
try:
exif = image.getexif()
image_id = exif.get(Image.ExifTags.Base.ImageID)
if isinstance(image_id, uuid.UUID):
return image_id.bytes
except Exception:
# Tolerate malformed EXIF metadata (e.g. invalid TIFF header)
# and fall back to serializing raw image data or bytes.
pass
return None
class MultiModalHasher:
@classmethod
def serialize_item(cls, obj: object) -> Iterable[bytes | memoryview]:
@@ -60,11 +73,9 @@ class MultiModalHasher:
return (np.array(obj).tobytes(),)
if isinstance(obj, Image.Image):
exif = obj.getexif()
if Image.ExifTags.Base.ImageID in exif and isinstance(
exif[Image.ExifTags.Base.ImageID], uuid.UUID
):
return (exif[Image.ExifTags.Base.ImageID].bytes,)
image_id = _get_image_id_bytes(obj)
if image_id is not None:
return (image_id,)
data = {"mode": obj.mode, "data": np.asarray(obj)}
palette = obj.palette
@@ -76,11 +87,9 @@ class MultiModalHasher:
return cls.iter_item_to_bytes("image", data)
if isinstance(obj, MediaWithBytes) and isinstance(obj.media, Image.Image):
exif = obj.media.getexif()
if Image.ExifTags.Base.ImageID in exif and isinstance(
exif[Image.ExifTags.Base.ImageID], uuid.UUID
):
return (exif[Image.ExifTags.Base.ImageID].bytes,)
image_id = _get_image_id_bytes(obj.media)
if image_id is not None:
return (image_id,)
if obj.io_config:
return cls.iter_item_to_bytes(
+41
View File
@@ -0,0 +1,41 @@
# vllm/parser
Model-output parsing for the serving layer. `Parser` (`abstract_parser.py`)
unifies reasoning extraction and tool-call extraction behind one object; the
serving layer calls `parse` / `parse_delta` and `is_reasoning_end` on it.
## Layout
- `abstract_parser.py` — `Parser`, and `DelegatingParser` for wrapping a legacy
`ReasoningParser` + `ToolParser` pair.
- `parser_manager.py` — resolves parser names to classes.
- `engine/` — the declarative parser engine. A `ParserEngineConfig`
(`parser_engine_config.py`) is a set of terminals, transitions and content
events; `parser_engine.py` and `streaming_parser_engine.py` drive it for the
non-streaming and streaming paths; `token_id_scanner.py` pre-lexes
special-token ids and `incremental_lexer.py` lexes text; `adapters.py`
exposes an engine through the legacy reasoning/tool parser interfaces.
- `<model>.py` — one file per model family. Most subclass `ParserEngine`
(`deepseek_v4.py`, `qwen3.py`, ...); a few are `DelegatingParser` wrappers
over legacy parsers (`harmony.py`, `kimi_k3.py`).
## Guidelines
- A new parser either subclasses `ParserEngine` with a declarative config, or
subclasses `Parser` and implements `parse` / `parse_delta` directly. Use the
engine when the format is expressible as terminals and transitions, and
subclass an existing config when it is a variant of one (`seed_oss.py`,
`ling3.py`). `DelegatingParser` exists for backward compatibility with the
separate `ReasoningParser` / `ToolParser` classes; do not use it for new
formats.
- Keep a model-specific fix inside that model's `<model>.py` where possible.
Change `engine/` or `abstract_parser.py` only when the behavior is wrong for
every parser, and say so in the PR.
- Text emitted in a state with no `content_events` entry is discarded. Make
that deliberate when adding a state.
- Special tokens reach the engine as token ids, not text. A text terminal that
spans a special token never matches in streaming; list it in
`token_id_terminals`.
- Comment only where the code alone would leave a reader with a question —
a non-obvious transition, a deliberate swallow, a model quirk being worked
around. Keep it minimal and concise.
+1
View File
@@ -0,0 +1 @@
@AGENTS.md
+19
View File
@@ -672,11 +672,30 @@ class ParserEngine(Parser):
return not wait_for_reasoning
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
config = self.parser_engine_config
wait_for_reasoning = config.wait_for_reasoning
if wait_for_reasoning is None:
wait_for_reasoning = config.initial_state is ParserState.REASONING
if not wait_for_reasoning:
return input_ids
end_id = self._reasoning_end_token_id
if end_id is not None:
for i in range(len(input_ids) - 1, -1, -1):
if input_ids[i] == end_id:
return input_ids[i + 1 :]
end_ids = self._reasoning_end_token_ids
if end_ids:
turn_start = 0
boundary_ids = self._turn_boundary_token_ids
for i in range(len(input_ids) - 1, -1, -1):
if input_ids[i] in boundary_ids:
turn_start = i + 1
break
for i in range(turn_start, len(input_ids)):
if input_ids[i] in end_ids:
return input_ids[i:]
return input_ids
def get_streaming_fallback_content(
+8 -6
View File
@@ -41,6 +41,13 @@ ARG_KEY_END = "</arg_key>"
ARG_VALUE_START = "<arg_value>"
ARG_VALUE_END = "</arg_value>"
# Special tokens that delimit conversation turns in a rendered GLM prompt.
# Reasoning markers belonging to earlier turns must not be mistaken for the
# state of the turn currently being generated.
GLM_TURN_BOUNDARIES = frozenset(
("<|system|>", "<|user|>", "<|assistant|>", "<|observation|>")
)
_ARG_RE = re.compile(
r"<arg_key>(?P<key>.*?)</arg_key>\s*"
r"<arg_value>(?P<value>.*?)</arg_value>",
@@ -123,7 +130,6 @@ def glm47_moe_config(thinking: bool = True) -> ParserEngineConfig:
return ParserEngineConfig(
name="glm47_moe",
initial_state=ParserState.REASONING if thinking else ParserState.CONTENT,
wait_for_reasoning=thinking,
terminals={
**reasoning_terminals,
"TOOL_START": TOOL_CALL_START,
@@ -166,6 +172,7 @@ def glm47_moe_config(thinking: bool = True) -> ParserEngineConfig:
),
**arg_tag_transitions,
},
turn_boundary_tokens=GLM_TURN_BOUNDARIES,
arg_converter=_glm47_arg_converter,
stream_arg_deltas=True,
tool_args_json=False,
@@ -207,11 +214,6 @@ class Glm47MoeParser(ParserEngine):
self._tool_slots[idx].name = self._tool_slots[idx].name.strip()
super()._handle_tool_end(event, deltas)
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
if not self.thinking_enabled:
return input_ids
return super().extract_content_ids(input_ids)
def extract_reasoning(
self,
model_output: str,
+9
View File
@@ -217,6 +217,15 @@ def cpu_platform_plugin() -> str | None:
"AMD Zen CPU detected but zentorch not installed, "
"falling back to CpuPlatform."
)
except OSError:
# An ABI-mismatched build fails here with an undefined-symbol
# error; other failures are not known to be safe to recover from.
logger.warning(
"AMD Zen CPU detected but zentorch failed to import, falling "
"back to CpuPlatform. This usually means the zentorch build "
"does not match the installed torch version.",
exc_info=True,
)
return "vllm.platforms.cpu.CpuPlatform"
+18
View File
@@ -1334,6 +1334,24 @@ class Platform:
"""
return False
@classmethod
def validate_environ(cls, hard_fail: bool) -> None:
"""
Validate environment variables for the current platform.
"""
from vllm import envs
for env in os.environ:
if env.startswith("VLLM_") and env not in envs.environment_variables:
if hard_fail:
raise ValueError(
f"Unknown vLLM environment variable detected: {env}"
)
else:
logger.warning(
"Unknown vLLM environment variable detected: %s", env
)
class UnspecifiedPlatform(Platform):
_enum = PlatformEnum.UNSPECIFIED
+54
View File
@@ -0,0 +1,54 @@
# vllm/reasoning
Every `--reasoning-parser` name resolves here. The parser has two consumers:
`ParserManager.get_parser` composes it with the tool parser into the unified
`Parser` the serving layer uses, and the structured-output manager
(`vllm/v1/structured_output/__init__.py`) instantiates it directly and calls
`is_reasoning_end` / `is_reasoning_end_streaming` every decode step to decide
when the grammar bitmask starts applying. The `ReasoningParser` interface
itself is legacy: it is kept so existing in-tree parsers and out-of-tree
plugins keep working, not as the base for new ones.
## Layout
- `abs_reasoning_parsers.py` — `ReasoningParser` (`extract_reasoning`,
`extract_reasoning_streaming`, `is_reasoning_end`,
`is_reasoning_end_streaming`, `extract_content_ids`, `adjust_request`) and
`ReasoningParserManager`.
- `__init__.py` — `_REASONING_PARSERS_TO_REGISTER`, the name → (file, class)
table that lazily registers every built-in parser.
- `basic_parsers.py` — `BaseThinkingReasoningParser`, the base for parsers
that only need a start/end token pair.
- `<model>_reasoning_parser.py` — one file per family. Either a subclass of a
`ParserReasoningAdapter` generated from a `vllm/parser/<model>.py` engine
grammar (`mistral_reasoning_parser.py`, `kimi_k2_reasoning_parser.py`; the
`_engine_` suffix is not applied consistently), or a self-contained
`ReasoningParser` subclass.
## Guidelines
- Do not change the `ReasoningParser` interface in `abs_reasoning_parsers.py`.
Out-of-tree plugins loaded through
`ReasoningParserManager.import_reasoning_parser` subclass it. New
capabilities go on `Parser` in `vllm/parser/`; if a legacy parser needs the
same capability, `DelegatingParser` supplies the default.
- For a new model, implement a unified `Parser` in `vllm/parser/` first, then
expose it here as a thin adapter subclass plus an entry in
`_REASONING_PARSERS_TO_REGISTER`.
- The `_REASONING_PARSERS_TO_REGISTER` key is what users pass on the command
line; treat renames as breaking.
- Keep adapters thin: registration, `adjust_request` tweaks, and class
attributes. Parsing logic belongs in `vllm/parser/`.
- Keep a model-specific fix inside that model's `<model>_reasoning_parser.py`,
or its grammar in `vllm/parser/<model>.py`, where possible. Touch
`abs_reasoning_parsers.py` or `basic_parsers.py` only for behavior every
parser needs, and say so in the PR.
- `is_reasoning_end_streaming` runs on the engine core for every decode step
of every structured-output request. Work on `delta_ids`; re-scanning the
full sequence makes the step cost grow with sequence length.
- The structured-output gate and the frontend's extraction must agree on
where reasoning ends, whether the turn continues with content or a tool
call. Test the end-detection methods with the same inputs as extraction.
- Comment only where the code alone would leave a reader with a question —
a request tweak in `adjust_request`, a marker handled unusually, a model
quirk being worked around. Keep it minimal and concise.
+1
View File
@@ -0,0 +1 @@
@AGENTS.md
+53 -6
View File
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from io import BytesIO
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Final
import pybase64
import torch
@@ -17,6 +17,27 @@ if TYPE_CHECKING:
from vllm.config import ModelConfig
# How much of torch's own reason is repeated back to the caller. The reason is
# built from bytes the caller sent, so it is bounded rather than echoed whole.
_MAX_EMBED_ERROR_REASON_CHARS: Final = 200
def _truncated_reason(exc: Exception) -> str:
"""`exc`'s message, capped, saying how much was left out when it is capped.
Without the count a capped reason reads like the whole one, so a caller
debugging a large payload cannot tell that torch said more than this.
"""
reason: Final = str(exc).strip() or type(exc).__name__
omitted: Final = len(reason) - _MAX_EMBED_ERROR_REASON_CHARS
if omitted <= 0:
return reason
return (
f"{reason[:_MAX_EMBED_ERROR_REASON_CHARS]}"
f"... ({omitted} more characters truncated)"
)
def safe_load_prompt_embeds(
model_config: "ModelConfig",
embed: bytes,
@@ -27,12 +48,38 @@ def safe_load_prompt_embeds(
parameter="prompt_embeds",
)
# Decoded outside the try below so that a body which is not base64 keeps
# raising binascii.Error, which is a ValueError and already answered 400.
payload: Final = pybase64.b64decode(embed, validate=True)
with check_sparse_tensor_invariants_threadsafe():
tensor = torch.load(
BytesIO(pybase64.b64decode(embed, validate=True)),
weights_only=True,
map_location=torch.device("cpu"),
)
try:
tensor = torch.load(
BytesIO(payload),
weights_only=True,
map_location=torch.device("cpu"),
)
except (torch.OutOfMemoryError, MemoryError):
# Running out of memory is the server's condition, not the caller's.
raise
except Exception as exc:
# Everything else here is about the bytes the caller sent, and torch
# has no single error type for "this is not a tensor file": the zip
# reader raises `RuntimeError`, the legacy pickle path raises
# `UnpicklingError`, `KeyError` or `IndexError`, an empty payload
# raises `EOFError`. None of them is a `ValueError`, so the
# entrypoints' fallback mapped them all to 500. Catching by
# behaviour rather than by type also survives torch changing which
# error it raises for a given malformed payload.
#
# torch's reason is worth keeping -- for a malformed sparse tensor
# it names the offending index -- but it is built from the caller's
# own bytes, so it is truncated rather than echoed whole.
raise VLLMValidationError(
"`prompt_embeds` could not be deserialized as a torch tensor: "
f"{_truncated_reason(exc)}",
parameter="prompt_embeds",
) from exc
tensor = safe_to_dense(tensor, parameter="prompt_embeds")
if tensor.dim() > 2:
+45
View File
@@ -0,0 +1,45 @@
# vllm/tool_parsers
Every `--tool-call-parser` name resolves here. `ParserManager.get_parser`
composes the tool parser chosen by name with the reasoning parser into the
unified `Parser` the serving layer uses. The `ToolParser` interface itself is
legacy: it is kept so existing in-tree parsers and out-of-tree plugins keep
working, not as the base for new ones.
## Layout
- `abstract_tool_parser.py` — `ToolParser` (`extract_tool_calls`,
`extract_tool_calls_streaming`, `adjust_request`) and `ToolParserManager`.
- `__init__.py` — `_TOOL_PARSERS_TO_REGISTER`, the name → (file, class) table
that lazily registers every built-in parser.
- `structural_tag_registry.py` — structural-tag builders used to constrain
tool-call output for `tool_choice="required"` / named tools.
- `<model>_tool_parser.py` — one file per family. Either a subclass of a
`ParserToolAdapter` generated from a `vllm/parser/<model>.py` engine grammar
(`mistral_tool_parser.py`, `kimi_k2_tool_parser.py`; the `_engine_` suffix
is not applied consistently), or a self-contained `ToolParser` subclass.
## Guidelines
- Do not change the `ToolParser` interface in `abstract_tool_parser.py`.
Out-of-tree plugins loaded through `ToolParserManager.import_tool_parser`
subclass it. New capabilities go on `Parser` in `vllm/parser/`; if a legacy
parser needs the same capability, `DelegatingParser` supplies the default.
- For a new model, implement a unified `Parser` in `vllm/parser/` first, then
expose it here as a thin adapter subclass plus an entry in
`_TOOL_PARSERS_TO_REGISTER`.
- The `_TOOL_PARSERS_TO_REGISTER` key is what users pass on the command line;
treat renames as breaking.
- Keep adapters thin: registration, `adjust_request` tweaks, and class
attributes. Parsing logic belongs in `vllm/parser/`.
- Keep a model-specific fix inside that model's `<model>_tool_parser.py`, or
its grammar in `vllm/parser/<model>.py`, where possible. Touch
`abstract_tool_parser.py` or `structural_tag_registry.py` only for behavior
every parser needs, and say so in the PR.
- Set `structural_tag_model` only to a key in `SUPPORTED_STRUCTURAL_TAG_MODELS`
(an xgrammar builtin, or one registered with `register_vllm_structural_tag`).
It only takes effect under `VLLM_ENFORCE_STRICT_TOOL_CALLING`, where it also
disables the generic required/named handling for that parser.
- Comment only where the code alone would leave a reader with a question —
a request tweak in `adjust_request`, a marker handled unusually, a model
quirk being worked around. Keep it minimal and concise.

Some files were not shown because too many files have changed in this diff Show More