Merge remote-tracking branch 'origin/main' into sync/lesj/qwen4exp-bugfixes
Signed-off-by: lesj0610 <lesj0610@users.noreply.github.com>
This commit is contained in:
@@ -540,6 +540,26 @@ prepare_ci_build_context() {
|
||||
echo "Using canonical CI Docker context: ${ROCM_BUILD_CONTEXT_ROOT}"
|
||||
}
|
||||
|
||||
configure_custom_rocm_stages() {
|
||||
using_custom_rocm_dockerfiles || return 0
|
||||
local context_root="${ROCM_BUILD_CONTEXT_ROOT:-.}"
|
||||
local dockerfile="${context_root}/${CI_BASE_DOCKERFILE}"
|
||||
local stages=""
|
||||
|
||||
(cd "${context_root}" && \
|
||||
validate_rocm_dockerfile "${CI_BASE_DOCKERFILE}" \
|
||||
ci_base test export_vllm export_test_smoke csrc-build rust-build) \
|
||||
|| return $?
|
||||
# Hash all stages for custom layouts, including additional helper stages.
|
||||
stages=$(rocm_dockerfile_stages "${dockerfile}" | paste -sd ' ')
|
||||
CI_BASE_DOCKERFILE_STAGES="${CI_BASE_DOCKERFILE_STAGES:-${stages}}"
|
||||
ROCM_CSRC_DOCKERFILE_STAGES="${ROCM_CSRC_DOCKERFILE_STAGES:-${stages}}"
|
||||
ROCM_RUST_DOCKERFILE_STAGES="${ROCM_RUST_DOCKERFILE_STAGES:-${stages}}"
|
||||
if [[ " ${stages} " != *" build_rocshmem "* ]]; then
|
||||
unset ROCSHMEM_BRANCH ROCSHMEM_CACHE_KEY DEEPEP_CACHE_KEY
|
||||
fi
|
||||
}
|
||||
|
||||
compose_dependency_cache_key() {
|
||||
local prefix="$1"
|
||||
local material="$2"
|
||||
@@ -569,11 +589,11 @@ hash_dockerfile_stages() {
|
||||
}
|
||||
emit = 0
|
||||
}
|
||||
$1 == "FROM" {
|
||||
toupper($1) == "FROM" {
|
||||
stage = ""
|
||||
for (idx = 1; idx <= NF; idx++) {
|
||||
if (tolower($idx) == "as" && idx < NF) {
|
||||
stage = $(idx + 1)
|
||||
stage = tolower($(idx + 1))
|
||||
}
|
||||
}
|
||||
emit = (stage in wanted)
|
||||
@@ -613,11 +633,11 @@ discover_dockerfile_stage_args() {
|
||||
}
|
||||
{
|
||||
line = $0
|
||||
if ($1 == "FROM") {
|
||||
if (toupper($1) == "FROM") {
|
||||
stage = ""
|
||||
for (idx = 1; idx <= NF; idx++) {
|
||||
if (tolower($idx) == "as" && idx < NF) {
|
||||
stage = $(idx + 1)
|
||||
stage = tolower($(idx + 1))
|
||||
}
|
||||
}
|
||||
emit = (stage in wanted)
|
||||
@@ -630,7 +650,7 @@ discover_dockerfile_stage_args() {
|
||||
for (idx = 1; idx <= line_count; idx++) {
|
||||
line = lines[idx]
|
||||
arg_name = line
|
||||
sub(/^[[:space:]]*ARG[[:space:]]+/, "", arg_name)
|
||||
sub(/^[[:space:]]*[Aa][Rr][Gg][[:space:]]+/, "", arg_name)
|
||||
if (arg_name != line) {
|
||||
sub(/[=[:space:]].*/, "", arg_name)
|
||||
if (arg_name ~ /^[A-Za-z_][A-Za-z0-9_]*$/) {
|
||||
@@ -716,7 +736,7 @@ extract_dockerfile_arg_default() {
|
||||
if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" && "${dockerfile}" != /* ]]; then
|
||||
physical_dockerfile="${ROCM_BUILD_CONTEXT_ROOT}/${dockerfile}"
|
||||
fi
|
||||
sed -n -E "s/^[[:space:]]*ARG[[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
|
||||
sed -n -E "s/^[[:space:]]*[Aa][Rr][Gg][[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
|
||||
"${physical_dockerfile}" | head -1
|
||||
}
|
||||
|
||||
@@ -1000,7 +1020,21 @@ create_and_bootstrap_builder() {
|
||||
}
|
||||
|
||||
init_config() {
|
||||
# shellcheck source=.buildkite/scripts/rocm/build-config.sh
|
||||
source "$(dirname "${BASH_SOURCE[0]}")/rocm/build-config.sh"
|
||||
configure_rocm_build
|
||||
|
||||
TARGET="${1:-test-ci}"
|
||||
if using_custom_rocm_dockerfiles; then
|
||||
if [[ -z "${BASE_IMAGE:-}" ]]; then
|
||||
echo "Custom ROCm builds require BASE_IMAGE from the selected base handoff" >&2
|
||||
return 1
|
||||
fi
|
||||
if ! is_ci_base_target && [[ -z "${CI_BASE_IMAGE:-}" ]]; then
|
||||
echo "Custom ROCm builds require CI_BASE_IMAGE from the selected ci_base handoff" >&2
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
BAKE_TARGETS=("${TARGET}")
|
||||
DEPENDENCY_CACHE_TARGETS=()
|
||||
CI_HCL_SOURCE="${CI_HCL_SOURCE:-${CI_HCL_FILE:-${DEFAULT_CI_HCL_SOURCE}}}"
|
||||
@@ -1010,10 +1044,12 @@ init_config() {
|
||||
PYTORCH_ROCM_ARCH="${PYTORCH_ROCM_ARCH:-gfx90a;gfx942;gfx950}"
|
||||
CI_BASE_CONTENT_FILES="${CI_BASE_CONTENT_FILES:-${DEFAULT_CI_BASE_CONTENT_FILES}}"
|
||||
CI_BASE_DOCKERFILE="${CI_BASE_DOCKERFILE:-${DEFAULT_CI_BASE_DOCKERFILE}}"
|
||||
CI_BASE_DOCKERFILE_STAGES="${CI_BASE_DOCKERFILE_STAGES:-${DEFAULT_CI_BASE_DOCKERFILE_STAGES}}"
|
||||
if ! using_custom_rocm_dockerfiles; then
|
||||
CI_BASE_DOCKERFILE_STAGES="${CI_BASE_DOCKERFILE_STAGES:-${DEFAULT_CI_BASE_DOCKERFILE_STAGES}}"
|
||||
fi
|
||||
CI_BASE_METADATA_VERSION="${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}"
|
||||
CI_BASE_IMAGE_TAG="${CI_BASE_IMAGE_TAG:-rocm/vllm-dev:ci_base}"
|
||||
export PYTORCH_ROCM_ARCH
|
||||
export PYTORCH_ROCM_ARCH CI_BASE_DOCKERFILE
|
||||
|
||||
SCRIPT_TMP_DIR=$(mktemp -d -t ci-bake-rocm.XXXXXX)
|
||||
CI_HCL_PATH="${SCRIPT_TMP_DIR}/ci.hcl"
|
||||
@@ -1042,6 +1078,11 @@ print_header() {
|
||||
}
|
||||
|
||||
validate_inputs() {
|
||||
if using_custom_rocm_dockerfiles; then
|
||||
validate_rocm_dockerfile "${ROCM_BASE_DOCKERFILE}" || return $?
|
||||
validate_rocm_dockerfile "${CI_BASE_DOCKERFILE}" \
|
||||
ci_base test export_vllm export_test_smoke csrc-build rust-build || return $?
|
||||
fi
|
||||
if [[ ! -f "${VLLM_BAKE_FILE}" ]]; then
|
||||
echo "Error: vLLM bake file not found at ${VLLM_BAKE_FILE}"
|
||||
echo "Make sure you're running from the vLLM repository root"
|
||||
@@ -1069,6 +1110,11 @@ load_ci_hcl() {
|
||||
|
||||
init_bake_files() {
|
||||
BAKE_FILES=(-f "${VLLM_BAKE_FILE}" -f "${CI_HCL_PATH}")
|
||||
if using_custom_rocm_dockerfiles; then
|
||||
# Custom stacks reuse local cache without replacing the
|
||||
# standard ROCm registry caches for the same commit or branch.
|
||||
BAKE_FILES+=(--set '*.cache-to=')
|
||||
fi
|
||||
}
|
||||
|
||||
compute_ci_base_hash_if_needed() {
|
||||
@@ -1804,7 +1850,7 @@ compute_rocm_csrc_content_hash() {
|
||||
local -a content_args=()
|
||||
|
||||
bake_dir=$(dirname "${VLLM_BAKE_FILE}")
|
||||
dockerfile_rocm="${bake_dir}/Dockerfile.rocm"
|
||||
dockerfile_rocm="${CI_BASE_DOCKERFILE:-${bake_dir}/Dockerfile.rocm}"
|
||||
read -r -a content_paths <<< "${content_files}"
|
||||
mapfile -t content_args < <(
|
||||
get_content_arg_names "${dockerfile_rocm}" "${stages}" "${ROCM_CSRC_CONTENT_ARGS:-}"
|
||||
@@ -1853,7 +1899,7 @@ compute_rocm_rust_content_hash() {
|
||||
local -a content_args=()
|
||||
|
||||
bake_dir=$(dirname "${VLLM_BAKE_FILE}")
|
||||
dockerfile_rocm="${bake_dir}/Dockerfile.rocm"
|
||||
dockerfile_rocm="${CI_BASE_DOCKERFILE:-${bake_dir}/Dockerfile.rocm}"
|
||||
read -r -a content_paths <<< "${content_files}"
|
||||
mapfile -t content_args < <(
|
||||
get_content_arg_names \
|
||||
@@ -1943,7 +1989,7 @@ write_rocm_build_arg_override() {
|
||||
local arg_value=""
|
||||
|
||||
bake_dir=$(dirname "${VLLM_BAKE_FILE}")
|
||||
dockerfile_rocm="${bake_dir}/Dockerfile.rocm"
|
||||
dockerfile_rocm="${CI_BASE_DOCKERFILE:-${bake_dir}/Dockerfile.rocm}"
|
||||
mapfile -t arg_names < <(
|
||||
{
|
||||
get_content_arg_names \
|
||||
@@ -2259,7 +2305,7 @@ extract_dependency_pins() {
|
||||
local val=""
|
||||
|
||||
bake_dir=$(dirname "${VLLM_BAKE_FILE}")
|
||||
dockerfile_rocm="${bake_dir}/Dockerfile.rocm"
|
||||
dockerfile_rocm="${CI_BASE_DOCKERFILE:-${bake_dir}/Dockerfile.rocm}"
|
||||
physical_dockerfile="${dockerfile_rocm}"
|
||||
if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" && "${dockerfile_rocm}" != /* ]]; then
|
||||
physical_dockerfile="${ROCM_BUILD_CONTEXT_ROOT}/${dockerfile_rocm}"
|
||||
@@ -2275,12 +2321,12 @@ extract_dependency_pins() {
|
||||
fi
|
||||
|
||||
val=$(
|
||||
sed -n -E "s/^[[:space:]]*ARG[[:space:]]+${var}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
|
||||
sed -n -E "s/^[[:space:]]*[Aa][Rr][Gg][[:space:]]+${var}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
|
||||
"${physical_dockerfile}" | head -1
|
||||
)
|
||||
if [[ -n "${val}" ]]; then
|
||||
export "${var}=${val}"
|
||||
echo "Extracted ${var}=${val} from Dockerfile.rocm"
|
||||
echo "Extracted ${var}=${val} from ${dockerfile_rocm}"
|
||||
fi
|
||||
done
|
||||
}
|
||||
@@ -2297,7 +2343,7 @@ compute_dependency_cache_keys() {
|
||||
local deepep_material=""
|
||||
|
||||
bake_dir=$(dirname "${VLLM_BAKE_FILE}")
|
||||
dockerfile_rocm="${bake_dir}/Dockerfile.rocm"
|
||||
dockerfile_rocm="${CI_BASE_DOCKERFILE:-${bake_dir}/Dockerfile.rocm}"
|
||||
nixl_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "NIXL_BRANCH")
|
||||
ucx_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "UCX_BRANCH")
|
||||
rocshmem_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "ROCSHMEM_BRANCH")
|
||||
@@ -2687,6 +2733,7 @@ main() {
|
||||
init_bake_files
|
||||
if is_ci_base_target; then
|
||||
prepare_ci_build_context
|
||||
configure_custom_rocm_stages
|
||||
fi
|
||||
compute_ci_base_hash_if_needed
|
||||
configure_ci_base_image_refs
|
||||
@@ -2697,6 +2744,7 @@ main() {
|
||||
# version metadata only after that lookup sees the available tag history.
|
||||
if ! is_ci_base_target; then
|
||||
prepare_ci_build_context
|
||||
configure_custom_rocm_stages
|
||||
fi
|
||||
extract_dependency_pins
|
||||
write_rocm_build_arg_override
|
||||
|
||||
@@ -34,6 +34,9 @@
|
||||
###############################################################################
|
||||
set -o pipefail
|
||||
|
||||
# shellcheck source=.buildkite/scripts/rocm/build-config.sh
|
||||
source "$(dirname "${BASH_SOURCE[0]}")/../rocm/build-config.sh" || exit $?
|
||||
|
||||
: "${BUILDKIT_PROGRESS:=plain}"
|
||||
: "${TERM:=xterm-256color}"
|
||||
: "${FORCE_COLOR:=1}"
|
||||
@@ -106,6 +109,10 @@ clear_ci_orchestration_env() {
|
||||
VLLM_CI_REQUIRE_WORKSPACE_MOUNT \
|
||||
VLLM_TEST_COMMANDS \
|
||||
VLLM_CI_BRANCH \
|
||||
CI_ROCM_DOCKERFILE_BASE \
|
||||
CI_ROCM_DOCKERFILE \
|
||||
ROCM_BASE_DOCKERFILE \
|
||||
CI_BASE_DOCKERFILE \
|
||||
VLLM_CI_BASE_IMAGE \
|
||||
VLLM_CI_FALLBACK_IMAGE \
|
||||
VLLM_CI_DOCKER_DISABLED \
|
||||
@@ -225,6 +232,9 @@ prepare_artifact_image() {
|
||||
metadata_file=$(find "${artifact_work_dir}" -name "ci-base-image.txt" -type f | head -1)
|
||||
if [[ -n "${metadata_file}" && -s "${metadata_file}" ]]; then
|
||||
base_image=$(tr -d '[:space:]' < "${metadata_file}")
|
||||
elif using_custom_rocm_dockerfiles; then
|
||||
echo "Custom ROCm ci_base metadata is missing; using the full CI image"
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "--- Preparing local ROCm test image"
|
||||
|
||||
@@ -83,14 +83,23 @@ rm -f "$build_log"
|
||||
|
||||
# Run the image, setting --shm-size=4g for tensor parallel. Default to
|
||||
# HF_HUB_OFFLINE so a warm ~/.cache/huggingface doesn't hit the network;
|
||||
# retry once online if the cache is missing something. vllm's get_config()
|
||||
# wraps the raw huggingface_hub offline-mode errors in a generic ValueError
|
||||
# (see transformers_utils/config.py), so match that message too or the
|
||||
# fallback never triggers for a config-lookup cache miss.
|
||||
OFFLINE_RETRY_PATTERN='huggingface_hub\.errors\.(LocalEntryNotFoundError|OfflineModeIsEnabled)|Invalid repository ID or local directory specified'
|
||||
# retry once online if the cache is missing something. vllm wraps offline
|
||||
# cache-miss errors in ways that don't preserve the raw huggingface_hub
|
||||
# exception name: get_config() re-raises as a generic ValueError
|
||||
# (transformers_utils/config.py), and default_loader.py raises its own
|
||||
# RuntimeError when a snapshot dir has a config but no weight files. Match
|
||||
# those messages too or the fallback never triggers for those cache misses.
|
||||
#
|
||||
# Also mount ~/.cache/vllm: multimodal test fixtures (e.g. VideoAsset, via
|
||||
# vllm/assets/video.py) download into VLLM_ASSETS_CACHE (~/.cache/vllm/assets
|
||||
# by default), a directory distinct from the HF hub cache above. Without a
|
||||
# persistent mount there, those assets never survive past one container's
|
||||
# lifetime, so every run re-fetches them from the network on the "offline"
|
||||
# attempt and unconditionally falls back to the online retry.
|
||||
OFFLINE_RETRY_PATTERN='huggingface_hub\.errors\.(LocalEntryNotFoundError|OfflineModeIsEnabled)|Invalid repository ID or local directory specified|Cannot find any model weights with'
|
||||
run_test() {
|
||||
local hf_offline=$1
|
||||
docker run --rm --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true -e HF_TOKEN -e VLLM_CPU_KVCACHE_SPACE=16 -e VLLM_CPU_CI_ENV=1 -e VLLM_CPU_SIM_MULTI_NUMA=1 -e VLLM_CPU_ATTN_SPLIT_KV=0 -e HF_HUB_OFFLINE="$hf_offline" -e HF_DATASETS_OFFLINE="$hf_offline" --shm-size=4g "$IMAGE_NAME" \
|
||||
docker run --rm --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" -v ~/.cache/huggingface:/root/.cache/huggingface -v ~/.cache/vllm:/root/.cache/vllm --privileged=true -e HF_TOKEN -e VLLM_CPU_KVCACHE_SPACE=16 -e VLLM_CPU_CI_ENV=1 -e VLLM_CPU_SIM_MULTI_NUMA=1 -e VLLM_CPU_ATTN_SPLIT_KV=0 -e HF_HUB_OFFLINE="$hf_offline" -e HF_DATASETS_OFFLINE="$hf_offline" --shm-size=4g "$IMAGE_NAME" \
|
||||
timeout "$TIMEOUT_VAL" bash -c "set -euox pipefail; echo \"--- Print packages\"; pip list; echo \"--- Running tests\"; ${TEST_COMMAND}"
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env bash
|
||||
# Shared Dockerfile selection for the normal AMD CI build chain.
|
||||
# CI_ROCM_DOCKERFILE_BASE and CI_ROCM_DOCKERFILE accept repository-relative paths.
|
||||
# For Rock, set CI_ROCM_DOCKERFILE_BASE=docker/Dockerfile.rock_base and
|
||||
# CI_ROCM_DOCKERFILE=docker/Dockerfile.rock.
|
||||
# Custom recipes must preserve the CI targets and artifact layout. Additional
|
||||
# COPY inputs also need the corresponding *_CONTENT_FILES overrides.
|
||||
|
||||
rocm_base_dockerfile() {
|
||||
local default="docker/Dockerfile.rocm_base"
|
||||
printf '%s\n' "${CI_ROCM_DOCKERFILE_BASE:-${ROCM_BASE_DOCKERFILE:-${default}}}"
|
||||
}
|
||||
|
||||
rocm_ci_dockerfile() {
|
||||
local default="docker/Dockerfile.rocm"
|
||||
printf '%s\n' "${CI_ROCM_DOCKERFILE:-${CI_BASE_DOCKERFILE:-${default}}}"
|
||||
}
|
||||
|
||||
using_custom_rocm_dockerfiles() {
|
||||
[[ "$(rocm_base_dockerfile)" != "docker/Dockerfile.rocm_base" \
|
||||
|| "$(rocm_ci_dockerfile)" != "docker/Dockerfile.rocm" ]]
|
||||
}
|
||||
|
||||
rocm_dockerfile_stages() {
|
||||
awk '
|
||||
toupper($1) == "FROM" {
|
||||
for (idx = 2; idx < NF; idx++) {
|
||||
if (tolower($idx) == "as") print tolower($(idx + 1))
|
||||
}
|
||||
}
|
||||
' "$1"
|
||||
}
|
||||
|
||||
validate_rocm_dockerfile() {
|
||||
local dockerfile="$1"
|
||||
local stage=""
|
||||
local stages=""
|
||||
shift
|
||||
|
||||
case "${dockerfile}" in
|
||||
/*|../*|*/../*|*/..|*[[:space:]]*)
|
||||
echo "ROCm Dockerfile must be a repository-relative path without '..' or whitespace: ${dockerfile}" >&2
|
||||
return 2
|
||||
;;
|
||||
esac
|
||||
if [[ ! -f "${dockerfile}" ]]; then
|
||||
echo "ROCm Dockerfile not found: ${dockerfile}" >&2
|
||||
return 2
|
||||
fi
|
||||
if (($#)) && ! awk '
|
||||
toupper($1) == "FROM" {
|
||||
named = 0
|
||||
for (idx = 2; idx < NF; idx++) {
|
||||
if (tolower($idx) == "as") named = 1
|
||||
}
|
||||
if (!named) exit 1
|
||||
}
|
||||
' "${dockerfile}"; then
|
||||
echo "ROCm Dockerfile must name every build stage: ${dockerfile}" >&2
|
||||
return 2
|
||||
fi
|
||||
stages=$(rocm_dockerfile_stages "${dockerfile}")
|
||||
for stage in "$@"; do
|
||||
if ! grep -Fxq -- "${stage}" <<< "${stages}"; then
|
||||
echo "ROCm Dockerfile ${dockerfile} is missing required stage: ${stage}" >&2
|
||||
return 2
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
configure_rocm_build() {
|
||||
ROCM_BASE_DOCKERFILE=$(rocm_base_dockerfile)
|
||||
CI_BASE_DOCKERFILE=$(rocm_ci_dockerfile)
|
||||
export ROCM_BASE_DOCKERFILE CI_BASE_DOCKERFILE
|
||||
using_custom_rocm_dockerfiles || return 0
|
||||
|
||||
# Custom stacks may supply dependencies without separate build stages.
|
||||
export ROCM_DEP_CACHE_EXPORT_MODE=never
|
||||
|
||||
# Preserve the standard runtime tags, which are unique to each build,
|
||||
# while preventing experimental builds from promoting stable images.
|
||||
export ROCM_BASE_PUSH_STABLE_TAG=0
|
||||
export CI_BASE_PUSH_STABLE_TAG=0
|
||||
echo "Custom ROCm Dockerfiles: ${ROCM_BASE_DOCKERFILE} and ${CI_BASE_DOCKERFILE}"
|
||||
}
|
||||
@@ -4,6 +4,10 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# shellcheck source=.buildkite/scripts/rocm/build-config.sh
|
||||
source "$(dirname "${BASH_SOURCE[0]}")/build-config.sh"
|
||||
configure_rocm_build
|
||||
|
||||
DOCKERFILE="${ROCM_BASE_DOCKERFILE:-docker/Dockerfile.rocm_base}"
|
||||
BASE_REPO="${ROCM_BASE_IMAGE_REPO:-rocm/vllm-dev}"
|
||||
CACHE_REPO="${ROCM_BASE_CACHE_REPO:-${DOCKERHUB_CACHE_REPO:-rocm/vllm-ci-cache}}"
|
||||
@@ -12,7 +16,12 @@ DEFAULT_ROCM_BASE_METADATA_VERSION="2"
|
||||
DEFAULT_ROCM_BASE_CONTENT_FILES="${DOCKERFILE}"
|
||||
|
||||
ROCM_BASE_LAYER_CACHE_REF=""
|
||||
ROCM_BASE_TRUSTED_LAYER_CACHE_REF="${CACHE_REPO}:rocm-base-main"
|
||||
BASE_CACHE_PREFIX="rocm-base"
|
||||
if using_custom_rocm_dockerfiles; then
|
||||
BASE_CACHE_PREFIX="rocm-base-$(printf '%s\n%s\n' \
|
||||
"${ROCM_BASE_DOCKERFILE}" "${CI_BASE_DOCKERFILE}" | sha256sum | cut -c1-12)"
|
||||
fi
|
||||
ROCM_BASE_TRUSTED_LAYER_CACHE_REF="${CACHE_REPO}:${BASE_CACHE_PREFIX}-main"
|
||||
ROCM_BASE_STABLE_TAG_UPDATED=0
|
||||
declare -a ROCM_BASE_CACHE_ARGS=()
|
||||
|
||||
@@ -120,7 +129,7 @@ configure_rocm_base_layer_cache() {
|
||||
fi
|
||||
|
||||
scope=$(rocm_base_layer_cache_scope)
|
||||
ROCM_BASE_LAYER_CACHE_REF="${CACHE_REPO}:rocm-base-${scope}"
|
||||
ROCM_BASE_LAYER_CACHE_REF="${CACHE_REPO}:${BASE_CACHE_PREFIX}-${scope}"
|
||||
ROCM_BASE_CACHE_ARGS+=(
|
||||
--cache-from "type=registry,ref=${ROCM_BASE_LAYER_CACHE_REF}"
|
||||
)
|
||||
@@ -139,7 +148,7 @@ configure_rocm_base_layer_cache() {
|
||||
extract_arg_default() {
|
||||
local arg_name="$1"
|
||||
|
||||
sed -n -E "s/^[[:space:]]*ARG[[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
|
||||
sed -n -E "s/^[[:space:]]*[Aa][Rr][Gg][[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
|
||||
"${DOCKERFILE}" | head -1
|
||||
}
|
||||
|
||||
@@ -419,11 +428,16 @@ build_base_image() {
|
||||
read -r -a content_paths <<< "${content_files}"
|
||||
content_files_hash="$(compute_content_hash "${content_paths[@]}")"
|
||||
base_hash=$(compute_base_content_hash "${use_sccache}" "${base_image_digest}")
|
||||
rocm_version="$(rocm_version_from_base_image "${base_image_arg}")"
|
||||
rocm_version="$(extract_arg_default ROCM_SDK_VERSION)"
|
||||
rocm_version="${rocm_version:-$(rocm_version_from_base_image "${base_image_arg}")}"
|
||||
triton_arg="$(extract_arg_default TRITON_BRANCH)"
|
||||
pytorch_arg="$(extract_arg_default PYTORCH_BRANCH)"
|
||||
pytorch_vision_arg="$(extract_arg_default PYTORCH_VISION_BRANCH)"
|
||||
pytorch_audio_arg="$(extract_arg_default PYTORCH_AUDIO_BRANCH)"
|
||||
triton_arg="${triton_arg:-$(extract_arg_default TRITON_VERSION)}"
|
||||
pytorch_arg="${pytorch_arg:-$(extract_arg_default TORCH_VERSION)}"
|
||||
pytorch_vision_arg="${pytorch_vision_arg:-$(extract_arg_default TORCHVISION_VERSION)}"
|
||||
pytorch_audio_arg="${pytorch_audio_arg:-$(extract_arg_default TORCHAUDIO_VERSION)}"
|
||||
fa_arg="$(extract_arg_default FA_BRANCH)"
|
||||
aiter_arg="$(extract_arg_default AITER_BRANCH)"
|
||||
mori_arg="$(extract_arg_default MORI_BRANCH)"
|
||||
@@ -569,6 +583,13 @@ build_base_image() {
|
||||
}
|
||||
|
||||
main() {
|
||||
if using_custom_rocm_dockerfiles; then
|
||||
if [[ "${ROCM_BASE_REFRESH_SKIP:-0}" == "1" ]]; then
|
||||
echo "Custom ROCm Dockerfiles require ROCM_BASE_REFRESH_SKIP=0" >&2
|
||||
return 2
|
||||
fi
|
||||
validate_rocm_dockerfile "${DOCKERFILE}" || return $?
|
||||
fi
|
||||
metadata_set "rocm-base-refresh" "0"
|
||||
|
||||
if [[ "${ROCM_BASE_REFRESH_SKIP:-0}" == "1" ]]; then
|
||||
|
||||
@@ -0,0 +1,272 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""FlashMLA mega attention vs the split-KV decode pipeline (DeepSeek V4.1).
|
||||
|
||||
Times the full set of per-layer ops each path runs in a decode step, captured
|
||||
in a CUDA graph -- eager timing is dominated by launch overhead and flatters
|
||||
whichever path launches fewer kernels.
|
||||
|
||||
mega: fused_qnorm_rope_kv_rope_quant_insert (Q pad + KV insert)
|
||||
+ mega kernel
|
||||
split-KV: fused_qnorm_rope_kv_rope_quant_insert (Q RoPE + pad + KV insert)
|
||||
+ flash_mla_with_kvcache + fused_inv_rope_fp8_quant
|
||||
|
||||
Both arms fuse their Q preparation into the KV insert, so that op is charged
|
||||
to both; the mega arm's Q side is only a zero-pad (nothing at all when the
|
||||
shard is already at the kernel head count) because its kernel does the Q RoPE,
|
||||
the inverse RoPE and the FP8 cast.
|
||||
|
||||
``--local-heads`` is the variable that decides the outcome: the mega kernel
|
||||
absorbs work proportional to the live head count while paying for the padded
|
||||
one, so it wins at 64 live heads and is a wash at 16 (TP4 on a 64-head model).
|
||||
|
||||
Both arms use the same KV records so the comparison is kernel-for-kernel:
|
||||
a V4.1 fp8 (528 B) sliding-window cache and a compressed cache that is either
|
||||
V4.1 fp8 or V4.1 NVFP4 (288 B) -- FlashMLA's SM100 sparse decode reads the
|
||||
NVFP4 compressed record in both the fused and unfused kernels.
|
||||
|
||||
Run: .venv/bin/python benchmarks/kernels/benchmark_dsv41_mega_attn.py
|
||||
"""
|
||||
|
||||
import argparse
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.v1.attention.ops.flashmla as fm
|
||||
from vllm.models.deepseek_v4.common.ops.fused_inv_rope_fp8_quant import (
|
||||
fused_inv_rope_fp8_quant,
|
||||
)
|
||||
from vllm.models.deepseek_v41.common.ops import quantize_and_insert_k_cache
|
||||
from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import (
|
||||
rope_quant_insert,
|
||||
)
|
||||
from vllm.models.deepseek_v41.common.ops.fused_layout import permute_q_to_fused
|
||||
from vllm.models.deepseek_v41.nvidia.flash_mla_mega_attn import (
|
||||
alloc_mega_attn_output,
|
||||
is_flashmla_mega_attn_supported,
|
||||
)
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
HEAD_DIM, ROPE_DIM = 512, 64
|
||||
V41_BYTES, V41_FP4_BYTES = 528, 288
|
||||
SWA_BLOCK, COMPRESSED_BLOCK = 32, 128
|
||||
|
||||
|
||||
def make_cos_sin_cache(max_pos, device):
|
||||
inv_freq = 1.0 / (
|
||||
10000 ** (torch.arange(0, ROPE_DIM, 2, device=device).float() / ROPE_DIM)
|
||||
)
|
||||
freqs = torch.outer(torch.arange(max_pos, device=device).float(), inv_freq)
|
||||
return torch.cat([freqs.cos(), freqs.sin()], -1)
|
||||
|
||||
|
||||
def empty_paged_cache(num_rows, block_size, bytes_per_token, device):
|
||||
"""A zeroed paged cache view [num_blocks, block_size, bytes_per_token]."""
|
||||
num_blocks = (num_rows + block_size - 1) // block_size + 1
|
||||
page = round_up(block_size * bytes_per_token, 512)
|
||||
backing = torch.zeros(num_blocks, page, dtype=torch.uint8, device=device)
|
||||
return backing, backing.as_strided(
|
||||
(num_blocks, block_size, bytes_per_token), (page, bytes_per_token, 1)
|
||||
)
|
||||
|
||||
|
||||
def fill_cache(view, bytes_per_token, cos_sin, device):
|
||||
n = view.shape[0] * view.shape[1]
|
||||
rows = torch.randn(n, HEAD_DIM, device=device, dtype=torch.bfloat16)
|
||||
slots = torch.arange(n, dtype=torch.int64, device=device)
|
||||
if bytes_per_token == V41_FP4_BYTES:
|
||||
# A page-aligned cache has more slots than the cos_sin table has rows;
|
||||
# wrap so the insert never indexes past it.
|
||||
positions = slots % cos_sin.shape[0]
|
||||
rope_quant_insert(rows, positions, cos_sin, view, slots, 1)
|
||||
else:
|
||||
quantize_and_insert_k_cache(
|
||||
rows,
|
||||
view.reshape(view.shape[0], -1),
|
||||
slots,
|
||||
block_size=view.shape[1],
|
||||
bytes_per_token=bytes_per_token,
|
||||
)
|
||||
|
||||
|
||||
def time_graph(fn, iters=100, warmup=5):
|
||||
"""Capture ``fn`` in a CUDA graph and time its replay."""
|
||||
s = torch.cuda.Stream()
|
||||
s.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(s):
|
||||
for _ in range(3):
|
||||
fn()
|
||||
torch.cuda.current_stream().wait_stream(s)
|
||||
g = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(g):
|
||||
fn()
|
||||
for _ in range(warmup):
|
||||
g.replay()
|
||||
torch.accelerator.synchronize()
|
||||
start = torch.cuda.Event(enable_timing=True)
|
||||
end = torch.cuda.Event(enable_timing=True)
|
||||
start.record()
|
||||
for _ in range(iters):
|
||||
g.replay()
|
||||
end.record()
|
||||
torch.accelerator.synchronize()
|
||||
return start.elapsed_time(end) * 1000 / iters
|
||||
|
||||
|
||||
def bench_decode(s_q, device, topk_extra, extra_bytes, local_heads):
|
||||
padded_heads, scale = 64, HEAD_DIM**-0.5
|
||||
n_wv_group = padded_heads // 8
|
||||
cos_sin = make_cos_sin_cache(65536, device)
|
||||
positions = torch.randint(0, 65536, (s_q,), device=device, dtype=torch.int64)
|
||||
pos32 = positions.to(torch.int32)
|
||||
|
||||
# Per-step inputs: the wq_b output (local heads) and this step's KV row.
|
||||
q_local = torch.randn(
|
||||
s_q, local_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
q_fused = permute_q_to_fused(q_local)
|
||||
kv = torch.randn(s_q, HEAD_DIM, device=device, dtype=torch.bfloat16)
|
||||
|
||||
n_swa = s_q * 128 + 128
|
||||
swa_backing, swa = empty_paged_cache(n_swa, SWA_BLOCK, V41_BYTES, device)
|
||||
fill_cache(swa, V41_BYTES, cos_sin, device)
|
||||
_, extra = empty_paged_cache(65536, COMPRESSED_BLOCK, extra_bytes, device)
|
||||
fill_cache(extra, extra_bytes, cos_sin, device)
|
||||
|
||||
slot_mapping = torch.arange(s_q, dtype=torch.int64, device=device)
|
||||
swa_idx = torch.randint(0, n_swa, (s_q, 128), device=device, dtype=torch.int32)
|
||||
swa_len = torch.full((s_q,), 128, device=device, dtype=torch.int32)
|
||||
ex_idx = torch.randint(
|
||||
0, 65536, (s_q, topk_extra), device=device, dtype=torch.int32
|
||||
)
|
||||
ex_len = torch.full((s_q,), topk_extra, device=device, dtype=torch.int32)
|
||||
sink = torch.zeros(padded_heads, device=device)
|
||||
sched = fm.FlashMLASchedMeta()
|
||||
out = alloc_mega_attn_output(s_q, n_wv_group, device)
|
||||
swa4 = swa.unsqueeze(-2)
|
||||
extra4 = extra.unsqueeze(-2)
|
||||
swa_2d = swa_backing
|
||||
|
||||
def mega():
|
||||
# One launch zero-pads the fused-layout Q and inserts the SWA KV.
|
||||
q_pad = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
q_fused,
|
||||
kv,
|
||||
swa_2d,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin,
|
||||
0 if padded_heads == local_heads else padded_heads,
|
||||
1e-6,
|
||||
SWA_BLOCK,
|
||||
False, # apply_q_norm
|
||||
True, # kv_mxfp8
|
||||
False, # apply_q_rope: the mega kernel rotates Q itself
|
||||
True, # is_q_interleaved
|
||||
)
|
||||
torch.ops._flashmla_C.fused_norm_rope_attn_rope_cast_decode(
|
||||
q_fused if padded_heads == local_heads else q_pad,
|
||||
swa4,
|
||||
swa_idx,
|
||||
scale,
|
||||
HEAD_DIM,
|
||||
sink,
|
||||
swa_len,
|
||||
extra4,
|
||||
ex_idx,
|
||||
ex_len,
|
||||
False,
|
||||
0.0,
|
||||
pos32,
|
||||
False,
|
||||
64,
|
||||
cos_sin,
|
||||
n_wv_group,
|
||||
32,
|
||||
True,
|
||||
True,
|
||||
True,
|
||||
out.data,
|
||||
out.scale,
|
||||
)
|
||||
|
||||
def split_kv():
|
||||
q_pad = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
q_local,
|
||||
kv,
|
||||
swa_2d,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin,
|
||||
padded_heads,
|
||||
1e-6,
|
||||
SWA_BLOCK,
|
||||
False,
|
||||
True,
|
||||
)
|
||||
o = fm.flash_mla_with_kvcache(
|
||||
q=q_pad.unsqueeze(1),
|
||||
k_cache=swa4,
|
||||
block_table=None,
|
||||
head_dim_v=HEAD_DIM,
|
||||
tile_scheduler_metadata=sched,
|
||||
cache_seqlens=None,
|
||||
is_fp8_kvcache=True,
|
||||
indices=swa_idx.view(s_q, 1, -1),
|
||||
topk_length=swa_len,
|
||||
softmax_scale=scale,
|
||||
attn_sink=sink,
|
||||
extra_k_cache=extra4,
|
||||
extra_indices_in_kvcache=ex_idx.view(s_q, 1, -1),
|
||||
extra_topk_length=ex_len,
|
||||
)[0]
|
||||
fused_inv_rope_fp8_quant(
|
||||
o.squeeze(1)[:, :local_heads],
|
||||
positions,
|
||||
cos_sin,
|
||||
n_groups=local_heads // 8,
|
||||
heads_per_group=8,
|
||||
quant_group_size=32,
|
||||
tma_aligned_scales=True,
|
||||
)
|
||||
|
||||
return time_graph(mega), time_graph(split_kv)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--decode-s-q",
|
||||
nargs="+",
|
||||
type=int,
|
||||
default=[1, 2, 4, 8, 16, 32, 64, 128, 256, 512],
|
||||
)
|
||||
parser.add_argument("--topk-extra", type=int, default=512)
|
||||
parser.add_argument("--local-heads", type=int, default=16)
|
||||
parser.add_argument(
|
||||
"--extra-bytes",
|
||||
type=int,
|
||||
choices=[V41_BYTES, V41_FP4_BYTES],
|
||||
default=V41_BYTES,
|
||||
help="compressed-cache record: 528 (V4.1 fp8) or 288 (V4.1 NVFP4)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
ok, reason = is_flashmla_mega_attn_supported()
|
||||
if not ok:
|
||||
raise SystemExit(reason)
|
||||
device = torch.device("cuda")
|
||||
torch.manual_seed(0)
|
||||
print(
|
||||
f"decode, cudagraph (topk_swa=128, topk_extra={args.topk_extra}, "
|
||||
f"local_heads={args.local_heads}, extra={args.extra_bytes}B)"
|
||||
)
|
||||
print(f"{'s_q':>6} {'mega_us':>9} {'split_kv_us':>12} {'speedup':>8}")
|
||||
for s_q in args.decode_s_q:
|
||||
m, s = bench_decode(
|
||||
s_q, device, args.topk_extra, args.extra_bytes, args.local_heads
|
||||
)
|
||||
print(f"{s_q:>6d} {m:9.1f} {s:12.1f} {s / m:7.2f}x")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -3,7 +3,10 @@
|
||||
* SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
*
|
||||
* Horizontally-fused DeepseekV4-MLA kernel:
|
||||
* - Q side: optional per-head RMSNorm + GPT-J RoPE on last ROPE_DIM
|
||||
* - Q side: optional per-head RMSNorm (apply_q_norm) and GPT-J RoPE on the
|
||||
* last ROPE_DIM (apply_q_rope), plus the zero-pad to
|
||||
* q_head_padded, in the head-major layout or (is_q_interleaved)
|
||||
* FlashMLA's mega-attention chunk-interleaved one
|
||||
* - KV side: GPT-J RoPE on last ROPE_DIM + UE8M0 FP8 quant on NoPE + paged
|
||||
* cache insert
|
||||
*
|
||||
@@ -174,6 +177,30 @@ __device__ __forceinline__ float warpSum(float val) {
|
||||
return val;
|
||||
}
|
||||
|
||||
// Offset of the 16 elements a lane owns inside a Q tensor of `num_heads`
|
||||
// heads. The default layout is head-major: a head's 512 dims are contiguous.
|
||||
// `Q_INTERLEAVED` is FlashMLA's mega-attention layout, 32 chunks of
|
||||
// [num_heads, 16] per token, which changes the address only -- lane L still
|
||||
// owns dims [16L, 16L+16) of its head either way, so `dim_base` stays the
|
||||
// true dim offset and the norm and RoPE below are equally valid in both.
|
||||
// What it does change is where the padding heads land: at the tail of every
|
||||
// chunk rather than after the live heads.
|
||||
template <bool Q_INTERLEAVED>
|
||||
__device__ __forceinline__ int64_t qElemOffset(int const tokenIdx,
|
||||
int const slotIdx,
|
||||
int const laneId,
|
||||
int const dim_base,
|
||||
int const num_heads) {
|
||||
if constexpr (Q_INTERLEAVED) {
|
||||
return static_cast<int64_t>(tokenIdx) * num_heads * kHeadDim +
|
||||
static_cast<int64_t>(laneId) * num_heads * kElemsPerLane +
|
||||
static_cast<int64_t>(slotIdx) * kElemsPerLane;
|
||||
} else {
|
||||
return (static_cast<int64_t>(tokenIdx) * num_heads + slotIdx) * kHeadDim +
|
||||
dim_base;
|
||||
}
|
||||
}
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
// Per-slot inner pipeline
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
@@ -192,10 +219,16 @@ __device__ __forceinline__ float warpSum(float val) {
|
||||
// + paged-cache
|
||||
// insert)
|
||||
//
|
||||
// `kv_mxfp8` is grid-uniform and only steers the KV branch, so it stays a
|
||||
// runtime argument rather than doubling every (head count, q-norm)
|
||||
// instantiation of a kernel whose hot path is the Q branch.
|
||||
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM>
|
||||
// `kv_mxfp8` and `apply_q_rope` are grid-uniform, so they stay runtime
|
||||
// arguments rather than multiplying every (head count, q-norm, q-layout)
|
||||
// instantiation of a kernel whose hot path is the Q branch. `Q_INTERLEAVED`
|
||||
// cannot: it picks the address arithmetic, which has to fold.
|
||||
//
|
||||
// The three Q knobs are independent. The mega-attention layer uses
|
||||
// Q_INTERLEAVED with both norm and RoPE off, because its attention kernel
|
||||
// applies those itself -- not because the layout forbids them.
|
||||
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM,
|
||||
bool Q_INTERLEAVED = false>
|
||||
__device__ __forceinline__ void processDeepseekV4Slot(
|
||||
uint4 v0, uint4 v1, int const tokenIdx, int const slotIdx,
|
||||
int const dim_base, int const laneId, int const num_heads_q,
|
||||
@@ -203,7 +236,7 @@ __device__ __forceinline__ void processDeepseekV4Slot(
|
||||
uint8_t* __restrict__ k_cache, int64_t const* __restrict__ slot_mapping,
|
||||
int64_t const* __restrict__ position_ids,
|
||||
float const* __restrict__ cos_sin_cache, int const cache_block_size,
|
||||
int const kv_block_stride, bool const kv_mxfp8) {
|
||||
int const kv_block_stride, bool const kv_mxfp8, bool const apply_q_rope) {
|
||||
using Converter = vllm::_typeConvert<scalar_t_in>;
|
||||
bool const isKV = (slotIdx == kNumHeadsQPadded);
|
||||
bool const isPadQ = !isKV && (slotIdx >= num_heads_q);
|
||||
@@ -213,10 +246,8 @@ __device__ __forceinline__ void processDeepseekV4Slot(
|
||||
// zero literal is correct. Matches the live-Q branch's vectorized store.
|
||||
if (isPadQ) {
|
||||
scalar_t_in* dst =
|
||||
q_out +
|
||||
(static_cast<int64_t>(tokenIdx) * kNumHeadsQPadded + slotIdx) *
|
||||
kHeadDim +
|
||||
dim_base;
|
||||
q_out + qElemOffset<Q_INTERLEAVED>(tokenIdx, slotIdx, laneId, dim_base,
|
||||
kNumHeadsQPadded);
|
||||
uint4 const zero4 = {0u, 0u, 0u, 0u};
|
||||
*reinterpret_cast<uint4*>(dst) = zero4;
|
||||
*reinterpret_cast<uint4*>(dst + 8) = zero4;
|
||||
@@ -264,7 +295,9 @@ __device__ __forceinline__ void processDeepseekV4Slot(
|
||||
|
||||
// ── GPT-J RoPE on dims [NOPE_DIM, HEAD_DIM) ─────────────────────────────
|
||||
// All math in fp32. cos_sin_cache is loaded as fp32 (its native storage).
|
||||
bool const is_rope_lane = dim_base >= kNopeDim;
|
||||
// KV is always rotated; Q only when the caller has not already had it
|
||||
// rotated elsewhere (the mega-attention kernel does its own).
|
||||
bool const is_rope_lane = dim_base >= kNopeDim && (isKV || apply_q_rope);
|
||||
if (is_rope_lane) {
|
||||
int64_t const pos = position_ids[tokenIdx];
|
||||
constexpr int kHalfRope = kRopeDim / 2;
|
||||
@@ -314,10 +347,8 @@ __device__ __forceinline__ void processDeepseekV4Slot(
|
||||
make_float2(elements[8 + 2 * i], elements[8 + 2 * i + 1]));
|
||||
}
|
||||
scalar_t_in* dst =
|
||||
q_out +
|
||||
(static_cast<int64_t>(tokenIdx) * kNumHeadsQPadded + slotIdx) *
|
||||
kHeadDim +
|
||||
dim_base;
|
||||
q_out + qElemOffset<Q_INTERLEAVED>(tokenIdx, slotIdx, laneId, dim_base,
|
||||
kNumHeadsQPadded);
|
||||
*reinterpret_cast<uint4*>(dst) = out0;
|
||||
*reinterpret_cast<uint4*>(dst + 8) = out1;
|
||||
} else {
|
||||
@@ -432,14 +463,17 @@ __device__ __forceinline__ void processDeepseekV4Slot(
|
||||
// `kNumHeadsQPadded` is a template parameter (compile-time constant) so the
|
||||
// divisions in the grid math and the KV-sentinel comparison fold to fast
|
||||
// constant operations. The launch wrapper dispatches the runtime value to
|
||||
// the matching instantiation.
|
||||
// the matching instantiation. `kNumHeadsQPadded == 0` leaves one slot per
|
||||
// token and is the KV-insert-only instantiation, for a Q that is already in
|
||||
// the shape its attention kernel wants.
|
||||
//
|
||||
// With DP padding, q/kv/position_ids can have more rows than slot_mapping.
|
||||
// The live-Q and pad-Q branches cover all `num_tokens_full` rows (downstream
|
||||
// attention uses them). The KV branch only inserts the first
|
||||
// `num_tokens_insert` tokens (= slot_mapping length) into the paged cache.
|
||||
//
|
||||
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM>
|
||||
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM,
|
||||
bool Q_INTERLEAVED>
|
||||
__global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel(
|
||||
scalar_t_in const* __restrict__ q_in, // [N, num_heads_q, 512]
|
||||
scalar_t_in* __restrict__ q_out, // [N, kNumHeadsQPadded, 512]
|
||||
@@ -454,7 +488,8 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel(
|
||||
int const num_heads_q, // live Q heads (input layout)
|
||||
int const cache_block_size, // tokens per paged-cache block
|
||||
int const kv_block_stride, // bytes per paged-cache block
|
||||
bool const kv_mxfp8) { // V4.1 all-dims MXFP8 record
|
||||
bool const kv_mxfp8, // V4.1 all-dims MXFP8 record
|
||||
bool const apply_q_rope) { // rotate Q too, not just KV
|
||||
#if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM)
|
||||
// BF16 _typeConvert specialization is unavailable on pre-Ampere. The
|
||||
// DeepseekV4 kernel only runs with bf16 inputs in practice, so compile a
|
||||
@@ -497,20 +532,18 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel(
|
||||
if (isKV) {
|
||||
src_ptr = kv_in + static_cast<int64_t>(tokenIdx) * kHeadDim + dim_base;
|
||||
} else {
|
||||
int64_t const q_row_offset =
|
||||
(static_cast<int64_t>(tokenIdx) * num_heads_q + slotIdx) *
|
||||
kHeadDim +
|
||||
dim_base;
|
||||
src_ptr = q_in + q_row_offset;
|
||||
src_ptr = q_in + qElemOffset<Q_INTERLEAVED>(tokenIdx, slotIdx, laneId,
|
||||
dim_base, num_heads_q);
|
||||
}
|
||||
v0 = *reinterpret_cast<uint4 const*>(src_ptr);
|
||||
v1 = *reinterpret_cast<uint4 const*>(src_ptr + 8);
|
||||
}
|
||||
|
||||
processDeepseekV4Slot<scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>(
|
||||
processDeepseekV4Slot<scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM,
|
||||
Q_INTERLEAVED>(
|
||||
v0, v1, tokenIdx, slotIdx, dim_base, laneId, num_heads_q, eps, q_out,
|
||||
k_cache, slot_mapping, position_ids, cos_sin_cache, cache_block_size,
|
||||
kv_block_stride, kv_mxfp8);
|
||||
kv_block_stride, kv_mxfp8, apply_q_rope);
|
||||
|
||||
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
|
||||
cudaTriggerProgrammaticLaunchCompletion();
|
||||
@@ -530,7 +563,8 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel(
|
||||
// Q branch (optional RMSNorm + RoPE, in place) head_slot == num_heads_q
|
||||
// KV branch (RoPE + UE8M0 quant + insert)
|
||||
//
|
||||
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM>
|
||||
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM,
|
||||
bool Q_INTERLEAVED>
|
||||
__global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid(
|
||||
scalar_t_in const* __restrict__ q_in, scalar_t_in* __restrict__ q_out,
|
||||
scalar_t_in const* __restrict__ kv_in, uint8_t* __restrict__ k_cache,
|
||||
@@ -539,7 +573,7 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid(
|
||||
float const* __restrict__ cos_sin_cache, float const eps,
|
||||
int const num_tokens_full, int const num_tokens_insert,
|
||||
int const num_heads_q, int const cache_block_size,
|
||||
int const kv_block_stride, bool const kv_mxfp8) {
|
||||
int const kv_block_stride, bool const kv_mxfp8, bool const apply_q_rope) {
|
||||
#if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM)
|
||||
if constexpr (std::is_same_v<scalar_t_in, c10::BFloat16>) {
|
||||
return;
|
||||
@@ -569,11 +603,8 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid(
|
||||
if (s == kNumHeadsQPadded) {
|
||||
src = kv_in + static_cast<int64_t>(tokenIdx) * kHeadDim + dim_base;
|
||||
} else {
|
||||
src = q_in +
|
||||
(static_cast<int64_t>(tokenIdx) * num_heads_q +
|
||||
static_cast<int64_t>(s)) *
|
||||
kHeadDim +
|
||||
dim_base;
|
||||
src = q_in + qElemOffset<Q_INTERLEAVED>(tokenIdx, s, laneId, dim_base,
|
||||
num_heads_q);
|
||||
}
|
||||
va = *reinterpret_cast<uint4 const*>(src);
|
||||
vb = *reinterpret_cast<uint4 const*>(src + 8);
|
||||
@@ -594,10 +625,12 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid(
|
||||
load_slot(next_slot, v0_next, v1_next);
|
||||
}
|
||||
|
||||
processDeepseekV4Slot<scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>(
|
||||
processDeepseekV4Slot<scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM,
|
||||
Q_INTERLEAVED>(
|
||||
v0_curr, v1_curr, tokenIdx, curr_slot, dim_base, laneId,
|
||||
num_heads_q, eps, q_out, k_cache, slot_mapping, position_ids,
|
||||
cos_sin_cache, cache_block_size, kv_block_stride, kv_mxfp8);
|
||||
cos_sin_cache, cache_block_size, kv_block_stride, kv_mxfp8,
|
||||
apply_q_rope);
|
||||
|
||||
// ── Buffer rotation: hand the prefetched LDGs to the next iter.
|
||||
v0_curr = v0_next;
|
||||
@@ -617,14 +650,15 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid(
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
// Launch wrapper
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM>
|
||||
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM,
|
||||
bool Q_INTERLEAVED>
|
||||
static void launchFusedDeepseekV4Templated(
|
||||
scalar_t_in const* q_in, scalar_t_in* q_out, scalar_t_in const* kv_in,
|
||||
uint8_t* k_cache, int64_t const* slot_mapping, int64_t const* position_ids,
|
||||
float const* cos_sin_cache, float const eps, int const num_tokens_full,
|
||||
int const num_tokens_insert, int const num_heads_q,
|
||||
int const cache_block_size, int const kv_block_stride, bool const kv_mxfp8,
|
||||
cudaStream_t stream) {
|
||||
bool const apply_q_rope, cudaStream_t stream) {
|
||||
constexpr int kBlockSize = 256;
|
||||
constexpr int kWarpsPerBlock = kBlockSize / 32;
|
||||
int64_t const total_warps =
|
||||
@@ -663,37 +697,40 @@ static void launchFusedDeepseekV4Templated(
|
||||
// grid instead. Only reachable above NUM_TOKEN_CUTOFF tokens in a single
|
||||
// insert, where it is worth ~2x.
|
||||
if (kNumHeadsQPadded == 0 || num_tokens_full < NUM_TOKEN_CUTOFF) {
|
||||
cudaLaunchKernelEx(&config,
|
||||
fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel<
|
||||
scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>,
|
||||
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids,
|
||||
cos_sin_cache, eps, num_tokens_full, num_tokens_insert,
|
||||
num_heads_q, cache_block_size, kv_block_stride,
|
||||
kv_mxfp8);
|
||||
cudaLaunchKernelEx(
|
||||
&config,
|
||||
fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel<
|
||||
scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM, Q_INTERLEAVED>,
|
||||
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, cos_sin_cache,
|
||||
eps, num_tokens_full, num_tokens_insert, num_heads_q, cache_block_size,
|
||||
kv_block_stride, kv_mxfp8, apply_q_rope);
|
||||
} else {
|
||||
config.gridDim = dim3(num_tokens_full);
|
||||
cudaLaunchKernelEx(
|
||||
&config,
|
||||
fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid<
|
||||
scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>,
|
||||
scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM, Q_INTERLEAVED>,
|
||||
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, cos_sin_cache,
|
||||
eps, num_tokens_full, num_tokens_insert, num_heads_q, cache_block_size,
|
||||
kv_block_stride, kv_mxfp8);
|
||||
kv_block_stride, kv_mxfp8, apply_q_rope);
|
||||
}
|
||||
#else
|
||||
// ROCm: use standard kernel launch syntax (no PDL/stream serialization)
|
||||
// clang-format off
|
||||
fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel<
|
||||
scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>
|
||||
scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM, Q_INTERLEAVED>
|
||||
<<<grid, kBlockSize, 0, stream>>>(
|
||||
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids,
|
||||
cos_sin_cache, eps, num_tokens_full, num_tokens_insert, num_heads_q,
|
||||
cache_block_size, kv_block_stride, kv_mxfp8);
|
||||
cache_block_size, kv_block_stride, kv_mxfp8, apply_q_rope);
|
||||
#endif
|
||||
}
|
||||
|
||||
// Runtime dispatch into one of the precompiled `kNumHeadsQPadded`
|
||||
// instantiations. Supported padded head counts: 8, 16, 32, 64, 128.
|
||||
// instantiations. Supported padded head counts: 8, 16, 32, 64, 128, plus 0
|
||||
// for a KV-only launch that writes no Q at all. `apply_q_norm` and
|
||||
// `is_q_interleaved` are template parameters, `apply_q_rope` is not (see
|
||||
// processDeepseekV4Slot), so each head count carries four instantiations.
|
||||
template <typename scalar_t_in>
|
||||
void launchFusedDeepseekV4QNormRopeKVRopeQuantInsert(
|
||||
scalar_t_in const* q_in, scalar_t_in* q_out, scalar_t_in const* kv_in,
|
||||
@@ -702,22 +739,32 @@ void launchFusedDeepseekV4QNormRopeKVRopeQuantInsert(
|
||||
int const num_tokens_full, int const num_tokens_insert,
|
||||
int const num_heads_q, int const num_heads_q_padded,
|
||||
int const cache_block_size, int const kv_block_stride,
|
||||
bool const apply_q_norm, bool const kv_mxfp8, cudaStream_t stream) {
|
||||
#define DISPATCH(N) \
|
||||
case N: \
|
||||
if (apply_q_norm) { \
|
||||
launchFusedDeepseekV4Templated<scalar_t_in, N, true>( \
|
||||
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, \
|
||||
cos_sin_cache, eps, num_tokens_full, num_tokens_insert, \
|
||||
num_heads_q, cache_block_size, kv_block_stride, kv_mxfp8, stream); \
|
||||
} else { \
|
||||
launchFusedDeepseekV4Templated<scalar_t_in, N, false>( \
|
||||
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, \
|
||||
cos_sin_cache, eps, num_tokens_full, num_tokens_insert, \
|
||||
num_heads_q, cache_block_size, kv_block_stride, kv_mxfp8, stream); \
|
||||
} \
|
||||
bool const apply_q_norm, bool const kv_mxfp8, bool const apply_q_rope,
|
||||
bool const is_q_interleaved, cudaStream_t stream) {
|
||||
#define LAUNCH(N, NORM, INTERLEAVED) \
|
||||
launchFusedDeepseekV4Templated<scalar_t_in, N, NORM, INTERLEAVED>( \
|
||||
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, cos_sin_cache, \
|
||||
eps, num_tokens_full, num_tokens_insert, num_heads_q, cache_block_size, \
|
||||
kv_block_stride, kv_mxfp8, apply_q_rope, stream)
|
||||
#define DISPATCH(N) \
|
||||
case N: \
|
||||
if (is_q_interleaved) { \
|
||||
if (apply_q_norm) { \
|
||||
LAUNCH(N, true, true); \
|
||||
} else { \
|
||||
LAUNCH(N, false, true); \
|
||||
} \
|
||||
} else if (apply_q_norm) { \
|
||||
LAUNCH(N, true, false); \
|
||||
} else { \
|
||||
LAUNCH(N, false, false); \
|
||||
} \
|
||||
return;
|
||||
|
||||
if (num_heads_q_padded == 0) {
|
||||
LAUNCH(0, false, false);
|
||||
return;
|
||||
}
|
||||
switch (num_heads_q_padded) {
|
||||
DISPATCH(8)
|
||||
DISPATCH(16)
|
||||
@@ -729,9 +776,10 @@ void launchFusedDeepseekV4QNormRopeKVRopeQuantInsert(
|
||||
"fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert: "
|
||||
"unsupported num_heads_q_padded=",
|
||||
num_heads_q_padded,
|
||||
" (compiled instantiations: 8, 16, 32, 64, 128).");
|
||||
" (compiled instantiations: 0, 8, 16, 32, 64, 128).");
|
||||
}
|
||||
#undef DISPATCH
|
||||
#undef LAUNCH
|
||||
}
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
@@ -1059,10 +1107,15 @@ void fused_deepseek_v4_kv_rope_insert(
|
||||
// Zero query heads schedules only the existing KV branch, preserving its
|
||||
// RoPE rounding and quantization without allocating any query tensors.
|
||||
if (packed) {
|
||||
vllm::deepseek_v4_fused_ops::launchFusedDeepseekV4Templated<scalar_t, 0, false>(
|
||||
// With zero query heads no Q slot is scheduled, so neither Q knob is
|
||||
// reachable: Q_INTERLEAVED picks addressing nothing uses, and apply_q_rope
|
||||
// gates a branch only a Q slot takes (KV is rotated unconditionally).
|
||||
vllm::deepseek_v4_fused_ops::launchFusedDeepseekV4Templated<scalar_t, 0,
|
||||
false, false>(
|
||||
nullptr, nullptr, input, cache, slots, positions, cos_sin, 0.0f,
|
||||
num_tokens, num_tokens, 0, static_cast<int>(cache_block_size),
|
||||
static_cast<int>(k_cache.stride(0)), kv_mxfp8, stream);
|
||||
static_cast<int>(k_cache.stride(0)), kv_mxfp8, /*apply_q_rope=*/true,
|
||||
stream);
|
||||
} else if (fp8) {
|
||||
vllm::deepseek_v4_fused_ops::launchFullCacheKernel<scalar_t, false, true>(
|
||||
nullptr, nullptr, 0, 0, input, cache, slots, positions, cos_sin,
|
||||
@@ -1085,8 +1138,9 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
torch::stable::Tensor const& slot_mapping, // [N] int64
|
||||
torch::stable::Tensor const& position_ids, // [N] int64
|
||||
torch::stable::Tensor const& cos_sin_cache, // [max_pos, rope_dim] bf16
|
||||
int64_t q_head_padded, // padded Q head count for output
|
||||
double eps, int64_t cache_block_size, bool apply_q_norm, bool kv_mxfp8) {
|
||||
int64_t q_head_padded, // padded Q head count for output, 0 for no Q
|
||||
double eps, int64_t cache_block_size, bool apply_q_norm, bool kv_mxfp8,
|
||||
bool apply_q_rope, bool is_q_interleaved) {
|
||||
STD_TORCH_CHECK(q_in.device().is_cuda() && q_in.is_contiguous(),
|
||||
"q_in must be contiguous CUDA");
|
||||
STD_TORCH_CHECK(kv.device().is_cuda() && kv.is_contiguous(),
|
||||
@@ -1106,8 +1160,11 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
STD_TORCH_CHECK(kv.dim() == 2 && kv.size(1) == 512, "kv shape [N, 512]");
|
||||
STD_TORCH_CHECK(q_in.scalar_type() == kv.scalar_type(),
|
||||
"q_in and kv dtype must match");
|
||||
STD_TORCH_CHECK(q_head_padded >= q_in.size(1),
|
||||
"q_head_padded must be >= q_in.size(1) (num_heads_q)");
|
||||
// q_head_padded == 0 asks for the KV insert alone: Q is left untouched and
|
||||
// an empty tensor comes back, for a caller whose Q is already in the shape
|
||||
// its attention kernel reads.
|
||||
STD_TORCH_CHECK(q_head_padded == 0 || q_head_padded >= q_in.size(1),
|
||||
"q_head_padded must be 0 or >= q_in.size(1) (num_heads_q)");
|
||||
STD_TORCH_CHECK(k_cache.scalar_type() == torch::headeronly::ScalarType::Byte,
|
||||
"k_cache must be uint8");
|
||||
STD_TORCH_CHECK(cos_sin_cache.dim() == 2 && cos_sin_cache.size(1) == 64,
|
||||
@@ -1138,8 +1195,12 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
// Allocate the padded q output. The kernel writes every element (live
|
||||
// region gets optional RMSNorm+RoPE; pad region gets zeros), so `empty` is
|
||||
// safe.
|
||||
auto q_out = torch::stable::new_empty(
|
||||
q_in, {q_in.size(0), q_head_padded, q_in.size(2)}, q_in.scalar_type());
|
||||
auto q_out =
|
||||
q_head_padded == 0
|
||||
? torch::stable::new_empty(q_in, {int64_t{0}}, q_in.scalar_type())
|
||||
: torch::stable::new_empty(
|
||||
q_in, {q_in.size(0), q_head_padded, q_in.size(2)},
|
||||
q_in.scalar_type());
|
||||
|
||||
VLLM_STABLE_DISPATCH_HALF_TYPES(
|
||||
q_in.scalar_type(), "fused_deepseek_v4_qnorm_rope_kv_insert", [&] {
|
||||
@@ -1155,7 +1216,8 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
cos_sin_cache.const_data_ptr<float>(), static_cast<float>(eps),
|
||||
num_tokens_full, num_tokens_insert, num_heads_q,
|
||||
num_heads_q_padded, cache_block_size_i, kv_block_stride,
|
||||
apply_q_norm, kv_mxfp8, stream);
|
||||
apply_q_norm, kv_mxfp8, apply_q_rope, is_q_interleaved,
|
||||
stream);
|
||||
});
|
||||
return q_out;
|
||||
}
|
||||
|
||||
@@ -271,7 +271,8 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
torch::stable::Tensor& k_cache, torch::stable::Tensor const& slot_mapping,
|
||||
torch::stable::Tensor const& position_ids,
|
||||
torch::stable::Tensor const& cos_sin_cache, int64_t q_head_padded,
|
||||
double eps, int64_t cache_block_size, bool apply_q_norm, bool kv_mxfp8);
|
||||
double eps, int64_t cache_block_size, bool apply_q_norm, bool kv_mxfp8,
|
||||
bool apply_q_rope, bool is_q_interleaved);
|
||||
|
||||
void fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert(
|
||||
torch::stable::Tensor& q, torch::stable::Tensor const& kv,
|
||||
|
||||
@@ -425,6 +425,12 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
"bool is_neox, Tensor position_ids, "
|
||||
"int forced_token_heads_per_warp=-1) -> ()");
|
||||
|
||||
// q_head_padded is the padded Q head count of the returned tensor, or 0 to
|
||||
// do the KV insert alone and return an empty tensor. The Q knobs are
|
||||
// independent: apply_q_norm and apply_q_rope each drop that step for Q
|
||||
// alone (KV is always rotated), and is_q_interleaved reads and writes Q in
|
||||
// FlashMLA's mega-attention chunk-interleaved layout, which moves the
|
||||
// padding heads to the tail of every head-dim chunk.
|
||||
ops.def(
|
||||
"fused_deepseek_v4_kv_rope_insert("
|
||||
"Tensor kv, Tensor! k_cache, Tensor slot_mapping, Tensor position_ids, "
|
||||
@@ -436,7 +442,8 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
"Tensor q_in, Tensor kv, Tensor! k_cache, "
|
||||
"Tensor slot_mapping, Tensor position_ids, Tensor cos_sin_cache, "
|
||||
"int q_head_padded, float eps, int cache_block_size, "
|
||||
"bool apply_q_norm=True, bool kv_mxfp8=False) -> Tensor");
|
||||
"bool apply_q_norm=True, bool kv_mxfp8=False, bool apply_q_rope=True, "
|
||||
"bool is_q_interleaved=False) -> Tensor");
|
||||
|
||||
// FlashInfer V4 full-cache variants: write Q in place (bf16) or to a separate
|
||||
// FP8 tensor, and KV into a contiguous 512-wide token-strided cache.
|
||||
|
||||
@@ -33,6 +33,10 @@ variable "CI_BASE_CONTENT_HASH" {
|
||||
default = ""
|
||||
}
|
||||
|
||||
variable "CI_BASE_DOCKERFILE" {
|
||||
default = "docker/Dockerfile.rocm"
|
||||
}
|
||||
|
||||
# REMOTE_VLLM=0: use local source via Docker build context (ONBUILD COPY ./ vllm/)
|
||||
# REMOTE_VLLM=1: clone from GitHub at VLLM_BRANCH (standalone builds without local source)
|
||||
variable "REMOTE_VLLM" {
|
||||
@@ -74,7 +78,7 @@ group "default" {
|
||||
}
|
||||
|
||||
target "_common-rocm" {
|
||||
dockerfile = "docker/Dockerfile.rocm"
|
||||
dockerfile = CI_BASE_DOCKERFILE
|
||||
context = "."
|
||||
args = {
|
||||
max_jobs = MAX_JOBS
|
||||
|
||||
@@ -20,6 +20,15 @@ Both approaches are under active development.
|
||||
|
||||
Due to the auto-regressive nature of decoding, every decoding step needs to compute a small amount of query tokens w.r.t. a large number of key/value tokens stored in the paged KV cache. The core of decode context parallel is how to shard the KV cache across GPUs.
|
||||
|
||||
`engine_client.vllm_config.cache_config.effective_attention_block_size` reports the
|
||||
initialized full-attention block size in tokens, including DCP: a physical block of
|
||||
16 tokens with DCP=4 represents 64 tokens. The same value is available through
|
||||
gRPC `Control.GetServerInfo.effective_attention_block_size`. Physical block-size
|
||||
fields retain their existing meaning. The new value is unavailable (`None` in
|
||||
Python, absent in gRPC) if there is no common full-attention size, the scheduler
|
||||
does not expose it, or an older engine omits it. gRPC also omits the value when
|
||||
engine replicas disagree. Partial cache events carry their own actual size.
|
||||
|
||||
For a model with `H` kv-heads, a request with `T` tokens in the context needs to store `H * T` key/value tensors in the KV cache.
|
||||
|
||||
1. If one GPU can hold them all, and the performance is good enough, then no parallelization is needed.
|
||||
|
||||
+1
-1
@@ -147,7 +147,7 @@ vllm-llm = { path = "src/llm" }
|
||||
vllm-managed-engine = { path = "src/managed-engine" }
|
||||
vllm-metrics = { path = "src/metrics" }
|
||||
vllm-parser = { path = "src/parser" }
|
||||
vllm-proto = { version = "0.2.0", path = "proto" }
|
||||
vllm-proto = { path = "proto" }
|
||||
vllm-server = { path = "src/server" }
|
||||
vllm-text = { path = "src/text" }
|
||||
vllm-tokenizer = { path = "src/tokenizer" }
|
||||
|
||||
@@ -2,6 +2,9 @@
|
||||
|
||||
This directory is the canonical source for vLLM's gRPC schema.
|
||||
|
||||
See [context parallel deployment](../../docs/serving/context_parallel_deployment.md)
|
||||
for effective attention block-size metadata in Python and `Control.GetServerInfo`.
|
||||
|
||||
Schema updates are no longer published to the Buf Schema Registry. Rust consumers
|
||||
should use `vllm-proto` from crates.io; consumers in other languages can generate
|
||||
bindings from the `.proto` files in this directory. Buf still builds and lints
|
||||
@@ -34,8 +37,7 @@ On pull requests and releases, `cargo-semver-checks` compares the crate with its
|
||||
latest published version. Include any required version bump in the protocol
|
||||
change PR. This check becomes available after the first manual publication.
|
||||
|
||||
1. Update the crate version and the `vllm-proto` workspace dependency together,
|
||||
and update `rust/Cargo.lock`.
|
||||
1. Update the crate version in `rust/proto/Cargo.toml` and update `rust/Cargo.lock`.
|
||||
2. Run `cargo publish --manifest-path rust/proto/Cargo.toml --locked --dry-run`
|
||||
and the frontend gRPC tests. Record the tested vLLM releases or revisions in
|
||||
the release notes; matching crate versions alone do not establish runtime compatibility.
|
||||
|
||||
@@ -50,6 +50,8 @@ message ServerInfo {
|
||||
uint64 max_batched_tokens = 9;
|
||||
uint32 max_loras = 10;
|
||||
RlCapabilities rl_capabilities = 11;
|
||||
// Full-attention block size in tokens after initialization; absent if unavailable.
|
||||
optional uint64 effective_attention_block_size = 14;
|
||||
}
|
||||
|
||||
message RlCapabilities {
|
||||
|
||||
@@ -510,6 +510,15 @@ impl EngineCoreClient {
|
||||
self.engines.iter().map(|engine| engine.ready_response.num_gpu_blocks).sum()
|
||||
}
|
||||
|
||||
/// Return the effective attention block size if all engines report the same value.
|
||||
pub fn effective_attention_block_size(&self) -> Option<u64> {
|
||||
let size = self.ready_response().effective_attention_block_size?;
|
||||
self.engines
|
||||
.iter()
|
||||
.all(|engine| engine.ready_response.effective_attention_block_size == Some(size))
|
||||
.then_some(size)
|
||||
}
|
||||
|
||||
/// Return the minimum engine-reported `max_model_len` across all engines.
|
||||
///
|
||||
/// This is the auto-fitted value after KV cache profiling and may differ
|
||||
|
||||
@@ -72,6 +72,7 @@ pub fn default_ready_response() -> EngineCoreReadyResponse {
|
||||
weight_transfer_backend: None,
|
||||
enable_sleep_mode: false,
|
||||
supports_draft_weight_updates: false,
|
||||
effective_attention_block_size: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -104,6 +104,9 @@ pub struct EngineCoreReadyResponse {
|
||||
/// Whether the engine has a speculative draft model that can be updated.
|
||||
#[serde(default)]
|
||||
pub supports_draft_weight_updates: bool,
|
||||
/// Full-attention block size in tokens after initialization, or unavailable.
|
||||
#[serde(default)]
|
||||
pub effective_attention_block_size: Option<u64>,
|
||||
}
|
||||
|
||||
/// Frontend-owned ZMQ addresses that are sent to the engine during startup
|
||||
|
||||
@@ -2846,6 +2846,11 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
|
||||
let ready_response: EngineCoreReadyResponse =
|
||||
rmp_serde::from_slice(&hex::decode(ready_response_hex).unwrap()).unwrap();
|
||||
let mut legacy_ready = serde_json::to_value(&ready_response).unwrap();
|
||||
legacy_ready.as_object_mut().unwrap().remove("effective_attention_block_size");
|
||||
let legacy_ready: EngineCoreReadyResponse =
|
||||
rmp_serde::from_slice(&rmp_serde::to_vec_named(&legacy_ready).unwrap()).unwrap();
|
||||
assert!(legacy_ready.effective_attention_block_size.is_none());
|
||||
assert!(ready_response.supports_lora);
|
||||
assert_eq!(ready_response.max_loras, 8);
|
||||
assert_eq!(
|
||||
@@ -2854,6 +2859,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
);
|
||||
assert!(ready_response.enable_sleep_mode);
|
||||
assert!(ready_response.supports_draft_weight_updates);
|
||||
assert_eq!(ready_response.effective_attention_block_size, Some(64));
|
||||
let kv_events_config = ready_response.kv_events_config.expect("KV events config should decode");
|
||||
assert!(kv_events_config.enable_kv_cache_events);
|
||||
assert_eq!(kv_events_config.publisher, "zmq");
|
||||
|
||||
@@ -423,6 +423,7 @@ class EngineCoreReadyResponse:
|
||||
weight_transfer_backend: str | None = None
|
||||
enable_sleep_mode: bool = False
|
||||
supports_draft_weight_updates: bool = False
|
||||
effective_attention_block_size: int | None = None
|
||||
|
||||
|
||||
ready_response = EngineCoreReadyResponse(
|
||||
@@ -446,6 +447,7 @@ ready_response = EngineCoreReadyResponse(
|
||||
weight_transfer_backend="nccl",
|
||||
enable_sleep_mode=True,
|
||||
supports_draft_weight_updates=True,
|
||||
effective_attention_block_size=64,
|
||||
kv_events_config=KVEventsConfig(
|
||||
enable_kv_cache_events=True,
|
||||
publisher="zmq",
|
||||
|
||||
@@ -195,6 +195,7 @@ impl pb::control_server::Control for ControlServiceImpl {
|
||||
max_batched_tokens: ready.max_num_batched_tokens,
|
||||
max_loras: ready.max_loras,
|
||||
rl_capabilities: Some(self.rl_capabilities()),
|
||||
effective_attention_block_size: self.client().effective_attention_block_size(),
|
||||
}))
|
||||
}
|
||||
|
||||
|
||||
@@ -1911,8 +1911,16 @@ async fn control_abort_resolves_external_id_and_empty_is_noop() {
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn control_reports_server_and_model_info() {
|
||||
let mut ready = default_ready_response();
|
||||
ready.effective_attention_block_size = Some(64);
|
||||
let (generate_service, control_service, engine_health, _engine_task) =
|
||||
setup_grpc_service(b"engine-grpc-info", default_stream_output_specs()).await;
|
||||
setup_grpc_service_with_engine_script(
|
||||
b"engine-grpc-info".to_vec(),
|
||||
ready,
|
||||
Arc::new(FakeTextBackend),
|
||||
|_, _| boxed_test_future(async {}),
|
||||
)
|
||||
.await;
|
||||
let (channel, server_task) = start_grpc_test_server(
|
||||
generate_service,
|
||||
control_service,
|
||||
@@ -1928,6 +1936,7 @@ async fn control_reports_server_and_model_info() {
|
||||
.expect("get server info")
|
||||
.into_inner();
|
||||
assert_eq!(server.engine_version, "test-vllm-version");
|
||||
assert_eq!(server.effective_attention_block_size, Some(64));
|
||||
assert_eq!(server.api_version, "vllm");
|
||||
assert_eq!(server.instance_id, "test-instance");
|
||||
assert_eq!(server.max_model_len, DEFAULT_MOCK_MAX_MODEL_LEN as u32);
|
||||
@@ -2195,6 +2204,7 @@ async fn control_aggregates_multi_engine_capacity() {
|
||||
ready_0.weight_transfer_backend = Some("nccl".to_string());
|
||||
ready_0.enable_sleep_mode = true;
|
||||
ready_0.supports_draft_weight_updates = true;
|
||||
ready_0.effective_attention_block_size = Some(64);
|
||||
|
||||
let mut ready_1 = default_ready_response();
|
||||
ready_1.max_model_len = 4_096;
|
||||
@@ -2248,6 +2258,7 @@ async fn control_aggregates_multi_engine_capacity() {
|
||||
.into_inner();
|
||||
assert_eq!(server.max_model_len, 4_096);
|
||||
assert_eq!(server.total_kv_blocks, 30);
|
||||
assert!(server.effective_attention_block_size.is_none());
|
||||
let rl = server.rl_capabilities.expect("RL capabilities");
|
||||
assert!(!rl.weight_transfer_enabled);
|
||||
assert!(rl.weight_transfer_backend.is_empty());
|
||||
|
||||
@@ -215,6 +215,9 @@ def test_cache_config_hash_ignores_kv_cache_sizing_knobs():
|
||||
base_hash = CacheConfig().compute_hash()
|
||||
assert CacheConfig(kv_cache_memory_bytes=1 << 30).compute_hash() == base_hash
|
||||
assert CacheConfig(gpu_memory_utilization=0.5).compute_hash() == base_hash
|
||||
config = CacheConfig()
|
||||
config.effective_attention_block_size = 64
|
||||
assert config.compute_hash() == base_hash
|
||||
|
||||
|
||||
def test_scheduler_config_hash_includes_max_num_seqs():
|
||||
|
||||
@@ -694,14 +694,43 @@ def test_check_shm_free_space_raises_when_insufficient(tmp_path):
|
||||
|
||||
|
||||
def test_check_shm_free_space_passes_when_sufficient(tmp_path):
|
||||
with mock.patch.object(
|
||||
shm_broadcast.shutil, "disk_usage", return_value=_fake_disk_usage(512 << 20)
|
||||
with (
|
||||
mock.patch.object(
|
||||
shm_broadcast.shutil,
|
||||
"disk_usage",
|
||||
return_value=_fake_disk_usage(512 << 20),
|
||||
),
|
||||
mock.patch.object(shm_broadcast, "check_cgroup_memory_available"),
|
||||
):
|
||||
check_shm_free_space(240 << 20, shm_path=str(tmp_path))
|
||||
|
||||
|
||||
def test_check_shm_free_space_skipped_when_path_missing(tmp_path):
|
||||
check_shm_free_space(1 << 60, shm_path=str(tmp_path / "does-not-exist"))
|
||||
with mock.patch.object(shm_broadcast, "check_cgroup_memory_available"):
|
||||
check_shm_free_space(1 << 60, shm_path=str(tmp_path / "does-not-exist"))
|
||||
|
||||
|
||||
def test_check_shm_free_space_checks_cgroup(tmp_path):
|
||||
with (
|
||||
mock.patch.object(
|
||||
shm_broadcast.shutil,
|
||||
"disk_usage",
|
||||
return_value=_fake_disk_usage(512 << 20),
|
||||
),
|
||||
mock.patch.object(
|
||||
shm_broadcast, "check_cgroup_memory_available"
|
||||
) as check_cgroup,
|
||||
):
|
||||
check_shm_free_space(
|
||||
240 << 20,
|
||||
shm_path=str(tmp_path),
|
||||
allocation_name="SHM mmap",
|
||||
)
|
||||
|
||||
check_cgroup.assert_called_once_with(
|
||||
240 << 20,
|
||||
"SHM mmap",
|
||||
)
|
||||
|
||||
|
||||
def test_shm_ring_buffer_creation_checks_free_space():
|
||||
|
||||
@@ -948,7 +948,7 @@ def test_structured_output_batched_with_non_structured_outputs_requests(
|
||||
prompts = [structured_outputs_prompt, non_structured_outputs_prompt]
|
||||
sampling_params = [
|
||||
SamplingParams(
|
||||
temperature=1.0,
|
||||
temperature=0,
|
||||
max_tokens=400,
|
||||
structured_outputs=StructuredOutputsParams(json=sample_json_schema),
|
||||
),
|
||||
|
||||
@@ -45,7 +45,8 @@ async def test_include_reasoning_true_non_streaming(client: openai.AsyncOpenAI):
|
||||
response = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=MESSAGES,
|
||||
max_tokens=200,
|
||||
max_tokens=512,
|
||||
temperature=0,
|
||||
extra_body={"include_reasoning": True},
|
||||
)
|
||||
|
||||
@@ -63,7 +64,8 @@ async def test_include_reasoning_false_non_streaming(client: openai.AsyncOpenAI)
|
||||
response = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=MESSAGES,
|
||||
max_tokens=200,
|
||||
max_tokens=512,
|
||||
temperature=0,
|
||||
extra_body={"include_reasoning": False},
|
||||
)
|
||||
|
||||
@@ -99,7 +101,8 @@ async def test_include_reasoning_true_streaming(client: openai.AsyncOpenAI):
|
||||
stream = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=MESSAGES,
|
||||
max_tokens=200,
|
||||
max_tokens=512,
|
||||
temperature=0,
|
||||
stream=True,
|
||||
extra_body={"include_reasoning": True},
|
||||
)
|
||||
@@ -130,7 +133,8 @@ async def test_include_reasoning_false_streaming(client: openai.AsyncOpenAI):
|
||||
stream = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=MESSAGES,
|
||||
max_tokens=200,
|
||||
max_tokens=512,
|
||||
temperature=0,
|
||||
stream=True,
|
||||
extra_body={"include_reasoning": False},
|
||||
)
|
||||
|
||||
@@ -353,7 +353,7 @@ async def test_parallel_no_streaming(client: openai.AsyncOpenAI, model_name: str
|
||||
|
||||
prompt = "What is an LLM?"
|
||||
n = 3
|
||||
max_tokens = 50 # we want some to finish earlier than others
|
||||
max_tokens = 50
|
||||
|
||||
# High temperature to maximize chance of unique completions.
|
||||
completion = await client.completions.create(
|
||||
@@ -371,16 +371,12 @@ async def test_parallel_no_streaming(client: openai.AsyncOpenAI, model_name: str
|
||||
num_completions = len(completion.choices)
|
||||
assert num_completions == n, f"Num completions {num_completions} but expected {n}."
|
||||
completion_repeats: dict[str, int] = {}
|
||||
output_token_lengths = set()
|
||||
for idx, choice in enumerate(completion.choices):
|
||||
# Assert correct completion index & some finish reason.
|
||||
assert choice.index == idx, f"Index {choice.index} but expected {idx}."
|
||||
assert choice.finish_reason is not None, "None finish_reason is invalid."
|
||||
text = choice.text
|
||||
completion_repeats[text] = completion_repeats.get(text, 0) + 1
|
||||
output_token_lengths.add(len(choice.logprobs.tokens))
|
||||
# Assert subrequests finished at different times
|
||||
assert len(output_token_lengths) > 1
|
||||
# Assert `n` unique completions
|
||||
num_unique = len(completion_repeats)
|
||||
if num_unique != n:
|
||||
@@ -403,7 +399,7 @@ async def test_parallel_streaming(client: openai.AsyncOpenAI, model_name: str):
|
||||
|
||||
prompt = "What is an LLM?"
|
||||
n = 3
|
||||
max_tokens = 50 # we want some to finish earlier than others
|
||||
max_tokens = 50
|
||||
|
||||
stream = await client.completions.create(
|
||||
model=model_name,
|
||||
@@ -427,19 +423,15 @@ async def test_parallel_streaming(client: openai.AsyncOpenAI, model_name: str):
|
||||
f"Expected {n} completions with valid indices and finish_reason."
|
||||
)
|
||||
completion_repeats: dict[str, int] = {}
|
||||
chunk_lengths = set()
|
||||
for chunk in chunks:
|
||||
chunk_len = len(chunk)
|
||||
# Assert correct number of completion tokens
|
||||
chunk_lengths.add(chunk_len)
|
||||
assert chunk_len <= max_tokens, (
|
||||
f"max_tokens={max_tokens} but chunk len is {chunk_len}."
|
||||
)
|
||||
text = "".join(chunk)
|
||||
completion_repeats[text] = completion_repeats.get(text, 0) + 1
|
||||
print(text)
|
||||
# Assert subrequests finished at different times
|
||||
assert len(chunk_lengths) > 1
|
||||
# Assert `n` unique completions
|
||||
num_unique = len(completion_repeats)
|
||||
if num_unique != n:
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from contextlib import AsyncExitStack
|
||||
from contextlib import AsyncExitStack, asynccontextmanager
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
@@ -41,6 +41,7 @@ from vllm.entrypoints.mcp.tool_server import ToolServer
|
||||
from vllm.entrypoints.openai.responses.context import (
|
||||
ConversationContext,
|
||||
HarmonyContext,
|
||||
ParsableContext,
|
||||
SimpleContext,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import (
|
||||
@@ -551,6 +552,87 @@ def test_responses_render_result_rejects_non_single_prompt(engine_inputs):
|
||||
class TestInitializeToolSessions:
|
||||
"""Test class for _initialize_tool_sessions method"""
|
||||
|
||||
@pytest.fixture(params=[ParsableContext, HarmonyContext])
|
||||
def tool_context(self, request):
|
||||
context = request.param.__new__(request.param)
|
||||
context.available_tools = ["browser", "python"]
|
||||
context._tool_sessions = {}
|
||||
context.called_tools = set()
|
||||
return context
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("available_tools", "called_tools"),
|
||||
[
|
||||
([], []),
|
||||
(["browser"], ["browser"]),
|
||||
(["browser", "python"], []),
|
||||
(["browser", "python"], ["browser"]),
|
||||
(["browser", "python"], ["python"]),
|
||||
(["browser", "python"], ["browser", "python"]),
|
||||
],
|
||||
ids=["empty", "single", "unused", "first", "last", "both"],
|
||||
)
|
||||
async def test_mcp_sessions_cleaned_once_before_close(
|
||||
self, tool_context, available_tools, called_tools
|
||||
):
|
||||
events = []
|
||||
sessions = {}
|
||||
|
||||
@asynccontextmanager
|
||||
async def new_session(name, *_args):
|
||||
session = MagicMock()
|
||||
session.call_tool = AsyncMock(
|
||||
side_effect=lambda *_: events.append(("cleanup", name))
|
||||
)
|
||||
sessions[name] = session
|
||||
try:
|
||||
yield session
|
||||
finally:
|
||||
events.append(("close", name))
|
||||
|
||||
tool_server = MagicMock(spec=ToolServer)
|
||||
tool_server.new_session.side_effect = new_session
|
||||
tool_context.available_tools = available_tools
|
||||
async with AsyncExitStack() as stack:
|
||||
await tool_context.init_tool_sessions(tool_server, stack, "req", {})
|
||||
# Reinitializing existing sessions must not duplicate cleanup either.
|
||||
await tool_context.init_tool_sessions(tool_server, stack, "req", {})
|
||||
tool_context.called_tools.update(called_tools)
|
||||
|
||||
for name, session in sessions.items():
|
||||
if name in called_tools:
|
||||
session.call_tool.assert_awaited_once_with("cleanup_session", {})
|
||||
assert events.index(("cleanup", name)) < events.index(("close", name))
|
||||
else:
|
||||
session.call_tool.assert_not_awaited()
|
||||
assert [name for event, name in events if event == "close"] == list(
|
||||
reversed(available_tools)
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mcp_partial_initialization_closes_opened_session(self, tool_context):
|
||||
closed = []
|
||||
session = MagicMock(call_tool=AsyncMock())
|
||||
|
||||
@asynccontextmanager
|
||||
async def new_session(name, *_args):
|
||||
if name == "python":
|
||||
raise RuntimeError("session initialization failed")
|
||||
try:
|
||||
yield session
|
||||
finally:
|
||||
closed.append(name)
|
||||
|
||||
tool_server = MagicMock(spec=ToolServer)
|
||||
tool_server.new_session.side_effect = new_session
|
||||
with pytest.raises(RuntimeError, match="session initialization failed"):
|
||||
async with AsyncExitStack() as stack:
|
||||
await tool_context.init_tool_sessions(tool_server, stack, "req", {})
|
||||
|
||||
assert closed == ["browser"]
|
||||
session.call_tool.assert_not_awaited()
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def serving_responses_instance(self):
|
||||
"""Create a real OpenAIServingResponses instance for testing"""
|
||||
|
||||
@@ -17,6 +17,88 @@ from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
|
||||
def assert_rope_close_rocm(
|
||||
actual: torch.Tensor,
|
||||
ref: torch.Tensor,
|
||||
rope_input: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
is_neox: bool,
|
||||
*,
|
||||
max_fail_fraction: float | None = None,
|
||||
atol: float | None = None,
|
||||
rtol: float | None = None,
|
||||
) -> None:
|
||||
"""assert_close for RoPE fused-vs-reference on ROCm, with a per-element
|
||||
product-ULP allowance for catastrophic-cancellation elements.
|
||||
|
||||
The fused and reference kernels may contract the RoPE multiply-add into an
|
||||
FMA with either product pre-rounded to the working dtype. The two valid
|
||||
orderings differ by at most 2 ULPs of the product at that element, which
|
||||
cancellation (x*cos ~= -y*sin) can amplify past the default tolerance.
|
||||
|
||||
Tier 1: |actual - ref| <= atol + rtol * |ref| (default tolerance).
|
||||
Tier 2: elements failing tier 1 must each satisfy
|
||||
|actual_e - ref_e| <= 2 * eps * |product_e|, where product_e is the larger
|
||||
of the two products combined at that element, and their fraction must be
|
||||
<= max_fail_fraction (1e-4 for fp16; 1e-2 for bf16, whose coarser eps
|
||||
puts 1-ULP divergences past the default tolerance far more often).
|
||||
"""
|
||||
if atol is None:
|
||||
atol = get_default_atol(ref)
|
||||
if rtol is None:
|
||||
rtol = get_default_rtol(ref)
|
||||
if max_fail_fraction is None:
|
||||
max_fail_fraction = 1e-2 if ref.dtype == torch.bfloat16 else 1e-4
|
||||
eps = torch.finfo(ref.dtype).eps
|
||||
|
||||
# per-element product magnitude, in the (seq, heads, rot_dim) layout
|
||||
seq_len, rot_dim = rope_input.shape[0], rope_input.shape[-1]
|
||||
half = rot_dim // 2
|
||||
inp = rope_input.reshape(seq_len, -1, rot_dim).to(torch.float32)
|
||||
cs = cos_sin_cache[positions].to(torch.float32) # (seq_len, 2*half)
|
||||
cos = cs[:, :half].unsqueeze(1) # broadcast over heads
|
||||
sin = cs[:, half:].unsqueeze(1)
|
||||
x, y = inp[..., :half], inp[..., half:]
|
||||
if is_neox:
|
||||
# out = [x*cos - y*sin, y*cos + x*sin]
|
||||
prod = torch.cat(
|
||||
[
|
||||
torch.maximum((x * cos).abs(), (y * sin).abs()),
|
||||
torch.maximum((y * cos).abs(), (x * sin).abs()),
|
||||
],
|
||||
dim=-1,
|
||||
)
|
||||
else:
|
||||
# out[2i] = x[2i]*cos[i] - x[2i+1]*sin[i], interleaved
|
||||
xe, xo = inp[..., 0::2], inp[..., 1::2]
|
||||
prod = torch.stack(
|
||||
[
|
||||
torch.maximum((xe * cos).abs(), (xo * sin).abs()),
|
||||
torch.maximum((xo * cos).abs(), (xe * sin).abs()),
|
||||
],
|
||||
dim=-1,
|
||||
).flatten(-2)
|
||||
prod = prod.reshape(rope_input.shape)
|
||||
|
||||
diff = (actual.to(torch.float32) - ref.to(torch.float32)).abs()
|
||||
tol = atol + rtol * ref.to(torch.float32).abs()
|
||||
fail = diff > tol
|
||||
n_fail = int(fail.sum())
|
||||
if n_fail == 0:
|
||||
return
|
||||
assert n_fail <= max_fail_fraction * diff.numel(), (
|
||||
f"{n_fail} elements beyond default tolerance "
|
||||
f"({n_fail / diff.numel():.2%} > {max_fail_fraction:.2%}); "
|
||||
f"max diff {diff.max().item()}"
|
||||
)
|
||||
excess = diff[fail] - 2 * eps * prod[fail]
|
||||
assert (excess <= 0).all(), (
|
||||
f"{int((excess > 0).sum())} cancellation elements exceed 2 product-ULPs; "
|
||||
f"worst excess {excess.max().item()}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def default_vllm_config(monkeypatch):
|
||||
"""Enable the AITER triton rope on ROCm for fp16-consistent numerics.
|
||||
@@ -99,6 +181,8 @@ def test_concat_and_cache_mla_rope_fused(
|
||||
|
||||
k_pe = torch.flatten(key[..., :qk_rope_head_dim], start_dim=1).to(device=device)
|
||||
kv_c = torch.flatten(key[..., qk_rope_head_dim:], start_dim=1).to(device=device)
|
||||
k_pe_orig = k_pe.clone()
|
||||
query_orig = query.clone()
|
||||
|
||||
if current_platform.is_rocm():
|
||||
# We use forward_hip for the same numerics as the fused custom kernel on ROCm
|
||||
@@ -181,12 +265,9 @@ def test_concat_and_cache_mla_rope_fused(
|
||||
kv_cache_scale,
|
||||
)
|
||||
|
||||
# On ROCm the AITER Triton rope diverges by ~1 ULP from the fused kernel,
|
||||
# which the tight CUDA tolerances don't cover for the low-precision paths:
|
||||
# - bf16: up to ~1 bf16 ULP (0.0156-0.03125 at values ~2-3), so relax the
|
||||
# kv-cache atol (bounded ~0.02) and the query atol (~0.04).
|
||||
# - neox-style fp8: one e4m3 ULP (~12.5%), so relax rtol.
|
||||
# Other paths use the CUDA defaults.
|
||||
# On ROCm the AITER Triton rope diverges by up to 2 product-ULPs from the
|
||||
# fused kernel, which the default tolerance misses at the rare
|
||||
# catastrophic-cancellation elements. fp8 keeps its calibrated tolerance.
|
||||
rocm_neox = current_platform.is_rocm() and is_neox_style
|
||||
rocm_bf16 = current_platform.is_rocm() and dtype == torch.bfloat16
|
||||
if kv_cache_dtype == "fp8":
|
||||
@@ -207,16 +288,40 @@ def test_concat_and_cache_mla_rope_fused(
|
||||
atol=0.004 if rocm_bf16 else 0.001,
|
||||
rtol=0.15 if rocm_neox or rocm_bf16 else 0.1,
|
||||
)
|
||||
elif rocm_bf16:
|
||||
torch.testing.assert_close(kv_cache, ref_kv_cache, atol=0.02, rtol=1e-3)
|
||||
elif rocm_neox:
|
||||
torch.testing.assert_close(kv_cache, ref_kv_cache, atol=1e-3, rtol=1e-3)
|
||||
elif current_platform.is_rocm() and dtype in (torch.half, torch.bfloat16):
|
||||
# gather the rope region of the cache back to token order
|
||||
block_idx = slot_mapping // block_size
|
||||
block_offset = slot_mapping % block_size
|
||||
fused_k_rope = kv_cache[block_idx, block_offset, kv_lora_rank:]
|
||||
ref_k_rope_gathered = ref_kv_cache[block_idx, block_offset, kv_lora_rank:]
|
||||
assert_rope_close_rocm(
|
||||
fused_k_rope,
|
||||
ref_k_rope_gathered,
|
||||
k_pe_orig,
|
||||
rope.cos_sin_cache,
|
||||
positions,
|
||||
is_neox_style,
|
||||
)
|
||||
# the nope region is a byte-exact copy
|
||||
torch.testing.assert_close(
|
||||
kv_cache[..., :kv_lora_rank], ref_kv_cache[..., :kv_lora_rank]
|
||||
)
|
||||
else:
|
||||
torch.testing.assert_close(kv_cache, ref_kv_cache)
|
||||
|
||||
torch.testing.assert_close(
|
||||
query,
|
||||
ref_q_pe,
|
||||
atol=0.04 if rocm_bf16 else get_default_atol(query),
|
||||
rtol=get_default_rtol(query),
|
||||
)
|
||||
if current_platform.is_rocm() and dtype in (torch.half, torch.bfloat16):
|
||||
assert_rope_close_rocm(
|
||||
query,
|
||||
ref_q_pe,
|
||||
query_orig,
|
||||
rope.cos_sin_cache,
|
||||
positions,
|
||||
is_neox_style,
|
||||
)
|
||||
else:
|
||||
torch.testing.assert_close(
|
||||
query,
|
||||
ref_q_pe,
|
||||
atol=get_default_atol(query),
|
||||
rtol=get_default_rtol(query),
|
||||
)
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the AITER norm support predicates.
|
||||
|
||||
The AITER norm wrappers flatten >2-D activations with ``Tensor.reshape``, which
|
||||
silently copies when the flattened shape is not expressible with the input's
|
||||
strides. The support predicates must reject those inputs so dispatch falls
|
||||
through to a provider that handles arbitrary strides.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
from vllm.kernels.aiter_ops import flatten_to_2d_is_free
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not is_aiter_found_and_supported(),
|
||||
reason="Only test on ROCm with AITER installed and supported",
|
||||
)
|
||||
|
||||
|
||||
def _qkv_slice_by_head(num_tokens, num_q_heads, num_kv_heads, head_dim):
|
||||
"""q viewed per-head from a fused QKV projection, as Qwen3-style QK-norm does."""
|
||||
q_size, kv_size = num_q_heads * head_dim, num_kv_heads * head_dim
|
||||
qkv = torch.empty(num_tokens, q_size + 2 * kv_size)
|
||||
q = qkv.split([q_size, kv_size, kv_size], dim=-1)[0]
|
||||
return q.view(*q.shape[:-1], num_q_heads, head_dim)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"x",
|
||||
[
|
||||
torch.empty(8, 16),
|
||||
torch.empty(16),
|
||||
torch.empty(2, 4, 16),
|
||||
torch.empty(2, 1, 16),
|
||||
# A contiguous tensor stays flattenable after a leading-dim slice.
|
||||
torch.empty(8, 4, 16)[2:6],
|
||||
],
|
||||
)
|
||||
def test_flattenable(x):
|
||||
assert flatten_to_2d_is_free(x)
|
||||
assert x.reshape(-1, x.shape[-1]).data_ptr() == x.data_ptr()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"x",
|
||||
[
|
||||
_qkv_slice_by_head(8, 32, 8, 128),
|
||||
# Non-unit last-dim stride.
|
||||
torch.empty(4, 8, 32).transpose(-1, -2),
|
||||
# Leading dims cannot be merged: a slice along the middle dim.
|
||||
torch.empty(4, 8, 32)[:, :4],
|
||||
],
|
||||
)
|
||||
def test_not_flattenable(x):
|
||||
assert not flatten_to_2d_is_free(x)
|
||||
# reshape has to copy, which is exactly what the predicate guards against.
|
||||
assert x.reshape(-1, x.shape[-1]).data_ptr() != x.data_ptr()
|
||||
@@ -9,6 +9,15 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.cpu import (
|
||||
CPUFp8PerTensorScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.ScaledMMLinearKernel import (
|
||||
FP8ScaledMMLinearLayerConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8StaticTensorSym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_cpu():
|
||||
@@ -17,6 +26,9 @@ if not current_platform.is_cpu():
|
||||
if not ops._supports_cpu_fp8_w8a16:
|
||||
pytest.skip("fp8_scaled_mm_cpu op not available", allow_module_level=True)
|
||||
|
||||
if not torch.cpu._is_amx_tile_supported():
|
||||
pytest.skip("requires AMX tile support", allow_module_level=True)
|
||||
|
||||
BLOCK_SIZE = [128, 128]
|
||||
|
||||
|
||||
@@ -160,3 +172,104 @@ def test_cpu_fp8_scaled_mm(M: int, N: int, K: int, use_bias: bool):
|
||||
|
||||
assert kernel_out.dtype == out_dtype
|
||||
torch.testing.assert_close(kernel_out, ref_out, rtol=0.02, atol=0.01)
|
||||
|
||||
|
||||
def quantize_weight_per_tensor_fp8(
|
||||
weight: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Quantize weight [N, K] to FP8 with a single per-tensor scale."""
|
||||
fp8_max = torch.finfo(torch.float8_e4m3fn).max
|
||||
scale = weight.abs().amax() / fp8_max
|
||||
scale = torch.where(scale == 0, torch.ones_like(scale), scale)
|
||||
q = (weight / scale).clamp(-fp8_max, fp8_max).to(torch.float8_e4m3fn)
|
||||
return q, scale
|
||||
|
||||
|
||||
def ref_fp8_per_tensor_scaled_mm(
|
||||
x: torch.Tensor,
|
||||
fp8_weight: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
out_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
w_dq = fp8_weight.float() * scale
|
||||
out = torch.mm(x.float(), w_dq.t())
|
||||
if bias is not None:
|
||||
out = out + bias.float()
|
||||
return out.to(out_dtype)
|
||||
|
||||
|
||||
NK_SIZES_PER_TENSOR = [
|
||||
(32, 64),
|
||||
(5120, 5120),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("M", [1, 64])
|
||||
@pytest.mark.parametrize("N,K", NK_SIZES_PER_TENSOR)
|
||||
@pytest.mark.parametrize("use_bias", [False, True])
|
||||
def test_cpu_fp8_per_tensor_scaled_mm_kernel(
|
||||
M: int, N: int, K: int, use_bias: bool, default_vllm_config
|
||||
):
|
||||
"""CPUFp8PerTensorScaledMMLinearKernel correctness against float reference.
|
||||
|
||||
Exercises the full kernel class (not just the raw op), including the
|
||||
weight-orientation fixup: Fp8LinearMethod stores `layer.weight` as
|
||||
[K, N] (torch._scaled_mm convention) before calling
|
||||
process_weights_after_loading, so the kernel must transpose back to
|
||||
[N, K] before VNNI-packing.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
out_dtype = torch.bfloat16
|
||||
|
||||
x = torch.randn(M, K, dtype=out_dtype) / (K**0.5)
|
||||
w_f32 = torch.randn(N, K, dtype=torch.float32) / (K**0.5)
|
||||
fp8_weight, scale = quantize_weight_per_tensor_fp8(w_f32)
|
||||
bias = torch.randn(N, dtype=torch.float32) * 0.1 if use_bias else None
|
||||
|
||||
ref_out = ref_fp8_per_tensor_scaled_mm(x, fp8_weight, scale, bias, out_dtype)
|
||||
|
||||
config = FP8ScaledMMLinearLayerConfig(
|
||||
weight_quant_key=kFp8StaticTensorSym,
|
||||
activation_quant_key=kFp8StaticTensorSym,
|
||||
weight_shape=(N, K),
|
||||
input_dtype=out_dtype,
|
||||
out_dtype=out_dtype,
|
||||
)
|
||||
kernel = CPUFp8PerTensorScaledMMLinearKernel(
|
||||
config,
|
||||
layer_param_names=["weight", "weight_scale", "input_scale", "input_scale_ub"],
|
||||
)
|
||||
|
||||
layer = torch.nn.Module()
|
||||
# Fp8LinearMethod stores the weight transposed to [K, N] before calling
|
||||
# process_weights_after_loading.
|
||||
layer.register_parameter(
|
||||
"weight", torch.nn.Parameter(fp8_weight.t().contiguous(), requires_grad=False)
|
||||
)
|
||||
layer.register_parameter(
|
||||
"weight_scale", torch.nn.Parameter(scale.clone(), requires_grad=False)
|
||||
)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
kernel_out = kernel.apply_weights(layer, x, bias)
|
||||
|
||||
assert kernel_out.dtype == out_dtype
|
||||
torch.testing.assert_close(kernel_out, ref_out, rtol=0.02, atol=0.01)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n", [16, 48])
|
||||
def test_cpu_fp8_per_tensor_kernel_rejects_non_multiple_of_32_n(n: int):
|
||||
"""The AMX tinygemm kernel tiles N in chunks of 32 and cannot handle a
|
||||
remainder tile smaller than that, so can_implement must reject any N
|
||||
that isn't a multiple of 32 rather than let it crash the process.
|
||||
"""
|
||||
config = FP8ScaledMMLinearLayerConfig(
|
||||
weight_quant_key=kFp8StaticTensorSym,
|
||||
activation_quant_key=kFp8StaticTensorSym,
|
||||
weight_shape=(n, 64),
|
||||
input_dtype=torch.bfloat16,
|
||||
out_dtype=torch.bfloat16,
|
||||
)
|
||||
supported, _ = CPUFp8PerTensorScaledMMLinearKernel.can_implement(config)
|
||||
assert not supported
|
||||
|
||||
@@ -331,6 +331,19 @@ def test_v41_fused_save_compress_and_insert(
|
||||
torch.testing.assert_close(cache_backing, expected_cache, rtol=0, atol=0)
|
||||
|
||||
|
||||
def _rotate_rope_tail(
|
||||
latent_row: torch.Tensor, cos_sin_row: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""GPT-J RoPE of one latent row's last 64 dims, in fp32."""
|
||||
row = latent_row.float()
|
||||
c, s = cos_sin_row.chunk(2)
|
||||
rotated = row.clone()
|
||||
even, odd = row[448::2], row[449::2]
|
||||
rotated[448::2] = even * c - odd * s
|
||||
rotated[449::2] = odd * c + even * s
|
||||
return rotated
|
||||
|
||||
|
||||
def _mxfp8_record_reference(
|
||||
latent_row: torch.Tensor, cos_sin_row: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
@@ -339,12 +352,7 @@ def _mxfp8_record_reference(
|
||||
Mirrors FlashMLA's ``KVCacheLayout.V41_FP8Sparse`` quantizer: rotate the
|
||||
RoPE tail first, then scale every 32-dim tile, RoPE tiles included.
|
||||
"""
|
||||
row = latent_row.float()
|
||||
c, s = cos_sin_row.chunk(2)
|
||||
rotated = row.clone()
|
||||
even, odd = row[448::2], row[449::2]
|
||||
rotated[448::2] = even * c - odd * s
|
||||
rotated[449::2] = odd * c + even * s
|
||||
rotated = _rotate_rope_tail(latent_row, cos_sin_row)
|
||||
quantized, scales = _ue8m0_reference(rotated, 32, 448.0)
|
||||
return quantized.view(torch.uint8), (scales.log2() + 127).to(torch.uint8)
|
||||
|
||||
@@ -461,6 +469,125 @@ def test_v41_mxfp8_cache_round_trip():
|
||||
assert (error <= tolerance.repeat_interleave(32, dim=-1)).all()
|
||||
|
||||
|
||||
# e2m1 magnitudes indexed by the low 3 bits of a code; bit 3 is the sign.
|
||||
_E2M1_MAGNITUDES = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]
|
||||
|
||||
|
||||
def _decode_nvfp4_row(packed: torch.Tensor, scales: torch.Tensor) -> torch.Tensor:
|
||||
"""Unpack one 256-byte e2m1 row and apply its 32 e4m3 tile scales."""
|
||||
mags = torch.tensor(_E2M1_MAGNITUDES, device=packed.device)
|
||||
codes = torch.empty(512, dtype=torch.uint8, device=packed.device)
|
||||
codes[0::2] = packed & 0xF # even element in the low nibble
|
||||
codes[1::2] = packed >> 4
|
||||
vals = mags[(codes & 7).long()] * torch.where(codes >= 8, -1.0, 1.0)
|
||||
return (
|
||||
vals.view(32, 16) * scales.view(torch.float8_e4m3fn).float().view(32, 1)
|
||||
).flatten()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
@pytest.mark.parametrize("compress_ratio", [1, 2])
|
||||
def test_v41_rope_insert_nvfp4_record(compress_ratio: int):
|
||||
"""The 288-byte V4.1 NVFP4 record: 256 B of e2m1 pairs then 32 e4m3 scales.
|
||||
|
||||
Scales are byte-exact against the reference (``amax / 6`` clamped to the
|
||||
e4m3 range); values are checked as round-to-nearest on the e2m1 grid, whose
|
||||
coarsest step is 2 between 4 and 6, so half a step is ``1.0 * scale``.
|
||||
"""
|
||||
from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import (
|
||||
rope_quant_insert,
|
||||
)
|
||||
|
||||
torch.manual_seed(13)
|
||||
device = "cuda"
|
||||
cache_block = 64
|
||||
cache_stride = math.ceil(cache_block * 288 / 512) * 512
|
||||
num_tokens = 12
|
||||
|
||||
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
latent = torch.randn(num_tokens, 512, dtype=torch.bfloat16, device=device)
|
||||
angles = torch.randn(64, 32, device=device)
|
||||
cos_sin = torch.cat((angles.cos(), angles.sin()), dim=-1)
|
||||
|
||||
cache_backing = torch.full((2, cache_stride), 165, dtype=torch.uint8, device=device)
|
||||
cache = cache_backing.as_strided((2, cache_block, 288), (cache_stride, 288, 1))
|
||||
slots = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
slots[3] = -1 # covers the negative-slot skip
|
||||
|
||||
rope_quant_insert(latent, positions, cos_sin, cache, slots, compress_ratio)
|
||||
|
||||
untouched = torch.full_like(cache_backing, 165)
|
||||
for t in range(num_tokens):
|
||||
slot = slots[t].item()
|
||||
pos = positions[t].item()
|
||||
if slot < 0 or (pos + 1) % compress_ratio:
|
||||
continue
|
||||
page, row = divmod(slot, cache_block)
|
||||
cs = cos_sin[pos // compress_ratio * compress_ratio]
|
||||
rotated = _rotate_rope_tail(latent[t], cs)
|
||||
amax = rotated.view(32, 16).abs().amax(-1)
|
||||
scale = (amax / 6.0).clamp(2.0**-9, 448.0).to(torch.float8_e4m3fn)
|
||||
|
||||
got_scales = cache_backing[page, cache_block * 256 + row * 32 :][:32]
|
||||
torch.testing.assert_close(got_scales, scale.view(torch.uint8), rtol=0, atol=0)
|
||||
decoded = _decode_nvfp4_row(
|
||||
cache_backing[page, row * 256 : (row + 1) * 256], got_scales
|
||||
)
|
||||
step = scale.float().repeat_interleave(16)
|
||||
assert ((decoded - rotated).abs() <= step).all()
|
||||
untouched[page, row * 256 : (row + 1) * 256] = cache_backing[
|
||||
page, row * 256 : (row + 1) * 256
|
||||
]
|
||||
untouched[page, cache_block * 256 + row * 32 :][:32] = got_scales
|
||||
# Rows the kernel must not have touched, page padding included.
|
||||
torch.testing.assert_close(cache_backing, untouched, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
def test_v41_nvfp4_gather_matches_insert():
|
||||
"""The NVFP4 gather dequantizes exactly what the insert kernel wrote."""
|
||||
from vllm.models.deepseek_v41.common.ops import dequantize_and_gather_k_cache
|
||||
from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import (
|
||||
rope_quant_insert,
|
||||
)
|
||||
|
||||
torch.manual_seed(17)
|
||||
device = "cuda"
|
||||
block_size = 64
|
||||
num_tokens = 70
|
||||
num_blocks = 4
|
||||
page_bytes = math.ceil(block_size * 288 / 512) * 512
|
||||
|
||||
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
latent = torch.randn(num_tokens, 512, dtype=torch.bfloat16, device=device)
|
||||
angles = torch.randn(128, 32, device=device)
|
||||
cos_sin = torch.cat((angles.cos(), angles.sin()), dim=-1)
|
||||
backing = torch.zeros(num_blocks, page_bytes, dtype=torch.uint8, device=device)
|
||||
cache = backing.as_strided((num_blocks, block_size, 288), (page_bytes, 288, 1))
|
||||
slots = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
rope_quant_insert(latent, positions, cos_sin, cache, slots, 1)
|
||||
|
||||
out = torch.zeros(1, num_tokens, 512, dtype=torch.bfloat16, device=device)
|
||||
dequantize_and_gather_k_cache(
|
||||
out,
|
||||
cache,
|
||||
seq_lens=torch.tensor([num_tokens], dtype=torch.int32, device=device),
|
||||
gather_lens=None,
|
||||
block_table=torch.arange(num_blocks, dtype=torch.int32, device=device).view(
|
||||
1, -1
|
||||
),
|
||||
block_size=block_size,
|
||||
offset=0,
|
||||
)
|
||||
for t in (0, 1, block_size, num_tokens - 1):
|
||||
page, row = divmod(t, block_size)
|
||||
expected = _decode_nvfp4_row(
|
||||
backing[page, row * 256 : (row + 1) * 256],
|
||||
backing[page, block_size * 256 + row * 32 :][:32],
|
||||
)
|
||||
torch.testing.assert_close(out[0, t].float(), expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
@pytest.mark.parametrize("compress_ratio", [1, 2])
|
||||
@pytest.mark.parametrize("store_fp8", [False, True])
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Layout plumbing for DeepSeek V4.1 mega attention.
|
||||
|
||||
The kernel's Q and O layouts are produced by permuting ``wq_b`` rows and
|
||||
``wo_a`` columns once at load, so what has to hold is that the permuted GEMMs
|
||||
agree with a reference that permutes the activation instead.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.models.deepseek_v41.common.ops.fused_layout import (
|
||||
WV_GROUP_SIZE,
|
||||
o_fused_chunk_permutation,
|
||||
o_fused_permutation,
|
||||
permute_q_to_fused,
|
||||
permute_wo_a_,
|
||||
permute_wq_b_,
|
||||
q_fused_permutation,
|
||||
)
|
||||
|
||||
HEAD_DIM = 512
|
||||
|
||||
|
||||
def test_permuted_wq_b_gemm_matches_permuted_activation():
|
||||
"""Permuting wq_b's rows makes the Q GEMM emit the fused layout directly."""
|
||||
torch.manual_seed(0)
|
||||
num_heads, q_lora_rank, num_tokens = 16, 64, 5
|
||||
weight = torch.randn(num_heads * HEAD_DIM, q_lora_rank)
|
||||
# One scale row per weight row, as an MXFP8 wq_b shard carries.
|
||||
scale = torch.randint(
|
||||
0, 256, (num_heads * HEAD_DIM, q_lora_rank // 32), dtype=torch.uint8
|
||||
)
|
||||
qr = torch.randn(num_tokens, q_lora_rank)
|
||||
orig_weight, orig_scale = weight.clone(), scale.clone()
|
||||
|
||||
standard = (qr @ weight.T).view(num_tokens, num_heads, HEAD_DIM)
|
||||
expected = permute_q_to_fused(standard)
|
||||
|
||||
permute_wq_b_(weight, scale, num_heads)
|
||||
fused = (qr @ weight.T).view(num_tokens, num_heads, HEAD_DIM)
|
||||
torch.testing.assert_close(fused, expected)
|
||||
# The scale has to follow its row, or dequant reads another row's scale.
|
||||
perm = q_fused_permutation(num_heads, HEAD_DIM)
|
||||
torch.testing.assert_close(weight, orig_weight[perm])
|
||||
torch.testing.assert_close(scale, orig_scale[perm])
|
||||
|
||||
|
||||
def test_permuted_wo_a_consumes_fused_output():
|
||||
"""Permuting wo_a's input columns lets it read the kernel's O layout."""
|
||||
torch.manual_seed(1)
|
||||
out_features, num_tokens = 32, 4
|
||||
in_features = WV_GROUP_SIZE * HEAD_DIM
|
||||
weight = torch.randn(out_features, in_features)
|
||||
scale = torch.randint(0, 256, (out_features, in_features // 32), dtype=torch.uint8)
|
||||
standard_o = torch.randn(num_tokens, in_features)
|
||||
orig_weight, orig_scale = weight.clone(), scale.clone()
|
||||
|
||||
expected = standard_o @ weight.T
|
||||
# The kernel emits the same values in the fused chunk order.
|
||||
perm = o_fused_permutation(WV_GROUP_SIZE, HEAD_DIM)
|
||||
fused_o = standard_o[:, perm]
|
||||
|
||||
permute_wo_a_(weight, scale, WV_GROUP_SIZE)
|
||||
torch.testing.assert_close(weight, orig_weight[:, perm])
|
||||
torch.testing.assert_close(
|
||||
scale, orig_scale[:, o_fused_chunk_permutation(WV_GROUP_SIZE, HEAD_DIM)]
|
||||
)
|
||||
# Unlike wq_b, this permutation sits inside the 4096-term reduction, so the
|
||||
# summation order changes and the result differs in the last few ulps.
|
||||
torch.testing.assert_close(fused_o @ weight.T, expected, rtol=1e-3, atol=1e-3)
|
||||
@@ -522,6 +522,189 @@ def test_kv_path_mxfp8_record_matches_reference(num_tokens: int, block_size: int
|
||||
)
|
||||
|
||||
|
||||
# ── Test 2d: mega-attention Q layout (is_q_interleaved) ──────────────────────
|
||||
|
||||
Q_CHUNK = 16
|
||||
NUM_Q_CHUNKS = HEAD_DIM // Q_CHUNK # 32
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 17, 2048])
|
||||
@pytest.mark.parametrize("n_heads,q_head_padded", [(16, 64), (64, 128), (64, 0)])
|
||||
def test_q_interleaved_pads_without_rope(
|
||||
num_tokens: int, n_heads: int, q_head_padded: int
|
||||
):
|
||||
"""``is_q_interleaved`` with no norm or RoPE only zero-pads Q.
|
||||
|
||||
The mega-attention kernel norms and rotates Q itself, so every live
|
||||
element must survive bit-exact while the padding heads -- which interleave
|
||||
with the live ones inside each of the 32 head-dim chunks -- come back
|
||||
zero. ``q_head_padded=0`` drops the Q pass entirely and returns nothing,
|
||||
for a shard already at the kernel's head count.
|
||||
"""
|
||||
torch.manual_seed(5)
|
||||
device = "cuda"
|
||||
dtype = torch.bfloat16
|
||||
block_size = 64
|
||||
|
||||
q = torch.randn(num_tokens, n_heads, HEAD_DIM, dtype=dtype, device=device)
|
||||
kv = torch.randn(num_tokens, HEAD_DIM, dtype=dtype, device=device)
|
||||
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
cos_sin_cache = make_cos_sin_cache(4096, ROPE_DIM, torch.float32, device)
|
||||
num_blocks = (num_tokens + block_size - 1) // block_size + 1
|
||||
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
k_cache = torch.zeros(
|
||||
num_blocks, block_size * V41_HEAD_BYTES, dtype=torch.uint8, device=device
|
||||
)
|
||||
|
||||
q_out = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
q,
|
||||
kv,
|
||||
k_cache,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
q_head_padded,
|
||||
1e-6,
|
||||
block_size,
|
||||
False, # apply_q_norm
|
||||
True, # kv_mxfp8
|
||||
False, # apply_q_rope
|
||||
True, # is_q_interleaved
|
||||
)
|
||||
|
||||
# The KV half is unaffected by the Q mode: it must still be inserted.
|
||||
assert k_cache.any()
|
||||
|
||||
if q_head_padded == 0:
|
||||
assert q_out.numel() == 0
|
||||
return
|
||||
assert q_out.shape == (num_tokens, q_head_padded, HEAD_DIM)
|
||||
got = q_out.view(num_tokens, NUM_Q_CHUNKS, q_head_padded, Q_CHUNK)
|
||||
want = q.view(num_tokens, NUM_Q_CHUNKS, n_heads, Q_CHUNK)
|
||||
torch.testing.assert_close(got[:, :, :n_heads], want, rtol=0, atol=0)
|
||||
assert (got[:, :, n_heads:] == 0).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n_heads", [8, 16, 32])
|
||||
def test_q_interleaved_survives_graph_replay(n_heads: int):
|
||||
"""A captured fused insert refills its padded Q buffer on every replay.
|
||||
|
||||
The op allocates ``q_out`` itself, so the tensor a capture hands back has
|
||||
to keep receiving the new padding when the graph is replayed over changed
|
||||
input -- that is what lets the layer prepare Q inside the captured region
|
||||
rather than in ``forward_mqa``.
|
||||
"""
|
||||
torch.manual_seed(6)
|
||||
device = "cuda"
|
||||
dtype = torch.bfloat16
|
||||
block_size, num_tokens, q_head_padded = 64, 7, 64
|
||||
|
||||
q = torch.randn(num_tokens, n_heads, HEAD_DIM, dtype=dtype, device=device)
|
||||
kv = torch.randn(num_tokens, HEAD_DIM, dtype=dtype, device=device)
|
||||
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
cos_sin_cache = make_cos_sin_cache(4096, ROPE_DIM, torch.float32, device)
|
||||
num_blocks = (num_tokens + block_size - 1) // block_size + 1
|
||||
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
k_cache = torch.zeros(
|
||||
num_blocks, block_size * V41_HEAD_BYTES, dtype=torch.uint8, device=device
|
||||
)
|
||||
|
||||
def prepare():
|
||||
return torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
q,
|
||||
kv,
|
||||
k_cache,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
q_head_padded,
|
||||
1e-6,
|
||||
block_size,
|
||||
False, # apply_q_norm
|
||||
True, # kv_mxfp8
|
||||
False, # apply_q_rope
|
||||
True, # is_q_interleaved
|
||||
)
|
||||
|
||||
# Capture takes its warmup on a side stream.
|
||||
side = torch.cuda.Stream()
|
||||
side.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(side):
|
||||
for _ in range(3):
|
||||
prepare()
|
||||
torch.cuda.current_stream().wait_stream(side)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
q_out = prepare()
|
||||
|
||||
for factor in (0.25, -0.75):
|
||||
q.copy_(torch.randn_like(q) * factor)
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
got = q_out.view(num_tokens, NUM_Q_CHUNKS, q_head_padded, Q_CHUNK)
|
||||
want = q.view(num_tokens, NUM_Q_CHUNKS, n_heads, Q_CHUNK)
|
||||
torch.testing.assert_close(got[:, :, :n_heads], want, rtol=0, atol=0)
|
||||
assert (got[:, :, n_heads:] == 0).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n_heads", [16, 64])
|
||||
@pytest.mark.parametrize("apply_q_norm", [False, True])
|
||||
@pytest.mark.parametrize("apply_q_rope", [False, True])
|
||||
def test_q_interleaved_is_orthogonal_to_norm_and_rope(
|
||||
n_heads: int, apply_q_norm: bool, apply_q_rope: bool
|
||||
):
|
||||
"""The Q layout composes with whatever norm and RoPE it is given.
|
||||
|
||||
A lane owns the same 512-dim slice of its head in either layout -- only
|
||||
the address differs -- so the head-major result permuted into the
|
||||
interleaved layout must equal running the interleaved path on a permuted
|
||||
input, bit for bit, for every norm/RoPE combination.
|
||||
"""
|
||||
from vllm.models.deepseek_v41.common.ops.fused_layout import permute_q_to_fused
|
||||
|
||||
torch.manual_seed(7)
|
||||
device = "cuda"
|
||||
dtype = torch.bfloat16
|
||||
block_size, num_tokens, q_head_padded = 64, 5, 64
|
||||
|
||||
q = torch.randn(num_tokens, n_heads, HEAD_DIM, dtype=dtype, device=device)
|
||||
kv = torch.randn(num_tokens, HEAD_DIM, dtype=dtype, device=device)
|
||||
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
cos_sin_cache = make_cos_sin_cache(4096, ROPE_DIM, torch.float32, device)
|
||||
num_blocks = (num_tokens + block_size - 1) // block_size + 1
|
||||
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
|
||||
def run(q_in, is_q_interleaved):
|
||||
cache = torch.zeros(
|
||||
num_blocks, block_size * V41_HEAD_BYTES, dtype=torch.uint8, device=device
|
||||
)
|
||||
out = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
q_in,
|
||||
kv,
|
||||
cache,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
q_head_padded,
|
||||
1e-6,
|
||||
block_size,
|
||||
apply_q_norm,
|
||||
True, # kv_mxfp8
|
||||
apply_q_rope,
|
||||
is_q_interleaved,
|
||||
)
|
||||
return out, cache
|
||||
|
||||
head_major, cache_major = run(q, False)
|
||||
interleaved, cache_inter = run(permute_q_to_fused(q), True)
|
||||
|
||||
torch.testing.assert_close(
|
||||
interleaved, permute_q_to_fused(head_major), rtol=0, atol=0
|
||||
)
|
||||
# The Q layout must not leak into the KV half.
|
||||
torch.testing.assert_close(cache_inter, cache_major, rtol=0, atol=0)
|
||||
|
||||
|
||||
# ── Test 2b: DP padding (slot_mapping shorter than q/kv) ─────────────────────
|
||||
|
||||
|
||||
|
||||
@@ -367,17 +367,21 @@ def test_deepseek_v4_mega_moe_padding_preserves_weights(monkeypatch, intermediat
|
||||
[
|
||||
pytest.param(128, 512, 128, True, False, id="aligned-block128"),
|
||||
pytest.param(128, 512, 128, True, True, id="aligned-block128-mxfp8"),
|
||||
pytest.param(5120, 2304, 32, False, False, id="deepseek-v41-flash"),
|
||||
pytest.param(5120, 2304, 32, True, False, id="deepseek-v41-flash"),
|
||||
pytest.param(128, 2304, 128, True, False, id="padded-block128"),
|
||||
pytest.param(5120, 2304, 32, True, True, id="deepseek-v41-flash-mxfp8"),
|
||||
pytest.param(128, 2304, 128, False, False, id="padded-packed-scale-fallback"),
|
||||
],
|
||||
)
|
||||
def test_deepseek_v4_mega_moe_finalizes_native_shared_expert_weights(
|
||||
monkeypatch, hidden_size, intermediate_size, block_size, fused, mxfp8
|
||||
):
|
||||
"""V4.1-Flash's routed padding must preserve the serial shared-expert fallback."""
|
||||
"""Shared fusion preserves checkpoint channels and zeros padded channels."""
|
||||
|
||||
class FakeDeepGemm:
|
||||
transformed_dims: list[tuple[int, int]] = []
|
||||
scale_inputs: list[tuple[int, ...]] = []
|
||||
scale_values: list[torch.Tensor] = []
|
||||
|
||||
@staticmethod
|
||||
def get_symm_buffer_for_mega_moe(*args, num_shared_experts=0, **kwargs):
|
||||
@@ -402,6 +406,7 @@ def test_deepseek_v4_mega_moe_finalizes_native_shared_expert_weights(
|
||||
@classmethod
|
||||
def transform_sf_into_required_layout(cls, sf, mn, k, *args, **kwargs):
|
||||
cls.scale_inputs.append(tuple(sf.shape))
|
||||
cls.scale_values.append(sf)
|
||||
return torch.empty((sf.shape[0], mn, k // 128), dtype=torch.int32)
|
||||
|
||||
@classmethod
|
||||
@@ -429,7 +434,10 @@ def test_deepseek_v4_mega_moe_finalizes_native_shared_expert_weights(
|
||||
|
||||
def fp8_parameter(*shape):
|
||||
return torch.nn.Parameter(
|
||||
torch.empty(*shape, dtype=torch.float8_e4m3fn), requires_grad=False
|
||||
torch.empty(*shape, dtype=torch.uint8)
|
||||
.random_(1, 119)
|
||||
.view(torch.float8_e4m3fn),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
def scale_parameter(*shape, dtype=torch.int32):
|
||||
@@ -460,21 +468,34 @@ def test_deepseek_v4_mega_moe_finalizes_native_shared_expert_weights(
|
||||
linear.weight_block_size = (1, 32)
|
||||
linear.weight_scale = torch.nn.Parameter(
|
||||
linear.weight_scale_inv.view(torch.uint8)
|
||||
.repeat_interleave(128, dim=0)
|
||||
.repeat_interleave(4, dim=1),
|
||||
.repeat_interleave(block_size, dim=0)
|
||||
.repeat_interleave(block_size // 32, dim=1),
|
||||
requires_grad=False,
|
||||
)
|
||||
del linear.weight_scale_inv
|
||||
originals = []
|
||||
for linear in (shared_experts.gate_up_proj, shared_experts.down_proj):
|
||||
scale = linear.weight_scale if mxfp8 else linear.weight_scale_inv
|
||||
if not fused:
|
||||
# Already packed scales cannot be padded as checkpoint UE8M0 bytes.
|
||||
linear.weight_scale_inv = scale_parameter(
|
||||
linear.weight.shape[0], linear.weight.shape[1] // 128
|
||||
)
|
||||
continue
|
||||
scale.data.view(torch.uint8).random_(124, 130)
|
||||
block_m, block_k = linear.weight_block_size
|
||||
scale_1x32 = (
|
||||
experts._ue8m0_uint8_to_float(scale.view(torch.uint8))
|
||||
.repeat_interleave(block_m, dim=0)
|
||||
.repeat_interleave(block_k // 32, dim=1)
|
||||
)
|
||||
originals.append((linear.weight.view(torch.uint8).clone(), scale_1x32))
|
||||
monkeypatch.setattr("vllm.utils.deep_gemm._import_deep_gemm", lambda: FakeDeepGemm)
|
||||
|
||||
original_gate_up_ptr = shared_experts.gate_up_proj.weight.data_ptr()
|
||||
original_down_ptr = shared_experts.down_proj.weight.data_ptr()
|
||||
experts.finalize_weights(shared_experts)
|
||||
|
||||
assert FakeDeepGemm.scale_inputs[-2:] == [
|
||||
(1, 2 * intermediate_size, hidden_size // 32),
|
||||
(1, hidden_size, intermediate_size // 32),
|
||||
]
|
||||
assert experts.has_fused_shared_experts is fused
|
||||
if not fused:
|
||||
assert experts.intermediate_size == 2560
|
||||
@@ -490,15 +511,56 @@ def test_deepseek_v4_mega_moe_finalizes_native_shared_expert_weights(
|
||||
return
|
||||
|
||||
assert FakeDeepGemm.transformed_dims == [(3, 3), (2, 2)]
|
||||
assert shared_experts.gate_up_proj.weight.data_ptr() != original_gate_up_ptr
|
||||
assert (
|
||||
experts._transformed_shared_l1_weights[0].data_ptr()
|
||||
== shared_experts.gate_up_proj.weight.data_ptr()
|
||||
)
|
||||
assert (
|
||||
experts._transformed_shared_l2_weights[0].data_ptr()
|
||||
== shared_experts.down_proj.weight.data_ptr()
|
||||
)
|
||||
padded_size = experts.intermediate_size
|
||||
assert FakeDeepGemm.scale_inputs[-2:] == [
|
||||
(1, 2 * padded_size, hidden_size // 32),
|
||||
(1, hidden_size, padded_size // 32),
|
||||
]
|
||||
for actual, original, fill in (
|
||||
(
|
||||
experts._transformed_shared_l1_weights[0].view(torch.uint8),
|
||||
originals[0][0],
|
||||
0,
|
||||
),
|
||||
(FakeDeepGemm.scale_values[-2][0], originals[0][1], 1),
|
||||
):
|
||||
actual = actual.unflatten(0, (2, padded_size))
|
||||
assert torch.equal(
|
||||
actual[:, :intermediate_size], original.unflatten(0, (2, intermediate_size))
|
||||
)
|
||||
assert torch.all(actual[:, intermediate_size:] == fill)
|
||||
for actual, original, fill in (
|
||||
(
|
||||
experts._transformed_shared_l2_weights[0].view(torch.uint8),
|
||||
originals[1][0],
|
||||
0,
|
||||
),
|
||||
(FakeDeepGemm.scale_values[-1][0], originals[1][1], 1),
|
||||
):
|
||||
assert torch.equal(actual[:, : original.shape[1]], original)
|
||||
assert torch.all(actual[:, original.shape[1] :] == fill)
|
||||
if padded_size != intermediate_size:
|
||||
# Generic linear post-load hooks still receive checkpoint-shaped weights.
|
||||
for linear, (weight, _) in zip(
|
||||
(shared_experts.gate_up_proj, shared_experts.down_proj), originals
|
||||
):
|
||||
assert torch.equal(linear.weight.view(torch.uint8), weight)
|
||||
assert shared_experts.gate_up_proj.weight.data_ptr() == original_gate_up_ptr
|
||||
assert shared_experts.down_proj.weight.data_ptr() == original_down_ptr
|
||||
else:
|
||||
assert shared_experts.gate_up_proj.weight.data_ptr() != original_gate_up_ptr
|
||||
assert (
|
||||
experts._transformed_shared_l1_weights[0].data_ptr()
|
||||
== shared_experts.gate_up_proj.weight.data_ptr()
|
||||
)
|
||||
assert (
|
||||
experts._transformed_shared_l2_weights[0].data_ptr()
|
||||
== shared_experts.down_proj.weight.data_ptr()
|
||||
)
|
||||
|
||||
transformed_l1 = experts._transformed_shared_l1_weights
|
||||
experts.finalize_weights(shared_experts)
|
||||
assert experts._transformed_shared_l1_weights is transformed_l1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fused", [False, True])
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import contextlib
|
||||
import hashlib
|
||||
import struct
|
||||
import uuid
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
@@ -8,7 +10,7 @@ from pathlib import Path
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image, ImageDraw
|
||||
from PIL import Image, ImageDraw, ImageOps
|
||||
|
||||
from vllm.config.multimodal import MMHasherAlgorithm
|
||||
from vllm.multimodal.hasher import MultiModalHasher
|
||||
@@ -199,6 +201,33 @@ def test_hash_image_exif_id():
|
||||
)
|
||||
|
||||
|
||||
def test_hash_image_malformed_exif():
|
||||
# Test that images with malformed EXIF headers (e.g. invalid TIFF header)
|
||||
# do not raise an unhandled exception during hashing and fall back to image data.
|
||||
buf = BytesIO()
|
||||
Image.new("RGB", (64, 48)).save(buf, "JPEG")
|
||||
jpg = buf.getvalue()
|
||||
rest = jpg[2:]
|
||||
rest = rest[2 + struct.unpack(">H", rest[2:4])[0] :]
|
||||
payload = b"Exif\x00\x00XXXX\x00\x00\x00\x08" + bytes(32)
|
||||
data = b"\xff\xd8\xff\xe1" + struct.pack(">H", len(payload) + 2) + payload + rest
|
||||
|
||||
image = Image.open(BytesIO(data))
|
||||
with contextlib.suppress(Exception):
|
||||
image = ImageOps.exif_transpose(image)
|
||||
image.load()
|
||||
|
||||
hasher = MultiModalHasher
|
||||
# Should hash without raising SyntaxError or any other exception
|
||||
hash_val = hasher.hash_kwargs("blake3", image=image)
|
||||
assert isinstance(hash_val, str) and len(hash_val) > 0
|
||||
|
||||
# Also verify MediaWithBytes wrapping the image with malformed EXIF
|
||||
media_item = MediaWithBytes(image, data)
|
||||
hash_media = hasher.hash_kwargs("blake3", image=media_item)
|
||||
assert isinstance(hash_media, str) and len(hash_media) > 0
|
||||
|
||||
|
||||
def _rgba_png_bytes() -> bytes:
|
||||
image = Image.new("RGBA", (8, 8), (255, 0, 0, 128))
|
||||
buf = BytesIO()
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Reasoning-boundary tests for the GLM-4.7 parser.
|
||||
|
||||
GLM can transition straight from reasoning into a tool call without ever
|
||||
emitting ``</think>``, so ``<tool_call>`` is an implicit reasoning
|
||||
terminator. These cover that path plus the multi-turn prompts where an
|
||||
earlier turn's markers must not be read as the current turn's state.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.parser.engine.conftest import make_mock_tokenizer
|
||||
from vllm.parser.glm47_moe import (
|
||||
THINK_END,
|
||||
THINK_START,
|
||||
TOOL_CALL_END,
|
||||
TOOL_CALL_START,
|
||||
Glm47MoeParser,
|
||||
)
|
||||
|
||||
THINK_S, THINK_E = 1000, 1001
|
||||
TOOL_S, TOOL_E = 1002, 1003
|
||||
ASSISTANT, OBSERVATION, USER = 1004, 1005, 1006
|
||||
TEXT = 42 # stand-in for an ordinary reasoning/content token
|
||||
|
||||
VOCAB = {
|
||||
THINK_START: THINK_S,
|
||||
THINK_END: THINK_E,
|
||||
TOOL_CALL_START: TOOL_S,
|
||||
TOOL_CALL_END: TOOL_E,
|
||||
"<|assistant|>": ASSISTANT,
|
||||
"<|observation|>": OBSERVATION,
|
||||
"<|user|>": USER,
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def parser():
|
||||
return Glm47MoeParser(make_mock_tokenizer(VOCAB))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def no_thinking_parser():
|
||||
return Glm47MoeParser(
|
||||
make_mock_tokenizer(VOCAB),
|
||||
chat_template_kwargs={"thinking": False},
|
||||
)
|
||||
|
||||
|
||||
class TestIsReasoningEnd:
|
||||
def test_open_reasoning(self, parser):
|
||||
assert not parser.is_reasoning_end([THINK_S, TEXT])
|
||||
|
||||
def test_think_end(self, parser):
|
||||
assert parser.is_reasoning_end([THINK_S, TEXT, THINK_E, TEXT])
|
||||
|
||||
def test_tool_call_without_think_end(self, parser):
|
||||
"""Reasoning -> tool call with no ``</think>`` still ends reasoning."""
|
||||
assert parser.is_reasoning_end([THINK_S, TEXT, TOOL_S, TEXT])
|
||||
|
||||
def test_previous_turn_tool_call_ignored(self, parser):
|
||||
"""A finished tool call from an earlier turn says nothing about the
|
||||
turn currently being generated."""
|
||||
prompt = [THINK_S, TEXT, TOOL_S, TEXT, TOOL_E, OBSERVATION, TEXT, ASSISTANT]
|
||||
assert not parser.is_reasoning_end(prompt)
|
||||
|
||||
def test_previous_turn_think_end_ignored(self, parser):
|
||||
prompt = [THINK_S, TEXT, THINK_E, TEXT, USER, TEXT, ASSISTANT]
|
||||
assert not parser.is_reasoning_end(prompt)
|
||||
|
||||
def test_empty_input(self, parser):
|
||||
assert not parser.is_reasoning_end([])
|
||||
|
||||
def test_thinking_disabled(self, no_thinking_parser):
|
||||
assert no_thinking_parser.is_reasoning_end([THINK_S, TEXT])
|
||||
|
||||
|
||||
class TestExtractContentIds:
|
||||
def test_think_end_wins_over_later_tool_call(self, parser):
|
||||
"""Content between ``</think>`` and a tool call must survive."""
|
||||
ids = [THINK_S, TEXT, THINK_E, 20, 21, TOOL_S, 30, TOOL_E]
|
||||
assert parser.extract_content_ids(ids) == [20, 21, TOOL_S, 30, TOOL_E]
|
||||
|
||||
def test_every_tool_call_after_think_end_kept(self, parser):
|
||||
ids = [THINK_E, 20, TOOL_S, 30, TOOL_E, 21, TOOL_S, 31, TOOL_E]
|
||||
assert parser.extract_content_ids(ids) == ids[1:]
|
||||
|
||||
def test_falls_back_to_tool_call(self, parser):
|
||||
"""Without ``</think>``, content starts at the opener itself so the
|
||||
tool parser still receives a well-formed call."""
|
||||
ids = [THINK_S, TEXT, TOOL_S, 30, TOOL_E]
|
||||
assert parser.extract_content_ids(ids) == [TOOL_S, 30, TOOL_E]
|
||||
|
||||
def test_falls_back_to_first_tool_call_of_turn(self, parser):
|
||||
ids = [THINK_S, TEXT, TOOL_S, 30, TOOL_E, TOOL_S, 31, TOOL_E]
|
||||
assert parser.extract_content_ids(ids) == ids[2:]
|
||||
|
||||
def test_previous_turn_tool_call_ignored(self, parser):
|
||||
ids = [TOOL_S, 30, TOOL_E, OBSERVATION, TEXT, ASSISTANT, THINK_S, TEXT]
|
||||
assert parser.extract_content_ids(ids) == ids
|
||||
|
||||
def test_no_markers_returns_input_ids(self, parser):
|
||||
assert parser.extract_content_ids([20, 21]) == [20, 21]
|
||||
|
||||
def test_thinking_disabled(self, no_thinking_parser):
|
||||
ids = [THINK_S, TEXT, TOOL_S]
|
||||
assert no_thinking_parser.extract_content_ids(ids) == ids
|
||||
@@ -0,0 +1,163 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""`prompt_embeds` payloads that are not tensor files must be client errors.
|
||||
|
||||
`prompt_embeds` is a base64 blob the caller writes, and every other way of
|
||||
getting it wrong -- wrong rank, wrong hidden_size, wrong dtype, not a tensor at
|
||||
all -- already answers 400. A payload that `torch.load` cannot read at all was
|
||||
the exception: torch has no single error type for that (the zip reader raises
|
||||
`RuntimeError`, the legacy pickle path raises an unpickling error, `KeyError` or
|
||||
`IndexError`, an empty payload raises `EOFError`), none of them is a
|
||||
`ValueError`, and the entrypoints' fallback therefore mapped them to 500.
|
||||
|
||||
The assertions go through `create_error_response`, which is what actually turns
|
||||
an exception into a status code, so they pin the HTTP answer rather than the
|
||||
exception class alone.
|
||||
"""
|
||||
|
||||
import io
|
||||
import zipfile
|
||||
from unittest.mock import patch
|
||||
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.entrypoints.serve.exception_handling.error_response import (
|
||||
create_error_response,
|
||||
)
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.renderers.embed_utils import (
|
||||
_MAX_EMBED_ERROR_REASON_CHARS,
|
||||
safe_load_prompt_embeds,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def model_config():
|
||||
from vllm.config import ModelConfig
|
||||
|
||||
return ModelConfig(
|
||||
model="facebook/opt-125m",
|
||||
tokenizer="facebook/opt-125m",
|
||||
tokenizer_mode="auto",
|
||||
trust_remote_code=False,
|
||||
dtype="float32",
|
||||
seed=0,
|
||||
enable_prompt_embeds=True,
|
||||
)
|
||||
|
||||
|
||||
def _encode(raw: bytes) -> bytes:
|
||||
return base64.b64encode(raw)
|
||||
|
||||
|
||||
def _zip_without_a_tensor() -> bytes:
|
||||
buffer = io.BytesIO()
|
||||
with zipfile.ZipFile(buffer, "w") as archive:
|
||||
archive.writestr("junk.txt", "hello")
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def _saved_tensor(tensor: torch.Tensor) -> bytes:
|
||||
buffer = io.BytesIO()
|
||||
torch.save(tensor, buffer)
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
UNPARSABLE_PAYLOADS = [
|
||||
pytest.param(b"", id="empty"),
|
||||
pytest.param(b"\x80", id="single-byte"),
|
||||
pytest.param(b"\x00\x01\x02\x03not-a-tensor", id="random-bytes"),
|
||||
pytest.param(b"hello world this is not a torch file", id="ascii-text"),
|
||||
pytest.param(b"PK\x03\x04" + b"\x00" * 20, id="truncated-zip"),
|
||||
pytest.param(_zip_without_a_tensor(), id="zip-without-a-tensor"),
|
||||
# pickle.dumps(42): a valid pickle stream that is not a torch checkpoint.
|
||||
# Spelled out rather than built, so this file does not import pickle.
|
||||
pytest.param(b"\x80\x04K*.", id="legacy-pickle-of-an-int"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("payload", UNPARSABLE_PAYLOADS)
|
||||
def test_unreadable_prompt_embeds_payload_is_a_client_error(model_config, payload):
|
||||
with pytest.raises(VLLMValidationError) as excinfo:
|
||||
safe_load_prompt_embeds(model_config, _encode(payload))
|
||||
|
||||
assert excinfo.value.parameter == "prompt_embeds"
|
||||
assert int(create_error_response(excinfo.value).error.code) == 400
|
||||
|
||||
|
||||
@pytest.mark.parametrize("payload", UNPARSABLE_PAYLOADS)
|
||||
def test_the_error_body_stays_bounded(model_config, payload):
|
||||
"""torch's reason is built from the caller's bytes, so it must be truncated.
|
||||
|
||||
An unpickler that reports the offending global, for instance, quotes a name
|
||||
the caller chose; without a bound a large one would be reflected whole.
|
||||
"""
|
||||
padding = b"A" * 100_000
|
||||
with pytest.raises(VLLMValidationError) as excinfo:
|
||||
safe_load_prompt_embeds(model_config, _encode(payload + padding))
|
||||
|
||||
assert len(str(excinfo.value)) < 500
|
||||
|
||||
|
||||
def test_a_long_reason_is_truncated_and_says_how_much_was_left_out(model_config):
|
||||
"""The cap is pinned against a reason of known length, not a real payload.
|
||||
|
||||
Which of torch's messages happen to exceed the cap is torch's business and
|
||||
changes between releases, so the reason is injected here: that keeps the
|
||||
assertion about *this* code's truncation rule rather than about whichever
|
||||
error text the installed torch produces.
|
||||
"""
|
||||
reason = "E" * (_MAX_EMBED_ERROR_REASON_CHARS + 37)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.renderers.embed_utils.torch.load", side_effect=RuntimeError(reason)
|
||||
),
|
||||
pytest.raises(VLLMValidationError) as excinfo,
|
||||
):
|
||||
safe_load_prompt_embeds(model_config, _encode(b"ignored"))
|
||||
|
||||
message = str(excinfo.value)
|
||||
assert "E" * _MAX_EMBED_ERROR_REASON_CHARS in message
|
||||
assert "E" * (_MAX_EMBED_ERROR_REASON_CHARS + 1) not in message
|
||||
assert "... (37 more characters truncated)" in message
|
||||
|
||||
|
||||
def test_a_short_reason_is_passed_through_whole(model_config):
|
||||
"""A reason that fits is not annotated -- the count only appears when it is
|
||||
telling the reader something."""
|
||||
reason = "invalid magic number; corrupt file?"
|
||||
assert len(reason) <= _MAX_EMBED_ERROR_REASON_CHARS
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.renderers.embed_utils.torch.load", side_effect=RuntimeError(reason)
|
||||
),
|
||||
pytest.raises(VLLMValidationError) as excinfo,
|
||||
):
|
||||
safe_load_prompt_embeds(model_config, _encode(b"ignored"))
|
||||
|
||||
message = str(excinfo.value)
|
||||
assert reason in message
|
||||
assert "truncated" not in message
|
||||
|
||||
|
||||
def test_a_well_formed_payload_still_loads(model_config):
|
||||
"""Positive control: the healthy path is untouched."""
|
||||
hidden_size = model_config.get_hidden_size()
|
||||
tensor = torch.zeros(3, hidden_size, dtype=torch.float32)
|
||||
|
||||
loaded = safe_load_prompt_embeds(model_config, _encode(_saved_tensor(tensor)))
|
||||
|
||||
assert loaded.shape == (3, hidden_size)
|
||||
assert loaded.dtype == model_config.dtype
|
||||
|
||||
|
||||
def test_a_payload_that_is_not_base64_is_still_a_client_error(model_config):
|
||||
"""Positive control: the base64 stage already answered 400 and still does."""
|
||||
with pytest.raises(ValueError) as excinfo:
|
||||
safe_load_prompt_embeds(model_config, b"!!!!not base64!!!!")
|
||||
|
||||
assert int(create_error_response(excinfo.value).error.code) == 400
|
||||
@@ -11,6 +11,7 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.multimodal.media import AudioEmbeddingMediaIO, ImageEmbeddingMediaIO
|
||||
from vllm.renderers.embed_utils import safe_load_prompt_embeds
|
||||
from vllm.utils.sparse_utils import check_sparse_tensor_invariants_threadsafe
|
||||
@@ -54,7 +55,7 @@ class TestNegativeControlWithoutRace:
|
||||
|
||||
def test_malicious_sparse_rejected_by_prompt_loader(self, model_config):
|
||||
encoded = _encode_tensor(_create_malicious_sparse_tensor())
|
||||
with pytest.raises((RuntimeError, ValueError)):
|
||||
with pytest.raises((RuntimeError, ValueError, VLLMValidationError)):
|
||||
safe_load_prompt_embeds(model_config, encoded)
|
||||
|
||||
def test_malicious_sparse_rejected_by_image_loader(self):
|
||||
@@ -147,7 +148,7 @@ class TestConcurrentRaceProtection:
|
||||
try:
|
||||
safe_load_prompt_embeds(model_config, malicious_encoded)
|
||||
bypassed.append(True)
|
||||
except (RuntimeError, ValueError):
|
||||
except (RuntimeError, ValueError, VLLMValidationError):
|
||||
rejected.append(True)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=num_workers) as pool:
|
||||
@@ -192,7 +193,7 @@ class TestGlobalFlagRestoration:
|
||||
initial = torch.sparse.check_sparse_tensor_invariants.is_enabled()
|
||||
malicious_encoded = _encode_tensor(_create_malicious_sparse_tensor())
|
||||
|
||||
with pytest.raises((RuntimeError, ValueError)):
|
||||
with pytest.raises((RuntimeError, ValueError, VLLMValidationError)):
|
||||
safe_load_prompt_embeds(model_config, malicious_encoded)
|
||||
|
||||
assert torch.sparse.check_sparse_tensor_invariants.is_enabled() == initial
|
||||
@@ -203,7 +204,7 @@ class TestGlobalFlagRestoration:
|
||||
malicious_encoded = _encode_tensor(_create_malicious_sparse_tensor())
|
||||
|
||||
def attempt_load(_):
|
||||
with contextlib.suppress(RuntimeError, ValueError):
|
||||
with contextlib.suppress(RuntimeError, ValueError, VLLMValidationError):
|
||||
safe_load_prompt_embeds(model_config, malicious_encoded)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=4) as pool:
|
||||
@@ -244,7 +245,7 @@ class TestCrossLoaderLockSharing:
|
||||
# serialization — one finishes before the other starts), AND invalid
|
||||
# tensors are still rejected.
|
||||
malicious = _encode_tensor(_create_malicious_sparse_tensor())
|
||||
with pytest.raises((RuntimeError, ValueError)):
|
||||
with pytest.raises((RuntimeError, ValueError, VLLMValidationError)):
|
||||
safe_load_prompt_embeds(model_config, malicious)
|
||||
with pytest.raises((RuntimeError, ValueError)):
|
||||
io_handler.load_base64("", malicious.decode("utf-8"))
|
||||
|
||||
@@ -117,8 +117,10 @@ class TestPromptEmbedsValidation:
|
||||
malicious_tensor = _create_malicious_sparse_tensor()
|
||||
encoded = _encode_tensor(malicious_tensor)
|
||||
|
||||
# Should raise RuntimeError due to invalid sparse tensor
|
||||
with pytest.raises((RuntimeError, ValueError)) as exc_info:
|
||||
# The invariant check raises RuntimeError from inside torch.load;
|
||||
# safe_load_prompt_embeds now reports that as a client error so the
|
||||
# caller gets a 400 rather than the 500 a bare RuntimeError produced.
|
||||
with pytest.raises(VLLMValidationError) as exc_info:
|
||||
safe_load_prompt_embeds(model_config, encoded)
|
||||
|
||||
# Error should indicate sparse tensor validation failure
|
||||
@@ -137,7 +139,7 @@ class TestPromptEmbedsValidation:
|
||||
)
|
||||
encoded = _encode_tensor(malicious_tensor)
|
||||
|
||||
with pytest.raises((RuntimeError, ValueError)):
|
||||
with pytest.raises(VLLMValidationError):
|
||||
safe_load_prompt_embeds(model_config, encoded)
|
||||
|
||||
def test_negative_indices_rejected(self, model_config):
|
||||
@@ -152,7 +154,7 @@ class TestPromptEmbedsValidation:
|
||||
)
|
||||
encoded = _encode_tensor(malicious_tensor)
|
||||
|
||||
with pytest.raises((RuntimeError, ValueError)):
|
||||
with pytest.raises(VLLMValidationError):
|
||||
safe_load_prompt_embeds(model_config, encoded)
|
||||
|
||||
def test_hidden_size_mismatch_rejected(self, model_config):
|
||||
@@ -379,8 +381,8 @@ class TestSparseTensorValidationIntegration:
|
||||
# Step 1-2: Attacker creates malicious payload
|
||||
attack_payload = _encode_tensor(_create_malicious_sparse_tensor())
|
||||
|
||||
# Step 3-4: Server processes and should reject
|
||||
with pytest.raises((RuntimeError, ValueError)):
|
||||
# Step 3-4: Server processes and should reject, as a client error
|
||||
with pytest.raises(VLLMValidationError):
|
||||
safe_load_prompt_embeds(model_config, attack_payload)
|
||||
|
||||
def test_attack_scenario_chat_api_image(self):
|
||||
|
||||
@@ -65,7 +65,7 @@ def test_cpu_target_selects_cpu_platform_from_non_cpu_wheel(
|
||||
rocm_plugin.assert_not_called()
|
||||
|
||||
|
||||
def test_platform_detection_logs_zentorch_import_failure(caplog):
|
||||
def test_broken_zentorch_falls_back_to_cpu_platform(caplog):
|
||||
original_import = builtins.__import__
|
||||
|
||||
def import_with_broken_zentorch(name, *args, **kwargs):
|
||||
@@ -88,6 +88,54 @@ def test_platform_detection_logs_zentorch_import_failure(caplog):
|
||||
):
|
||||
platform = resolve_current_platform_cls_qualname()
|
||||
|
||||
assert platform == "vllm.platforms.cpu.CpuPlatform"
|
||||
assert "zentorch failed to import" in caplog.text
|
||||
assert "OSError: incompatible shared library" in caplog.text
|
||||
|
||||
|
||||
def test_zentorch_failure_other_than_oserror_is_not_recovered(caplog):
|
||||
original_import = builtins.__import__
|
||||
|
||||
def import_with_broken_zentorch(name, *args, **kwargs):
|
||||
if name == "zentorch":
|
||||
raise RuntimeError("unknown zentorch failure")
|
||||
return original_import(name, *args, **kwargs)
|
||||
|
||||
with (
|
||||
patch("vllm.platforms.envs.VLLM_TARGET_DEVICE", "cuda"),
|
||||
patch.dict(
|
||||
"vllm.platforms.builtin_platform_plugins",
|
||||
{"cpu": cpu_platform_plugin},
|
||||
clear=True,
|
||||
),
|
||||
patch("vllm.platforms.load_plugins_by_group", return_value={}),
|
||||
patch("vllm.platforms.vllm_version_matches_substr", return_value=True),
|
||||
patch("vllm.platforms._is_amd_zen_cpu", return_value=True),
|
||||
patch.object(builtins, "__import__", side_effect=import_with_broken_zentorch),
|
||||
caplog.at_level(logging.DEBUG, logger="vllm.platforms"),
|
||||
):
|
||||
platform = resolve_current_platform_cls_qualname()
|
||||
|
||||
assert platform == "vllm.platforms.interface.UnspecifiedPlatform"
|
||||
assert "RuntimeError: unknown zentorch failure" in caplog.text
|
||||
|
||||
|
||||
def test_platform_plugin_failure_is_logged(caplog):
|
||||
def failing_plugin():
|
||||
raise RuntimeError("plugin exploded")
|
||||
|
||||
with (
|
||||
patch("vllm.platforms.envs.VLLM_TARGET_DEVICE", "cuda"),
|
||||
patch.dict(
|
||||
"vllm.platforms.builtin_platform_plugins",
|
||||
{"cpu": failing_plugin},
|
||||
clear=True,
|
||||
),
|
||||
patch("vllm.platforms.load_plugins_by_group", return_value={}),
|
||||
caplog.at_level(logging.DEBUG, logger="vllm.platforms"),
|
||||
):
|
||||
platform = resolve_current_platform_cls_qualname()
|
||||
|
||||
assert platform == "vllm.platforms.interface.UnspecifiedPlatform"
|
||||
assert "Platform plugin cpu failed during detection" in caplog.text
|
||||
assert "OSError: incompatible shared library" in caplog.text
|
||||
assert "RuntimeError: plugin exploded" in caplog.text
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Cgroup memory readers and SHM allocation preflight checks."""
|
||||
|
||||
import sys
|
||||
from io import StringIO
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.utils import cpu_resource_utils as cru
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
|
||||
_V2_LIMIT_PATH = "/sys/fs/cgroup/memory.max"
|
||||
_V2_USAGE_PATH = "/sys/fs/cgroup/memory.current"
|
||||
_V1_LIMIT_PATH = "/sys/fs/cgroup/memory/memory.limit_in_bytes"
|
||||
_V1_USAGE_PATH = "/sys/fs/cgroup/memory/memory.usage_in_bytes"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear_cgroup_cache():
|
||||
cru.get_cgroup_memory_limit.cache_clear()
|
||||
yield
|
||||
cru.get_cgroup_memory_limit.cache_clear()
|
||||
|
||||
|
||||
def _stub_files(monkeypatch, files: dict):
|
||||
"""Stub ``open()`` for a fixed set of paths; ``None`` -> OSError."""
|
||||
real_open = open
|
||||
|
||||
def fake_open(path, *args, **kwargs):
|
||||
if path in files:
|
||||
content = files[path]
|
||||
if content is None:
|
||||
raise OSError(f"no such file: {path}")
|
||||
return StringIO(content)
|
||||
return real_open(path, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr("builtins.open", fake_open)
|
||||
|
||||
|
||||
def test_cgroup_v2_limit_and_usage(monkeypatch):
|
||||
monkeypatch.setattr(sys, "platform", "linux")
|
||||
_stub_files(
|
||||
monkeypatch,
|
||||
{
|
||||
_V2_LIMIT_PATH: f"{20 * GiB_bytes}\n",
|
||||
_V2_USAGE_PATH: f"{5 * GiB_bytes}\n",
|
||||
},
|
||||
)
|
||||
|
||||
assert cru.get_cgroup_memory_limit() == 20 * GiB_bytes
|
||||
assert cru.get_cgroup_memory_usage() == 5 * GiB_bytes
|
||||
|
||||
|
||||
def test_cgroup_v2_unlimited_falls_back_to_v1(monkeypatch):
|
||||
monkeypatch.setattr(sys, "platform", "linux")
|
||||
_stub_files(
|
||||
monkeypatch,
|
||||
{
|
||||
_V2_LIMIT_PATH: "max\n",
|
||||
_V1_LIMIT_PATH: f"{8 * GiB_bytes}\n",
|
||||
_V1_USAGE_PATH: f"{1 * GiB_bytes}\n",
|
||||
},
|
||||
)
|
||||
|
||||
assert cru.get_cgroup_memory_limit() == 8 * GiB_bytes
|
||||
assert cru.get_cgroup_memory_usage() == 1 * GiB_bytes
|
||||
|
||||
|
||||
def test_cgroup_v1_limit_and_usage(monkeypatch):
|
||||
monkeypatch.setattr(sys, "platform", "linux")
|
||||
_stub_files(
|
||||
monkeypatch,
|
||||
{
|
||||
_V2_LIMIT_PATH: None,
|
||||
_V1_LIMIT_PATH: f"{8 * GiB_bytes}\n",
|
||||
_V1_USAGE_PATH: f"{2 * GiB_bytes}\n",
|
||||
},
|
||||
)
|
||||
|
||||
assert cru.get_cgroup_memory_limit() == 8 * GiB_bytes
|
||||
assert cru.get_cgroup_memory_usage() == 2 * GiB_bytes
|
||||
|
||||
|
||||
def test_cgroup_v1_unlimited_sentinel_is_ignored(monkeypatch):
|
||||
"""An unlimited cgroup v1 sentinel must not be treated as a real limit."""
|
||||
monkeypatch.setattr(sys, "platform", "linux")
|
||||
_stub_files(
|
||||
monkeypatch,
|
||||
{_V2_LIMIT_PATH: None, _V1_LIMIT_PATH: f"{(1 << 63) - 1}\n"},
|
||||
)
|
||||
|
||||
assert cru.get_cgroup_memory_limit() is None
|
||||
assert cru.get_cgroup_memory_usage() is None
|
||||
|
||||
|
||||
def test_cgroup_no_limit_files_present(monkeypatch):
|
||||
monkeypatch.setattr(sys, "platform", "linux")
|
||||
_stub_files(monkeypatch, {_V2_LIMIT_PATH: None, _V1_LIMIT_PATH: None})
|
||||
|
||||
assert cru.get_cgroup_memory_limit() is None
|
||||
assert cru.get_cgroup_memory_usage() is None
|
||||
|
||||
|
||||
def test_cgroup_skipped_on_non_linux(monkeypatch):
|
||||
monkeypatch.setattr(sys, "platform", "darwin")
|
||||
_stub_files(monkeypatch, {_V2_LIMIT_PATH: f"{1 * GiB_bytes}\n"})
|
||||
|
||||
assert cru.get_cgroup_memory_limit() is None
|
||||
assert cru.get_cgroup_memory_usage() is None
|
||||
|
||||
|
||||
def test_cgroup_limit_is_cached_until_explicitly_cleared(monkeypatch):
|
||||
monkeypatch.setattr(sys, "platform", "linux")
|
||||
files = {
|
||||
_V2_LIMIT_PATH: f"{1 * GiB_bytes}\n",
|
||||
_V2_USAGE_PATH: f"{512 << 20}\n",
|
||||
}
|
||||
_stub_files(monkeypatch, files)
|
||||
|
||||
assert cru.get_cgroup_memory_limit() == 1 * GiB_bytes
|
||||
files[_V2_LIMIT_PATH] = f"{2 * GiB_bytes}\n"
|
||||
assert cru.get_cgroup_memory_limit() == 1 * GiB_bytes
|
||||
|
||||
cru.get_cgroup_memory_limit.cache_clear()
|
||||
assert cru.get_cgroup_memory_limit() == 2 * GiB_bytes
|
||||
|
||||
|
||||
def test_cgroup_usage_is_read_without_cache(monkeypatch):
|
||||
monkeypatch.setattr(sys, "platform", "linux")
|
||||
files = {
|
||||
_V2_LIMIT_PATH: f"{1 * GiB_bytes}\n",
|
||||
_V2_USAGE_PATH: f"{512 << 20}\n",
|
||||
}
|
||||
_stub_files(monkeypatch, files)
|
||||
|
||||
assert cru.get_cgroup_memory_usage() == 512 << 20
|
||||
files[_V2_USAGE_PATH] = f"{768 << 20}\n"
|
||||
assert cru.get_cgroup_memory_usage() == 768 << 20
|
||||
|
||||
|
||||
def test_check_cgroup_memory_available_warns_on_low_headroom(monkeypatch):
|
||||
monkeypatch.setattr(cru, "get_cgroup_memory_limit", lambda: 1 << 30)
|
||||
monkeypatch.setattr(cru, "get_cgroup_memory_usage", lambda: 512 << 20)
|
||||
|
||||
with (
|
||||
mock.patch.object(cru.logger, "debug") as log_debug,
|
||||
mock.patch.object(cru.logger, "warning") as log_warning,
|
||||
):
|
||||
cru.check_cgroup_memory_available(600 << 20, "mmap")
|
||||
|
||||
log_debug.assert_not_called()
|
||||
log_warning.assert_called_once()
|
||||
assert "current headroom is below" in log_warning.call_args.args[-1]
|
||||
|
||||
|
||||
def test_check_cgroup_memory_available_logs_success_at_debug(monkeypatch):
|
||||
monkeypatch.setattr(cru, "get_cgroup_memory_limit", lambda: 1 << 30)
|
||||
monkeypatch.setattr(cru, "get_cgroup_memory_usage", lambda: 512 << 20)
|
||||
|
||||
with (
|
||||
mock.patch.object(cru.logger, "debug") as log_debug,
|
||||
mock.patch.object(cru.logger, "warning") as log_warning,
|
||||
):
|
||||
cru.check_cgroup_memory_available(256 << 20, "mmap")
|
||||
|
||||
log_debug.assert_called_once()
|
||||
log_warning.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("limit", "usage"),
|
||||
[(None, 512 << 20), (1 << 30, None)],
|
||||
)
|
||||
def test_check_cgroup_memory_available_skips_without_snapshot(
|
||||
monkeypatch, limit, usage
|
||||
):
|
||||
monkeypatch.setattr(cru, "get_cgroup_memory_limit", lambda: limit)
|
||||
monkeypatch.setattr(cru, "get_cgroup_memory_usage", lambda: usage)
|
||||
|
||||
with (
|
||||
mock.patch.object(cru.logger, "debug") as log_debug,
|
||||
mock.patch.object(cru.logger, "warning") as log_warning,
|
||||
):
|
||||
cru.check_cgroup_memory_available(1 << 60, "mmap")
|
||||
|
||||
log_debug.assert_not_called()
|
||||
log_warning.assert_not_called()
|
||||
@@ -20,10 +20,10 @@ kernel is documented to replicate.
|
||||
|
||||
import torch
|
||||
|
||||
# Bootstrap the glm5next package before entering the indexer module: its
|
||||
# kpool_compress import runs glm5next/__init__, which pulls model ->
|
||||
# attention -> back into sparse_attn_indexer_kpool (attention.py imports the
|
||||
# class at module scope). Production always enters via attention.py first.
|
||||
# Bootstrap the glm5next package before importing kpool_compress directly: its
|
||||
# import runs glm5next/__init__, which pulls model -> attention -> the indexer
|
||||
# dispatcher (sparse_indexer) -> back into the nvidia/amd sparse_indexer
|
||||
# modules. Production always enters via attention.py first.
|
||||
# isort: off
|
||||
import vllm.models.glm5next # noqa: F401
|
||||
from vllm.models.glm5next.nvidia.ops.kpool_compress import ( # noqa: E402
|
||||
@@ -32,13 +32,17 @@ from vllm.models.glm5next.nvidia.ops.kpool_compress import ( # noqa: E402
|
||||
)
|
||||
# isort: on
|
||||
|
||||
import vllm.model_executor.layers.sparse_attn_indexer_kpool as indexer_mod
|
||||
from vllm.model_executor.layers.sparse_attn_indexer_kpool import (
|
||||
from vllm.models.glm5next.common.sparse_indexer import (
|
||||
_decode_topk_seq_lens,
|
||||
_fill_short_decode_causal_indices,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.models.glm5next.amd import sparse_indexer as indexer_mod
|
||||
else:
|
||||
from vllm.models.glm5next.nvidia import sparse_indexer as indexer_mod
|
||||
|
||||
KPOOL = 4
|
||||
TOPK_TOKENS = 16
|
||||
SELECT_K = TOPK_TOKENS // KPOOL
|
||||
|
||||
@@ -279,6 +279,44 @@ def make_request(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dcp", [1, 4])
|
||||
def test_effective_attention_block_size_matches_events(dcp):
|
||||
from vllm.distributed.kv_events import BlockStored
|
||||
from vllm.v1.engine.core import EngineCore
|
||||
|
||||
config = KVCacheConfig(
|
||||
num_blocks=32,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["attention"], new_kv_cache_spec()),
|
||||
],
|
||||
)
|
||||
manager = KVCacheManager(
|
||||
generate_scheduler_kv_cache_config([config]),
|
||||
max_model_len=256,
|
||||
scheduler_block_size=16 * dcp,
|
||||
hash_block_size=16 * dcp,
|
||||
dcp_world_size=dcp,
|
||||
enable_kv_cache_events=True,
|
||||
)
|
||||
core = EngineCore.__new__(EngineCore)
|
||||
core.vllm_config = SimpleNamespace(cache_config=CacheConfig(block_size=16))
|
||||
core.scheduler = SimpleNamespace(kv_cache_manager=manager)
|
||||
core._initialize_effective_attention_block_size()
|
||||
block_size = core.vllm_config.cache_config.effective_attention_block_size
|
||||
assert block_size == 16 * dcp
|
||||
|
||||
request = make_request(
|
||||
"block-size", list(range(64)), block_size=16 * dcp, hash_fn=sha256
|
||||
)
|
||||
assert manager.allocate_slots(request, 64) is not None
|
||||
assert [
|
||||
event.block_size
|
||||
for event in manager.take_events()
|
||||
if isinstance(event, BlockStored)
|
||||
] == [block_size]
|
||||
|
||||
|
||||
def new_kv_cache_spec(
|
||||
block_size=16,
|
||||
num_kv_heads=2,
|
||||
|
||||
@@ -179,6 +179,63 @@ def test_mamba_retirement_bounds_prefill_states(block_size, in_flight_chunks):
|
||||
assert pool.get_num_free_blocks() == initial_free
|
||||
|
||||
|
||||
@pytest.mark.parametrize("block_size", [896, 1536])
|
||||
@pytest.mark.parametrize("num_speculative_blocks", [0, 1, 4])
|
||||
@pytest.mark.parametrize("prompt_tokens", [25121, 704547])
|
||||
def test_mamba_checkpoint_admission_matches_allocation(
|
||||
block_size, num_speculative_blocks, prompt_tokens
|
||||
):
|
||||
"""Checkpoint admission must match the subsequent physical allocation."""
|
||||
spec = MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
num_speculative_blocks=num_speculative_blocks,
|
||||
num_prefill_checkpoint_blocks=1,
|
||||
prefill_checkpoint_alignment=64,
|
||||
)
|
||||
pool = BlockPool(
|
||||
num_gpu_blocks=2048,
|
||||
enable_caching=True,
|
||||
hash_block_size=128,
|
||||
)
|
||||
manager = MambaManager(
|
||||
spec,
|
||||
block_pool=pool,
|
||||
enable_caching=True,
|
||||
kv_cache_group_id=0,
|
||||
scheduler_block_size=block_size,
|
||||
)
|
||||
request_id = "prefill"
|
||||
computed_tokens = 23040
|
||||
|
||||
def estimate(num_tokens, total_computed_tokens, apply_admission_cap):
|
||||
return manager.get_num_blocks_to_allocate(
|
||||
request_id=request_id,
|
||||
num_tokens=num_tokens,
|
||||
new_computed_blocks=[],
|
||||
total_computed_tokens=total_computed_tokens,
|
||||
num_local_computed_tokens=total_computed_tokens,
|
||||
num_tokens_main_model=num_tokens,
|
||||
apply_admission_cap=apply_admission_cap,
|
||||
)
|
||||
|
||||
estimate(computed_tokens, 0, False)
|
||||
manager.allocate_new_blocks(request_id, computed_tokens, computed_tokens)
|
||||
assert request_id in manager._allocated_block_reqs
|
||||
|
||||
admission_estimate = estimate(prompt_tokens, computed_tokens, True)
|
||||
allocation_estimate = estimate(prompt_tokens, computed_tokens, False)
|
||||
assert request_id in manager._checkpoints
|
||||
|
||||
free_before = pool.get_num_free_blocks()
|
||||
manager.allocate_new_blocks(request_id, prompt_tokens, prompt_tokens)
|
||||
allocated = free_before - pool.get_num_free_blocks()
|
||||
|
||||
assert admission_estimate == allocation_estimate == allocated
|
||||
|
||||
|
||||
def get_sliding_window_manager(
|
||||
sliding_window_spec,
|
||||
block_pool,
|
||||
|
||||
@@ -131,7 +131,7 @@ SMOKE_CONFIGS = [
|
||||
id="dflash",
|
||||
),
|
||||
pytest.param(
|
||||
"Qwen/Qwen3.5-0.8B-Base",
|
||||
"Qwen/Qwen3.5-0.8B",
|
||||
{
|
||||
"method": "mtp",
|
||||
"num_speculative_tokens": 3,
|
||||
|
||||
@@ -158,7 +158,10 @@ def test_mp_client_uses_env_timeout(monkeypatch: pytest.MonkeyPatch):
|
||||
local_engines_only=False,
|
||||
enable_elastic_ep=False,
|
||||
)
|
||||
vllm_config = SimpleNamespace(parallel_config=parallel_config)
|
||||
vllm_config = SimpleNamespace(
|
||||
parallel_config=parallel_config,
|
||||
cache_config=SimpleNamespace(effective_attention_block_size=None),
|
||||
)
|
||||
|
||||
client = core_client_mod.MPClient(
|
||||
asyncio_mode=False,
|
||||
@@ -368,10 +371,15 @@ def test_dplb_finished_requests_release_inflight():
|
||||
assert req.request_id not in client.reqs_in_flight
|
||||
|
||||
|
||||
def test_apply_ready_response_syncs_block_size():
|
||||
@pytest.mark.parametrize(
|
||||
("effective_size", "other_size"),
|
||||
[(None, None), (4224, 4224), (4224, 1056), (4224, None)],
|
||||
)
|
||||
def test_apply_ready_response_syncs_block_size(effective_size, other_size):
|
||||
import msgspec
|
||||
|
||||
client = object.__new__(MPClient)
|
||||
client._effective_attention_block_sizes = set()
|
||||
client.vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
|
||||
model_config=SimpleNamespace(max_model_len=8192),
|
||||
@@ -399,14 +407,31 @@ def test_apply_ready_response_syncs_block_size():
|
||||
max_loras=0,
|
||||
)
|
||||
)
|
||||
client._apply_ready_response(payload)
|
||||
fields = msgspec.msgpack.decode(payload)
|
||||
if effective_size is None:
|
||||
del fields["effective_attention_block_size"]
|
||||
else:
|
||||
fields["effective_attention_block_size"] = effective_size
|
||||
client._apply_ready_response(msgspec.msgpack.encode(fields))
|
||||
assert client.vllm_config.cache_config.block_size == 1056
|
||||
cache_config = client.vllm_config.cache_config
|
||||
assert cache_config.effective_attention_block_size == effective_size
|
||||
|
||||
fields["effective_attention_block_size"] = other_size
|
||||
client._apply_ready_response(msgspec.msgpack.encode(fields))
|
||||
assert cache_config.effective_attention_block_size == (
|
||||
effective_size if effective_size == other_size else None
|
||||
)
|
||||
|
||||
client._apply_ready_response(b"")
|
||||
assert cache_config.effective_attention_block_size is None
|
||||
|
||||
|
||||
def test_apply_ready_response_syncs_mamba_block_size():
|
||||
import msgspec
|
||||
|
||||
client = object.__new__(MPClient)
|
||||
client._effective_attention_block_sizes = set()
|
||||
client.vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
|
||||
model_config=SimpleNamespace(max_model_len=8192),
|
||||
|
||||
@@ -79,20 +79,31 @@ from vllm.v1.outputs import KVConnectorOutput, ModelRunnerOutput
|
||||
from vllm.v1.request import RequestStatus
|
||||
|
||||
|
||||
@pytest.mark.parametrize("boundary", [3840, 3904])
|
||||
@pytest.mark.parametrize(
|
||||
"window,extra_retained", [(128, 0), (128, 3), (256, 1), (256, 3)]
|
||||
)
|
||||
@pytest.mark.parametrize("boundary", [3840, 3904, 3968])
|
||||
@pytest.mark.parametrize("eagle", [False, True])
|
||||
@pytest.mark.parametrize("left_state", ["present", "missing", "pending", "loading"])
|
||||
def test_swa_offload_window_covers_unaligned_hit(boundary, eagle, left_state):
|
||||
def test_swa_offload_window_covers_unaligned_hit(
|
||||
boundary, eagle, left_state, window, extra_retained
|
||||
):
|
||||
"""A smaller SWA group can move the hit inside another group's CPU chunk."""
|
||||
groups = []
|
||||
for i, (block_size, window) in enumerate([(256, None), (64, 128), (8, 128)]):
|
||||
for i, (block_size, group_window) in enumerate(
|
||||
[(256, None), (64, window), (8, 128)]
|
||||
):
|
||||
kwargs = dict(
|
||||
block_size=block_size, num_kv_heads=1, head_size=1, dtype=torch.float32
|
||||
)
|
||||
kv_spec = (
|
||||
FullAttentionSpec(**kwargs)
|
||||
if window is None
|
||||
else SlidingWindowSpec(**kwargs, sliding_window=window)
|
||||
if group_window is None
|
||||
else SlidingWindowSpec(
|
||||
**kwargs,
|
||||
sliding_window=group_window,
|
||||
extra_retained_tokens=extra_retained if i == 1 else 0,
|
||||
)
|
||||
)
|
||||
groups.append(
|
||||
KVCacheGroupSpec([f"layer{i}"], kv_spec, is_eagle_group=eagle and i == 1)
|
||||
@@ -153,7 +164,9 @@ def test_swa_offload_window_covers_unaligned_hit(boundary, eagle, left_state):
|
||||
sched._chunks_being_loaded.add(left_key)
|
||||
|
||||
external, load_async = sched.get_num_new_matched_tokens(request, 0)
|
||||
if boundary == 3904 and left_state != "present":
|
||||
first_retained_block = (boundary - window + 1 - extra_retained) // 64
|
||||
needs_left = first_retained_block // 4 < end - 1
|
||||
if needs_left and left_state != "present":
|
||||
assert external == (0 if left_state == "missing" else None)
|
||||
assert not load_async
|
||||
return
|
||||
@@ -177,8 +190,8 @@ def test_swa_offload_window_covers_unaligned_hit(boundary, eagle, left_state):
|
||||
sched.update_state_after_alloc(request, blocks, external)
|
||||
assert len(sched._jobs) == 1
|
||||
job = next(iter(sched._jobs.values()))
|
||||
if boundary == 3904:
|
||||
assert {swa[14], swa[15]} <= job.keys
|
||||
if needs_left:
|
||||
assert {left_key, swa[end - 1]} <= job.keys
|
||||
manager.complete_load(job.keys, state.req_context)
|
||||
|
||||
|
||||
@@ -245,6 +258,9 @@ def test_partial_tail_store_uses_attention_and_recurrent_cow_sources():
|
||||
jobs = scheduler._build_partial_tail_store_jobs(output)
|
||||
|
||||
assert len(jobs) == 1
|
||||
offered_keys = scheduler.manager.prepare_store.call_args.args[0]
|
||||
req_context = scheduler._req_status["req"].req_context
|
||||
assert all(req_context.get_offload_key_position(key) == 28 for key in offered_keys)
|
||||
[job_id] = jobs
|
||||
src_spec = jobs[job_id].src_spec
|
||||
assert isinstance(src_spec, GPULoadStoreSpec)
|
||||
@@ -367,6 +383,13 @@ def test_normal_store_excludes_align_mode_mamba_sources():
|
||||
req_status.group_states[0].block_ids[:] = [11]
|
||||
req_status.group_states[1].block_ids[:] = [99]
|
||||
req_status.update_offload_keys()
|
||||
for group_config, group_state in zip(
|
||||
scheduler.config.kv_group_configs, req_status.group_states
|
||||
):
|
||||
assert (
|
||||
req_status.req_context.get_offload_key_position(group_state.offload_keys[0])
|
||||
== group_config.tokens_per_chunk
|
||||
)
|
||||
scheduler.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
@@ -908,7 +931,6 @@ def test_offloading_connector(request_runner, async_scheduling: bool):
|
||||
runner.run(decoded_tokens=[0] * (tokens_per_chunk + 1))
|
||||
|
||||
# 1 more block (+ token for kicking off offloading)
|
||||
# now check touch was called with all 6 blocks
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
@@ -916,9 +938,6 @@ def test_offloading_connector(request_runner, async_scheduling: bool):
|
||||
decoded_tokens=[0] * (tokens_per_chunk + 1),
|
||||
expected_stored=(15, 16, 17),
|
||||
)
|
||||
runner.manager.touch.assert_called()
|
||||
block_hashes1 = list(runner.manager.touch.call_args.args[0])
|
||||
assert len(block_hashes1) == 6
|
||||
|
||||
# terminate request
|
||||
runner.run(decoded_tokens=[EOS_TOKEN_ID])
|
||||
@@ -926,13 +945,7 @@ def test_offloading_connector(request_runner, async_scheduling: bool):
|
||||
# create a new request differing only on the last token
|
||||
runner.new_request(token_ids=[0] * (tokens_per_chunk * 6 - 1) + [1])
|
||||
runner.run(decoded_tokens=[0])
|
||||
runner.manager.touch.assert_called()
|
||||
block_hashes2 = list(runner.manager.touch.call_args.args[0])
|
||||
assert len(block_hashes2) == 6
|
||||
|
||||
# verify hashes are the same, except for the last block
|
||||
assert block_hashes1[:5] == block_hashes2[:5]
|
||||
assert block_hashes1[5] != block_hashes2[5]
|
||||
runner.manager.touch.assert_not_called()
|
||||
|
||||
# terminate request
|
||||
runner.run(
|
||||
@@ -1340,14 +1353,7 @@ def test_two_groups_full_and_sliding_window(request_runner, async_scheduling: bo
|
||||
generate_store_output(keys)
|
||||
)
|
||||
runner.run(decoded_tokens=[0])
|
||||
# _touch called from get_num_new_matched_tokens (2 groups) and
|
||||
# _get_reqs_to_store (2 groups) → 4 touch calls total.
|
||||
touch_calls = runner.manager.touch.call_args_list
|
||||
assert len(touch_calls) == 4
|
||||
assert len(touch_calls[0].args[0]) == 3
|
||||
assert len(touch_calls[1].args[0]) == 3
|
||||
assert len(touch_calls[2].args[0]) == 3
|
||||
assert len(touch_calls[3].args[0]) == 3
|
||||
runner.manager.touch.assert_not_called()
|
||||
|
||||
# store 3 more block
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
@@ -1358,9 +1364,7 @@ def test_two_groups_full_and_sliding_window(request_runner, async_scheduling: bo
|
||||
expected_stored=(0, 1, 2, 3, 4, 5),
|
||||
)
|
||||
|
||||
# touch called from _get_reqs_to_store * 3 blocks, once for each group
|
||||
touch_calls = runner.manager.touch.call_args_list
|
||||
assert len(touch_calls) == 6
|
||||
runner.manager.touch.assert_not_called()
|
||||
|
||||
# EOS lands in the last slot of the 7th block (offset 6). No forward pass
|
||||
# writes that slot, so the finishing step declines the block.
|
||||
@@ -1379,13 +1383,7 @@ def test_two_groups_full_and_sliding_window(request_runner, async_scheduling: bo
|
||||
expected_loaded=((0, 0), (0, 1), (0, 2), (1, 1), (1, 2)),
|
||||
)
|
||||
|
||||
# 2 touch calls from get_num_new_matched_tokens (2 groups)
|
||||
touch_calls = runner.manager.touch.call_args_list
|
||||
assert len(touch_calls) == 2
|
||||
# full attention group touched all 3 blocks
|
||||
assert len(touch_calls[0].args[0]) == 3
|
||||
# sliding window group touched just the last 2 blocks
|
||||
assert len(touch_calls[1].args[0]) == 2
|
||||
runner.manager.touch.assert_not_called()
|
||||
|
||||
# 3 blocks are hit on GPU [0, 1, 2]
|
||||
# 1 block loaded [3,]
|
||||
@@ -1451,15 +1449,7 @@ def test_two_groups_different_block_sizes(request_runner, async_scheduling: bool
|
||||
generate_store_output(keys)
|
||||
)
|
||||
runner.run(decoded_tokens=[0])
|
||||
# _touch called from get_num_new_matched_tokens (2 groups) and
|
||||
# _get_reqs_to_store (2 groups) → 4 touch calls total.
|
||||
# Group 0 has 2 offload keys, group 1 has 1.
|
||||
touch_calls = runner.manager.touch.call_args_list
|
||||
assert len(touch_calls) == 4
|
||||
assert len(touch_calls[0].args[0]) == 2
|
||||
assert len(touch_calls[1].args[0]) == 1
|
||||
assert len(touch_calls[2].args[0]) == 2
|
||||
assert len(touch_calls[3].args[0]) == 1
|
||||
runner.manager.touch.assert_not_called()
|
||||
|
||||
# Get to 31 tokens
|
||||
# No further blocks offloaded
|
||||
@@ -1469,11 +1459,7 @@ def test_two_groups_different_block_sizes(request_runner, async_scheduling: bool
|
||||
# Group 0 blocks: [0, 1], ending_token_offset = 24
|
||||
# Group 1 blocks: [0, 1], ending_token_offset = 32
|
||||
runner.run(decoded_tokens=[0])
|
||||
# _get_reqs_to_store touch: only group 1 has a new block to store
|
||||
touch_calls = runner.manager.touch.call_args_list
|
||||
assert len(touch_calls) == 2
|
||||
assert len(touch_calls[0].args[0]) == 2
|
||||
assert len(touch_calls[1].args[0]) == 2
|
||||
runner.manager.touch.assert_not_called()
|
||||
|
||||
# Get to 35 tokens
|
||||
# No further blocks offloaded
|
||||
@@ -1483,11 +1469,7 @@ def test_two_groups_different_block_sizes(request_runner, async_scheduling: bool
|
||||
# Group 0 blocks: [0, 1, 2], ending_token_offset = 36
|
||||
# Group 1 blocks: [0, 1], ending_token_offset = 32
|
||||
runner.run(decoded_tokens=[0])
|
||||
# _get_reqs_to_store touch: only group 0 has a new block to store
|
||||
touch_calls = runner.manager.touch.call_args_list
|
||||
assert len(touch_calls) == 2
|
||||
assert len(touch_calls[0].args[0]) == 3
|
||||
assert len(touch_calls[1].args[0]) == 2
|
||||
runner.manager.touch.assert_not_called()
|
||||
|
||||
# Get to 47 tokens
|
||||
# No further blocks offloaded
|
||||
@@ -1497,11 +1479,7 @@ def test_two_groups_different_block_sizes(request_runner, async_scheduling: bool
|
||||
# Group 0 blocks: [0, 1, 2, 3], ending_token_offset = 4
|
||||
# Group 1 blocks: [0, 1, 2], ending_token_offset = 48
|
||||
runner.run(decoded_tokens=[0])
|
||||
# _get_reqs_to_store touch: both groups have a new block, each with 1 key
|
||||
touch_calls = runner.manager.touch.call_args_list
|
||||
assert len(touch_calls) == 2
|
||||
assert len(touch_calls[0].args[0]) == 4
|
||||
assert len(touch_calls[1].args[0]) == 3
|
||||
runner.manager.touch.assert_not_called()
|
||||
|
||||
runner.run(decoded_tokens=[0], expected_stored=((0, 3), (1, 2)))
|
||||
|
||||
@@ -2629,27 +2607,14 @@ def test_async_preempt_readmit_before_transfer_output_is_deferred(request_runner
|
||||
assert req_status.transfer_jobs == pending_store_jobs
|
||||
|
||||
|
||||
@pytest.mark.parametrize("extra_retained", [0, 1, 3])
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_swa_alignment_skip(request_runner, async_scheduling: bool):
|
||||
"""SWA blocks unreachable by the load path are skipped during store.
|
||||
def test_swa_alignment_skip(request_runner, async_scheduling: bool, extra_retained):
|
||||
"""Store the window a cold consumer must restore at each FA boundary.
|
||||
|
||||
Simulates a DeepSeek V4-like hybrid architecture where SWA groups have
|
||||
much smaller block sizes than the full-attention (MLA) group, causing
|
||||
most SWA blocks to be unreachable by the alignment-based load path.
|
||||
|
||||
Setup:
|
||||
- Group 0: full attention (MLA-like), block_size=16
|
||||
- Group 1: SWA, block_size=4, sliding_window=8
|
||||
|
||||
alignment_tokens = 16, so 16 / 4 = 4 SWA blocks per alignment segment.
|
||||
sliding_window_size_in_chunks = ceil(8 / 4) = 2.
|
||||
Within each segment of 4 SWA blocks, only the trailing 2 are stored.
|
||||
|
||||
With 32 tokens (2 full-attn blocks, 8 SWA blocks):
|
||||
- Group 0 stores: blocks 0, 1 (all full-attn blocks)
|
||||
- Group 1 stores: blocks 2, 3, 6, 7 (skip 0,1,4,5)
|
||||
|
||||
For real DeepSeek V4 (100K tokens), this reduces SWA stores by ~78%.
|
||||
With FA blocks of 16 tokens and SWA blocks of 4 tokens, a window of 8
|
||||
needs the last two blocks per segment. Retaining three extra tokens for
|
||||
MTP requires one more block; retaining just one token does not.
|
||||
"""
|
||||
full_attn_block_size = 16
|
||||
swa_block_size = 4
|
||||
@@ -2674,6 +2639,7 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool):
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=sliding_window,
|
||||
extra_retained_tokens=extra_retained,
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -2691,16 +2657,17 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool):
|
||||
# Group 0: full attention
|
||||
assert kv_group_configs[0].sliding_window_size_in_chunks is None
|
||||
assert kv_group_configs[0].tokens_per_chunk == full_attn_block_size
|
||||
# Group 1: SWA with sliding_window_size of 2 chunks
|
||||
assert kv_group_configs[1].sliding_window_size_in_chunks == 2
|
||||
# Group 1: SWA, including any additional retained blocks
|
||||
assert kv_group_configs[1].sliding_window_size_in_chunks == (
|
||||
3 if extra_retained > 1 else 2
|
||||
)
|
||||
assert kv_group_configs[1].tokens_per_chunk == swa_block_size
|
||||
# alignment_tokens = full_attn_block_size = 16
|
||||
assert runner.connector_scheduler.config.alignment_tokens == full_attn_block_size
|
||||
|
||||
# Send 32 tokens = 2 full-attn blocks (block_size=16) = 8 SWA blocks
|
||||
# (block_size=4). Decode 1 token to kick off processing (stores are
|
||||
# deferred to next step).
|
||||
num_tokens = 32
|
||||
# Keep a partial last block so decoding does not reuse the producer's
|
||||
# freed blocks before the harness records its deferred stores.
|
||||
num_tokens = 33
|
||||
runner.new_request(token_ids=[0] * num_tokens)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
@@ -2713,12 +2680,7 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool):
|
||||
)
|
||||
runner.run(
|
||||
decoded_tokens=[EOS_TOKEN_ID],
|
||||
# Group 0 (full attn, block_size=16): 2 offloaded chunks
|
||||
# -> GPU blocks (0, 0) and (0, 1)
|
||||
# Group 1 (SWA, block_size=4): 8 offloaded chunks, skip first 2
|
||||
# per segment of 4:
|
||||
# Segment 0 (blocks 0-3): skip 0,1 -> store (1, 2), (1, 3)
|
||||
# Segment 1 (blocks 4-7): skip 4,5 -> store (1, 6), (1, 7)
|
||||
# Retain each segment's tail, including MTP history when needed.
|
||||
expected_stored=(
|
||||
(0, 0),
|
||||
(0, 1),
|
||||
@@ -2726,27 +2688,26 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool):
|
||||
(1, 3),
|
||||
(1, 6),
|
||||
(1, 7),
|
||||
),
|
||||
)
|
||||
+ (((1, 1), (1, 5)) if extra_retained > 1 else ()),
|
||||
)
|
||||
|
||||
# Verify that loads still work correctly for the stored SWA blocks.
|
||||
runner.scheduler.reset_prefix_cache()
|
||||
runner.new_request(token_ids=[0] * num_tokens + [1])
|
||||
runner.manager.lookup.return_value = LookupResult.HIT
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: 2
|
||||
runner.manager.lookup.side_effect = lambda key, ctx: (
|
||||
LookupResult.HIT if (key, 0) in runner.offloaded else LookupResult.MISS
|
||||
)
|
||||
runner.run(
|
||||
decoded_tokens=[EOS_TOKEN_ID],
|
||||
# Group 0: full prefix lookup hits 2 offloaded chunks
|
||||
# -> loads GPU blocks (0, 0), (0, 1)
|
||||
# Group 1: sliding window lookup finds trailing 2 from last segment
|
||||
# (blocks 6, 7 which were stored)
|
||||
# -> loads GPU blocks (1, 6), (1, 7)
|
||||
# The consumer restores the full FA prefix and the final SWA tail.
|
||||
expected_loaded=(
|
||||
(0, 0),
|
||||
(0, 1),
|
||||
(1, 6),
|
||||
(1, 7),
|
||||
),
|
||||
)
|
||||
+ (((1, 5),) if extra_retained > 1 else ()),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@ from vllm.v1.kv_offload.cpu.common import (
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
|
||||
from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy
|
||||
from vllm.v1.kv_offload.cpu.policies.lru import LRUCachePolicy
|
||||
|
||||
|
||||
def make_req_context(
|
||||
@@ -430,31 +431,34 @@ def test_cpu_manager():
|
||||
assert cpu_manager.lookup(to_key(0), _EMPTY_REQ_CTX) is LookupResult.MISS
|
||||
|
||||
# prepare load [2, 3]
|
||||
prepare_load_output = cpu_manager.prepare_load(to_keys([2, 3]), _EMPTY_REQ_CTX)
|
||||
load_ctx = make_req_context("load-2-3")
|
||||
prepare_load_output = cpu_manager.prepare_load(to_keys([2, 3]), load_ctx)
|
||||
verify_load_output(prepare_load_output, [1, 2])
|
||||
|
||||
# prepare store with no space ([2, 3] is being loaded)
|
||||
assert cpu_manager.prepare_store(to_keys([6, 7, 8]), _EMPTY_REQ_CTX) is None
|
||||
|
||||
# complete load [2, 3]. Load changes the eviction list, making 2, 3 recent.
|
||||
cpu_manager.complete_load(to_keys([2, 3]), _EMPTY_REQ_CTX)
|
||||
cpu_manager.complete_load(to_keys([2, 3]), load_ctx)
|
||||
cpu_manager.on_request_finished(load_ctx)
|
||||
|
||||
# prepare store [6, 7, 8] -> evicts [4, 5, 2] (oldest)
|
||||
# prepare store [6, 7, 8] -> evicts [4, 5, 3] (oldest). Within
|
||||
# the accessed prefix [2, 3], the tail is less valuable than the head.
|
||||
prepare_store_output = cpu_manager.prepare_store(to_keys([6, 7, 8]), _EMPTY_REQ_CTX)
|
||||
verify_store_output(
|
||||
prepare_store_output,
|
||||
ExpectedPrepareStoreOutput(
|
||||
keys_to_store=[6, 7, 8],
|
||||
store_chunk_ids=[1, 0, 3],
|
||||
evicted_keys=[4, 5, 2],
|
||||
store_chunk_ids=[2, 0, 3],
|
||||
evicted_keys=[4, 5, 3],
|
||||
),
|
||||
)
|
||||
|
||||
# complete store [6, 7, 8]
|
||||
cpu_manager.complete_store(to_keys([6, 7, 8]), _EMPTY_REQ_CTX)
|
||||
|
||||
# touch [3, 6, 7] (move to end of LRU order)
|
||||
cpu_manager.touch(to_keys([3, 6, 7]), _EMPTY_REQ_CTX)
|
||||
# touch [2, 6, 7] (move to end of LRU order)
|
||||
cpu_manager.touch(to_keys([2, 6, 7]), _EMPTY_REQ_CTX)
|
||||
|
||||
# prepare store [7, 9] -> evicts [8] (oldest following previous touch)
|
||||
prepare_store_output = cpu_manager.prepare_store(to_keys([9]), _EMPTY_REQ_CTX)
|
||||
@@ -477,7 +481,7 @@ def test_cpu_manager():
|
||||
verify_events(
|
||||
cpu_manager.take_events(),
|
||||
expected_stores=({3, 4, 5}, {6, 7, 8}),
|
||||
expected_evictions=({4, 5, 2}, {8}),
|
||||
expected_evictions=({4, 5, 3}, {8}),
|
||||
)
|
||||
|
||||
|
||||
@@ -518,6 +522,65 @@ def test_prepare_load_preserves_key_order():
|
||||
manager.complete_load([key_a, key_b, key_c], _EMPTY_REQ_CTX) # order irrelevant
|
||||
|
||||
|
||||
def test_lru_batch_eviction_failure_is_atomic():
|
||||
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
|
||||
policy = manager._policy
|
||||
assert isinstance(policy, LRUCachePolicy)
|
||||
keys = to_keys([1, 2, 3, 4])
|
||||
assert manager.prepare_store(keys, _EMPTY_REQ_CTX) is not None
|
||||
manager.complete_store(keys, _EMPTY_REQ_CTX)
|
||||
|
||||
protected = set(keys[1:])
|
||||
assert policy.evict(2, protected) is None
|
||||
assert all(manager.lookup(key, _EMPTY_REQ_CTX) is LookupResult.HIT for key in keys)
|
||||
|
||||
evicted = policy.evict(1, protected)
|
||||
assert evicted is not None
|
||||
assert [key for key, _ in evicted] == [keys[0]]
|
||||
|
||||
|
||||
def test_lru_repeated_pin_cycles_compact_lazy_heap_entries():
|
||||
manager = make_cpu_manager(num_chunks=1, cache_policy="lru")
|
||||
policy = manager._policy
|
||||
assert isinstance(policy, LRUCachePolicy)
|
||||
key = to_key(1)
|
||||
assert manager.prepare_store([key], _EMPTY_REQ_CTX) is not None
|
||||
manager.complete_store([key], _EMPTY_REQ_CTX)
|
||||
|
||||
for _ in range(128):
|
||||
manager.prepare_load([key], _EMPTY_REQ_CTX)
|
||||
manager.complete_load([key], _EMPTY_REQ_CTX)
|
||||
|
||||
assert len(policy._heap) < 64
|
||||
evicted = policy.evict(1, set())
|
||||
assert evicted is not None
|
||||
assert [evicted_key for evicted_key, _ in evicted] == [key]
|
||||
|
||||
|
||||
def test_reset_discards_stale_request_access_classification():
|
||||
manager = make_cpu_manager(num_chunks=1, cache_policy="arc")
|
||||
policy = manager._policy
|
||||
assert isinstance(policy, ARCCachePolicy)
|
||||
key = to_key(1)
|
||||
resumed_ctx = make_req_context("resumed")
|
||||
|
||||
assert manager.prepare_store([key], resumed_ctx) is not None
|
||||
manager.complete_store([key], resumed_ctx)
|
||||
manager.reset_cache()
|
||||
|
||||
replacement_ctx = make_req_context("replacement")
|
||||
assert manager.prepare_store([key], replacement_ctx) is not None
|
||||
manager.complete_store([key], replacement_ctx)
|
||||
manager.on_request_finished(replacement_ctx)
|
||||
assert key in policy.t1
|
||||
|
||||
manager.prepare_load([key], resumed_ctx)
|
||||
manager.complete_load([key], resumed_ctx)
|
||||
manager.on_request_finished(resumed_ctx)
|
||||
|
||||
assert key in policy.t2
|
||||
|
||||
|
||||
class TestARCPolicy:
|
||||
"""Unit tests for CPUOffloadingManager with ARC eviction policy."""
|
||||
|
||||
@@ -1042,6 +1105,48 @@ def test_filter_reused_manager_oversized_offer_makes_progress():
|
||||
assert stored_keys == set(keys)
|
||||
|
||||
|
||||
def test_store_threshold_does_not_hide_ready_reuse():
|
||||
manager = make_cpu_manager(
|
||||
num_chunks=2,
|
||||
cache_policy="lru",
|
||||
store_threshold=2,
|
||||
max_tracker_size=2,
|
||||
)
|
||||
head, tail, noise_1, noise_2, replacement = to_keys(list(range(5)))
|
||||
|
||||
seed_ctx = make_req_context("seed")
|
||||
output = manager.prepare_store([head, tail], seed_ctx)
|
||||
assert output is not None
|
||||
assert not output.keys_to_store
|
||||
output = manager.prepare_store([head, tail], seed_ctx)
|
||||
assert output is not None
|
||||
assert output.keys_to_store == [head, tail]
|
||||
manager.complete_store([head, tail], seed_ctx)
|
||||
manager.on_request_finished(seed_ctx)
|
||||
|
||||
# Age both resident keys out of the admission tracker without storing the
|
||||
# noise keys. Reusing the tail recreates its counter at one, below the
|
||||
# threshold, but must still refresh its cache recency.
|
||||
output = manager.prepare_store([noise_1, noise_2], make_req_context("noise"))
|
||||
assert output is not None
|
||||
assert not output.keys_to_store
|
||||
reuse_ctx = make_req_context("reuse")
|
||||
skipped_before_reuse = manager.stores_skipped_in_current_batch
|
||||
output = manager.prepare_store([tail], reuse_ctx)
|
||||
assert output is not None
|
||||
assert not output.keys_to_store
|
||||
assert manager.stores_skipped_in_current_batch == skipped_before_reuse
|
||||
manager.on_request_finished(reuse_ctx)
|
||||
|
||||
replacement_ctx = make_req_context("replacement")
|
||||
output = manager.prepare_store([replacement], replacement_ctx)
|
||||
assert output is not None
|
||||
assert not output.keys_to_store
|
||||
output = manager.prepare_store([replacement], replacement_ctx)
|
||||
assert output is not None
|
||||
assert output.evicted_keys == [head]
|
||||
|
||||
|
||||
def test_evictable_cache_chunk_count():
|
||||
"""
|
||||
Verifies _num_evictable_cache_chunks is maintained correctly through the
|
||||
@@ -1156,3 +1261,318 @@ def test_touch_forwards_req_context_to_policy(monkeypatch):
|
||||
assert len(received) == 1
|
||||
assert received[0][0] == keys
|
||||
assert received[0][1] is ctx
|
||||
|
||||
|
||||
@pytest.mark.parametrize("finish_before_completion", [False, True])
|
||||
def test_request_finish_orders_lru_prefix_independent_of_store_completion(
|
||||
finish_before_completion: bool,
|
||||
):
|
||||
"""A transfer's completion order must not make the prefix head LRU."""
|
||||
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
|
||||
prefix = to_keys([1, 2, 3, 4])
|
||||
ctx = make_req_context("prefix")
|
||||
|
||||
output = manager.prepare_store(prefix, ctx)
|
||||
assert output is not None
|
||||
if finish_before_completion:
|
||||
manager.on_request_finished(ctx)
|
||||
manager.complete_store(set(prefix), ctx)
|
||||
if not finish_before_completion:
|
||||
manager.on_request_finished(ctx)
|
||||
|
||||
output = manager.prepare_store(to_keys([5]), make_req_context("evict"))
|
||||
assert output is not None
|
||||
assert output.evicted_keys == to_keys([4])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("finish_before_completion", [False, True])
|
||||
def test_request_finish_orders_lru_prefix_independent_of_load_completion(
|
||||
finish_before_completion: bool,
|
||||
):
|
||||
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
|
||||
prefix = to_keys([1, 2, 3, 4])
|
||||
|
||||
seed_ctx = make_req_context("seed")
|
||||
output = manager.prepare_store(prefix, seed_ctx)
|
||||
assert output is not None
|
||||
manager.complete_store(prefix, seed_ctx)
|
||||
|
||||
reuse_ctx = make_req_context("reuse")
|
||||
manager.prepare_load(prefix, reuse_ctx)
|
||||
if finish_before_completion:
|
||||
manager.on_request_finished(reuse_ctx)
|
||||
manager.complete_load(set(prefix), reuse_ctx)
|
||||
if not finish_before_completion:
|
||||
manager.on_request_finished(reuse_ctx)
|
||||
|
||||
output = manager.prepare_store(to_keys([5]), make_req_context("evict"))
|
||||
assert output is not None
|
||||
assert output.evicted_keys == to_keys([4])
|
||||
|
||||
|
||||
def test_late_old_completion_does_not_override_newer_request_recency():
|
||||
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
|
||||
old_keys = to_keys([1, 2])
|
||||
new_keys = to_keys([3, 4])
|
||||
|
||||
old_ctx = make_req_context("old")
|
||||
assert manager.prepare_store(old_keys, old_ctx) is not None
|
||||
manager.on_request_finished(old_ctx)
|
||||
|
||||
new_ctx = make_req_context("new")
|
||||
assert manager.prepare_store(new_keys, new_ctx) is not None
|
||||
manager.complete_store(new_keys, new_ctx)
|
||||
manager.on_request_finished(new_ctx)
|
||||
|
||||
# Physical completion is late, but the logical access is still older.
|
||||
manager.complete_store(old_keys, old_ctx)
|
||||
|
||||
output = manager.prepare_store(to_keys([5]), make_req_context("evict"))
|
||||
assert output is not None
|
||||
assert output.evicted_keys == [old_keys[-1]]
|
||||
|
||||
|
||||
def test_arc_counts_reuse_once_and_keeps_insertions_in_t1():
|
||||
manager = make_cpu_manager(num_chunks=4, cache_policy="arc")
|
||||
policy = manager._policy
|
||||
assert isinstance(policy, ARCCachePolicy)
|
||||
prefix = to_keys([1, 2, 3, 4])
|
||||
|
||||
seed_ctx = make_req_context("seed")
|
||||
assert manager.prepare_store(prefix, seed_ctx) is not None
|
||||
manager.complete_store(prefix, seed_ctx)
|
||||
# Repeated store offers from the same request must not become ARC hits.
|
||||
assert manager.prepare_store(prefix, seed_ctx) is not None
|
||||
# Nor should an internal read used to cascade the new chunks to a tier.
|
||||
manager.prepare_load(prefix, seed_ctx)
|
||||
manager.complete_load(prefix, seed_ctx)
|
||||
manager.on_request_finished(seed_ctx)
|
||||
|
||||
assert list(policy.t1) == list(reversed(prefix))
|
||||
assert not policy.t2
|
||||
|
||||
reuse_ctx = make_req_context("reuse")
|
||||
manager.prepare_load(prefix, reuse_ctx)
|
||||
manager.complete_load(prefix, reuse_ctx)
|
||||
# A duplicate internal pin, as used by tiering cascades, is deduplicated.
|
||||
manager.prepare_load(prefix, reuse_ctx)
|
||||
manager.complete_load(prefix, reuse_ctx)
|
||||
manager.on_request_finished(reuse_ctx)
|
||||
|
||||
assert not policy.t1
|
||||
assert list(policy.t2) == list(reversed(prefix))
|
||||
|
||||
|
||||
def test_arc_reuse_wins_if_same_request_later_reinserts_key():
|
||||
manager = make_cpu_manager(num_chunks=1, cache_policy="arc")
|
||||
policy = manager._policy
|
||||
assert isinstance(policy, ARCCachePolicy)
|
||||
key_1, key_2 = to_keys([1, 2])
|
||||
|
||||
seed_ctx = make_req_context("seed")
|
||||
assert manager.prepare_store([key_1], seed_ctx) is not None
|
||||
manager.complete_store([key_1], seed_ctx)
|
||||
manager.on_request_finished(seed_ctx)
|
||||
|
||||
reuse_ctx = make_req_context("reuse-and-reinsert")
|
||||
manager.prepare_load([key_1], reuse_ctx)
|
||||
manager.complete_load([key_1], reuse_ctx)
|
||||
|
||||
other_ctx = make_req_context("evict-reused-key")
|
||||
assert manager.prepare_store([key_2], other_ctx) is not None
|
||||
manager.complete_store([key_2], other_ctx)
|
||||
manager.on_request_finished(other_ctx)
|
||||
|
||||
assert manager.prepare_store([key_1], reuse_ctx) is not None
|
||||
manager.complete_store([key_1], reuse_ctx)
|
||||
manager.on_request_finished(reuse_ctx)
|
||||
|
||||
assert key_1 not in policy.t1
|
||||
assert list(policy.t2) == [key_1]
|
||||
|
||||
|
||||
def test_request_finish_forwards_grouped_deduplicated_accesses(monkeypatch):
|
||||
manager = make_cpu_manager(num_chunks=5)
|
||||
existing = make_offload_key(b"existing", 0)
|
||||
group_0_new = make_offload_key(b"group-0-new", 0)
|
||||
group_1_head = make_offload_key(b"group-1-head", 1)
|
||||
group_1_tail = make_offload_key(b"group-1-tail", 1)
|
||||
|
||||
seed_ctx = make_req_context("seed")
|
||||
assert manager.prepare_store([existing], seed_ctx) is not None
|
||||
manager.complete_store([existing], seed_ctx)
|
||||
|
||||
ctx = make_req_context("grouped")
|
||||
# Group 1 is observed first, but policy finalization follows group index.
|
||||
offered = [group_1_head, existing, group_0_new]
|
||||
assert manager.prepare_store(offered, ctx) is not None
|
||||
# Later offers extend a group's prefix and may repeat earlier keys.
|
||||
assert (
|
||||
manager.prepare_store([group_0_new, group_1_head, group_1_tail], ctx)
|
||||
is not None
|
||||
)
|
||||
|
||||
received = []
|
||||
|
||||
def on_request_finished(key_groups, insertion_only_keys, reused_keys, req_context):
|
||||
received.append((key_groups, insertion_only_keys, reused_keys, req_context))
|
||||
|
||||
monkeypatch.setattr(manager._policy, "on_request_finished", on_request_finished)
|
||||
manager.on_request_finished(ctx)
|
||||
manager.on_request_finished(ctx)
|
||||
|
||||
assert received == [
|
||||
(
|
||||
((existing, group_0_new), (group_1_head, group_1_tail)),
|
||||
{group_0_new, group_1_head, group_1_tail},
|
||||
{existing},
|
||||
ctx,
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_failed_store_records_access_once_and_uses_store_miss_hook(monkeypatch):
|
||||
manager = make_cpu_manager(num_chunks=1)
|
||||
resident, candidate = to_keys([1, 2])
|
||||
|
||||
seed_ctx = make_req_context("seed")
|
||||
assert manager.prepare_store([resident], seed_ctx) is not None
|
||||
manager.complete_store([resident], seed_ctx)
|
||||
|
||||
pin_ctx = make_req_context("pin")
|
||||
manager.prepare_load([resident], pin_ctx)
|
||||
|
||||
record_calls = 0
|
||||
original_record = manager._record_request_cache_access
|
||||
|
||||
def record_access(*args, **kwargs):
|
||||
nonlocal record_calls
|
||||
record_calls += 1
|
||||
return original_record(*args, **kwargs)
|
||||
|
||||
store_miss_calls = []
|
||||
|
||||
def on_store_miss(keys, req_context):
|
||||
store_miss_calls.append((list(keys), req_context))
|
||||
|
||||
finalized = []
|
||||
|
||||
def on_request_finished(key_groups, insertion_only_keys, reused_keys, req_context):
|
||||
finalized.append((key_groups, insertion_only_keys, reused_keys, req_context))
|
||||
|
||||
monkeypatch.setattr(manager, "_record_request_cache_access", record_access)
|
||||
monkeypatch.setattr(manager._policy, "on_store_miss", on_store_miss)
|
||||
monkeypatch.setattr(manager._policy, "on_request_finished", on_request_finished)
|
||||
|
||||
ctx = make_req_context("failed-store")
|
||||
assert manager.prepare_store([candidate, resident], ctx) is None
|
||||
assert record_calls == 1
|
||||
assert store_miss_calls == [([candidate], ctx)]
|
||||
|
||||
manager.on_request_finished(ctx)
|
||||
assert finalized == [
|
||||
(((candidate, resident),), set(), {resident}, ctx),
|
||||
]
|
||||
|
||||
manager.complete_load([resident], pin_ctx)
|
||||
|
||||
|
||||
def test_arc_ghost_hit_adapts_once_per_request_before_insertion():
|
||||
manager = make_cpu_manager(num_chunks=2, cache_policy="arc")
|
||||
policy = manager._policy
|
||||
assert isinstance(policy, ARCCachePolicy)
|
||||
keys = to_keys([1, 2, 3])
|
||||
|
||||
seed_ctx = make_req_context("seed")
|
||||
assert manager.prepare_store(keys[:2], seed_ctx) is not None
|
||||
manager.complete_store(keys[:2], seed_ctx)
|
||||
manager.on_request_finished(seed_ctx)
|
||||
|
||||
evict_ctx = make_req_context("create-ghost")
|
||||
output = manager.prepare_store([keys[2]], evict_ctx)
|
||||
assert output is not None
|
||||
[ghost_key] = output.evicted_keys
|
||||
manager.complete_store([keys[2]], evict_ctx)
|
||||
manager.on_request_finished(evict_ctx)
|
||||
assert ghost_key in policy.b1
|
||||
|
||||
resident_keys = [key for key in keys if key != ghost_key]
|
||||
pin_ctx = make_req_context("pin-residents")
|
||||
manager.prepare_load(resident_keys, pin_ctx)
|
||||
|
||||
ghost_ctx = make_req_context("ghost-hit")
|
||||
target_before = policy.target_t1_size
|
||||
assert manager.prepare_store([ghost_key], ghost_ctx) is None
|
||||
target_after_first_offer = policy.target_t1_size
|
||||
assert target_after_first_offer > target_before
|
||||
assert manager.prepare_store([ghost_key], ghost_ctx) is None
|
||||
assert policy.target_t1_size == target_after_first_offer
|
||||
|
||||
manager.complete_load(resident_keys, pin_ctx)
|
||||
output = manager.prepare_store([ghost_key], ghost_ctx)
|
||||
assert output is not None
|
||||
assert policy.target_t1_size == target_after_first_offer
|
||||
manager.complete_store([ghost_key], ghost_ctx)
|
||||
manager.on_request_finished(ghost_ctx)
|
||||
assert ghost_key in policy.t1
|
||||
assert ghost_key not in policy.t2
|
||||
|
||||
|
||||
def test_arc_pending_chunk_is_not_a_frequency_hit():
|
||||
manager = make_cpu_manager(num_chunks=2, cache_policy="arc")
|
||||
policy = manager._policy
|
||||
assert isinstance(policy, ARCCachePolicy)
|
||||
key = to_key(1)
|
||||
|
||||
writer_ctx = make_req_context("writer")
|
||||
assert manager.prepare_store([key], writer_ctx) is not None
|
||||
|
||||
observer_ctx = make_req_context("pending-observer")
|
||||
output = manager.prepare_store([key], observer_ctx)
|
||||
assert output is not None and not output.keys_to_store
|
||||
manager.on_request_finished(observer_ctx)
|
||||
|
||||
chunk = policy.get(key)
|
||||
assert chunk is not None and not chunk.is_ready
|
||||
assert key in policy.t1
|
||||
assert key not in policy.t2
|
||||
|
||||
|
||||
def test_request_key_positions_override_store_observation_order():
|
||||
manager = make_cpu_manager(num_chunks=2, cache_policy="lru")
|
||||
head, tail = to_keys([1, 2])
|
||||
ctx = make_req_context("out-of-order-store")
|
||||
ctx.set_offload_key_position(head, 16)
|
||||
ctx.set_offload_key_position(tail, 32)
|
||||
|
||||
# A backfill or partial-tail path may offer a later key first.
|
||||
assert manager.prepare_store([tail], ctx) is not None
|
||||
assert manager.prepare_store([head], ctx) is not None
|
||||
manager.on_request_finished(ctx)
|
||||
manager.complete_store({head, tail}, ctx)
|
||||
|
||||
output = manager.prepare_store([to_key(3)], make_req_context("evict-tail"))
|
||||
assert output is not None
|
||||
assert output.evicted_keys == [tail]
|
||||
|
||||
|
||||
def test_request_key_positions_order_tails_across_kv_groups():
|
||||
manager = make_cpu_manager(num_chunks=4, cache_policy="lru")
|
||||
group_0_head = make_offload_key(b"group-0-head", 0)
|
||||
group_0_tail = make_offload_key(b"group-0-tail", 0)
|
||||
group_1_head = make_offload_key(b"group-1-head", 1)
|
||||
group_1_tail = make_offload_key(b"group-1-tail", 1)
|
||||
keys = [group_0_head, group_0_tail, group_1_head, group_1_tail]
|
||||
ctx = make_req_context("hybrid-groups")
|
||||
for key in (group_0_head, group_1_head):
|
||||
ctx.set_offload_key_position(key, 16)
|
||||
for key in (group_0_tail, group_1_tail):
|
||||
ctx.set_offload_key_position(key, 32)
|
||||
|
||||
assert manager.prepare_store(keys, ctx) is not None
|
||||
manager.complete_store(keys, ctx)
|
||||
manager.on_request_finished(ctx)
|
||||
|
||||
output = manager.prepare_store(to_keys([10, 11]), make_req_context("evict"))
|
||||
assert output is not None
|
||||
assert set(output.evicted_keys) == {group_0_tail, group_1_tail}
|
||||
|
||||
@@ -906,12 +906,13 @@ def test_insufficient_space_raises_clear_error(monkeypatch):
|
||||
monkeypatch.setattr(region.os, "open", mock_open)
|
||||
monkeypatch.setattr(region.os, "unlink", mock_unlink)
|
||||
monkeypatch.setattr(region.os, "close", mock_close)
|
||||
mock_check = MagicMock(
|
||||
side_effect=RuntimeError("Insufficient space in /dev/shm: 30 GB required.")
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
region,
|
||||
"check_shm_free_space",
|
||||
lambda *a, **kw: (_ for _ in ()).throw(
|
||||
RuntimeError("Insufficient space in /dev/shm: 30 GB required.")
|
||||
),
|
||||
mock_check,
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="Insufficient space"):
|
||||
@@ -925,6 +926,10 @@ def test_insufficient_space_raises_clear_error(monkeypatch):
|
||||
|
||||
mock_unlink.assert_called_once_with(mmap_path)
|
||||
mock_close.assert_called_once_with(9999)
|
||||
mock_check.assert_called_once_with(
|
||||
4 * PAGE_SIZE,
|
||||
allocation_name="CPU KV offload shared region in /dev/shm",
|
||||
)
|
||||
|
||||
|
||||
def test_ftruncate_failure_cleans_up_creator(monkeypatch):
|
||||
|
||||
@@ -40,6 +40,7 @@ from vllm.v1.kv_offload.base import (
|
||||
TierMatcher,
|
||||
make_offload_key,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.policies.lru import LRUCachePolicy
|
||||
from vllm.v1.kv_offload.tiering.base import (
|
||||
JobResult,
|
||||
SecondaryTierManager,
|
||||
@@ -753,14 +754,64 @@ class TestTieringOffloadingManager:
|
||||
# Touch chunks
|
||||
self.manager.touch(chunks, _CTX)
|
||||
|
||||
# Verify touch was called on primary tier (check LRU order)
|
||||
primary_keys = list(self.primary_tier._policy.evictable_chunks.keys())
|
||||
assert primary_keys[-3:] == list(reversed(chunks))
|
||||
# Verify touch was called on the primary tier (head is most recent).
|
||||
policy = self.primary_tier._policy
|
||||
assert isinstance(policy, LRUCachePolicy)
|
||||
ranks = policy._ranks
|
||||
assert ranks[chunks[2]] < ranks[chunks[1]] < ranks[chunks[0]]
|
||||
|
||||
# Verify touch was propagated to all secondary tiers
|
||||
self.secondary_tier1.touch.assert_called_once_with(chunks, _CTX)
|
||||
self.secondary_tier2.touch.assert_called_once_with(chunks, _CTX)
|
||||
|
||||
def test_request_order_survives_late_cascade_completion(self, manager_setup):
|
||||
"""Cascade completion must not replace request order with I/O order."""
|
||||
chunks = to_keys(range(5))
|
||||
self._start_request()
|
||||
output = self.manager.prepare_store(chunks, _CTX)
|
||||
assert output is not None
|
||||
self.manager.complete_store(chunks, _CTX, success=True)
|
||||
|
||||
# Finalize while both secondary-tier cascades still pin primary chunks.
|
||||
self.manager.on_request_finished(_CTX)
|
||||
self._simulate_on_schedule_end()
|
||||
self._simulate_on_schedule_end()
|
||||
|
||||
evict_ctx = ReqContext(req_id="evict")
|
||||
self._start_request(evict_ctx)
|
||||
output = self.manager.prepare_store(to_keys([5]), evict_ctx)
|
||||
assert output is not None
|
||||
assert output.evicted_keys == [chunks[-1]]
|
||||
|
||||
def test_late_old_store_submission_does_not_override_newer_request_order(
|
||||
self, manager_setup
|
||||
):
|
||||
"""Cascade submission after finish is not a new primary access."""
|
||||
old_blocks = to_keys(range(2))
|
||||
old_ctx = ReqContext(req_id="old")
|
||||
self._start_request(old_ctx)
|
||||
assert self.manager.prepare_store(old_blocks, old_ctx) is not None
|
||||
self.manager.on_request_finished(old_ctx)
|
||||
|
||||
new_blocks = to_keys(range(2, 5))
|
||||
new_ctx = ReqContext(req_id="new")
|
||||
self._start_request(new_ctx)
|
||||
assert self.manager.prepare_store(new_blocks, new_ctx) is not None
|
||||
self.manager.complete_store(new_blocks, new_ctx, success=True)
|
||||
self.manager.on_request_finished(new_ctx)
|
||||
|
||||
# The old GPU->primary write lands after the newer request finished.
|
||||
# Its cascade pins must not turn it into the more recent request.
|
||||
self.manager.complete_store(old_blocks, old_ctx, success=True)
|
||||
self._simulate_on_schedule_end()
|
||||
self._simulate_on_schedule_end()
|
||||
|
||||
evict_ctx = ReqContext(req_id="evict")
|
||||
self._start_request(evict_ctx)
|
||||
output = self.manager.prepare_store(to_keys([5]), evict_ctx)
|
||||
assert output is not None
|
||||
assert output.evicted_keys == [old_blocks[-1]]
|
||||
|
||||
def test_failed_store_no_cascade(self, manager_setup):
|
||||
"""Test that failed GPU→primary store doesn't cascade."""
|
||||
chunks = to_keys(range(3))
|
||||
@@ -873,10 +924,10 @@ class TestTieringOffloadingManager:
|
||||
job_metadata = self.secondary_tier1.submit_store.call_args.args[0]
|
||||
assert job_metadata.req_context is ctx
|
||||
|
||||
def test_on_request_finished_delays_secondary_until_store_submitted(
|
||||
def test_on_request_finished_delays_only_secondary_until_store_submitted(
|
||||
self, manager_setup
|
||||
):
|
||||
"""Manager hook is eager; secondary hooks wait for cascade submission."""
|
||||
"""Primary order commits immediately; secondary cleanup waits."""
|
||||
chunks = to_keys(range(2))
|
||||
ctx = ReqContext(req_id="req_delayed_secondary")
|
||||
calls: list[tuple[str, str]] = []
|
||||
|
||||
@@ -21,7 +21,12 @@ pytest.importorskip("triton")
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA required for Gumbel sampler tests", allow_module_level=True)
|
||||
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.worker.gpu.sample.gumbel import (
|
||||
_uniform64_from_random53,
|
||||
gumbel_sample,
|
||||
murmur3_hash32,
|
||||
)
|
||||
|
||||
DEVICE = "cuda"
|
||||
VOCAB_SIZE = 200_000
|
||||
@@ -33,6 +38,103 @@ HEAD_LOG_GAP = 18.0
|
||||
Z_TOLERANCE = 10.0
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _murmur3_known_answer_kernel(
|
||||
seeds_ptr,
|
||||
positions_ptr,
|
||||
offsets_ptr,
|
||||
hashes_ptr,
|
||||
domain_hashes_ptr,
|
||||
numel: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
idx = tl.arange(0, BLOCK_SIZE)
|
||||
mask = idx < numel
|
||||
seed = tl.load(seeds_ptr + idx, mask=mask)
|
||||
pos = tl.load(positions_ptr + idx, mask=mask)
|
||||
offset = tl.load(offsets_ptr + idx, mask=mask)
|
||||
hash32 = murmur3_hash32(seed, pos, offset).to(tl.int64)
|
||||
domain_hash32 = murmur3_hash32(seed, pos, offset, domain=0x9E3779B9).to(tl.int64)
|
||||
tl.store(hashes_ptr + idx, hash32, mask=mask)
|
||||
tl.store(domain_hashes_ptr + idx, domain_hash32, mask=mask)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _uniform64_endpoint_kernel(
|
||||
random53_ptr,
|
||||
uniform_ptr,
|
||||
numel: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
idx = tl.arange(0, BLOCK_SIZE)
|
||||
mask = idx < numel
|
||||
random53 = tl.load(random53_ptr + idx, mask=mask).to(tl.uint64)
|
||||
tl.store(uniform_ptr + idx, _uniform64_from_random53(random53), mask=mask)
|
||||
|
||||
|
||||
def test_murmur3_known_answers():
|
||||
seeds = torch.tensor(
|
||||
[0, 1, -1, 0x123456789ABCDEF, -(1 << 63), (1 << 63) - 1],
|
||||
dtype=torch.int64,
|
||||
device=DEVICE,
|
||||
)
|
||||
positions = torch.tensor(
|
||||
[0, 2, 17, 123456789, 1 << 30, (1 << 30) + 7],
|
||||
dtype=torch.int64,
|
||||
device=DEVICE,
|
||||
)
|
||||
offsets = torch.tensor(
|
||||
[0, 3, 1023, 199999, 4096, 0xFFFFFFFF],
|
||||
dtype=torch.int64,
|
||||
device=DEVICE,
|
||||
)
|
||||
hashes = torch.empty_like(seeds)
|
||||
domain_hashes = torch.empty_like(seeds)
|
||||
|
||||
_murmur3_known_answer_kernel[(1,)](
|
||||
seeds,
|
||||
positions,
|
||||
offsets,
|
||||
hashes,
|
||||
domain_hashes,
|
||||
numel=seeds.numel(),
|
||||
BLOCK_SIZE=8,
|
||||
)
|
||||
|
||||
assert hashes.tolist() == [
|
||||
0x8134CDF8,
|
||||
0x23B53F91,
|
||||
0xC8C13C14,
|
||||
0x5DDE974F,
|
||||
0xF1A9355B,
|
||||
0xD952F9AA,
|
||||
]
|
||||
assert domain_hashes.tolist() == [
|
||||
0xD28AD962,
|
||||
0xE3AAF818,
|
||||
0xD223C51C,
|
||||
0x0547D2FE,
|
||||
0x3AED160F,
|
||||
0x49E8EA7F,
|
||||
]
|
||||
|
||||
|
||||
def test_uniform64_excludes_endpoints():
|
||||
random53 = torch.tensor([0, (1 << 53) - 1], dtype=torch.int64, device=DEVICE)
|
||||
uniform = torch.empty(2, dtype=torch.float64, device=DEVICE)
|
||||
|
||||
_uniform64_endpoint_kernel[(1,)](
|
||||
random53,
|
||||
uniform,
|
||||
numel=random53.numel(),
|
||||
BLOCK_SIZE=2,
|
||||
)
|
||||
|
||||
assert torch.isfinite(uniform).all()
|
||||
assert ((uniform > 0.0) & (uniform < 1.0)).all()
|
||||
assert uniform.tolist() == [2.0**-54, 1.0 - 2.0**-53]
|
||||
|
||||
|
||||
def _make_heavy_tailed_counts(seed: int = 1234) -> torch.Tensor:
|
||||
"""Non-negative int64 counts of shape [VOCAB_SIZE]; target prob = counts/N."""
|
||||
gen = torch.Generator(device=DEVICE).manual_seed(seed)
|
||||
@@ -180,7 +282,7 @@ def test_full_vocab_distribution_fidelity():
|
||||
|
||||
|
||||
def test_drafting_uses_a_separate_noise_stream():
|
||||
"""is_drafting salts the Philox offset: same inputs, different draws.
|
||||
"""is_drafting salts the RNG position: same inputs, different draws.
|
||||
|
||||
The draft proposal and the residual resample after a rejection must be
|
||||
independent. They key noise by the same (seed, pos), so only the salt keeps
|
||||
@@ -189,7 +291,7 @@ def test_drafting_uses_a_separate_noise_stream():
|
||||
tests/v1/spec_decode/test_rejection_sampler_utils.py for the distributional
|
||||
consequence.
|
||||
|
||||
Relocating the offset must not distort the draw either, so both streams are
|
||||
Salting the position must not distort the draw either, so both streams are
|
||||
checked against the target's far-tail mass, which sits HEAD_LOG_GAP logits
|
||||
below the head -- the regime where fp32 Gumbel precision matters.
|
||||
"""
|
||||
|
||||
+8
-2
@@ -1482,8 +1482,14 @@ def cutlass_w4a8_moe_mm(
|
||||
Cumulative token offsets
|
||||
problem_sizes:
|
||||
Per-expert (M, N, K) GEMM sizes used by the grouped GEMM launcher.
|
||||
a/b/c/group_scale_strides:
|
||||
Strides describing the memory layout of the input tensors.
|
||||
a_strides:
|
||||
Strides describing the memory layout of a_tensors.
|
||||
b_strides:
|
||||
Strides describing the memory layout of b_tensors.
|
||||
c_strides:
|
||||
Strides describing the memory layout of out_tensors.
|
||||
group_scale_strides:
|
||||
Strides describing the memory layout of b_group_scales.
|
||||
maybe_schedule:
|
||||
Optional override to choose a specific kernel or epilogue schedule.
|
||||
|
||||
|
||||
@@ -211,6 +211,8 @@ class CacheConfig:
|
||||
"""The number of blocks to allocate for CPU memory."""
|
||||
|
||||
# Set after KV cache initialization.
|
||||
effective_attention_block_size: int | None = field(default=None, init=False)
|
||||
"""Full-attention block size in tokens, including DCP, or None if unavailable."""
|
||||
kv_cache_size_tokens: int | None = field(default=None, init=False)
|
||||
"""Per-DP-engine KV cache capacity in tokens (group-aware). Uses
|
||||
group-aware capacity since num_gpu_blocks * block_size can be wrong
|
||||
@@ -277,6 +279,7 @@ class CacheConfig:
|
||||
# Post-init/derived counters
|
||||
"num_gpu_blocks",
|
||||
"num_cpu_blocks",
|
||||
"effective_attention_block_size",
|
||||
"kv_cache_size_tokens",
|
||||
"kv_cache_max_concurrency",
|
||||
# WIP feature toggle not impacting compiled graph shape
|
||||
|
||||
@@ -34,6 +34,7 @@ import vllm.envs as envs
|
||||
from vllm.distributed.utils import StatelessProcessGroup, sched_yield
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.cpu_resource_utils import check_cgroup_memory_available
|
||||
from vllm.utils.network_utils import (
|
||||
get_ip,
|
||||
get_open_zmq_inproc_path,
|
||||
@@ -224,26 +225,37 @@ class SpinCondition:
|
||||
SHM_PATH = "/dev/shm"
|
||||
|
||||
|
||||
def check_shm_free_space(required_bytes: int, shm_path: str = SHM_PATH) -> None:
|
||||
"""Raise if ``shm_path`` cannot fit a ``required_bytes`` shared segment.
|
||||
def check_shm_free_space(
|
||||
required_bytes: int,
|
||||
shm_path: str = SHM_PATH,
|
||||
*,
|
||||
allocation_name: str = "shared-memory allocation",
|
||||
) -> None:
|
||||
"""Raise if SHM cannot fit a shared segment and log cgroup headroom.
|
||||
|
||||
Args:
|
||||
required_bytes: Size of the shared-memory segment to be created.
|
||||
shm_path: Mount point backing POSIX shared memory; skipped if absent.
|
||||
shm_path: Mount point backing POSIX shared memory; its filesystem
|
||||
check is skipped if absent.
|
||||
allocation_name: Human-readable name used in errors and logs.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If ``required_bytes`` exceeds the free space.
|
||||
RuntimeError: If the SHM filesystem has insufficient space.
|
||||
"""
|
||||
if not os.path.isdir(shm_path):
|
||||
return
|
||||
free_bytes = shutil.disk_usage(shm_path).free
|
||||
if required_bytes <= free_bytes:
|
||||
return
|
||||
mib = 1 << 20
|
||||
raise RuntimeError(
|
||||
f"Insufficient space in {shm_path}: {required_bytes / mib:.0f} MiB "
|
||||
f"required, {free_bytes / mib:.0f} MiB free. Increase {shm_path} "
|
||||
"(e.g. --shm-size or --ipc=host)."
|
||||
if os.path.isdir(shm_path):
|
||||
free_bytes = shutil.disk_usage(shm_path).free
|
||||
if required_bytes > free_bytes:
|
||||
mib = 1 << 20
|
||||
raise RuntimeError(
|
||||
f"Insufficient space in {shm_path} for {allocation_name}: "
|
||||
f"{required_bytes / mib:.0f} MiB required, "
|
||||
f"{free_bytes / mib:.0f} MiB free. Increase {shm_path} "
|
||||
"(e.g. --shm-size or --ipc=host)."
|
||||
)
|
||||
|
||||
check_cgroup_memory_available(
|
||||
required_bytes,
|
||||
allocation_name,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import time
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass, field, replace
|
||||
from itertools import chain, islice
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
@@ -114,7 +114,10 @@ class GroupOffloadConfig(NamedTuple):
|
||||
window = max(
|
||||
window,
|
||||
cdiv(
|
||||
self.kv_cache_spec.sliding_window - 1 + right_padding,
|
||||
self.kv_cache_spec.sliding_window
|
||||
- 1
|
||||
+ self.kv_cache_spec.extra_retained_tokens
|
||||
+ right_padding,
|
||||
self.tokens_per_chunk,
|
||||
),
|
||||
)
|
||||
@@ -126,7 +129,13 @@ def get_sliding_window_size_in_chunks(
|
||||
) -> int | None:
|
||||
if isinstance(kv_cache_spec, SlidingWindowSpec):
|
||||
assert kv_cache_spec.sliding_window > 0
|
||||
return cdiv(kv_cache_spec.sliding_window, tokens_per_chunk)
|
||||
return max(
|
||||
cdiv(kv_cache_spec.sliding_window, tokens_per_chunk),
|
||||
cdiv(
|
||||
kv_cache_spec.sliding_window - 1 + kv_cache_spec.extra_retained_tokens,
|
||||
tokens_per_chunk,
|
||||
),
|
||||
)
|
||||
|
||||
if isinstance(kv_cache_spec, ChunkedLocalAttentionSpec):
|
||||
# Attention never reaches back past one chunk
|
||||
@@ -418,8 +427,11 @@ class RequestOffloadState:
|
||||
None,
|
||||
group_config.hashes_per_chunk,
|
||||
):
|
||||
group_state.offload_keys.append(
|
||||
make_offload_key(req_block_hash, group_config.group_idx)
|
||||
key = make_offload_key(req_block_hash, group_config.group_idx)
|
||||
group_state.offload_keys.append(key)
|
||||
self.req_context.set_offload_key_position(
|
||||
key,
|
||||
len(group_state.offload_keys) * group_config.tokens_per_chunk,
|
||||
)
|
||||
|
||||
def update_block_id_groups(
|
||||
@@ -733,37 +745,6 @@ class OffloadingConnectorScheduler:
|
||||
)
|
||||
return None if defer_lookup or pending_in_window else consecutive_hits
|
||||
|
||||
def _touch(self, req_status: RequestOffloadState):
|
||||
for group_config, group_state in zip(
|
||||
self.config.kv_group_configs, req_status.group_states
|
||||
):
|
||||
if group_config.sliding_window_size_in_chunks is None:
|
||||
self.manager.touch(group_state.offload_keys, req_status.req_context)
|
||||
else:
|
||||
# Keep only chunks needed to hit the original request, plus
|
||||
# decoded chunks.
|
||||
chunks_to_skip = max(
|
||||
0,
|
||||
group_state.num_hit_chunks
|
||||
- group_config.sliding_window_size_in_chunks,
|
||||
)
|
||||
self.manager.touch(
|
||||
group_state.offload_keys[chunks_to_skip:],
|
||||
req_status.req_context,
|
||||
)
|
||||
if req_status.partial_tail_boundary is not None:
|
||||
self.manager.touch(
|
||||
tuple(
|
||||
self._make_boundary_key(
|
||||
req_status.req,
|
||||
group.group_idx,
|
||||
req_status.partial_tail_boundary,
|
||||
)
|
||||
for group in self.config.kv_group_configs
|
||||
),
|
||||
req_status.req_context,
|
||||
)
|
||||
|
||||
def _lookup_complete_chunks(
|
||||
self,
|
||||
req_status: RequestOffloadState,
|
||||
@@ -968,10 +949,16 @@ class OffloadingConnectorScheduler:
|
||||
return num_hit_tokens
|
||||
|
||||
def _make_boundary_key(
|
||||
self, request: Request, group_idx: int, boundary_tokens: int
|
||||
self,
|
||||
request: Request,
|
||||
group_idx: int,
|
||||
boundary_tokens: int,
|
||||
req_context: ReqContext,
|
||||
) -> OffloadKey:
|
||||
hash_idx = boundary_tokens // self.config.tokens_per_hash - 1
|
||||
return make_offload_key(request.block_hashes[hash_idx], group_idx)
|
||||
key = make_offload_key(request.block_hashes[hash_idx], group_idx)
|
||||
req_context.set_offload_key_position(key, boundary_tokens)
|
||||
return key
|
||||
|
||||
def _lookup(
|
||||
self,
|
||||
@@ -1006,7 +993,10 @@ class OffloadingConnectorScheduler:
|
||||
boundary_keys = []
|
||||
for group_config in self.config.kv_group_configs:
|
||||
key = self._make_boundary_key(
|
||||
req_status.req, group_config.group_idx, boundary
|
||||
req_status.req,
|
||||
group_config.group_idx,
|
||||
boundary,
|
||||
req_status.req_context,
|
||||
)
|
||||
boundary_keys.append(key)
|
||||
result = self.manager.lookup(key, req_status.req_context)
|
||||
@@ -1099,8 +1089,6 @@ class OffloadingConnectorScheduler:
|
||||
self._maybe_observe_lookup_async_delay(req_status)
|
||||
req_status.update_num_hit_chunks(num_computed_tokens + (num_hit_tokens or 0))
|
||||
|
||||
self._touch(req_status)
|
||||
|
||||
return num_hit_tokens, bool(num_hit_tokens)
|
||||
|
||||
def update_state_after_alloc(
|
||||
@@ -1174,7 +1162,10 @@ class OffloadingConnectorScheduler:
|
||||
if partial_tail_boundary is not None:
|
||||
keys_to_load.append(
|
||||
self._make_boundary_key(
|
||||
request, group_config.group_idx, partial_tail_boundary
|
||||
request,
|
||||
group_config.group_idx,
|
||||
partial_tail_boundary,
|
||||
req_status.req_context,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -1298,7 +1289,9 @@ class OffloadingConnectorScheduler:
|
||||
):
|
||||
continue
|
||||
|
||||
key = self._make_boundary_key(req, group_idx, boundary)
|
||||
key = self._make_boundary_key(
|
||||
req, group_idx, boundary, req_status.req_context
|
||||
)
|
||||
store_output = self.manager.prepare_store([key], req_status.req_context)
|
||||
if store_output is None:
|
||||
self._connector_stats.increase_counter(
|
||||
@@ -1393,7 +1386,9 @@ class OffloadingConnectorScheduler:
|
||||
):
|
||||
continue
|
||||
keys = [
|
||||
self._make_boundary_key(req, group.group_idx, boundary)
|
||||
self._make_boundary_key(
|
||||
req, group.group_idx, boundary, req_status.req_context
|
||||
)
|
||||
for group in self.config.kv_group_configs
|
||||
]
|
||||
block_ids = [
|
||||
@@ -1461,11 +1456,24 @@ class OffloadingConnectorScheduler:
|
||||
) -> list[bool] | None:
|
||||
"""Build the block mask for a range of candidate offload chunks."""
|
||||
blocks_per_chunk = self.config.blocks_per_chunk
|
||||
kv_cache_spec = group_config.kv_cache_spec
|
||||
if isinstance(kv_cache_spec, SlidingWindowSpec) and (
|
||||
kv_cache_spec.extra_retained_tokens
|
||||
):
|
||||
# Offload restores all allocated history, including MTP re-prefill
|
||||
# tokens. Widen only the store mask, not the model's attention window.
|
||||
kv_cache_spec = replace(
|
||||
kv_cache_spec,
|
||||
sliding_window=(
|
||||
kv_cache_spec.sliding_window + kv_cache_spec.extra_retained_tokens
|
||||
),
|
||||
extra_retained_tokens=0,
|
||||
)
|
||||
return group_config.manager_cls.reachable_block_mask(
|
||||
start_block=start_chunk_idx * blocks_per_chunk,
|
||||
end_block=end_chunk_idx * blocks_per_chunk,
|
||||
alignment_tokens=self.config.alignment_tokens,
|
||||
kv_cache_spec=group_config.kv_cache_spec,
|
||||
kv_cache_spec=kv_cache_spec,
|
||||
use_eagle=group_config.is_eagle_group,
|
||||
retention_interval=self.config.retention_interval,
|
||||
reachable_boundaries=reachable_boundaries,
|
||||
@@ -1667,8 +1675,6 @@ class OffloadingConnectorScheduler:
|
||||
req_status.advance_stored_idx(num_offloadable_tokens)
|
||||
continue
|
||||
|
||||
self._touch(req_status)
|
||||
|
||||
keys_to_store = set(store_output.keys_to_store)
|
||||
|
||||
group_sizes: list[int] = []
|
||||
|
||||
@@ -2051,7 +2051,7 @@ class EngineArgs:
|
||||
|
||||
device_config = DeviceConfig(device=cast(Device, current_platform.device_type))
|
||||
|
||||
envs.validate_environ(self.fail_on_environ_validation)
|
||||
current_platform.validate_environ(self.fail_on_environ_validation)
|
||||
|
||||
# Check if the model is a speculator and override model/tokenizer/config
|
||||
# BEFORE creating ModelConfig, so the config is created with the target model
|
||||
|
||||
@@ -46,21 +46,35 @@ def _interactive_cli(args: argparse.Namespace) -> tuple[str, OpenAI]:
|
||||
|
||||
|
||||
def _print_chat_stream(stream, stats: bool = False) -> str:
|
||||
output = ""
|
||||
start = time.perf_counter()
|
||||
output: str = ""
|
||||
in_reasoning: bool = False
|
||||
start: float = time.perf_counter()
|
||||
ttft: float | None = None
|
||||
completion_tokens = 0
|
||||
completion_tokens: int = 0
|
||||
for chunk in stream:
|
||||
if chunk.usage is not None:
|
||||
completion_tokens = chunk.usage.completion_tokens
|
||||
if not chunk.choices:
|
||||
continue
|
||||
delta = chunk.choices[0].delta
|
||||
reasoning = getattr(delta, "reasoning", None) or getattr(
|
||||
delta, "reasoning_content", None
|
||||
)
|
||||
if ttft is None and (reasoning or delta.content):
|
||||
ttft = time.perf_counter() - start
|
||||
if reasoning:
|
||||
if not in_reasoning:
|
||||
print("<think>", flush=True)
|
||||
in_reasoning = True
|
||||
print(reasoning, end="", flush=True)
|
||||
if delta.content:
|
||||
if ttft is None:
|
||||
ttft = time.perf_counter() - start
|
||||
if in_reasoning:
|
||||
print("\n</think>", flush=True)
|
||||
in_reasoning = False
|
||||
output += delta.content
|
||||
print(delta.content, end="", flush=True)
|
||||
if in_reasoning:
|
||||
print("\n</think>", end="", flush=True)
|
||||
print()
|
||||
if stats:
|
||||
_print_metrics(start, ttft, completion_tokens)
|
||||
|
||||
@@ -565,6 +565,7 @@ class ParsableContext(ConversationContext):
|
||||
mcp_tools: dict[str, Mcp],
|
||||
):
|
||||
if tool_server:
|
||||
initialized_session = False
|
||||
for tool_name in self.available_tools:
|
||||
if tool_name in self._tool_sessions:
|
||||
continue
|
||||
@@ -577,6 +578,8 @@ class ParsableContext(ConversationContext):
|
||||
tool_server.new_session(tool_name, request_id, headers)
|
||||
)
|
||||
self._tool_sessions[tool_name] = tool_session
|
||||
initialized_session = True
|
||||
if initialized_session:
|
||||
exit_stack.push_async_exit(self.cleanup_session)
|
||||
|
||||
async def cleanup_session(self, *args, **kwargs) -> None:
|
||||
@@ -863,6 +866,7 @@ class HarmonyContext(ConversationContext):
|
||||
mcp_tools: dict[str, Mcp],
|
||||
):
|
||||
if tool_server:
|
||||
initialized_session = False
|
||||
for tool_name in self.available_tools:
|
||||
if tool_name not in self._tool_sessions:
|
||||
tool_type = _map_tool_name_to_tool_type(tool_name)
|
||||
@@ -873,7 +877,9 @@ class HarmonyContext(ConversationContext):
|
||||
tool_server.new_session(tool_name, request_id, headers)
|
||||
)
|
||||
self._tool_sessions[tool_name] = tool_session
|
||||
exit_stack.push_async_exit(self.cleanup_session)
|
||||
initialized_session = True
|
||||
if initialized_session:
|
||||
exit_stack.push_async_exit(self.cleanup_session)
|
||||
|
||||
async def call_container_tool(
|
||||
self, tool_session: Union["ClientSession", Tool], last_msg: Message
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import logging
|
||||
|
||||
|
||||
import model_hosting_container_standards.sagemaker as sagemaker_standards
|
||||
from fastapi import APIRouter, Depends, FastAPI, Request
|
||||
from fastapi.responses import JSONResponse, Response
|
||||
|
||||
@@ -22,6 +21,12 @@ router = APIRouter()
|
||||
|
||||
|
||||
def attach_router(app: FastAPI):
|
||||
"""Attach the LoRA adapter load/unload endpoints to the API server.
|
||||
|
||||
Does nothing when dynamic LoRA updating is disabled. Handler levels are
|
||||
snapshotted and restored because importing
|
||||
model_hosting_container_standards may reconfigure root logging.
|
||||
"""
|
||||
if not envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
|
||||
"""If LoRA dynamic loading & unloading is not enabled, do nothing."""
|
||||
return
|
||||
@@ -30,6 +35,23 @@ def attach_router(app: FastAPI):
|
||||
"This should ONLY be used for local development!"
|
||||
)
|
||||
|
||||
snapshot = [
|
||||
(h, h.level)
|
||||
for lg in (logging.getLogger(), logging.getLogger("vllm"))
|
||||
for h in lg.handlers
|
||||
]
|
||||
|
||||
try:
|
||||
_attach_router(app)
|
||||
finally:
|
||||
for handler, level in snapshot:
|
||||
handler.setLevel(level)
|
||||
|
||||
|
||||
def _attach_router(app: FastAPI):
|
||||
"""Register the LoRA adapter load/unload routes on the app."""
|
||||
import model_hosting_container_standards.sagemaker as sagemaker_standards
|
||||
|
||||
@sagemaker_standards.register_load_adapter_handler(
|
||||
request_shape={
|
||||
"lora_name": "body.name",
|
||||
@@ -40,6 +62,8 @@ def attach_router(app: FastAPI):
|
||||
)
|
||||
@router.post("/v1/load_lora_adapter", dependencies=[Depends(validate_json_request)])
|
||||
async def load_lora_adapter(request: LoadLoRAAdapterRequest, raw_request: Request):
|
||||
"""Handle POST /v1/load_lora_adapter: load a LoRA adapter into the
|
||||
serving engine."""
|
||||
handler: OpenAIServingModels = models(raw_request)
|
||||
response = await handler.load_lora_adapter(request)
|
||||
if isinstance(response, ErrorResponse):
|
||||
@@ -60,6 +84,8 @@ def attach_router(app: FastAPI):
|
||||
async def unload_lora_adapter(
|
||||
request: UnloadLoRAAdapterRequest, raw_request: Request
|
||||
):
|
||||
"""Handle POST /v1/unload_lora_adapter: unload a LoRA adapter from
|
||||
the serving engine."""
|
||||
handler: OpenAIServingModels = models(raw_request)
|
||||
response = await handler.unload_lora_adapter(request)
|
||||
if isinstance(response, ErrorResponse):
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Awaitable, Callable
|
||||
from http import HTTPStatus
|
||||
from typing import Any
|
||||
|
||||
import model_hosting_container_standards.sagemaker as sagemaker_standards
|
||||
import pydantic
|
||||
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
|
||||
from fastapi.responses import JSONResponse, Response
|
||||
@@ -27,11 +27,55 @@ GetHandlerFn = Callable[[Request], BaseServing | None]
|
||||
EndpointFn = Callable[[RequestType, Request], Awaitable[Any]]
|
||||
|
||||
|
||||
def _snapshot_handler_levels() -> list[tuple[logging.Handler, int]]:
|
||||
"""Snapshot handler levels for loggers that may be affected by third-party
|
||||
logging configuration side effects (e.g. model_hosting_container_standards
|
||||
calling configure_root_logger() at import time)."""
|
||||
loggers = [logging.getLogger(), logging.getLogger("vllm")]
|
||||
seen: set[int] = set()
|
||||
snapshot: list[tuple[logging.Handler, int]] = []
|
||||
for logger in loggers:
|
||||
for handler in logger.handlers:
|
||||
if id(handler) not in seen:
|
||||
seen.add(id(handler))
|
||||
snapshot.append((handler, handler.level))
|
||||
return snapshot
|
||||
|
||||
|
||||
def _restore_handler_levels(
|
||||
snapshot: list[tuple[logging.Handler, int]],
|
||||
) -> None:
|
||||
"""Restore handler levels from a snapshot."""
|
||||
for handler, level in snapshot:
|
||||
handler.setLevel(level)
|
||||
|
||||
|
||||
def attach_router(
|
||||
app: FastAPI,
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
model_config: ModelConfig | None = None,
|
||||
):
|
||||
"""Attach the SageMaker hosting endpoints to the API server.
|
||||
|
||||
Handler levels are snapshotted and restored because importing
|
||||
model_hosting_container_standards may reconfigure root logging.
|
||||
"""
|
||||
snapshot = _snapshot_handler_levels()
|
||||
try:
|
||||
_attach_router(app, supported_tasks, model_config)
|
||||
finally:
|
||||
_restore_handler_levels(snapshot)
|
||||
|
||||
|
||||
def _attach_router(
|
||||
app: FastAPI,
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
model_config: ModelConfig | None = None,
|
||||
):
|
||||
"""Register the SageMaker hosting routes (/ping, /invocations) on the
|
||||
app."""
|
||||
import model_hosting_container_standards.sagemaker as sagemaker_standards
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# NOTE: Construct the TypeAdapters only once
|
||||
@@ -99,4 +143,17 @@ def attach_router(
|
||||
|
||||
|
||||
def sagemaker_standards_bootstrap(app: FastAPI) -> FastAPI:
|
||||
return sagemaker_standards.bootstrap(app)
|
||||
"""Bootstrap the app with the SageMaker hosting standards.
|
||||
|
||||
Handler levels are restored right after the import because importing
|
||||
model_hosting_container_standards may reconfigure root logging, and
|
||||
bootstrap must run with the original levels.
|
||||
"""
|
||||
snapshot = _snapshot_handler_levels()
|
||||
try:
|
||||
import model_hosting_container_standards.sagemaker as sagemaker_standards
|
||||
|
||||
app = sagemaker_standards.bootstrap(app)
|
||||
finally:
|
||||
_restore_handler_levels(snapshot)
|
||||
return app
|
||||
|
||||
@@ -2249,15 +2249,6 @@ def is_set(name: str):
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
|
||||
def validate_environ(hard_fail: bool) -> None:
|
||||
for env in os.environ:
|
||||
if env.startswith("VLLM_") and env not in environment_variables:
|
||||
if hard_fail:
|
||||
raise ValueError(f"Unknown vLLM environment variable detected: {env}")
|
||||
else:
|
||||
logger.warning("Unknown vLLM environment variable detected: %s", env)
|
||||
|
||||
|
||||
def compile_factors() -> dict[str, object]:
|
||||
"""Return env vars used for torch.compile cache keys.
|
||||
|
||||
|
||||
@@ -34,13 +34,38 @@ direct_register_aiter_op = functools.partial(
|
||||
AITER_SUPPORTED = is_aiter_found()
|
||||
"""Most kernels in this file are supported if AITER is installed."""
|
||||
|
||||
|
||||
def flatten_to_2d_is_free(x: Tensor) -> bool:
|
||||
"""Whether ``x.reshape(-1, x.shape[-1])`` is a view with unit-stride rows.
|
||||
|
||||
The AITER norm kernels only take dense 2D inputs, so the wrappers below
|
||||
flatten the leading dims with ``Tensor.reshape``, which silently falls back
|
||||
to ``contiguous()`` when the flattened shape is not expressible with the
|
||||
existing strides, adding a whole-tensor device copy that costs more than
|
||||
the kernel saves. A strided last dim flattens without a copy but not into a
|
||||
dense buffer, so it is rejected too.
|
||||
"""
|
||||
if x.dim() <= 2:
|
||||
return True
|
||||
if x.stride(-1) != 1:
|
||||
return False
|
||||
expected_stride = x.size(-1)
|
||||
for i in range(x.dim() - 2, -1, -1):
|
||||
if x.size(i) != 1 and x.stride(i) != expected_stride:
|
||||
return False
|
||||
expected_stride *= x.size(i)
|
||||
return True
|
||||
|
||||
|
||||
rms_no_var_16bit_only = (
|
||||
lambda x, weight, epsilon, variance_size=None: variance_size is None
|
||||
and x.dtype in (torch.float16, torch.bfloat16)
|
||||
and (weight is None or weight.dtype == x.dtype)
|
||||
and flatten_to_2d_is_free(x)
|
||||
)
|
||||
"""AITER rms_norm only supports float16 and bfloat16 acts, no var_size override,
|
||||
and requires weight dtype to match x dtype."""
|
||||
requires weight dtype to match x dtype, and requires flattening the activation
|
||||
to 2D to be free."""
|
||||
|
||||
|
||||
@ir.ops.rms_norm.register_impl(
|
||||
@@ -79,10 +104,13 @@ rms_add_no_var_16bit_only = (
|
||||
lambda x, x_residual, weight, epsilon, variance_size=None: variance_size is None
|
||||
and x.dtype in (torch.float16, torch.bfloat16)
|
||||
and (weight is None or weight.dtype == x.dtype)
|
||||
and flatten_to_2d_is_free(x)
|
||||
and flatten_to_2d_is_free(x_residual)
|
||||
)
|
||||
"""
|
||||
AITER fused_add_rms_norm only supports 16-bit activations and no var_size override.
|
||||
Requires weight dtype to match x dtype.
|
||||
Requires weight dtype to match x dtype, and flattening both the activation and the
|
||||
residual to 2D to be free.
|
||||
"""
|
||||
|
||||
|
||||
|
||||
@@ -190,9 +190,11 @@ def candidate_blocks_to_sparse_indices(
|
||||
candidate_blocks: [rows, K] int32 request-local candidate block ids
|
||||
(-1 padded), in units of ``candidate_block_size`` positions. K
|
||||
must be a power of two (the in-kernel sort); rows may be strided.
|
||||
row_ks/row_ke: [rows] int32 per-row K range; bounds are in the same
|
||||
row_ks: [rows] int32 per-row K range start; bounds are in the same
|
||||
(packed-workspace) coordinates the sparse kernel iterates over.
|
||||
Pass zeros for the paged path, whose blocks are context-relative.
|
||||
row_ke: [rows] int32 per-row K range end, in the same coordinates as
|
||||
``row_ks``.
|
||||
candidate_block_size: Positions per candidate block.
|
||||
sparse_block_kv: Positions per sparse block (8 or 16).
|
||||
out: Optional ``(sparse_indices, end)`` buffers to write into,
|
||||
@@ -322,12 +324,16 @@ def sparse_mqa_logits_prefill_chunk(
|
||||
k_scale: [total_kv] int32 packed UE8M0 K scales.
|
||||
weights: [rows, H] bf16 per-head weights; the sparse kernels take
|
||||
bf16 and do not fold the Q scale in.
|
||||
cu_seqlen_ks/cu_seqlen_ke: [rows] int32 per-token K bounds in the
|
||||
packed workspace.
|
||||
cu_seqlen_ks: [rows] int32 per-token K start bounds in the packed
|
||||
workspace.
|
||||
cu_seqlen_ke: [rows] int32 per-token K end bounds in the packed
|
||||
workspace.
|
||||
candidate_blocks: [rows, K] int32 request-local candidate block ids.
|
||||
topk_indices: [rows, topk_tokens] output buffer.
|
||||
sparse_indices/end/col_indices: Caller-owned scratch, see
|
||||
`candidate_blocks_to_sparse_indices` and `sparse_topk_remap`.
|
||||
sparse_indices: Caller-owned scratch, see
|
||||
`candidate_blocks_to_sparse_indices`.
|
||||
end: Caller-owned scratch, see `candidate_blocks_to_sparse_indices`.
|
||||
col_indices: Caller-owned scratch, see `sparse_topk_remap`.
|
||||
kernel_metadata: DeepGEMM schedule from a previous call with the same
|
||||
candidates and bounds (i.e. another indexer layer in the same
|
||||
step). When given, the candidate expansion is skipped and
|
||||
@@ -413,7 +419,9 @@ def sparse_mqa_logits_paged_decode(
|
||||
candidate_blocks: [rows, K] int32 candidate block ids.
|
||||
topk_indices: [rows, topk_tokens] output buffer.
|
||||
row_ks: [rows] int32 zeros (paged blocks are context-relative).
|
||||
sparse_indices/end/col_indices: Caller-owned scratch.
|
||||
sparse_indices: Caller-owned scratch.
|
||||
end: Caller-owned scratch.
|
||||
col_indices: Caller-owned scratch.
|
||||
kernel_metadata: See `sparse_mqa_logits_prefill_chunk`.
|
||||
|
||||
Returns:
|
||||
|
||||
@@ -187,6 +187,7 @@ from vllm.model_executor.kernels.linear.scaled_mm.b12x import (
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.cpu import (
|
||||
CPUFp8BlockScaledMMKernel,
|
||||
CPUFp8PerTensorScaledMMLinearKernel,
|
||||
CPUInt8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.cutlass import (
|
||||
@@ -438,6 +439,7 @@ _POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] =
|
||||
ChannelWiseTorchFP8ScaledMMLinearKernel,
|
||||
],
|
||||
PlatformEnum.CPU: [
|
||||
CPUFp8PerTensorScaledMMLinearKernel,
|
||||
PerTensorTorchFP8ScaledMMLinearKernel,
|
||||
ChannelWiseTorchFP8ScaledMMLinearKernel,
|
||||
],
|
||||
|
||||
@@ -9,6 +9,7 @@ from vllm.model_executor.kernels.linear.scaled_mm.BlockScaledMMLinearKernel impo
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.cpu import (
|
||||
CPUFp8BlockScaledMMKernel,
|
||||
CPUFp8PerTensorScaledMMLinearKernel,
|
||||
CPUInt8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.cutlass import (
|
||||
@@ -67,5 +68,6 @@ __all__ = [
|
||||
"ZentorchInt8ScaledMMLinearKernel",
|
||||
"Fp8BlockScaledMMLinearKernel",
|
||||
"CPUFp8BlockScaledMMKernel",
|
||||
"CPUFp8PerTensorScaledMMLinearKernel",
|
||||
"XPUFp8BlockScaledMMKernel",
|
||||
]
|
||||
|
||||
@@ -21,10 +21,14 @@ from .BlockScaledMMLinearKernel import (
|
||||
FP8ScaledMMLinearLayerConfig,
|
||||
)
|
||||
from .ScaledMMLinearKernel import (
|
||||
FP8ScaledMMLinearKernel,
|
||||
Int8ScaledMMLinearKernel,
|
||||
Int8ScaledMMLinearLayerConfig,
|
||||
)
|
||||
|
||||
# BLOCK_K in csrc/cpu/sgl-kernels/gemm.h — the AMX kernel's fixed K-block size.
|
||||
_BLOCK_SIZE_K = 128
|
||||
|
||||
|
||||
class CPUInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel):
|
||||
@classmethod
|
||||
@@ -330,3 +334,99 @@ class CPUFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
raise NotImplementedError(
|
||||
"CPUFp8BlockScaledMMKernel overrides apply_weights directly."
|
||||
)
|
||||
|
||||
|
||||
class CPUFp8PerTensorScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
"""FP8 W8A16 per-tensor-scaled GEMM via AMX BRGEMM on CPU.
|
||||
|
||||
Reuses the block-scaled AMX kernel (fp8_scaled_mm_cpu) with a single
|
||||
synthetic block spanning the whole weight, so activations stay BF16/FP32
|
||||
— no FP8 activation quantization, unlike PerTensorTorchFP8ScaledMMLinearKernel.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_cpu():
|
||||
return False, "requires CPU platform."
|
||||
if not torch.cpu._is_amx_tile_supported():
|
||||
return False, "requires AMX tile support (Sapphire Rapids or newer)."
|
||||
if not ops._supports_cpu_fp8_w8a16:
|
||||
return False, "fp8_scaled_mm_cpu op not available."
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: FP8ScaledMMLinearLayerConfig) -> tuple[bool, str | None]:
|
||||
if not c.weight_quant_key.scale.group_shape.is_per_tensor():
|
||||
return False, "requires a per-tensor weight scale."
|
||||
if not c.activation_quant_key.scale.group_shape.is_per_tensor():
|
||||
return False, "requires a per-tensor activation scale."
|
||||
if c.out_dtype not in (torch.bfloat16, torch.float32):
|
||||
return False, "Only bfloat16/float32 output dtype supported."
|
||||
n = c.weight_shape[0]
|
||||
if n % 32 != 0:
|
||||
# The AMX tinygemm kernel tiles N in chunks of 32 and cannot
|
||||
# handle a remainder tile smaller than that.
|
||||
return False, f"requires weight output dim (N={n}) to be a multiple of 32."
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
w_name, w_s_name, _, _ = self.layer_param_names
|
||||
# Fp8LinearMethod transposes weight to (K, N) for torch._scaled_mm-style
|
||||
# kernels; convert_weight_packed/fp8_scaled_mm_cpu expect (N, K), as
|
||||
# produced by CUDA nn.Linear-style checkpoints.
|
||||
weight = getattr(layer, w_name).t().contiguous()
|
||||
n, k = weight.shape
|
||||
|
||||
packed_weight = torch.ops._C.convert_weight_packed(weight)
|
||||
replace_parameter(
|
||||
layer, w_name, torch.nn.Parameter(packed_weight, requires_grad=False)
|
||||
)
|
||||
|
||||
# Synthesize a single-block "block scale" tensor from the scalar
|
||||
# per-tensor weight scale so fp8_scaled_mm_cpu can be reused as-is.
|
||||
weight_scale = getattr(layer, w_s_name)
|
||||
self._block_size_n = -(-n // 32) * 32 # round up to a multiple of 32
|
||||
num_k_blocks = -(-k // _BLOCK_SIZE_K)
|
||||
block_scale = weight_scale.reshape(()).expand(1, num_k_blocks).contiguous()
|
||||
replace_parameter(
|
||||
layer, w_s_name, torch.nn.Parameter(block_scale, requires_grad=False)
|
||||
)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
w_name, w_s_name, _, _ = self.layer_param_names
|
||||
weight = getattr(layer, w_name)
|
||||
weight_scale = getattr(layer, w_s_name)
|
||||
|
||||
x_2d = x.reshape(-1, x.shape[-1]) if x.dim() > 2 else x
|
||||
out = torch.ops._C.fp8_scaled_mm_cpu(
|
||||
x_2d,
|
||||
weight,
|
||||
weight_scale,
|
||||
[self._block_size_n, _BLOCK_SIZE_K],
|
||||
bias,
|
||||
x.dtype,
|
||||
True, # is_vnni (weight already prepacked)
|
||||
)
|
||||
return out.reshape(x.shape[:-1] + (out.size(-1),)) if x.dim() > 2 else out
|
||||
|
||||
def apply_scaled_mm(
|
||||
self,
|
||||
*,
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
As: torch.Tensor,
|
||||
Bs: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
output_shape: list,
|
||||
) -> torch.Tensor:
|
||||
raise NotImplementedError(
|
||||
"CPUFp8PerTensorScaledMMLinearKernel overrides apply_weights directly."
|
||||
)
|
||||
|
||||
@@ -615,14 +615,15 @@ class RoutedExperts(PluggableLayer):
|
||||
# compressed-tensors checkpoints with packed weights are stored flipped
|
||||
# TODO (mgoin): check self.quant_method.quant_config.quant_format
|
||||
# against known CompressionFormat enum values that have this quality
|
||||
if quant_method_name in (
|
||||
"CompressedTensorsWNA16MoEMethod",
|
||||
"CompressedTensorsW4A16FlydslMoEMethod",
|
||||
if (
|
||||
quant_method_name
|
||||
in (
|
||||
"CompressedTensorsWNA16MoEMethod",
|
||||
"CompressedTensorsW4A16FlydslMoEMethod",
|
||||
)
|
||||
and is_transposed
|
||||
):
|
||||
if is_transposed:
|
||||
loaded_weight = loaded_weight.t().contiguous()
|
||||
else:
|
||||
loaded_weight = loaded_weight
|
||||
loaded_weight = loaded_weight.t().contiguous()
|
||||
|
||||
if shard_id not in ("w1", "w2", "w3"):
|
||||
raise ValueError(f"shard_id must be ['w1','w2','w3'] but got {shard_id}.")
|
||||
|
||||
@@ -727,11 +727,15 @@ def _get_available_ram_bytes() -> int:
|
||||
|
||||
host_available = psutil.virtual_memory().available
|
||||
|
||||
from vllm.utils.cpu_resource_utils import get_cgroup_memory_limit
|
||||
from vllm.utils.cpu_resource_utils import (
|
||||
get_cgroup_memory_limit,
|
||||
get_cgroup_memory_usage,
|
||||
)
|
||||
|
||||
cgroup_limit, cgroup_usage = get_cgroup_memory_limit()
|
||||
cgroup_limit = get_cgroup_memory_limit()
|
||||
if cgroup_limit is None:
|
||||
return host_available
|
||||
cgroup_usage = get_cgroup_memory_usage()
|
||||
cgroup_available = (
|
||||
cgroup_limit if cgroup_usage is None else max(0, cgroup_limit - cgroup_usage)
|
||||
)
|
||||
|
||||
@@ -780,9 +780,7 @@ class DeepCLIPVisionEmbeddings(CLIPVisionEmbeddings):
|
||||
self, pixel_values: torch.Tensor, patch_embeds: torch.Tensor | None = None
|
||||
) -> torch.Tensor:
|
||||
batch_size = pixel_values.shape[0]
|
||||
if patch_embeds is not None:
|
||||
patch_embeds = patch_embeds
|
||||
else:
|
||||
if patch_embeds is None:
|
||||
patch_embeds = self.patch_embedding(pixel_values)
|
||||
patch_embeds = patch_embeds.flatten(2).transpose(1, 2)
|
||||
|
||||
|
||||
@@ -964,7 +964,6 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
|
||||
assert isinstance(va_item, (list, tuple)) and len(va_item) == 2
|
||||
vid, audio_src = va_item
|
||||
va_item = VideoAudioInput(video=vid, audio=audio_src)
|
||||
vid = vid
|
||||
# Convert video frames to (TCHW, timestamps) if needed
|
||||
if (
|
||||
isinstance(vid, tuple)
|
||||
|
||||
@@ -253,6 +253,7 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessing
|
||||
hf_processor_mm_kwargs = dict(
|
||||
**hf_processor_mm_kwargs,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
truncation=True,
|
||||
)
|
||||
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
|
||||
@@ -173,7 +173,6 @@ class Step3p5Attention(nn.Module):
|
||||
rope_scaling = None
|
||||
|
||||
if sliding_window is not None and enable_sliding_window:
|
||||
sliding_window = sliding_window
|
||||
if swa_num_attention_heads is not None:
|
||||
num_heads = swa_num_attention_heads
|
||||
self.total_num_heads = swa_num_attention_heads
|
||||
|
||||
@@ -204,6 +204,7 @@ class DeepseekV4MegaMoEExperts(nn.Module):
|
||||
self.top_k = top_k
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.unpadded_intermediate_size = intermediate_size
|
||||
self.num_shared_experts = num_shared_experts
|
||||
self.max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
|
||||
@@ -386,6 +387,17 @@ class DeepseekV4MegaMoEExperts(nn.Module):
|
||||
# the generic linear post-load hook replaces the raw checkpoint scales
|
||||
# with its 128x128 DeepGEMM layout.
|
||||
checkpoint_scale_dtypes = (torch.float8_e8m0fnu, torch.uint8)
|
||||
unpadded_size = self.unpadded_intermediate_size * self.num_shared_experts
|
||||
padding = self.intermediate_size * self.num_shared_experts - unpadded_size
|
||||
pad_weights = (
|
||||
padding > 0
|
||||
and gate_up_weight.dtype == torch.float8_e4m3fn
|
||||
and down_weight.dtype == torch.float8_e4m3fn
|
||||
and gate_up_weight.shape == (2 * unpadded_size, self.hidden_size)
|
||||
and down_weight.shape == (self.hidden_size, unpadded_size)
|
||||
and gate_up_scale.dtype in checkpoint_scale_dtypes
|
||||
and down_scale.dtype in checkpoint_scale_dtypes
|
||||
)
|
||||
if (
|
||||
gate_up_scale.dtype in checkpoint_scale_dtypes
|
||||
and down_scale.dtype in checkpoint_scale_dtypes
|
||||
@@ -396,6 +408,7 @@ class DeepseekV4MegaMoEExperts(nn.Module):
|
||||
gate_up_scale,
|
||||
gate_up_weight.shape[0],
|
||||
gate_up_weight.shape[1],
|
||||
padding=(padding, 0) if pad_weights else (0, 0),
|
||||
)
|
||||
down_scale = self._prepare_shared_expert_scale(
|
||||
deep_gemm,
|
||||
@@ -403,12 +416,27 @@ class DeepseekV4MegaMoEExperts(nn.Module):
|
||||
down_scale,
|
||||
down_weight.shape[0],
|
||||
down_weight.shape[1],
|
||||
padding=(0, padding) if pad_weights else (0, 0),
|
||||
)
|
||||
|
||||
if gate_up_scale is None or down_scale is None:
|
||||
self.num_shared_experts = 0
|
||||
return
|
||||
|
||||
if pad_weights:
|
||||
# Pad gate/up separately so the SwiGLU split stays at the midpoint.
|
||||
gate_up_weight = (
|
||||
torch.nn.functional.pad(
|
||||
gate_up_weight.view(torch.uint8).unflatten(0, (2, unpadded_size)),
|
||||
(0, 0, 0, padding),
|
||||
)
|
||||
.flatten(0, 1)
|
||||
.view(gate_up_weight.dtype)
|
||||
)
|
||||
down_weight = torch.nn.functional.pad(
|
||||
down_weight.view(torch.uint8), (0, padding)
|
||||
).view(down_weight.dtype)
|
||||
|
||||
shared_intermediate_size = self.intermediate_size * self.num_shared_experts
|
||||
expected_gate_up_shape = (
|
||||
2 * shared_intermediate_size,
|
||||
@@ -449,11 +477,11 @@ class DeepseekV4MegaMoEExperts(nn.Module):
|
||||
# released instead of adding roughly 0.7 GiB per rank on DSV4-Flash.
|
||||
# The generic linear post-load hook may still repack the serial scales,
|
||||
# but this shared MLP is never called after native fusion is enabled.
|
||||
gate_up.weight.data = transformed_l1[0]
|
||||
self._transformed_shared_l1_weights = (
|
||||
gate_up.weight.data,
|
||||
transformed_l1[1],
|
||||
)
|
||||
# Padded weights need separate storage: the generic linear post-load
|
||||
# hooks still expect the original checkpoint weight/scale shapes.
|
||||
if not pad_weights:
|
||||
gate_up.weight.data = transformed_l1[0]
|
||||
self._transformed_shared_l1_weights = transformed_l1
|
||||
self._transformed_shared_l2_weights = transformed_l2
|
||||
|
||||
def _prepare_shared_expert_scale(
|
||||
@@ -463,6 +491,8 @@ class DeepseekV4MegaMoEExperts(nn.Module):
|
||||
scale: torch.Tensor,
|
||||
mn: int,
|
||||
k: int,
|
||||
*,
|
||||
padding: tuple[int, int] = (0, 0),
|
||||
) -> torch.Tensor | None:
|
||||
block_size = getattr(linear, "weight_block_size", None)
|
||||
if block_size is None or len(block_size) != 2:
|
||||
@@ -497,6 +527,19 @@ class DeepseekV4MegaMoEExperts(nn.Module):
|
||||
.repeat_interleave(block_k // 32, dim=1)[:mn, : k // 32]
|
||||
.contiguous()
|
||||
)
|
||||
pad_m, pad_k = padding
|
||||
if pad_m:
|
||||
scale_1x32 = torch.nn.functional.pad(
|
||||
scale_1x32.unflatten(0, (2, mn // 2)),
|
||||
(0, 0, 0, pad_m),
|
||||
value=1.0,
|
||||
).flatten(0, 1)
|
||||
mn += 2 * pad_m
|
||||
if pad_k:
|
||||
scale_1x32 = torch.nn.functional.pad(
|
||||
scale_1x32, (0, pad_k // 32), value=1.0
|
||||
)
|
||||
k += pad_k
|
||||
# The grouped API is used with a singleton dimension to request the
|
||||
# MN-major, TMA-aligned packed-UE8M0 strides, then squeezed back to the
|
||||
# 2D layout required for a shared expert.
|
||||
|
||||
@@ -636,7 +636,8 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
|
||||
transpose_scale=False,
|
||||
)
|
||||
|
||||
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
||||
def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
||||
o = attn_out[:, : self.n_local_heads, :]
|
||||
# ROCm BF16 reference wo_a path (inverse RoPE + einsum) + wo_b.
|
||||
z = rocm_inv_rope_einsum(
|
||||
self.rotary_emb,
|
||||
|
||||
@@ -44,6 +44,7 @@ from vllm.config import (
|
||||
VllmConfig,
|
||||
get_current_vllm_config,
|
||||
)
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
@@ -127,34 +128,40 @@ def _use_v41_mxfp8_kv_record() -> bool:
|
||||
|
||||
def _resolve_dsv4_kv_cache_dtype(
|
||||
use_fp8_ds_mla_layout: bool,
|
||||
kv_cache_dtype: str,
|
||||
kv_cache_dtype: CacheDType,
|
||||
cache_config: CacheConfig | None,
|
||||
) -> tuple[str, torch.dtype]:
|
||||
packed_kv_cache_dtype: CacheDType = "fp8_ds_mla",
|
||||
) -> tuple[CacheDType, torch.dtype]:
|
||||
"""Map ``(layout, --kv-cache-dtype)`` to ``(cache_dtype_str, torch_dtype)``.
|
||||
|
||||
Both layouts are paged; they differ in the per-token block format. The
|
||||
``fp8_ds_mla`` format is UE8M0 block-scaled fp8 packed as ``uint8`` (the
|
||||
canonical ``fp8_ds_mla`` string is written back onto ``cache_config`` so the
|
||||
page-size specs pick the 576B per-token slot). Plain-row backends store each
|
||||
token's KV row in its element dtype: bf16 or per-tensor FP8 E4M3.
|
||||
packed formats are ``uint8``-backed: ``fp8_ds_mla`` is UE8M0 block-scaled
|
||||
fp8 throughout, ``nvfp4_ds_mla`` keeps that sliding-window record and
|
||||
stores the compressed cache as NVFP4. An unspecific ``--kv-cache-dtype``
|
||||
(``auto`` / ``fp8``) resolves to ``packed_kv_cache_dtype``, the record this
|
||||
layer's kernel prefers, and the canonical string is written back onto
|
||||
``cache_config`` so the page-size specs pick the right per-token slot.
|
||||
Plain-row backends store each token's KV row in its element dtype: bf16 or
|
||||
per-tensor FP8 E4M3.
|
||||
"""
|
||||
if use_fp8_ds_mla_layout:
|
||||
# fp8_ds_mla block format: UE8M0 block-scaled fp8 packed as uint8.
|
||||
if kv_cache_dtype == "auto":
|
||||
kv_cache_dtype = "fp8"
|
||||
if not kv_cache_dtype.startswith("fp8"):
|
||||
if kv_cache_dtype in ("auto", "fp8"):
|
||||
kv_cache_dtype = packed_kv_cache_dtype
|
||||
elif not kv_cache_dtype.endswith("_ds_mla"):
|
||||
raise ValueError(
|
||||
"DeepseekV4 fp8_ds_mla layout only supports fp8 "
|
||||
f"kv-cache, got {kv_cache_dtype}. Please set "
|
||||
"`--kv-cache-dtype fp8` or select a backend that supports "
|
||||
"bfloat16 KV cache."
|
||||
"DeepseekV4 packed KV layouts only support fp8 kv-cache, got "
|
||||
f"{kv_cache_dtype}. Please set `--kv-cache-dtype fp8` or "
|
||||
"select a backend that supports bfloat16 KV cache."
|
||||
)
|
||||
if kv_cache_dtype != "fp8_ds_mla":
|
||||
if cache_config is not None:
|
||||
cache_config.cache_dtype = "fp8_ds_mla"
|
||||
kv_cache_dtype = "fp8_ds_mla"
|
||||
logger.info_once("Using DeepSeek's fp8_ds_mla KV cache format.")
|
||||
if cache_config is not None and cache_config.cache_dtype != kv_cache_dtype:
|
||||
cache_config.cache_dtype = kv_cache_dtype
|
||||
logger.info_once("Using DeepSeek's %s KV cache format.", kv_cache_dtype)
|
||||
return kv_cache_dtype, torch.uint8
|
||||
if kv_cache_dtype.endswith("_ds_mla"):
|
||||
raise ValueError(
|
||||
f"{kv_cache_dtype} is a packed FlashMLA DeepSeek V4.1 KV cache "
|
||||
"format; the selected backend stores plain KV rows."
|
||||
)
|
||||
|
||||
# Plain bf16 / per-tensor fp8 KV row (FlashInfer).
|
||||
if kv_cache_dtype.startswith("fp8"):
|
||||
@@ -189,6 +196,28 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
# path to pre-reserve that workspace.
|
||||
PREFILL_CHUNK_SIZE: ClassVar[int] = 4
|
||||
|
||||
# ---- attention-interface contract, declared by the platform subclass ----
|
||||
|
||||
@property
|
||||
def accepts_unnormed_unroped_query(self) -> bool:
|
||||
"""Whether ``forward_mqa``'s ``q`` is the raw ``wq_b`` output.
|
||||
|
||||
True when the attention kernel applies the Q norm and RoPE itself, and
|
||||
reads Q in its own chunk-interleaved layout, so the layer only
|
||||
zero-pads Q to ``padded_heads`` and inserts KV before calling it.
|
||||
"""
|
||||
return False
|
||||
|
||||
@property
|
||||
def packed_kv_cache_dtype(self) -> CacheDType:
|
||||
"""The packed KV record this layer's kernel prefers.
|
||||
|
||||
What an unspecific ``--kv-cache-dtype`` (``auto`` / ``fp8``) resolves
|
||||
to. Mega attention overrides it: its kernel is the one that can read
|
||||
an NVFP4 compressed cache.
|
||||
"""
|
||||
return "fp8_ds_mla"
|
||||
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def get_padded_num_q_heads(cls, num_heads: int) -> int:
|
||||
@@ -213,8 +242,19 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
||||
"""Inverse-RoPE + wo_a + wo_b output projection (platform-specific)."""
|
||||
def _o_proj(
|
||||
self,
|
||||
attn_out: "torch.Tensor | QuantizedActivation",
|
||||
positions: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Project whatever ``_alloc_attn_out`` produced through wo_a and wo_b.
|
||||
|
||||
Takes the buffer whole, so each layer owns the shape it allocated: the
|
||||
bf16 layers slice off their padding heads and apply the inverse RoPE,
|
||||
while a layer whose attention kernel already did the inverse RoPE and
|
||||
the FP8 cast gets a QuantizedActivation and has only wo_a and wo_b
|
||||
left.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def _uses_fp8_ds_mla_layout(self) -> bool:
|
||||
@@ -459,11 +499,27 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
# Resolve the kv-cache dtype from this backend's block format. The same
|
||||
# resolution drives the SWA cache tensor dtype below.
|
||||
self.kv_cache_dtype, self.kv_cache_torch_dtype = _resolve_dsv4_kv_cache_dtype(
|
||||
self._uses_fp8_ds_mla_layout(), cache_config.cache_dtype, cache_config
|
||||
self._uses_fp8_ds_mla_layout(),
|
||||
cache_config.cache_dtype,
|
||||
cache_config,
|
||||
self.packed_kv_cache_dtype,
|
||||
)
|
||||
self.kv_mxfp8 = _use_v41_mxfp8_kv_record()
|
||||
self.kv_bytes_per_token = 528 if self.kv_mxfp8 else 584
|
||||
self.swa_bytes_per_token = 528 if self.kv_mxfp8 else 584
|
||||
# nvfp4_ds_mla keeps the MXFP8 sliding-window record and stores the
|
||||
# compressed cache as NVFP4 (256 B of e2m1 pairs + 32 e4m3 scales).
|
||||
self.compressed_bytes_per_token = (
|
||||
288 if self.kv_cache_dtype == "nvfp4_ds_mla" else self.swa_bytes_per_token
|
||||
)
|
||||
# One alignment for every page in the block: the block stride is their
|
||||
# sum, and 512 satisfies both TMA strides in play (512 for the V4.1
|
||||
# fp8 record, 256 for NVFP4).
|
||||
self.kv_page_alignment = 512 if self.kv_mxfp8 else 576
|
||||
if self.kv_cache_dtype == "nvfp4_ds_mla" and not self.kv_mxfp8:
|
||||
raise ValueError(
|
||||
"nvfp4_ds_mla needs the V4.1 KV records, which FlashMLA "
|
||||
"decodes only on SM100."
|
||||
)
|
||||
|
||||
self.swa_cache_layer = DeepseekV4SWACache(
|
||||
head_dim=self.head_dim,
|
||||
@@ -473,7 +529,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
cache_config=cache_config,
|
||||
backend_cls=self.swa_backend_cls,
|
||||
block_size=32,
|
||||
packed_bytes_per_token=self.kv_bytes_per_token,
|
||||
packed_bytes_per_token=self.swa_bytes_per_token,
|
||||
packed_page_alignment=self.kv_page_alignment,
|
||||
)
|
||||
|
||||
@@ -553,8 +609,12 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
block_size=self.swa_cache_layer.block_size,
|
||||
)
|
||||
|
||||
# Every backend that gathers a chunk's KV through
|
||||
# combine_topk_swa_indices needs its Triton kernel warmed, mega
|
||||
# attention included -- it calls it from _forward_prefill_mega.
|
||||
if self.backend_cls.get_name() in (
|
||||
"FLASHMLA_SPARSE_DSV41",
|
||||
"FLASHMLA_MEGA_ATTN_DSV41",
|
||||
"ROCM_FLASHMLA_SPARSE_DSV4",
|
||||
):
|
||||
from vllm.models.deepseek_v41.common.ops.cache_utils import (
|
||||
@@ -569,14 +629,10 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
hidden_states: torch.Tensor,
|
||||
llama_4_scaling: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
# Pre-allocate attention output with FlashMLA-padded head count.
|
||||
# The op writes into `o_padded`; we slice to n_local_heads after.
|
||||
num_tokens = hidden_states.shape[0]
|
||||
o_padded = torch.empty(
|
||||
(num_tokens, self.padded_heads, self.head_dim),
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
# The eager attention region writes into a caller-owned buffer
|
||||
# (breakable_cudagraph needs in-place outputs); its shape and how it is
|
||||
# projected afterwards follow the interface contract above.
|
||||
attn_out = self._alloc_attn_out(hidden_states.shape[0], hidden_states)
|
||||
|
||||
# Keep the attention input preparation in the captured graph. Only the
|
||||
# sparse indexer and MLA attention run in the eager break below.
|
||||
@@ -593,12 +649,19 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
kv_score,
|
||||
indexer_weights,
|
||||
positions,
|
||||
o_padded,
|
||||
attn_out,
|
||||
)
|
||||
o = o_padded[:, : self.n_local_heads, :]
|
||||
return self._o_proj(attn_out, positions)
|
||||
|
||||
# Inverse-RoPE + wo_a + wo_b output projection (platform-specific).
|
||||
return self._o_proj(o, positions)
|
||||
def _alloc_attn_out(
|
||||
self, num_tokens: int, hidden_states: torch.Tensor
|
||||
) -> "torch.Tensor | QuantizedActivation":
|
||||
"""The buffer ``forward_mqa`` fills, per the interface contract."""
|
||||
return torch.empty(
|
||||
(num_tokens, self.padded_heads, self.head_dim),
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def _can_fuse_query_quant(self) -> bool:
|
||||
@@ -652,7 +715,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
kv_score: torch.Tensor,
|
||||
indexer_weights: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
o_padded: torch.Tensor,
|
||||
attn_out: "torch.Tensor | QuantizedActivation",
|
||||
) -> None:
|
||||
"""Wide eager region: the whole of ``_prepare_and_attn`` runs eagerly.
|
||||
|
||||
@@ -667,7 +730,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
kv_score,
|
||||
indexer_weights,
|
||||
positions,
|
||||
o_padded,
|
||||
attn_out,
|
||||
)
|
||||
|
||||
def _prepare_and_attn(
|
||||
@@ -679,7 +742,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
kv_score: torch.Tensor,
|
||||
indexer_weights: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
o_padded: torch.Tensor,
|
||||
attn_out: "torch.Tensor | QuantizedActivation",
|
||||
) -> None:
|
||||
"""Attention input preparation followed by the sparse indexer and MLA.
|
||||
|
||||
@@ -751,7 +814,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
q,
|
||||
kv,
|
||||
positions,
|
||||
o_padded,
|
||||
attn_out,
|
||||
)
|
||||
|
||||
def _fused_wqa_wkv_gemm(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
@@ -858,16 +921,28 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
dict[str, AttentionMetadata] | list[dict[str, AttentionMetadata]] | None
|
||||
),
|
||||
) -> torch.Tensor:
|
||||
"""Ready ``q`` for the attention kernel and publish this step's KV.
|
||||
|
||||
One launch does both. With ``accepts_unnormed_unroped_query`` the
|
||||
attention kernel norms and rotates Q itself and reads it in its own
|
||||
chunk-interleaved layout, so the Q half of the launch is a zero-pad to
|
||||
``padded_heads`` -- and nothing at all once the shard is that wide.
|
||||
"""
|
||||
if not isinstance(attn_metadata, dict):
|
||||
# Profile run: kernel doesn't fire; produce a padded tensor so
|
||||
# downstream FlashMLA gets the right shape.
|
||||
if self.n_local_heads < self.padded_heads:
|
||||
return F.pad(
|
||||
q,
|
||||
(0, 0, 0, self.padded_heads - self.n_local_heads),
|
||||
value=0.0,
|
||||
)
|
||||
return q
|
||||
if self.n_local_heads >= self.padded_heads:
|
||||
return q
|
||||
if self.accepts_unnormed_unroped_query:
|
||||
# Padding heads sit at the tail of every head-dim chunk in
|
||||
# that layout, so no head-major pad of `q` reproduces it --
|
||||
# and nothing reads it on a profile run.
|
||||
return q.new_zeros((q.shape[0], self.padded_heads, q.shape[2]))
|
||||
return F.pad(
|
||||
q,
|
||||
(0, 0, 0, self.padded_heads - self.n_local_heads),
|
||||
value=0.0,
|
||||
)
|
||||
|
||||
swa_metadata = cast(
|
||||
"DeepseekSparseSWAMetadata | None",
|
||||
@@ -886,23 +961,38 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
if cache_dtype == torch.uint8:
|
||||
# fp8_ds_mla UE8M0 paged path. Horizontally fused:
|
||||
# Q side: GPT-J RoPE, zero-filling the padding head slots; the
|
||||
# kernel allocates and returns the padded q tensor.
|
||||
# kernel allocates and returns the padded q tensor. An
|
||||
# interleaved Q skips the RoPE its attention kernel owns
|
||||
# and keeps only the pad, which q_head_padded=0 drops
|
||||
# too once the shard is already padded_heads wide.
|
||||
# KV side: GPT-J RoPE + UE8M0 FP8 quant + paged cache insert.
|
||||
swa_kv_cache_2d = swa_kv_cache.view(swa_kv_cache.shape[0], -1)
|
||||
return torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
pad_to = (
|
||||
0
|
||||
if self.accepts_unnormed_unroped_query
|
||||
and self.n_local_heads == self.padded_heads
|
||||
else self.padded_heads
|
||||
)
|
||||
q_padded = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
q,
|
||||
kv,
|
||||
swa_kv_cache_2d,
|
||||
swa_metadata.slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
self.padded_heads,
|
||||
pad_to,
|
||||
self.eps,
|
||||
swa_metadata.block_size,
|
||||
False,
|
||||
False, # apply_q_norm: qr is normed before wq_b
|
||||
self.kv_mxfp8,
|
||||
not self.accepts_unnormed_unroped_query, # apply_q_rope
|
||||
self.accepts_unnormed_unroped_query, # is_q_interleaved
|
||||
)
|
||||
return q if pad_to == 0 else q_padded
|
||||
|
||||
assert not self.accepts_unnormed_unroped_query, (
|
||||
"the chunk-interleaved Q layout only pairs with a packed KV record"
|
||||
)
|
||||
# Plain-row path: the [num_blocks, block_size, 512] cache stores the KV
|
||||
# row in its element dtype (no Q padding). bf16 rewrites q in place;
|
||||
# per-tensor fp8 writes a separately-allocated fp8 q and quantizes the
|
||||
@@ -959,7 +1049,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
# fp8_ds_mla is a UE8M0 block-scaled uint8 layout whose page rounds up
|
||||
# to the decode kernel's TMA stride; plain bf16 / per-tensor fp8 rows
|
||||
# use natural element-size pages.
|
||||
uses_fp8_ds_mla_layout = self.kv_cache_dtype == "fp8_ds_mla"
|
||||
uses_fp8_ds_mla_layout = self.kv_cache_dtype in ("fp8_ds_mla", "nvfp4_ds_mla")
|
||||
return MLAAttentionSpec(
|
||||
block_size=vllm_config.cache_config.block_size,
|
||||
num_kv_heads=1,
|
||||
@@ -972,7 +1062,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
|
||||
# Packed record width; head_size stays semantic (512).
|
||||
state_content_bytes=(
|
||||
self.kv_bytes_per_token if uses_fp8_ds_mla_layout else None
|
||||
self.compressed_bytes_per_token if uses_fp8_ds_mla_layout else None
|
||||
),
|
||||
)
|
||||
|
||||
@@ -1016,7 +1106,13 @@ class DeepseekV4IndexerCache(torch.nn.Module, AttentionLayerBase):
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
|
||||
# head_dim already carries the fp8 scale padding
|
||||
# tokens_per_state=1 for V3.2, >1 for DeepseekV4; same cache layout.
|
||||
uses_fp8_ds_mla_layout = vllm_config.cache_config.cache_dtype == "fp8_ds_mla"
|
||||
# nvfp4_ds_mla is packed too: its compressed record is NVFP4 but the
|
||||
# sliding-window record stays the V4.1 MXFP8 one, so the indexer page
|
||||
# takes the same alignment either way.
|
||||
uses_fp8_ds_mla_layout = vllm_config.cache_config.cache_dtype in (
|
||||
"fp8_ds_mla",
|
||||
"nvfp4_ds_mla",
|
||||
)
|
||||
page_alignment = (
|
||||
576 if uses_fp8_ds_mla_layout and not _use_v41_mxfp8_kv_record() else 512
|
||||
)
|
||||
|
||||
@@ -43,10 +43,16 @@ from vllm.utils.math_utils import next_power_of_2
|
||||
# dims each plus a pad byte.
|
||||
# V4.1 (528 B): all 512 dims as fp8 e4m3 (RoPE included), then 16 UE8M0
|
||||
# scales of 32 dims each. FlashMLA's ``ModelType::V41``.
|
||||
# V4.1 NVFP4 (288 B): 512 e2m1 values packed two per byte (even element in
|
||||
# the low nibble), then 32 e4m3 scales of 16 dims each.
|
||||
# FlashMLA's ``ModelType::V41_FP4``, compressed cache only.
|
||||
V4_BYTES_PER_TOKEN = 584
|
||||
V41_BYTES_PER_TOKEN = 528
|
||||
V41_QUANT_BLOCK = 32
|
||||
V41_NUM_SCALES = 512 // V41_QUANT_BLOCK # 16
|
||||
V41_NVFP4_BYTES_PER_TOKEN = 288
|
||||
V41_NVFP4_QUANT_BLOCK = 16
|
||||
V41_NVFP4_NUM_SCALES = 512 // V41_NVFP4_QUANT_BLOCK # 32
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -496,6 +502,73 @@ def _dequantize_and_gather_k_mxfp8_kernel(
|
||||
tl.store(output_row_ptr + d, dequant.to(tl.bfloat16))
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _dequantize_and_gather_k_nvfp4_kernel(
|
||||
out_ptr,
|
||||
out_stride0,
|
||||
out_stride1,
|
||||
k_cache_ptr,
|
||||
seq_lens_ptr,
|
||||
block_table_ptr,
|
||||
offset,
|
||||
gather_lens_ptr,
|
||||
max_blocks_per_seq: tl.constexpr,
|
||||
head_dim: tl.constexpr, # 512
|
||||
scale_dim: tl.constexpr, # 32
|
||||
quant_block: tl.constexpr, # 16
|
||||
cache_block_size: tl.constexpr,
|
||||
block_stride: tl.constexpr,
|
||||
):
|
||||
"""Gather and dequantize V4.1 NVFP4 rows into a bf16 workspace."""
|
||||
batch_idx = tl.program_id(0)
|
||||
worker_id = tl.program_id(1)
|
||||
num_workers = tl.num_programs(1)
|
||||
|
||||
seq_len = tl.load(seq_lens_ptr + batch_idx)
|
||||
if gather_lens_ptr is not None: # noqa: SIM108
|
||||
gather_len = tl.load(gather_lens_ptr + batch_idx)
|
||||
else:
|
||||
gather_len = seq_len
|
||||
start_pos = seq_len - gather_len
|
||||
|
||||
packed_bytes: tl.constexpr = head_dim // 2
|
||||
d = tl.arange(0, head_dim)
|
||||
for i in range(worker_id, gather_len, num_workers):
|
||||
pos = start_pos + i
|
||||
block_in_seq = pos // cache_block_size
|
||||
pos_in_block = pos % cache_block_size
|
||||
|
||||
physical_block_idx = tl.load(
|
||||
block_table_ptr + batch_idx * max_blocks_per_seq + block_in_seq
|
||||
)
|
||||
page = k_cache_ptr + physical_block_idx.to(tl.int64) * block_stride
|
||||
|
||||
packed = tl.load(
|
||||
page + pos_in_block * packed_bytes + tl.arange(0, packed_bytes)
|
||||
)
|
||||
# Even element in the low nibble, odd in the high nibble.
|
||||
codes = tl.interleave((packed & 0xF).to(tl.int32), (packed >> 4).to(tl.int32))
|
||||
# e2m1 magnitudes: 0, 0.5, 1, 1.5, 2, 3, 4, 6.
|
||||
mag_code = codes & 7
|
||||
e = (mag_code >> 1).to(tl.float32)
|
||||
m = (mag_code & 1).to(tl.float32)
|
||||
mag = tl.where(mag_code < 2, m * 0.5, (1.0 + m * 0.5) * tl.exp2(e - 1.0))
|
||||
vals = tl.where(codes >= 8, -mag, mag)
|
||||
|
||||
sf = tl.load(
|
||||
page
|
||||
+ cache_block_size * packed_bytes
|
||||
+ pos_in_block * scale_dim
|
||||
+ tl.arange(0, scale_dim)
|
||||
)
|
||||
scale = sf.to(tl.float8e4nv, bitcast=True).to(tl.float32)
|
||||
tiles = tl.reshape(vals, (scale_dim, quant_block))
|
||||
dequant = tl.reshape(tiles * tl.reshape(scale, (scale_dim, 1)), (head_dim,))
|
||||
|
||||
output_row_ptr = out_ptr + batch_idx * out_stride0 + (offset + i) * out_stride1
|
||||
tl.store(output_row_ptr + d, dequant.to(tl.bfloat16))
|
||||
|
||||
|
||||
def dequantize_and_gather_k_cache_triton(
|
||||
# [num_reqs, max_num_tokens, head_size]
|
||||
out: torch.Tensor,
|
||||
@@ -513,6 +586,25 @@ def dequantize_and_gather_k_cache_triton(
|
||||
) -> None:
|
||||
num_reqs = seq_lens.shape[0]
|
||||
NUM_WORKERS = 128
|
||||
if k_cache.shape[-1] == V41_NVFP4_BYTES_PER_TOKEN:
|
||||
_dequantize_and_gather_k_nvfp4_kernel[(num_reqs, NUM_WORKERS)](
|
||||
out,
|
||||
out.stride(0),
|
||||
out.stride(1),
|
||||
k_cache,
|
||||
seq_lens,
|
||||
block_table,
|
||||
offset,
|
||||
gather_lens,
|
||||
max_blocks_per_seq=block_table.shape[-1],
|
||||
head_dim=512,
|
||||
scale_dim=V41_NVFP4_NUM_SCALES,
|
||||
quant_block=V41_NVFP4_QUANT_BLOCK,
|
||||
cache_block_size=block_size,
|
||||
block_stride=k_cache.stride(0),
|
||||
)
|
||||
return
|
||||
|
||||
if k_cache.shape[-1] == V41_BYTES_PER_TOKEN:
|
||||
_dequantize_and_gather_k_mxfp8_kernel[(num_reqs, NUM_WORKERS)](
|
||||
out,
|
||||
@@ -581,14 +673,16 @@ def dequantize_and_gather_k_cache(
|
||||
) -> None:
|
||||
"""Dequantize and gather a paged DSv4 K cache.
|
||||
|
||||
The record is read off ``k_cache.shape[-1]``; see the module header.
|
||||
The record is read off ``k_cache.shape[-1]``; see the module header. Only
|
||||
the fp8 records have a CuteDSL gather, so NVFP4 always takes the Triton
|
||||
path.
|
||||
|
||||
``use_fnuz`` MUST match the encoder of the specific cache being read:
|
||||
``False`` for ``compressed_k_cache`` (Triton encoder is OCP everywhere),
|
||||
``current_platform.is_fp8_fnuz()`` for ``swa_k_cache`` (C++ encoder
|
||||
writes FNUZ on gfx942 and OCP on gfx950).
|
||||
"""
|
||||
if has_cutedsl():
|
||||
if has_cutedsl() and k_cache.shape[-1] != V41_NVFP4_BYTES_PER_TOKEN:
|
||||
# lazily import, otherwise some tests fail due to CUDA driver init failure.
|
||||
from vllm.models.deepseek_v4.nvidia.ops.dequant_gather_k_cutedsl import (
|
||||
_DEQUANT_GATHER_K_CACHE_CUTEDSL_KERNEL,
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.models.deepseek_v4.common.ops.fused_indexer_q import _fp32x2_to_fp4x2
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
@@ -232,13 +233,14 @@ def rope_quant_insert(
|
||||
|
||||
The BF16 latent supplies both NoPE quantization and RoPE input. It is read
|
||||
only for valid slots at group boundaries. The cache dtype selects the
|
||||
layout: ``uint8`` is the fp8_ds_mla paged layout, whose record the
|
||||
per-token byte width names -- 584 B for V4 (576 value bytes and eight
|
||||
segregated UE8M0 scale bytes, including one zero padding scale) or 528 B
|
||||
for V4.1 (512 MXFP8 value bytes covering the RoPE dims too, then 16 UE8M0
|
||||
scales of 32 dims each). ``bfloat16`` and ``float8_e4m3fn`` are the plain
|
||||
[448 NoPE | 64 RoPE] rows read by FlashInfer, the latter scaled by the
|
||||
per-tensor ``fp8_scale``.
|
||||
layout: ``uint8`` is a paged FlashMLA layout, whose record the per-token
|
||||
byte width names -- 584 B for V4 (576 value bytes and eight segregated
|
||||
UE8M0 scale bytes, including one zero padding scale), 528 B for V4.1
|
||||
(512 MXFP8 value bytes covering the RoPE dims too, then 16 UE8M0 scales of
|
||||
32 dims each), or 288 B for V4.1 NVFP4 (256 bytes of e2m1 pairs then 32
|
||||
e4m3 scales of 16 dims each), which only the compressed cache uses.
|
||||
``bfloat16`` and ``float8_e4m3fn`` are the plain [448 NoPE | 64 RoPE] rows
|
||||
read by FlashInfer, the latter scaled by the per-tensor ``fp8_scale``.
|
||||
"""
|
||||
assert compress_ratio in (1, 2)
|
||||
assert latent.shape[1] == 512 and latent.dtype == torch.bfloat16
|
||||
@@ -249,14 +251,14 @@ def rope_quant_insert(
|
||||
return
|
||||
launch_kwargs = {"launch_pdl": False} if current_platform.is_cuda() else {}
|
||||
if kv_cache.dtype == torch.uint8:
|
||||
assert kv_cache.shape[-1] in (584, 528), (
|
||||
kernel = {
|
||||
584: _rope_quant_insert_kernel,
|
||||
528: _rope_quant_insert_mxfp8_kernel,
|
||||
288: _rope_quant_insert_nvfp4_kernel,
|
||||
}.get(kv_cache.shape[-1])
|
||||
assert kernel is not None, (
|
||||
f"unsupported paged KV record width {kv_cache.shape[-1]}"
|
||||
)
|
||||
kernel = (
|
||||
_rope_quant_insert_mxfp8_kernel
|
||||
if kv_cache.shape[-1] == 528
|
||||
else _rope_quant_insert_kernel
|
||||
)
|
||||
kernel[(num_tokens,)](
|
||||
latent,
|
||||
positions,
|
||||
@@ -402,6 +404,62 @@ def _rope_quant_insert_mxfp8_kernel(
|
||||
tl.store(scales + tl.arange(0, 16), encoded.to(tl.uint8))
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _rope_quant_insert_nvfp4_kernel(
|
||||
latent,
|
||||
positions,
|
||||
cos_sin,
|
||||
cache,
|
||||
cache_slots,
|
||||
COS_STRIDE: tl.constexpr,
|
||||
CACHE_STRIDE: tl.constexpr,
|
||||
CACHE_BLOCK: tl.constexpr,
|
||||
COMPRESS_RATIO: tl.constexpr,
|
||||
SANITIZE_CACHE_NANS: tl.constexpr,
|
||||
):
|
||||
"""V4.1 NVFP4 record: RoPE, then e2m1 with one e4m3 scale per 16 dims.
|
||||
|
||||
The scale is ``amax / 6`` (6 is e2m1's largest magnitude) clamped to the
|
||||
e4m3 range, with no per-tensor scale on top.
|
||||
"""
|
||||
t = tl.program_id(0)
|
||||
slot = tl.load(cache_slots + t)
|
||||
if slot < 0:
|
||||
return
|
||||
position = tl.load(positions + t)
|
||||
if (position + 1) % COMPRESS_RATIO != 0:
|
||||
return
|
||||
d = tl.arange(0, 512)
|
||||
normed = tl.load(latent + t.to(tl.int64) * 512 + d).to(tl.float32)
|
||||
|
||||
# NoPE pairs load (cos, sin) = (1, 0), so the rotation is the identity there.
|
||||
even, odd = tl.split(tl.reshape(normed, (256, 2)))
|
||||
pair = tl.arange(0, 256) - 224
|
||||
cs = cos_sin + (position // COMPRESS_RATIO * COMPRESS_RATIO) * COS_STRIDE
|
||||
c = tl.load(cs + tl.maximum(pair, 0), pair >= 0, other=1.0).to(tl.float32)
|
||||
s = tl.load(cs + 32 + tl.maximum(pair, 0), pair >= 0, other=0.0).to(tl.float32)
|
||||
rotated = tl.interleave(even * c - odd * s, odd * c + even * s)
|
||||
if SANITIZE_CACHE_NANS:
|
||||
rotated = tl.where(rotated == rotated, rotated, 0.0)
|
||||
|
||||
tiles = tl.reshape(rotated, (32, 16))
|
||||
amax = tl.max(tl.abs(tiles), 1)
|
||||
# 2**-9 is the smallest normal e4m3 magnitude; 448 the largest.
|
||||
scale = tl.clamp(amax * (1.0 / 6.0), 0.001953125, 448.0).to(tl.float8e4nv)
|
||||
# Round-to-nearest division: Triton's default div.full misplaces values
|
||||
# that land exactly on an e2m1 tie.
|
||||
scaled = tl.math.div_rn(tiles, tl.reshape(scale.to(tl.float32), (32, 1)))
|
||||
lo, hi = tl.split(tl.reshape(tl.reshape(scaled, (512,)), (256, 2)))
|
||||
|
||||
page = cache + (slot // CACHE_BLOCK).to(tl.int64) * CACHE_STRIDE
|
||||
tl.store(
|
||||
page + (slot % CACHE_BLOCK) * 256 + tl.arange(0, 256),
|
||||
_fp32x2_to_fp4x2(lo, hi),
|
||||
)
|
||||
scales = page + CACHE_BLOCK * 256 + (slot % CACHE_BLOCK) * 32
|
||||
tl.store(scales + tl.arange(0, 32), scale.to(tl.uint8, bitcast=True))
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _rope_plain_insert_kernel(
|
||||
latent,
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Weight permutations for FlashMLA's mega-attention kernel.
|
||||
|
||||
The kernel reads Q with 16-element head-dim chunks interleaved across heads and
|
||||
writes O with 32-element chunks interleaved across the 8 heads of a ``wo_a``
|
||||
group. Every permutation here satisfies ``fused = standard[perm]``, and is
|
||||
applied once to ``wq_b`` rows and ``wo_a`` columns at load time so the
|
||||
surrounding GEMMs produce and consume the kernel's layouts directly -- no
|
||||
per-step shuffle.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
HEAD_DIM = 512
|
||||
Q_CHUNK = 16
|
||||
O_CHUNK = 32
|
||||
WV_GROUP_SIZE = 8
|
||||
|
||||
|
||||
def q_fused_permutation(num_heads: int, head_dim: int = HEAD_DIM) -> torch.Tensor:
|
||||
"""``fused[(d // 16) * (H * 16) + h * 16 + d % 16] = standard[h * D + d]``."""
|
||||
h = torch.arange(num_heads).view(num_heads, 1)
|
||||
d = torch.arange(head_dim).view(1, head_dim)
|
||||
fused_index = (d // Q_CHUNK) * (num_heads * Q_CHUNK) + h * Q_CHUNK + d % Q_CHUNK
|
||||
perm = torch.empty(num_heads * head_dim, dtype=torch.long)
|
||||
perm[fused_index.reshape(-1)] = torch.arange(num_heads * head_dim)
|
||||
return perm
|
||||
|
||||
|
||||
def o_fused_permutation(
|
||||
heads_per_group: int = WV_GROUP_SIZE, head_dim: int = HEAD_DIM
|
||||
) -> torch.Tensor:
|
||||
"""``fused[(c * G + h) * 32 + j] = standard[h * D + c * 32 + j]`` per group."""
|
||||
h = torch.arange(heads_per_group).view(-1, 1, 1)
|
||||
c = torch.arange(head_dim // O_CHUNK).view(1, -1, 1)
|
||||
j = torch.arange(O_CHUNK).view(1, 1, -1)
|
||||
fused_index = (c * heads_per_group + h) * O_CHUNK + j
|
||||
perm = torch.empty(heads_per_group * head_dim, dtype=torch.long)
|
||||
perm[fused_index.reshape(-1)] = torch.arange(heads_per_group * head_dim)
|
||||
return perm
|
||||
|
||||
|
||||
def o_fused_chunk_permutation(
|
||||
heads_per_group: int = WV_GROUP_SIZE, head_dim: int = HEAD_DIM
|
||||
) -> torch.Tensor:
|
||||
"""Per-32-element-chunk form of :func:`o_fused_permutation`, for scales."""
|
||||
return o_fused_permutation(heads_per_group, head_dim)[::O_CHUNK] // O_CHUNK
|
||||
|
||||
|
||||
def permute_q_to_fused(q: torch.Tensor) -> torch.Tensor:
|
||||
"""``[N, H, D]`` standard layout -> the same shape in the fused layout."""
|
||||
n, h, d = q.shape
|
||||
perm = q_fused_permutation(h, d).to(q.device)
|
||||
return q.reshape(n, h * d)[:, perm].view(n, h, d)
|
||||
|
||||
|
||||
def _bytes_view(t: torch.Tensor) -> torch.Tensor:
|
||||
"""Byte view of a 1-byte-element tensor, so fp8/ue8m0 can be gathered."""
|
||||
return t.view(torch.uint8) if t.element_size() == 1 else t
|
||||
|
||||
|
||||
def permute_wq_b_(
|
||||
weight: torch.Tensor, weight_scale: torch.Tensor, num_local_heads: int
|
||||
) -> None:
|
||||
"""Permute an MXFP8 ``wq_b`` shard's rows and per-row scales, in place."""
|
||||
head_dim = weight.shape[0] // num_local_heads
|
||||
perm = q_fused_permutation(num_local_heads, head_dim).to(weight.device)
|
||||
for t in (weight, weight_scale):
|
||||
b = _bytes_view(t)
|
||||
b.copy_(b[perm])
|
||||
|
||||
|
||||
def permute_wo_a_(
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
heads_per_group: int = WV_GROUP_SIZE,
|
||||
) -> None:
|
||||
"""Permute an MXFP8 ``wo_a`` shard's input columns and scales, in place."""
|
||||
head_dim = weight.shape[1] // heads_per_group
|
||||
perm = o_fused_permutation(heads_per_group, head_dim).to(weight.device)
|
||||
w = _bytes_view(weight)
|
||||
w.copy_(w[:, perm])
|
||||
chunk_perm = o_fused_chunk_permutation(heads_per_group, head_dim)
|
||||
s = _bytes_view(weight_scale)
|
||||
s.copy_(s[:, chunk_perm.to(weight.device)])
|
||||
@@ -477,8 +477,22 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
for layer in self.model.layers:
|
||||
layer.ffn.finalize_mega_moe_weights()
|
||||
|
||||
def _finalize_attn(self) -> None:
|
||||
"""Run the attention backend's post-load weight step on the draft layers.
|
||||
|
||||
They are ordinary DeepseekV4DecoderLayers, so they take the engine-wide
|
||||
attention backend and owe it whatever the target model owes it -- mega
|
||||
attention permutes wq_b / wo_a here and refuses to run without it.
|
||||
Idempotent, like the target's.
|
||||
"""
|
||||
for layer in self.model.layers:
|
||||
finalize = getattr(layer.attn, "finalize_loaded_weights", None)
|
||||
if finalize is not None:
|
||||
finalize()
|
||||
|
||||
def process_weights_after_loading(self) -> None:
|
||||
self._finalize_moe()
|
||||
self._finalize_attn()
|
||||
|
||||
def _remap_dspark_name(self, name: str) -> str | None:
|
||||
"""Map a checkpoint ``mtp.{i}.*`` name to this model's parameter path.
|
||||
|
||||
@@ -0,0 +1,497 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepSeek V4.1 attention on FlashMLA's mega-attention kernel (SM100).
|
||||
|
||||
One kernel does Q RoPE, sparse attention, the inverse RoPE of the output and
|
||||
its FP8 cast, and writes straight into the buffer ``wo_a`` consumes. So the
|
||||
layer declares ``accepts_unnormed_unroped_query`` -- the kernel RoPEs Q itself,
|
||||
leaving the fused KV insert only the zero-pad to the kernel's head count -- and
|
||||
its ``_alloc_attn_out`` / ``_o_proj`` pair speaks QuantizedActivation instead
|
||||
of bf16, since the output needs no rotation or quantization here.
|
||||
|
||||
``wq_b`` rows and ``wo_a`` columns are permuted once at load so the surrounding
|
||||
GEMMs speak the kernel's chunk-interleaved layouts directly. A step's prefill
|
||||
and decode segments write disjoint token ranges of one output buffer pair, so
|
||||
a single ``wo_a`` einsum covers the whole step.
|
||||
"""
|
||||
|
||||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Dynamic
|
||||
from vllm.models.deepseek_v41.common.ops import (
|
||||
combine_topk_swa_indices,
|
||||
compute_global_topk_indices_and_lens,
|
||||
dequantize_and_gather_k_cache,
|
||||
)
|
||||
from vllm.models.deepseek_v41.common.ops.fused_layout import (
|
||||
WV_GROUP_SIZE,
|
||||
permute_wo_a_,
|
||||
permute_wq_b_,
|
||||
)
|
||||
from vllm.models.deepseek_v41.nvidia.flashmla import DeepseekV4FlashMLAAttention
|
||||
from vllm.models.deepseek_v41.sparse_mla import (
|
||||
DeepseekV4FlashMLAMetadata,
|
||||
FlashMLAMegaAttnBackend,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import fp8_einsum, get_tma_aligned_size
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.v1.attention.ops.flashmla import is_flashmla_sparse_supported
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
|
||||
_HEAD_DIM_V = 512
|
||||
_ROPE_DIM = 64
|
||||
# The kernel always emits per-32 scales, whatever num_per_channels says.
|
||||
_QUANT_GROUP = 32
|
||||
# Fixed kernel conventions: GPT-J (non-neox) RoPE, TMA-aligned col-major
|
||||
# scales, round-to-nearest, packed ue8m0 -- i.e. what DeepGEMM's fp8_einsum
|
||||
# consumes. Q norm is off: the V4.1 layer norms qr before wq_b.
|
||||
_ROPE_ARGS = (False, _ROPE_DIM)
|
||||
_SF_ARGS = (_QUANT_GROUP, True, True, True)
|
||||
|
||||
# Decode always goes through the mega kernel: there is no batch size at which
|
||||
# falling back to the split-KV decode pays, so there is no minimum-token
|
||||
# threshold and no fallback path to maintain. What decides whether this layer
|
||||
# is worth selecting at all is TP, not the batch size.
|
||||
#
|
||||
# Measured on one GB300 with benchmarks/kernels/benchmark_dsv41_mega_attn.py,
|
||||
# CUDA graphs on, counting every per-step op each path runs (64-head model,
|
||||
# topk_swa=128, topk_extra=512, NVFP4 compressed cache), as mega / split-KV
|
||||
# microseconds:
|
||||
#
|
||||
# s_q 1 32 128 256 512
|
||||
# TP1 (64 live heads) 29/29 31/31 31/40 53/72 96/140
|
||||
# TP2 (32 live) 29/29 31/30 35/35 61/63 110/119
|
||||
# TP4 (16 live) 29/29 31/29 34/34 59/58 109/109
|
||||
# TP8 (8 live) 29/29 31/29 33/33 59/57 107/104
|
||||
#
|
||||
# The kernel absorbs the inverse RoPE and the FP8 cast, whose cost scales with
|
||||
# *live* heads, while its own cost scales with *padded* heads -- always 64,
|
||||
# since FlashMLA's FP8 decode takes only h_q in {64, 128}. So it wins at TP1
|
||||
# (up to 1.45x, and TP1 is also where the fused insert skips the Q pad
|
||||
# entirely) and from TP2 down the two cancel to within a few percent, with a
|
||||
# fixed ~2 us penalty at small s_q where the padded-head cost dominates.
|
||||
# Measure before assuming a win at high TP -- and measure under CUDA graphs:
|
||||
# in eager mode launch overhead swamps both paths and overstates the fused one
|
||||
# several-fold.
|
||||
|
||||
|
||||
def is_flashmla_mega_attn_supported() -> tuple[bool, str | None]:
|
||||
"""Whether FlashMLA's fused mega-attention kernels are usable here."""
|
||||
is_available, maybe_reason = is_flashmla_sparse_supported()
|
||||
if not is_available:
|
||||
return False, maybe_reason
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
return False, "FlashMLA mega attention requires sm_10x GPUs."
|
||||
if not hasattr(torch.ops._flashmla_C, "fused_norm_rope_attn_rope_cast_decode"):
|
||||
return False, "vllm._flashmla_C was built without the mega attention ops."
|
||||
if not hasattr(torch.ops._flashmla_C, "permute_q_b_proj"):
|
||||
return False, "vllm._flashmla_C predates the mega attention weight permutes."
|
||||
return True, None
|
||||
|
||||
|
||||
def alloc_mega_attn_output(
|
||||
num_tokens: int,
|
||||
n_wv_group: int,
|
||||
device: torch.device,
|
||||
) -> QuantizedActivation:
|
||||
"""Allocate the output buffer pair the mega-attention kernels write into.
|
||||
|
||||
One pair per forward step: the prefill and decode segments fill disjoint
|
||||
token ranges, so a single ``fp8_einsum`` over all ``num_tokens`` can
|
||||
consume the result instead of one call per segment.
|
||||
|
||||
The scale buffer is MN-major -- stride 1 along tokens, head-dim stride
|
||||
``ceil4(num_tokens)`` -- which is both what the kernel requires of a
|
||||
caller-provided buffer and what DeepGEMM expects for this N.
|
||||
"""
|
||||
d = WV_GROUP_SIZE * _HEAD_DIM_V
|
||||
data = torch.empty(
|
||||
(num_tokens, n_wv_group, d), dtype=torch.float8_e4m3fn, device=device
|
||||
)
|
||||
aligned = get_tma_aligned_size(num_tokens, torch.int32.itemsize)
|
||||
scale = torch.empty(
|
||||
(n_wv_group, d // (_QUANT_GROUP * 4), aligned),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
).permute(2, 0, 1)[:num_tokens]
|
||||
return QuantizedActivation(
|
||||
data=data,
|
||||
scale=scale,
|
||||
orig_dtype=torch.bfloat16,
|
||||
orig_shape=data.shape,
|
||||
quant_key=kMxfp8Dynamic,
|
||||
)
|
||||
|
||||
|
||||
def _token_slice(out: QuantizedActivation, start: int, end: int) -> QuantizedActivation:
|
||||
"""The ``[start, end)`` token slice of a mega-attention output buffer."""
|
||||
data = out.data[start:end]
|
||||
return replace(out, data=data, scale=out.scale[start:end], orig_shape=data.shape)
|
||||
|
||||
|
||||
class DeepseekV4MegaAttnAttention(DeepseekV4FlashMLAAttention):
|
||||
"""FlashMLA mega-attention layer for DeepSeek V4.1 (SM100)."""
|
||||
|
||||
backend_cls = FlashMLAMegaAttnBackend
|
||||
|
||||
@classmethod
|
||||
def is_available_for(cls, vllm_config: VllmConfig) -> bool:
|
||||
"""Whether this layer can serve the configured model on this device.
|
||||
|
||||
The kernel is SM100-only and wants ``WV_GROUP_SIZE`` heads per ``wo_a``
|
||||
group, which stops holding once TP divides the head count far enough
|
||||
(TP16 on a 64-head model leaves 4). ``__init__`` raises on both, so the
|
||||
default-backend selector asks here first rather than turning an
|
||||
unsupported topology into a startup crash.
|
||||
"""
|
||||
if not is_flashmla_mega_attn_supported()[0]:
|
||||
return False
|
||||
config = vllm_config.model_config.hf_config
|
||||
config = getattr(config, "text_config", config)
|
||||
n_heads = getattr(config, "num_attention_heads", 0) or 0
|
||||
n_groups = getattr(config, "o_groups", 0) or 0
|
||||
tp_size = vllm_config.parallel_config.tensor_parallel_size
|
||||
if n_heads % tp_size or n_groups % tp_size:
|
||||
return False
|
||||
n_local_heads, n_local_groups = n_heads // tp_size, n_groups // tp_size
|
||||
if not n_local_groups or n_local_heads % WV_GROUP_SIZE:
|
||||
return False
|
||||
return n_local_heads // n_local_groups == WV_GROUP_SIZE
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, *args, **kwargs) -> None:
|
||||
super().__init__(vllm_config, *args, **kwargs)
|
||||
if self.n_local_heads % WV_GROUP_SIZE:
|
||||
raise ValueError(
|
||||
f"{self.prefix}: mega attention needs the local head count "
|
||||
f"({self.n_local_heads}) to be a multiple of {WV_GROUP_SIZE}."
|
||||
)
|
||||
if self.n_local_heads // self.n_local_groups != WV_GROUP_SIZE:
|
||||
raise ValueError(
|
||||
f"{self.prefix}: mega attention needs {WV_GROUP_SIZE} heads per "
|
||||
"wo_a group."
|
||||
)
|
||||
self.n_wv_group = self.padded_heads // WV_GROUP_SIZE
|
||||
self._fused_layouts_ready = False
|
||||
|
||||
# ---- interface contract ------------------------------------------------
|
||||
|
||||
@property
|
||||
def accepts_unnormed_unroped_query(self) -> bool:
|
||||
return True
|
||||
|
||||
@property
|
||||
def packed_kv_cache_dtype(self) -> CacheDType:
|
||||
# This kernel is the only one that reads an NVFP4 compressed cache, so
|
||||
# it is what an unspecific --kv-cache-dtype resolves to here.
|
||||
return "nvfp4_ds_mla"
|
||||
|
||||
def _alloc_attn_out(
|
||||
self, num_tokens: int, hidden_states: torch.Tensor
|
||||
) -> QuantizedActivation:
|
||||
return alloc_mega_attn_output(num_tokens, self.n_wv_group, hidden_states.device)
|
||||
|
||||
def _o_proj(
|
||||
self, attn_out: QuantizedActivation, positions: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Grouped ``wo_a`` then ``wo_b`` over the kernel's quantized output.
|
||||
|
||||
The inverse RoPE and the FP8 cast happened inside the attention
|
||||
kernel, and ``wo_a`` is permuted for its output layout, so there is
|
||||
nothing to rotate or quantize here and ``positions`` is unused. The
|
||||
scale already is DeepGEMM's packed-ue8m0 MN-major layout, so the
|
||||
einsum consumes the kernel's output with no repacking.
|
||||
"""
|
||||
groups = self.n_local_groups
|
||||
z = torch.empty(
|
||||
(attn_out.data.shape[0], groups, self.o_lora_rank),
|
||||
dtype=torch.bfloat16,
|
||||
device=attn_out.data.device,
|
||||
)
|
||||
fp8_einsum(
|
||||
"bhr,hdr->bhd",
|
||||
(attn_out.data[:, :groups], attn_out.scale[:, :groups]),
|
||||
(self.wo_a.weight, self.wo_a.weight_scale),
|
||||
z,
|
||||
recipe=self._einsum_recipe,
|
||||
)
|
||||
return self.wo_b(z.flatten(1))
|
||||
|
||||
# ---- weights -----------------------------------------------------------
|
||||
|
||||
def finalize_loaded_weights(self) -> None:
|
||||
"""Permute wq_b rows / wo_a columns into the kernel's layouts.
|
||||
|
||||
Idempotent: a second post-load pass must not permute twice.
|
||||
"""
|
||||
if self._fused_layouts_ready:
|
||||
return
|
||||
permute_wq_b_(
|
||||
self.wq_b.weight.data, self.wq_b.weight_scale.data, self.n_local_heads
|
||||
)
|
||||
permute_wo_a_(
|
||||
self.wo_a.weight.data,
|
||||
self.wo_a.weight_scale.data,
|
||||
self.n_local_heads // self.n_local_groups,
|
||||
)
|
||||
self._fused_layouts_ready = True
|
||||
|
||||
# ---- forward -----------------------------------------------------------
|
||||
|
||||
def forward_mqa(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
output: QuantizedActivation,
|
||||
) -> None:
|
||||
if not self._fused_layouts_ready:
|
||||
raise RuntimeError(
|
||||
f"{self.prefix}: wq_b / wo_a were never permuted for the mega "
|
||||
"attention kernel; refusing to run with mismatched layouts."
|
||||
)
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
if attn_metadata is None:
|
||||
# Warmup dummy run: reserve the prefill workspace, produce zeros.
|
||||
self._reserve_prefill_workspace(q)
|
||||
output.data.zero_()
|
||||
output.scale.zero_()
|
||||
return
|
||||
|
||||
assert isinstance(attn_metadata, dict)
|
||||
# q was zero-padded to the kernel's head count by the fused KV insert.
|
||||
assert q.shape[1] == self.padded_heads
|
||||
flashmla_metadata = cast(
|
||||
DeepseekV4FlashMLAMetadata | None,
|
||||
attn_metadata.get(self.compressed_cache_prefix)
|
||||
if self.compressed_cache_prefix is not None
|
||||
else None,
|
||||
)
|
||||
swa_metadata = cast(
|
||||
"DeepseekSparseSWAMetadata", attn_metadata[self.swa_cache_layer.prefix]
|
||||
)
|
||||
# The kernel takes int32 RoPE positions.
|
||||
positions_int32 = positions.to(torch.int32)
|
||||
num_decode_tokens = swa_metadata.num_decode_tokens
|
||||
|
||||
if swa_metadata.num_prefills > 0:
|
||||
self._forward_prefill_mega(
|
||||
q[num_decode_tokens:],
|
||||
positions_int32[num_decode_tokens:],
|
||||
flashmla_metadata,
|
||||
swa_metadata,
|
||||
output,
|
||||
num_decode_tokens,
|
||||
)
|
||||
if swa_metadata.num_decodes > 0:
|
||||
self._forward_decode_mega(
|
||||
q[:num_decode_tokens],
|
||||
positions_int32[:num_decode_tokens],
|
||||
flashmla_metadata,
|
||||
swa_metadata,
|
||||
_token_slice(output, 0, num_decode_tokens),
|
||||
)
|
||||
|
||||
def _reserve_prefill_workspace(self, q: torch.Tensor) -> None:
|
||||
swa_only = self.compress_ratio == 0
|
||||
n = 0 if swa_only else -(-self.max_model_len // self.compress_ratio)
|
||||
m = n + self.window_size + self.max_num_batched_tokens
|
||||
if swa_only:
|
||||
top_k = 0
|
||||
else:
|
||||
assert self.topk_indices_buffer is not None
|
||||
top_k = self.topk_indices_buffer.shape[-1]
|
||||
combined_topk = round_up(top_k + self.window_size + self.max_image_tokens, 128)
|
||||
current_workspace_manager().get_simultaneous(
|
||||
((self.PREFILL_CHUNK_SIZE, m, q.shape[-1]), torch.bfloat16),
|
||||
((self.max_num_batched_tokens, combined_topk), torch.int32),
|
||||
((self.max_num_batched_tokens,), torch.int32),
|
||||
)
|
||||
|
||||
def _decode_compressed_kv_and_topk(
|
||||
self,
|
||||
flashmla_metadata: DeepseekV4FlashMLAMetadata | None,
|
||||
swa_metadata: "DeepseekSparseSWAMetadata",
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]:
|
||||
"""The compressed cache and its ``[n, topk]`` slot indices / lengths."""
|
||||
if self.compress_ratio == 0:
|
||||
return None, None, None
|
||||
assert flashmla_metadata is not None
|
||||
assert swa_metadata.is_valid_token is not None
|
||||
assert self.topk_indices_buffer is not None
|
||||
num_decode_tokens = swa_metadata.num_decode_tokens
|
||||
indices, lens = compute_global_topk_indices_and_lens(
|
||||
self.topk_indices_buffer[:num_decode_tokens],
|
||||
swa_metadata.token_to_req_indices,
|
||||
flashmla_metadata.block_table[: swa_metadata.num_decodes],
|
||||
flashmla_metadata.block_size // self.compress_ratio,
|
||||
swa_metadata.is_valid_token[:num_decode_tokens],
|
||||
)
|
||||
return self._compressed_kv_cache().unsqueeze(-2), indices, lens
|
||||
|
||||
def _forward_decode_mega(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
positions_int32: torch.Tensor,
|
||||
flashmla_metadata: DeepseekV4FlashMLAMetadata | None,
|
||||
swa_metadata: "DeepseekSparseSWAMetadata",
|
||||
out: QuantizedActivation,
|
||||
) -> None:
|
||||
"""Mega attention over the paged quantized caches, in place into ``out``.
|
||||
|
||||
The caches are ``[num_blocks, page, 1, bytes]``, the record taken from
|
||||
``bytes`` (528 V4.1 fp8, 288 V4.1 NVFP4; NVFP4 only as the compressed
|
||||
cache beside an fp8 SWA one), and the indices ``[s_q, topk]`` int32
|
||||
slot ids (``block * page + offset``, ``-1`` invalid).
|
||||
"""
|
||||
extra_cache, extra_idx, extra_len = self._decode_compressed_kv_and_topk(
|
||||
flashmla_metadata, swa_metadata
|
||||
)
|
||||
assert swa_metadata.decode_swa_indices is not None
|
||||
torch.ops._flashmla_C.fused_norm_rope_attn_rope_cast_decode(
|
||||
q,
|
||||
self.swa_cache_layer.kv_cache.unsqueeze(-2),
|
||||
swa_metadata.decode_swa_indices.view(q.shape[0], -1),
|
||||
self.scale,
|
||||
_HEAD_DIM_V,
|
||||
self.attn_sink,
|
||||
swa_metadata.decode_swa_lens,
|
||||
extra_cache,
|
||||
extra_idx,
|
||||
extra_len,
|
||||
False, # enable_q_norm
|
||||
0.0, # rms_norm_eps
|
||||
positions_int32,
|
||||
*_ROPE_ARGS,
|
||||
self.rotary_emb.cos_sin_cache,
|
||||
self.n_wv_group,
|
||||
*_SF_ARGS,
|
||||
out.data,
|
||||
out.scale,
|
||||
)
|
||||
|
||||
def _forward_prefill_mega(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
positions_int32: torch.Tensor,
|
||||
flashmla_metadata: DeepseekV4FlashMLAMetadata | None,
|
||||
swa_metadata: "DeepseekSparseSWAMetadata",
|
||||
out: QuantizedActivation,
|
||||
token_base: int,
|
||||
) -> None:
|
||||
swa_only = self.compress_ratio == 0
|
||||
num_decodes = swa_metadata.num_decodes
|
||||
num_decode_tokens = swa_metadata.num_decode_tokens
|
||||
seq_lens = swa_metadata.prefill_seq_lens
|
||||
gather_lens = swa_metadata.prefill_gather_lens
|
||||
query_start_loc_cpu = swa_metadata.query_start_loc_cpu
|
||||
query_start_loc = swa_metadata.query_start_loc
|
||||
assert seq_lens is not None and gather_lens is not None
|
||||
assert query_start_loc_cpu is not None and query_start_loc is not None
|
||||
prefill_token_base = query_start_loc_cpu[num_decodes]
|
||||
assert self.topk_indices_buffer is not None
|
||||
topk_indices = self.topk_indices_buffer[num_decode_tokens:][
|
||||
: swa_metadata.num_prefill_tokens
|
||||
]
|
||||
top_k = 0 if swa_only else topk_indices.shape[-1]
|
||||
chunk_plan = swa_metadata.get_prefill_chunk_plan(
|
||||
compress_ratio=self.compress_ratio,
|
||||
prefill_chunk_size=self.PREFILL_CHUNK_SIZE,
|
||||
has_compressed=not swa_only,
|
||||
)
|
||||
assert chunk_plan, "prefill chunk plan must be non-empty when num_prefills > 0"
|
||||
workspace_manager = current_workspace_manager()
|
||||
combined_topk = round_up(top_k + self.window_size + self.max_image_tokens, 128)
|
||||
for chunk_start, chunk_end, chunk_n, chunk_m in chunk_plan:
|
||||
chunk_size = chunk_end - chunk_start
|
||||
kv_ws, idx_ws, lens_ws = workspace_manager.get_simultaneous(
|
||||
((chunk_size, chunk_m, q.shape[-1]), torch.bfloat16),
|
||||
((self.max_num_batched_tokens, combined_topk), torch.int32),
|
||||
((self.max_num_batched_tokens,), torch.int32),
|
||||
)
|
||||
if not swa_only:
|
||||
assert flashmla_metadata is not None
|
||||
dequantize_and_gather_k_cache(
|
||||
kv_ws[:chunk_size],
|
||||
self._compressed_kv_cache(),
|
||||
seq_lens=seq_lens[chunk_start:chunk_end] // self.compress_ratio,
|
||||
gather_lens=None,
|
||||
block_table=flashmla_metadata.block_table[num_decodes:][
|
||||
chunk_start:chunk_end
|
||||
],
|
||||
block_size=flashmla_metadata.block_size // self.compress_ratio,
|
||||
offset=0,
|
||||
)
|
||||
dequantize_and_gather_k_cache(
|
||||
kv_ws[:chunk_size],
|
||||
self.swa_cache_layer.kv_cache,
|
||||
seq_lens=seq_lens[chunk_start:chunk_end],
|
||||
gather_lens=gather_lens[chunk_start:chunk_end],
|
||||
block_table=swa_metadata.block_table[num_decodes:][
|
||||
chunk_start:chunk_end
|
||||
],
|
||||
block_size=swa_metadata.block_size,
|
||||
offset=chunk_n,
|
||||
)
|
||||
first, last = num_decodes + chunk_start, num_decodes + chunk_end
|
||||
qs = int(query_start_loc_cpu[first] - prefill_token_base)
|
||||
qe = int(query_start_loc_cpu[last] - prefill_token_base)
|
||||
combined_indices, combined_lens = combine_topk_swa_indices(
|
||||
topk_indices[qs:qe],
|
||||
query_start_loc[first : last + 1],
|
||||
seq_lens[chunk_start:chunk_end],
|
||||
gather_lens[chunk_start:chunk_end],
|
||||
self.window_size,
|
||||
self.compress_ratio,
|
||||
top_k,
|
||||
chunk_m,
|
||||
chunk_n,
|
||||
out=(idx_ws[: qe - qs], lens_ws[: qe - qs]),
|
||||
left_visible=(
|
||||
swa_metadata.prefill_left_visible[
|
||||
num_decode_tokens + qs : num_decode_tokens + qe
|
||||
]
|
||||
if swa_metadata.prefill_left_visible is not None
|
||||
else None
|
||||
),
|
||||
right_visible=(
|
||||
swa_metadata.prefill_right_visible[
|
||||
num_decode_tokens + qs : num_decode_tokens + qe
|
||||
]
|
||||
if swa_metadata.prefill_right_visible is not None
|
||||
else None
|
||||
),
|
||||
max_image_tokens=self.max_image_tokens,
|
||||
)
|
||||
chunk_out = _token_slice(out, token_base + qs, token_base + qe)
|
||||
# Mega attention over the gathered non-paged bf16 KV (RoPE already
|
||||
# applied), in place into this chunk's token range. `indices`
|
||||
# entries outside [0, s_kv) skip.
|
||||
torch.ops._flashmla_C.fused_norm_rope_attn_rope_cast_fwd(
|
||||
q[qs:qe],
|
||||
kv_ws.view(-1, 1, q.shape[-1]),
|
||||
combined_indices.unsqueeze(1),
|
||||
self.scale,
|
||||
_HEAD_DIM_V,
|
||||
self.attn_sink,
|
||||
combined_lens,
|
||||
False, # enable_q_norm
|
||||
0.0, # rms_norm_eps
|
||||
positions_int32[qs:qe],
|
||||
*_ROPE_ARGS,
|
||||
self.rotary_emb.cos_sin_cache,
|
||||
self.n_wv_group,
|
||||
*_SF_ARGS,
|
||||
chunk_out.data,
|
||||
chunk_out.scale,
|
||||
)
|
||||
@@ -201,7 +201,8 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention):
|
||||
def get_padded_num_q_heads(cls, num_heads: int) -> int:
|
||||
return _pad_to_supported_q_heads(num_heads)
|
||||
|
||||
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
||||
def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
||||
o = attn_out[:, : self.n_local_heads, :]
|
||||
return deep_gemm_fp8_o_proj(
|
||||
o,
|
||||
positions,
|
||||
@@ -562,7 +563,8 @@ class DeepseekV4FlashInferSM120Attention(DeepseekV4Attention):
|
||||
def get_padded_num_q_heads(cls, num_heads: int) -> int:
|
||||
return _pad_to_supported_q_heads(num_heads)
|
||||
|
||||
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
||||
def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
||||
o = attn_out[:, : self.n_local_heads, :]
|
||||
return deep_gemm_fp8_o_proj(
|
||||
o,
|
||||
positions,
|
||||
|
||||
@@ -58,7 +58,8 @@ class DeepseekV4FlashMLAAttention(DeepseekV4Attention):
|
||||
self._o_proj_block_size
|
||||
)
|
||||
|
||||
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
||||
def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
||||
o = attn_out[:, : self.n_local_heads, :]
|
||||
return deep_gemm_fp8_o_proj(
|
||||
o,
|
||||
positions,
|
||||
|
||||
@@ -64,6 +64,9 @@ from vllm.models.deepseek_v4.nvidia.model import (
|
||||
make_deepseek_v4_expert_params_mapping,
|
||||
)
|
||||
from vllm.models.deepseek_v41.attention import DeepseekV4Attention
|
||||
from vllm.models.deepseek_v41.nvidia.flash_mla_mega_attn import (
|
||||
DeepseekV4MegaAttnAttention,
|
||||
)
|
||||
from vllm.models.deepseek_v41.nvidia.flashinfer_sparse import (
|
||||
DeepseekV4FlashInferMLAAttention,
|
||||
DeepseekV4FlashInferSM120Attention,
|
||||
@@ -119,8 +122,9 @@ def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]:
|
||||
|
||||
The generic CUDA backend selector does not instantiate DSv4 layers directly,
|
||||
so map generic sparse-MLA choices to the DSv4-specialized attention class.
|
||||
Without an explicit backend, SM12 defaults to FlashInfer while the other
|
||||
CUDA arches keep the FlashMLA path.
|
||||
Without an explicit backend: SM12 takes FlashInfer, SM100 takes mega
|
||||
attention where the topology allows it, and everything else keeps the
|
||||
FlashMLA path.
|
||||
"""
|
||||
backend = vllm_config.attention_config.backend
|
||||
device_capability = current_platform.get_device_capability()
|
||||
@@ -140,6 +144,8 @@ def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]:
|
||||
if device_capability is not None and device_capability.major == 12:
|
||||
return DeepseekV4FlashInferSM120Attention
|
||||
return DeepseekV4FlashInferMLAAttention
|
||||
if backend is AttentionBackendEnum.FLASHMLA_MEGA_ATTN_DSV41:
|
||||
return DeepseekV4MegaAttnAttention
|
||||
if backend in (
|
||||
AttentionBackendEnum.FLASHMLA_SPARSE,
|
||||
AttentionBackendEnum.FLASHMLA_SPARSE_DSV4,
|
||||
@@ -149,6 +155,14 @@ def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]:
|
||||
|
||||
if device_capability is not None and device_capability.major == 12:
|
||||
return DeepseekV4FlashInferSM120Attention
|
||||
# Mega attention is the SM100 default: it fuses Q RoPE, sparse attention,
|
||||
# the output's inverse RoPE and its FP8 cast into one launch, and brings
|
||||
# the 288 B NVFP4 compressed record -- the format the reference
|
||||
# implementation itself stores. It declines topologies it cannot serve
|
||||
# (non-SM100, TP that leaves fewer than WV_GROUP_SIZE heads per wo_a
|
||||
# group, a build without the kernel), which then fall through to FlashMLA.
|
||||
if DeepseekV4MegaAttnAttention.is_available_for(vllm_config):
|
||||
return DeepseekV4MegaAttnAttention
|
||||
return DeepseekV4FlashMLAAttention
|
||||
|
||||
|
||||
@@ -936,6 +950,17 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
for layer in islice(self.layers, self.start_layer, self.end_layer):
|
||||
layer.ffn.finalize_mega_moe_weights()
|
||||
|
||||
def finalize_mega_attn_weights(self) -> None:
|
||||
"""Permute wq_b / wo_a into FlashMLA's mega-attention layouts.
|
||||
|
||||
A no-op for every other attention layer, and idempotent, so a second
|
||||
post-load pass cannot permute twice.
|
||||
"""
|
||||
for layer in islice(self.layers, self.start_layer, self.end_layer):
|
||||
finalize = getattr(layer.attn, "finalize_loaded_weights", None)
|
||||
if finalize is not None:
|
||||
finalize()
|
||||
|
||||
def finalize_mhc_broadcast_weights(self) -> None:
|
||||
if not get_pp_group().is_first_rank or self.start_layer >= self.end_layer:
|
||||
return
|
||||
@@ -1190,6 +1215,7 @@ class DeepseekV41LLMForCausalLM(
|
||||
def process_weights_after_loading(self) -> None:
|
||||
self.model.finalize_mega_moe_weights()
|
||||
self.model.finalize_mhc_broadcast_weights()
|
||||
self.model.finalize_mega_attn_weights()
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return self.model.get_expert_mapping()
|
||||
|
||||
@@ -229,3 +229,54 @@ class DeepseekV4FlashMLABackend(DeepseekV4SparseMLABackend):
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type[DeepseekV4FlashMLAMetadataBuilder]:
|
||||
return DeepseekV4FlashMLAMetadataBuilder
|
||||
|
||||
|
||||
class FlashMLAMegaAttnBackend(DeepseekV4FlashMLABackend):
|
||||
"""FlashMLA's mega-attention kernel: Q RoPE + sparse attention + inverse
|
||||
RoPE + FP8 cast of the output, in one launch.
|
||||
|
||||
Same metadata and KV-cache geometry as ``FLASHMLA_SPARSE_DSV41`` -- what
|
||||
differs is the attention layer's interface contract (it takes an unnormed,
|
||||
unroped Q and returns an already-inverse-RoPE'd, quantized output) and the
|
||||
extra ``nvfp4_ds_mla`` compressed-cache record only this kernel can read.
|
||||
SM100 only; the kernel has no SM90 instantiation.
|
||||
"""
|
||||
|
||||
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
|
||||
"auto",
|
||||
"fp8_ds_mla",
|
||||
"fp8", # alias for fp8_ds_mla
|
||||
"nvfp4_ds_mla", # V4.1 fp8 SWA cache + NVFP4 compressed cache
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "FLASHMLA_MEGA_ATTN_DSV41"
|
||||
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes() -> list[int | MultipleOf]:
|
||||
return [128]
|
||||
|
||||
@classmethod
|
||||
def supports_compute_capability(cls, capability: DeviceCapability) -> bool:
|
||||
return capability.major == 10
|
||||
|
||||
@classmethod
|
||||
def supports_combination(
|
||||
cls,
|
||||
head_size: int,
|
||||
dtype: torch.dtype,
|
||||
kv_cache_dtype: CacheDType | None,
|
||||
block_size: int | None,
|
||||
use_mla: bool,
|
||||
has_sink: bool,
|
||||
use_sparse: bool,
|
||||
use_mm_prefix: bool,
|
||||
device_capability: DeviceCapability,
|
||||
) -> str | None:
|
||||
# Imported here: the layer module imports this backend class.
|
||||
from vllm.models.deepseek_v41.nvidia.flash_mla_mega_attn import (
|
||||
is_flashmla_mega_attn_supported,
|
||||
)
|
||||
|
||||
return is_flashmla_mega_attn_supported()[1]
|
||||
|
||||
@@ -0,0 +1,729 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Custom Sparse Attention Indexer layers."""
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm import _custom_ops # noqa: F401 # registers the torch.ops._C kernels
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import get_current_vllm_config_or_none
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
from vllm.models.glm5next.amd.ops import kpool_compress as kpool_ops
|
||||
from vllm.models.glm5next.common.sparse_indexer import (
|
||||
RADIX_TOPK_WORKSPACE_SIZE,
|
||||
_build_decode_scatter_indices,
|
||||
_decode_topk_seq_lens,
|
||||
_fill_causal_indices,
|
||||
_fill_short_decode_causal_indices,
|
||||
_gather_workspace_shapes,
|
||||
_scatter_decode_tokens_by_request,
|
||||
kv_cache_as_quant_view,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
_resolve_layer_name,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
DeepseekV32IndexerMetadata,
|
||||
)
|
||||
from vllm.v1.attention.ops.common import pack_seq_triton, unpack_seq_triton
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# kpool write helper: form pools from the current token batch and compress them
|
||||
# into the index K cache via the fused Triton kernel.
|
||||
|
||||
|
||||
def _kpool_compress_insert(
|
||||
k: torch.Tensor,
|
||||
gate_score: torch.Tensor,
|
||||
ape: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
kpool: int,
|
||||
head_dim: int,
|
||||
round_scale: bool,
|
||||
) -> None:
|
||||
"""Pool ``kpool`` consecutive tokens into one fp8 K and write at pool slots.
|
||||
|
||||
``slot_mapping`` is pool-granular (compress_ratio == kpool on the spec):
|
||||
only the *last* token of each complete pool carries a valid (>=0) slot;
|
||||
intra-pool tokens are -1. Every position is treated as a pool-completion
|
||||
candidate and non-completions are masked off inside the kernel. Compacting
|
||||
the valid rows first costs two device syncs on the eager prefill path and
|
||||
buys nothing numerically. Assumes pool-aligned chunk starts.
|
||||
"""
|
||||
n = slot_mapping.shape[0]
|
||||
# No pool can complete in a batch smaller than one pool; also keeps the
|
||||
# clamped gather indices below in bounds.
|
||||
if n < kpool:
|
||||
return
|
||||
pos = torch.arange(n, device=k.device)
|
||||
valid = slot_mapping >= 0
|
||||
# Drop pools whose start falls before the batch (leading padding); their
|
||||
# gate/k data is undefined anyway.
|
||||
write_mask = valid & (pos >= kpool - 1)
|
||||
offs = torch.arange(kpool, device=k.device)
|
||||
idx = (pos - (kpool - 1)).clamp_min(0)[:, None] + offs[None, :]
|
||||
kpool_ops.kpool_compress_and_write_cache(
|
||||
kv_cache,
|
||||
k[idx], # [n, kpool, head_dim]
|
||||
gate_score[idx],
|
||||
ape,
|
||||
slot_mapping.to(torch.int64),
|
||||
pool_size=kpool,
|
||||
head_dim=head_dim,
|
||||
write_mask=write_mask,
|
||||
round_scale=round_scale,
|
||||
write_cache=True,
|
||||
return_compressed=False,
|
||||
)
|
||||
|
||||
|
||||
@eager_break_during_capture
|
||||
def sparse_attn_indexer_kpool(
|
||||
hidden_states: torch.Tensor,
|
||||
k_cache_prefix: LayerNameType,
|
||||
kv_cache: torch.Tensor,
|
||||
q_quant: torch.Tensor,
|
||||
q_scale: torch.Tensor | None,
|
||||
k: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
quant_block_size: int,
|
||||
scale_fmt: str | None,
|
||||
topk_tokens: int,
|
||||
head_dim: int,
|
||||
max_model_len: int,
|
||||
total_seq_lens: int,
|
||||
topk_indices_buffer: torch.Tensor,
|
||||
skip_k_cache_insert: bool,
|
||||
use_fp4_cache: bool = False,
|
||||
# kpool params (Plan-A: gate is consumed at write time and read back at
|
||||
# topk time to softmax-weight the pool).
|
||||
gate_score: torch.Tensor | None = None,
|
||||
compress_ape: torch.Tensor | None = None,
|
||||
index_kpool: int = 1,
|
||||
positions: torch.Tensor | None = None,
|
||||
# Paged tail cache (in-progress pool's raw K + gate score), replacing the
|
||||
# transient _DECODE_TAIL ring. tail_prefix resolves attn_metadata[tail_prefix]
|
||||
# for the tail group's token-granular slot_mapping. None on the dummy/profiling
|
||||
# path and when the tail cache is disabled.
|
||||
tail_kv_cache: torch.Tensor | None = None,
|
||||
tail_prefix: str | None = None,
|
||||
) -> torch.Tensor:
|
||||
# careful! this will be None in dummy run
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
k_cache_prefix = _resolve_layer_name(k_cache_prefix)
|
||||
|
||||
# assert isinstance(attn_metadata, dict)
|
||||
if not isinstance(attn_metadata, dict):
|
||||
# Reserve workspace for indexer during profiling run
|
||||
values_spec, scales_spec = _gather_workspace_shapes(
|
||||
total_seq_lens, head_dim, fp8_dtype, use_fp4_cache
|
||||
)
|
||||
current_workspace_manager().get_simultaneous(
|
||||
values_spec,
|
||||
scales_spec,
|
||||
((RADIX_TOPK_WORKSPACE_SIZE,), torch.uint8),
|
||||
)
|
||||
|
||||
# Reserve profiler-visible memory for the worst-case decode logits,
|
||||
# whose shape is [B * next_n, max_model_len]. This profiling branch
|
||||
# returns before invoking the logits kernel itself.
|
||||
cfg = get_current_vllm_config_or_none()
|
||||
worst_decode_tokens = 0
|
||||
if cfg is not None:
|
||||
sched = cfg.scheduler_config
|
||||
num_spec = (
|
||||
cfg.speculative_config.num_speculative_tokens
|
||||
if cfg.speculative_config is not None
|
||||
else 0
|
||||
)
|
||||
worst_decode_tokens = min(
|
||||
sched.max_num_seqs * (num_spec + 1),
|
||||
sched.max_num_batched_tokens,
|
||||
)
|
||||
# float32 logits -> 4 bytes/element; uint8 sentinel so elems == bytes.
|
||||
decode_logits_elems = worst_decode_tokens * max_model_len * 4
|
||||
prefill_cap_elems = envs.VLLM_SPARSE_INDEXER_MAX_LOGITS_MB * 1024 * 1024
|
||||
max_logits_elems = max(decode_logits_elems, prefill_cap_elems)
|
||||
_ = torch.empty(
|
||||
max_logits_elems, dtype=torch.uint8, device=hidden_states.device
|
||||
)
|
||||
|
||||
return topk_indices_buffer
|
||||
attn_metadata_narrowed = attn_metadata[k_cache_prefix]
|
||||
assert isinstance(attn_metadata_narrowed, DeepseekV32IndexerMetadata)
|
||||
slot_mapping = attn_metadata_narrowed.slot_mapping
|
||||
has_decode = attn_metadata_narrowed.num_decodes > 0
|
||||
has_prefill = attn_metadata_narrowed.num_prefills > 0
|
||||
num_decode_tokens = attn_metadata_narrowed.num_decode_tokens
|
||||
|
||||
# q_scale is required iff the FP4 cache path is enabled; the FP8 path
|
||||
# folds the Q scale into `weights` inside fused_indexer_q_rope_quant.
|
||||
if use_fp4_cache:
|
||||
assert q_scale is not None, "use_fp4_cache=True requires q_scale"
|
||||
else:
|
||||
assert q_scale is None, "q_scale must be None when use_fp4_cache=False"
|
||||
|
||||
# During speculative decoding, k may be padded to the CUDA graph batch
|
||||
# size while slot_mapping only covers actual tokens. Truncate k to avoid
|
||||
# out-of-bounds reads in the kernel.
|
||||
num_tokens = slot_mapping.shape[0]
|
||||
if k is not None:
|
||||
k = k[:num_tokens]
|
||||
|
||||
if not skip_k_cache_insert:
|
||||
assert not use_fp4_cache, "Unfused FP4 Insert is not supported yet"
|
||||
if index_kpool > 1 and gate_score is not None and compress_ape is not None:
|
||||
# kpool prefill write: pool kpool consecutive prefill tokens via
|
||||
# softmax(gate+ape)-weighted sum -> Hadamard -> fp8 -> pool slots.
|
||||
# Decode tokens (the first num_decode_tokens in the batch) cannot be
|
||||
# pooled here — their pool's earlier tokens are not in this batch —
|
||||
# so they are deferred to the tail-buffer kernel in has_decode.
|
||||
# compress_ratio == index_kpool makes slot_mapping pool-granular.
|
||||
n_prefill = num_tokens - num_decode_tokens
|
||||
if n_prefill > 0:
|
||||
# decode tokens are batched first; prefill tokens follow.
|
||||
prefill_slice = slice(num_decode_tokens, num_tokens)
|
||||
_kpool_compress_insert(
|
||||
k[prefill_slice],
|
||||
gate_score[prefill_slice],
|
||||
compress_ape,
|
||||
kv_cache,
|
||||
slot_mapping[prefill_slice],
|
||||
index_kpool,
|
||||
head_dim,
|
||||
round_scale=(scale_fmt is not None),
|
||||
)
|
||||
# Persist each request's incomplete prefill pool so decode can
|
||||
# finish it, including after PD transfer. Tail slots use
|
||||
# ``pos % kpool`` within the request's tail block. Processing
|
||||
# only the batch's trailing tokens would miss all but the last
|
||||
# request in a multi-request prefill.
|
||||
if tail_kv_cache is not None and tail_prefix is not None:
|
||||
tail_meta = attn_metadata.get(_resolve_layer_name(tail_prefix))
|
||||
if tail_meta is not None:
|
||||
assert isinstance(tail_meta, DeepseekV32IndexerMetadata)
|
||||
kpool_ops.kpool_seed_tail_cache(
|
||||
tail_kv_cache,
|
||||
k[prefill_slice],
|
||||
gate_score[prefill_slice],
|
||||
tail_meta.slot_mapping[prefill_slice],
|
||||
index_kpool,
|
||||
head_dim,
|
||||
)
|
||||
else:
|
||||
# standard: per-token fp8 quant + scatter (all tokens).
|
||||
assert scale_fmt is not None
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
indexer_k_quant_and_cache_triton,
|
||||
)
|
||||
|
||||
indexer_k_quant_and_cache_triton(
|
||||
k,
|
||||
kv_cache,
|
||||
slot_mapping,
|
||||
quant_block_size,
|
||||
scale_fmt,
|
||||
)
|
||||
|
||||
topk_indices_buffer[: hidden_states.shape[0]] = -1
|
||||
if has_prefill:
|
||||
prefill_metadata = attn_metadata_narrowed.prefill
|
||||
assert prefill_metadata is not None
|
||||
|
||||
# Short sequences select every pool, so skip sparse scoring and fill
|
||||
# the top-k buffer with all causal token indices. The index-K cache was
|
||||
# already written above.
|
||||
n_prefill_sf = num_tokens - num_decode_tokens
|
||||
# Host-side short-prefill predicate: max_prefill_seq_len is computed
|
||||
# in the metadata builder (exact for prefill rows) and equals
|
||||
# positions[prefill_slice].max() + 1, so this replaces a
|
||||
# positions.max().item() device sync per layer. -1 (unknown metadata)
|
||||
# falls back to the device-side check.
|
||||
if prefill_metadata.max_prefill_seq_len >= 0:
|
||||
short_prefill = (
|
||||
n_prefill_sf > 0
|
||||
and positions is not None
|
||||
and prefill_metadata.max_prefill_seq_len <= topk_tokens
|
||||
)
|
||||
else:
|
||||
short_prefill = (
|
||||
n_prefill_sf > 0
|
||||
and positions is not None
|
||||
and int(positions[num_decode_tokens:num_tokens].max().item()) + 1
|
||||
<= topk_tokens
|
||||
)
|
||||
if short_prefill:
|
||||
# short_prefill is only True when positions is not None (above),
|
||||
# but narrow explicitly for the indexer below.
|
||||
assert positions is not None
|
||||
_pos = positions[num_decode_tokens:num_tokens].to(torch.int32)
|
||||
_buf = topk_indices_buffer[num_decode_tokens:num_tokens]
|
||||
_fill_causal_indices(_buf, _pos)
|
||||
|
||||
# Get the full shared workspace buffers once (will allocate on first use).
|
||||
# Layout switches between FP8 (head_dim bytes + 4-byte fp32 scale) and
|
||||
# MXFP4 (head_dim/2 bytes packed + head_dim/MXFP4_BLOCK_SIZE ue8m0
|
||||
# scales) based on use_fp4_cache.
|
||||
workspace_manager = current_workspace_manager()
|
||||
values_spec, scales_spec = _gather_workspace_shapes(
|
||||
total_seq_lens, head_dim, fp8_dtype, use_fp4_cache
|
||||
)
|
||||
k_quant_full, k_scale_full = workspace_manager.get_simultaneous(
|
||||
values_spec,
|
||||
scales_spec,
|
||||
)
|
||||
for chunk in prefill_metadata.chunks if not short_prefill else ():
|
||||
k_quant = k_quant_full[: chunk.total_seq_lens]
|
||||
k_scale = k_scale_full[: chunk.total_seq_lens]
|
||||
|
||||
if not chunk.skip_kv_gather:
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
cp_gather_indexer_k_quant_cache_triton,
|
||||
)
|
||||
|
||||
cp_gather_indexer_k_quant_cache_triton(
|
||||
kv_cache,
|
||||
k_quant,
|
||||
k_scale,
|
||||
chunk.block_table,
|
||||
chunk.cu_seq_lens,
|
||||
token_to_seq=chunk.token_to_seq,
|
||||
)
|
||||
|
||||
q_slice = q_quant[chunk.token_start : chunk.token_end]
|
||||
q_scale_slice = (
|
||||
q_scale[chunk.token_start : chunk.token_end]
|
||||
if q_scale is not None
|
||||
else None
|
||||
)
|
||||
# DeepGEMM scalar-type tags (zero-copy): MXFP4 values → int8
|
||||
# (kPackedFP4), scales → int32 squeezed to 1-D kv_sf / 2-D q_sf.
|
||||
if use_fp4_cache:
|
||||
q_slice_cast = q_slice.view(torch.int8)
|
||||
k_quant_cast = k_quant.view(torch.int8)
|
||||
k_scale_cast = k_scale.view(torch.int32).squeeze(-1)
|
||||
else:
|
||||
q_slice_cast = q_slice
|
||||
k_quant_cast = k_quant
|
||||
k_scale_cast = k_scale.view(torch.float32).squeeze(-1)
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
rocm_fp8_mqa_logits,
|
||||
)
|
||||
|
||||
assert q_scale_slice is None
|
||||
logits = rocm_fp8_mqa_logits(
|
||||
q_slice_cast,
|
||||
(k_quant_cast, k_scale_cast),
|
||||
weights[chunk.token_start : chunk.token_end],
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
)
|
||||
num_rows = logits.shape[0]
|
||||
|
||||
# kpool: logits are pool-granular (compress_ratio == index_kpool),
|
||||
# so topk selects pools. We pick topk_tokens // kpool pools then
|
||||
# expand each pool back to its kpool constituent tokens.
|
||||
select_k = topk_tokens // index_kpool if index_kpool > 1 else topk_tokens
|
||||
if index_kpool > 1:
|
||||
pool_topk = torch.empty(
|
||||
(num_rows, select_k), dtype=torch.int32, device=logits.device
|
||||
)
|
||||
topk_dst = pool_topk
|
||||
else:
|
||||
topk_dst = topk_indices_buffer[
|
||||
chunk.token_start : chunk.token_end, :topk_tokens
|
||||
]
|
||||
|
||||
torch.ops._C.top_k_per_row_prefill(
|
||||
logits,
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
topk_dst,
|
||||
num_rows,
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
select_k,
|
||||
)
|
||||
|
||||
if index_kpool > 1:
|
||||
pool_ids = pool_topk.to(torch.int64)
|
||||
if positions is not None:
|
||||
# Fused expand-pools + append-tail into one Triton kernel
|
||||
# (replaces ~25 elementwise ops). seq_len is token-granular
|
||||
# (pos+1); the kernel derives pool_len internally.
|
||||
q_seq = (
|
||||
positions[chunk.token_start : chunk.token_end].to(torch.int32)
|
||||
+ 1
|
||||
)
|
||||
expanded = kpool_ops.expand_pools_and_append_tail(
|
||||
pool_ids, q_seq, index_kpool
|
||||
)
|
||||
else:
|
||||
valid = pool_ids >= 0
|
||||
expanded = kpool_ops.expand_pools_to_tokens(
|
||||
pool_ids, valid, topk_tokens, index_kpool
|
||||
)
|
||||
topk_indices_buffer[
|
||||
chunk.token_start : chunk.token_end, : expanded.shape[-1]
|
||||
] = expanded
|
||||
|
||||
if has_decode:
|
||||
decode_metadata = attn_metadata_narrowed.decode
|
||||
assert decode_metadata is not None
|
||||
kv_cache_raw = kv_cache # raw [num_blocks, block_size, head_dim+4] for writes
|
||||
kv_cache = kv_cache_as_quant_view(kv_cache, head_dim, use_fp4_cache)
|
||||
|
||||
# Update the tail before reading logits; completed pools are compressed
|
||||
# into the slot supplied by slot_mapping.
|
||||
# Spec verification groups tokens by request and preserves position
|
||||
# order so each token is stashed before the next completes its pool.
|
||||
# Positions must remain token-granular because the kernel derives the
|
||||
# pool phase and tail index from ``pos % kpool``.
|
||||
if (
|
||||
index_kpool > 1
|
||||
and gate_score is not None
|
||||
and compress_ape is not None
|
||||
and positions is not None
|
||||
and not skip_k_cache_insert
|
||||
):
|
||||
num_requests = attn_metadata_narrowed.num_decodes
|
||||
# Kpool writes must recover the original request grouping after the
|
||||
# indexer's flattened decode path. Host metadata avoids a CUDA graph
|
||||
# sync when choosing the uniform or padded layout.
|
||||
per_req_lens = decode_metadata.per_req_decode_lens
|
||||
if per_req_lens is not None:
|
||||
use_uniform = (
|
||||
decode_metadata.decode_is_uniform
|
||||
and num_decode_tokens
|
||||
== num_requests * decode_metadata.write_max_decode_len
|
||||
)
|
||||
group_lens = per_req_lens
|
||||
lmax = decode_metadata.write_max_decode_len
|
||||
else:
|
||||
# Legacy metadata without per-request lens: fall back to the
|
||||
# host-side requires_padding flag. Unreached now (per-request
|
||||
# lens is always populated for decode), kept defensive.
|
||||
use_uniform = not decode_metadata.requires_padding
|
||||
group_lens = decode_metadata.decode_lens
|
||||
lmax = int(decode_metadata.decode_lens.max().item())
|
||||
if not use_uniform:
|
||||
# Non-uniform decode_lens (mixed plain-decode + spec-verify, or
|
||||
# a variable MTP-verify batch): scatter actual tokens into a
|
||||
# padded [B, lmax] layout. int32 tensors can't go through
|
||||
# pack_seq_triton (float/uint8 only). The scatter indices are
|
||||
# shared by all five scatters below (and the tail slot one).
|
||||
scatter_idx = _build_decode_scatter_indices(
|
||||
group_lens, num_requests, num_decode_tokens
|
||||
)
|
||||
dec_k = _scatter_decode_tokens_by_request(
|
||||
k[:num_decode_tokens], 0, num_requests, lmax, scatter_idx
|
||||
)
|
||||
dec_gate = _scatter_decode_tokens_by_request(
|
||||
gate_score[:num_decode_tokens],
|
||||
0,
|
||||
num_requests,
|
||||
lmax,
|
||||
scatter_idx,
|
||||
)
|
||||
dec_slot = _scatter_decode_tokens_by_request(
|
||||
slot_mapping[:num_decode_tokens],
|
||||
-1,
|
||||
num_requests,
|
||||
lmax,
|
||||
scatter_idx,
|
||||
)
|
||||
dec_pos = _scatter_decode_tokens_by_request(
|
||||
positions[:num_decode_tokens].to(torch.int32),
|
||||
-1,
|
||||
num_requests,
|
||||
lmax,
|
||||
scatter_idx,
|
||||
)
|
||||
else:
|
||||
next_n = num_decode_tokens // num_requests
|
||||
shape2 = (num_requests, next_n)
|
||||
dec_k = k[:num_decode_tokens].view(*shape2, head_dim)
|
||||
dec_gate = gate_score[:num_decode_tokens].view(*shape2, head_dim)
|
||||
dec_slot = slot_mapping[:num_decode_tokens].view(shape2)
|
||||
dec_pos = positions[:num_decode_tokens].to(torch.int32).view(shape2)
|
||||
tail_meta = (
|
||||
attn_metadata.get(_resolve_layer_name(tail_prefix))
|
||||
if tail_prefix is not None
|
||||
else None
|
||||
)
|
||||
# Paged tail cache replaces the transient _DECODE_TAIL ring. Group
|
||||
# the tail group's token-granular slot_mapping per-request, mirroring
|
||||
# dec_slot / dec_pos, so the kernel gets each request's current-token
|
||||
# tail slot (block * kpool + pos % kpool).
|
||||
if tail_meta is not None:
|
||||
assert isinstance(tail_meta, DeepseekV32IndexerMetadata)
|
||||
if tail_meta is None or tail_kv_cache is None:
|
||||
dec_tail_slot = None
|
||||
elif not use_uniform:
|
||||
dec_tail_slot = _scatter_decode_tokens_by_request(
|
||||
tail_meta.slot_mapping[:num_decode_tokens],
|
||||
-1,
|
||||
num_requests,
|
||||
lmax,
|
||||
scatter_idx,
|
||||
)
|
||||
else:
|
||||
dec_tail_slot = tail_meta.slot_mapping[:num_decode_tokens].view(shape2)
|
||||
# The compress kernel writes the raw fp8 cache (not the quant view);
|
||||
# pass the underlying kv_cache, not kv_cache_quant_view.
|
||||
if dec_tail_slot is not None:
|
||||
# Single batched launch over [num_requests, next_n] replaces the
|
||||
# per-token sequential loop. The kernel iterates each request's
|
||||
# tokens in position order internally, preserving the
|
||||
# pool-completion read-after-stash dependency that the loop
|
||||
# provided. Inputs are already grouped per request (uniform:
|
||||
# view; non-uniform: _scatter_decode_tokens_by_request padded to
|
||||
# [B, lmax]) — no per-token .contiguous() copies needed.
|
||||
kpool_ops.kpool_decode_update_and_maybe_write_cache_batched(
|
||||
kv_cache_raw,
|
||||
tail_kv_cache,
|
||||
dec_tail_slot,
|
||||
dec_k,
|
||||
dec_gate,
|
||||
compress_ape,
|
||||
dec_slot,
|
||||
dec_pos,
|
||||
index_kpool,
|
||||
head_dim,
|
||||
round_scale=(scale_fmt is not None),
|
||||
)
|
||||
if current_platform.is_cuda_alike() and _fill_short_decode_causal_indices(
|
||||
topk_indices_buffer,
|
||||
positions,
|
||||
num_decode_tokens,
|
||||
attn_metadata_narrowed.max_seq_len,
|
||||
topk_tokens,
|
||||
):
|
||||
return topk_indices_buffer
|
||||
decode_lens = decode_metadata.decode_lens
|
||||
if decode_metadata.requires_padding:
|
||||
# Padding also covers short chunked prefills classified as decode.
|
||||
# MXFP4 uses zero-byte padding so padded slots dequantize to zero.
|
||||
if q_scale is not None:
|
||||
padded_q_quant_decode_tokens = pack_seq_triton(
|
||||
q_quant[:num_decode_tokens], decode_lens, pad_value=0
|
||||
)
|
||||
padded_q_scale = pack_seq_triton(
|
||||
q_scale[:num_decode_tokens], decode_lens, pad_value=0
|
||||
)
|
||||
else:
|
||||
padded_q_quant_decode_tokens = pack_seq_triton(
|
||||
q_quant[:num_decode_tokens], decode_lens
|
||||
)
|
||||
padded_q_scale = None
|
||||
padded_weights = pack_seq_triton(
|
||||
weights[:num_decode_tokens], decode_lens, pad_value=0
|
||||
).reshape(-1, *weights.shape[1:])
|
||||
else:
|
||||
padded_q_quant_decode_tokens = q_quant[:num_decode_tokens].reshape(
|
||||
decode_lens.shape[0], -1, *q_quant.shape[1:]
|
||||
)
|
||||
if q_scale is not None:
|
||||
padded_q_scale = q_scale[:num_decode_tokens].reshape(
|
||||
decode_lens.shape[0], -1, *q_scale.shape[1:]
|
||||
)
|
||||
else:
|
||||
padded_q_scale = None
|
||||
padded_weights = weights[:num_decode_tokens]
|
||||
# TODO: move and optimize below logic with triton kernels
|
||||
batch_size = padded_q_quant_decode_tokens.shape[0]
|
||||
next_n = padded_q_quant_decode_tokens.shape[1]
|
||||
num_padded_tokens = batch_size * next_n
|
||||
seq_lens = decode_metadata.seq_lens[:batch_size]
|
||||
# seq_lens is always 2D: (B, next_n) for native spec decode, (B, 1)
|
||||
# otherwise. deep_gemm fp8_fp4_paged_mqa_logits requires 2D context_lens;
|
||||
# the downstream topk kernels accept both 1D and 2D.
|
||||
padded_q_quant_cast = (
|
||||
padded_q_quant_decode_tokens.view(torch.int8)
|
||||
if use_fp4_cache
|
||||
else padded_q_quant_decode_tokens
|
||||
)
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
rocm_fp8_paged_mqa_logits,
|
||||
)
|
||||
|
||||
assert padded_q_scale is None
|
||||
logits = rocm_fp8_paged_mqa_logits(
|
||||
padded_q_quant_cast,
|
||||
kv_cache,
|
||||
padded_weights[:num_padded_tokens],
|
||||
seq_lens,
|
||||
decode_metadata.block_table,
|
||||
decode_metadata.schedule_metadata,
|
||||
max_model_len=max_model_len,
|
||||
)
|
||||
num_rows = logits.shape[0]
|
||||
# kpool: logits are pool-granular -> select topk_tokens//kpool pools,
|
||||
# then expand each pool back to its kpool tokens.
|
||||
select_k = topk_tokens // index_kpool if index_kpool > 1 else topk_tokens
|
||||
if index_kpool > 1:
|
||||
pool_topk = torch.empty(
|
||||
(num_rows, select_k), dtype=torch.int32, device=logits.device
|
||||
)
|
||||
topk_dst = pool_topk
|
||||
else:
|
||||
topk_dst = topk_indices_buffer[:num_padded_tokens, :topk_tokens]
|
||||
|
||||
torch.ops._C.top_k_per_row_decode(
|
||||
logits,
|
||||
next_n,
|
||||
seq_lens,
|
||||
topk_dst,
|
||||
num_rows,
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
select_k,
|
||||
)
|
||||
|
||||
# Resolve to token-level indices in the output buffer.
|
||||
if index_kpool > 1:
|
||||
pool_ids = pool_topk.to(torch.int64)
|
||||
n = pool_topk.shape[0]
|
||||
# Decode seq_lens are pool-granular; recover token lengths from
|
||||
# positions using the padded [B, next_n] row layout when needed.
|
||||
if positions is not None:
|
||||
dec_seq = _decode_topk_seq_lens(
|
||||
positions,
|
||||
decode_lens,
|
||||
num_decode_tokens,
|
||||
batch_size,
|
||||
next_n,
|
||||
decode_metadata.requires_padding,
|
||||
)
|
||||
else:
|
||||
dec_seq = decode_metadata.seq_lens[:n]
|
||||
if dec_seq.ndim == 2:
|
||||
dec_seq = dec_seq[:, -1]
|
||||
dec_seq = dec_seq.to(torch.int32)
|
||||
out = kpool_ops.expand_pools_and_append_tail(pool_ids, dec_seq, index_kpool)
|
||||
else:
|
||||
out = topk_dst
|
||||
|
||||
if decode_metadata.requires_padding:
|
||||
# Drop padded query rows introduced by the next_n padding above.
|
||||
out = unpack_seq_triton(
|
||||
out.reshape(batch_size, -1, out.shape[-1]), decode_lens
|
||||
)
|
||||
topk_indices_buffer[: out.shape[0], : out.shape[-1]] = out
|
||||
|
||||
return topk_indices_buffer
|
||||
|
||||
|
||||
@CustomOp.register("sparse_attn_indexer_kpool")
|
||||
class SparseAttnIndexerKpool(CustomOp):
|
||||
"""Sparse Attention Indexer Custom Op Layer. This layer is extracted as a
|
||||
separate custom op since it involves heavy custom kernels like `mqa_logits`,
|
||||
`paged_mqa_logits` and `top_k_per_row`, etc. Those kernels maybe requires
|
||||
specific memory layout or implementation for different hardware backends to
|
||||
achieve optimal performance.
|
||||
|
||||
For now, the default native path will use CUDA backend path. Other platform
|
||||
may requires add the corresponding Custom Op name `sparse_attn_indexer` to
|
||||
`custom_ops` in `CompilationConfig` to enable the platform specific path.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
k_cache,
|
||||
quant_block_size: int,
|
||||
scale_fmt: str,
|
||||
topk_tokens: int,
|
||||
head_dim: int,
|
||||
max_model_len: int,
|
||||
max_total_seq_len: int,
|
||||
topk_indices_buffer: torch.Tensor,
|
||||
skip_k_cache_insert: bool = False,
|
||||
use_fp4_cache: bool = False,
|
||||
tail_cache=None,
|
||||
):
|
||||
super().__init__()
|
||||
self.k_cache = k_cache
|
||||
self.tail_cache = tail_cache
|
||||
self.quant_block_size = quant_block_size
|
||||
self.scale_fmt = scale_fmt
|
||||
self.topk_tokens = topk_tokens
|
||||
self.head_dim = head_dim
|
||||
self.max_model_len = max_model_len
|
||||
self.max_total_seq_len = max_total_seq_len
|
||||
self.topk_indices_buffer = topk_indices_buffer
|
||||
self.skip_k_cache_insert = skip_k_cache_insert
|
||||
self.use_fp4_cache = use_fp4_cache
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
q_quant: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
k: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
*,
|
||||
gate_score: torch.Tensor | None = None,
|
||||
compress_ape: torch.Tensor | None = None,
|
||||
index_kpool: int = 1,
|
||||
positions: torch.Tensor | None = None,
|
||||
):
|
||||
assert not self.use_fp4_cache, "AMD platform doesn't support fp4 cache yet"
|
||||
assert isinstance(q_quant, torch.Tensor), (
|
||||
"AMD sparse_attn_indexer expects a single FP8 q_quant tensor"
|
||||
)
|
||||
if not rocm_aiter_ops.is_enabled():
|
||||
raise RuntimeError(
|
||||
"Sparse attention indexer ROCm path is only supported on AITER. "
|
||||
"Please enable aiter with VLLM_ROCM_USE_AITER=1"
|
||||
)
|
||||
if index_kpool <= 1:
|
||||
return torch.ops.vllm.rocm_aiter_sparse_attn_indexer(
|
||||
hidden_states,
|
||||
_encode_layer_name(self.k_cache.prefix),
|
||||
self.k_cache.kv_cache,
|
||||
q_quant,
|
||||
k,
|
||||
weights,
|
||||
self.quant_block_size,
|
||||
self.scale_fmt,
|
||||
self.topk_tokens,
|
||||
self.head_dim,
|
||||
self.max_model_len,
|
||||
self.max_total_seq_len,
|
||||
self.topk_indices_buffer,
|
||||
skip_k_cache_insert=self.skip_k_cache_insert,
|
||||
)
|
||||
return sparse_attn_indexer_kpool(
|
||||
hidden_states,
|
||||
self.k_cache.prefix,
|
||||
self.k_cache.kv_cache,
|
||||
q_quant,
|
||||
None,
|
||||
k,
|
||||
weights,
|
||||
self.quant_block_size,
|
||||
self.scale_fmt,
|
||||
self.topk_tokens,
|
||||
self.head_dim,
|
||||
self.max_model_len,
|
||||
self.max_total_seq_len,
|
||||
self.topk_indices_buffer,
|
||||
self.skip_k_cache_insert,
|
||||
self.use_fp4_cache,
|
||||
gate_score,
|
||||
compress_ape,
|
||||
index_kpool,
|
||||
positions,
|
||||
self.tail_cache.kv_cache if self.tail_cache is not None else None,
|
||||
self.tail_cache.prefix if self.tail_cache is not None else None,
|
||||
)
|
||||
@@ -23,7 +23,6 @@ from vllm.model_executor.layers.linear import (
|
||||
from vllm.model_executor.layers.mla import MLAModules, MultiHeadLatentAttentionWrapper
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding, get_rope
|
||||
from vllm.model_executor.layers.sparse_attn_indexer_kpool import SparseAttnIndexerKpool
|
||||
from vllm.model_executor.models.deepseek_v2 import (
|
||||
DeepSeekV2FusedQkvAProjLinear,
|
||||
DeepseekV32IndexerCache,
|
||||
@@ -31,6 +30,7 @@ from vllm.model_executor.models.deepseek_v2 import (
|
||||
)
|
||||
from vllm.model_executor.utils import maybe_disable_graph_partition
|
||||
from vllm.models.glm5next.nvidia.ops.kpool_compress import fwht128_quant_fp8
|
||||
from vllm.models.glm5next.sparse_indexer import SparseAttnIndexerKpool
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.configs.glm5_next import Glm5NextConfig
|
||||
from vllm.utils.deep_gemm import PAGED_MQA_PAGE_SIZES
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared helpers for the glm5next sparse attention indexer (kpool) layers."""
|
||||
|
||||
import torch
|
||||
|
||||
RADIX_TOPK_WORKSPACE_SIZE = 1024 * 1024
|
||||
|
||||
# MXFP4 layout: 2 values packed per byte, ue8m0 (1-byte) scale per block of 32.
|
||||
MXFP4_BLOCK_SIZE = 32
|
||||
|
||||
|
||||
def _build_decode_scatter_indices(
|
||||
decode_lens: torch.Tensor,
|
||||
num_requests: int,
|
||||
n: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Per-token (request id, intra-request index) for a non-uniform decode
|
||||
batch, with ``n == decode_lens.sum()`` as a host int (avoids a
|
||||
device sync and keeps both repeat_interleaves sync-free).
|
||||
|
||||
Shared by every ``_scatter_decode_tokens_by_request`` call in a step:
|
||||
building it per call would repeat the same repeat_interleave/cumsum chain
|
||||
up to 5x per layer on the eager decode break.
|
||||
"""
|
||||
device = decode_lens.device
|
||||
dl = decode_lens.to(torch.int64)
|
||||
req_id = torch.repeat_interleave(
|
||||
torch.arange(num_requests, device=device, dtype=torch.int64),
|
||||
dl,
|
||||
output_size=n,
|
||||
)
|
||||
req_starts = torch.cumsum(
|
||||
torch.cat([torch.zeros(1, device=device, dtype=torch.int64), dl[:-1]]),
|
||||
dim=0,
|
||||
)
|
||||
# Broadcast the per-request start offsets to per-token (length n ==
|
||||
# dl.sum()) so each token's intra-request index subtracts its own
|
||||
# request's start.
|
||||
starts = torch.repeat_interleave(req_starts, dl, output_size=n)
|
||||
intra = torch.arange(n, device=device, dtype=torch.int64) - starts
|
||||
return req_id, intra
|
||||
|
||||
|
||||
def _scatter_decode_tokens_by_request(
|
||||
tokens: torch.Tensor,
|
||||
pad_value,
|
||||
num_requests: int,
|
||||
lmax: int,
|
||||
scatter_indices: tuple[torch.Tensor, torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
"""Group ``[N, ...]`` decode tokens into a padded ``[num_requests, lmax, ...]``
|
||||
layout: request ``r``'s tokens at row ``r`` in order; short requests padded.
|
||||
|
||||
Unlike ``pack_seq_triton`` this is dtype-agnostic (needed for the int32
|
||||
slot/pos tensors) — it scatters with the shared per-step indices from
|
||||
``_build_decode_scatter_indices``. Used only for the non-uniform
|
||||
(``requires_padding``) decode batch; uniform batches use a zero-copy
|
||||
reshape.
|
||||
"""
|
||||
req_id, intra = scatter_indices
|
||||
out = torch.full(
|
||||
(num_requests, lmax, *tokens.shape[1:]),
|
||||
pad_value,
|
||||
dtype=tokens.dtype,
|
||||
device=tokens.device,
|
||||
)
|
||||
out[req_id, intra] = tokens
|
||||
return out
|
||||
|
||||
|
||||
def _decode_topk_seq_lens(
|
||||
positions: torch.Tensor,
|
||||
decode_lens: torch.Tensor,
|
||||
num_decode_tokens: int,
|
||||
batch_size: int,
|
||||
next_n: int,
|
||||
requires_padding: bool,
|
||||
) -> torch.Tensor:
|
||||
"""Token-granular seq_len (pos + 1) per pool-topk row, layout-aware.
|
||||
|
||||
``pool_topk`` (and the logits it comes from) follow the padded
|
||||
``[batch_size, next_n]`` grid whenever ``requires_padding`` is set, so row
|
||||
``(b, t)`` corresponds to flat decode token ``offset_b + t`` -- NOT
|
||||
``b * next_n + t``. Slicing flat ``positions[: batch_size * next_n]``
|
||||
(the uniform-layout shortcut) misaligns every row after the first
|
||||
non-uniform request and, past the decode region, reads prefill tokens'
|
||||
positions; ``expand_pools_and_append_tail`` then anchors the tail at
|
||||
another request's length, dropping the row's real tail tokens or emitting
|
||||
indices past its sequence (out-of-bounds block-table reads). Padded rows
|
||||
get 0 (empty tail); they are dropped by ``unpack_seq_triton`` anyway.
|
||||
"""
|
||||
n = batch_size * next_n
|
||||
if not requires_padding:
|
||||
return positions[:n].to(torch.int32) + 1
|
||||
scatter_idx = _build_decode_scatter_indices(
|
||||
decode_lens, batch_size, num_decode_tokens
|
||||
)
|
||||
padded = _scatter_decode_tokens_by_request(
|
||||
positions[:num_decode_tokens].to(torch.int32),
|
||||
-1,
|
||||
batch_size,
|
||||
next_n,
|
||||
scatter_idx,
|
||||
)
|
||||
return padded.reshape(n) + 1 # pad rows: -1 + 1 = 0 -> empty tail
|
||||
|
||||
|
||||
def _fill_causal_indices(rows: torch.Tensor, positions: torch.Tensor) -> None:
|
||||
causal_range = torch.arange(rows.shape[1], device=rows.device, dtype=torch.int32)
|
||||
positions = positions.to(torch.int32)
|
||||
rows[:] = causal_range[None, :]
|
||||
rows[causal_range[None, :] > positions[:, None]] = -1
|
||||
|
||||
|
||||
def _fill_short_decode_causal_indices(
|
||||
topk_indices_buffer: torch.Tensor,
|
||||
positions: torch.Tensor | None,
|
||||
num_decode_tokens: int,
|
||||
max_seq_len: int,
|
||||
topk_tokens: int,
|
||||
) -> bool:
|
||||
"""Fill exact causal rows when sparse decode would select every token."""
|
||||
if positions is None or positions.numel() == 0 or max_seq_len > topk_tokens:
|
||||
return False
|
||||
_fill_causal_indices(
|
||||
topk_indices_buffer[:num_decode_tokens], positions[:num_decode_tokens]
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def _gather_workspace_shapes(
|
||||
total_seq_lens: int,
|
||||
head_dim: int,
|
||||
fp8_dtype: torch.dtype,
|
||||
use_fp4_cache: bool,
|
||||
) -> tuple[tuple[tuple[int, int], torch.dtype], tuple[tuple[int, int], torch.dtype]]:
|
||||
"""Return ((values_shape, values_dtype), (scales_shape, scales_dtype)) for
|
||||
the K-gather workspace. FP8 path: (T, head_dim) fp8 + (T, 4) uint8 fp32
|
||||
scales. MXFP4 path: (T, head_dim // 2) uint8 packed mxfp4 +
|
||||
(T, head_dim // MXFP4_BLOCK_SIZE) uint8 ue8m0 scales."""
|
||||
if use_fp4_cache:
|
||||
return (
|
||||
((total_seq_lens, head_dim // 2), torch.uint8),
|
||||
((total_seq_lens, head_dim // MXFP4_BLOCK_SIZE), torch.uint8),
|
||||
)
|
||||
return (
|
||||
((total_seq_lens, head_dim), fp8_dtype),
|
||||
((total_seq_lens, 4), torch.uint8),
|
||||
)
|
||||
|
||||
|
||||
def kv_cache_as_quant_view(
|
||||
kv_cache: torch.Tensor,
|
||||
head_dim: int,
|
||||
use_fp4_cache: bool,
|
||||
) -> torch.Tensor:
|
||||
"""4D ``[num_blocks, block_size, 1, head_width]`` view expected by
|
||||
DeepGEMM, from the 3D indexer kv-cache allocation."""
|
||||
if use_fp4_cache:
|
||||
assert kv_cache.ndim == 3 and kv_cache.dtype == torch.uint8
|
||||
num_blocks, block_size, _ = kv_cache.shape
|
||||
page_bytes = int(kv_cache.stride(0))
|
||||
fp4_bytes = head_dim // 2 + head_dim // MXFP4_BLOCK_SIZE
|
||||
return torch.as_strided(
|
||||
kv_cache,
|
||||
size=(num_blocks, block_size, 1, fp4_bytes),
|
||||
stride=(page_bytes, fp4_bytes, fp4_bytes, 1),
|
||||
)
|
||||
return kv_cache.unsqueeze(-2)
|
||||
+75
-390
@@ -2,30 +2,29 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Custom Sparse Attention Indexer layers."""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import get_current_vllm_config_or_none
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
from vllm.models.glm5next.common.sparse_indexer import (
|
||||
RADIX_TOPK_WORKSPACE_SIZE,
|
||||
_build_decode_scatter_indices,
|
||||
_decode_topk_seq_lens,
|
||||
_fill_causal_indices,
|
||||
_fill_short_decode_causal_indices,
|
||||
_gather_workspace_shapes,
|
||||
_scatter_decode_tokens_by_request,
|
||||
kv_cache_as_quant_view,
|
||||
)
|
||||
from vllm.models.glm5next.nvidia.ops import kpool_compress as kpool_ops
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.models.glm5next.nvidia.ops import kpool_compress as kpool_ops
|
||||
elif current_platform.is_rocm():
|
||||
from vllm.models.glm5next.amd.ops import kpool_compress as kpool_ops
|
||||
else:
|
||||
from vllm.models.glm5next.nvidia.ops import kpool_compress as kpool_ops
|
||||
|
||||
from vllm.utils.deep_gemm import has_deep_gemm
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
_resolve_layer_name,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
@@ -36,16 +35,9 @@ from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
if current_platform.is_cuda_alike():
|
||||
from vllm import _custom_ops as ops
|
||||
elif current_platform.is_xpu():
|
||||
from vllm._xpu_ops import xpu_ops
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
RADIX_TOPK_WORKSPACE_SIZE = 1024 * 1024
|
||||
|
||||
# MXFP4 layout: 2 values packed per byte, ue8m0 (1-byte) scale per block of 32.
|
||||
MXFP4_BLOCK_SIZE = 32
|
||||
|
||||
# kpool write helper: form pools from the current token batch and compress them
|
||||
# into the index K cache via the fused Triton kernel.
|
||||
|
||||
@@ -96,166 +88,6 @@ def _kpool_compress_insert(
|
||||
)
|
||||
|
||||
|
||||
def _build_decode_scatter_indices(
|
||||
decode_lens: torch.Tensor,
|
||||
num_requests: int,
|
||||
n: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Per-token (request id, intra-request index) for a non-uniform decode
|
||||
batch, with ``n == decode_lens.sum()`` as a host int (avoids a
|
||||
device sync and keeps both repeat_interleaves sync-free).
|
||||
|
||||
Shared by every ``_scatter_decode_tokens_by_request`` call in a step:
|
||||
building it per call would repeat the same repeat_interleave/cumsum chain
|
||||
up to 5x per layer on the eager decode break.
|
||||
"""
|
||||
device = decode_lens.device
|
||||
dl = decode_lens.to(torch.int64)
|
||||
req_id = torch.repeat_interleave(
|
||||
torch.arange(num_requests, device=device, dtype=torch.int64),
|
||||
dl,
|
||||
output_size=n,
|
||||
)
|
||||
req_starts = torch.cumsum(
|
||||
torch.cat([torch.zeros(1, device=device, dtype=torch.int64), dl[:-1]]),
|
||||
dim=0,
|
||||
)
|
||||
# Broadcast the per-request start offsets to per-token (length n ==
|
||||
# dl.sum()) so each token's intra-request index subtracts its own
|
||||
# request's start.
|
||||
starts = torch.repeat_interleave(req_starts, dl, output_size=n)
|
||||
intra = torch.arange(n, device=device, dtype=torch.int64) - starts
|
||||
return req_id, intra
|
||||
|
||||
|
||||
def _scatter_decode_tokens_by_request(
|
||||
tokens: torch.Tensor,
|
||||
pad_value,
|
||||
num_requests: int,
|
||||
lmax: int,
|
||||
scatter_indices: tuple[torch.Tensor, torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
"""Group ``[N, ...]`` decode tokens into a padded ``[num_requests, lmax, ...]``
|
||||
layout: request ``r``'s tokens at row ``r`` in order; short requests padded.
|
||||
|
||||
Unlike ``pack_seq_triton`` this is dtype-agnostic (needed for the int32
|
||||
slot/pos tensors) — it scatters with the shared per-step indices from
|
||||
``_build_decode_scatter_indices``. Used only for the non-uniform
|
||||
(``requires_padding``) decode batch; uniform batches use a zero-copy
|
||||
reshape.
|
||||
"""
|
||||
req_id, intra = scatter_indices
|
||||
out = torch.full(
|
||||
(num_requests, lmax, *tokens.shape[1:]),
|
||||
pad_value,
|
||||
dtype=tokens.dtype,
|
||||
device=tokens.device,
|
||||
)
|
||||
out[req_id, intra] = tokens
|
||||
return out
|
||||
|
||||
|
||||
def _decode_topk_seq_lens(
|
||||
positions: torch.Tensor,
|
||||
decode_lens: torch.Tensor,
|
||||
num_decode_tokens: int,
|
||||
batch_size: int,
|
||||
next_n: int,
|
||||
requires_padding: bool,
|
||||
) -> torch.Tensor:
|
||||
"""Token-granular seq_len (pos + 1) per pool-topk row, layout-aware.
|
||||
|
||||
``pool_topk`` (and the logits it comes from) follow the padded
|
||||
``[batch_size, next_n]`` grid whenever ``requires_padding`` is set, so row
|
||||
``(b, t)`` corresponds to flat decode token ``offset_b + t`` -- NOT
|
||||
``b * next_n + t``. Slicing flat ``positions[: batch_size * next_n]``
|
||||
(the uniform-layout shortcut) misaligns every row after the first
|
||||
non-uniform request and, past the decode region, reads prefill tokens'
|
||||
positions; ``expand_pools_and_append_tail`` then anchors the tail at
|
||||
another request's length, dropping the row's real tail tokens or emitting
|
||||
indices past its sequence (out-of-bounds block-table reads). Padded rows
|
||||
get 0 (empty tail); they are dropped by ``unpack_seq_triton`` anyway.
|
||||
"""
|
||||
n = batch_size * next_n
|
||||
if not requires_padding:
|
||||
return positions[:n].to(torch.int32) + 1
|
||||
scatter_idx = _build_decode_scatter_indices(
|
||||
decode_lens, batch_size, num_decode_tokens
|
||||
)
|
||||
padded = _scatter_decode_tokens_by_request(
|
||||
positions[:num_decode_tokens].to(torch.int32),
|
||||
-1,
|
||||
batch_size,
|
||||
next_n,
|
||||
scatter_idx,
|
||||
)
|
||||
return padded.reshape(n) + 1 # pad rows: -1 + 1 = 0 -> empty tail
|
||||
|
||||
|
||||
def _fill_causal_indices(rows: torch.Tensor, positions: torch.Tensor) -> None:
|
||||
causal_range = torch.arange(rows.shape[1], device=rows.device, dtype=torch.int32)
|
||||
positions = positions.to(torch.int32)
|
||||
rows[:] = causal_range[None, :]
|
||||
rows[causal_range[None, :] > positions[:, None]] = -1
|
||||
|
||||
|
||||
def _fill_short_decode_causal_indices(
|
||||
topk_indices_buffer: torch.Tensor,
|
||||
positions: torch.Tensor | None,
|
||||
num_decode_tokens: int,
|
||||
max_seq_len: int,
|
||||
topk_tokens: int,
|
||||
) -> bool:
|
||||
"""Fill exact causal rows when sparse decode would select every token."""
|
||||
if positions is None or positions.numel() == 0 or max_seq_len > topk_tokens:
|
||||
return False
|
||||
_fill_causal_indices(
|
||||
topk_indices_buffer[:num_decode_tokens], positions[:num_decode_tokens]
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def _gather_workspace_shapes(
|
||||
total_seq_lens: int,
|
||||
head_dim: int,
|
||||
fp8_dtype: torch.dtype,
|
||||
use_fp4_cache: bool,
|
||||
) -> tuple[tuple[tuple[int, int], torch.dtype], tuple[tuple[int, int], torch.dtype]]:
|
||||
"""Return ((values_shape, values_dtype), (scales_shape, scales_dtype)) for
|
||||
the K-gather workspace. FP8 path: (T, head_dim) fp8 + (T, 4) uint8 fp32
|
||||
scales. MXFP4 path: (T, head_dim // 2) uint8 packed mxfp4 +
|
||||
(T, head_dim // MXFP4_BLOCK_SIZE) uint8 ue8m0 scales."""
|
||||
if use_fp4_cache:
|
||||
return (
|
||||
((total_seq_lens, head_dim // 2), torch.uint8),
|
||||
((total_seq_lens, head_dim // MXFP4_BLOCK_SIZE), torch.uint8),
|
||||
)
|
||||
return (
|
||||
((total_seq_lens, head_dim), fp8_dtype),
|
||||
((total_seq_lens, 4), torch.uint8),
|
||||
)
|
||||
|
||||
|
||||
def kv_cache_as_quant_view(
|
||||
kv_cache: torch.Tensor,
|
||||
head_dim: int,
|
||||
use_fp4_cache: bool,
|
||||
) -> torch.Tensor:
|
||||
"""4D ``[num_blocks, block_size, 1, head_width]`` view expected by
|
||||
DeepGEMM, from the 3D indexer kv-cache allocation."""
|
||||
if use_fp4_cache:
|
||||
assert kv_cache.ndim == 3 and kv_cache.dtype == torch.uint8
|
||||
num_blocks, block_size, _ = kv_cache.shape
|
||||
page_bytes = int(kv_cache.stride(0))
|
||||
fp4_bytes = head_dim // 2 + head_dim // MXFP4_BLOCK_SIZE
|
||||
return torch.as_strided(
|
||||
kv_cache,
|
||||
size=(num_blocks, block_size, 1, fp4_bytes),
|
||||
stride=(page_bytes, fp4_bytes, fp4_bytes, 1),
|
||||
)
|
||||
return kv_cache.unsqueeze(-2)
|
||||
|
||||
|
||||
@eager_break_during_capture
|
||||
def sparse_attn_indexer_kpool(
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -393,26 +225,13 @@ def sparse_attn_indexer_kpool(
|
||||
else:
|
||||
# standard: per-token fp8 quant + scatter (all tokens).
|
||||
assert scale_fmt is not None
|
||||
if current_platform.is_rocm():
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
indexer_k_quant_and_cache_triton,
|
||||
)
|
||||
|
||||
indexer_k_quant_and_cache_triton(
|
||||
k,
|
||||
kv_cache,
|
||||
slot_mapping,
|
||||
quant_block_size,
|
||||
scale_fmt,
|
||||
)
|
||||
else:
|
||||
ops.indexer_k_quant_and_cache(
|
||||
k,
|
||||
kv_cache,
|
||||
slot_mapping,
|
||||
quant_block_size,
|
||||
scale_fmt,
|
||||
)
|
||||
ops.indexer_k_quant_and_cache(
|
||||
k,
|
||||
kv_cache,
|
||||
slot_mapping,
|
||||
quant_block_size,
|
||||
scale_fmt,
|
||||
)
|
||||
|
||||
topk_indices_buffer[: hidden_states.shape[0]] = -1
|
||||
if has_prefill:
|
||||
@@ -466,27 +285,13 @@ def sparse_attn_indexer_kpool(
|
||||
k_scale = k_scale_full[: chunk.total_seq_lens]
|
||||
|
||||
if not chunk.skip_kv_gather:
|
||||
if current_platform.is_rocm():
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
cp_gather_indexer_k_quant_cache_triton,
|
||||
)
|
||||
|
||||
cp_gather_indexer_k_quant_cache_triton(
|
||||
kv_cache,
|
||||
k_quant,
|
||||
k_scale,
|
||||
chunk.block_table,
|
||||
chunk.cu_seq_lens,
|
||||
token_to_seq=chunk.token_to_seq,
|
||||
)
|
||||
else:
|
||||
ops.cp_gather_indexer_k_quant_cache(
|
||||
kv_cache,
|
||||
k_quant,
|
||||
k_scale,
|
||||
chunk.block_table,
|
||||
chunk.cu_seq_lens,
|
||||
)
|
||||
ops.cp_gather_indexer_k_quant_cache(
|
||||
kv_cache,
|
||||
k_quant,
|
||||
k_scale,
|
||||
chunk.block_table,
|
||||
chunk.cu_seq_lens,
|
||||
)
|
||||
|
||||
q_slice = q_quant[chunk.token_start : chunk.token_end]
|
||||
q_scale_slice = (
|
||||
@@ -504,30 +309,16 @@ def sparse_attn_indexer_kpool(
|
||||
q_slice_cast = q_slice
|
||||
k_quant_cast = k_quant
|
||||
k_scale_cast = k_scale.view(torch.float32).squeeze(-1)
|
||||
if current_platform.is_rocm():
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
rocm_fp8_mqa_logits,
|
||||
)
|
||||
from vllm.utils.deep_gemm import fp8_fp4_mqa_logits
|
||||
|
||||
assert q_scale_slice is None
|
||||
logits = rocm_fp8_mqa_logits(
|
||||
q_slice_cast,
|
||||
(k_quant_cast, k_scale_cast),
|
||||
weights[chunk.token_start : chunk.token_end],
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
)
|
||||
else:
|
||||
from vllm.utils.deep_gemm import fp8_fp4_mqa_logits
|
||||
|
||||
logits = fp8_fp4_mqa_logits(
|
||||
(q_slice_cast, q_scale_slice),
|
||||
(k_quant_cast, k_scale_cast),
|
||||
weights[chunk.token_start : chunk.token_end],
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
clean_logits=False,
|
||||
)
|
||||
logits = fp8_fp4_mqa_logits(
|
||||
(q_slice_cast, q_scale_slice),
|
||||
(k_quant_cast, k_scale_cast),
|
||||
weights[chunk.token_start : chunk.token_end],
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
clean_logits=False,
|
||||
)
|
||||
num_rows = logits.shape[0]
|
||||
|
||||
# kpool: logits are pool-granular (compress_ratio == index_kpool),
|
||||
@@ -544,28 +335,16 @@ def sparse_attn_indexer_kpool(
|
||||
chunk.token_start : chunk.token_end, :topk_tokens
|
||||
]
|
||||
|
||||
if current_platform.is_xpu():
|
||||
xpu_ops.top_k_per_row_prefill( # type: ignore[attr-defined]
|
||||
logits,
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
topk_dst,
|
||||
num_rows,
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
select_k,
|
||||
)
|
||||
else:
|
||||
torch.ops._C.top_k_per_row_prefill(
|
||||
logits,
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
topk_dst,
|
||||
num_rows,
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
select_k,
|
||||
)
|
||||
torch.ops._C.top_k_per_row_prefill(
|
||||
logits,
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
topk_dst,
|
||||
num_rows,
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
select_k,
|
||||
)
|
||||
|
||||
if index_kpool > 1:
|
||||
pool_ids = pool_topk.to(torch.int64)
|
||||
@@ -765,34 +544,18 @@ def sparse_attn_indexer_kpool(
|
||||
if use_fp4_cache
|
||||
else padded_q_quant_decode_tokens
|
||||
)
|
||||
if current_platform.is_rocm():
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
rocm_fp8_paged_mqa_logits,
|
||||
)
|
||||
from vllm.utils.deep_gemm import fp8_fp4_paged_mqa_logits
|
||||
|
||||
assert padded_q_scale is None
|
||||
logits = rocm_fp8_paged_mqa_logits(
|
||||
padded_q_quant_cast,
|
||||
kv_cache,
|
||||
padded_weights[:num_padded_tokens],
|
||||
seq_lens,
|
||||
decode_metadata.block_table,
|
||||
decode_metadata.schedule_metadata,
|
||||
max_model_len=max_model_len,
|
||||
)
|
||||
else:
|
||||
from vllm.utils.deep_gemm import fp8_fp4_paged_mqa_logits
|
||||
|
||||
logits = fp8_fp4_paged_mqa_logits(
|
||||
(padded_q_quant_cast, padded_q_scale),
|
||||
kv_cache,
|
||||
padded_weights[:num_padded_tokens],
|
||||
seq_lens,
|
||||
decode_metadata.block_table,
|
||||
decode_metadata.schedule_metadata,
|
||||
max_model_len=max_model_len,
|
||||
clean_logits=False,
|
||||
)
|
||||
logits = fp8_fp4_paged_mqa_logits(
|
||||
(padded_q_quant_cast, padded_q_scale),
|
||||
kv_cache,
|
||||
padded_weights[:num_padded_tokens],
|
||||
seq_lens,
|
||||
decode_metadata.block_table,
|
||||
decode_metadata.schedule_metadata,
|
||||
max_model_len=max_model_len,
|
||||
clean_logits=False,
|
||||
)
|
||||
num_rows = logits.shape[0]
|
||||
# kpool: logits are pool-granular -> select topk_tokens//kpool pools,
|
||||
# then expand each pool back to its kpool tokens.
|
||||
@@ -819,28 +582,16 @@ def sparse_attn_indexer_kpool(
|
||||
attn_metadata_narrowed.max_seq_len,
|
||||
)
|
||||
else:
|
||||
if current_platform.is_xpu():
|
||||
xpu_ops.top_k_per_row_decode( # type: ignore[attr-defined]
|
||||
logits,
|
||||
next_n,
|
||||
seq_lens,
|
||||
topk_dst,
|
||||
num_rows,
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
select_k,
|
||||
)
|
||||
else:
|
||||
torch.ops._C.top_k_per_row_decode(
|
||||
logits,
|
||||
next_n,
|
||||
seq_lens,
|
||||
topk_dst,
|
||||
num_rows,
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
select_k,
|
||||
)
|
||||
torch.ops._C.top_k_per_row_decode(
|
||||
logits,
|
||||
next_n,
|
||||
seq_lens,
|
||||
topk_dst,
|
||||
num_rows,
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
select_k,
|
||||
)
|
||||
|
||||
# Resolve to token-level indices in the output buffer.
|
||||
if index_kpool > 1:
|
||||
@@ -942,33 +693,16 @@ class SparseAttnIndexerKpool(CustomOp):
|
||||
index_kpool: int = 1,
|
||||
positions: torch.Tensor | None = None,
|
||||
):
|
||||
if current_platform.is_cuda() or current_platform.is_xpu():
|
||||
return self.forward_cuda(
|
||||
hidden_states,
|
||||
q_quant,
|
||||
k,
|
||||
weights,
|
||||
gate_score=gate_score,
|
||||
compress_ape=compress_ape,
|
||||
index_kpool=index_kpool,
|
||||
positions=positions,
|
||||
)
|
||||
elif current_platform.is_rocm():
|
||||
return self.forward_hip(
|
||||
hidden_states,
|
||||
q_quant,
|
||||
k,
|
||||
weights,
|
||||
gate_score=gate_score,
|
||||
compress_ape=compress_ape,
|
||||
index_kpool=index_kpool,
|
||||
positions=positions,
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
"SparseAttnIndexer native forward is only implemented for "
|
||||
"CUDA, ROCm and XPU platforms."
|
||||
)
|
||||
return self.forward_cuda(
|
||||
hidden_states,
|
||||
q_quant,
|
||||
k,
|
||||
weights,
|
||||
gate_score=gate_score,
|
||||
compress_ape=compress_ape,
|
||||
index_kpool=index_kpool,
|
||||
positions=positions,
|
||||
)
|
||||
|
||||
def forward_cuda(
|
||||
self,
|
||||
@@ -1012,52 +746,3 @@ class SparseAttnIndexerKpool(CustomOp):
|
||||
self.tail_cache.kv_cache if self.tail_cache is not None else None,
|
||||
self.tail_cache.prefix if self.tail_cache is not None else None,
|
||||
)
|
||||
|
||||
def forward_hip(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
q_quant: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
k: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
*,
|
||||
gate_score: torch.Tensor | None = None,
|
||||
compress_ape: torch.Tensor | None = None,
|
||||
index_kpool: int = 1,
|
||||
positions: torch.Tensor | None = None,
|
||||
):
|
||||
assert not self.use_fp4_cache, "AMD platform doesn't support fp4 cache yet"
|
||||
assert isinstance(q_quant, torch.Tensor), (
|
||||
"AMD sparse_attn_indexer expects a single FP8 q_quant tensor"
|
||||
)
|
||||
if rocm_aiter_ops.is_enabled():
|
||||
if index_kpool <= 1:
|
||||
return torch.ops.vllm.rocm_aiter_sparse_attn_indexer(
|
||||
hidden_states,
|
||||
_encode_layer_name(self.k_cache.prefix),
|
||||
self.k_cache.kv_cache,
|
||||
q_quant,
|
||||
k,
|
||||
weights,
|
||||
self.quant_block_size,
|
||||
self.scale_fmt,
|
||||
self.topk_tokens,
|
||||
self.head_dim,
|
||||
self.max_model_len,
|
||||
self.max_total_seq_len,
|
||||
self.topk_indices_buffer,
|
||||
skip_k_cache_insert=self.skip_k_cache_insert,
|
||||
)
|
||||
return self.forward_cuda(
|
||||
hidden_states,
|
||||
q_quant,
|
||||
k,
|
||||
weights,
|
||||
gate_score=gate_score,
|
||||
compress_ape=compress_ape,
|
||||
index_kpool=index_kpool,
|
||||
positions=positions,
|
||||
)
|
||||
raise RuntimeError(
|
||||
"Sparse attention indexer ROCm path is only supported on AITER. "
|
||||
"Please enable aiter with VLLM_ROCM_USE_AITER=1"
|
||||
)
|
||||
@@ -0,0 +1,22 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Sparse attention indexer (kpool) — hardware-isolated entry point.
|
||||
|
||||
The implementation lives under ``nvidia/`` and ``amd/`` (with shared helpers
|
||||
in ``common/``); this module picks the right one for the current platform.
|
||||
"""
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from .amd.sparse_indexer import (
|
||||
SparseAttnIndexerKpool,
|
||||
)
|
||||
else:
|
||||
from .nvidia.sparse_indexer import (
|
||||
SparseAttnIndexerKpool,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"SparseAttnIndexerKpool",
|
||||
]
|
||||
+19
-10
@@ -48,6 +48,19 @@ def _get_hasher_factory(
|
||||
raise ValueError(f"Unsupported hash algorithm: {algorithm}")
|
||||
|
||||
|
||||
def _get_image_id_bytes(image: Image.Image) -> bytes | None:
|
||||
try:
|
||||
exif = image.getexif()
|
||||
image_id = exif.get(Image.ExifTags.Base.ImageID)
|
||||
if isinstance(image_id, uuid.UUID):
|
||||
return image_id.bytes
|
||||
except Exception:
|
||||
# Tolerate malformed EXIF metadata (e.g. invalid TIFF header)
|
||||
# and fall back to serializing raw image data or bytes.
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
class MultiModalHasher:
|
||||
@classmethod
|
||||
def serialize_item(cls, obj: object) -> Iterable[bytes | memoryview]:
|
||||
@@ -60,11 +73,9 @@ class MultiModalHasher:
|
||||
return (np.array(obj).tobytes(),)
|
||||
|
||||
if isinstance(obj, Image.Image):
|
||||
exif = obj.getexif()
|
||||
if Image.ExifTags.Base.ImageID in exif and isinstance(
|
||||
exif[Image.ExifTags.Base.ImageID], uuid.UUID
|
||||
):
|
||||
return (exif[Image.ExifTags.Base.ImageID].bytes,)
|
||||
image_id = _get_image_id_bytes(obj)
|
||||
if image_id is not None:
|
||||
return (image_id,)
|
||||
|
||||
data = {"mode": obj.mode, "data": np.asarray(obj)}
|
||||
palette = obj.palette
|
||||
@@ -76,11 +87,9 @@ class MultiModalHasher:
|
||||
return cls.iter_item_to_bytes("image", data)
|
||||
|
||||
if isinstance(obj, MediaWithBytes) and isinstance(obj.media, Image.Image):
|
||||
exif = obj.media.getexif()
|
||||
if Image.ExifTags.Base.ImageID in exif and isinstance(
|
||||
exif[Image.ExifTags.Base.ImageID], uuid.UUID
|
||||
):
|
||||
return (exif[Image.ExifTags.Base.ImageID].bytes,)
|
||||
image_id = _get_image_id_bytes(obj.media)
|
||||
if image_id is not None:
|
||||
return (image_id,)
|
||||
|
||||
if obj.io_config:
|
||||
return cls.iter_item_to_bytes(
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
# vllm/parser
|
||||
|
||||
Model-output parsing for the serving layer. `Parser` (`abstract_parser.py`)
|
||||
unifies reasoning extraction and tool-call extraction behind one object; the
|
||||
serving layer calls `parse` / `parse_delta` and `is_reasoning_end` on it.
|
||||
|
||||
## Layout
|
||||
|
||||
- `abstract_parser.py` — `Parser`, and `DelegatingParser` for wrapping a legacy
|
||||
`ReasoningParser` + `ToolParser` pair.
|
||||
- `parser_manager.py` — resolves parser names to classes.
|
||||
- `engine/` — the declarative parser engine. A `ParserEngineConfig`
|
||||
(`parser_engine_config.py`) is a set of terminals, transitions and content
|
||||
events; `parser_engine.py` and `streaming_parser_engine.py` drive it for the
|
||||
non-streaming and streaming paths; `token_id_scanner.py` pre-lexes
|
||||
special-token ids and `incremental_lexer.py` lexes text; `adapters.py`
|
||||
exposes an engine through the legacy reasoning/tool parser interfaces.
|
||||
- `<model>.py` — one file per model family. Most subclass `ParserEngine`
|
||||
(`deepseek_v4.py`, `qwen3.py`, ...); a few are `DelegatingParser` wrappers
|
||||
over legacy parsers (`harmony.py`, `kimi_k3.py`).
|
||||
|
||||
## Guidelines
|
||||
|
||||
- A new parser either subclasses `ParserEngine` with a declarative config, or
|
||||
subclasses `Parser` and implements `parse` / `parse_delta` directly. Use the
|
||||
engine when the format is expressible as terminals and transitions, and
|
||||
subclass an existing config when it is a variant of one (`seed_oss.py`,
|
||||
`ling3.py`). `DelegatingParser` exists for backward compatibility with the
|
||||
separate `ReasoningParser` / `ToolParser` classes; do not use it for new
|
||||
formats.
|
||||
- Keep a model-specific fix inside that model's `<model>.py` where possible.
|
||||
Change `engine/` or `abstract_parser.py` only when the behavior is wrong for
|
||||
every parser, and say so in the PR.
|
||||
- Text emitted in a state with no `content_events` entry is discarded. Make
|
||||
that deliberate when adding a state.
|
||||
- Special tokens reach the engine as token ids, not text. A text terminal that
|
||||
spans a special token never matches in streaming; list it in
|
||||
`token_id_terminals`.
|
||||
- Comment only where the code alone would leave a reader with a question —
|
||||
a non-obvious transition, a deliberate swallow, a model quirk being worked
|
||||
around. Keep it minimal and concise.
|
||||
@@ -0,0 +1 @@
|
||||
@AGENTS.md
|
||||
@@ -672,11 +672,30 @@ class ParserEngine(Parser):
|
||||
return not wait_for_reasoning
|
||||
|
||||
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
|
||||
config = self.parser_engine_config
|
||||
wait_for_reasoning = config.wait_for_reasoning
|
||||
if wait_for_reasoning is None:
|
||||
wait_for_reasoning = config.initial_state is ParserState.REASONING
|
||||
if not wait_for_reasoning:
|
||||
return input_ids
|
||||
|
||||
end_id = self._reasoning_end_token_id
|
||||
if end_id is not None:
|
||||
for i in range(len(input_ids) - 1, -1, -1):
|
||||
if input_ids[i] == end_id:
|
||||
return input_ids[i + 1 :]
|
||||
|
||||
end_ids = self._reasoning_end_token_ids
|
||||
if end_ids:
|
||||
turn_start = 0
|
||||
boundary_ids = self._turn_boundary_token_ids
|
||||
for i in range(len(input_ids) - 1, -1, -1):
|
||||
if input_ids[i] in boundary_ids:
|
||||
turn_start = i + 1
|
||||
break
|
||||
for i in range(turn_start, len(input_ids)):
|
||||
if input_ids[i] in end_ids:
|
||||
return input_ids[i:]
|
||||
return input_ids
|
||||
|
||||
def get_streaming_fallback_content(
|
||||
|
||||
@@ -41,6 +41,13 @@ ARG_KEY_END = "</arg_key>"
|
||||
ARG_VALUE_START = "<arg_value>"
|
||||
ARG_VALUE_END = "</arg_value>"
|
||||
|
||||
# Special tokens that delimit conversation turns in a rendered GLM prompt.
|
||||
# Reasoning markers belonging to earlier turns must not be mistaken for the
|
||||
# state of the turn currently being generated.
|
||||
GLM_TURN_BOUNDARIES = frozenset(
|
||||
("<|system|>", "<|user|>", "<|assistant|>", "<|observation|>")
|
||||
)
|
||||
|
||||
_ARG_RE = re.compile(
|
||||
r"<arg_key>(?P<key>.*?)</arg_key>\s*"
|
||||
r"<arg_value>(?P<value>.*?)</arg_value>",
|
||||
@@ -123,7 +130,6 @@ def glm47_moe_config(thinking: bool = True) -> ParserEngineConfig:
|
||||
return ParserEngineConfig(
|
||||
name="glm47_moe",
|
||||
initial_state=ParserState.REASONING if thinking else ParserState.CONTENT,
|
||||
wait_for_reasoning=thinking,
|
||||
terminals={
|
||||
**reasoning_terminals,
|
||||
"TOOL_START": TOOL_CALL_START,
|
||||
@@ -166,6 +172,7 @@ def glm47_moe_config(thinking: bool = True) -> ParserEngineConfig:
|
||||
),
|
||||
**arg_tag_transitions,
|
||||
},
|
||||
turn_boundary_tokens=GLM_TURN_BOUNDARIES,
|
||||
arg_converter=_glm47_arg_converter,
|
||||
stream_arg_deltas=True,
|
||||
tool_args_json=False,
|
||||
@@ -207,11 +214,6 @@ class Glm47MoeParser(ParserEngine):
|
||||
self._tool_slots[idx].name = self._tool_slots[idx].name.strip()
|
||||
super()._handle_tool_end(event, deltas)
|
||||
|
||||
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
|
||||
if not self.thinking_enabled:
|
||||
return input_ids
|
||||
return super().extract_content_ids(input_ids)
|
||||
|
||||
def extract_reasoning(
|
||||
self,
|
||||
model_output: str,
|
||||
|
||||
@@ -217,6 +217,15 @@ def cpu_platform_plugin() -> str | None:
|
||||
"AMD Zen CPU detected but zentorch not installed, "
|
||||
"falling back to CpuPlatform."
|
||||
)
|
||||
except OSError:
|
||||
# An ABI-mismatched build fails here with an undefined-symbol
|
||||
# error; other failures are not known to be safe to recover from.
|
||||
logger.warning(
|
||||
"AMD Zen CPU detected but zentorch failed to import, falling "
|
||||
"back to CpuPlatform. This usually means the zentorch build "
|
||||
"does not match the installed torch version.",
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
return "vllm.platforms.cpu.CpuPlatform"
|
||||
|
||||
|
||||
@@ -1334,6 +1334,24 @@ class Platform:
|
||||
"""
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def validate_environ(cls, hard_fail: bool) -> None:
|
||||
"""
|
||||
Validate environment variables for the current platform.
|
||||
"""
|
||||
from vllm import envs
|
||||
|
||||
for env in os.environ:
|
||||
if env.startswith("VLLM_") and env not in envs.environment_variables:
|
||||
if hard_fail:
|
||||
raise ValueError(
|
||||
f"Unknown vLLM environment variable detected: {env}"
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"Unknown vLLM environment variable detected: %s", env
|
||||
)
|
||||
|
||||
|
||||
class UnspecifiedPlatform(Platform):
|
||||
_enum = PlatformEnum.UNSPECIFIED
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
# vllm/reasoning
|
||||
|
||||
Every `--reasoning-parser` name resolves here. The parser has two consumers:
|
||||
`ParserManager.get_parser` composes it with the tool parser into the unified
|
||||
`Parser` the serving layer uses, and the structured-output manager
|
||||
(`vllm/v1/structured_output/__init__.py`) instantiates it directly and calls
|
||||
`is_reasoning_end` / `is_reasoning_end_streaming` every decode step to decide
|
||||
when the grammar bitmask starts applying. The `ReasoningParser` interface
|
||||
itself is legacy: it is kept so existing in-tree parsers and out-of-tree
|
||||
plugins keep working, not as the base for new ones.
|
||||
|
||||
## Layout
|
||||
|
||||
- `abs_reasoning_parsers.py` — `ReasoningParser` (`extract_reasoning`,
|
||||
`extract_reasoning_streaming`, `is_reasoning_end`,
|
||||
`is_reasoning_end_streaming`, `extract_content_ids`, `adjust_request`) and
|
||||
`ReasoningParserManager`.
|
||||
- `__init__.py` — `_REASONING_PARSERS_TO_REGISTER`, the name → (file, class)
|
||||
table that lazily registers every built-in parser.
|
||||
- `basic_parsers.py` — `BaseThinkingReasoningParser`, the base for parsers
|
||||
that only need a start/end token pair.
|
||||
- `<model>_reasoning_parser.py` — one file per family. Either a subclass of a
|
||||
`ParserReasoningAdapter` generated from a `vllm/parser/<model>.py` engine
|
||||
grammar (`mistral_reasoning_parser.py`, `kimi_k2_reasoning_parser.py`; the
|
||||
`_engine_` suffix is not applied consistently), or a self-contained
|
||||
`ReasoningParser` subclass.
|
||||
|
||||
## Guidelines
|
||||
|
||||
- Do not change the `ReasoningParser` interface in `abs_reasoning_parsers.py`.
|
||||
Out-of-tree plugins loaded through
|
||||
`ReasoningParserManager.import_reasoning_parser` subclass it. New
|
||||
capabilities go on `Parser` in `vllm/parser/`; if a legacy parser needs the
|
||||
same capability, `DelegatingParser` supplies the default.
|
||||
- For a new model, implement a unified `Parser` in `vllm/parser/` first, then
|
||||
expose it here as a thin adapter subclass plus an entry in
|
||||
`_REASONING_PARSERS_TO_REGISTER`.
|
||||
- The `_REASONING_PARSERS_TO_REGISTER` key is what users pass on the command
|
||||
line; treat renames as breaking.
|
||||
- Keep adapters thin: registration, `adjust_request` tweaks, and class
|
||||
attributes. Parsing logic belongs in `vllm/parser/`.
|
||||
- Keep a model-specific fix inside that model's `<model>_reasoning_parser.py`,
|
||||
or its grammar in `vllm/parser/<model>.py`, where possible. Touch
|
||||
`abs_reasoning_parsers.py` or `basic_parsers.py` only for behavior every
|
||||
parser needs, and say so in the PR.
|
||||
- `is_reasoning_end_streaming` runs on the engine core for every decode step
|
||||
of every structured-output request. Work on `delta_ids`; re-scanning the
|
||||
full sequence makes the step cost grow with sequence length.
|
||||
- The structured-output gate and the frontend's extraction must agree on
|
||||
where reasoning ends, whether the turn continues with content or a tool
|
||||
call. Test the end-detection methods with the same inputs as extraction.
|
||||
- Comment only where the code alone would leave a reader with a question —
|
||||
a request tweak in `adjust_request`, a marker handled unusually, a model
|
||||
quirk being worked around. Keep it minimal and concise.
|
||||
@@ -0,0 +1 @@
|
||||
@AGENTS.md
|
||||
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from io import BytesIO
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import pybase64
|
||||
import torch
|
||||
@@ -17,6 +17,27 @@ if TYPE_CHECKING:
|
||||
from vllm.config import ModelConfig
|
||||
|
||||
|
||||
# How much of torch's own reason is repeated back to the caller. The reason is
|
||||
# built from bytes the caller sent, so it is bounded rather than echoed whole.
|
||||
_MAX_EMBED_ERROR_REASON_CHARS: Final = 200
|
||||
|
||||
|
||||
def _truncated_reason(exc: Exception) -> str:
|
||||
"""`exc`'s message, capped, saying how much was left out when it is capped.
|
||||
|
||||
Without the count a capped reason reads like the whole one, so a caller
|
||||
debugging a large payload cannot tell that torch said more than this.
|
||||
"""
|
||||
reason: Final = str(exc).strip() or type(exc).__name__
|
||||
omitted: Final = len(reason) - _MAX_EMBED_ERROR_REASON_CHARS
|
||||
if omitted <= 0:
|
||||
return reason
|
||||
return (
|
||||
f"{reason[:_MAX_EMBED_ERROR_REASON_CHARS]}"
|
||||
f"... ({omitted} more characters truncated)"
|
||||
)
|
||||
|
||||
|
||||
def safe_load_prompt_embeds(
|
||||
model_config: "ModelConfig",
|
||||
embed: bytes,
|
||||
@@ -27,12 +48,38 @@ def safe_load_prompt_embeds(
|
||||
parameter="prompt_embeds",
|
||||
)
|
||||
|
||||
# Decoded outside the try below so that a body which is not base64 keeps
|
||||
# raising binascii.Error, which is a ValueError and already answered 400.
|
||||
payload: Final = pybase64.b64decode(embed, validate=True)
|
||||
|
||||
with check_sparse_tensor_invariants_threadsafe():
|
||||
tensor = torch.load(
|
||||
BytesIO(pybase64.b64decode(embed, validate=True)),
|
||||
weights_only=True,
|
||||
map_location=torch.device("cpu"),
|
||||
)
|
||||
try:
|
||||
tensor = torch.load(
|
||||
BytesIO(payload),
|
||||
weights_only=True,
|
||||
map_location=torch.device("cpu"),
|
||||
)
|
||||
except (torch.OutOfMemoryError, MemoryError):
|
||||
# Running out of memory is the server's condition, not the caller's.
|
||||
raise
|
||||
except Exception as exc:
|
||||
# Everything else here is about the bytes the caller sent, and torch
|
||||
# has no single error type for "this is not a tensor file": the zip
|
||||
# reader raises `RuntimeError`, the legacy pickle path raises
|
||||
# `UnpicklingError`, `KeyError` or `IndexError`, an empty payload
|
||||
# raises `EOFError`. None of them is a `ValueError`, so the
|
||||
# entrypoints' fallback mapped them all to 500. Catching by
|
||||
# behaviour rather than by type also survives torch changing which
|
||||
# error it raises for a given malformed payload.
|
||||
#
|
||||
# torch's reason is worth keeping -- for a malformed sparse tensor
|
||||
# it names the offending index -- but it is built from the caller's
|
||||
# own bytes, so it is truncated rather than echoed whole.
|
||||
raise VLLMValidationError(
|
||||
"`prompt_embeds` could not be deserialized as a torch tensor: "
|
||||
f"{_truncated_reason(exc)}",
|
||||
parameter="prompt_embeds",
|
||||
) from exc
|
||||
tensor = safe_to_dense(tensor, parameter="prompt_embeds")
|
||||
|
||||
if tensor.dim() > 2:
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
# vllm/tool_parsers
|
||||
|
||||
Every `--tool-call-parser` name resolves here. `ParserManager.get_parser`
|
||||
composes the tool parser chosen by name with the reasoning parser into the
|
||||
unified `Parser` the serving layer uses. The `ToolParser` interface itself is
|
||||
legacy: it is kept so existing in-tree parsers and out-of-tree plugins keep
|
||||
working, not as the base for new ones.
|
||||
|
||||
## Layout
|
||||
|
||||
- `abstract_tool_parser.py` — `ToolParser` (`extract_tool_calls`,
|
||||
`extract_tool_calls_streaming`, `adjust_request`) and `ToolParserManager`.
|
||||
- `__init__.py` — `_TOOL_PARSERS_TO_REGISTER`, the name → (file, class) table
|
||||
that lazily registers every built-in parser.
|
||||
- `structural_tag_registry.py` — structural-tag builders used to constrain
|
||||
tool-call output for `tool_choice="required"` / named tools.
|
||||
- `<model>_tool_parser.py` — one file per family. Either a subclass of a
|
||||
`ParserToolAdapter` generated from a `vllm/parser/<model>.py` engine grammar
|
||||
(`mistral_tool_parser.py`, `kimi_k2_tool_parser.py`; the `_engine_` suffix
|
||||
is not applied consistently), or a self-contained `ToolParser` subclass.
|
||||
|
||||
## Guidelines
|
||||
|
||||
- Do not change the `ToolParser` interface in `abstract_tool_parser.py`.
|
||||
Out-of-tree plugins loaded through `ToolParserManager.import_tool_parser`
|
||||
subclass it. New capabilities go on `Parser` in `vllm/parser/`; if a legacy
|
||||
parser needs the same capability, `DelegatingParser` supplies the default.
|
||||
- For a new model, implement a unified `Parser` in `vllm/parser/` first, then
|
||||
expose it here as a thin adapter subclass plus an entry in
|
||||
`_TOOL_PARSERS_TO_REGISTER`.
|
||||
- The `_TOOL_PARSERS_TO_REGISTER` key is what users pass on the command line;
|
||||
treat renames as breaking.
|
||||
- Keep adapters thin: registration, `adjust_request` tweaks, and class
|
||||
attributes. Parsing logic belongs in `vllm/parser/`.
|
||||
- Keep a model-specific fix inside that model's `<model>_tool_parser.py`, or
|
||||
its grammar in `vllm/parser/<model>.py`, where possible. Touch
|
||||
`abstract_tool_parser.py` or `structural_tag_registry.py` only for behavior
|
||||
every parser needs, and say so in the PR.
|
||||
- Set `structural_tag_model` only to a key in `SUPPORTED_STRUCTURAL_TAG_MODELS`
|
||||
(an xgrammar builtin, or one registered with `register_vllm_structural_tag`).
|
||||
It only takes effect under `VLLM_ENFORCE_STRICT_TOOL_CALLING`, where it also
|
||||
disables the generic required/named handling for that parser.
|
||||
- Comment only where the code alone would leave a reader with a question —
|
||||
a request tweak in `adjust_request`, a marker handled unusually, a model
|
||||
quirk being worked around. Keep it minimal and concise.
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user