From e126687a9a828d513c01a07cd69f025f27d63280 Mon Sep 17 00:00:00 2001 From: Harry Huang Date: Mon, 31 Aug 2026 13:57:56 +0800 Subject: [PATCH] [Model] Support Qwen3.8-Flash-Next (#53896) Signed-off-by: huanghaoyan.hhy Signed-off-by: andyluo7 Signed-off-by: lishunyang12 Signed-off-by: Harry Huang Signed-off-by: Thien Tran Signed-off-by: Isotr0py Signed-off-by: Lucas Wilkinson Signed-off-by: Yifan Qiao Signed-off-by: zjy0516 Co-authored-by: zjy0516 Co-authored-by: KungYork Co-authored-by: Thien Tran Co-authored-by: Yifan Qiao Co-authored-by: andyluo7 <43718156+andyluo7@users.noreply.github.com> Co-authored-by: Isotr0py Co-authored-by: andyluo7 Co-authored-by: OpenAI Codex Co-authored-by: lishunyang12 Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com> Co-authored-by: Isotr0py Co-authored-by: Lucas Wilkinson Co-authored-by: Claude Opus 5 Co-authored-by: OpenAI Codex --- .buildkite/test_areas/lm_eval.yaml | 36 + .buildkite/test_areas/models_basic.yaml | 37 + .buildkite/test_areas/spec_decode.yaml | 1 + .../gdn/fused_gdn_decode_kernel.cu | 36 +- csrc/libtorch_stable/ops.h | 2 +- csrc/libtorch_stable/torch_bindings.cpp | 3 +- tests/config/test_config_utils.py | 19 + .../test_speculative_draft_hf_overrides.py | 53 +- tests/distributed/test_custom_all_reduce.py | 23 + tests/evals/qwen4_exp/README.md | 14 + tests/evals/qwen4_exp/__init__.py | 3 + .../configs/Qwen3.8-Flash-Next-FP8.yaml | 26 + tests/evals/qwen4_exp/configs/models-b200.txt | 2 + tests/evals/qwen4_exp/configs/models-h200.txt | 2 + tests/evals/qwen4_exp/conftest.py | 42 + tests/evals/qwen4_exp/test_accuracy.py | 68 + tests/kernels/mamba/test_gdn_fused_mtp.py | 11 +- .../kernels/mamba/test_precopy_mamba_align.py | 4 +- tests/kernels/moe/test_block_fp8.py | 104 ++ tests/kernels/moe/test_trtllm_bf16_moe.py | 16 +- tests/kernels/test_fused_gdn_post_conv.py | 5 +- .../layers/test_fused_shared_expert.py | 80 ++ tests/models/qwen4_exp/__init__.py | 2 + tests/models/qwen4_exp/test_config.py | 200 +++ tests/models/qwen4_exp/test_hc_ops.py | 94 ++ tests/models/qwen4_exp/test_ple.py | 401 ++++++ tests/models/qwen4_exp/test_qsa_amd.py | 297 ++++ .../models/qwen4_exp/test_qsa_pre_indexer.py | 297 ++++ tests/models/qwen4_exp/test_qsa_reference.py | 973 +++++++++++++ tests/models/registry.py | 6 + tests/quantization/test_fp8.py | 44 + tests/quantization/test_gfx950_moe.py | 86 ++ tests/quantization/test_online_mxfp4.py | 21 + .../test_trtllm_nvfp4_hidden_dim_padding.py | 51 + tests/v1/core/test_contiguous_kv_packing.py | 312 ++++ tests/v1/core/test_kv_cache_utils.py | 87 +- tests/v1/core/test_scheduler.py | 46 + .../core/test_single_type_kv_cache_manager.py | 63 + .../v1/e2e/general/test_mamba_prefix_cache.py | 21 +- .../unit/test_hidden_states_connector.py | 59 +- .../unit/test_nixl_connector_hma.py | 5 + .../unit/test_nixl_desc_geometry.py | 299 ++++ .../unit/test_nixl_push_connector.py | 70 + tests/v1/kv_connector/unit/test_tp_mapping.py | 31 + .../test_gpu_autoregressive_speculator.py | 49 +- tests/v1/worker/test_gpu_block_table.py | 36 + tests/v1/worker/test_gpu_model_runner_v2.py | 92 ++ tests/v1/worker/test_gpu_warmup_blocks.py | 17 +- tests/v1/worker/test_mamba_utils.py | 200 ++- vllm/_custom_ops.py | 2 + vllm/config/compilation.py | 3 + vllm/config/scheduler.py | 4 + vllm/config/speculative.py | 43 + .../device_communicators/custom_all_reduce.py | 2 + .../kv_connector/v1/nixl/base_worker.py | 239 +++- .../kv_connector/v1/nixl/push_worker.py | 3 +- .../layers/fused_moe/experts/triton_moe.py | 14 + .../layers/fused_moe/oracle/fp8.py | 34 + .../layers/fused_moe/oracle/mxfp4.py | 48 + .../layers/fused_moe/oracle/unquantized.py | 15 +- .../layers/fused_moe/routed_experts.py | 14 + vllm/model_executor/layers/mamba/abstract.py | 5 + .../layers/mamba/gdn/qwen_gdn_linear_attn.py | 9 +- .../layers/mamba/mamba_utils.py | 5 + .../model_executor/layers/quantization/fp8.py | 66 +- .../quark/schemes/quark_ocp_mx.py | 10 + .../layers/quantization/utils/config_utils.py | 35 + .../quantization/utils/flashinfer_utils.py | 27 +- .../layers/vocab_parallel_embedding.py | 17 +- .../model_loader/weight_utils.py | 9 +- vllm/model_executor/models/config.py | 80 ++ vllm/model_executor/models/interfaces.py | 28 +- vllm/model_executor/models/qwen2_moe.py | 6 +- vllm/model_executor/models/qwen3_next.py | 65 +- vllm/model_executor/models/registry.py | 9 + vllm/models/qwen4_exp/__init__.py | 61 + vllm/models/qwen4_exp/amd/__init__.py | 2 + vllm/models/qwen4_exp/amd/hyperconnection.py | 205 +++ vllm/models/qwen4_exp/amd/indexer_qsa.py | 321 +++++ vllm/models/qwen4_exp/amd/low_latency_gemm.py | 15 + vllm/models/qwen4_exp/amd/model.py | 1065 ++++++++++++++ vllm/models/qwen4_exp/amd/model_state.py | 143 ++ vllm/models/qwen4_exp/amd/mtp.py | 462 ++++++ vllm/models/qwen4_exp/amd/ops/__init__.py | 3 + vllm/models/qwen4_exp/amd/ops/hc.py | 487 +++++++ vllm/models/qwen4_exp/amd/ops/qsa.py | 1128 +++++++++++++++ vllm/models/qwen4_exp/amd/ple_layer.py | 1131 +++++++++++++++ vllm/models/qwen4_exp/amd/qsa.py | 474 +++++++ vllm/models/qwen4_exp/common/__init__.py | 17 + .../qwen4_exp/common/hyperconnection.py | 250 ++++ vllm/models/qwen4_exp/common/ple.py | 100 ++ vllm/models/qwen4_exp/common/qsa_cache.py | 807 +++++++++++ vllm/models/qwen4_exp/config.py | 252 ++++ vllm/models/qwen4_exp/nvidia/__init__.py | 2 + .../qwen4_exp/nvidia/hyperconnection.py | 205 +++ vllm/models/qwen4_exp/nvidia/indexer_qsa.py | 356 +++++ .../qwen4_exp/nvidia/low_latency_gemm.py | 185 +++ vllm/models/qwen4_exp/nvidia/model.py | 1065 ++++++++++++++ vllm/models/qwen4_exp/nvidia/model_state.py | 143 ++ vllm/models/qwen4_exp/nvidia/mtp.py | 462 ++++++ vllm/models/qwen4_exp/nvidia/ops/__init__.py | 3 + vllm/models/qwen4_exp/nvidia/ops/hc.py | 487 +++++++ vllm/models/qwen4_exp/nvidia/ops/qsa.py | 1115 +++++++++++++++ .../qwen4_exp/nvidia/ops/qsa_pre_indexer.py | 508 +++++++ vllm/models/qwen4_exp/nvidia/ple_layer.py | 1256 +++++++++++++++++ vllm/models/qwen4_exp/nvidia/qsa.py | 487 +++++++ vllm/platforms/interface.py | 17 +- vllm/transformers_utils/config.py | 2 + vllm/transformers_utils/configs/__init__.py | 6 + vllm/transformers_utils/configs/qwen4_exp.py | 15 + .../model_arch_config_convertor.py | 10 + vllm/v1/attention/backends/short_conv_attn.py | 529 ++++++- vllm/v1/core/kv_cache_coordinator.py | 19 +- vllm/v1/core/kv_cache_manager.py | 8 + vllm/v1/core/kv_cache_utils.py | 349 +++-- vllm/v1/core/sched/scheduler.py | 13 + vllm/v1/core/single_type_kv_cache_manager.py | 99 +- vllm/v1/kv_cache_interface.py | 63 +- vllm/v1/worker/gpu/block_table.py | 16 +- vllm/v1/worker/gpu/model_runner.py | 19 +- .../worker/gpu/model_states/mamba_hybrid.py | 44 +- vllm/v1/worker/gpu/warmup.py | 13 +- vllm/v1/worker/gpu_model_runner.py | 18 +- vllm/v1/worker/mamba_utils.py | 199 ++- 124 files changed, 19424 insertions(+), 450 deletions(-) create mode 100644 tests/evals/qwen4_exp/README.md create mode 100644 tests/evals/qwen4_exp/__init__.py create mode 100644 tests/evals/qwen4_exp/configs/Qwen3.8-Flash-Next-FP8.yaml create mode 100644 tests/evals/qwen4_exp/configs/models-b200.txt create mode 100644 tests/evals/qwen4_exp/configs/models-h200.txt create mode 100644 tests/evals/qwen4_exp/conftest.py create mode 100644 tests/evals/qwen4_exp/test_accuracy.py create mode 100644 tests/models/qwen4_exp/__init__.py create mode 100644 tests/models/qwen4_exp/test_config.py create mode 100644 tests/models/qwen4_exp/test_hc_ops.py create mode 100644 tests/models/qwen4_exp/test_ple.py create mode 100644 tests/models/qwen4_exp/test_qsa_amd.py create mode 100644 tests/models/qwen4_exp/test_qsa_pre_indexer.py create mode 100644 tests/models/qwen4_exp/test_qsa_reference.py create mode 100644 vllm/models/qwen4_exp/__init__.py create mode 100644 vllm/models/qwen4_exp/amd/__init__.py create mode 100644 vllm/models/qwen4_exp/amd/hyperconnection.py create mode 100644 vllm/models/qwen4_exp/amd/indexer_qsa.py create mode 100644 vllm/models/qwen4_exp/amd/low_latency_gemm.py create mode 100644 vllm/models/qwen4_exp/amd/model.py create mode 100644 vllm/models/qwen4_exp/amd/model_state.py create mode 100644 vllm/models/qwen4_exp/amd/mtp.py create mode 100644 vllm/models/qwen4_exp/amd/ops/__init__.py create mode 100644 vllm/models/qwen4_exp/amd/ops/hc.py create mode 100644 vllm/models/qwen4_exp/amd/ops/qsa.py create mode 100644 vllm/models/qwen4_exp/amd/ple_layer.py create mode 100644 vllm/models/qwen4_exp/amd/qsa.py create mode 100644 vllm/models/qwen4_exp/common/__init__.py create mode 100644 vllm/models/qwen4_exp/common/hyperconnection.py create mode 100644 vllm/models/qwen4_exp/common/ple.py create mode 100644 vllm/models/qwen4_exp/common/qsa_cache.py create mode 100644 vllm/models/qwen4_exp/config.py create mode 100644 vllm/models/qwen4_exp/nvidia/__init__.py create mode 100644 vllm/models/qwen4_exp/nvidia/hyperconnection.py create mode 100644 vllm/models/qwen4_exp/nvidia/indexer_qsa.py create mode 100644 vllm/models/qwen4_exp/nvidia/low_latency_gemm.py create mode 100644 vllm/models/qwen4_exp/nvidia/model.py create mode 100644 vllm/models/qwen4_exp/nvidia/model_state.py create mode 100644 vllm/models/qwen4_exp/nvidia/mtp.py create mode 100644 vllm/models/qwen4_exp/nvidia/ops/__init__.py create mode 100644 vllm/models/qwen4_exp/nvidia/ops/hc.py create mode 100644 vllm/models/qwen4_exp/nvidia/ops/qsa.py create mode 100644 vllm/models/qwen4_exp/nvidia/ops/qsa_pre_indexer.py create mode 100644 vllm/models/qwen4_exp/nvidia/ple_layer.py create mode 100644 vllm/models/qwen4_exp/nvidia/qsa.py create mode 100644 vllm/transformers_utils/configs/qwen4_exp.py diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index d52eccf1ee..86b6bf0042 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -331,6 +331,42 @@ steps: - uv pip install --system 'gpt-oss[eval]==0.0.5' - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-b200.txt +- label: ":nvidia: (B200) Qwen3.8-Flash-Next-FP8 Accuracy Eval" + key: accuracy-eval-qwen3-8-flash-next-fp8-b200 + timeout_in_minutes: 60 + device: b200-k8s + optional: true + num_devices: 4 + source_file_dependencies: + - csrc/libtorch_stable/gdn/ + - tests/evals/qwen4_exp/ + - vllm/model_executor/layers/mamba/ + - vllm/models/qwen4_exp/ + - vllm/transformers_utils/configs/qwen4_exp.py + - vllm/v1/attention/backends/short_conv_attn.py + - vllm/v1/spec_decode/qwen4_exp.py + commands: + - uv pip install --system 'evalscope==1.10.0' + - pytest -s -v evals/qwen4_exp/test_accuracy.py --config-list-file=configs/models-b200.txt + +- label: ":nvidia: (H200) Qwen3.8-Flash-Next-FP8 Accuracy Eval" + key: accuracy-eval-qwen3-8-flash-next-fp8-h200 + timeout_in_minutes: 60 + device: h200 + optional: true + num_devices: 4 + source_file_dependencies: + - csrc/libtorch_stable/gdn/ + - tests/evals/qwen4_exp/ + - vllm/model_executor/layers/mamba/ + - vllm/models/qwen4_exp/ + - vllm/transformers_utils/configs/qwen4_exp.py + - vllm/v1/attention/backends/short_conv_attn.py + - vllm/v1/spec_decode/qwen4_exp.py + commands: + - uv pip install --system 'evalscope==1.10.0' + - pytest -s -v evals/qwen4_exp/test_accuracy.py --config-list-file=configs/models-h200.txt + - label: ":nvidia: (DGX) Spark GPQA Eval (GPT-OSS)" key: gpqa-eval-gpt-oss-spark timeout_in_minutes: 35 diff --git a/.buildkite/test_areas/models_basic.yaml b/.buildkite/test_areas/models_basic.yaml index 780eef68ce..5669f0f1fc 100644 --- a/.buildkite/test_areas/models_basic.yaml +++ b/.buildkite/test_areas/models_basic.yaml @@ -86,6 +86,43 @@ steps: # The native NVIDIA Kimi K3 kernels require the SM100 family. - pytest -v -s models/kimi_k3 kernels/attention/test_kimi_k3_mla_fused_epilogue.py kernels/test_bf16_skinny_gemm.py +- label: ":nvidia: (H200) Qwen4 Exp" + key: qwen4-exp-unit-tests + timeout_in_minutes: 40 + device: h200_35gb + source_file_dependencies: + - vllm/models/qwen4_exp/ + - vllm/transformers_utils/configs/qwen4_exp.py + - vllm/config/compilation.py + - vllm/v1/attention/backends/short_conv_attn.py + - tests/models/qwen4_exp/ + commands: + - pytest -v -s models/qwen4_exp/test_hc_ops.py models/qwen4_exp/test_qsa_pre_indexer.py models/qwen4_exp/test_qsa_reference.py + mirror: + amd: + label: ":amd: (MI300) Qwen4 Exp" + dind: false + device: mi300_1 + depends_on: + - image-build-amd + commands: + - pytest -v -s models/qwen4_exp/test_qsa_amd.py + +- label: ":computer: (CPU) Qwen4 Exp" + key: qwen4-exp-unit-tests-cpu + depends_on: + - image-build-cpu + timeout_in_minutes: 20 + device: cpu-small + source_file_dependencies: + - vllm/models/qwen4_exp/ + - vllm/transformers_utils/configs/qwen4_exp.py + - vllm/config/compilation.py + - vllm/v1/attention/backends/short_conv_attn.py + - tests/models/qwen4_exp/ + commands: + - pytest -v -s models/qwen4_exp/test_config.py models/qwen4_exp/test_ple.py + - label: ":computer: (CPU) Basic Models Other" key: basic-models-test-other-cpu depends_on: diff --git a/.buildkite/test_areas/spec_decode.yaml b/.buildkite/test_areas/spec_decode.yaml index 1907dc44b9..08aa05befd 100644 --- a/.buildkite/test_areas/spec_decode.yaml +++ b/.buildkite/test_areas/spec_decode.yaml @@ -11,6 +11,7 @@ steps: - vllm/distributed/ - vllm/inputs/ - vllm/model_executor/ + - vllm/models/qwen4_exp/ - vllm/platforms/ - vllm/sampling_params.py - vllm/transformers_utils/ diff --git a/csrc/libtorch_stable/gdn/fused_gdn_decode_kernel.cu b/csrc/libtorch_stable/gdn/fused_gdn_decode_kernel.cu index 1166b41868..1229404dc6 100644 --- a/csrc/libtorch_stable/gdn/fused_gdn_decode_kernel.cu +++ b/csrc/libtorch_stable/gdn/fused_gdn_decode_kernel.cu @@ -4,6 +4,7 @@ */ #include +#include #include #include #include @@ -146,7 +147,7 @@ __device__ __forceinline__ Sum2 warp_reduce_sum_pair(float x, float y) { return {x, y}; } -template +template __global__ __launch_bounds__(kThreads, 2) void gdn_decode_post_conv_mtp_kernel( const __nv_bfloat16* __restrict__ mixed_qkv, const __nv_bfloat16* __restrict__ a, const __nv_bfloat16* __restrict__ b, @@ -354,9 +355,11 @@ __global__ __launch_bounds__(kThreads, 2) void gdn_decode_post_conv_mtp_kernel( #pragma unroll for (int i = 0; i < 4; ++i) { const int value = lane + i * 32; - const float gate = silu_fast(__bfloat162float( + const float gate_input = __bfloat162float( output_gate[static_cast(token) * strides.gate_row + - value_head * kDimV + value])); + value_head * kDimV + value]); + const float gate = + SigmoidGate ? sigmoid_fast(gate_input) : silu_fast(gate_input); const float weight = norm_weight_is_bf16 ? __bfloat162float( @@ -370,7 +373,7 @@ __global__ __launch_bounds__(kThreads, 2) void gdn_decode_post_conv_mtp_kernel( } } -template +template void launch_gdn_decode_post_conv_mtp( torch::stable::Tensor const& mixed_qkv, torch::stable::Tensor const& a_log, torch::stable::Tensor const& dt_bias, @@ -395,7 +398,7 @@ void launch_gdn_decode_post_conv_mtp( get_current_cuda_stream(mixed_qkv.get_device_index()); const int num_requests = static_cast(state_indices.size(0)); const dim3 grid(num_requests, num_value_heads); - gdn_decode_post_conv_mtp_kernel + gdn_decode_post_conv_mtp_kernel <<>>( static_cast(mixed_qkv.data_ptr()), a, b, static_cast(a_log.data_ptr()), dt_bias.data_ptr(), @@ -425,7 +428,7 @@ void fused_gdn_decode_post_conv_mtp( torch::stable::Tensor const& num_accepted_tokens, torch::stable::Tensor& state, torch::stable::Tensor const& output_gate, torch::stable::Tensor const& norm_weight, torch::stable::Tensor& out, - double scale, double norm_eps) { + double scale, double norm_eps, const std::string& output_gate_activation) { using torch::headeronly::ScalarType; STD_TORCH_CHECK( @@ -466,6 +469,9 @@ void fused_gdn_decode_post_conv_mtp( "norm_weight must be a CUDA float32 or bfloat16 tensor"); STD_TORCH_CHECK(out.is_cuda() && out.scalar_type() == ScalarType::BFloat16, "out must be a CUDA bfloat16 tensor"); + STD_TORCH_CHECK( + output_gate_activation == "silu" || output_gate_activation == "sigmoid", + "output_gate_activation must be 'silu' or 'sigmoid'"); STD_TORCH_CHECK(mixed_qkv.dim() == 2, "mixed_qkv must have shape [L, 2 * H * 128 + HV * 128]"); @@ -547,8 +553,9 @@ void fused_gdn_decode_post_conv_mtp( const auto* b_ptr = static_cast(b.data_ptr()); const auto* output_gate_ptr = static_cast(output_gate.data_ptr()); - const auto launch = [&]() { - launch_gdn_decode_post_conv_mtp( + const auto launch = [&]() { + launch_gdn_decode_post_conv_mtp( mixed_qkv, a_log, dt_bias, state_indices, cu_seqlens, num_accepted_tokens, state, norm_weight, out, a_ptr, b_ptr, output_gate_ptr, num_key_heads, num_value_heads, scale, norm_eps, @@ -556,9 +563,18 @@ void fused_gdn_decode_post_conv_mtp( }; const auto dispatch_state_type = [&]() { if (state_scalar_type == ScalarType::Float) { - launch.template operator()(); + if (output_gate_activation == "sigmoid") { + launch.template operator()(); + } else { + launch.template operator()(); + } } else { - launch.template operator()<__nv_bfloat16, ValueHeadsPerKeyHead>(); + if (output_gate_activation == "sigmoid") { + launch.template operator()<__nv_bfloat16, ValueHeadsPerKeyHead, true>(); + } else { + launch + .template operator()<__nv_bfloat16, ValueHeadsPerKeyHead, false>(); + } } }; switch (value_heads_per_key_head) { diff --git a/csrc/libtorch_stable/ops.h b/csrc/libtorch_stable/ops.h index 31983fc372..75f95e4bfd 100644 --- a/csrc/libtorch_stable/ops.h +++ b/csrc/libtorch_stable/ops.h @@ -405,7 +405,7 @@ void fused_gdn_decode_post_conv_mtp( torch::stable::Tensor const& num_accepted_tokens, torch::stable::Tensor& state, torch::stable::Tensor const& output_gate, torch::stable::Tensor const& norm_weight, torch::stable::Tensor& out, - double scale, double norm_eps); + double scale, double norm_eps, const std::string& output_gate_activation); #endif diff --git a/csrc/libtorch_stable/torch_bindings.cpp b/csrc/libtorch_stable/torch_bindings.cpp index c9bcba1549..ec1963b8a9 100644 --- a/csrc/libtorch_stable/torch_bindings.cpp +++ b/csrc/libtorch_stable/torch_bindings.cpp @@ -535,7 +535,8 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) { "Tensor mixed_qkv, Tensor a, Tensor b, Tensor A_log, Tensor dt_bias, " "Tensor state_indices, Tensor cu_seqlens, Tensor num_accepted_tokens, " "Tensor! state, Tensor output_gate, Tensor norm_weight, Tensor! out, " - "float scale, float norm_eps=1e-5) -> ()"); + "float scale, float norm_eps=1e-5, " + "str output_gate_activation='silu') -> ()"); #endif #ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES diff --git a/tests/config/test_config_utils.py b/tests/config/test_config_utils.py index 24ef1b52a9..94060d1724 100644 --- a/tests/config/test_config_utils.py +++ b/tests/config/test_config_utils.py @@ -7,6 +7,7 @@ from enum import Enum import pytest from vllm.config.cache import CacheConfig +from vllm.config.scheduler import SchedulerConfig from vllm.config.utils import get_hash_factors, hash_factors, normalize_value # Helpers @@ -216,6 +217,24 @@ def test_cache_config_hash_ignores_kv_cache_sizing_knobs(): assert CacheConfig(gpu_memory_utilization=0.5).compute_hash() == base_hash +def test_scheduler_config_hash_includes_max_num_seqs(): + """Per-request workspace sizes must invalidate compiled graphs.""" + base_hash = SchedulerConfig( + max_model_len=8192, + is_encoder_decoder=False, + max_num_batched_tokens=8192, + max_num_seqs=128, + ).compute_hash() + larger_batch_hash = SchedulerConfig( + max_model_len=8192, + is_encoder_decoder=False, + max_num_batched_tokens=8192, + max_num_seqs=1024, + ).compute_hash() + + assert larger_batch_hash != base_hash + + def test_cache_config_hash_ignores_prefix_cache_retention_interval(): base_hash = CacheConfig().compute_hash() assert CacheConfig(prefix_cache_retention_interval=64).compute_hash() == base_hash diff --git a/tests/config/test_speculative_draft_hf_overrides.py b/tests/config/test_speculative_draft_hf_overrides.py index 75b4bd2963..8505172da3 100644 --- a/tests/config/test_speculative_draft_hf_overrides.py +++ b/tests/config/test_speculative_draft_hf_overrides.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for SpeculativeConfig.compose_draft_hf_overrides. +"""Tests for draft config overrides used by SpeculativeConfig. Callable ``hf_overrides`` on the target model config (e.g. the ``dummy_hf_overrides`` shrink used by ``tests/models/test_initialization.py``) @@ -12,10 +12,12 @@ when the target itself is shrunk — which is what kept spec-decode archs like """ import functools +from unittest.mock import MagicMock, patch import pytest from transformers import PretrainedConfig +from vllm.config.parallel import ParallelConfig from vllm.config.speculative import SpeculativeConfig @@ -134,3 +136,52 @@ def test_composed_override_is_picklable(): out = composed(_make_hf_config()) assert out.num_hidden_layers == 1 + + +def _make_mtp_speculative_config( + override: bool | None, + checkpoint_value: bool, +) -> SpeculativeConfig: + draft_hf_config = _make_hf_config( + architectures=["Qwen4ExpMTP"], + model_type="qwen4_exp_mtp", + n_predict=1, + index_share_for_mtp_iteration=checkpoint_value, + ) + draft_model_config = MagicMock( + model="draft", + hf_config=draft_hf_config, + architectures=draft_hf_config.architectures, + max_model_len=128, + ) + target_model_config = MagicMock( + model="target", + max_model_len=128, + quantization=None, + hf_overrides={}, + ) + + with patch("vllm.config.speculative.ModelConfig", return_value=draft_model_config): + return SpeculativeConfig( + model="draft", + method="mtp", + num_speculative_tokens=1, + index_share_for_mtp_iteration=override, + target_model_config=target_model_config, + target_parallel_config=ParallelConfig(), + ) + + +@pytest.mark.cpu_test +@pytest.mark.parametrize( + ("override", "checkpoint_value", "expected"), + [(None, True, True), (False, True, False), (True, False, True)], +) +def test_mtp_index_share_override( + override: bool | None, checkpoint_value: bool, expected: bool +): + speculative_config = _make_mtp_speculative_config(override, checkpoint_value) + assert ( + speculative_config.draft_model_config.hf_config.index_share_for_mtp_iteration + is expected + ) diff --git a/tests/distributed/test_custom_all_reduce.py b/tests/distributed/test_custom_all_reduce.py index b6769e4d51..7f91f11d7c 100644 --- a/tests/distributed/test_custom_all_reduce.py +++ b/tests/distributed/test_custom_all_reduce.py @@ -24,6 +24,29 @@ for i, v in enumerate(test_sizes): test_sizes[i] -= v % 8 +@pytest.mark.parametrize( + ("dtype", "expected"), + [ + (torch.float32, True), + (torch.float16, True), + (torch.bfloat16, True), + (torch.int8, False), + (torch.float8_e4m3fn, False), + ], +) +def test_custom_allreduce_filters_dtype( + dtype: torch.dtype, + expected: bool, +) -> None: + communicator = car.CustomAllreduce.__new__(car.CustomAllreduce) + communicator.disabled = False + communicator.world_size = 2 + communicator.max_size = 1024 + communicator._ptr = 0 + + assert communicator.should_custom_ar(torch.empty(16, dtype=dtype)) is expected + + @pytest.mark.parametrize( ("major", "local_multicast", "expected"), [ diff --git a/tests/evals/qwen4_exp/README.md b/tests/evals/qwen4_exp/README.md new file mode 100644 index 0000000000..33ad13499a --- /dev/null +++ b/tests/evals/qwen4_exp/README.md @@ -0,0 +1,14 @@ +# Qwen4Exp accuracy evaluation + +This suite starts a Qwen3.8-Flash-Next-FP8 OpenAI-compatible server once and +uses EvalScope to evaluate GSM8K and AIME25. + +```bash +# B200 +pytest -s -v tests/evals/qwen4_exp/test_accuracy.py \ + --config-list-file=configs/models-b200.txt + +# H200 +pytest -s -v tests/evals/qwen4_exp/test_accuracy.py \ + --config-list-file=configs/models-h200.txt +``` diff --git a/tests/evals/qwen4_exp/__init__.py b/tests/evals/qwen4_exp/__init__.py new file mode 100644 index 0000000000..6655f89136 --- /dev/null +++ b/tests/evals/qwen4_exp/__init__.py @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + diff --git a/tests/evals/qwen4_exp/configs/Qwen3.8-Flash-Next-FP8.yaml b/tests/evals/qwen4_exp/configs/Qwen3.8-Flash-Next-FP8.yaml new file mode 100644 index 0000000000..c35614cd5f --- /dev/null +++ b/tests/evals/qwen4_exp/configs/Qwen3.8-Flash-Next-FP8.yaml @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +model_name: "Qwen/Qwen3.8-Flash-Next-FP8" +datasets: + gsm8k: + metric_threshold: 0.98 + tolerance: 0.02 + aime25: + metric_threshold: 0.90 + tolerance: 0.05 +eval_batch_size: 32 +startup_max_wait_seconds: 1800 +generation_config: + seed: 1236 + do_sample: true + temperature: 0.7 + top_p: 0.8 + top_k: 20 + repetition_penalty: 1.0 + presence_penalty: 1.5 + max_tokens: 32768 + stream: true + extra_body: + chat_template_kwargs: + enable_thinking: true +server_args: "--tensor-parallel-size 4" diff --git a/tests/evals/qwen4_exp/configs/models-b200.txt b/tests/evals/qwen4_exp/configs/models-b200.txt new file mode 100644 index 0000000000..63615558e2 --- /dev/null +++ b/tests/evals/qwen4_exp/configs/models-b200.txt @@ -0,0 +1,2 @@ +# B200 configuration for Qwen3.8-Flash-Next-FP8 accuracy evaluation +Qwen3.8-Flash-Next-FP8.yaml diff --git a/tests/evals/qwen4_exp/configs/models-h200.txt b/tests/evals/qwen4_exp/configs/models-h200.txt new file mode 100644 index 0000000000..a3357d31fb --- /dev/null +++ b/tests/evals/qwen4_exp/configs/models-h200.txt @@ -0,0 +1,2 @@ +# H200 configuration for Qwen3.8-Flash-Next-FP8 accuracy evaluation +Qwen3.8-Flash-Next-FP8.yaml diff --git a/tests/evals/qwen4_exp/conftest.py b/tests/evals/qwen4_exp/conftest.py new file mode 100644 index 0000000000..a4b4127c0b --- /dev/null +++ b/tests/evals/qwen4_exp/conftest.py @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from pathlib import Path + + +def pytest_addoption(parser): + parser.addoption( + "--config-list-file", + required=True, + help="File containing Qwen4Exp evaluation config files", + ) + + +def pytest_generate_tests(metafunc): + if "config_filename" not in metafunc.fixturenames: + return + + config_list_path = Path(metafunc.config.getoption("--config-list-file")) + if not config_list_path.is_absolute(): + test_relative_path = Path(__file__).parent / config_list_path + config_list_path = ( + test_relative_path + if test_relative_path.exists() + else Path.cwd() / config_list_path + ) + + config_dir = config_list_path.parent + config_files = [ + config_dir / line.strip() + for line in config_list_path.read_text(encoding="utf-8").splitlines() + if line.strip() and not line.lstrip().startswith("#") + ] + missing = [path for path in config_files if not path.exists()] + if missing: + raise FileNotFoundError(f"Missing evaluation configs: {missing}") + + metafunc.parametrize( + "config_filename", + config_files, + ids=[config_file.stem for config_file in config_files], + ) diff --git a/tests/evals/qwen4_exp/test_accuracy.py b/tests/evals/qwen4_exp/test_accuracy.py new file mode 100644 index 0000000000..d5dee5aff0 --- /dev/null +++ b/tests/evals/qwen4_exp/test_accuracy.py @@ -0,0 +1,68 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""GSM8K and AIME25 accuracy evaluation for Qwen3.8-Flash-Next-FP8.""" + +import shlex +from pathlib import Path +from typing import Any + +import yaml + +from tests.utils import RemoteOpenAIServer + + +def run_evalscope( + eval_config: dict[str, Any], base_url: str, work_dir: Path +) -> dict[str, float]: + from evalscope.run import run_task + + reports = run_task( + { + "model": eval_config["model_name"], + "api_url": base_url, + "api_key": "EMPTY_TOKEN", + "datasets": list(eval_config["datasets"]), + "eval_batch_size": eval_config.get("eval_batch_size", 32), + "generation_config": eval_config["generation_config"], + "work_dir": str(work_dir), + "no_timestamp": True, + } + ) + if not isinstance(reports, dict): + raise TypeError(f"Expected EvalScope reports to be a dict, got {type(reports)}") + + return {dataset: float(report.score) for dataset, report in reports.items()} + + +def test_qwen4_exp_accuracy(config_filename: Path, tmp_path: Path): + eval_config = yaml.safe_load(config_filename.read_text(encoding="utf-8")) + server_args = shlex.split(eval_config.get("server_args", "")) + server_args.extend(["--trust-remote-code", "--disable-uvicorn-access-log"]) + + model_name = eval_config["model_name"] + print(f"Starting Qwen4Exp evaluation for model: {model_name}") + print(f"Datasets: {', '.join(eval_config['datasets'])}") + print(f"Server args: {' '.join(server_args)}") + + with RemoteOpenAIServer( + model_name, + server_args, + env_dict=eval_config.get("env"), + max_wait_seconds=eval_config.get("startup_max_wait_seconds", 1800), + ) as remote_server: + scores = run_evalscope(eval_config, remote_server.url_for("v1"), tmp_path) + + for dataset, metric_config in eval_config["datasets"].items(): + score = scores[dataset] + threshold = metric_config["metric_threshold"] + tolerance = metric_config.get("tolerance", 0.0) + minimum_score = threshold - tolerance + + print( + f"{dataset}: measured={score:.4f}, expected={threshold:.4f}, " + f"tolerance={tolerance:.4f}" + ) + assert score >= minimum_score, ( + f"{dataset} score too low: {score:.4f} < {threshold:.4f} - " + f"{tolerance:.4f} = {minimum_score:.4f}" + ) diff --git a/tests/kernels/mamba/test_gdn_fused_mtp.py b/tests/kernels/mamba/test_gdn_fused_mtp.py index 0d78f8ceeb..ea062b50c2 100644 --- a/tests/kernels/mamba/test_gdn_fused_mtp.py +++ b/tests/kernels/mamba/test_gdn_fused_mtp.py @@ -57,13 +57,13 @@ EPS = 1e-6 class _TestGatedNorm: - def __init__(self, weight: torch.Tensor) -> None: + def __init__(self, weight: torch.Tensor, activation: str) -> None: self.weight = weight self.bias = None self.eps = EPS self.group_size = None self.norm_before_gate = True - self.activation = "silu" + self.activation = activation def __call__(self, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor: return rmsnorm_fn( @@ -99,6 +99,7 @@ def _build_layer( dt_bias: torch.Tensor, conv_weight: torch.Tensor, norm_weight: torch.Tensor, + output_gate_activation: str, ): layer = types.SimpleNamespace( prefix=PREFIX, @@ -116,7 +117,7 @@ def _build_layer( dt_bias=dt_bias, conv1d=types.SimpleNamespace(weight=conv_weight, bias=None), kv_cache=(conv_state, ssm_state), - norm=_TestGatedNorm(norm_weight), + norm=_TestGatedNorm(norm_weight, output_gate_activation), layer_norm_epsilon=EPS, gdn_decode_kernel="cuda", ) @@ -250,12 +251,14 @@ def test_fused_forward_uses_packed_entrypoint() -> None: pytest.param([128], [1], [-1], 0, id="pure-decode"), ], ) +@pytest.mark.parametrize("output_gate_activation", ["silu", "sigmoid"]) @torch.inference_mode() def test_fused_model_path_matches_reference( seq_lens: list[int], query_lens: list[int], draft_tokens: list[int], expected_fused_calls: int, + output_gate_activation: str, ) -> None: """Fused MTP and its mixed/prefill/decode fallbacks match the reference.""" torch.manual_seed(1) @@ -334,6 +337,7 @@ def test_fused_model_path_matches_reference( dt_bias, conv_weight, norm_weight, + output_gate_activation, ) reference_out = torch.zeros_like(output_gate) with patch.object( @@ -355,6 +359,7 @@ def test_fused_model_path_matches_reference( dt_bias, conv_weight, norm_weight, + output_gate_activation, ) context.no_compile_layers = {PREFIX: fused_layer} fused_out = torch.zeros_like(output_gate) diff --git a/tests/kernels/mamba/test_precopy_mamba_align.py b/tests/kernels/mamba/test_precopy_mamba_align.py index d5a3f56f13..22b73c936e 100644 --- a/tests/kernels/mamba/test_precopy_mamba_align.py +++ b/tests/kernels/mamba/test_precopy_mamba_align.py @@ -366,7 +366,7 @@ def test_preprocess_fused_align_matches_scalar_bookkeeping(monkeypatch, token_bi input_batch=scalar_case[1], requests=scalar_case[2], forward_context={}, - mamba_state_copy_funcs=(), + mamba_state_copy_funcs={}, copy_bufs=scalar_copy_bufs, ) @@ -384,7 +384,7 @@ def test_preprocess_fused_align_matches_scalar_bookkeeping(monkeypatch, token_bi input_batch=fused_case[1], requests=fused_case[2], forward_context={}, - mamba_state_copy_funcs=(), + mamba_state_copy_funcs={}, copy_bufs=fused_copy_bufs, align_ctx=ctx, ) diff --git a/tests/kernels/moe/test_block_fp8.py b/tests/kernels/moe/test_block_fp8.py index 131e303fe3..1cbbbb45fb 100644 --- a/tests/kernels/moe/test_block_fp8.py +++ b/tests/kernels/moe/test_block_fp8.py @@ -351,3 +351,107 @@ def test_w8a8_block_fp8_deep_gemm_fused_moe(M, N, K, E, topk, seed, monkeypatch) torch.accelerator.synchronize() torch.testing.assert_close(out, ref_out, atol=0.035, rtol=0.035) + + +def _dequant_block_fp8( + w_fp8: torch.Tensor, w_s: torch.Tensor, block_shape: list[int] +) -> torch.Tensor: + """Dequantize a [N, K] fp8 weight with [Nb, Kb] block scales to fp32.""" + n, k = w_fp8.shape + bn, bk = block_shape + s = w_s.repeat_interleave(bn, dim=0).repeat_interleave(bk, dim=1) + return w_fp8.to(torch.float32) * s[:n, :k] + + +@pytest.mark.parametrize("tp_rank", [0, 1, 2, 3]) +@torch.inference_mode() +def test_w8a8_block_fp8_fused_moe_refined_block_scales(tp_rank, workspace_init): + """TP-misaligned blockwise FP8 MoE (e.g. Qwen4Exp, intermediate + 640 at TP=4 -> 160 per rank, not divisible by the checkpoint's 128 block). + + The per-rank scale grid is refined from 128 to 32 (lossless: each 32-block + lies inside one global 128-block) so the Triton kernel can consume exact + per-shard scales. Simulates one TP rank's shard and checks: + 1. the refined+sharded scales dequantize exactly like the global scales; + 2. the Triton fused MoE kernel with block_shape=[32, 32] matches the + native-torch blockwise reference on the same shard. + """ + torch.manual_seed(0) + dtype = torch.bfloat16 + M, E, topk = 8, 4, 2 + hidden = 256 + inter_full = 640 + tp_size = 4 + n_shard = inter_full // tp_size # 160 + ckpt_block = [128, 128] + refined_block = [32, 32] + factor = ckpt_block[0] // refined_block[0] # 4 + + a = torch.randn((M, hidden), dtype=dtype) / 10 + score = torch.randn((M, E), dtype=dtype) + + (_, w1_full, w1_s_full, _), (_, w2_full, w2_s_full, _) = make_test_weights( + E, + inter_full, + hidden, + dtype, + torch.float8_e4m3fn, + per_out_ch_quant=False, + block_shape=ckpt_block, + ) + assert w1_s_full is not None + assert w2_s_full is not None + + # TP shard the weights (exact slicing, no scale involvement). + lo, hi = tp_rank * n_shard, (tp_rank + 1) * n_shard + w1 = torch.cat( + [w1_full[:, lo:hi], w1_full[:, inter_full + lo : inter_full + hi]], dim=1 + ) + w2 = w2_full[:, :, lo:hi] + + # Refine the global 128-block scales to 32 blocks, then take the shard's + # slice -- mirrors the Fp8MoEMethod/RoutedExperts loading path. + w1_s32 = w1_s_full.repeat_interleave(factor, dim=-2).repeat_interleave( + factor, dim=-1 + ) + nb = n_shard // refined_block[0] # 5 local 32-blocks per projection + gate_hi = w1_s32.shape[1] // 2 + w1_s = torch.cat( + [ + w1_s32[:, tp_rank * nb : (tp_rank + 1) * nb], + w1_s32[:, gate_hi + tp_rank * nb : gate_hi + (tp_rank + 1) * nb], + ], + dim=1, + ) + w2_s32 = w2_s_full.repeat_interleave(factor, dim=-2).repeat_interleave( + factor, dim=-1 + ) + w2_s = w2_s32[:, :, tp_rank * nb : (tp_rank + 1) * nb] + + # The refined per-shard scales must reproduce the global-scale dequant + # exactly (dequant full-width, then slice to the shard). + for e in range(E): + d32 = _dequant_block_fp8(w1[e, :n_shard], w1_s[e, :nb], refined_block) + d128 = _dequant_block_fp8(w1_full[e], w1_s_full[e], ckpt_block)[lo:hi] + assert torch.equal(d32, d128) + d32 = _dequant_block_fp8(w2[e], w2_s[e], refined_block) + d128 = _dequant_block_fp8(w2_full[e], w2_s_full[e], ckpt_block)[:, lo:hi] + assert torch.equal(d32, d128) + + quant_config = fp8_w8a8_moe_quant_config( + w1_scale=w1_s, + w2_scale=w2_s, + block_shape=refined_block, + ) + + topk_weights, topk_ids, _ = fused_topk(a, score.float(), topk, False) + + with set_current_vllm_config(vllm_config): + ref_out = torch_w8a8_block_fp8_moe( + a, w1, w2, w1_s, w2_s, topk_weights, topk_ids, refined_block + ) + out = fused_experts( + a, w1, w2, topk_weights, topk_ids, quant_config=quant_config + ) + + torch.testing.assert_close(out, ref_out, atol=0.035, rtol=0.035) diff --git a/tests/kernels/moe/test_trtllm_bf16_moe.py b/tests/kernels/moe/test_trtllm_bf16_moe.py index 22dff237cb..76b7fd8b6d 100644 --- a/tests/kernels/moe/test_trtllm_bf16_moe.py +++ b/tests/kernels/moe/test_trtllm_bf16_moe.py @@ -28,8 +28,9 @@ from vllm.model_executor.layers.fused_moe.config import ( from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import ( TrtLlmBf16ExpertsModular, ) -from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( - convert_moe_weights_to_flashinfer_trtllm_block_layout, +from vllm.model_executor.layers.fused_moe.oracle.unquantized import ( + UnquantizedMoeBackend, + convert_to_unquantized_kernel_format, ) from vllm.platforms import current_platform from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe @@ -46,9 +47,8 @@ if pytest and ( ) # (m, n, k) = (tokens, intermediate_size_per_partition, hidden_dim). -# Covers larger NvFP4-like shapes while keeping BF16's FlashInfer TRTLLM -# intermediate-size multiple-of-128 requirement. MNK_FACTORS = [ + (2, 160, 2560), (2, 1024, 1024), (64, 2048, 1536), (64, 1024, 4096), @@ -97,11 +97,15 @@ def test_trtllm_bf16_moe_modular_no_graph( max_num_tokens=next_power_of_2(m), ) - trtllm_w1, trtllm_w2 = convert_moe_weights_to_flashinfer_trtllm_block_layout( - {}, + trtllm_w1, trtllm_w2 = convert_to_unquantized_kernel_format( + UnquantizedMoeBackend.FLASHINFER_TRTLLM, + moe_config, w1, w2, ) + expected_n = (n + 127) // 128 * 128 + assert moe_config.intermediate_size_per_partition == expected_n + assert trtllm_w2.numel() == e * k * expected_n trtllm_experts = mk.FusedMoEKernel( maybe_make_prepare_finalize( diff --git a/tests/kernels/test_fused_gdn_post_conv.py b/tests/kernels/test_fused_gdn_post_conv.py index 7cdeb55b9b..0be75159da 100644 --- a/tests/kernels/test_fused_gdn_post_conv.py +++ b/tests/kernels/test_fused_gdn_post_conv.py @@ -267,6 +267,7 @@ def test_fused_post_conv_l0(): ), ], ) +@pytest.mark.parametrize("output_gate_activation", ["silu", "sigmoid"]) @torch.inference_mode() def test_fused_gdn_decode_post_conv_mtp_head_ratios( head_ratio: int, @@ -274,6 +275,7 @@ def test_fused_gdn_decode_post_conv_mtp_head_ratios( query_lengths: tuple[int, ...], state_dtype: torch.dtype, norm_dtype: torch.dtype, + output_gate_activation: str, ) -> None: if torch.cuda.get_device_capability() < (8, 0): pytest.skip("fused GDN decode MTP requires compute capability 8.0+") @@ -357,7 +359,7 @@ def test_fused_gdn_decode_post_conv_mtp_head_ratios( z=output_gate, eps=eps, norm_before_gate=True, - activation="silu", + activation=output_gate_activation, ) actual = ops.fused_gdn_decode_post_conv_mtp( mixed_qkv=mixed_qkv, @@ -374,6 +376,7 @@ def test_fused_gdn_decode_post_conv_mtp_head_ratios( out=torch.empty_like(output_gate), scale=scale, norm_eps=eps, + output_gate_activation=output_gate_activation, ) output_error = (actual.float() - expected.float()).norm() diff --git a/tests/model_executor/layers/test_fused_shared_expert.py b/tests/model_executor/layers/test_fused_shared_expert.py index 968a4d57d1..4b4e10df58 100644 --- a/tests/model_executor/layers/test_fused_shared_expert.py +++ b/tests/model_executor/layers/test_fused_shared_expert.py @@ -18,8 +18,12 @@ from vllm.model_executor.layers.fused_moe.layer import determine_expert_counts from vllm.model_executor.layers.quantization.fp8 import Fp8Config from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig from vllm.model_executor.layers.quantization.utils.config_utils import ( + get_quark_ocp_mx_group_size, is_shared_expert_quant_fse_compatible, ) +from vllm.model_executor.models.qwen3_next import ( + _should_replicate_misaligned_shared_expert, +) from vllm.model_executor.models.utils import PPMissingLayer from vllm.models.deepseek_v4 import quant_config as deepseek_v4_quant_config from vllm.models.minimax_m3.amd import model as minimax_m3_model @@ -605,6 +609,82 @@ def test_quark_shared_expert_fse_compatibility( ) +def test_quark_ocp_mx_group_size_for_layer() -> None: + quant_config = QuarkConfig({**_QUARK_FSE_CONFIG, "exclude": []}) + assert ( + get_quark_ocp_mx_group_size( + quant_config, + "model.layers.0.mlp.shared_expert.down_proj", + ) + == 32 + ) + + quant_config.quant_config["exclude"] = ["model.layers.0.mlp.shared_expert_gate"] + assert ( + get_quark_ocp_mx_group_size( + quant_config, + "model.layers.0.mlp.shared_expert.down_proj", + ) + == 32 + ) + + quant_config.quant_config["exclude"] = [ + "model.layers.0.mlp.shared_expert.down_proj" + ] + assert ( + get_quark_ocp_mx_group_size( + quant_config, + "model.layers.0.mlp.shared_expert.down_proj", + ) + is None + ) + + +@pytest.mark.parametrize("tp_size", [1, 2, 4]) +def test_quark_ocp_mx_shared_expert_uses_supported_tp_shards(tp_size: int) -> None: + assert not _should_replicate_misaligned_shared_expert( + intermediate_size=640, + tp_size=tp_size, + group_size=32, + enable_expert_parallel=False, + is_sequence_parallel=False, + ) + + +def test_quark_ocp_mx_shared_expert_rejects_plain_tp8() -> None: + with pytest.raises( + ValueError, + match="TP size 8 produces a partition of 80", + ): + _should_replicate_misaligned_shared_expert( + intermediate_size=640, + tp_size=8, + group_size=32, + enable_expert_parallel=False, + is_sequence_parallel=False, + ) + + +def test_quark_ocp_mx_shared_expert_is_replicated_for_tep8() -> None: + assert _should_replicate_misaligned_shared_expert( + intermediate_size=640, + tp_size=8, + group_size=32, + enable_expert_parallel=True, + is_sequence_parallel=False, + ) + + +def test_quark_ocp_mx_shared_expert_is_replicated_when_not_tp_divisible() -> None: + assert _should_replicate_misaligned_shared_expert( + intermediate_size=672, + tp_size=8, + group_size=32, + enable_expert_parallel=True, + is_sequence_parallel=False, + ) + + def test_quark_shared_expert_fse_requires_matching_layer_quant_configs() -> None: global_quant_config = {"weight": {"dtype": "fp4"}} quant_config = QuarkConfig( diff --git a/tests/models/qwen4_exp/__init__.py b/tests/models/qwen4_exp/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/tests/models/qwen4_exp/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/tests/models/qwen4_exp/test_config.py b/tests/models/qwen4_exp/test_config.py new file mode 100644 index 0000000000..b883aa24b8 --- /dev/null +++ b/tests/models/qwen4_exp/test_config.py @@ -0,0 +1,200 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +import torch + +from vllm.config.speculative import SpeculativeConfig +from vllm.model_executor.models.config import ( + Qwen3_5ForConditionalGenerationConfig, + Qwen4ExpForConditionalGenerationConfig, +) +from vllm.models.qwen4_exp.config import ( + Qwen4ExpConfig, + Qwen4ExpTextConfig, +) +from vllm.models.qwen4_exp.nvidia.model_state import Qwen4ExpModelState +from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState + + +def _text_config(**kwargs) -> Qwen4ExpTextConfig: + values = { + "vocab_size": 64, + "hidden_size": 16, + "intermediate_size": 32, + "num_hidden_layers": 2, + "num_attention_heads": 2, + "num_key_value_heads": 1, + "head_dim": 8, + "layer_types": ["linear_attention", "full_attention"], + "linear_num_key_heads": 2, + "linear_num_value_heads": 2, + "linear_key_head_dim": 8, + "linear_value_head_dim": 8, + "num_experts": 0, + "hc_count": 2, + "hc_lowrank": 4, + "ple_layer_ids": [1], + "mtp_num_hidden_layers": 1, + "mtp": {"hybrid": True}, + } + values.update(kwargs) + return Qwen4ExpTextConfig(**values) + + +def test_qwen4_exp_mtp_returns_sample_and_multi_streams() -> None: + from vllm.models.qwen4_exp.nvidia.mtp import ( + Qwen4ExpMultiTokenPredictor, + ) + + model = object.__new__(Qwen4ExpMultiTokenPredictor) + torch.nn.Module.__init__(model) + model.hc_count = 2 + model.hidden_size = 4 + model.num_mtp_layers = 1 + model.layers = [ + lambda **kwargs: ( + kwargs["hidden_states"], + kwargs["hidden_states"], + torch.zeros(kwargs["hidden_states"].shape[0], 2), + ), + ] + model.hyper_connection_mixer = SimpleNamespace( + combine_and_mix=lambda hidden_states, block_output, injection: ( + hidden_states, + hidden_states.unflatten(-1, (2, 4)).mean(-2), + None, + ), + ) + multi_hidden = torch.arange(16, dtype=torch.float32).reshape(2, 8) + pp_group = SimpleNamespace(is_first_rank=False, is_last_rank=True) + + with patch( + "vllm.models.qwen4_exp.nvidia.mtp.get_pp_group", + return_value=pp_group, + ): + sample_hidden, returned_multi_hidden = model.forward( + input_ids=None, + positions=torch.arange(2), + intermediate_tensors={"hidden_states": multi_hidden}, + ) + + torch.testing.assert_close( + sample_hidden, + multi_hidden.unflatten(-1, (2, 4)).mean(dim=-2), + ) + assert returned_multi_hidden is multi_hidden + + +@pytest.mark.parametrize("wrapped_config", [False, True]) +def test_qwen4_exp_mtp_override_sets_draft_config( + wrapped_config: bool, +) -> None: + text_config = _text_config( + architectures=["Qwen4ExpForCausalLM"], + index_share_for_mtp_iteration=True, + ) + config = ( + Qwen4ExpConfig( + architectures=["Qwen4ExpForConditionalGeneration"], + text_config=text_config, + ) + if wrapped_config + else text_config + ) + + draft_config = SpeculativeConfig.hf_config_override(config) + + assert draft_config.index_share_for_mtp_iteration is True + assert draft_config.model_type == "qwen4_exp_mtp" + assert draft_config.architectures == ["Qwen4ExpMTP"] + assert draft_config.hc_mult == 2 + assert draft_config.n_predict == 1 + + +@pytest.mark.parametrize("ple_layer_ids", [[1], []]) +def test_qwen4_exp_rejects_pipeline_parallel_only_with_ple(ple_layer_ids) -> None: + """PLE needs raw input_ids, which non-first pipeline ranks never see. The + rest of the architecture is PP-capable, so the refusal must be conditional + -- and must land before the engine spends time loading weights.""" + vllm_config = SimpleNamespace( + model_config=SimpleNamespace( + hf_text_config=_text_config(ple_layer_ids=ple_layer_ids), + multimodal_config=None, + ), + parallel_config=SimpleNamespace( + pipeline_parallel_size=2, enable_dbo=False, ubatch_size=1 + ), + speculative_config=None, + ) + with patch.object( + Qwen3_5ForConditionalGenerationConfig, "verify_and_update_config" + ): + if ple_layer_ids: + with pytest.raises(NotImplementedError, match="pipeline_parallel_size=1"): + Qwen4ExpForConditionalGenerationConfig.verify_and_update_config( + vllm_config + ) + else: + Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(vllm_config) + + +def test_qwen4_exp_model_state_prepares_ngram_context() -> None: + model_state = object.__new__(Qwen4ExpModelState) + model_state.uses_ngram_embedding = True + model_state.ngram_context_len = 3 + model_state.ngram_eos_token_id = 99 + model_state.ngram_context = torch.empty((4, 3), dtype=torch.int32) + model_state.ngram_context_offsets = torch.arange(-3, 0, dtype=torch.int64) + model_state.ple_query_start_loc = torch.empty(5, dtype=torch.int32) + + input_batch = SimpleNamespace( + num_reqs=2, + num_reqs_after_padding=3, + idx_mapping=torch.tensor([1, 0]), + query_start_loc=torch.tensor([0, 2, 3, 3], dtype=torch.int32), + ) + req_states = SimpleNamespace( + num_computed_tokens=SimpleNamespace(gpu=torch.tensor([3, 1])), + all_token_ids=SimpleNamespace( + gpu=torch.tensor([[1, 2, 3, 4], [20, 21, 22, 23]], dtype=torch.int32) + ), + ) + + with patch.object(MambaHybridModelState, "prepare_inputs", return_value={}): + model_inputs = model_state.prepare_inputs(input_batch, req_states) + + torch.testing.assert_close( + model_inputs["query_start_loc"], + torch.tensor([0, 2, 3, 3], dtype=torch.int32), + ) + torch.testing.assert_close( + model_inputs["ngram_context"], + torch.tensor([[99, 99, 20], [1, 2, 3], [99, 99, 99]], dtype=torch.int32), + ) + + +def test_qwen4_exp_model_state_prepares_stable_dummy_ngram_inputs() -> None: + model_state = object.__new__(Qwen4ExpModelState) + model_state.uses_ngram_embedding = True + model_state.ngram_eos_token_id = 99 + model_state.ngram_context = torch.empty((4, 3), dtype=torch.int32) + model_state.ple_query_start_loc = torch.empty(5, dtype=torch.int32) + + with patch.object(MambaHybridModelState, "prepare_dummy_inputs", return_value={}): + first = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=8) + query_start_loc_ptr = first["query_start_loc"].data_ptr() + ngram_context_ptr = first["ngram_context"].data_ptr() + second = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=8) + + torch.testing.assert_close( + second["query_start_loc"], torch.tensor([0, 2, 5, 8], dtype=torch.int32) + ) + torch.testing.assert_close( + second["ngram_context"], torch.full((3, 3), 99, dtype=torch.int32) + ) + assert second["query_start_loc"].data_ptr() == query_start_loc_ptr + assert second["ngram_context"].data_ptr() == ngram_context_ptr diff --git a/tests/models/qwen4_exp/test_hc_ops.py b/tests/models/qwen4_exp/test_hc_ops.py new file mode 100644 index 0000000000..4af4e55732 --- /dev/null +++ b/tests/models/qwen4_exp/test_hc_ops.py @@ -0,0 +1,94 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch + +from vllm.models.qwen4_exp.nvidia.ops.hc import ( + grouped_gemma_rmsnorm, + hc_combine, + hc_combine_norm, + hc_gate_mix, +) +from vllm.platforms import current_platform +from vllm.triton_utils import HAS_TRITON + +pytestmark = pytest.mark.skipif( + not current_platform.is_cuda() or not HAS_TRITON, + reason="HC kernels require CUDA and Triton", +) + +HC = 4 +HIDDEN_SIZE = 2560 +HYPER_HIDDEN_SIZE = HC * HIDDEN_SIZE +EPS = 1e-6 + + +def test_grouped_gemma_rmsnorm() -> None: + torch.manual_seed(0) + x = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda") + weight = torch.randn(HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda") + + actual = grouped_gemma_rmsnorm(x, weight, EPS, HC) + + grouped = x.float().unflatten(-1, (HC, HIDDEN_SIZE)) + variance = grouped.square().mean(-1, keepdim=True) + expected = grouped * torch.rsqrt(variance + EPS) + expected = expected.flatten(-2) * (1.0 + weight.float()) + torch.testing.assert_close(actual, expected.to(torch.bfloat16)) + + +def test_hc_gate_mix() -> None: + torch.manual_seed(0) + x = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda") + gate = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda") + + actual = hc_gate_mix(x, gate, HC) + expected = ( + torch.sigmoid(gate.float().unflatten(-1, (HC, HIDDEN_SIZE))) + * x.float().unflatten(-1, (HC, HIDDEN_SIZE)) + ).mean(-2) + + torch.testing.assert_close(actual, expected.to(torch.bfloat16)) + + +def test_hc_combine() -> None: + torch.manual_seed(0) + block_output = torch.randn(2, HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda") + residual = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda") + injection = torch.randn(2, HC, dtype=torch.bfloat16, device="cuda") + + actual = hc_combine(residual, block_output, injection, HC) + injection_weight = 2.0 * torch.sigmoid(injection.float() / HC) + expected = residual.float().unflatten(-1, (HC, HIDDEN_SIZE)) + expected = expected + block_output.float().unsqueeze( + -2 + ) * injection_weight.unsqueeze(-1) + + torch.testing.assert_close(actual, expected.flatten(-2).to(torch.bfloat16)) + + +def test_hc_combine_norm() -> None: + torch.manual_seed(0) + block_output = torch.randn(2, HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda") + residual = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda") + injection = torch.randn(2, HC, dtype=torch.bfloat16, device="cuda") + weight = torch.randn(HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda") + + actual, actual_norm = hc_combine_norm( + residual, block_output, injection, weight, EPS, HC + ) + + injection_weight = 2.0 * torch.sigmoid(injection.float() / HC) + expected = residual.float().unflatten(-1, (HC, HIDDEN_SIZE)) + expected = expected + block_output.float().unsqueeze( + -2 + ) * injection_weight.unsqueeze(-1) + expected = expected.flatten(-2).to(residual.dtype) + grouped = expected.float().unflatten(-1, (HC, HIDDEN_SIZE)) + variance = grouped.square().mean(-1, keepdim=True) + expected_norm = grouped * torch.rsqrt(variance + EPS) + expected_norm = expected_norm.flatten(-2) * (1.0 + weight.float()) + + torch.testing.assert_close(actual, expected) + torch.testing.assert_close(actual_norm, expected_norm.to(torch.bfloat16)) diff --git a/tests/models/qwen4_exp/test_ple.py b/tests/models/qwen4_exp/test_ple.py new file mode 100644 index 0000000000..da832a3e68 --- /dev/null +++ b/tests/models/qwen4_exp/test_ple.py @@ -0,0 +1,401 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from functools import partial +from types import SimpleNamespace + +import pytest +import torch +from torch import nn +from torch.nn import functional as F + +import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module +import vllm.model_executor.parameter as parameter_module +from vllm.model_executor.layers.quantization.fp8 import Fp8Config +from vllm.models.qwen4_exp.common.ple import ( + PLEShardOverlap, + compute_ple_shard_overlap, + copy_ple_embedding_shard_, +) +from vllm.models.qwen4_exp.nvidia import ple_layer as ple_layer_module +from vllm.models.qwen4_exp.nvidia.ple_layer import ( + Qwen4ExpNGramEmbedding, + Qwen4ExpPLEFp8EmbeddingMethod, + Qwen4ExpPLELayer, + _get_ple_embedding_quant_method, +) + + +def _make_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding: + module = Qwen4ExpNGramEmbedding.__new__(Qwen4ExpNGramEmbedding) + nn.Module.__init__(module) + module.split_ngram_parts = 2 + module.register_buffer("layer_multipliers", torch.zeros(1, dtype=torch.long)) + module.register_buffer("ngram_heads_offsets", torch.zeros(1, dtype=torch.long)) + module.register_buffer("ngram_heads_vocab_sizes", torch.zeros(1, dtype=torch.long)) + module.ngram_embedding = SimpleNamespace( + org_vocab_size=8, + embedding_dim=2, + weight=nn.Parameter(torch.full((4, 2), -1.0)), + shard_indices=SimpleNamespace( + org_vocab_start_index=2, + org_vocab_end_index=6, + ), + ) + _set_test_embedding_weight_loader(module.ngram_embedding) + return module + + +def _set_test_embedding_weight_loader(embedding) -> None: + embedding.weight.weight_loader = partial( + copy_ple_embedding_shard_, + tp_start=embedding.shard_indices.org_vocab_start_index, + tp_end=embedding.shard_indices.org_vocab_end_index, + ) + + +def _make_fp8_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding: + module = _make_ngram_embedding_for_load_test() + embedding = nn.Module() + embedding.org_vocab_size = 8 + embedding.embedding_dim = 2 + embedding.shard_indices = SimpleNamespace( + org_vocab_start_index=2, + org_vocab_end_index=6, + ) + embedding.register_parameter( + "weight", + nn.Parameter( + torch.full((4, 2), -1.0).to(torch.float8_e4m3fn), + requires_grad=False, + ), + ) + embedding.register_parameter( + "weight_scale", + nn.Parameter(torch.zeros(1, dtype=torch.bfloat16), requires_grad=False), + ) + _set_test_embedding_weight_loader(embedding) + module.ngram_embedding = embedding + return module + + +def test_ple_shard_overlap_and_copy() -> None: + overlap = compute_ple_shard_overlap( + checkpoint_start=2, checkpoint_rows=5, tp_start=4, tp_end=8 + ) + assert overlap == PLEShardOverlap(source_start=2, destination_start=0, row_count=3) + + destination = torch.full((4, 2), -1.0) + loaded = torch.arange(10, dtype=torch.float64).reshape(5, 2) + copied = copy_ple_embedding_shard_( + destination, + loaded, + checkpoint_start=2, + tp_start=4, + tp_end=8, + ) + + assert copied == 3 + torch.testing.assert_close(destination[:3], loaded[2:5].float()) + torch.testing.assert_close(destination[3], torch.tensor([-1.0, -1.0])) + + +def test_ple_shard_copy_is_a_noop_without_overlap() -> None: + destination = torch.ones(4, 2) + copied = copy_ple_embedding_shard_( + destination, + torch.zeros(2, 2), + checkpoint_start=10, + tp_start=4, + tp_end=8, + ) + + assert copied == 0 + assert torch.equal(destination, torch.ones_like(destination)) + + +def test_ngram_embedding_loads_shards_and_ignores_legacy_token_lookup() -> None: + module = _make_ngram_embedding_for_load_test() + shard_0 = torch.arange(8, dtype=torch.float32).reshape(4, 2) + shard_1 = torch.arange(8, 16, dtype=torch.float32).reshape(4, 2) + + loaded = module.load_weights( + [ + ("ngram_embedding.shard_0.weight", shard_0), + ("ngram_embedding.shard_1.weight", shard_1), + ("token_lookup", torch.tensor([2, 1, 0])), + ] + ) + + assert loaded == {"ngram_embedding.weight"} + torch.testing.assert_close( + module.ngram_embedding.weight, + torch.cat((shard_0[2:4], shard_1[0:2])), + ) + + +def test_ngram_embedding_loads_fp8_shards_and_global_scale() -> None: + module = _make_fp8_ngram_embedding_for_load_test() + shard_0 = torch.arange(8, dtype=torch.float32).reshape(4, 2).to(torch.float8_e4m3fn) + shard_1 = ( + torch.arange(8, 16, dtype=torch.float32).reshape(4, 2).to(torch.float8_e4m3fn) + ) + weight_scale = torch.tensor([0.25], dtype=torch.bfloat16) + + loaded = module.load_weights( + [ + ("ngram_embedding.shard_0.weight", shard_0), + ("ngram_embedding.shard_1.weight", shard_1), + ("ngram_embedding.weight_scale", weight_scale), + ] + ) + + assert loaded == {"ngram_embedding.weight", "ngram_embedding.weight_scale"} + assert module.ngram_embedding.weight.dtype == torch.float8_e4m3fn + assert torch.equal( + module.ngram_embedding.weight.float(), + torch.cat((shard_0[2:4], shard_1[0:2])).float(), + ) + assert torch.equal(module.ngram_embedding.weight_scale, weight_scale) + + +def _make_fp8_embedding_layer( + monkeypatch: pytest.MonkeyPatch, +) -> embedding_module.VocabParallelEmbedding: + monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0) + monkeypatch.setattr( + embedding_module, "get_tensor_model_parallel_world_size", lambda: 1 + ) + monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0) + monkeypatch.setattr( + parameter_module, "get_tensor_model_parallel_world_size", lambda: 1 + ) + method = Qwen4ExpPLEFp8EmbeddingMethod() + layer = embedding_module.VocabParallelEmbedding( + 3, + 2, + params_dtype=torch.bfloat16, + padding_size=1, + quant_method=method, + ) + weight = torch.tensor([[1.0, 2.0], [4.0, 8.0], [16.0, 32.0]]) + layer.weight.data.copy_(weight.to(torch.float8_e4m3fn)) + layer.weight_scale.data.copy_(torch.tensor([0.25], dtype=torch.bfloat16)) + return layer + + +def test_ple_fp8_embedding_dequantizes_in_ple_layer(monkeypatch) -> None: + layer = _make_fp8_embedding_layer(monkeypatch) + quantized_output = layer(torch.tensor([2, 0])) + ple_layer = Qwen4ExpPLELayer.__new__(Qwen4ExpPLELayer) + nn.Module.__init__(ple_layer) + ple_layer.ple_embedding = nn.Module() + ple_layer.ple_embedding.ngram_embedding = layer + + output = ple_layer._dequantize_embeddings( + quantized_output, + torch.bfloat16, + ) + + assert layer.weight.dtype == torch.float8_e4m3fn + assert layer.weight_scale.dtype == torch.bfloat16 + assert quantized_output.dtype == torch.float8_e4m3fn + assert output.dtype == torch.bfloat16 + weight = torch.tensor([[1.0, 2.0], [4.0, 8.0], [16.0, 32.0]]) + torch.testing.assert_close(output, (weight[[2, 0]] * 0.25).bfloat16()) + + +def test_ple_fp8_embedding_uses_int8_for_tp_reduce(monkeypatch) -> None: + monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0) + monkeypatch.setattr( + embedding_module, "get_tensor_model_parallel_world_size", lambda: 2 + ) + monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0) + monkeypatch.setattr( + parameter_module, "get_tensor_model_parallel_world_size", lambda: 2 + ) + monkeypatch.setattr( + embedding_module, + "get_masked_input_and_mask", + lambda *args: ( + torch.tensor([0, 0]), + torch.tensor([False, True]), + ), + ) + reduced_dtypes = [] + + def all_reduce(tensor: torch.Tensor) -> torch.Tensor: + reduced_dtypes.append(tensor.dtype) + return tensor.clone() + + monkeypatch.setattr( + embedding_module, + "tensor_model_parallel_all_reduce", + all_reduce, + ) + layer = embedding_module.VocabParallelEmbedding( + 4, + 2, + params_dtype=torch.bfloat16, + padding_size=1, + quant_method=Qwen4ExpPLEFp8EmbeddingMethod(), + ) + layer.weight.data.copy_( + torch.tensor([[1.0, 2.0], [4.0, 8.0]]).to(torch.float8_e4m3fn) + ) + + output = layer(torch.tensor([0, 2])) + + assert reduced_dtypes == [torch.int8] + assert output.dtype == torch.float8_e4m3fn + torch.testing.assert_close(output[0].float(), layer.weight[0].float()) + assert torch.count_nonzero(output[1].float()) == 0 + + +def test_ple_fp8_embedding_respects_checkpoint_shard_exclusions() -> None: + prefix = "model.layers.1.ple.ple_embedding.ngram_embedding" + quant_config = Fp8Config( + is_checkpoint_fp8_serialized=True, + ignored_layers=[], + weight_block_size=[128, 128], + ) + assert isinstance( + _get_ple_embedding_quant_method(quant_config, prefix), + Qwen4ExpPLEFp8EmbeddingMethod, + ) + + quant_config.ignored_layers = [f"{prefix}.shard_0"] + assert _get_ple_embedding_quant_method(quant_config, prefix) is None + + +def test_ple_ngram_ids_custom_op_uses_current_request_layout(monkeypatch) -> None: + class RuntimeNGramEmbedding(nn.Module): + def compute_ngram_ids( + self, + input_ids: torch.Tensor, + query_start_loc: torch.Tensor, + ngram_context: torch.Tensor, + ) -> torch.Tensor: + del input_ids, ngram_context + num_reqs = query_start_loc.numel() - 1 + return torch.full((4, 2), num_reqs, dtype=torch.long) + + layer = Qwen4ExpPLELayer.__new__(Qwen4ExpPLELayer) + nn.Module.__init__(layer) + layer.ple_embedding = RuntimeNGramEmbedding() + monkeypatch.setattr( + ple_layer_module, + "get_forward_context", + lambda: SimpleNamespace(no_compile_layers={"ple": layer}), + ) + input_ids = torch.arange(4) + ngram_context = torch.zeros(2, 2, dtype=torch.long) + output = torch.empty(4, 2, dtype=torch.long) + + ple_layer_module.qwen4_exp_compute_ple_ngram_ids( + input_ids, + torch.tensor([0, 4]), + ngram_context, + output, + "ple", + ) + assert torch.equal(output, torch.ones_like(output)) + + ple_layer_module.qwen4_exp_compute_ple_ngram_ids( + input_ids, + torch.tensor([0, 2, 4]), + ngram_context, + output, + "ple", + ) + assert torch.equal(output, torch.full_like(output, 2)) + + +def test_dilated_ple_spec_state_rolls_back_before_next_forward() -> None: + module = Qwen4ExpPLELayer.__new__(Qwen4ExpPLELayer) + nn.Module.__init__(module) + module.conv_state_len = 6 + module.short_conv_dilation = 2 + + conv_weights = torch.tensor([[0.25, -0.5, 0.75, 1.0]]) + conv_state = torch.zeros(2, 1, 9) + conv_state[1] = torch.arange(1, 10, dtype=torch.float32).reshape(1, 9) + first_inputs = torch.tensor([[10.0], [20.0], [30.0], [40.0]]) + initial_state = conv_state[1:].clone() + first_history = torch.cat( + (initial_state[..., : module.conv_state_len], first_inputs.T.unsqueeze(0)), + dim=-1, + ) + + graph_padded_inputs = F.pad(first_inputs, (0, 0, 0, 4)) + first_output = module._short_conv_dilated_spec_batched( + graph_padded_inputs, + conv_state, + conv_weights, + torch.tensor([1, 0]), + torch.tensor([0, 4, 4]), + torch.tensor([1, 0]), + spec_query_len=4, + ) + + expected_first_output = F.silu( + F.conv1d( + first_history, + conv_weights.unsqueeze(1), + groups=1, + dilation=module.short_conv_dilation, + ) + ).transpose(1, 2)[0] + expected_first_state = first_history[..., 1:10] + torch.testing.assert_close(first_output[:4], expected_first_output) + assert torch.count_nonzero(first_output[4:]) == 0 + assert torch.count_nonzero(conv_state[0]) == 0 + torch.testing.assert_close(conv_state[1:], expected_first_state) + + second_inputs = torch.tensor([[50.0], [60.0]]) + rollback_state = expected_first_state[..., 1:7] + padded_second_inputs = F.pad(second_inputs.T.unsqueeze(0), (0, 2)) + second_history = torch.cat((rollback_state, padded_second_inputs), dim=-1) + expected_second_state = expected_first_state.clone() + expected_second_state[..., :7] = second_history[..., 1:8] + + second_output = module._short_conv_dilated_spec_batched( + second_inputs, + conv_state, + conv_weights, + torch.tensor([1]), + torch.tensor([0, 2]), + torch.tensor([2]), + spec_query_len=4, + ) + + expected_second_output = F.silu( + F.conv1d( + second_history, + conv_weights.unsqueeze(1), + groups=1, + dilation=module.short_conv_dilation, + ) + ).transpose(1, 2)[0, :2] + torch.testing.assert_close(second_output, expected_second_output) + torch.testing.assert_close(conv_state[1:], expected_second_state) + + +def test_ple_short_conv_uses_fallback_when_profile_metadata_is_omitted( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = Qwen4ExpPLELayer.__new__(Qwen4ExpPLELayer) + nn.Module.__init__(module) + module.prefix = "model.layers.1.ple" + inputs = torch.arange(6, dtype=torch.float32).reshape(3, 2) + expected = inputs + 1 + monkeypatch.setattr(module, "_short_conv_fallback", lambda _: expected) + monkeypatch.setattr( + ple_layer_module, + "get_forward_context", + lambda: SimpleNamespace(attn_metadata={"model.layers.0.self_attn": object()}), + ) + + output = module._short_conv(inputs) + + assert output is expected diff --git a/tests/models/qwen4_exp/test_qsa_amd.py b/tests/models/qwen4_exp/test_qsa_amd.py new file mode 100644 index 0000000000..e8e39a7666 --- /dev/null +++ b/tests/models/qwen4_exp/test_qsa_amd.py @@ -0,0 +1,297 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import math +from types import SimpleNamespace +from typing import Any + +import pytest +import torch + +from vllm.model_executor.layers.layernorm import GemmaRMSNorm +from vllm.models.qwen4_exp.amd import ( + model as _qwen4_exp_model, # noqa: F401 +) +from vllm.models.qwen4_exp.amd import ple_layer as ple_layer_module +from vllm.models.qwen4_exp.amd.indexer_qsa import ( + apply_qsa_rmsnorm, + apply_qsa_rope, +) +from vllm.models.qwen4_exp.amd.ops import qsa as qsa_ops +from vllm.platforms import current_platform +from vllm.triton_utils import HAS_TRITON + +pytestmark = pytest.mark.skipif( + not current_platform.is_rocm(), + reason="AMD QSA requires ROCm", +) + +requires_qsa_kernels = pytest.mark.skipif( + not HAS_TRITON, + reason="AMD QSA kernels require Triton", +) + + +def test_ple_ngram_embedding_custom_op_uses_resident_weight( + monkeypatch: pytest.MonkeyPatch, +) -> None: + layer_name = "model.layers.0.ple" + layer = ple_layer_module.Qwen4ExpPLELayer.__new__(ple_layer_module.Qwen4ExpPLELayer) + torch.nn.Module.__init__(layer) + layer.ple_embedding = torch.nn.Module() + layer.ple_embedding.ngram_embedding = torch.nn.Embedding(8, 3) + context = SimpleNamespace(no_compile_layers={layer_name: layer}) + monkeypatch.setattr(ple_layer_module, "get_forward_context", lambda: context) + + ngram_ids = torch.tensor([[0, 1], [2, 3]]) + output = torch.empty(2, 6) + ple_layer_module.qwen4_exp_amd_ple_ngram_embedding( + ngram_ids, + output, + layer_name, + ) + + expected = layer.ple_embedding.ngram_embedding(ngram_ids).flatten(-2) + torch.testing.assert_close(output, expected) + + +def _qsa_sparse_paged_attention_reference( + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + logical_indices: torch.Tensor, + block_table: torch.Tensor, + token_to_req: torch.Tensor, + softmax_scale: float, +) -> torch.Tensor: + output = torch.zeros_like(q) + repeats = q.shape[1] // k_cache.shape[2] + page_size = k_cache.shape[1] + for row in range(q.shape[0]): + logical = logical_indices[row] + logical = logical[logical >= 0].long() + if not logical.numel(): + continue + request = token_to_req[row].long() + pages = block_table[request, logical // page_size].long() + offsets = logical % page_size + keys = k_cache[pages, offsets].repeat_interleave(repeats, dim=1) + values = v_cache[pages, offsets].repeat_interleave(repeats, dim=1) + scores = torch.einsum("hd,khd->hk", q[row].float(), keys.float()) + probabilities = torch.softmax(scores * softmax_scale, dim=-1) + output[row] = torch.einsum("hk,khd->hd", probabilities, values.float()).to( + q.dtype + ) + return output + + +def test_qsa_rope_uses_platform_dispatch() -> None: + tensor = torch.arange(16, dtype=torch.float32).reshape(2, 2, 4) + positions = torch.tensor([0, 1]) + calls = [] + + def apply_rotary_emb( + rotary_input: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + ) -> torch.Tensor: + calls.append((rotary_input, cos, sin)) + return rotary_input + 1 + + rotary_emb = SimpleNamespace( + rotary_dim=2, + apply_rotary_emb=apply_rotary_emb, + _match_cos_sin_cache_dtype=lambda _: torch.zeros(2, 4), + ) + + output = apply_qsa_rope(rotary_emb, positions, tensor) + + assert len(calls) == 1 + torch.testing.assert_close(output[..., :2], tensor[..., :2] + 1) + torch.testing.assert_close(output[..., 2:], tensor[..., 2:]) + + +def test_qsa_rmsnorm_uses_portable_implementation(default_vllm_config) -> None: + norm = GemmaRMSNorm(4, eps=1e-6) + norm.weight.data.copy_(torch.tensor([0.1, -0.2, 0.3, -0.4])) + tensor = torch.arange(8, dtype=torch.float32).reshape(2, 4) + output = apply_qsa_rmsnorm(norm, tensor) + + torch.testing.assert_close(output, norm.forward_native(tensor)) + + +def test_qsa_selection_uses_portable_topk_on_rocm( + monkeypatch: pytest.MonkeyPatch, +) -> None: + rows = 2 + token_topk = 8 + compress_ratio = 4 + block_topk = token_topk // compress_ratio + blocks = torch.empty((rows, block_topk), dtype=torch.int32) + visible_blocks = torch.tensor([4, 6], dtype=torch.int32) + logits = torch.empty((rows, 8), dtype=torch.float32) + selection_output = torch.empty( + (rows, token_topk + compress_ratio - 1), dtype=torch.int32 + ) + topk_call: dict[str, Any] = {} + + monkeypatch.setattr( + qsa_ops, + "qsa_mqa_paged", + lambda *args, **kwargs: (logits, visible_blocks), + ) + monkeypatch.setattr(qsa_ops.current_platform, "is_cuda", lambda: False) + + def top_k_per_row_decode( + topk_logits, + next_n, + seq_lens, + raw_topk_indices, + num_rows, + stride0, + stride1, + topk_tokens, + ) -> None: + topk_call.update( + logits=topk_logits, + next_n=next_n, + seq_lens=seq_lens, + raw_topk_indices=raw_topk_indices, + num_rows=num_rows, + strides=(stride0, stride1), + topk_tokens=topk_tokens, + ) + raw_topk_indices.zero_() + + def expand_qsa_block_indices(*args) -> None: + args[-1].fill_(-1) + + monkeypatch.setattr(qsa_ops.ops, "top_k_per_row_decode", top_k_per_row_decode) + monkeypatch.setattr( + qsa_ops, + "expand_qsa_block_indices_cuda", + expand_qsa_block_indices, + ) + + output = qsa_ops.qsa_select_paged_tokens( + torch.empty((rows, 1, 1)), + torch.empty((1, 4, 1, 1)), + torch.empty((1, 2), dtype=torch.int32), + torch.zeros(rows, dtype=torch.int32), + torch.arange(rows, dtype=torch.int32), + torch.tensor([8], dtype=torch.int32), + token_topk, + compress_ratio, + selection_output, + ) + + assert output is selection_output + assert torch.all(output == -1) + assert topk_call["logits"] is logits + assert topk_call["next_n"] == 1 + assert topk_call["seq_lens"] is visible_blocks + assert topk_call["raw_topk_indices"].shape == blocks.shape + assert topk_call["num_rows"] == rows + assert topk_call["strides"] == (logits.stride(0), logits.stride(1)) + assert topk_call["topk_tokens"] == block_topk + + +@requires_qsa_kernels +@pytest.mark.parametrize( + ("num_rows", "num_query_heads", "num_kv_heads", "page_size"), + [ + pytest.param(1, 24, 2, 1792, id="tp1_split64"), + pytest.param(16, 12, 1, 1792, id="tp2_split32"), + pytest.param(32, 6, 1, 1024, id="tp4_split8"), + pytest.param(257, 6, 1, 1024, id="tp4_split4"), + pytest.param(513, 6, 1, 1024, id="tp4_split1"), + ], +) +def test_qsa_sparse_paged_attention_matches_reference( + num_rows: int, + num_query_heads: int, + num_kv_heads: int, + page_size: int, +) -> None: + torch.manual_seed(2) + head_dim = 256 + num_requests = 2 + num_selected_pages = 64 + num_pages_per_request = num_selected_pages + 1 + num_cache_blocks = num_requests * num_pages_per_request + indexer_budget = 2048 + indexer_compress_ratio = 4 + selection_width = indexer_budget + indexer_compress_ratio - 1 + q = torch.randn( + num_rows, num_query_heads, head_dim, device="cuda", dtype=torch.bfloat16 + ) + kv_cache = torch.randn( + num_cache_blocks, + page_size, + num_kv_heads, + 2 * head_dim, + device="cuda", + dtype=torch.bfloat16, + ) + k_cache, v_cache = kv_cache.split(head_dim, dim=-1) + block_table = ( + torch.randperm(num_cache_blocks, device="cuda") + .reshape(num_requests, num_pages_per_request) + .to(torch.int32) + ) + rows_per_request = math.ceil(num_rows / num_requests) + row_indices = torch.arange(num_rows, device="cuda", dtype=torch.int32) + token_to_req = row_indices // rows_per_request + request_row_counts = torch.tensor( + [rows_per_request, num_rows - rows_per_request], + device="cuda", + dtype=torch.int32, + ) + + context_length = num_pages_per_request * page_size - 1 + block_topk = indexer_budget // indexer_compress_ratio + compressed_blocks_per_page = page_size // indexer_compress_ratio + selection = torch.arange(block_topk, device="cuda") + selected_pages = selection % num_selected_pages + selected_offsets = selection // num_selected_pages + row_shifts = 2 * row_indices.unsqueeze(1) + selected_offsets = (selected_offsets + row_shifts) % compressed_blocks_per_page + block_indices = (selected_pages * compressed_blocks_per_page + selected_offsets).to( + torch.int32 + ) + rows_within_request = row_indices % rows_per_request + query_positions = ( + context_length - request_row_counts[token_to_req.long()] + rows_within_request + ).to(torch.int64) + sequence_lengths = torch.full( + (num_requests,), context_length, device="cuda", dtype=torch.int32 + ) + logical_indices = qsa_ops.expand_qsa_block_indices_cuda( + block_indices, + query_positions, + sequence_lengths, + token_to_req, + indexer_compress_ratio, + indexer_budget, + ) + assert logical_indices.shape == (num_rows, selection_width) + + actual = qsa_ops.qsa_sparse_paged_attention( + q, + k_cache, + v_cache, + logical_indices, + block_table, + token_to_req, + ) + expected = _qsa_sparse_paged_attention_reference( + q, + k_cache, + v_cache, + logical_indices, + block_table, + token_to_req, + q.shape[-1] ** -0.5, + ) + + torch.testing.assert_close(actual, expected, rtol=2e-2, atol=2e-2) diff --git a/tests/models/qwen4_exp/test_qsa_pre_indexer.py b/tests/models/qwen4_exp/test_qsa_pre_indexer.py new file mode 100644 index 0000000000..991c73af00 --- /dev/null +++ b/tests/models/qwen4_exp/test_qsa_pre_indexer.py @@ -0,0 +1,297 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Correctness tests for the fused QSA pre-indexer.""" + +import pytest +import torch + +from vllm.models.qwen4_exp.common.qsa_cache import ( + canonical_qsa_rope_positions, + circular_qsa_slot_mapping, + compressed_qsa_slot_mapping, +) +from vllm.models.qwen4_exp.nvidia.indexer_qsa import apply_qsa_rope +from vllm.models.qwen4_exp.nvidia.ops.qsa import ( + qsa_compress_groups_with_ratio, + qsa_store_cache_rows, +) +from vllm.models.qwen4_exp.nvidia.ops.qsa_pre_indexer import ( + qsa_pre_indexer, +) +from vllm.platforms import current_platform +from vllm.triton_utils import HAS_TRITON + +requires_qsa_kernels = pytest.mark.skipif( + not current_platform.is_cuda() or not HAS_TRITON, + reason="QSA kernels require CUDA and Triton", +) + +HQ, D = 4, 128 +CR = 4 +MROPE_SECTION = (11, 11, 10) +EPS = 1e-6 +BLOCK_SIZE = 16 +COMP_PAGE = BLOCK_SIZE // CR +ROPE_POS_OFFSET = D +RTOL = 1.6e-2 +ATOL = 1e-2 +MIXED_BATCH = ([260, 259, 138], [1, 1, 37], [8, 8, 8]) + + +def _make_block_table(block_counts): + num_blocks = sum(block_counts) + table = torch.full((len(block_counts), max(block_counts)), -1, dtype=torch.int32) + physical_blocks = torch.randperm(num_blocks) + offset = 0 + for request, count in enumerate(block_counts): + table[request, :count] = physical_blocks[offset : offset + count] + offset += count + return table, num_blocks + + +@requires_qsa_kernels +@pytest.mark.usefixtures("default_vllm_config") +@pytest.mark.parametrize( + "mrope,is_2d_positions,cache_rope_positions,state_size,seq_lens,query_lens,history_lens", + [ + pytest.param(True, True, True, 4, *MIXED_BATCH, id="mrope"), + pytest.param(False, False, False, 4, *MIXED_BATCH, id="text"), + pytest.param(True, False, True, 8, *MIXED_BATCH, id="mrope-model-1d"), + pytest.param(True, True, False, 4, *MIXED_BATCH, id="mrope-no-position-cache"), + pytest.param( + True, + False, + False, + 8, + *MIXED_BATCH, + id="mrope-model-1d-no-position-cache", + ), + pytest.param( + False, False, True, 4, *MIXED_BATCH, id="text-with-position-cache" + ), + pytest.param(True, True, True, 4, [37], [37], [0], id="fresh"), + pytest.param(True, True, True, 4, [4097], [4097], [0], id="tiled"), + ], +) +def test_qsa_fused_pre_indexer_matches_unfused( + mrope, + is_2d_positions, + cache_rope_positions, + state_size, + seq_lens, + query_lens, + history_lens, +) -> None: + from flashinfer.norm import gemma_rmsnorm + + from vllm.model_executor.layers.rotary_embedding import get_rope + + device = "cuda" + rope_params = { + "partial_rotary_factor": 0.25, + "rope_theta": 10000000, + "rope_type": "default", + } + if mrope: + rope_params["mrope_interleaved"] = True + rope_params["mrope_section"] = list(MROPE_SECTION) + with torch.device(device): + rope = get_rope( + head_size=256, + max_position=32768, + rope_parameters=rope_params, + dtype=torch.bfloat16, + ) + + token_to_req = torch.cat( + [ + torch.full((length,), request, dtype=torch.int32) + for request, length in enumerate(query_lens) + ] + ).to(device) + logical_positions = torch.cat( + [ + torch.arange(seq_len - query_len, seq_len, dtype=torch.int64) + for seq_len, query_len in zip(seq_lens, query_lens) + ] + ).to(device) + num_tokens = logical_positions.numel() + query_start_loc = torch.tensor( + [0, *torch.tensor(query_lens).cumsum(0).tolist()], dtype=torch.int32 + ).to(device) + positions = ( + torch.stack( + [ + logical_positions, + logical_positions // 7 + 3, + logical_positions // 13 + 11, + ] + ) + if is_2d_positions + else logical_positions + ) + position_rows = ( + canonical_qsa_rope_positions(positions) + if cache_rope_positions + else logical_positions.view(-1, 1, 1).expand(-1, 1, 3) + ) + + compressed_block_counts = [ + (seq_len // CR + COMP_PAGE - 1) // COMP_PAGE for seq_len in seq_lens + ] + raw_block_table, num_raw_blocks = _make_block_table([1] * len(seq_lens)) + compressed_block_table, num_compressed_blocks = _make_block_table( + compressed_block_counts + ) + raw_block_table = raw_block_table.to(device) + raw_slots = circular_qsa_slot_mapping( + raw_block_table, + token_to_req, + logical_positions, + state_size, + query_start_loc, + ) + compressed_slots = compressed_qsa_slot_mapping( + compressed_block_table.to(device), + token_to_req, + logical_positions, + COMP_PAGE, + CR, + ) + group_counts = torch.tensor( + [ + seq_len // CR - (seq_len - query_len) // CR + for seq_len, query_len in zip(seq_lens, query_lens) + ], + dtype=torch.int32, + device=device, + ) + k_work_counts = torch.maximum(group_counts, torch.ones_like(group_counts)) + k_start_loc = torch.cat([k_work_counts.new_zeros(1), k_work_counts.cumsum(0)]) + work_requests = torch.repeat_interleave( + torch.arange(len(query_lens), dtype=torch.int32, device=device), + k_work_counts, + ) + local_work = torch.arange( + int(k_start_loc[-1]), dtype=torch.int32, device=device + ) - torch.repeat_interleave(k_start_loc[:-1], k_work_counts) + max_k_work = (num_tokens + (CR - 1) * len(query_lens)) // CR + k_work_metadata = torch.full((max_k_work, 2), -1, dtype=torch.int32, device=device) + k_work_metadata[: work_requests.numel()] = torch.stack( + (work_requests, local_work), dim=1 + ) + + raw_width = D + 12 if cache_rope_positions else D + # Match vLLM's padded-page cache layout: rows are contiguous, while physical + # blocks have a larger stride than their logical contents. + raw_page_elements = state_size * raw_width + fused_raw_storage = torch.zeros( + num_raw_blocks, + raw_page_elements + 16, + dtype=torch.bfloat16, + device=device, + ) + fused_raw = torch.as_strided( + fused_raw_storage, + (num_raw_blocks, state_size, 1, raw_width), + (raw_page_elements + 16, raw_width, raw_width, 1), + ) + for request, history_len in enumerate(history_lens): + history_end = seq_lens[request] - query_lens[request] + for position in range(history_end - history_len, history_end): + block = int(raw_block_table[request, 0]) + row = fused_raw[block, position % state_size, 0] + row[:D] = torch.randn(D, dtype=torch.bfloat16, device=device) + if cache_rope_positions: + row[ROPE_POS_OFFSET:].view(torch.int64).copy_( + torch.tensor( + [position, position // 7 + 3, position // 13 + 11], + dtype=torch.int64, + device=device, + ) + ) + unfused_raw = fused_raw.clone() + compressed_page_elements = COMP_PAGE * D + fused_compressed_storage = torch.zeros( + num_compressed_blocks, + compressed_page_elements + 16, + dtype=torch.bfloat16, + device=device, + ) + fused_compressed = torch.as_strided( + fused_compressed_storage, + (num_compressed_blocks, COMP_PAGE, 1, D), + (compressed_page_elements + 16, D, D, 1), + ) + unfused_compressed = fused_compressed.clone() + + projected_qk = torch.randn( + num_tokens, (HQ + 1) * D, dtype=torch.bfloat16, device=device + ) + q_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2 + k_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2 + + fused_query = torch.empty(num_tokens, HQ, D, dtype=torch.bfloat16, device=device) + qsa_pre_indexer( + projected_qk[:, : HQ * D], + projected_qk[:, HQ * D :], + positions, + rope.cos_sin_cache, + q_weight, + k_weight, + EPS, + fused_query, + fused_raw, + raw_slots, + raw_block_table, + query_start_loc, + logical_positions, + fused_compressed, + compressed_slots, + k_work_metadata, + compress_ratio=CR, + mrope_section=MROPE_SECTION if mrope else None, + rope_pos_offset=ROPE_POS_OFFSET if cache_rope_positions else None, + ) + + unfused_query = projected_qk[:, : HQ * D].reshape(num_tokens, HQ, D) + unfused_query = gemma_rmsnorm( + unfused_query.reshape(-1, D), q_weight, EPS + ).reshape_as(unfused_query) + unfused_query = apply_qsa_rope(rope, positions, unfused_query) + + raw_keys = unfused_raw[..., :D] + rope_positions = ( + unfused_raw[..., ROPE_POS_OFFSET:].view(torch.int64) + if cache_rope_positions + else None + ) + pooled, first_positions = qsa_compress_groups_with_ratio( + projected_qk[:, HQ * D :].reshape(-1, 1, D), + position_rows, + raw_keys, + raw_block_table, + token_to_req, + query_start_loc, + logical_positions, + compressed_slots, + CR, + rope_positions, + ) + compressed_rows = gemma_rmsnorm(pooled.reshape(-1, D), k_weight, EPS).reshape( + -1, 1, D + ) + group_positions = ( + first_positions.transpose(0, 1) if mrope else first_positions[:, 0] + ) + compressed_rows = apply_qsa_rope(rope, group_positions, compressed_rows) + qsa_store_cache_rows(unfused_compressed, compressed_slots, compressed_rows) + qsa_store_cache_rows(raw_keys, raw_slots, projected_qk[:, HQ * D :]) + if rope_positions is not None: + qsa_store_cache_rows(rope_positions, raw_slots, position_rows) + + torch.testing.assert_close(fused_query, unfused_query, rtol=RTOL, atol=ATOL) + assert torch.equal(fused_raw.view(torch.int16), unfused_raw.view(torch.int16)) + torch.testing.assert_close( + fused_compressed, unfused_compressed, rtol=RTOL, atol=ATOL + ) diff --git a/tests/models/qwen4_exp/test_qsa_reference.py b/tests/models/qwen4_exp/test_qsa_reference.py new file mode 100644 index 0000000000..276eb31b17 --- /dev/null +++ b/tests/models/qwen4_exp/test_qsa_reference.py @@ -0,0 +1,973 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import math +from types import SimpleNamespace + +import pytest +import torch + +from vllm.models.qwen4_exp.common import qsa_cache +from vllm.models.qwen4_exp.common.qsa_cache import QSAMetadataBuilder +from vllm.models.qwen4_exp.nvidia import indexer_qsa +from vllm.models.qwen4_exp.nvidia import ( + model as _qwen4_exp_model, # noqa: F401 +) +from vllm.models.qwen4_exp.nvidia.ops import qsa as qsa_ops +from vllm.platforms import current_platform +from vllm.triton_utils import HAS_TRITON + +requires_qsa_kernels = pytest.mark.skipif( + not current_platform.is_cuda() or not HAS_TRITON, + reason="QSA kernels require CUDA and Triton", +) + + +def test_qsa_mtp_index_share_updates_cache_but_skips_selection( + monkeypatch: pytest.MonkeyPatch, +) -> None: + rows = torch.tensor([[3, 1, -1], [5, 2, 0]], dtype=torch.int32) + raw_metadata = SimpleNamespace( + num_actual_tokens=2, + slot_mapping=torch.arange(2), + block_table=torch.empty(0), + query_start_loc=torch.arange(3), + logical_positions=torch.arange(2), + ) + compressed_metadata = SimpleNamespace( + num_actual_tokens=2, + slot_mapping=torch.arange(2), + k_work_metadata=torch.empty(0), + ) + updates = [] + selections = [] + indexer = SimpleNamespace( + skip_topk=True, + _metadata=lambda: (raw_metadata, compressed_metadata), + index_qk_proj=lambda hidden: (torch.zeros(2, 2), None), + index_n_heads=1, + index_kv_heads=1, + index_head_dim=1, + raw_key_cache=SimpleNamespace( + kv_cache=torch.empty(0), + rope_position_cache=None, + rope_position_offset=0, + ), + compressed_key_cache=SimpleNamespace(kv_cache=torch.empty(0)), + use_fused_pre_indexer=True, + rotary_emb=SimpleNamespace(cos_sin_cache=torch.empty(0)), + q_layernorm=SimpleNamespace(weight=torch.ones(1), variance_epsilon=1e-6), + k_layernorm=SimpleNamespace(weight=torch.ones(1)), + compress_ratio=2, + ) + + monkeypatch.setattr( + indexer_qsa, + "qsa_pre_indexer", + lambda *args, **kwargs: updates.append((args, kwargs)), + ) + monkeypatch.setattr( + qsa_ops, + "qsa_select_paged_tokens", + lambda *args, **kwargs: selections.append((args, kwargs)), + ) + + actual = indexer_qsa.QSAIndexer.forward( + indexer, + torch.zeros(2, 4), + torch.tensor([7, 8]), + rows, + ) + + assert actual is rows + assert len(updates) == 1 + assert not selections + + +def _qsa_mqa_paged_reference( + q: torch.Tensor, + k_cache: torch.Tensor, + page_table: torch.Tensor, + token_to_req: torch.Tensor, + visible_lengths: torch.Tensor, +) -> torch.Tensor: + pages = page_table.index_select(0, token_to_req.long()).long() + keys = k_cache[pages, :, 0, :].flatten(1, 2) + scores = torch.einsum("rhd,rnd->rnh", q.float(), keys.float()) + logits = torch.relu(scores).sum(dim=-1) / math.sqrt(q.shape[-1]) + positions = torch.arange(keys.shape[1], device=q.device).unsqueeze(0) + return logits.masked_fill(positions >= visible_lengths.unsqueeze(1), -torch.inf) + + +def _qsa_relative_topk_reference( + logits: torch.Tensor, + row_starts: torch.Tensor, + row_ends: torch.Tensor, + topk: int, +) -> torch.Tensor: + output = torch.full( + (logits.shape[0], topk), -1, dtype=torch.int32, device=logits.device + ) + for row in range(logits.shape[0]): + start = int(row_starts[row].item()) + length = int((row_ends[row] - row_starts[row]).item()) + width = min(length, topk) + if width: + output[row, :width] = torch.topk( + logits[row, start : start + length], width + ).indices.to(torch.int32) + return output + + +def _expand_qsa_indices_reference( + block_indices: torch.Tensor, + query_positions: torch.Tensor, + sequence_lengths: torch.Tensor, + compress_ratio: int, + token_topk: int, +) -> torch.Tensor: + rows = block_indices.shape[0] + block_topk = token_topk // compress_ratio + output_width = token_topk + compress_ratio - 1 + offsets = torch.arange(compress_ratio, device=block_indices.device) + blocks = block_indices.long() + expanded = blocks.unsqueeze(-1) * compress_ratio + offsets + expanded = torch.where( + blocks.unsqueeze(-1) >= 0, expanded, torch.full_like(expanded, -1) + ).reshape(rows, block_topk * compress_ratio) + expanded = expanded[:, :token_topk] + expanded = torch.where( + (expanded >= 0) & (expanded < sequence_lengths.unsqueeze(1)), + expanded, + torch.full_like(expanded, -1), + ) + + tail_offsets = torch.arange(compress_ratio - 1, device=block_indices.device) + visible_tokens = query_positions + 1 + tail_start = visible_tokens // compress_ratio * compress_ratio + tail = tail_start.unsqueeze(1) + tail_offsets.unsqueeze(0) + tail_count = (visible_tokens - tail_start).unsqueeze(1) + tail_valid = (tail_offsets.unsqueeze(0) < tail_count) & ( + tail < sequence_lengths.unsqueeze(1) + ) + tail = torch.where(tail_valid, tail, torch.full_like(tail, -1)) + + result = torch.cat((expanded, tail), dim=1) + order = torch.arange(output_width, device=result.device).expand(rows, -1) + sort_key = torch.where(result >= 0, order, order + output_width) + return result.gather(1, torch.argsort(sort_key, dim=1, stable=True)).to(torch.int32) + + +def _qsa_select_paged_tokens_reference( + q: torch.Tensor, + k_cache: torch.Tensor, + page_table: torch.Tensor, + token_to_req: torch.Tensor, + query_positions: torch.Tensor, + sequence_lengths: torch.Tensor, + token_topk: int, + compress_ratio: int, +) -> torch.Tensor: + row_sequence_lengths = sequence_lengths.index_select(0, token_to_req.long()) + visible_blocks = torch.minimum( + (query_positions + 1) // compress_ratio, + row_sequence_lengths // compress_ratio, + ).to(torch.int32) + logits = _qsa_mqa_paged_reference( + q, + k_cache, + page_table, + token_to_req, + visible_blocks, + ) + starts = torch.zeros_like(visible_blocks) + blocks = _qsa_relative_topk_reference( + logits, + starts, + visible_blocks, + token_topk // compress_ratio, + ) + return _expand_qsa_indices_reference( + blocks, + query_positions, + row_sequence_lengths, + compress_ratio, + token_topk, + ) + + +def _qsa_sparse_paged_attention_reference( + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + logical_indices: torch.Tensor, + block_table: torch.Tensor, + token_to_req: torch.Tensor, + softmax_scale: float, +) -> torch.Tensor: + output = torch.zeros_like(q) + repeats = q.shape[1] // k_cache.shape[2] + page_size = k_cache.shape[1] + for row in range(q.shape[0]): + logical = logical_indices[row] + logical = logical[logical >= 0].long() + if not logical.numel(): + continue + request = token_to_req[row].long() + pages = block_table[request, logical // page_size].long() + offsets = logical % page_size + keys = k_cache[pages, offsets].repeat_interleave(repeats, dim=1) + values = v_cache[pages, offsets].repeat_interleave(repeats, dim=1) + scores = torch.einsum("hd,khd->hk", q[row].float(), keys.float()) + probabilities = torch.softmax(scores * softmax_scale, dim=-1) + output[row] = torch.einsum("hk,khd->hd", probabilities, values.float()).to( + q.dtype + ) + return output + + +@requires_qsa_kernels +def test_qsa_side_metadata_marks_cudagraph_padding_inert() -> None: + device = torch.device("cuda") + builder = QSAMetadataBuilder.__new__(QSAMetadataBuilder) + builder.compress_ratio = 1 + builder.is_circular_buffer = False + builder.storage_block_size = 64 + builder.token_to_req_buffer = torch.empty(16, dtype=torch.int32, device=device) + builder.slot_mapping_buffer = torch.empty(16, dtype=torch.int64, device=device) + builder.logical_positions_buffer = torch.empty(16, dtype=torch.int64, device=device) + builder.k_work_metadata_buffer = torch.empty(0, 2, dtype=torch.int32, device=device) + query_start_loc = torch.tensor([0, 4, 8, 12, 12], dtype=torch.int32, device=device) + token_to_req = torch.tensor([0] * 4 + [1] * 4 + [2] * 4 + [0] * 4, device=device) + common = SimpleNamespace( + num_actual_tokens=16, + query_start_loc=query_start_loc, + query_start_loc_cpu=query_start_loc.cpu(), + seq_lens=torch.tensor([68, 68, 68, 0], dtype=torch.int32, device=device), + slot_mapping=torch.tensor(list(range(12)) + [-1] * 4, device=device), + block_table_tensor=torch.empty((4, 0), dtype=torch.int32, device=device), + token_to_req_indices=lambda buffer: buffer.copy_(token_to_req), + ) + + metadata = builder.build(0, common) + + assert metadata.logical_positions.tolist() == [ + 64, + 65, + 66, + 67, + 64, + 65, + 66, + 67, + 64, + 65, + 66, + 67, + -1, + -1, + -1, + -1, + ] + assert metadata.slot_mapping.tolist() == list(range(12)) + [-1] * 4 + + +@requires_qsa_kernels +def test_qsa_circular_buffer_metadata_keeps_only_each_requests_suffix() -> None: + device = torch.device("cuda") + builder = QSAMetadataBuilder.__new__(QSAMetadataBuilder) + builder.compress_ratio = 4 + builder.is_circular_buffer = True + builder.kv_cache_spec = SimpleNamespace(block_size=4) + builder.storage_block_size = 4 + builder.token_to_req_buffer = torch.empty(16, dtype=torch.int32, device=device) + builder.slot_mapping_buffer = torch.empty(16, dtype=torch.int64, device=device) + builder.logical_positions_buffer = torch.empty(16, dtype=torch.int64, device=device) + builder.k_work_metadata_buffer = torch.empty(0, 2, dtype=torch.int32, device=device) + query_start_loc = torch.tensor([0, 7, 13, 13], dtype=torch.int32, device=device) + token_to_req = torch.tensor([0] * 7 + [1] * 6 + [0] * 3, device=device) + block_table = torch.tensor([[1], [0], [2]], dtype=torch.int32, device=device) + common = SimpleNamespace( + num_actual_tokens=16, + query_start_loc=query_start_loc, + query_start_loc_cpu=query_start_loc.cpu(), + seq_lens=torch.tensor([9, 11, 0], dtype=torch.int32, device=device), + slot_mapping=torch.full((16,), -1, dtype=torch.int64, device=device), + block_table_tensor=block_table, + token_to_req_indices=lambda buffer: buffer.copy_(token_to_req), + ) + + metadata = builder.build(0, common) + expected = [ + -1, + -1, + -1, + 5, + 6, + 7, + 4, + -1, + -1, + 3, + 0, + 1, + 2, + -1, + -1, + -1, + ] + + assert metadata.slot_mapping.tolist() == expected + + +@pytest.mark.parametrize("chunk_start", list(range(8))) +def test_qsa_circular_buffer_survives_one_speculative_step(chunk_start: int) -> None: + """A speculative step must not overwrite the open group's committed keys. + + The step stores every row it computes, drafts included, before acceptance + is known, while the next step still reads the earlier members of the group + being compressed from the ring. A ring sized at the compression ratio makes + those rows alias, so a rejected draft silently replaces a committed key. + """ + compress_ratio = 4 + num_spec = 3 + capacity = compress_ratio * -(-(compress_ratio + num_spec) // compress_ratio) + query_len = num_spec + 1 + + slots = qsa_cache.circular_qsa_slot_mapping( + torch.tensor([[0]], dtype=torch.int32), + torch.zeros(query_len, dtype=torch.int32), + torch.arange(chunk_start, chunk_start + query_len), + capacity, + query_start_loc=torch.tensor([0, query_len], dtype=torch.int32), + ) + + committed = torch.arange(chunk_start - chunk_start % compress_ratio, chunk_start) + assert set(slots.tolist()).isdisjoint((committed % capacity).tolist()) + + +def _qsa_key_cache(block_size: int, compress_ratio: int) -> qsa_cache.QSAKeyStateCache: + return qsa_cache.QSAKeyStateCache( + head_size=64, + dtype=torch.bfloat16, + cache_config=SimpleNamespace(block_size=block_size), + prefix=f"raw.{block_size}.{compress_ratio}", + vllm_config=SimpleNamespace( + compilation_config=SimpleNamespace(static_forward_context={}) + ), + compress_ratio=compress_ratio, + ) + + +def test_qsa_state_caches_adapt_the_unified_logical_layout() -> None: + raw_cache = _qsa_key_cache(block_size=32, compress_ratio=4) + compressed_cache = qsa_cache.QSACompressedKeyCache( + head_size=64, + dtype=torch.bfloat16, + cache_config=SimpleNamespace(block_size=32), + prefix="compressed.bind", + vllm_config=SimpleNamespace( + compilation_config=SimpleNamespace(static_forward_context={}) + ), + compress_ratio=4, + ) + raw_view = torch.empty(2, 1, 8, 64, dtype=torch.bfloat16) + compressed_view = torch.empty(2, 1, 8, 64, dtype=torch.bfloat16) + + raw_cache.bind_kv_cache(raw_view) + compressed_cache.bind_kv_cache(compressed_view) + + assert raw_cache.kv_cache.shape == (2, 8, 1, 64) + assert compressed_cache.kv_cache.shape == (2, 8, 1, 64) + assert raw_cache.kv_cache.data_ptr() == raw_view.data_ptr() + assert compressed_cache.kv_cache.data_ptr() == compressed_view.data_ptr() + + +@pytest.mark.parametrize( + ("compress_ratio", "num_spec", "expected"), + [(4, 0, 4), (4, 1, 8), (4, 3, 8), (4, 4, 8), (4, 5, 12), (2, 3, 6)], +) +def test_qsa_ring_capacity_covers_one_speculative_step( + compress_ratio: int, num_spec: int, expected: int +) -> None: + """Capacity spans the open group plus one speculative step, in whole groups.""" + spec = _qsa_key_cache( + block_size=48, compress_ratio=compress_ratio + ).get_kv_cache_spec(SimpleNamespace(num_speculative_tokens=num_spec)) + assert spec.block_size == expected + + +@requires_qsa_kernels +def test_qsa_compressed_metadata_keeps_dummy_slots_inert() -> None: + device = torch.device("cuda") + builder = QSAMetadataBuilder.__new__(QSAMetadataBuilder) + builder.compress_ratio = 4 + builder.is_circular_buffer = False + builder.storage_block_size = 16 + builder.token_to_req_buffer = torch.empty(8, dtype=torch.int32, device=device) + builder.slot_mapping_buffer = torch.empty(8, dtype=torch.int64, device=device) + builder.logical_positions_buffer = torch.empty(8, dtype=torch.int64, device=device) + # Simulate max_num_seqs exceeding the three live requests below. + builder.request_capacity = 8 + builder.k_work_metadata_buffer = torch.empty(4, 2, dtype=torch.int32, device=device) + query_start_loc = torch.tensor([0, 3, 3, 8], dtype=torch.int32, device=device) + token_to_req = torch.tensor( + [0, 0, 0, 2, 2, 2, 2, 2], dtype=torch.int32, device=device + ) + common = SimpleNamespace( + num_actual_tokens=8, + query_start_loc=query_start_loc, + query_start_loc_cpu=query_start_loc.cpu(), + seq_lens=torch.tensor([7, 0, 12], dtype=torch.int32, device=device), + slot_mapping=torch.full((8,), -1, dtype=torch.int64, device=device), + block_table_tensor=torch.zeros((3, 1), dtype=torch.int32, device=device), + token_to_req_indices=lambda buffer: buffer.copy_(token_to_req), + ) + + metadata = builder.build(0, common) + + assert metadata.slot_mapping.tolist() == [-1] * 8 + assert metadata.k_work_metadata.tolist() == [[0, 0], [2, 0], [2, 1], [-1, -1]] + + +@requires_qsa_kernels +@pytest.mark.parametrize("compress_ratio", [1, 4]) +@pytest.mark.parametrize("num_reqs", [2, 3, 4, 7, 8, 9]) +def test_qsa_triton_metadata_matches_pytorch( + compress_ratio: int, num_reqs: int +) -> None: + device = torch.device("cuda") + num_tokens = 8 + query_start_loc = torch.tensor( + [0, 3, *([3] * (num_reqs - 2)), 8], dtype=torch.int32, device=device + ) + token_to_req = torch.tensor( + [0, 0, 0, *([num_reqs - 1] * 5)], + dtype=torch.int32, + device=device, + ) + block_table_rows = torch.tensor( + [ + [4, -1, 8, -1, 12, -1], + [1, -1, 2, -1, 3, -1], + [7, -1, 9, -1, 11, -1], + ], + dtype=torch.int32, + device=device, + ) + block_table_storage = block_table_rows[ + torch.arange(num_reqs, device=device) % block_table_rows.shape[0] + ] + seq_lens = torch.zeros(num_reqs, dtype=torch.int32, device=device) + seq_lens[0] = 10 + seq_lens[-1] = 20 + common = SimpleNamespace( + num_actual_tokens=num_tokens, + query_start_loc=query_start_loc, + query_start_loc_cpu=query_start_loc.cpu(), + seq_lens=seq_lens, + slot_mapping=torch.tensor( + [0, 1, -1, 3, 4, -1, -1, -1], dtype=torch.int64, device=device + ), + block_table_tensor=block_table_storage[:, ::2], + token_to_req_indices=lambda buffer: buffer.copy_(token_to_req), + ) + + def make_buffers() -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + return ( + torch.empty(num_tokens, dtype=torch.int32, device=device), + torch.empty(num_tokens, dtype=torch.int64, device=device), + torch.empty(num_tokens, dtype=torch.int64, device=device), + ) + + max_num_work = ( + (num_tokens + (compress_ratio - 1) * num_reqs) // compress_ratio + if compress_ratio != 1 + else 0 + ) + actual_k_work = ( + torch.empty(max_num_work, 2, dtype=torch.int32, device=device) + if max_num_work + else None + ) + actual_buffers = make_buffers() + actual = qsa_cache.build_qsa_metadata_triton( + common, + *actual_buffers, + storage_block_size=2, + compress_ratio=compress_ratio, + k_work_metadata_buffer=actual_k_work, + request_capacity=num_reqs, + ) + + expected_k_work = ( + torch.empty_like(actual_k_work) if actual_k_work is not None else None + ) + expected_buffers = make_buffers() + expected = qsa_cache._build_qsa_metadata_torch( + common, + *expected_buffers, + storage_block_size=2, + compress_ratio=compress_ratio, + k_work_metadata_buffer=expected_k_work, + request_capacity=num_reqs, + ) + + for actual_tensor, expected_tensor in zip(actual, expected): + torch.testing.assert_close(actual_tensor, expected_tensor) + if actual_k_work is not None: + torch.testing.assert_close(actual_k_work, expected_k_work) + + +@requires_qsa_kernels +def test_qsa_fused_metadata_matches_pytorch_for_large_padded_prefill() -> None: + device = torch.device("cuda") + num_mapped_tokens = 4096 + num_tokens = 4224 + query_start_loc = torch.tensor( + [0, num_mapped_tokens], dtype=torch.int32, device=device + ) + common = SimpleNamespace( + num_actual_tokens=num_tokens, + query_start_loc=query_start_loc, + query_start_loc_cpu=query_start_loc.cpu(), + seq_lens=torch.tensor( + [num_mapped_tokens + 32], dtype=torch.int32, device=device + ), + block_table_tensor=torch.arange(256, dtype=torch.int32, device=device)[None], + slot_mapping=torch.tensor( + [0] * num_mapped_tokens + [-1] * (num_tokens - num_mapped_tokens), + dtype=torch.int64, + device=device, + ), + token_to_req_indices=lambda buffer: buffer.zero_(), + ) + + def make_buffers() -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + return ( + torch.empty(num_tokens, dtype=torch.int32, device=device), + torch.empty(num_tokens, dtype=torch.int64, device=device), + torch.empty(num_tokens, dtype=torch.int64, device=device), + ) + + max_num_work = (num_tokens + 3) // 4 + actual_k_work = torch.empty(max_num_work, 2, dtype=torch.int32, device=device) + expected_k_work = torch.empty_like(actual_k_work) + actual = qsa_cache.build_qsa_metadata_triton( + common, + *make_buffers(), + storage_block_size=8, + compress_ratio=4, + k_work_metadata_buffer=actual_k_work, + ) + expected = qsa_cache._build_qsa_metadata_torch( + common, + *make_buffers(), + storage_block_size=8, + compress_ratio=4, + k_work_metadata_buffer=expected_k_work, + ) + + for actual_tensor, expected_tensor in zip(actual, expected): + torch.testing.assert_close(actual_tensor, expected_tensor) + torch.testing.assert_close(actual_k_work, expected_k_work) + + +@requires_qsa_kernels +@pytest.mark.parametrize( + "num_rows", + [ + pytest.param(3, id="one_tile_per_program"), + pytest.param(33, id="looped_tiles"), + ], +) +def test_qsa_mqa_paged_matches_test_reference(num_rows: int) -> None: + torch.manual_seed(1) + head_dim = 128 + q = torch.randn(num_rows, 4, head_dim, device="cuda", dtype=torch.bfloat16) + cache = torch.randn(40, 4, 1, head_dim, device="cuda", dtype=torch.bfloat16) + page_table = torch.randperm(40, device="cuda", dtype=torch.int32).reshape(2, 20) + request_rows = [(num_rows + 1) // 2, num_rows // 2] + token_to_req = torch.repeat_interleave( + torch.arange(2, device="cuda", dtype=torch.int32), + torch.tensor(request_rows, device="cuda"), + ) + sequence_length_values = [320, 264] + sequence_lengths = torch.tensor( + sequence_length_values, device="cuda", dtype=torch.int32 + ) + query_positions = torch.cat( + [ + torch.arange(length - rows, length, device="cuda", dtype=torch.int32) + for rows, length in zip(request_rows, sequence_length_values, strict=True) + ] + ) + compress_ratio = 4 + visible_lengths = (query_positions + 1) // compress_ratio + + actual, actual_visible_blocks = qsa_ops.qsa_mqa_paged( + q, + cache, + page_table, + token_to_req, + query_positions, + sequence_lengths, + compress_ratio=compress_ratio, + ) + expected = _qsa_mqa_paged_reference( + q, cache, page_table, token_to_req, visible_lengths + ) + + torch.testing.assert_close(actual_visible_blocks, visible_lengths) + # Top-k is bounded by visible_blocks; later columns are intentionally unwritten. + columns = torch.arange(actual.shape[1], device=actual.device) + visible = columns[None, :] < actual_visible_blocks[:, None] + torch.testing.assert_close(actual[visible], expected[visible], rtol=1e-3, atol=1e-3) + + +@requires_qsa_kernels +def test_qsa_block_expansion_matches_test_reference() -> None: + blocks = torch.tensor([[0, -1], [1, 0]], device="cuda", dtype=torch.int32) + query_positions = torch.tensor([5, 10], device="cuda") + sequence_lengths = torch.tensor([6, 11], device="cuda") + token_to_req = torch.tensor([0, 1], device="cuda", dtype=torch.int32) + + actual = qsa_ops.expand_qsa_block_indices_cuda( + blocks, + query_positions, + sequence_lengths, + token_to_req, + compress_ratio=4, + token_topk=8, + ) + expected = _expand_qsa_indices_reference( + blocks, + query_positions, + sequence_lengths, + compress_ratio=4, + token_topk=8, + ) + + torch.testing.assert_close(actual, expected) + + +@requires_qsa_kernels +@pytest.mark.parametrize( + ("num_rows", "num_query_heads", "num_kv_heads", "page_size"), + [ + # Kernel-visible pages with --block-size 256 and hybrid-cache alignment. + pytest.param(1, 24, 2, 1792, id="tp1_split64"), + pytest.param(16, 12, 1, 1792, id="tp2_split32"), + pytest.param(32, 6, 1, 1024, id="tp4_split8"), + pytest.param(257, 6, 1, 1024, id="tp4_split4"), + pytest.param(513, 6, 1, 1024, id="tp4_split1"), + ], +) +def test_qsa_sparse_paged_attention_matches_test_reference( + num_rows: int, + num_query_heads: int, + num_kv_heads: int, + page_size: int, +) -> None: + torch.manual_seed(2) + head_dim = 256 + num_requests = 2 + num_selected_pages = 64 + # Keep the newest page outside the synthetic top-k as causal headroom. + num_pages_per_request = num_selected_pages + 1 + num_cache_blocks = num_requests * num_pages_per_request + indexer_budget = 2048 + indexer_compress_ratio = 4 + selection_width = indexer_budget + indexer_compress_ratio - 1 + q = torch.randn( + num_rows, num_query_heads, head_dim, device="cuda", dtype=torch.bfloat16 + ) + kv_cache = torch.randn( + num_cache_blocks, + page_size, + num_kv_heads, + 2 * head_dim, + device="cuda", + dtype=torch.bfloat16, + ) + k_cache, v_cache = kv_cache.split(head_dim, dim=-1) + block_table = ( + torch.randperm(num_cache_blocks, device="cuda") + .reshape(num_requests, num_pages_per_request) + .to(torch.int32) + ) + rows_per_request = math.ceil(num_rows / num_requests) + row_indices = torch.arange(num_rows, device="cuda", dtype=torch.int32) + token_to_req = row_indices // rows_per_request + request_row_counts = torch.tensor( + [rows_per_request, num_rows - rows_per_request], + device="cuda", + dtype=torch.int32, + ) + + context_length = num_pages_per_request * page_size - 1 + block_topk = indexer_budget // indexer_compress_ratio + compressed_blocks_per_page = page_size // indexer_compress_ratio + selection = torch.arange(block_topk, device="cuda") + selected_pages = selection % num_selected_pages + selected_offsets = selection // num_selected_pages + row_shifts = 2 * row_indices.unsqueeze(1) + # Eight blocks per page; adjacent rows overlap by six of those eight. + selected_offsets = (selected_offsets + row_shifts) % compressed_blocks_per_page + block_indices = (selected_pages * compressed_blocks_per_page + selected_offsets).to( + torch.int32 + ) + rows_within_request = row_indices % rows_per_request + query_positions = ( + context_length - request_row_counts[token_to_req.long()] + rows_within_request + ).to(torch.int64) + sequence_lengths = torch.full( + (num_requests,), context_length, device="cuda", dtype=torch.int32 + ) + logical_indices = qsa_ops.expand_qsa_block_indices_cuda( + block_indices, + query_positions, + sequence_lengths, + token_to_req, + indexer_compress_ratio, + indexer_budget, + ) + assert logical_indices.shape == (num_rows, selection_width) + scale = q.shape[-1] ** -0.5 + + actual = qsa_ops.qsa_sparse_paged_attention( + q, + k_cache, + v_cache, + logical_indices, + block_table, + token_to_req, + ) + expected = _qsa_sparse_paged_attention_reference( + q, + k_cache, + v_cache, + logical_indices, + block_table, + token_to_req, + scale, + ) + + torch.testing.assert_close(actual, expected, rtol=2e-2, atol=2e-2) + + +@requires_qsa_kernels +def test_qsa_selection_chunks_workspace_and_matches_test_reference( + monkeypatch: pytest.MonkeyPatch, + workspace_init, +) -> None: + rows, keys, heads, head_dim = 65, 640, 4, 16 + token_topk, compress_ratio = 2048, 4 + torch.manual_seed(3) + q = torch.randn(rows, heads, head_dim, device="cuda", dtype=torch.bfloat16) + cache = torch.randn(40, 16, 1, head_dim, device="cuda", dtype=torch.bfloat16) + page_table = torch.randperm(40, device="cuda", dtype=torch.int32).unsqueeze(0) + token_to_req = torch.zeros(rows, device="cuda", dtype=torch.int32) + query_positions = torch.full((rows,), 2559, device="cuda", dtype=torch.int32) + sequence_lengths = torch.tensor([2560], device="cuda", dtype=torch.int32) + monkeypatch.setattr(qsa_ops, "_LOGITS_WORKSPACE_BYTES", 32 * keys * 4) + original_score = qsa_ops.qsa_mqa_paged + scored_row_counts = [] + + def record_score(query: torch.Tensor, *args, **kwargs): + scored_row_counts.append(query.shape[0]) + return original_score(query, *args, **kwargs) + + monkeypatch.setattr(qsa_ops, "qsa_mqa_paged", record_score) + + actual = qsa_ops.qsa_select_paged_tokens( + q, + cache, + page_table, + token_to_req, + query_positions, + sequence_lengths, + token_topk, + compress_ratio, + ) + expected = _qsa_select_paged_tokens_reference( + q, + cache, + page_table, + token_to_req, + query_positions, + sequence_lengths, + token_topk, + compress_ratio, + ) + + torch.testing.assert_close(actual.sort().values, expected.sort().values) + assert scored_row_counts == [32, 32, 1] + + +@requires_qsa_kernels +def test_qsa_selection_handles_no_complete_compressed_blocks(workspace_init) -> None: + q = torch.zeros(2, 4, 8, device="cuda", dtype=torch.bfloat16) + cache = torch.zeros(1, 16, 1, 8, device="cuda", dtype=torch.bfloat16) + page_table = torch.zeros(1, 1, device="cuda", dtype=torch.int32) + token_to_req = torch.zeros(2, device="cuda", dtype=torch.int32) + query_positions = torch.tensor([1, 2], device="cuda", dtype=torch.int32) + sequence_lengths = torch.tensor([3], device="cuda", dtype=torch.int32) + + selected = qsa_ops.qsa_select_paged_tokens( + q, + cache, + page_table, + token_to_req, + query_positions, + sequence_lengths, + token_topk=2048, + compress_ratio=4, + ) + + assert selected[0, :2].tolist() == [0, 1] + assert selected[1, :3].tolist() == [0, 1, 2] + assert torch.all(selected[0, 2:] == -1) + assert torch.all(selected[1, 3:] == -1) + + +@requires_qsa_kernels +def test_qsa_streaming_compression_and_compressor_state_store_match_reference() -> None: + head_dim = 8 + current_pairs = [ + *((0, position) for position in range(2, 9)), + *((1, position) for position in range(5, 11)), + ] + + def key_row(request: int, position: int) -> torch.Tensor: + return ( + torch.arange(head_dim, dtype=torch.float32) + request * 1000 + position * 10 + ) + + def position_row(request: int, position: int) -> torch.Tensor: + return torch.tensor( + [ + request * 1000 + position, + request * 1000 + position + 100, + request * 1000 + position + 200, + ], + dtype=torch.int64, + ) + + raw_keys = ( + torch.stack([key_row(request, position) for request, position in current_pairs]) + .unsqueeze(1) + .to(device="cuda", dtype=torch.bfloat16) + ) + raw_positions = ( + torch.stack( + [position_row(request, position) for request, position in current_pairs] + ) + .unsqueeze(1) + .to(device="cuda") + ) + token_to_req = torch.tensor( + [request for request, _ in current_pairs], + dtype=torch.int32, + device="cuda", + ) + logical_positions = torch.tensor( + [position for _, position in current_pairs], + dtype=torch.int64, + device="cuda", + ) + query_start_loc = torch.tensor([0, 7, 13], dtype=torch.int32, device="cuda") + compressor_state_block_table = torch.tensor( + [[1], [0]], dtype=torch.int32, device="cuda" + ) + compressor_state_cache = torch.zeros( + 2, 4, 1, head_dim, dtype=torch.bfloat16, device="cuda" + ) + rope_cache = torch.zeros(2, 4, 1, 3, dtype=torch.int64, device="cuda") + for request, position, block in ((0, 0, 1), (0, 1, 1), (1, 4, 0)): + compressor_state_cache[block, position % 4, 0] = key_row(request, position).to( + device="cuda", dtype=torch.bfloat16 + ) + rope_cache[block, position % 4, 0] = position_row(request, position).to("cuda") + + compressed_slots = torch.full( + (len(current_pairs),), -1, dtype=torch.int64, device="cuda" + ) + valid_rows = torch.tensor([1, 5, 9], dtype=torch.int64, device="cuda") + compressed_slots[valid_rows] = torch.arange(3, device="cuda") + pooled, first_positions = qsa_ops.qsa_compress_groups_with_ratio( + raw_keys, + raw_positions, + compressor_state_cache, + compressor_state_block_table, + token_to_req, + query_start_loc, + logical_positions, + compressed_slots, + compress_ratio=4, + rope_cache=rope_cache, + ) + pooled_without_rope, scalar_first_positions = ( + qsa_ops.qsa_compress_groups_with_ratio( + raw_keys, + raw_positions, + compressor_state_cache, + compressor_state_block_table, + token_to_req, + query_start_loc, + logical_positions, + compressed_slots, + compress_ratio=4, + ) + ) + + groups = [ + [(0, position) for position in range(0, 4)], + [(0, position) for position in range(4, 8)], + [(1, position) for position in range(4, 8)], + ] + expected_pooled = ( + torch.stack( + [ + torch.stack([key_row(*pair) for pair in group]).mean(dim=0) + for group in groups + ] + ) + .unsqueeze(1) + .to(device="cuda", dtype=torch.bfloat16) + ) + expected_positions = torch.stack( + [position_row(0, 0), position_row(0, 4), position_row(1, 4)] + ).to("cuda") + expected_scalar_positions = torch.tensor( + [[0, 0, 0], [4, 4, 4], [4, 4, 4]], + dtype=torch.int64, + device="cuda", + ) + + torch.testing.assert_close(pooled[valid_rows], expected_pooled) + torch.testing.assert_close(pooled_without_rope[valid_rows], expected_pooled) + torch.testing.assert_close(first_positions[valid_rows], expected_positions) + torch.testing.assert_close( + scalar_first_positions[valid_rows], expected_scalar_positions + ) + + compressor_state_slots = torch.tensor( + [-1, -1, -1, 5, 6, 7, 4, -1, -1, 3, 0, 1, 2], + dtype=torch.int64, + device="cuda", + ) + qsa_ops.qsa_store_cache_rows( + compressor_state_cache, compressor_state_slots, raw_keys + ) + qsa_ops.qsa_store_cache_rows(rope_cache, compressor_state_slots, raw_positions) + for request, positions, block in ((0, range(5, 9), 1), (1, range(7, 11), 0)): + for position in positions: + torch.testing.assert_close( + compressor_state_cache[block, position % 4, 0], + key_row(request, position).to(device="cuda", dtype=torch.bfloat16), + ) + torch.testing.assert_close( + rope_cache[block, position % 4, 0], + position_row(request, position).to("cuda"), + ) diff --git a/tests/models/registry.py b/tests/models/registry.py index 033ea454b6..ba16453511 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -508,6 +508,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = { extras={"native-prefix": "imdatta0/small_qwen3_5_20b"}, max_model_len=4096, ), + "Qwen4ExpForCausalLM": _HfExamplesInfo("", is_available_online=False), "MellumForCausalLM": _HfExamplesInfo("JetBrains/Mellum2-12B-A2.5B-Base"), "Qwen3NextForCausalLM": _HfExamplesInfo( "Qwen/Qwen3-Next-80B-A3B-Instruct", @@ -1348,6 +1349,10 @@ _MULTIMODAL_EXAMPLE_MODELS = { "Qwen/Qwen3.5-35B-A3B", max_model_len=4096, ), + "Qwen4ExpForConditionalGeneration": _HfExamplesInfo( + "", + is_available_online=False, + ), "Qwen3OmniMoeForConditionalGeneration": _HfExamplesInfo( "Qwen/Qwen3-Omni-30B-A3B-Instruct", max_model_len=4096, @@ -1797,6 +1802,7 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = { "Qwen/Qwen3.5-35B-A3B", speculative_model="Qwen/Qwen3.5-35B-A3B", ), + "Qwen4ExpMTP": _HfExamplesInfo("", is_available_online=False), "Step3p5MTP": _HfExamplesInfo( "stepfun-ai/Step-3.5-Flash", speculative_model="stepfun-ai/Step-3.5-Flash", diff --git a/tests/quantization/test_fp8.py b/tests/quantization/test_fp8.py index 1e96b630fa..0a89d2a035 100644 --- a/tests/quantization/test_fp8.py +++ b/tests/quantization/test_fp8.py @@ -6,6 +6,7 @@ Run `pytest tests/quantization/test_fp8.py --forked`. """ import logging +from types import SimpleNamespace import pytest import regex as re @@ -31,6 +32,10 @@ from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod from vllm.model_executor.layers.quantization.online.fp8 import ( Fp8PerTensorOnlineLinearMethod, ) +from vllm.model_executor.layers.quantization.utils import flashinfer_utils +from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( + prepare_fp8_moe_layer_for_fi, +) from vllm.model_executor.layers.quantization.utils.fp8_utils import ( process_fp8_input_tensor_strategy_moe, ) @@ -50,6 +55,45 @@ MODELS = [ ] +def test_prepare_gated_trtllm_fp8_moe_weights_pads_each_projection(monkeypatch): + monkeypatch.setattr( + flashinfer_utils, + "rotate_weights_for_fi_trtllm_fp8_per_tensor_moe", + lambda *args: None, + ) + intermediate = 17 + padded_intermediate = 32 + hidden_size = 4 + gate = torch.ones((1, intermediate, hidden_size), dtype=torch.float8_e4m3fn) + up = torch.full_like(gate, 2) + w13 = torch.cat((gate, up), dim=1) + w2 = torch.ones((1, hidden_size, intermediate), dtype=torch.float8_e4m3fn) + layer = SimpleNamespace( + activation=SimpleNamespace(is_gated=True), + moe_config=SimpleNamespace( + is_act_and_mul=True, + intermediate_size_per_partition=intermediate, + ), + ) + + padded_w31, _, _, _ = prepare_fp8_moe_layer_for_fi( + layer, + w13, + w2, + w13_scale=torch.ones(1), + w13_input_scale=torch.ones(1), + w2_scale=torch.ones(1), + w2_input_scale=torch.ones(1), + is_trtllm=True, + ) + + expected = w13.new_zeros((1, 2 * padded_intermediate, hidden_size)) + expected[:, :intermediate] = up + expected[:, padded_intermediate : padded_intermediate + intermediate] = gate + assert layer.moe_config.intermediate_size_per_partition == padded_intermediate + assert torch.equal(padded_w31, expected) + + def test_static_fp8_moe_input_scales_remain_scalar() -> None: a1_scale, a2_scale = process_fp8_input_tensor_strategy_moe( torch.tensor([0.25, 0.5]), diff --git a/tests/quantization/test_gfx950_moe.py b/tests/quantization/test_gfx950_moe.py index a4dac1c163..67728a2da8 100644 --- a/tests/quantization/test_gfx950_moe.py +++ b/tests/quantization/test_gfx950_moe.py @@ -16,6 +16,7 @@ from vllm.model_executor.layers.fused_moe.config import ( ) from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import ( Mxfp4MoeBackend, + _requires_qwen38_tep8_emulation, select_mxfp4_moe_backend, ) from vllm.model_executor.layers.quantization.utils.quant_utils import ( @@ -72,6 +73,91 @@ def _make_w4a4_moe_config(moe_backend: str = "auto") -> FusedMoEConfig: ) +def _make_qwen38_tep8_moe_config() -> FusedMoEConfig: + from vllm.model_executor.layers.fused_moe.activation import MoEActivation + + return FusedMoEConfig( + num_experts=512, + experts_per_token=9, + hidden_dim=2560, + intermediate_size=640, + num_local_experts=64, + num_logical_experts=512, + moe_parallel_config=FusedMoEParallelConfig( + tp_size=1, + tp_rank=0, + pcp_size=1, + pcp_rank=0, + dp_size=1, + dp_rank=0, + ep_size=8, + ep_rank=0, + sp_size=1, + use_ep=True, + all2all_backend="allgather_reducescatter", + enable_eplb=False, + ), + activation=MoEActivation.SILU, + in_dtype=torch.bfloat16, + device="cuda", + routing_method=RoutingMethodType.Renormalize, + ) + + +def test_qwen38_tep8_requires_emulation_only_on_gfx950(monkeypatch): + import vllm.model_executor.layers.fused_moe.oracle.mxfp4 as mxfp4_oracle + + config = _make_qwen38_tep8_moe_config() + monkeypatch.setattr(current_platform, "is_rocm", lambda: True) + monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: True) + + assert _requires_qwen38_tep8_emulation(config, kMxfp4Dynamic) + + config.moe_parallel_config.ep_size = 4 + config.num_local_experts = 128 + assert not _requires_qwen38_tep8_emulation(config, kMxfp4Dynamic) + + config = _make_qwen38_tep8_moe_config() + monkeypatch.setattr(mxfp4_oracle.current_platform, "is_rocm", lambda: False) + assert not _requires_qwen38_tep8_emulation(config, kMxfp4Dynamic) + + +@pytest.mark.parametrize( + "requested_backend,expected_backend", + [ + ("auto", Mxfp4MoeBackend.EMULATION), + ("aiter", Mxfp4MoeBackend.AITER_MXFP4_MXFP4), + ], +) +def test_qwen38_tep8_auto_fallback_respects_explicit_backend( + requested_backend, + expected_backend, + monkeypatch, +): + import vllm.model_executor.layers.fused_moe.oracle.mxfp4 as mxfp4_oracle + + class SupportedExperts: + @staticmethod + def is_supported_config(*args, **kwargs): + return True, None + + config = _make_qwen38_tep8_moe_config() + config.moe_backend = requested_backend + monkeypatch.setattr(current_platform, "is_rocm", lambda: True) + monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: True) + monkeypatch.setattr(mxfp4_oracle, "_user_moe_activation_override", lambda: None) + monkeypatch.setattr( + mxfp4_oracle, "backend_to_kernel_cls", lambda backend: [SupportedExperts] + ) + + backend, experts_cls = select_mxfp4_moe_backend( + config, activation_key=kMxfp4Dynamic + ) + + assert backend == expected_backend + assert experts_cls is SupportedExperts + + @pytest.fixture def mxfp4_oracle_config(): """Stub the config the oracle reads (``model_config.quantization_config``) diff --git a/tests/quantization/test_online_mxfp4.py b/tests/quantization/test_online_mxfp4.py index 660d404054..95c814e9fc 100644 --- a/tests/quantization/test_online_mxfp4.py +++ b/tests/quantization/test_online_mxfp4.py @@ -604,3 +604,24 @@ def test_online_mxfp4_dense_matches_quark( keys=("weight", "weight_scale"), packed_weight_keys=("weight",), ) + + +def test_quark_ocp_mx_dense_rejects_unaligned_input_partition(): + scheme = object.__new__(QuarkOCP_MX) + layer = torch.nn.Module() + layer.prefix = "model.layers.0.mlp.shared_expert.down_proj" + + with pytest.raises( + ValueError, + match=( + "input size per partition of 80, which must be divisible by " + "the OCP MX group size 32" + ), + ): + scheme.create_weights( + layer=layer, + output_partition_sizes=[2560], + input_size_per_partition=80, + params_dtype=torch.bfloat16, + weight_loader=lambda *_: None, + ) diff --git a/tests/quantization/test_trtllm_nvfp4_hidden_dim_padding.py b/tests/quantization/test_trtllm_nvfp4_hidden_dim_padding.py index 9bec23b1b4..be8f984853 100644 --- a/tests/quantization/test_trtllm_nvfp4_hidden_dim_padding.py +++ b/tests/quantization/test_trtllm_nvfp4_hidden_dim_padding.py @@ -11,6 +11,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import ( prepare_nvfp4_moe_layer_for_fi_or_cutlass, ) from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( + align_fp4_moe_weights_for_fi, align_trtllm_fp4_moe_hidden_dim_for_fi, ) @@ -107,3 +108,53 @@ def test_align_trtllm_fp4_moe_hidden_dim_pads_to_256_multiple(): assert torch.count_nonzero(out_w13_scale[:, :, hidden_dim // 16 :]) == 0 assert torch.count_nonzero(out_w2[:, hidden_dim:, :]) == 0 assert torch.count_nonzero(out_w2_scale[:, hidden_dim:, :]) == 0 + + +def test_align_trtllm_fp4_moe_intermediate_pads_gate_and_up_separately(): + # Qwen3.8-Flash-Next has intermediate_size=640. At TP4 each rank owns + # 160 values, which TRTLLM pads to the next 64-value boundary (192). + intermediate = 160 + padded_intermediate = 192 + hidden_dim = 32 + + gate = torch.ones((1, intermediate, hidden_dim // 2), dtype=torch.uint8) + up = torch.full_like(gate, 2) + w13 = torch.cat((gate, up), dim=1) + gate_scale = torch.full((1, intermediate, hidden_dim // 16), 3, dtype=torch.uint8) + up_scale = torch.full_like(gate_scale, 4) + w13_scale = torch.cat((gate_scale, up_scale), dim=1) + w2 = torch.full((1, hidden_dim, intermediate // 2), 5, dtype=torch.uint8) + w2_scale = torch.full((1, hidden_dim, intermediate // 16), 6, dtype=torch.uint8) + + out_w13, out_w13_scale, out_w2, out_w2_scale, out_intermediate = ( + align_fp4_moe_weights_for_fi( + w13, + w13_scale, + w2, + w2_scale, + is_act_and_mul=True, + min_alignment=64, + ) + ) + + assert out_intermediate == padded_intermediate + torch.testing.assert_close(out_w13[:, :intermediate], gate) + torch.testing.assert_close( + out_w13[:, padded_intermediate : padded_intermediate + intermediate], up + ) + torch.testing.assert_close(out_w13_scale[:, :intermediate], gate_scale) + torch.testing.assert_close( + out_w13_scale[:, padded_intermediate : padded_intermediate + intermediate], + up_scale, + ) + + assert torch.count_nonzero(out_w13[:, intermediate:padded_intermediate]) == 0 + assert torch.count_nonzero(out_w13[:, padded_intermediate + intermediate :]) == 0 + assert torch.count_nonzero(out_w13_scale[:, intermediate:padded_intermediate]) == 0 + assert ( + torch.count_nonzero(out_w13_scale[:, padded_intermediate + intermediate :]) == 0 + ) + torch.testing.assert_close(out_w2[:, :, : intermediate // 2], w2) + torch.testing.assert_close(out_w2_scale[:, :, : intermediate // 16], w2_scale) + assert torch.count_nonzero(out_w2[:, :, intermediate // 2 :]) == 0 + assert torch.count_nonzero(out_w2_scale[:, :, intermediate // 16 :]) == 0 diff --git a/tests/v1/core/test_contiguous_kv_packing.py b/tests/v1/core/test_contiguous_kv_packing.py index 1be6078043..5249de7f8c 100644 --- a/tests/v1/core/test_contiguous_kv_packing.py +++ b/tests/v1/core/test_contiguous_kv_packing.py @@ -9,23 +9,34 @@ block dim (a contiguous region per layer) or inside it (all layers' pages within block); the allocation is the same either way. """ +from dataclasses import replace from unittest.mock import MagicMock import pytest import torch from vllm.config import CacheConfig +from vllm.v1.core.kv_cache_manager import KVCacheManager from vllm.v1.core.kv_cache_utils import ( _get_kv_cache_bytes_per_block, + _get_packed_kv_cache_groups, _pool_bytes_per_block, + generate_scheduler_kv_cache_config, get_kv_cache_config_from_groups, + get_kv_cache_groups, + resolve_kv_cache_block_sizes, ) from vllm.v1.kv_cache_interface import ( + CircularBufferSpec, FullAttentionSpec, KVCacheGroupSpec, KVCacheLayout, + KVCacheSpec, + MambaSpec, MLAAttentionSpec, + SlidingWindowMLASpec, UniformTypeKVCacheSpecs, + iter_layer_specs, ) from vllm.v1.worker.utils import allocate_kv_cache @@ -86,6 +97,307 @@ def _bind(config, layout: str): return allocate_kv_cache(config, torch.device("cpu"), KVCacheLayout[layout], None) +MAIN_KV_PAGE_BYTES = 2_048 +COMPRESSED_PAGE_BYTES = 128 +NUM_CACHE_TUPLES = 3 +BYTES_PER_BLOCK = NUM_CACHE_TUPLES * (MAIN_KV_PAGE_BYTES + COMPRESSED_PAGE_BYTES) + + +def _main_kv_name(layer_index: int) -> str: + return f"model.layers.{layer_index}.self_attn" + + +def _compressed_name(layer_index: int) -> str: + return f"model.layers.{layer_index}.self_attn.indexer.compressed_key_cache" + + +def _compressor_state_name(layer_index: int) -> str: + return f"model.layers.{layer_index}.self_attn.indexer.raw_key_cache" + + +def _make_csa_linear_specs( + num_mamba: int = 7, + num_tuples: int = NUM_CACHE_TUPLES, +) -> dict[str, KVCacheSpec]: + specs: dict[str, KVCacheSpec] = {} + for layer_index in range(num_mamba): + specs[f"model.layers.{layer_index}.linear_attn"] = MambaSpec( + block_size=16, + shapes=((32,),), + dtypes=(torch.bfloat16,), + ) + specs["model.layers.0.ple"] = MambaSpec( + block_size=16, + shapes=((24,),), + dtypes=(torch.bfloat16,), + tp_replicated=True, + ) + for layer_index in range(num_tuples): + specs[_main_kv_name(layer_index)] = FullAttentionSpec( + block_size=16, + num_kv_heads=2, + head_size=16, + head_size_v=16, + dtype=torch.bfloat16, + ) + specs[_compressed_name(layer_index)] = MLAAttentionSpec( + block_size=16, + num_kv_heads=1, + head_size=16, + dtype=torch.bfloat16, + tokens_per_state=4, + ) + specs[_compressor_state_name(layer_index)] = CircularBufferSpec( + block_size=4, + num_kv_heads=1, + head_size=8, + head_size_v=0, + dtype=torch.bfloat16, + ) + return specs + + +def _shared_layout_config(): + config = _mock_vllm_config("BLNHC") + config.scheduler_config.disable_hybrid_kv_cache_manager = False + config.speculative_config = None + config.model_config.max_model_len = 16 + config.model_config.get_total_num_hidden_layers.return_value = 64 + config.model_config.get_total_num_kv_heads.return_value = 2 + config.model_config.get_num_kv_heads.return_value = 2 + config.parallel_config.pipeline_parallel_size = 1 + config.parallel_config.decode_context_parallel_size = 1 + config.cache_config.block_size = 16 + config.cache_config.enable_prefix_caching = False + config.cache_config.prefix_match_unit = None + config.cache_config.mamba_cache_mode = "none" + return config + + +class TestCSALinearGrouping: + """A CSA + linear-attention model (sparse attention with a compressor ring, + plus sharded GDN and one TP-replicated PLE state) goes through the generic + packed-group path; no model-specific branch is involved.""" + + @staticmethod + def _mamba_groups(groups): + """(group, per-layer mamba spec) for every group holding mamba state.""" + out = [] + for group in groups: + spec = next(iter(iter_layer_specs(group.kv_cache_spec))) + if isinstance(spec, MambaSpec): + out.append((group, spec)) + return out + + def test_replicated_state_gets_its_own_group(self): + """The PLE state is TP-replicated and a different size, so it must not + share a manager group with the sharded GDN states (the NIXL worker + requires exactly one single-layer replicated group).""" + groups = get_kv_cache_groups(_shared_layout_config(), _make_csa_linear_specs()) + + mamba_groups = self._mamba_groups(groups) + replicated = [g for g, spec in mamba_groups if spec.tp_replicated] + assert len(replicated) == 1 + assert replicated[0].layer_names == ["model.layers.0.ple"] + sharded = [g for g, spec in mamba_groups if not spec.tp_replicated] + assert sharded, "GDN states must keep their own groups" + assert sorted(n for g in sharded for n in g.layer_names) == sorted( + f"model.layers.{i}.linear_attn" for i in range(7) + ) + + def test_roles_land_in_separate_groups(self): + groups = get_kv_cache_groups(_shared_layout_config(), _make_csa_linear_specs()) + + owner = next(g for g in groups if _main_kv_name(0) in g.layer_names) + assert sorted(owner.layer_names) == sorted( + [ + *(_main_kv_name(i) for i in range(NUM_CACHE_TUPLES)), + *(_compressed_name(i) for i in range(NUM_CACHE_TUPLES)), + ] + ) + assert owner.kv_cache_spec.prefix_cacheable + + scratch = next(g for g in groups if _compressor_state_name(0) in g.layer_names) + assert scratch.layer_names == [ + _compressor_state_name(i) for i in range(NUM_CACHE_TUPLES) + ] + assert not scratch.kv_cache_spec.prefix_cacheable + + def test_every_group_fits_one_packed_block(self): + config = _shared_layout_config() + groups = get_kv_cache_groups(config, _make_csa_linear_specs()) + bytes_per_block = _get_kv_cache_bytes_per_block(groups) + + pages = _pages(groups) + for group in groups: + assert sum(pages[n] for n in group.layer_names) <= bytes_per_block + + kv_cache_config = get_kv_cache_config_from_groups( + config, groups, available_memory=bytes_per_block * 32 + ) + assert kv_cache_config.num_blocks == 32 + # Groups overlay from byte 0 of each block, so a layer never addresses + # past the block it belongs to. + for tensor in kv_cache_config.kv_cache_tensors: + assert tensor.block_stride == bytes_per_block + assert tensor.offset < bytes_per_block + + def test_scratch_group_survives_computed_block_truncation(self): + """The scratch group contributes no computed blocks, so truncating a + lookup result must skip it: its block size is the ring capacity, which + neither divides the hit length nor bounds the (empty) block list.""" + config = _shared_layout_config() + config.cache_config.enable_prefix_caching = True + groups = get_kv_cache_groups(config, _make_csa_linear_specs()) + kv_cache_config = get_kv_cache_config_from_groups( + config, groups, available_memory=BYTES_PER_BLOCK * 64 + ) + scheduler_config = generate_scheduler_kv_cache_config([kv_cache_config]) + manager = KVCacheManager( + scheduler_config, + max_model_len=8192, + enable_caching=True, + hash_block_size=16, + scheduler_block_size=16, + ) + blocks = manager.create_kv_cache_blocks( + tuple( + manager.block_pool.get_new_blocks(3) + if group.kv_cache_spec.prefix_cacheable + else [] + for group in scheduler_config.kv_cache_groups + ) + ) + truncated = manager.truncate_computed_blocks(blocks, 48) + scratch_index = next( + i + for i, group in enumerate(scheduler_config.kv_cache_groups) + if not group.kv_cache_spec.prefix_cacheable + ) + assert truncated.blocks[scratch_index] == [] + + def test_prefix_hits_respect_compression_alignment(self): + config = _shared_layout_config() + config.cache_config.enable_prefix_caching = True + config.cache_config.prefix_match_unit = 16 + config.cache_config.mamba_cache_mode = "align" + specs = { + name: replace(spec, mamba_cache_mode="align") + if isinstance(spec, MambaSpec) + else spec + for name, spec in _make_csa_linear_specs(num_tuples=1).items() + } + groups = get_kv_cache_groups(config, specs) + kv_cache_config = get_kv_cache_config_from_groups( + config, groups, available_memory=8 * MAIN_KV_PAGE_BYTES + ) + assert resolve_kv_cache_block_sizes(kv_cache_config, config) == (16, 16) + + # 2 tokens is not a multiple of the compression ratio 4: a prefix hit + # could land inside a partially filled compressed state. + config.cache_config.prefix_match_unit = 2 + with pytest.raises(ValueError, match="per-state compression"): + resolve_kv_cache_block_sizes(kv_cache_config, config) + + def test_scratch_ring_does_not_drag_hash_granularity(self): + config = _shared_layout_config() + config.cache_config.enable_prefix_caching = True + config.cache_config.mamba_cache_mode = "align" + specs = { + name: replace(spec, mamba_cache_mode="align") + if isinstance(spec, MambaSpec) + else spec + for name, spec in _make_csa_linear_specs(num_tuples=1).items() + } + groups = get_kv_cache_groups(config, specs) + kv_cache_config = get_kv_cache_config_from_groups( + config, groups, available_memory=8 * MAIN_KV_PAGE_BYTES + ) + # The hash granularity is the GCD over prefix-cacheable groups only; + # the 4-token scratch ring is excluded (it would drag it to 4). + assert resolve_kv_cache_block_sizes(kv_cache_config, config) == (16, 16) + + def test_compressed_attention_hashes_can_be_finer_than_cache_hits(self): + config = _shared_layout_config() + config.cache_config.enable_prefix_caching = True + specs = { + "compressed.4": MLAAttentionSpec( + block_size=256, + num_kv_heads=1, + head_size=16, + dtype=torch.bfloat16, + tokens_per_state=4, + ), + "compressed.128": MLAAttentionSpec( + block_size=256, + num_kv_heads=1, + head_size=16, + dtype=torch.bfloat16, + tokens_per_state=128, + ), + "compressor_state.4": SlidingWindowMLASpec( + block_size=4, + num_kv_heads=1, + head_size=8, + head_size_v=0, + dtype=torch.bfloat16, + sliding_window=8, + ), + } + groups = get_kv_cache_groups(config, specs) + kv_cache_config = get_kv_cache_config_from_groups( + config, groups, available_memory=8 * MAIN_KV_PAGE_BYTES + ) + + # Hashes are computed every 4 tokens, but without an align-mode Mamba + # group cache hits remain on the 256-token scheduler boundary. + assert resolve_kv_cache_block_sizes(kv_cache_config, config) == (256, 4) + + @pytest.mark.parametrize("wide", ["unbalanced_attention", "unsplittable_state"]) + def test_mamba_split_measures_the_block_the_other_groups_already_force(self, wide): + """Whatever fixes the block stride -- a bucket with unequal layer counts + per page size, which is emitted whole, or a single-layer state that + cannot be split at all -- the mamba layers must be sized against it. + Sizing them against a narrower bucket splits them past what the block + already fits, spending a pool block per extra group for no saving.""" + config = _mock_vllm_config("BLNHC") + config.speculative_config = None + specs = {} + if wide == "unbalanced_attention": + for i in range(3): + specs[f"wide.{i}"] = _mla(1024) + specs["wide.odd"] = _mla(800) + else: + specs["wide.state"] = MambaSpec( + block_size=16, + shapes=((51_200,),), + dtypes=(torch.bfloat16,), + tp_replicated=True, + ) + # Mixed and balanced, but far narrower than the bucket above. + for i in range(10): + specs[f"narrow.a.{i}"] = FullAttentionSpec( + block_size=16, num_kv_heads=1, head_size=8, dtype=torch.uint8 + ) + specs[f"narrow.b.{i}"] = FullAttentionSpec( + block_size=16, num_kv_heads=1, head_size=16, dtype=torch.uint8 + ) + for i in range(100): + specs[f"gdn.{i}"] = MambaSpec( + block_size=16, shapes=((512,),), dtypes=(torch.bfloat16,) + ) + + groups = _get_packed_kv_cache_groups(config, specs) + gdn = [g for g in groups if g.layer_names[0].startswith("gdn.")] + + assert _get_kv_cache_bytes_per_block(groups) == sum( + specs[name].page_size_bytes for name in specs if name.startswith("wide.") + ) + # That block holds every GDN state at once, so the repeat pattern alone + # decides the split; the cap must not add groups on top of it. + assert len(gdn) == 10 + + class TestDensePacking: def test_bytes_per_block_is_largest_group(self): groups, g1, g2 = _mixed_page_groups() diff --git a/tests/v1/core/test_kv_cache_utils.py b/tests/v1/core/test_kv_cache_utils.py index 1e6255a4c2..9ca04c5c94 100644 --- a/tests/v1/core/test_kv_cache_utils.py +++ b/tests/v1/core/test_kv_cache_utils.py @@ -12,7 +12,7 @@ import pytest import torch import vllm.v1.core.kv_cache_utils as kv_cache_utils -from vllm.config import ModelConfig, SchedulerConfig, VllmConfig +from vllm.config import CacheConfig, ModelConfig, SchedulerConfig, VllmConfig from vllm.config.kv_events import KVEventsConfig from vllm.lora.request import LoRARequest from vllm.multimodal.inputs import ( @@ -37,7 +37,6 @@ from vllm.v1.core.kv_cache_utils import ( get_kv_cache_groups, get_max_concurrency_for_kv_cache_config, get_request_block_hasher, - group_and_unify_kv_cache_specs, hash_block_tokens, init_none_hash, is_kv_cache_spec_uniform, @@ -2226,38 +2225,6 @@ def new_swa_mla_spec(head_size=576, sliding_window=128, model_version=None): ) -def test_group_and_unify_kv_cache_specs_no_swa_mla_returns_none(): - # Without any SlidingWindowMLASpec the function does not apply. - specs = {"mla.0": new_mla_spec(), "mla.1": new_mla_spec()} - assert group_and_unify_kv_cache_specs(specs) is None - - -def test_group_and_unify_kv_cache_specs_uniform_page_size_returns_none(): - # A non-DeepseekV4 model that mixes full MLA and sliding-window MLA layers - # with a uniform page size must not fall into the DeepseekV4 tuple-packing - # path; it should defer to the generic uniform-page-size grouping instead. - mla_spec = new_mla_spec() - swa_spec = new_swa_mla_spec() - assert mla_spec.page_size_bytes == swa_spec.page_size_bytes - specs = {"mla.0": mla_spec, "mla.1": new_mla_spec(), "swa.0": swa_spec} - assert group_and_unify_kv_cache_specs(specs) is None - - -def test_group_and_unify_kv_cache_specs_mixed_page_size_groups(): - # DeepseekV4-style: differing page sizes across MLA and sliding-window MLA - # layers do require tuple packing, so grouping must still be produced. - mla_spec = new_mla_spec() - swa_spec = new_swa_mla_spec(head_size=1024) - assert mla_spec.page_size_bytes != swa_spec.page_size_bytes - specs = {"mla.0": mla_spec, "mla.1": new_mla_spec(), "swa.0": swa_spec} - grouped = group_and_unify_kv_cache_specs(specs) - assert grouped is not None - # One MLA group plus one sliding-window MLA group. - assert len(grouped) == 2 - layer_names = {name for g in grouped for name in g.kv_cache_specs} - assert layer_names == {"mla.0", "mla.1", "swa.0"} - - def new_indexer_mla_spec(block_size=16): # Sparse-attention indexer k_cache: an MLAAttentionSpec with a much smaller # page size than the main MLA attention (uint8, small head), so their pages @@ -2270,10 +2237,30 @@ def new_indexer_mla_spec(block_size=16): ) +def test_mixed_page_size_groups_use_spec_compatibility(): + specs = {} + for i in range(3): + specs[f"mla.{i}"] = new_mla_spec() + specs[f"indexer.{i}"] = new_indexer_mla_spec() + specs.update({f"swa.{i}": new_swa_mla_spec(head_size=1024) for i in range(5)}) + + config = _grouping_config() + config.cache_config = CacheConfig() + config.cache_config.kv_cache_layout = "BLNHC" + groups = get_kv_cache_groups(config, specs) + + assert len(groups) == 3 + assert {name for group in groups for name in group.layer_names} == set(specs) + assert sorted(len(group.layer_names) for group in groups) == [2, 3, 6] + + def _grouping_config(): + cache_config = CacheConfig() + cache_config.kv_cache_layout = "LBNHC" return SimpleNamespace( scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False), speculative_config=None, + cache_config=cache_config, ) @@ -3281,10 +3268,16 @@ def test_iter_layer_specs_returns_group_members(): assert list(iter_layer_specs(wrapped)) == [full, mla] -def _spec_decode_grouping_config(method="dspark"): +def _spec_decode_grouping_config(method="dspark", model_type=None): """Grouping config with an EAGLE-family speculative method enabled.""" return SimpleNamespace( scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False), + cache_config=SimpleNamespace( + get_resolved_kv_cache_layout=lambda: SimpleNamespace( + is_block_outermost=True + ) + ), + model_config=SimpleNamespace(hf_config=SimpleNamespace(model_type=model_type)), speculative_config=SimpleNamespace( method=method, use_eagle=lambda: True, @@ -3322,13 +3315,12 @@ def test_draft_group_annotated_on_hybrid_general_path(): flagged = [g for g in groups if g.is_eagle_group] assert len(flagged) == 1 - assert flagged[0].layer_names == ["draft.attn.0"] + assert "draft.attn.0" in flagged[0].layer_names def test_mamba_groups_never_flagged_even_when_draft_shares_a_group(): - # MLAAttentionSpec.merge ORs non_causal_multi_token_decode rather than - # requiring equality, so a draft layer can share a group with target - # layers; that combined group still holds volatile draft KV and must be + # Packed uniform-type groups can contain distinct target and draft layer + # specs; the combined group still holds volatile draft KV and must be # flagged. What must never happen is a Mamba group being flagged: that # widens its lookup window to two consecutive chunks, which align-mode # checkpointing never produces, zeroing every lookup. @@ -3347,10 +3339,8 @@ def test_mamba_groups_never_flagged_even_when_draft_shares_a_group(): def test_draft_group_not_annotated_without_spec_decode(): # The marker alone must not flag anything; the eagle semantics only apply # when a speculative method is actually enabled. - config = SimpleNamespace( - scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False), - speculative_config=None, - ) + config = _spec_decode_grouping_config() + config.speculative_config = None groups = get_kv_cache_groups(config, _hybrid_specs_with_draft(draft=True)) assert not any(g.is_eagle_group for g in groups) @@ -3396,12 +3386,13 @@ def _deepseek_v4_specs(model_version="deepseek_v4"): } -def test_deepseek_v4_draft_group_annotated_on_group_and_unify_path(): +def test_deepseek_v4_draft_group_annotated_on_packed_path(): # DeepseekV4's MTP block reuses the target's decoder layer, so its spec # carries no draft marker and only the positional rule can find it. This # pins the pre-existing behaviour that the unified annotator must preserve. groups = get_kv_cache_groups( - _spec_decode_grouping_config(method="mtp"), _deepseek_v4_specs() + _spec_decode_grouping_config(method="mtp", model_type="deepseek_v4"), + _deepseek_v4_specs(model_version=None), ) flagged = [g for g in groups if g.is_eagle_group] @@ -3409,13 +3400,13 @@ def test_deepseek_v4_draft_group_annotated_on_group_and_unify_path(): assert "model.layers.3.self_attn.attn" in flagged[0].layer_names -def test_deepseek_v4_annotation_requires_model_version(): +def test_deepseek_v4_annotation_requires_model_type(): # The positional rule is only sound for DeepseekV4, where the draft layer # is known to be registered last. Without that model gate nothing may be # flagged, however the grouping happens to fall out. groups = get_kv_cache_groups( - _spec_decode_grouping_config(method="mtp"), - _deepseek_v4_specs(model_version=None), + _spec_decode_grouping_config(method="mtp", model_type="other"), + _deepseek_v4_specs(), ) assert not any(g.is_eagle_group for g in groups) diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index b9d709b8c1..7b730523fa 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -1761,6 +1761,52 @@ def test_spec_decode_padding_skipped_for_diffusion(): assert r2.request_id not in out.scheduled_spec_decode_tokens +@pytest.mark.parametrize("aligned_tokens", [1, 2, 3]) +def test_spec_decode_padding_dropped_when_recurrent_alignment_clips( + aligned_tokens: int, +): + """A clipped padded request must drop its padding, not shorten it. + + The sampler derives a request's row window from its draft count, so a + padded request that keeps fewer than 1 + num_spec query rows would select + the preceding request's hidden states. + """ + num_spec = 3 + scheduler = create_scheduler( + num_speculative_tokens=num_spec, + enable_prefix_caching=True, + block_size=16, + ) + r1, r2 = create_requests( + num_requests=2, num_tokens=33, same_prompt=True, max_tokens=16 + ) + + scheduler.add_request(r1) + out = scheduler.schedule() + assert out.num_scheduled_tokens[r1.request_id] == 33 + _model_output(scheduler, out, [[100]]) + scheduler.update_draft_token_ids(DraftTokenIds([r1.request_id], [[1, 2, 3]])) + + # Model the recurrent align split that clips a padded 1+3 prompt tail at + # the next cache boundary. The real QSA block size produces all three cases + # depending on prompt length modulo four. + scheduler.need_mamba_block_aligned_split = True + original_split = scheduler._mamba_block_aligned_split + + def align_prompt_tail(request, num_new_tokens, *args): + if request is r2: + return aligned_tokens + return original_split(request, num_new_tokens, *args) + + scheduler._mamba_block_aligned_split = align_prompt_tail + scheduler.add_request(r2) + out = scheduler.schedule() + + assert out.scheduled_spec_decode_tokens[r1.request_id] == [1, 2, 3] + assert out.num_scheduled_tokens[r2.request_id] == 1 + assert r2.request_id not in out.scheduled_spec_decode_tokens + + def test_spec_decode_padding_skipped_with_prefill_in_batch(): """Padding is skipped when the batch contains a prefill chunk: the batch is already mixed/non-uniform, so padding a new decode request buys nothing. diff --git a/tests/v1/core/test_single_type_kv_cache_manager.py b/tests/v1/core/test_single_type_kv_cache_manager.py index 074a3faf87..57885c9cb9 100644 --- a/tests/v1/core/test_single_type_kv_cache_manager.py +++ b/tests/v1/core/test_single_type_kv_cache_manager.py @@ -14,6 +14,7 @@ from vllm.v1.core.kv_cache_utils import ( ) from vllm.v1.core.single_type_kv_cache_manager import ( ChunkedLocalAttentionManager, + CircularBufferManager, FullAttentionManager, MambaManager, RSWAManager, @@ -21,6 +22,7 @@ from vllm.v1.core.single_type_kv_cache_manager import ( ) from vllm.v1.kv_cache_interface import ( ChunkedLocalAttentionSpec, + CircularBufferSpec, FullAttentionSpec, MambaSpec, RSWASpec, @@ -136,6 +138,67 @@ def get_chunked_local_attention_manager( ) +def test_circular_buffer_allocates_one_block_for_the_request_lifetime(): + block_size = 4 + spec = CircularBufferSpec( + block_size=block_size, + num_kv_heads=1, + head_size=8, + dtype=torch.bfloat16, + ) + block_pool = BlockPool( + num_gpu_blocks=10, enable_caching=True, hash_block_size=block_size + ) + manager = CircularBufferManager( + spec, + block_pool=block_pool, + enable_caching=True, + kv_cache_group_id=0, + scheduler_block_size=block_size, + needs_kv_cache_zeroing=True, + max_admission_blocks_per_request=1, + ) + request_id = "request" + + assert ( + manager.get_num_blocks_to_allocate( + request_id, + num_tokens=128, + new_computed_blocks=[block_pool.blocks[1]], + total_computed_tokens=0, + num_local_computed_tokens=0, + num_tokens_main_model=128, + ) + == 1 + ) + blocks = manager.allocate_new_blocks( + request_id, num_tokens=128, num_tokens_main_model=128 + ) + assert len(blocks) == 1 + assert not manager.records_new_block_ids + assert manager.take_new_block_ids() == [] + + for num_tokens in (256, 1024): + assert ( + manager.get_num_blocks_to_allocate( + request_id, + num_tokens, + [], + total_computed_tokens=num_tokens - 1, + num_local_computed_tokens=0, + num_tokens_main_model=num_tokens, + ) + == 0 + ) + assert manager.allocate_new_blocks(request_id, num_tokens, num_tokens) == [] + + manager.cache_blocks(request_id, 1024) + manager.remove_skipped_blocks(request_id, 1024) + assert manager.get_num_common_prefix_blocks(request_id) == 0 + assert manager.get_num_skipped_tokens(1024) == 0 + assert manager.req_to_blocks[request_id] == blocks + + def test_sliding_window_records_new_blocks_for_zeroing(): block_size = 2 spec = SlidingWindowSpec( diff --git a/tests/v1/e2e/general/test_mamba_prefix_cache.py b/tests/v1/e2e/general/test_mamba_prefix_cache.py index d8b42b9abf..27331516cb 100644 --- a/tests/v1/e2e/general/test_mamba_prefix_cache.py +++ b/tests/v1/e2e/general/test_mamba_prefix_cache.py @@ -234,7 +234,8 @@ def get_fake_execute_model_fn(original_execute_model_fn: Callable): iter(scheduler_output.num_scheduled_tokens.values()) ) assert num_scheduled_tokens == cur_step_action.num_scheduled_tokens - mamba_group_ids, mamba_spec = get_mamba_groups(self.kv_cache_config) + mamba_groups = get_mamba_groups(self.kv_cache_config) + mamba_spec, mamba_group_ids = next(iter(mamba_groups.items())) mamba_group_id = mamba_group_ids[0] mamba_layer_name = self.kv_cache_config.kv_cache_groups[ mamba_group_id @@ -321,7 +322,8 @@ def get_fake_process_mamba_fn( assert len(copy_info[0]) == len(copy_info[1]) == len(copy_info[2]) == 0 else: assert len(copy_info[0]) == len(copy_info[1]) == len(copy_info[2]) == 2 - mamba_group_ids, mamba_spec = get_mamba_groups(kv_cache_config) + mamba_groups = get_mamba_groups(kv_cache_config) + mamba_spec, mamba_group_ids = next(iter(mamba_groups.items())) mamba_group_id = mamba_group_ids[0] mamba_layer_name = kv_cache_config.kv_cache_groups[ mamba_group_id @@ -936,12 +938,12 @@ def _run_mamba_prefix_cache_mrv1( cleanup_dist_env_and_memory() -@create_new_process_for_each_test() +@create_new_process_for_each_test("spawn") def test_mamba_prefix_cache_mrv1(monkeypatch: pytest.MonkeyPatch): _run_mamba_prefix_cache_mrv1(monkeypatch, async_scheduling=False) -@create_new_process_for_each_test() +@create_new_process_for_each_test("spawn") def test_mamba_prefix_cache_mrv1_async(monkeypatch: pytest.MonkeyPatch): _run_mamba_prefix_cache_mrv1(monkeypatch, async_scheduling=True) @@ -975,11 +977,12 @@ def _run_mamba_prefix_cache_mrv2( forward_context = ( model_state.vllm_config.compilation_config.static_forward_context ) - group_ids, _ = get_mamba_groups(kv_cache_config) - for group_id in group_ids: - block_table = block_tables[group_id] - for layer_name in kv_cache_config.kv_cache_groups[group_id].layer_names: - yield forward_context[layer_name].kv_cache[-1], block_table + mamba_groups = get_mamba_groups(kv_cache_config) + for group_ids in mamba_groups.values(): + for group_id in group_ids: + block_table = block_tables[group_id] + for layer_name in kv_cache_config.kv_cache_groups[group_id].layer_names: + yield forward_context[layer_name].kv_cache[-1], block_table def temporal_block(temporal_state, block_table, col): # Resolving the block id for assertions is a deliberate D2H. diff --git a/tests/v1/kv_connector/unit/test_hidden_states_connector.py b/tests/v1/kv_connector/unit/test_hidden_states_connector.py index ffd6482891..8427833b12 100644 --- a/tests/v1/kv_connector/unit/test_hidden_states_connector.py +++ b/tests/v1/kv_connector/unit/test_hidden_states_connector.py @@ -15,6 +15,8 @@ from vllm.v1.kv_cache_interface import ( FullAttentionSpec, HiddenStateCacheSpec, KVCacheGroupSpec, + KVCacheLayout, + MambaSpec, MLAAttentionSpec, SlidingWindowMLASpec, ) @@ -99,12 +101,12 @@ def test_get_block_size_falls_back_to_cache_config_when_no_kv_cache_config(): assert block_size == 16 -# ---- MLA-verifier absorption ------------------------------------------------ +# ---- Packed MLA grouping ---------------------------------------------------- -def test_find_group_id_errors_clearly_when_absorbed_by_mla_swa_verifier(): - # HiddenStateCacheSpec subclasses MLAAttentionSpec, so an MLA + sliding- - # window MLA verifier absorbs it into the MLA group instead of isolating it. +def test_hidden_state_group_isolated_from_packed_mla_groups(): + # HiddenStateCacheSpec subclasses MLAAttentionSpec, but grouping must pull + # it out before packing compatible MLA cache specs. dt = torch.bfloat16 spec = { "layers.0.mla": MLAAttentionSpec( @@ -116,11 +118,54 @@ def test_find_group_id_errors_clearly_when_absorbed_by_mla_swa_verifier(): "cache_only_layers.61": _hidden(64), } vllm_config = SimpleNamespace( + cache_config=SimpleNamespace( + get_resolved_kv_cache_layout=lambda: KVCacheLayout.BLHNC + ), scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False), speculative_config=None, ) groups = get_kv_cache_groups(vllm_config, spec) - assert not any(isinstance(g.kv_cache_spec, HiddenStateCacheSpec) for g in groups) + hidden_group_ids = [ + i + for i, group in enumerate(groups) + if isinstance(group.kv_cache_spec, HiddenStateCacheSpec) + ] + assert len(hidden_group_ids) == 1 cfg = SimpleNamespace(kv_cache_groups=groups) - with pytest.raises(ValueError, match="MLA verifiers are unsupported"): - ExampleHiddenStatesConnector._find_cache_kv_group_id(cfg) + assert ( + ExampleHiddenStatesConnector._find_cache_kv_group_id(cfg) == hidden_group_ids[0] + ) + + +def test_hidden_state_group_isolated_from_packed_mixed_page_groups(): + # Packed grouping keeps groups with unequal page sizes (blocks are strided + # by the widest group), so the hidden group is appended without padding. + hidden = _hidden(16) + spec = { + "layers.0.attn": _full(16), + "layers.1.mamba": MambaSpec( + block_size=16, shapes=((1024,),), dtypes=(torch.float32,) + ), + "cache_only_layers.61": hidden, + } + vllm_config = SimpleNamespace( + cache_config=SimpleNamespace( + get_resolved_kv_cache_layout=lambda: KVCacheLayout.BLHNC + ), + scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False), + speculative_config=None, + ) + groups = get_kv_cache_groups(vllm_config, spec) + page_sizes = {g.kv_cache_spec.page_size_bytes for g in groups} + assert len(page_sizes) > 1, "spec must exercise the packed grouping path" + hidden_group_ids = [ + i + for i, group in enumerate(groups) + if isinstance(group.kv_cache_spec, HiddenStateCacheSpec) + ] + assert len(hidden_group_ids) == 1 + assert groups[hidden_group_ids[0]].kv_cache_spec == hidden + cfg = SimpleNamespace(kv_cache_groups=groups) + assert ( + ExampleHiddenStatesConnector._find_cache_kv_group_id(cfg) == hidden_group_ids[0] + ) diff --git a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py index 1a0f70d288..4dc017fade 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py @@ -865,6 +865,10 @@ def _make_mock_worker_for_desc_ids( worker._group_spec_types = group_spec_types worker.block_len_per_layer = block_len_per_layer or [100] worker._conv_decomp = None + worker._is_csa_linear = False + worker._ssm_region_indices = [0] if has_mamba else [] + worker._ple_group_index = None + worker._ple_region_index = None if has_mamba: from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import ( # noqa: E501 MambaConvSplitInfo, @@ -1823,6 +1827,7 @@ def test_push_write_hybrid_mla_replicates_attention(): worker.shutdown = lambda: None # skeleton worker: silence __del__ worker.use_mla = True worker._has_mamba = True + worker._is_csa_linear = False worker._group_spec_types = (MLAAttentionSpec, MambaSpec) worker.transfer_topo = MagicMock() worker.transfer_topo.tp_ratio.return_value = -2 diff --git a/tests/v1/kv_connector/unit/test_nixl_desc_geometry.py b/tests/v1/kv_connector/unit/test_nixl_desc_geometry.py index daf6251dce..561457157d 100644 --- a/tests/v1/kv_connector/unit/test_nixl_desc_geometry.py +++ b/tests/v1/kv_connector/unit/test_nixl_desc_geometry.py @@ -645,3 +645,302 @@ def test_mismatched_mla_kernel_page_rejected_for_mla_hybrid(): meta_r.block_lens = [x // 2 for x in worker.block_len_per_layer] with pytest.raises((AssertionError, RuntimeError)): worker.add_remote_agent(meta_r, remote_tp_rank=0, remote_tp_size=2) + + +def _make_csa_linear_ple_worker(scratch_aliases: str = "compressed"): + """``scratch_aliases`` selects which pages the compressor ring overlays.""" + from unittest.mock import MagicMock + + from vllm.config import set_current_vllm_config + from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker as bw + from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import ( + NixlConnectorWorker, + ) + from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum + from vllm.v1.kv_cache_interface import ( + CircularBufferSpec, + FullAttentionSpec, + KVCacheConfig, + KVCacheGroupSpec, + KVCacheTensor, + MambaSpec, + MLAAttentionSpec, + UniformTypeKVCacheSpecs, + ) + + main_kv_specs = { + f"main_kv.{index}": FullAttentionSpec( + block_size=4, + num_kv_heads=1, + head_size=16, + dtype=torch.float16, + ) + for index in range(2) + } + compressed_specs = { + f"compressed.{index}": MLAAttentionSpec( + block_size=4, + num_kv_heads=1, + head_size=64, + dtype=torch.float16, + tokens_per_state=2, + ) + for index in range(2) + } + compressor_state_specs = { + f"compressor_state.{index}": CircularBufferSpec( + block_size=2, + num_kv_heads=1, + head_size=32, + dtype=torch.float16, + ) + for index in range(2) + } + sharded_mamba_spec = MambaSpec( + block_size=1, + shapes=((8, 3), (1, 4, 4)), + dtypes=(torch.float16, torch.float32), + page_size_padded=256, + mamba_type=MambaAttentionBackendEnum.GDN_ATTN, + ) + ple_spec = MambaSpec( + block_size=1, + shapes=((6, 3),), + dtypes=(torch.float16,), + page_size_padded=256, + mamba_type=MambaAttentionBackendEnum.SHORT_CONV, + tp_replicated=True, + ) + compressed_sparse_specs = { + name: spec + for index in range(2) + for name, spec in ( + (f"main_kv.{index}", main_kv_specs[f"main_kv.{index}"]), + (f"compressed.{index}", compressed_specs[f"compressed.{index}"]), + ) + } + + def group(specs): + uniform = UniformTypeKVCacheSpecs.from_specs(specs) + assert uniform is not None + return KVCacheGroupSpec(list(specs), uniform) + + tensor_regions: tuple[tuple[str, ...], ...] + if scratch_aliases == "compressed": + tensor_regions = ( + ("main_kv.0", "mamba.sharded", "mamba.ple"), + ("main_kv.1",), + ("compressed.0", "compressor_state.0"), + ("compressed.1", "compressor_state.1"), + ) + else: + assert scratch_aliases == "main_kv" + tensor_regions = ( + ("main_kv.0", "mamba.sharded", "mamba.ple", "compressor_state.0"), + ("main_kv.1", "compressor_state.1"), + ("compressed.0",), + ("compressed.1",), + ) + region_size = 512 + page_size = 256 + kv_cache_config = KVCacheConfig( + num_blocks=2, + kv_cache_tensors=[ + KVCacheTensor( + size=len(tensor_regions) * region_size, + layers=[layer_name], + layer_stride=region_size, + block_stride=page_size, + offset=region_index * region_size, + ) + for region_index, layer_names in enumerate(tensor_regions) + for layer_name in layer_names + ], + kv_cache_groups=[ + group(compressed_sparse_specs), + group(compressor_state_specs), + KVCacheGroupSpec(["mamba.sharded"], sharded_mamba_spec), + KVCacheGroupSpec(["mamba.ple"], ple_spec), + ], + ) + + vllm_config = create_vllm_config(block_size=4) + vllm_config.cache_config.enable_prefix_caching = False + vllm_config.kv_transfer_config.kv_buffer_device = "cuda" + fake_backend = MagicMock() + fake_backend.get_supported_kernel_block_sizes.return_value = [4] + fake_backend.get_name.return_value = "TEST_ATTN" + fake_backend.full_cls_name.return_value = "test.AttentionBackend" + fake_platform = MagicMock() + fake_platform.device_type = "cuda" + fake_platform.get_nixl_memory_type.return_value = "VRAM" + + with ( + patch.object(bw, "NixlWrapper", _RecordingNixl), + patch.object(bw, "get_tensor_model_parallel_rank", return_value=0), + patch.object(bw, "get_tensor_model_parallel_world_size", return_value=1), + patch.object(bw, "get_current_attn_backends", return_value=[fake_backend]), + patch.object(bw, "current_platform", fake_platform), + patch( + "vllm.model_executor.layers.mamba.mamba_utils.get_conv_state_layout", + return_value="DS", + ), + set_current_vllm_config(vllm_config), + ): + worker = NixlConnectorWorker(vllm_config, "local-engine", kv_cache_config) + tensors = [torch.zeros((2, 256), dtype=torch.uint8) for _ in range(4)] + worker.register_kv_caches( + { + layer_name: tensors[region_index] + for region_index, layer_names in enumerate(tensor_regions) + for layer_name in layer_names + } + ) + return worker + + +@pytest.mark.cpu_test +def test_csa_linear_registration_discovers_shared_regions(): + worker = _make_csa_linear_ple_worker() + + assert worker._ssm_region_indices == [0] + assert worker._ple_group_index == 3 + assert worker._ple_region_index == 0 + assert worker._region_is_mla == [False, False, True, True] + assert worker._scratch_region_indices == [2, 3] + + +@pytest.mark.cpu_test +def test_csa_linear_scratch_descs_follow_the_pages_they_overlay(): + """The scratch ring is addressed by the regions it actually registered in, + not by inferring that it must sit on the MLA pages: overlaying the main KV + pages instead moves its descriptors with it.""" + worker = _make_csa_linear_ple_worker(scratch_aliases="main_kv") + + # Same MLA regions as before, but the ring now lives in the non-MLA ones. + assert worker._region_is_mla == [False, False, True, True] + assert worker._scratch_region_indices == [0, 1] + + desc_ids = worker._compute_desc_ids( + block_ids=([1], [0], [1], [1]), + dst_num_blocks=2, + block_size_ratio=None, + physical_blocks_per_logical=1, + ) + # Scratch group (block 0) now maps to regions 0 and 1 -> descs 0 and 2, + # where the compressed-page placement gave 4 and 6. + assert desc_ids.tolist()[4:6] == [0, 2] + + +@pytest.mark.cpu_test +def test_csa_linear_selects_compressed_sparse_compressor_state_and_mamba_regions(): + worker = _make_csa_linear_ple_worker() + + desc_ids = worker._compute_desc_ids( + block_ids=([1], [0], [1], [1]), + dst_num_blocks=2, + block_size_ratio=None, + physical_blocks_per_logical=1, + ) + assert desc_ids.tolist() == [ + 1, + 3, + 5, + 7, + 4, + 6, + 9, + 11, + 13, + 15, + 17, + ] + + mamba_only = worker._compute_desc_ids( + block_ids=([], [], [1], [1]), + dst_num_blocks=2, + block_size_ratio=None, + physical_blocks_per_logical=1, + ) + assert mamba_only.tolist() == [9, 11, 13, 15, 17] + + +@pytest.mark.cpu_test +def test_csa_linear_ple_descriptor_is_not_split(): + from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import ( + TPMapping, + ) + + worker = _make_csa_linear_ple_worker() + bases = [0x10000, 0x20000, 0x30000, 0x40000] + attention = worker._build_fa_local(bases, block_size_ratio=1) + mamba = worker._build_mamba_local(bases) + assert attention.shape == (8, 3) + assert mamba.shape == (10, 3) + assert mamba[-2:, 0].tolist() == [0x10000, 0x10100] + assert mamba[-2:, 1].tolist() == [256, 256] + + mapping = TPMapping( + source_ranks_per_group=((0,), (0,), (0, 1), (0, 1)), + all_source_ranks=(0, 1), + rank_to_attention_slot={0: 0, 1: 0}, + rank_offset_factor=0, + ) + splits = list( + worker._build_local_splits_from_plan( + mapping, + np.concatenate([attention, mamba]), + num_fa_descs=len(attention), + ) + ) + assert len(splits) == 2 + assert [row[1] for row in splits[0][-2:]] == [256, 256] + assert [row[1] for row in splits[1][-2:]] == [256, 256] + assert splits[0][-3][1] * 2 == mamba[-3, 1] + assert splits[1][-3][1] * 2 == mamba[-3, 1] + + +@pytest.mark.cpu_test +def test_csa_linear_remote_ple_is_copied_whole(): + from types import SimpleNamespace + + from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( + NixlAgentMetadata, + ) + + worker = _make_csa_linear_ple_worker() + metadata = NixlAgentMetadata( + engine_id="remote", + agent_metadata=b"agent", + kv_caches_base_addr=[ + 0x10000, + 0x20000, + 0x30000, + 0x40000, + ], + device_id=0, + num_blocks=2, + block_lens=[256] * 4, + block_strides=[256] * 4, + kv_cache_layout="HND", + block_size=4, + ssm_sizes=(24, 32), + attn_backend_name="test", + physical_blocks_per_logical_kv_block=1, + ) + descriptors = worker._build_mamba_remote( + metadata, + tp_ratio=-2, + transfer_info=SimpleNamespace(remote_physical_blocks_per_logical=1), + ) + assert descriptors.shape == (10, 3) + assert descriptors[-2:, 0].tolist() == [0x10000, 0x10100] + assert descriptors[-2:, 1].tolist() == [256, 256] + + metadata.block_lens[0] = 128 + with pytest.raises(ValueError, match="PLE pages require identical"): + worker._build_mamba_remote( + metadata, + tp_ratio=-2, + transfer_info=SimpleNamespace(remote_physical_blocks_per_logical=1), + ) diff --git a/tests/v1/kv_connector/unit/test_nixl_push_connector.py b/tests/v1/kv_connector/unit/test_nixl_push_connector.py index a7ea078d6c..1b78b896a4 100644 --- a/tests/v1/kv_connector/unit/test_nixl_push_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_push_connector.py @@ -359,6 +359,7 @@ class _StubWriterWorker(NixlPushConnectorWorker): w._physical_blocks_per_logical_kv_block = 1 # Single non-hybrid attention group, matching the stub block id lists. w._has_mamba = False + w._is_csa_linear = False w._group_spec_types = (FullAttentionSpec,) w._engine_ttl = 0.0 w._engine_last_active = {} @@ -1198,6 +1199,75 @@ class TestPushWriterMlaReplication: # engine thread never sees a partial set and double-frees the request. assert sorted(w._sending_transfers["p-req"]) == [1000, 1001] + def test_csa_linear_writes_every_group_to_each_target(self): + from types import SimpleNamespace + + from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( + RemoteMeta, + ReqMeta, + ) + from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import ( + TPMapping, + ) + + engine_id = "decode-engine" + w = _StubWriterWorker.fresh() + w.world_size = 2 + w.use_mla = False + w._has_mamba = True + w._is_csa_linear = True + w.nixl_wrapper = MagicMock() + w.transfer_topo = MagicMock() + w.transfer_topo.get_engine_info.return_value = SimpleNamespace( + remote_tp_size=4, + remote_block_size=16, + remote_physical_blocks_per_logical=1, + ) + w.transfer_topo.tp_ratio.return_value = -2 + w.transfer_topo.handshake_target_ranks.return_value = (0, 1) + w.tp_mappings = { + engine_id: TPMapping( + source_ranks_per_group=((0,), (0,), (0, 1), (0, 1)), + all_source_ranks=(0, 1), + rank_to_attention_slot={0: 0, 1: 0}, + rank_offset_factor=0, + ) + } + w._logical_to_kernel_block_ids = lambda block_ids, ratio: block_ids + w.dst_xfer_side_handles = {engine_id: {0: 1000, 1: 1001}} + w.src_xfer_handles_by_tp_ratio = {(-2, 16): [2000, 2001]} + + writes = [] + + def _fake_xfer(**kwargs): + writes.append(kwargs) + return 3000 + kwargs["read_spec"].remote_rank + + w._xfer_blocks = _fake_xfer + block_ids = ([10], [20], [30], [40]) + meta = ReqMeta( + local_block_ids=block_ids, + local_physical_block_ids=block_ids, + tp_size=2, + remote=RemoteMeta( + block_ids=([50], [60], [70], [80]), + host="", + port=0, + engine_id=engine_id, + request_id="d-req", + ), + ) + + w._xfer_blocks_for_req(req_id="p-req", meta=meta) + + assert [write["read_spec"].remote_rank for write in writes] == [0, 1] + assert [write["read_spec"].local_block_ids for write in writes] == [ + list(block_ids), + list(block_ids), + ] + assert [write["local_xfer_side_handle"] for write in writes] == [2000, 2001] + assert sorted(w._sending_transfers["p-req"]) == [3000, 3001] + class TestPushPrefixCaching: """Partial prefix-cache hit on D: D preallocates only its *uncomputed* diff --git a/tests/v1/kv_connector/unit/test_tp_mapping.py b/tests/v1/kv_connector/unit/test_tp_mapping.py index a08da3cd15..52067bc2fa 100644 --- a/tests/v1/kv_connector/unit/test_tp_mapping.py +++ b/tests/v1/kv_connector/unit/test_tp_mapping.py @@ -154,6 +154,9 @@ def _make_mock_worker_for_splits(group_spec_types): worker.block_len_per_layer = [] worker.num_regions = 0 worker._region_is_mla = [] + worker._conv_decomp = SimpleNamespace(local_conv_offsets=()) + worker._ssm_region_indices = [0] if MambaSpec in group_spec_types else [] + worker._ple_region_index = None return worker @@ -282,3 +285,31 @@ class TestMambaPlanSplitHandles: with pytest.raises(AssertionError, match="Head-sharded"): list(worker._build_local_splits_from_plan(plan, src_blocks_data, 2, 2)) + + +@pytest.mark.parametrize( + ("total_kv_heads", "local_tp", "remote_tp", "compatible"), + [ + # 2 heads: TP1 packs both heads into a page, TP >= 2 packs one. + (2, 1, 2, False), + (2, 2, 1, False), + (2, 1, 1, True), + (2, 2, 4, True), + (2, 4, 2, True), + # 4 heads: the boundary moves with the head count. + (4, 2, 4, False), + (4, 2, 2, True), + (4, 4, 8, True), + ], +) +def test_csa_linear_tp_layout_boundary(total_kv_heads, local_tp, remote_tp, compatible): + worker = object.__new__(NixlConnectorWorker) + worker.world_size = local_tp + worker._is_csa_linear = True + worker.transfer_topo = SimpleNamespace(total_num_kv_heads=total_kv_heads) + + if compatible: + worker._validate_csa_linear_tp_layout(remote_tp) + else: + with pytest.raises(ValueError, match="KV-head sharding boundary"): + worker._validate_csa_linear_tp_layout(remote_tp) diff --git a/tests/v1/worker/test_gpu_autoregressive_speculator.py b/tests/v1/worker/test_gpu_autoregressive_speculator.py index b5234a6468..6d8b8cbdbc 100644 --- a/tests/v1/worker/test_gpu_autoregressive_speculator.py +++ b/tests/v1/worker/test_gpu_autoregressive_speculator.py @@ -104,6 +104,47 @@ def _make_speculator( return speculator +@pytest.mark.parametrize(("hc_mult", "expected"), [(None, 64), (4, 256)]) +def test_speculator_uses_draft_model_hidden_size(monkeypatch, hc_mult, expected): + # Qwen4Exp targets expose multi-stream HC residuals to the drafter. + monkeypatch.setattr(base_spec_module, "_target_feeds_hc_residual", lambda _: True) + hf_config = SimpleNamespace() + if hc_mult is not None: + hf_config.hc_mult = hc_mult + draft_model_config = SimpleNamespace( + hf_config=hf_config, + get_hidden_size=lambda: 64, + get_vocab_size=lambda: 32, + ) + speculative_config = SimpleNamespace( + method="mtp", + num_speculative_tokens=3, + draft_model_config=draft_model_config, + use_local_argmax_reduction=False, + draft_sample_method="greedy", + ) + vllm_config = SimpleNamespace( + speculative_config=speculative_config, + scheduler_config=SimpleNamespace( + max_num_seqs=2, + max_num_batched_tokens=8, + ), + model_config=SimpleNamespace( + max_model_len=32, + dtype=torch.float32, + use_fp64_gumbel=False, + ), + parallel_config=SimpleNamespace( + data_parallel_size=1, + data_parallel_rank=0, + ), + ) + + speculator = _TestSpeculator(vllm_config, torch.device("cpu")) + + assert speculator.hidden_size == expected + + def test_mm_support_configured_after_model_load(monkeypatch): target_model_config = object() draft_model_config = object() @@ -197,9 +238,11 @@ def test_load_model_disables_mm_support_for_text_only_drafter(monkeypatch): assert not speculator.supports_mm_inputs assert warning_messages == [ - "Draft model _TextOnlyDraftModel does not support external multimodal " - "embeddings. Embeddings from the target model will not be passed to the " - "drafter; using text-only draft inputs instead." + ( + "Draft model _TextOnlyDraftModel does not support external multimodal " + "embeddings. Embeddings from the target model will not be passed to the " + "drafter; using text-only draft inputs instead." + ) ] diff --git a/tests/v1/worker/test_gpu_block_table.py b/tests/v1/worker/test_gpu_block_table.py index ee44ff24d5..e77156eef9 100644 --- a/tests/v1/worker/test_gpu_block_table.py +++ b/tests/v1/worker/test_gpu_block_table.py @@ -132,6 +132,42 @@ def test_block_tables_apply_staged_writes_single_group(): ) +def test_block_tables_skip_custom_slot_mapping_groups(): + device = torch.device("cuda") + block_tables = BlockTables( + block_sizes=[8, 262144], + max_num_reqs=1, + max_num_batched_tokens=4, + max_num_blocks_per_group=[1, 1], + device=device, + kernel_block_sizes=[8, 262144], + slot_mapping_enabled=[False, True], + ) + block_tables.append_block_ids( + req_index=0, + new_block_ids=([7], [12]), + overwrite=True, + ) + block_tables.apply_staged_writes() + + idx_mapping = torch.tensor([0], dtype=torch.int32, device=device) + query_start_loc = torch.tensor([0, 2], dtype=torch.int32, device=device) + positions = torch.tensor([153797, 165757], dtype=torch.int64, device=device) + slot_mappings = block_tables.compute_slot_mappings( + idx_mapping, + query_start_loc, + positions, + num_tokens_padded=2, + ) + torch.accelerator.synchronize() + + assert slot_mappings[0].tolist() == [-1, -1] + assert slot_mappings[1].tolist() == [ + 12 * 262144 + 153797, + 12 * 262144 + 165757, + ] + + @pytest.mark.parametrize("cp_rank", range(4)) def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int): """DCP interleave is expressed in logical-block token coordinates.""" diff --git a/tests/v1/worker/test_gpu_model_runner_v2.py b/tests/v1/worker/test_gpu_model_runner_v2.py index 8b48993c47..291cbbb312 100644 --- a/tests/v1/worker/test_gpu_model_runner_v2.py +++ b/tests/v1/worker/test_gpu_model_runner_v2.py @@ -8,15 +8,107 @@ import torch import vllm.v1.worker.gpu.model_runner as model_runner_module from vllm.v1.kv_cache_interface import ( + CircularBufferSpec, FullAttentionSpec, KVCacheConfig, KVCacheGroupSpec, MambaSpec, + UniformTypeKVCacheSpecs, ) from vllm.v1.worker.gpu.block_table import BlockTables from vllm.v1.worker.gpu.model_runner import GPUModelRunner +def test_qsa_circular_group_uses_custom_slot_mapping(monkeypatch): + runner = GPUModelRunner.__new__(GPUModelRunner) + runner.max_model_len = 262144 + runner.is_encoder_decoder = False + runner.dcp_size = 1 + runner.dcp_rank = 0 + runner.cp_interleave = 1 + runner.cache_config = SimpleNamespace(enable_prefix_caching=True) + parallel_config = SimpleNamespace( + decode_context_parallel_size=1, + cp_kv_cache_interleave_size=1, + ) + runner.parallel_config = parallel_config + runner.vllm_config = SimpleNamespace( + parallel_config=parallel_config, + cache_config=SimpleNamespace(mamba_cache_mode="none"), + ) + runner.model_state = SimpleNamespace( + get_additional_cg_support=lambda: (), + num_new_sampled_tokens_per_step=1, + ) + runner.speculator = None + runner.req_states = [] + runner.input_buffers = SimpleNamespace(query_start_loc=None) + runner.vocab_size = 1 + runner.max_num_reqs = 1 + runner.max_num_tokens = 2 + runner.device = torch.device("cuda") + + raw_spec = CircularBufferSpec( + block_size=8, + num_kv_heads=1, + head_size=128, + dtype=torch.bfloat16, + ) + compressed_spec = FullAttentionSpec( + block_size=262144, + num_kv_heads=1, + head_size=128, + dtype=torch.bfloat16, + ) + kv_cache_config = KVCacheConfig( + num_blocks=1, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec( + layer_names=["raw"], + kv_cache_spec=UniformTypeKVCacheSpecs( + block_size=8, + kv_cache_specs={"raw": raw_spec}, + ), + ), + KVCacheGroupSpec(layer_names=["compressed"], kv_cache_spec=compressed_spec), + ], + ) + + class FakeAttnCGSupport: + def narrow(self, *args): + return self + + attn_cg_support = FakeAttnCGSupport() + monkeypatch.setattr( + model_runner_module, + "init_attn_backend", + lambda *args: ([], attn_cg_support, [8, 262144]), + ) + monkeypatch.setattr( + model_runner_module, + "maybe_create_adaptive_verification_manager", + lambda **kwargs: None, + ) + + captured = {} + + class BlockTablesCaptured(Exception): + pass + + def capture_block_tables(**kwargs): + captured.update(kwargs) + raise BlockTablesCaptured + + monkeypatch.setattr(model_runner_module, "BlockTables", capture_block_tables) + + with pytest.raises(BlockTablesCaptured): + runner.initialize_kv_cache(kv_cache_config) + + assert captured["max_num_blocks_per_group"] == [1, 1] + assert captured["slot_mapping_enabled"] == [False, True] + + @pytest.mark.parametrize( ("mamba_cache_mode", "num_speculative_blocks", "expected"), [ diff --git a/tests/v1/worker/test_gpu_warmup_blocks.py b/tests/v1/worker/test_gpu_warmup_blocks.py index 11be99f208..6c196a258e 100644 --- a/tests/v1/worker/test_gpu_warmup_blocks.py +++ b/tests/v1/worker/test_gpu_warmup_blocks.py @@ -20,6 +20,7 @@ from vllm.config.vllm import VllmConfig from vllm.platforms import current_platform from vllm.utils.math_utils import cdiv from vllm.v1.kv_cache_interface import ( + CircularBufferSpec, FullAttentionSpec, KVCacheConfig, KVCacheGroupSpec, @@ -53,6 +54,18 @@ def _attention_group() -> KVCacheGroupSpec: ) +def _circular_group() -> KVCacheGroupSpec: + return KVCacheGroupSpec( + ["circular"], + CircularBufferSpec( + block_size=BLOCK_SIZE, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + ), + ) + + def _mamba_group(mamba_cache_mode: str) -> KVCacheGroupSpec: # Name carries the mode so several groups can coexist in one config. return KVCacheGroupSpec( @@ -212,8 +225,7 @@ def test_warmup_reserves_mamba_speculative_blocks(mamba_cache_mode): def _hybrid_kv_cache_config(num_blocks: int) -> KVCacheConfig: - """Full attention plus one Mamba group per cache mode, so a single manager - exercises every branch `_reserved_block_count` has. + """Attention, circular, and Mamba groups exercise every reservation branch. "none" and "all" reach the same branch of both `_reserved_block_count` and `MambaManager`, which tests only for "align". They are still both listed: @@ -226,6 +238,7 @@ def _hybrid_kv_cache_config(num_blocks: int) -> KVCacheConfig: kv_cache_tensors=[], kv_cache_groups=[ _attention_group(), + _circular_group(), _mamba_group("none"), _mamba_group("all"), _mamba_group("align"), diff --git a/tests/v1/worker/test_mamba_utils.py b/tests/v1/worker/test_mamba_utils.py index c31116042f..a162c39a86 100644 --- a/tests/v1/worker/test_mamba_utils.py +++ b/tests/v1/worker/test_mamba_utils.py @@ -1,6 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from collections.abc import Callable from dataclasses import dataclass from typing import Any from unittest.mock import MagicMock, patch @@ -10,11 +9,20 @@ import pytest import torch from vllm.model_executor.layers.mamba.mamba_utils import ( + MambaStateCopyFunc, + MambaStateCopyFuncsByType, get_conv_copy_spec, get_temporal_copy_spec, ) +from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput -from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheGroupSpec, MambaSpec +from vllm.v1.kv_cache_interface import ( + KVCacheConfig, + KVCacheGroupSpec, + MambaSpec, + UniformTypeKVCacheSpecs, +) +from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState from vllm.v1.worker.mamba_utils import ( MambaCopyBuffers, MambaSpecDecodeGPUContext, @@ -22,17 +30,21 @@ from vllm.v1.worker.mamba_utils import ( batch_memcpy, collect_mamba_copy_meta, do_mamba_copy_block, + get_mamba_groups, preprocess_mamba, stage_postprocess_inputs_to_gpu, ) -MambaStateCopyFunc = Callable[..., Any] - # Conv + temporal copy specs, in the order the tests' MambaSpec shapes expect. -_COPY_FUNCS: tuple[MambaStateCopyFunc, ...] = ( +_DEFAULT_COPY_FUNCS: tuple[MambaStateCopyFunc, ...] = ( get_conv_copy_spec, get_temporal_copy_spec, ) +_COPY_FUNCS: MambaStateCopyFuncsByType = { + MambaAttentionBackendEnum.MAMBA2: _DEFAULT_COPY_FUNCS, + MambaAttentionBackendEnum.GDN_ATTN: _DEFAULT_COPY_FUNCS, + MambaAttentionBackendEnum.SHORT_CONV: (get_conv_copy_spec,), +} def postprocess_mamba( @@ -41,7 +53,7 @@ def postprocess_mamba( input_batch: Any, requests: dict[str, Any], forward_context: dict[str, Any], - mamba_state_copy_funcs: tuple[MambaStateCopyFunc, ...], + mamba_state_copy_funcs: MambaStateCopyFuncsByType, copy_bufs: "MambaCopyBuffers", ): """CPU reference for the align-mode postprocess. @@ -147,7 +159,7 @@ def test_resumed_req_ids_cleared_from_mamba_state_idx(): input_batch, {}, # requests {}, # forward_context - (), # mamba_state_copy_funcs + {}, # mamba_state_copy_funcs copy_bufs, ) @@ -430,7 +442,10 @@ def _make_requests( def _make_copy_bufs( - cfg: _TestConfig, kv_cache_config: KVCacheConfig, device: torch.device + cfg: _TestConfig, + kv_cache_config: KVCacheConfig, + device: torch.device, + copy_funcs: MambaStateCopyFuncsByType = _COPY_FUNCS, ) -> MambaCopyBuffers: """Create MambaCopyBuffers for the Python path.""" @@ -440,13 +455,16 @@ def _make_copy_bufs( return MambaCopyBuffers.create( max_num_reqs=cfg.max_num_reqs, kv_cache_config=kv_cache_config, - copy_funcs=(get_conv_copy_spec, get_temporal_copy_spec), + copy_funcs=copy_funcs, make_buffer=make_buffer, ) def _make_gpu_ctx( - cfg: _TestConfig, kv_cache_config: KVCacheConfig, device: torch.device + cfg: _TestConfig, + kv_cache_config: KVCacheConfig, + device: torch.device, + copy_funcs: MambaStateCopyFuncsByType = _COPY_FUNCS, ) -> MambaSpecDecodeGPUContext: """Create MambaSpecDecodeGPUContext for the GPU path.""" @@ -456,12 +474,170 @@ def _make_gpu_ctx( return MambaSpecDecodeGPUContext.create( max_num_reqs=cfg.max_num_reqs, kv_cache_config=kv_cache_config, - num_state_types=2, + copy_funcs=copy_funcs, device=device, make_buffer=make_buffer, ) +def test_mamba_groups_support_different_state_specs(): + gdn_spec = MambaSpec( + block_size=16, + shapes=((4, 4), (2, 4, 4)), + dtypes=(torch.float16, torch.float32), + mamba_type=MambaAttentionBackendEnum.GDN_ATTN, + mamba_cache_mode="align", + num_speculative_blocks=3, + ) + short_conv_spec = MambaSpec( + block_size=16, + shapes=((12, 8),), + dtypes=(torch.float16,), + mamba_type=MambaAttentionBackendEnum.SHORT_CONV, + mamba_cache_mode="align", + num_speculative_blocks=3, + ) + kv_cache_config = KVCacheConfig( + num_blocks=8, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec(["gdn.0", "gdn.1"], gdn_spec), + KVCacheGroupSpec(["ple.0"], short_conv_spec), + ], + ) + + mamba_groups = get_mamba_groups(kv_cache_config) + assert mamba_groups == {gdn_spec: [0], short_conv_spec: [1]} + + model_state = object.__new__(MambaHybridModelState) + model_state._mamba_group_ids = [] + model_state._mamba_spec = None + group_ids, representative_spec = model_state._get_mamba_group_info(kv_cache_config) + assert group_ids == [0, 1] + assert representative_spec is gdn_spec + + cfg = _TestConfig() + copy_bufs = _make_copy_bufs(cfg, kv_cache_config, torch.device("cpu")) + assert copy_bufs.src_ptrs.cpu.numel() == cfg.max_num_reqs * 5 + assert copy_bufs.dst_ptrs.cpu.numel() == cfg.max_num_reqs * 5 + assert copy_bufs.sizes.cpu.numel() == cfg.max_num_reqs * 5 + + ctx = _make_gpu_ctx(cfg, kv_cache_config, torch.device("cpu")) + assert ctx.num_states == 5 + + forward_context = {} + gdn_spec_layer_names = ["gdn.0", "gdn.1"] + for layer_name in gdn_spec_layer_names: + attention = MagicMock() + attention.kv_cache = ( + torch.empty(8, 4, 4, dtype=torch.float16), + torch.empty(8, 2, 4, 4, dtype=torch.float32), + ) + forward_context[layer_name] = attention + ple_attention = MagicMock() + ple_attention.kv_cache = (torch.empty(8, 12, 8, dtype=torch.float16),) + forward_context["ple.0"] = ple_attention + assert gdn_spec_layer_names == kv_cache_config.kv_cache_groups[0].layer_names + + block_tables = [ + torch.zeros(4, 8, dtype=torch.int32), + torch.zeros(4, 8, dtype=torch.int32), + ] + ctx.initialize_from_forward_context( + kv_cache_config, + forward_context, + _COPY_FUNCS, + block_tables, + ) + assert ctx.is_initialized + assert ctx.state_group_indices.tolist() == [0, 0, 0, 0, 1] + assert ctx.state_conv_widths.tolist() == [4, 0, 4, 0, 12] + + +def test_mamba_groups_support_mixed_specs_in_uniform_group(): + gdn_spec = MambaSpec( + block_size=16, + shapes=((4, 4), (2, 4, 4)), + dtypes=(torch.float16, torch.float32), + mamba_type=MambaAttentionBackendEnum.GDN_ATTN, + mamba_cache_mode="align", + num_speculative_blocks=3, + ) + short_conv_spec = MambaSpec( + block_size=16, + shapes=((12, 8),), + dtypes=(torch.float16,), + mamba_type=MambaAttentionBackendEnum.SHORT_CONV, + mamba_cache_mode="align", + num_speculative_blocks=3, + ) + layer_specs = { + "gdn.0": gdn_spec, + "gdn.1": gdn_spec, + "ple.0": short_conv_spec, + } + kv_cache_config = KVCacheConfig( + num_blocks=8, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec( + list(layer_specs), + UniformTypeKVCacheSpecs( + block_size=16, + kv_cache_specs=layer_specs, + ), + ) + ], + ) + + assert kv_cache_config.has_mamba_layers + assert get_mamba_groups(kv_cache_config) == { + gdn_spec: [0], + short_conv_spec: [0], + } + + # The group is listed once per spec, so its id must not be repeated. + model_state = object.__new__(MambaHybridModelState) + model_state._mamba_group_ids = [] + model_state._mamba_spec = None + group_ids, representative_spec = model_state._get_mamba_group_info(kv_cache_config) + assert group_ids == [0] + assert representative_spec is gdn_spec + + cfg = _TestConfig() + copy_bufs = _make_copy_bufs(cfg, kv_cache_config, torch.device("cpu")) + assert copy_bufs.mamba_group_ids == [0] + assert copy_bufs.src_ptrs.cpu.numel() == cfg.max_num_reqs * 5 + + ctx = _make_gpu_ctx(cfg, kv_cache_config, torch.device("cpu")) + assert ctx.mamba_group_ids == [0] + assert ctx.num_states == 5 + + forward_context = { + "gdn.0": MagicMock( + kv_cache=( + torch.empty(8, 4, 4, dtype=torch.float16), + torch.empty(8, 2, 4, 4, dtype=torch.float32), + ) + ), + "gdn.1": MagicMock( + kv_cache=( + torch.empty(8, 4, 4, dtype=torch.float16), + torch.empty(8, 2, 4, 4, dtype=torch.float32), + ) + ), + "ple.0": MagicMock(kv_cache=(torch.empty(8, 12, 8, dtype=torch.float16),)), + } + ctx.initialize_from_forward_context( + kv_cache_config, + forward_context, + _COPY_FUNCS, + [torch.zeros(4, 8, dtype=torch.int32)], + ) + assert ctx.state_group_indices.tolist() == [0, 0, 0, 0, 0] + assert ctx.state_conv_widths.tolist() == [4, 0, 4, 0, 12] + + # ----------------------------------------------------------------------------- # stage_postprocess_inputs_to_gpu: single-pass staging into pinned views # ----------------------------------------------------------------------------- @@ -620,7 +796,7 @@ def _run_gpu_postprocess( *, kv_cache_config: KVCacheConfig, forward_context: dict[str, Any], - copy_funcs: tuple, + copy_funcs: MambaStateCopyFuncsByType, block_table: torch.Tensor, req_ids: list[str], num_accepted_tokens: list[int], diff --git a/vllm/_custom_ops.py b/vllm/_custom_ops.py index e54cba02b0..4d8c1e40a5 100644 --- a/vllm/_custom_ops.py +++ b/vllm/_custom_ops.py @@ -2817,6 +2817,7 @@ def fused_gdn_decode_post_conv_mtp( out: torch.Tensor | None = None, scale: float = 128**-0.5, norm_eps: float = 1e-5, + output_gate_activation: str = "silu", ) -> torch.Tensor: if out is None: out = torch.empty_like(output_gate) @@ -2835,6 +2836,7 @@ def fused_gdn_decode_post_conv_mtp( out, scale, norm_eps, + output_gate_activation, ) return out diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py index 84bce0468c..9989c4d2a0 100644 --- a/vllm/config/compilation.py +++ b/vllm/config/compilation.py @@ -768,6 +768,9 @@ class CompilationConfig: "vllm::mamba_mixer2", "vllm::mamba_mixer", "vllm::short_conv", + "vllm::qwen4_exp_compute_ple_ngram_ids", + "vllm::qwen4_exp_ple_short_conv", + "vllm::qwen4_exp_qsa_with_output", "vllm::linear_attention", "vllm::qwen_gdn_attention_core", "vllm::qwen_gdn_attention_core_fused_norm_packed", diff --git a/vllm/config/scheduler.py b/vllm/config/scheduler.py index 1b82b2c512..bcff098c34 100644 --- a/vllm/config/scheduler.py +++ b/vllm/config/scheduler.py @@ -215,6 +215,10 @@ class SchedulerConfig: # https://github.com/vllm-project/vllm/issues/29585 factors.append(self.max_num_batched_tokens) + # PLE and other model components allocate static per-request buffers. + # Their shapes are captured in compiled graphs. + factors.append(self.max_num_seqs) + hash_str = safe_hash(str(factors).encode(), usedforsecurity=False).hexdigest() return hash_str diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index 564f13cddf..fde84a2622 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -47,6 +47,7 @@ MTPModelTypes = Literal[ "exaone_moe_mtp", "exaone4_5_mtp", "qwen3_next_mtp", + "qwen4_exp_mtp", "qwen3_5_mtp", "longcat_flash_mtp", "bailing_hybrid_v3_mtp", @@ -425,6 +426,9 @@ class SpeculativeConfig: """The specific revision to use for the draft model code on Hugging Face Hub. It can be a branch name, a tag name, or a commit id. If unspecified, will use the default version.""" + index_share_for_mtp_iteration: bool | None = None + """Override whether MTP iterations reuse the first step's sparse indices. + If `None`, use the value from the draft model's Hugging Face config.""" # Advanced control disable_padded_drafter_batch: bool = False @@ -623,6 +627,15 @@ class SpeculativeConfig: # Convert to tuple to make it hashable factors.append(tuple(layer_ids)) + if self.method == "mtp" and self.draft_model_config is not None: + factors.append( + getattr( + self.draft_model_config.hf_config, + "index_share_for_mtp_iteration", + False, + ) + ) + hash_str = safe_hash(str(factors).encode(), usedforsecurity=False).hexdigest() return hash_str @@ -805,6 +818,27 @@ class SpeculativeConfig: hf_config.update( {"n_predict": n_predict, "architectures": ["Qwen3NextMTP"]} ) + if hf_config.model_type in {"qwen4_exp", "qwen4_exp_text"}: + hf_config.model_type = "qwen4_exp_mtp" + if hf_config.model_type == "qwen4_exp_mtp": + text_config = get_hf_text_config(hf_config) + n_predict = getattr( + text_config, + "mtp_num_hidden_layers", + getattr(text_config, "num_nextn_predict_layers", None), + ) + share_mtp_indices = getattr( + text_config, "index_share_for_mtp_iteration", False + ) + hf_config.update( + { + # hc_count is the HC stream multiplier for Qwen MTP feedback. + "hc_mult": int(text_config.hc_count), + "n_predict": n_predict, + "architectures": ["Qwen4ExpMTP"], + "index_share_for_mtp_iteration": share_mtp_indices, + } + ) architectures = getattr(hf_config, "architectures", []) or [] if initial_architecture == "BailingMoeV3ForCausalLM": @@ -1461,6 +1495,15 @@ class SpeculativeConfig: ) ) + if self.index_share_for_mtp_iteration is not None: + if self.method != "mtp" or self.draft_model_config is None: + raise ValueError( + "index_share_for_mtp_iteration is only supported with method='mtp'" + ) + self.draft_model_config.hf_config.index_share_for_mtp_iteration = ( + self.index_share_for_mtp_iteration + ) + if self.method != "dspark" and self.enable_adaptive_verification: raise ValueError("Adaptive verification only supported with DSpark") diff --git a/vllm/distributed/device_communicators/custom_all_reduce.py b/vllm/distributed/device_communicators/custom_all_reduce.py index d1a841e259..6c15a98274 100644 --- a/vllm/distributed/device_communicators/custom_all_reduce.py +++ b/vllm/distributed/device_communicators/custom_all_reduce.py @@ -405,6 +405,8 @@ class CustomAllreduce: def should_custom_ar(self, inp: torch.Tensor): if self.disabled or self.world_size > 8: return False + if inp.dtype not in (torch.float32, torch.float16, torch.bfloat16): + return False inp_size = inp.numel() * inp.element_size() # custom allreduce requires input byte size to be multiples of 16 if inp_size % 16 != 0: diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py index 8cb465b6b1..19b35b8253 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py @@ -72,12 +72,15 @@ from vllm.utils.gpu_sync_debug import gpu_sync_allowed from vllm.utils.network_utils import make_zmq_path from vllm.utils.torch_utils import async_tensor_h2d from vllm.v1.kv_cache_interface import ( + CircularBufferSpec, FullAttentionSpec, KVCacheLayout, + KVCacheSpec, MambaSpec, MLAAttentionSpec, SlidingWindowMLASpec, UniformTypeKVCacheSpecs, + iter_layer_specs, ) from vllm.v1.worker.block_table import BlockTable from vllm.v1.worker.utils import select_common_block_size @@ -118,7 +121,12 @@ class NixlBaseConnectorWorker: # NIXL regions per SSM layer = conv sub-projections + 1 SSM temporal # (Mamba2/GDN: 3+1=4; Mamba1: 1+1=2). ssm_regions_per_layer = len(self._conv_decomp.local_conv_offsets) + 1 - num_ssm_regions = len(self.block_len_per_layer) * ssm_regions_per_layer + # Count only the regions that actually hold SSM state; must match + # the descriptors emitted by _build_mamba_local. + num_ssm_regions = ( + len(self._ssm_region_indices or self.block_len_per_layer) + * ssm_regions_per_layer + ) num_blocks = dst_num_blocks if block_size_ratio is not None: @@ -146,12 +154,19 @@ class NixlBaseConnectorWorker: all_descs: list[np.ndarray] = [] for i, group in enumerate(block_ids): group_arr = np.asarray(group) - if _is_attention_spec(self._group_spec_types[i]): - fa_region_ids = np.arange(self.num_regions)[:, None] + spec_type = self._group_spec_types[i] + if _is_attention_spec(spec_type): + # A scratch cache lives only in its own regions; every other + # attention group spans all of them. + fa_region_ids = ( + np.asarray(self._scratch_region_indices, dtype=np.int64) + if spec_type is CircularBufferSpec + else np.arange(self.num_regions) + )[:, None] all_descs.append( (fa_region_ids * num_blocks + group_arr[None, :]).flatten() ) - elif _is_ssm_spec(self._group_spec_types[i]): + elif _is_ssm_spec(spec_type): # NOTE (NickLucche) SSM and Attention block regions can # be exchanged arbitrarily by manager. Therefore, descs # are laid out as: @@ -159,7 +174,11 @@ class NixlBaseConnectorWorker: # num_fa_descs offset must be computed per-engine since # P and D can have different num_blocks (and thus # different FA desc counts). - ssm_region_ids = np.arange(num_ssm_regions)[:, None] + ssm_region_ids = ( + np.arange(num_ssm_regions, num_ssm_regions + 1) + if i == self._ple_group_index + else np.arange(num_ssm_regions) + )[:, None] all_descs.append( ( ssm_region_ids * logical_blocks @@ -206,6 +225,10 @@ class NixlBaseConnectorWorker: # Per-FA-descriptor replicate flag, in _build_fa_local emission order. fa_desc_replicated = self._fa_desc_replicated(num_fa_descs) + sharded_desc_end = len(src_blocks_data) - ( + self._logical_num_blocks if self._ple_region_index is not None else 0 + ) + assert num_fa_descs <= sharded_desc_end assert block_size_ratio == 1 or fa_num_splits == 1 or all(fa_desc_replicated), ( "Head-sharded attention reads with P_TP > D_TP and heterogeneous " @@ -226,9 +249,11 @@ class NixlBaseConnectorWorker: # SPLIT (full-attn): this rank's head slice. chunk = local_len // fa_num_splits handle.append((addr + fa_slot * chunk, chunk, dev)) - else: + elif j < sharded_desc_end: chunk = local_len // ssm_num_splits handle.append((addr + p_idx * chunk, chunk, dev)) + else: + handle.append((addr, local_len, dev)) yield handle def _needs_split_local_xfer_handles(self, tp_ratio: int, plan: TPMapping) -> bool: @@ -307,19 +332,22 @@ class NixlBaseConnectorWorker: ) ) + self.kv_cache_config = kv_cache_config + # Per-layer specs, unwrapping UniformTypeKVCacheSpecs group wrappers. + self._layer_specs: dict[str, KVCacheSpec] = {} + for group in kv_cache_config.transfer_groups: + group_spec = group.kv_cache_spec + if isinstance(group_spec, UniformTypeKVCacheSpecs): + self._layer_specs.update(group_spec.kv_cache_specs) + else: + self._layer_specs.update(dict.fromkeys(group.layer_names, group_spec)) self._is_hma_required = ( not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager and any( - not isinstance(g.kv_cache_spec, FullAttentionSpec) - for g in kv_cache_config.transfer_groups + not isinstance(spec, FullAttentionSpec) + for spec in self._layer_specs.values() ) ) - self.kv_cache_config = kv_cache_config - self._layer_specs = { - layer: group.kv_cache_spec - for group in kv_cache_config.transfer_groups - for layer in group.layer_names - } # ---- Model state (derived from model config) ---- mamba_ssm_size = (0, 0) @@ -328,9 +356,29 @@ class NixlBaseConnectorWorker: # conv sub-projections are contiguous in memory. self._conv_decomp: MambaConvSplitInfo | None = None self._has_mamba = any( - isinstance(g.kv_cache_spec, MambaSpec) - for g in kv_cache_config.transfer_groups + isinstance(spec, MambaSpec) for spec in self._layer_specs.values() ) + self._is_csa_linear = any( + get_representative_spec_type(group.kv_cache_spec) is CircularBufferSpec + for group in kv_cache_config.kv_cache_groups + ) + self._ple_group_index: int | None = None + if self._is_csa_linear: + ple_groups = [ + (index, group) + for index, group in enumerate(kv_cache_config.kv_cache_groups) + if group.layer_names + and all( + isinstance(spec, MambaSpec) and spec.tp_replicated + for spec in iter_layer_specs(group.kv_cache_spec) + ) + ] + if len(ple_groups) != 1 or len(ple_groups[0][1].layer_names) != 1: + raise ValueError( + "CSA-linear NIXL requires exactly one PLE cache owner." + ) + self._ple_group_index = ple_groups[0][0] + if self._has_mamba: assert self._is_hma_required from vllm.model_executor.layers.mamba.mamba_utils import ( @@ -345,6 +393,7 @@ class NixlBaseConnectorWorker: spec for spec in self._layer_specs.values() if isinstance(spec, MambaSpec) + and (not self._is_csa_linear or not spec.tp_replicated) ) self._conv_decomp = derive_mamba_conv_split( mamba_spec, @@ -579,6 +628,12 @@ class NixlBaseConnectorWorker: # (MLA), False -> SPLIT (head-sharded full-attn). Mixed only for models # combining both (e.g. GQA main + MLA Eagle-3 draft). self._region_is_mla = list[bool]() + self._ssm_region_indices = list[int]() + # Regions holding a scratch cache (the CSA compressor circular buffer). + # Tracked explicitly because a scratch page shares its address with the + # page it overlays, so its regions cannot be recovered from spec type. + self._scratch_region_indices = list[int]() + self._ple_region_index: int | None = None # Enable different block lengths for different layers *only* when MLA is used. # This is not used for SSM layers, which use the counterpart `mamba_ssm_size`. @@ -632,6 +687,24 @@ class NixlBaseConnectorWorker: self.block_size = kernel_block_size self.num_blocks *= self._physical_blocks_per_logical_kv_block + def _validate_csa_linear_tp_layout(self, remote_tp_size: int) -> None: + """Reject P/D pairs whose main-KV pages have different head layouts. + + A rank at TP < total KV heads holds several heads per page, while a + rank at TP >= total KV heads holds exactly one (replicated beyond + that). CSA-linear pages are shared tensors that transfer as whole + block units, so endpoints on opposite sides of that boundary lay + them out differently and cannot exchange them. + """ + assert self.transfer_topo is not None + total_kv_heads = self.transfer_topo.total_num_kv_heads + if (self.world_size < total_kv_heads) != (remote_tp_size < total_kv_heads): + raise ValueError( + "CSA-linear NIXL requires both endpoints on the same side of " + f"the KV-head sharding boundary ({total_kv_heads} KV heads): " + f"got local TP {self.world_size}, remote TP {remote_tp_size}." + ) + def _nixl_handshake( self, host: str, @@ -643,6 +716,8 @@ class NixlBaseConnectorWorker: notif_agents_only: bool = False, ) -> tuple[dict[tuple[int, int], str], float]: """Do a NIXL handshake with a remote instance.""" + if self._is_csa_linear: + self._validate_csa_linear_tp_layout(remote_tp_size) # the first time we connect to a remote agent. # be careful, the handshake happens in a background thread. @@ -1034,6 +1109,11 @@ class NixlBaseConnectorWorker: transfer_mode=self._TRANSFER_MODE, ) + if self._is_csa_linear and self.use_host_buffer: + raise NotImplementedError( + "NIXL host staging does not preserve CSA-linear shared tensors." + ) + if self.use_host_buffer: self.initialize_host_xfer_buffer(kv_caches=kv_caches) assert len(self.host_xfer_buffers) == len(kv_caches), ( @@ -1059,8 +1139,14 @@ class NixlBaseConnectorWorker: caches_data = [] seen_storage_addresses: set[int] = set() seen_base_addresses: list[int] = [] + self._ssm_region_indices = [] + self._scratch_region_indices = [] + self._ple_region_index = None packed_storage = _share_storage_and_block_stride(list(xfer_buffers.values())) + # CSA-linear needs separate logical regions for attention and state + # aliases even though every view shares one block-major allocation. + packed_storage = packed_storage and not self._is_csa_linear # K and V are packed into the content dim, so each attention layer is a # single NIXL region whose block transfers as one unit. Mamba layers instead @@ -1112,14 +1198,16 @@ class NixlBaseConnectorWorker: ) if isinstance(layer_spec, MambaSpec): + physical_ratio = self._physical_blocks_per_logical_kv_block + block_len = physical_page_size // physical_ratio + block_stride = physical_page_size + if self._is_csa_linear and cache.ndim != 1: + # CSA-linear packs multiple cache owners in each block, so + # state views advance by the packed block stride. + block_stride = cache.stride(0) * cache.element_size() + assert block_stride % physical_ratio == 0 region_specs = [ - ( - cache.data_ptr(), - physical_page_size - // self._physical_blocks_per_logical_kv_block, - physical_page_size - // self._physical_blocks_per_logical_kv_block, - ) + (cache.data_ptr(), block_len, block_stride // physical_ratio) ] else: if cache.ndim == 1: @@ -1131,19 +1219,18 @@ class NixlBaseConnectorWorker: block_stride = cache.stride(0) * cache.element_size() storage_is_block_major = num_blocks * block_stride == storage.nbytes() # A layer whose [H, N, C] interior is dense addresses its own page - # as one chunk, so it can own a region. When it does not, or when - # every layer is packed into this one storage, the block's whole - # row is the only contiguous unit. + # as one chunk. Otherwise the block's whole row is the only + # contiguous transfer unit. hnc_contiguous = ( cache.ndim == 4 and cache.stride(2) == cache.shape[3] and cache.stride(1) == cache.shape[2] * cache.shape[3] ) if storage_is_block_major and ( - (packed_storage and is_mla_region) or not hnc_contiguous + (packed_storage and is_mla_region) + or (not hnc_contiguous and not self._is_csa_linear) ): - # TODO(Lucas): handle TP slicing for packed_storage; for now - # restrict to MLA (DSv4) where kv is replicated. + # Packed MLA layouts transfer the complete storage row. storage_block_len = storage.nbytes() // num_blocks region_specs = [ (storage_addr, storage_block_len, storage_block_len) @@ -1170,17 +1257,27 @@ class NixlBaseConnectorWorker: for base_addr, block_len, block_stride in region_specs: if base_addr in seen_base_addresses: - if is_mla_region: - # Dual-purpose HMA tensor: an MLA layer shares a region - # that a non-MLA layer registered first. MLA is not - # head-sharded, so the region must be flagged MLA. - idx = seen_base_addresses.index(base_addr) - self._region_is_mla[idx] = True - continue - seen_base_addresses.append(base_addr) - self.block_len_per_layer.append(block_len) - self.block_stride_per_layer.append(block_stride) - self._region_is_mla.append(is_mla_region) + region_index = seen_base_addresses.index(base_addr) + self._region_is_mla[region_index] |= is_mla_region + else: + region_index = len(seen_base_addresses) + seen_base_addresses.append(base_addr) + self.block_len_per_layer.append(block_len) + self.block_stride_per_layer.append(block_stride) + self._region_is_mla.append(is_mla_region) + + if isinstance(layer_spec, MambaSpec): + if layer_spec.tp_replicated: + assert self._is_csa_linear + assert self._ple_region_index in (None, region_index) + self._ple_region_index = region_index + elif region_index not in self._ssm_region_indices: + self._ssm_region_indices.append(region_index) + elif ( + isinstance(layer_spec, CircularBufferSpec) + and region_index not in self._scratch_region_indices + ): + self._scratch_region_indices.append(region_index) # When there's a mismatch between kbs<>bs, we rely on HMA to ensure # caches are either [NB, PS] or [NB*r, PS/r] where r is bs/kbs. @@ -1210,6 +1307,8 @@ class NixlBaseConnectorWorker: == len(self._region_is_mla) == len(self.block_stride_per_layer) ) + # Descriptor ids must be region-ordered, matching the remote side. + self._scratch_region_indices.sort() self.kv_caches_base_addr[self.engine_id][self.tp_rank] = seen_base_addresses self.num_regions = len(seen_base_addresses) @@ -1324,17 +1423,26 @@ class NixlBaseConnectorWorker: physical_per_logical = self._physical_blocks_per_logical_kv_block device_id = self.device_id block_arange = np.arange(num_blocks, dtype=np.uint64) - parts: list[np.ndarray] = [] - for i, base_addr in enumerate(base_addresses): - # Jump one page_size, but ssm page_size may be bigger when kernel - # locks block size to a specific value (physical_per_logical scale). - page_stride = self.block_len_per_layer[i] * physical_per_logical - blk_addrs = base_addr + block_arange * page_stride + + region_indices = self._ssm_region_indices or range(len(base_addresses)) + for i in region_indices: + base_addr = base_addresses[i] + block_stride = self.block_stride_per_layer[i] * physical_per_logical + blk_addrs = base_addr + block_arange * block_stride for off, sz in conv_offsets: parts.append(self._stack_descs(blk_addrs + off, sz, device_id)) # SSM temporal state follows the conv state. parts.append(self._stack_descs(blk_addrs + conv_size, ssm_size, device_id)) + + if (region_index := self._ple_region_index) is not None: + block_len = self.block_len_per_layer[region_index] * physical_per_logical + block_stride = ( + self.block_stride_per_layer[region_index] * physical_per_logical + ) + block_addrs = base_addresses[region_index] + block_arange * block_stride + parts.append(self._stack_descs(block_addrs, block_len, self.device_id)) + return np.concatenate(parts) def _build_mamba_remote( @@ -1370,14 +1478,44 @@ class NixlBaseConnectorWorker: parts: list[np.ndarray] = [] # NOTE (ZhanqiuHu): use per-layer block_lens[i], not [0], in case # block lengths vary across layers (e.g. MLA). - for i, base_addr in enumerate(nixl_agent_meta.kv_caches_base_addr): - page_stride = nixl_agent_meta.block_lens[i] * remote_physical_per_logical - blk_addrs = base_addr + block_arange * page_stride + region_indices = self._ssm_region_indices or range( + len(nixl_agent_meta.kv_caches_base_addr) + ) + for i in region_indices: + base_addr = nixl_agent_meta.kv_caches_base_addr[i] + block_stride = ( + nixl_agent_meta.block_strides[i] * remote_physical_per_logical + ) + blk_addrs = base_addr + block_arange * block_stride for off, sz in conv_offsets: parts.append(self._stack_descs(blk_addrs + off, sz, device_id)) # SSM temporal state is also TP-sharded on the heads dimension. ssm_addrs = blk_addrs + conv_size_remote + local_offset * ssm_read_size parts.append(self._stack_descs(ssm_addrs, ssm_read_size, device_id)) + + if (region_index := self._ple_region_index) is not None: + local_block_len = ( + self.block_len_per_layer[region_index] + * self._physical_blocks_per_logical_kv_block + ) + remote_block_len = ( + nixl_agent_meta.block_lens[region_index] * remote_physical_per_logical + ) + if local_block_len != remote_block_len: + raise ValueError( + "PLE pages require identical P/D geometry: " + f"local={local_block_len}, remote={remote_block_len}." + ) + remote_block_stride = ( + nixl_agent_meta.block_strides[region_index] + * remote_physical_per_logical + ) + block_addrs = ( + nixl_agent_meta.kv_caches_base_addr[region_index] + + block_arange * remote_block_stride + ) + parts.append(self._stack_descs(block_addrs, remote_block_len, device_id)) + return np.concatenate(parts) @staticmethod @@ -2410,8 +2548,7 @@ class NixlBaseConnectorWorker: group_specs = self.kv_cache_config.transfer_groups physical_block_ids = [] for i, group in enumerate(block_ids): - spec = group_specs[i].kv_cache_spec - if isinstance(spec, MambaSpec): + if _is_ssm_spec(get_representative_spec_type(group_specs[i].kv_cache_spec)): physical_block_ids.append(group) else: physical_block_ids.append( diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py index 78fe49bb57..9d62dfa563 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py @@ -536,7 +536,8 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): def group_ids(block_ids: BlockIds, rank: int) -> BlockIds: return [ list(block_ids[g]) - if (replicate_attn and _is_attention_spec(self._group_spec_types[g])) + if (self._is_csa_linear and tp_ratio < 0) + or (replicate_attn and _is_attention_spec(self._group_spec_types[g])) or rank in plan.source_ranks_per_group[g] else [] for g in range(num_groups) diff --git a/vllm/model_executor/layers/fused_moe/experts/triton_moe.py b/vllm/model_executor/layers/fused_moe/experts/triton_moe.py index 9887423e79..9f982644b5 100644 --- a/vllm/model_executor/layers/fused_moe/experts/triton_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/triton_moe.py @@ -39,6 +39,7 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import ( is_deep_gemm_e8m0_used, ) from vllm.model_executor.layers.quantization.utils.quant_utils import ( + FP8_DTYPE, QuantKey, kFp8Dynamic128Sym, kFp8DynamicTensorSym, @@ -153,6 +154,19 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular): (kFp8StaticTensorSym, kFp8StaticTensorSym), (kFp8StaticTensorSym, kFp8DynamicTensorSym), ] + # Block-quantized FP8 with arbitrary block shape: the Triton + # kernels take the block shape as a runtime argument. + if ( + weight_key is not None + and activation_key == kFp8Dynamic128Sym + and weight_key.dtype == FP8_DTYPE + and weight_key.symmetric + and weight_key.scale.static + and weight_key.scale.dtype == torch.float32 + and weight_key.scale.group_shape.row > 1 + and weight_key.scale.group_shape.col > 1 + ): + return True return (weight_key, activation_key) in supported @staticmethod diff --git a/vllm/model_executor/layers/fused_moe/oracle/fp8.py b/vllm/model_executor/layers/fused_moe/oracle/fp8.py index b2d76e4ef9..abe47373de 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/fp8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/fp8.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import math from enum import Enum from typing import Any @@ -268,6 +269,39 @@ def map_fp8_backend(runner_backend: MoEBackend) -> Fp8MoeBackend: ) +def refine_fp8_moe_block_shape( + config: FusedMoEConfig, + weight_block_size: list[int], +) -> list[int] | None: + """ + Compute a refined block shape for block-quantized FP8 MoE weights whose + checkpoint blocks cannot be sharded exactly across TP ranks. + + TP shards the intermediate dim of the expert weights, so a per-shard size + that is not a multiple of the checkpoint's block size makes the + checkpoint's block scales impossible to shard exactly. When a finer block + size (>= 32) divides both the checkpoint blocks and all involved dims, + the weight scales can be refined to that granularity at load time (a + lossless upsampling, since the refined block divides the checkpoint + block). Only Triton-based kernels can consume the refined block shape: + they take it as a runtime argument, while the other backends require the + native 128x128 blocks. The refined shape is encoded in the QuantKey used + for backend selection, so backends that only support 128x128 blocks are + rejected by the oracle automatically. + + Returns the refined [block_n, block_k] shape, or None if no refinement + is needed or possible. + """ + block_n, block_k = weight_block_size + ispp = config.intermediate_size_per_partition + if ispp % block_n == 0 and (config.tp_size == 1 or ispp % block_k == 0): + return None + refine = math.gcd(block_n, block_k, ispp, config.hidden_dim) + if refine < 32: + return None + return [refine, refine] + + def select_fp8_moe_backend( config: FusedMoEConfig, weight_key: QuantKey | None, diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py index 4a8c723278..f0de8522fa 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py @@ -474,6 +474,39 @@ def _get_requested_backends( return _filter_by_activation(backends, requested_activation_key) +def _requires_qwen38_tep8_emulation( + config: FusedMoEConfig, + activation_key: QuantKey | None, +) -> bool: + """Avoid the inaccurate native gfx950 kernel for Qwen3.8 TEP8. + + Qwen3.8 Flash Next's routed experts use the distinctive + ``E=512, H=2560, N=640`` W4A4 shape. With eight-way expert parallelism, + AITER operates on 64 local experts and pads ``N`` to 768. That native path + is not numerically reliable on gfx950, while OCP MX emulation preserves + model accuracy. Keep this guard exact so other MXFP4 shapes and smaller EP + configurations retain the native backend. + """ + parallel = config.moe_parallel_config + if not ( + activation_key == kMxfp4Dynamic + and parallel.use_ep + and parallel.ep_size == 8 + and config.num_experts == 512 + and config.num_local_experts == 64 + and config.hidden_dim == 2560 + and config.intermediate_size == 640 + ): + return False + + if not current_platform.is_rocm(): + return False + + from vllm.platforms.rocm import on_gfx950 + + return on_gfx950() + + def select_mxfp4_moe_backend( config: FusedMoEConfig, activation_key: QuantKey | None = None, @@ -532,6 +565,21 @@ def select_mxfp4_moe_backend( assert last_error is not None raise last_error + if _requires_qwen38_tep8_emulation(config, requested_activation_key): + backend = Mxfp4MoeBackend.EMULATION + logger.warning_once( + "Using OCP MX emulation for the Qwen3.8 Flash Next TEP8 routed " + "experts on gfx950 because the native AITER W4A4 kernel is not " + "numerically reliable for this shape. Performance will be lower." + ) + return _return_or_raise( + backend, + config, + kMxfp4Static, + requested_activation_key, + activation_format, + ) + # Select kernels in order of backend. AVAILABLE_BACKENDS = _filter_by_activation( _get_priority_backends_for_gpt_oss(), requested_activation_key diff --git a/vllm/model_executor/layers/fused_moe/oracle/unquantized.py b/vllm/model_executor/layers/fused_moe/oracle/unquantized.py index cf884df501..5d9b69529b 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/unquantized.py +++ b/vllm/model_executor/layers/fused_moe/oracle/unquantized.py @@ -345,14 +345,13 @@ def convert_to_unquantized_kernel_format( elif unquantized_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM: is_act_and_mul = moe_config.is_act_and_mul - if not is_act_and_mul: - # Kernel requires intermediate_size_per_partition % 128 == 0 (BlockMajorK - # weight layout uses block_k=128). Pad along the intermediate dim when - # the model + TP split don't satisfy the constraint. - w13_weight, w2_weight, padded_intermediate = align_moe_weights_for_fi( - w13_weight, w2_weight, is_act_and_mul, min_alignment=128 - ) - moe_config.intermediate_size_per_partition = padded_intermediate + # Kernel requires intermediate_size_per_partition % 128 == 0 (BlockMajorK + # weight layout uses block_k=128). Pad along the intermediate dim when + # the model + TP split don't satisfy the constraint. + w13_weight, w2_weight, padded_intermediate = align_moe_weights_for_fi( + w13_weight, w2_weight, is_act_and_mul, min_alignment=128 + ) + moe_config.intermediate_size_per_partition = padded_intermediate _cache_permute_indices: dict[torch.Size, torch.Tensor] = {} w13_weight, w2_weight = convert_moe_weights_to_flashinfer_trtllm_block_layout( diff --git a/vllm/model_executor/layers/fused_moe/routed_experts.py b/vllm/model_executor/layers/fused_moe/routed_experts.py index 574d165ef7..59562ed8be 100644 --- a/vllm/model_executor/layers/fused_moe/routed_experts.py +++ b/vllm/model_executor/layers/fused_moe/routed_experts.py @@ -832,6 +832,20 @@ class RoutedExperts(PluggableLayer): FusedMoeWeightScaleSupported.GROUP.value, FusedMoeWeightScaleSupported.BLOCK.value, ]: + scale_refine = getattr(self.quant_method, "weight_scale_refine", None) + if ( + quant_method == FusedMoeWeightScaleSupported.BLOCK.value + and scale_refine is not None + ): + # FP8 block scales are stored per (block_n, block_k) tile + # of the unsharded weight, while the TP-sharded parameters + # use a refined block grid (see Fp8MoEMethod). Upsample the + # scales to the refined grid (lossless: the refined block + # divides the checkpoint block) so per-rank slicing stays + # exact. Dim -2 is the weight's N dim, dim -1 is K. + loaded_weight = loaded_weight.repeat_interleave( + scale_refine[0], dim=-2 + ).repeat_interleave(scale_refine[1], dim=-1) self._load_model_weight_or_group_weight_scale( shard_id=shard_id, shard_dim=shard_dim, diff --git a/vllm/model_executor/layers/mamba/abstract.py b/vllm/model_executor/layers/mamba/abstract.py index ef350237dc..10ed7d05fc 100644 --- a/vllm/model_executor/layers/mamba/abstract.py +++ b/vllm/model_executor/layers/mamba/abstract.py @@ -56,6 +56,10 @@ class MambaBase(AttentionLayerBase): def mamba_type(self) -> MambaAttentionBackendEnum: pass + @property + def is_kv_cache_tp_replicated(self) -> bool: + return False + @abstractmethod def get_state_dtype(self) -> tuple[torch.dtype, ...]: pass @@ -70,6 +74,7 @@ class MambaBase(AttentionLayerBase): block_size=mamba_block_size, page_size_padded=page_size_padded, mamba_type=self.mamba_type, + tp_replicated=self.is_kv_cache_tp_replicated, mamba_cache_mode=vllm_config.cache_config.mamba_cache_mode, # RecoverSSM verifies the whole window off one checkpoint, so it # never writes the baseline's per-draft-token state slots. diff --git a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py index 84ea15d1a6..e733cd9a78 100644 --- a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py +++ b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py @@ -519,7 +519,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention): self.gqa_interleaved_layout or self.head_k_dim != 128 or self.head_v_dim != 128 - or self.norm.activation != "silu" + or self.norm.activation not in ("silu", "sigmoid") or vllm_config.model_config.dtype != torch.bfloat16 or conv_state_dtype != torch.bfloat16 or recurrent_state_dtype not in FUSED_GDN_STATE_DTYPES @@ -527,9 +527,9 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention): ): return ( "the fused CUDA kernel requires a BF16 GDN model with " - "K=V=128, SiLU gating, non-interleaved GQA layout, BF16 " - "convolution cache, BF16 or FP32 recurrent state, and a " - "GPU with compute capability 8.0+" + "K=V=128, SiLU or sigmoid gating, non-interleaved GQA " + "layout, BF16 convolution cache, BF16 or FP32 recurrent " + "state, and a GPU with compute capability 8.0+" ) if not hasattr(torch.ops._C, "fused_gdn_decode_post_conv_mtp"): return "torch.ops._C.fused_gdn_decode_post_conv_mtp is not built" @@ -1766,6 +1766,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention): out=core_attn_out, scale=self.head_k_dim**-0.5, norm_eps=self.layer_norm_epsilon, + output_gate_activation=self.norm.activation, ) def _forward_core_fused_norm_packed( diff --git a/vllm/model_executor/layers/mamba/mamba_utils.py b/vllm/model_executor/layers/mamba/mamba_utils.py index 71872de47c..99059c6b54 100644 --- a/vllm/model_executor/layers/mamba/mamba_utils.py +++ b/vllm/model_executor/layers/mamba/mamba_utils.py @@ -17,6 +17,7 @@ from vllm.utils.torch_utils import ( STR_DTYPE_TO_TORCH_DTYPE, get_kv_cache_torch_dtype, ) +from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum logger = init_logger(__name__) @@ -352,6 +353,10 @@ Parameters: num_accepted_tokens: int - number of accepted tokens used to compute the copy offset. Range: 1 .. 1 + num_speculative_tokens (inclusive). """ +MambaStateCopyFuncs: TypeAlias = tuple[MambaStateCopyFunc, ...] +MambaStateCopyFuncsByType: TypeAlias = dict[ + MambaAttentionBackendEnum, MambaStateCopyFuncs +] def get_conv_copy_spec( diff --git a/vllm/model_executor/layers/quantization/fp8.py b/vllm/model_executor/layers/quantization/fp8.py index 20699484fc..7d74bd9876 100644 --- a/vllm/model_executor/layers/quantization/fp8.py +++ b/vllm/model_executor/layers/quantization/fp8.py @@ -31,6 +31,7 @@ from vllm.model_executor.layers.fused_moe.oracle.fp8 import ( convert_to_fp8_moe_kernel_format, make_fp8_moe_kernel, make_fp8_moe_quant_config, + refine_fp8_moe_block_shape, select_fp8_moe_backend, ) from vllm.model_executor.layers.linear import ( @@ -63,7 +64,6 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( kFp8Dynamic128Sym, kFp8DynamicTensorSym, kFp8DynamicTokenSym, - kFp8Static128BlockSym, kFp8StaticTensorSym, ) from vllm.model_executor.layers.quantization.utils.w8a8_utils import ( @@ -481,9 +481,41 @@ class Fp8MoEMethod(FusedMoEMethodBase): "weight_scale_inv" if self.block_quant else "weight_scale" ) + self.weight_scale_refine: tuple[int, int] | None = None + self.moe_block_shape = self.weight_block_size + # Set weight key and activation key for kernel compatibility if self.block_quant: - weight_key = kFp8Static128BlockSym + assert self.weight_block_size is not None + # TP shards the intermediate dim of the expert weights, so a + # per-shard size that is not a multiple of the checkpoint's block + # size makes the checkpoint's block scales impossible to shard + # exactly. When a finer block size (>= 32) divides both the + # checkpoint blocks and all involved dims, the weight scales are + # refined to that granularity at load time (a lossless upsampling, + # since the refined block divides the checkpoint block). The + # refined block shape is encoded in the weight key, so the oracle + # only selects kernels that support it (e.g. Triton, which takes + # the block shape as a runtime argument). + refined_shape = refine_fp8_moe_block_shape(self.moe, self.weight_block_size) + if refined_shape is not None: + block_n, block_k = self.weight_block_size + self.weight_scale_refine = ( + block_n // refined_shape[0], + block_k // refined_shape[1], + ) + self.moe_block_shape = refined_shape + logger.info_once( + "FP8 MoE block scales refined from %s to %s to fit " + "the TP-sharded intermediate size %d.", + str(self.weight_block_size), + str(refined_shape), + self.moe.intermediate_size_per_partition, + ) + assert self.moe_block_shape is not None + weight_key = create_fp8_quant_key( + static=True, group_shape=GroupShape(*self.moe_block_shape) + ) activation_key = kFp8Dynamic128Sym else: weight_key = kFp8StaticTensorSym @@ -519,6 +551,8 @@ class Fp8MoEMethod(FusedMoEMethodBase): if self.block_quant: assert self.weight_block_size is not None + assert self.moe_block_shape is not None + moe_block_shape = self.moe_block_shape layer.weight_block_size = self.weight_block_size tp_size = get_tensor_model_parallel_world_size() block_n, block_k = ( @@ -530,18 +564,24 @@ class Fp8MoEMethod(FusedMoEMethodBase): # layers must be divisible by block_n. # Required by column parallel or enabling merged weights if intermediate_size_per_partition % block_n != 0: - raise ValueError( - f"The output_size of gate's and up's weight = " - f"{intermediate_size_per_partition} is not divisible by " - f"weight quantization block_n = {block_n}." - ) + if self.weight_scale_refine is None: + raise ValueError( + f"The output_size of gate's and up's weight = " + f"{intermediate_size_per_partition} is not divisible by " + f"weight quantization block_n = {block_n}." + ) + # Use the refined block grid for the scale parameters; the + # loader upsamples the checkpoint scales accordingly. + block_n, block_k = moe_block_shape if tp_size > 1 and intermediate_size_per_partition % block_k != 0: # Required by row parallel - raise ValueError( - f"The input_size of down's weight = " - f"{intermediate_size_per_partition} is not divisible by " - f"weight quantization block_k = {block_k}." - ) + if self.weight_scale_refine is None: + raise ValueError( + f"The input_size of down's weight = " + f"{intermediate_size_per_partition} is not divisible by " + f"weight quantization block_k = {block_k}." + ) + block_n, block_k = moe_block_shape # WEIGHTS w13_weight = torch.nn.Parameter( @@ -747,7 +787,7 @@ class Fp8MoEMethod(FusedMoEMethodBase): w2_scale=w2_scale, a1_scale=a1_scale, a2_scale=a2_scale, - block_shape=self.weight_block_size, + block_shape=self.moe_block_shape, swiglu_limit=getattr(layer, "swiglu_limit", None), gemm1_alpha=getattr(layer, "swiglu_alpha", None), gemm1_beta=getattr(layer, "swiglu_beta", None), diff --git a/vllm/model_executor/layers/quantization/quark/schemes/quark_ocp_mx.py b/vllm/model_executor/layers/quantization/quark/schemes/quark_ocp_mx.py index 59499a1060..cea1c55ac5 100644 --- a/vllm/model_executor/layers/quantization/quark/schemes/quark_ocp_mx.py +++ b/vllm/model_executor/layers/quantization/quark/schemes/quark_ocp_mx.py @@ -161,6 +161,16 @@ class QuarkOCP_MX(QuarkScheme): weight_loader: Callable, **kwargs, ): + if input_size_per_partition % OCP_MX_BLOCK_SIZE != 0: + layer_name = getattr(layer, "prefix", "") or type(layer).__name__ + raise ValueError( + f"OCP MX linear layer {layer_name!r} has an input size per " + f"partition of {input_size_per_partition}, which must be " + f"divisible by the OCP MX group size {OCP_MX_BLOCK_SIZE}. " + "Choose a compatible tensor-parallel size or avoid " + "tensor-parallel sharding for this layer." + ) + if self.dynamic_mxfp4_quant: weight = ModelWeightParameter( data=torch.empty( diff --git a/vllm/model_executor/layers/quantization/utils/config_utils.py b/vllm/model_executor/layers/quantization/utils/config_utils.py index 33247c4440..4f3587841d 100644 --- a/vllm/model_executor/layers/quantization/utils/config_utils.py +++ b/vllm/model_executor/layers/quantization/utils/config_utils.py @@ -9,6 +9,41 @@ if TYPE_CHECKING: from vllm.model_executor.layers.quantization.base_config import QuantizationConfig +def get_quark_ocp_mx_group_size( + quant_config: "QuantizationConfig | None", + layer_name: str, +) -> int | None: + """Return the OCP MX group size for a quantized Quark linear layer.""" + if quant_config is None: + return None + + from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig + from vllm.model_executor.layers.quantization.quark.utils import should_ignore_layer + + if not isinstance(quant_config, QuarkConfig): + return None + + if should_ignore_layer( + layer_name, + ignore=quant_config.quant_config.get("exclude") or [], + fused_mapping=quant_config.packed_modules_mapping, + ): + return None + + layer_quant_config = ( + quant_config.get_layer_quant_config_from_name(layer_name) + or quant_config.quant_config.get("global_quant_config") + or {} + ) + weight_quant = layer_quant_config.get("weight") + input_quant = layer_quant_config.get("input_tensors") + if not quant_config._is_w_ocp_mx_a_x(weight_quant, input_quant): + return None + + assert weight_quant is not None + return int(weight_quant["group_size"]) + + def is_shared_expert_quant_fse_compatible( quant_config: "QuantizationConfig | None", expert_prefix: str, diff --git a/vllm/model_executor/layers/quantization/utils/flashinfer_utils.py b/vllm/model_executor/layers/quantization/utils/flashinfer_utils.py index 28c3bbae4c..7bccc373dd 100644 --- a/vllm/model_executor/layers/quantization/utils/flashinfer_utils.py +++ b/vllm/model_executor/layers/quantization/utils/flashinfer_utils.py @@ -232,7 +232,16 @@ def align_fp4_moe_weights_for_fi( # Pad w13 and w2 along its intermediate dimension. padded_w13 = w13.new_zeros((num_experts, padded_gate_up_dim, hidden_size // 2)) - padded_w13[:, : w13.shape[1], :] = w13 + if is_act_and_mul: + # Keep the two logical projections independently aligned. Copying the + # fused [gate, up] tensor contiguously would move the up projection into + # the padded tail of gate when intermediate is rounded up. + padded_w13[:, :intermediate, :] = w13[:, :intermediate, :] + padded_w13[:, padded_intermediate : padded_intermediate + intermediate, :] = ( + w13[:, intermediate:, :] + ) + else: + padded_w13[:, : w13.shape[1], :] = w13 padded_w2 = w2.new_zeros((num_experts, hidden_size, padded_intermediate // 2)) padded_w2[:, :, : w2.shape[2]] = w2 @@ -240,7 +249,13 @@ def align_fp4_moe_weights_for_fi( padded_w13_scale = w13_scale.new_zeros( (num_experts, padded_gate_up_dim, hidden_size // 16) ) - padded_w13_scale[:, : w13_scale.shape[1], :] = w13_scale + if is_act_and_mul: + padded_w13_scale[:, :intermediate, :] = w13_scale[:, :intermediate, :] + padded_w13_scale[ + :, padded_intermediate : padded_intermediate + intermediate, : + ] = w13_scale[:, intermediate:, :] + else: + padded_w13_scale[:, : w13_scale.shape[1], :] = w13_scale padded_w2_scale = w2_scale.new_zeros( (num_experts, hidden_size, padded_intermediate // 16) @@ -322,7 +337,13 @@ def align_moe_weights_for_fi( # Pad w13 and w2 along its intermediate dimension. padded_w13 = w13.new_zeros((num_experts, padded_gate_up_dim, hidden_size)) - padded_w13[:, : w13.shape[1], :] = w13 + if is_act_and_mul: + padded_w13[:, :intermediate, :] = w13[:, :intermediate, :] + padded_w13[:, padded_intermediate : padded_intermediate + intermediate, :] = ( + w13[:, intermediate:, :] + ) + else: + padded_w13[:, :intermediate, :] = w13 padded_w2 = w2.new_zeros((num_experts, hidden_size, padded_intermediate)) padded_w2[:, :, :intermediate] = w2 diff --git a/vllm/model_executor/layers/vocab_parallel_embedding.py b/vllm/model_executor/layers/vocab_parallel_embedding.py index 3057270c62..37241c8a01 100644 --- a/vllm/model_executor/layers/vocab_parallel_embedding.py +++ b/vllm/model_executor/layers/vocab_parallel_embedding.py @@ -233,6 +233,7 @@ class VocabParallelEmbedding(PluggableLayer): quant_config: quant config for the layer prefix: full name of the layer in the state dict disable_tp: If true, tensor parallelism will be disabled for this layer. + quant_method: Preselected quantization method for model-specific layers. """ # noqa: E501 # --8<-- [end:vocab_parallel_embedding] @@ -248,6 +249,7 @@ class VocabParallelEmbedding(PluggableLayer): prefix: str = "", *, disable_tp: bool = False, + quant_method: QuantizeMethodBase | None = None, ): super().__init__() @@ -281,8 +283,9 @@ class VocabParallelEmbedding(PluggableLayer): ) self.embedding_dim = embedding_dim - quant_method = None - if quant_config is not None: + # Avoid overriding a preselected model-specific method with generic + # config-based dispatch. + if quant_method is None and quant_config is not None: quant_method = quant_config.get_quant_method(self, prefix=prefix) if quant_method is None: quant_method = UnquantizedEmbeddingMethod() @@ -501,8 +504,16 @@ class VocabParallelEmbedding(PluggableLayer): output_parallel = self.quant_method.embedding(self, masked_input.long()) # Mask the output embedding. if self.tp_size > 1: + if output_parallel.dtype in ( + torch.float8_e4m3fn, + torch.float8_e5m2, + ): + # Each vocab token has one owner, so FP8 bytes can use int8 SUM. + comm_output = output_parallel.view(torch.int8) + comm_output.masked_fill_(input_mask.unsqueeze(-1), 0) + output = tensor_model_parallel_all_reduce(comm_output) + return output.view(output_parallel.dtype) output_parallel.masked_fill_(input_mask.unsqueeze(-1), 0) - # Reduce across all the model parallel GPUs. return tensor_model_parallel_all_reduce(output_parallel) return output_parallel diff --git a/vllm/model_executor/model_loader/weight_utils.py b/vllm/model_executor/model_loader/weight_utils.py index 772835c9f1..c8b166d134 100644 --- a/vllm/model_executor/model_loader/weight_utils.py +++ b/vllm/model_executor/model_loader/weight_utils.py @@ -289,12 +289,17 @@ def get_quant_config( # if hf_quant_config is None, we will try to get config from # hf_overrides hf_overrides = model_config.hf_overrides + if callable(hf_overrides): + # A callable hf_overrides is a config-to-config transform (e.g. the + # one SpeculativeConfig installs on draft model configs); it cannot + # carry quantization config entries, so treat it as no overrides. + hf_overrides = {} if not isinstance(hf_overrides, dict): raise ValueError( "hf_overrides must be a dict for get_quant_config " "to get the quantization config from it." ) - quantization_config_file = hf_overrides.get("quantization_config_file", None) + quantization_config_file = hf_overrides.get("quantization_config_file") if quantization_config_file is not None: if hasattr(quant_cls, "from_config_file"): return quant_cls.from_config_file(quantization_config_file) @@ -304,7 +309,7 @@ def get_quant_config( "but quant_cls.from_config_file is not implemented in " f"{quant_cls}" ) - quantization_config_json = hf_overrides.get("quantization_config_dict_json", None) + quantization_config_json = hf_overrides.get("quantization_config_dict_json") if quantization_config_json is not None: if hasattr(quant_cls, "from_config_dict_json"): return quant_cls.from_config_dict_json(quantization_config_json) diff --git a/vllm/model_executor/models/config.py b/vllm/model_executor/models/config.py index fe7eca2757..263c8a4035 100644 --- a/vllm/model_executor/models/config.py +++ b/vllm/model_executor/models/config.py @@ -839,6 +839,83 @@ class Qwen3_5ForCausalLMConfig(Qwen3_5ForConditionalGenerationConfig): rope_parameters.pop("mrope_interleaved", None) +def _strip_qwen4_exp_mrope(model_config: "ModelConfig") -> None: + configs = { + id(config): config + for config in ( + getattr(model_config, "hf_config", None), + model_config.hf_text_config, + ) + if config is not None + } + for config in configs.values(): + rope_parameters = getattr(config, "rope_parameters", None) + if rope_parameters is not None: + rope_parameters.pop("mrope_section", None) + rope_parameters.pop("mrope_interleaved", None) + + +class Qwen4ExpForConditionalGenerationConfig(Qwen3_5ForConditionalGenerationConfig): + """Apply the Qwen3.5 hybrid-cache contract to Qwen4Exp.""" + + @staticmethod + def verify_and_update_config(vllm_config: "VllmConfig") -> None: + Qwen3_5ForConditionalGenerationConfig.verify_and_update_config(vllm_config) + text_config = vllm_config.model_config.hf_text_config + if text_config.hc_count <= 1: + raise ValueError("Qwen4Exp requires hc_count > 1") + parallel_config = vllm_config.parallel_config + uses_ple_or_qsa = bool(text_config.ple_layer_ids) or ( + getattr(text_config, "indexer_n_heads", None) is not None + ) + if uses_ple_or_qsa and ( + parallel_config.enable_dbo or parallel_config.ubatch_size > 1 + ): + raise NotImplementedError( + "Qwen4Exp PLE/QSA does not support dual-batch overlap or microbatching" + ) + # Checked again in Qwen4ExpModelState; rejecting it here keeps the + # engine from loading weights first. + if text_config.ple_layer_ids and parallel_config.pipeline_parallel_size > 1: + raise NotImplementedError( + "Qwen4Exp N-gram PLE embedding requires pipeline_parallel_size=1 " + "because non-first pipeline ranks do not receive the raw input_ids " + "it needs. Please run with PP=1." + ) + multimodal_config = vllm_config.model_config.multimodal_config + if multimodal_config is not None and multimodal_config.language_model_only: + _strip_qwen4_exp_mrope(vllm_config.model_config) + spec_config = vllm_config.speculative_config + if spec_config is not None and spec_config.method not in { + "mtp", + "ngram", + "ngram_gpu", + }: + raise NotImplementedError( + "Qwen4Exp speculative decoding supports only its native MTP " + "checkpoint and linear n-gram proposers" + ) + + +class Qwen4ExpForCausalLMConfig(Qwen4ExpForConditionalGenerationConfig): + @staticmethod + def verify_and_update_config(vllm_config: "VllmConfig") -> None: + Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(vllm_config) + + _strip_qwen4_exp_mrope(vllm_config.model_config) + + +class Qwen4ExpMTPConfig(Qwen4ExpForConditionalGenerationConfig): + """Preserve MRoPE for a VL target and use 1D RoPE for a text target.""" + + @staticmethod + def verify_and_update_config(vllm_config: "VllmConfig") -> None: + Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(vllm_config) + if hasattr(vllm_config.model_config.hf_config, "vision_config"): + return + _strip_qwen4_exp_mrope(vllm_config.model_config) + + class ColQwen3_5Config(Qwen3_5ForConditionalGenerationConfig): """Apply the attention contract declared by a ColQwen3.5 checkpoint.""" @@ -963,6 +1040,9 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = { "Qwen3_5ForConditionalGeneration": Qwen3_5ForConditionalGenerationConfig, "Qwen3_5MoeForCausalLM": Qwen3_5ForCausalLMConfig, "Qwen3_5MoeForConditionalGeneration": Qwen3_5ForConditionalGenerationConfig, + "Qwen4ExpForCausalLM": Qwen4ExpForCausalLMConfig, + "Qwen4ExpForConditionalGeneration": (Qwen4ExpForConditionalGenerationConfig), + "Qwen4ExpMTP": Qwen4ExpMTPConfig, "UnlimitedOCRForCausalLM": UnlimitedOCRForCausalLMConfig, "VoyageQwen3BidirectionalEmbedModel": VoyageQwen3BidirectionalEmbedModelConfig, "XLMRobertaModel": JinaRobertaModelConfig, diff --git a/vllm/model_executor/models/interfaces.py b/vllm/model_executor/models/interfaces.py index f35dc3b9cc..2404e44eb8 100644 --- a/vllm/model_executor/models/interfaces.py +++ b/vllm/model_executor/models/interfaces.py @@ -49,13 +49,17 @@ if TYPE_CHECKING: from vllm.lora.model_manager import LoRAModelManager from vllm.model_executor.layers.fused_moe import MoERunner from vllm.model_executor.layers.logits_processor import LogitsProcessor - from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc + from vllm.model_executor.layers.mamba.mamba_utils import ( + MambaStateCopyFunc, + MambaStateCopyFuncsByType, + ) from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead from vllm.model_executor.models.utils import WeightsMapper from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalKwargsItem from vllm.multimodal.registry import _ProcessorFactories from vllm.sequence import IntermediateTensors from vllm.tasks import ScoreType + from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum from vllm.v1.worker.encoder_cudagraph_defs import ( EncoderCudaGraphCaptureInputs, EncoderCudaGraphConfig, @@ -970,6 +974,14 @@ class IsHybrid(Protocol): """ ... + @classmethod + def get_mamba_state_copy_funcs( + cls, + mamba_types: set["MambaAttentionBackendEnum"], + ) -> "MambaStateCopyFuncsByType": + copy_funcs = cls.get_mamba_state_copy_func() + return {mamba_type: copy_funcs for mamba_type in mamba_types} + @overload def is_hybrid(model: type[object]) -> TypeIs[type[IsHybrid]]: ... @@ -1131,6 +1143,20 @@ class SupportsMambaPrefixCaching(Protocol): supports_mamba_prefix_caching: ClassVar[Literal[True]] = True + @classmethod + def get_mamba_state_copy_func(cls) -> tuple["MambaStateCopyFunc", ...]: + """Return copy functions for the model's Mamba states.""" + ... + + @classmethod + def get_mamba_state_copy_funcs( + cls, + mamba_types: set["MambaAttentionBackendEnum"], + ) -> "MambaStateCopyFuncsByType": + """Map legacy copy functions to each requested Mamba backend.""" + copy_funcs = cls.get_mamba_state_copy_func() + return {mamba_type: copy_funcs for mamba_type in mamba_types} + @overload def supports_mamba_prefix_caching( diff --git a/vllm/model_executor/models/qwen2_moe.py b/vllm/model_executor/models/qwen2_moe.py index df1794d289..3d1345397f 100644 --- a/vllm/model_executor/models/qwen2_moe.py +++ b/vllm/model_executor/models/qwen2_moe.py @@ -80,15 +80,17 @@ class Qwen2MoeMLP(nn.Module): reduce_results: bool = True, expert_gate: torch.nn.Linear | None = None, is_sequence_parallel: bool = False, + disable_tp: bool = False, prefix: str = "", ) -> None: super().__init__() + disable_tp = disable_tp or is_sequence_parallel self.gate_up_proj = MergedColumnParallelLinear( hidden_size, [intermediate_size] * 2, bias=False, quant_config=quant_config, - disable_tp=is_sequence_parallel, + disable_tp=disable_tp, prefix=f"{prefix}.gate_up_proj", ) self.down_proj = RowParallelLinear( @@ -97,7 +99,7 @@ class Qwen2MoeMLP(nn.Module): bias=False, quant_config=quant_config, reduce_results=reduce_results, - disable_tp=is_sequence_parallel, + disable_tp=disable_tp, prefix=f"{prefix}.down_proj", ) if hidden_act != "silu": diff --git a/vllm/model_executor/models/qwen3_next.py b/vllm/model_executor/models/qwen3_next.py index 3dc38f3a2d..2b16a2e494 100644 --- a/vllm/model_executor/models/qwen3_next.py +++ b/vllm/model_executor/models/qwen3_next.py @@ -43,6 +43,9 @@ from vllm.model_executor.layers.mamba.mamba_utils import ( MambaStateShapeCalculator, ) from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.quantization.utils.config_utils import ( + get_quark_ocp_mx_group_size, +) from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding, get_rope from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, @@ -90,6 +93,38 @@ def _should_use_sequence_parallel(vllm_config: VllmConfig) -> bool: ) +def _should_replicate_misaligned_shared_expert( + intermediate_size: int, + tp_size: int, + group_size: int | None, + enable_expert_parallel: bool, + is_sequence_parallel: bool, +) -> bool: + if intermediate_size <= 0 or group_size is None: + return False + + partition_size, remainder = divmod(intermediate_size, tp_size) + if remainder == 0 and partition_size % group_size == 0: + return False + + if enable_expert_parallel or is_sequence_parallel: + return True + + if remainder != 0: + raise ValueError( + f"Shared-expert intermediate size {intermediate_size} must be " + f"divisible by tensor-parallel size {tp_size}." + ) + + raise ValueError( + "The Quark OCP MX shared expert cannot be tensor-parallelized: " + f"intermediate size {intermediate_size} with TP size {tp_size} " + f"produces a partition of {partition_size}, which is not divisible by " + f"the OCP MX group size {group_size}. Choose a compatible " + "tensor-parallel size or enable expert parallelism." + ) + + class Qwen3NextSparseMoeBlock(nn.Module): def __init__(self, vllm_config: VllmConfig, prefix: str = ""): super().__init__() @@ -106,6 +141,17 @@ class Qwen3NextSparseMoeBlock(nn.Module): self.is_sequence_parallel = parallel_config.use_sequence_parallel_moe + shared_expert_group_size = get_quark_ocp_mx_group_size( + quant_config, + f"{prefix}.shared_expert.down_proj", + ) + self.replicate_shared_expert = _should_replicate_misaligned_shared_expert( + config.shared_expert_intermediate_size, + self.tp_size, + shared_expert_group_size, + parallel_config.enable_expert_parallel, + self.is_sequence_parallel, + ) if self.tp_size > config.num_experts: raise ValueError( f"Tensor parallel size {self.tp_size} is greater than " @@ -138,7 +184,10 @@ class Qwen3NextSparseMoeBlock(nn.Module): ) self.is_fused_shared_expert_enabled = False - if config.shared_expert_intermediate_size > 0: + if ( + config.shared_expert_intermediate_size > 0 + and not self.replicate_shared_expert + ): self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert( quant_config, prefix, @@ -159,11 +208,14 @@ class Qwen3NextSparseMoeBlock(nn.Module): reduce_results=False, expert_gate=self.shared_expert_gate, is_sequence_parallel=self.is_sequence_parallel, + disable_tp=self.replicate_shared_expert, prefix=f"{prefix}.shared_expert", ) self.experts = FusedMoEFactory( - shared_experts=self.shared_expert, + shared_experts=( + None if self.replicate_shared_expert else self.shared_expert + ), gate=self.gate, num_experts=self.n_routed_experts, top_k=config.num_experts_per_tok, @@ -195,9 +247,16 @@ class Qwen3NextSparseMoeBlock(nn.Module): if self.is_sequence_parallel and not already_sequence_parallel: hidden_states = sequence_parallel_chunk(hidden_states) + replicated_shared_output = ( + self.shared_expert(hidden_states) + if self.replicate_shared_expert and self.shared_expert is not None + else None + ) final_hidden_states = self.experts( hidden_states=hidden_states, router_logits=hidden_states ) + if replicated_shared_output is not None: + final_hidden_states += replicated_shared_output if self.is_sequence_parallel and not already_sequence_parallel: final_hidden_states = tensor_model_parallel_all_gather( @@ -488,7 +547,7 @@ class Qwen3NextDecoderLayer(nn.Module): self, hidden_states: torch.Tensor, residual: torch.Tensor | None, - positions: torch.Tensor = None, + positions: torch.Tensor, **kwargs: object, ): full_num_tokens = positions.shape[-1] diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 05af1e04fe..40f4a27d28 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -111,6 +111,10 @@ _TEXT_GENERATION_MODELS = { "Gemma3nForCausalLM": ("gemma3n", "Gemma3nForCausalLM"), "Gemma4ForCausalLM": ("gemma4", "Gemma4ForCausalLM"), "Qwen3NextForCausalLM": ("qwen3_next", "Qwen3NextForCausalLM"), + "Qwen4ExpForCausalLM": ( + "vllm.models.qwen4_exp", + "Qwen4ExpForCausalLM", + ), "GlmForCausalLM": ("glm", "GlmForCausalLM"), "Glm4ForCausalLM": ("glm4", "Glm4ForCausalLM"), "Glm4MoeForCausalLM": ("glm4_moe", "Glm4MoeForCausalLM"), @@ -573,6 +577,10 @@ _MULTIMODAL_MODELS = { "qwen3_5", "Qwen3_5MoeForConditionalGeneration", ), + "Qwen4ExpForConditionalGeneration": ( + "vllm.models.qwen4_exp", + "Qwen4ExpForConditionalGeneration", + ), "RForConditionalGeneration": ("rvl", "RForConditionalGeneration"), "SkyworkR1VChatModel": ("skyworkr1v", "SkyworkR1VChatModel"), "SmolVLMForConditionalGeneration": ("smolvlm", "SmolVLMForConditionalGeneration"), @@ -659,6 +667,7 @@ _SPECULATIVE_DECODING_MODELS = { "MedusaModel": ("medusa", "Medusa"), "OpenPanguMTPModel": ("openpangu_mtp", "OpenPanguMTP"), "Qwen3NextMTP": ("qwen3_next_mtp", "Qwen3NextMTP"), + "Qwen4ExpMTP": ("vllm.models.qwen4_exp", "Qwen4ExpMTP"), "Step3p5MTP": ("step3p5_mtp", "Step3p5MTP"), "Qwen3_5MTP": ("qwen3_5_mtp", "Qwen3_5MTP"), "Qwen3_5MoeMTP": ("qwen3_5_mtp", "Qwen3_5MoeMTP"), diff --git a/vllm/models/qwen4_exp/__init__.py b/vllm/models/qwen4_exp/__init__.py new file mode 100644 index 0000000000..9e533e28b4 --- /dev/null +++ b/vllm/models/qwen4_exp/__init__.py @@ -0,0 +1,61 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Qwen4Exp model package.""" + +from typing import TYPE_CHECKING, Any + +from .common.hyperconnection import ( + GatedResidual, + GroupedGemmaRMSNorm, + HyperConnectionBase, + HyperConnectionConfig, +) + +if TYPE_CHECKING: + from .nvidia.model import ( + Qwen4ExpForCausalLM, + Qwen4ExpForConditionalGeneration, + ) + from .nvidia.mtp import Qwen4ExpMTP + + +def __getattr__(name: str) -> Any: + if name in { + "Qwen4ExpForCausalLM", + "Qwen4ExpForConditionalGeneration", + "Qwen4ExpMTP", + }: + from vllm.platforms import current_platform + + if current_platform.is_xpu() or current_platform.is_tpu(): + raise NotImplementedError("Qwen4Exp currently supports CUDA and ROCm only") + if current_platform.is_rocm(): + from .amd.model import ( + Qwen4ExpForCausalLM, + Qwen4ExpForConditionalGeneration, + ) + from .amd.mtp import Qwen4ExpMTP + else: + from .nvidia.model import ( + Qwen4ExpForCausalLM, + Qwen4ExpForConditionalGeneration, + ) + from .nvidia.mtp import Qwen4ExpMTP + + return { + "Qwen4ExpForCausalLM": Qwen4ExpForCausalLM, + "Qwen4ExpForConditionalGeneration": (Qwen4ExpForConditionalGeneration), + "Qwen4ExpMTP": Qwen4ExpMTP, + }[name] + raise AttributeError(name) + + +__all__ = [ + "GatedResidual", + "GroupedGemmaRMSNorm", + "HyperConnectionBase", + "HyperConnectionConfig", + "Qwen4ExpForCausalLM", + "Qwen4ExpForConditionalGeneration", + "Qwen4ExpMTP", +] diff --git a/vllm/models/qwen4_exp/amd/__init__.py b/vllm/models/qwen4_exp/amd/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/vllm/models/qwen4_exp/amd/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm/models/qwen4_exp/amd/hyperconnection.py b/vllm/models/qwen4_exp/amd/hyperconnection.py new file mode 100644 index 0000000000..087f03fdcf --- /dev/null +++ b/vllm/models/qwen4_exp/amd/hyperconnection.py @@ -0,0 +1,205 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""HyperConnection (Gated Residual) utilities for the AMD model variant. + +Implements the HyperConnection residual scheme proposed in +"HyperConnections" (https://arxiv.org/abs/2409.19606). This AMD variant +delays each HC combine to the following HC mix boundary. HC glue kernels, +including fused combine+RMSNorm, live in ``ops/hc.py``; projections remain +standard vLLM Linear modules. + +Hidden states between layers have shape ``[..., HC*HS]`` with HS inner +(HC outer, HS inner — checkpoint-native layout). + +Typical usage inside a transformer decoder layer:: + + self.attn_hc = GatedResidual(hc_config) + + hidden_states, block_input, injection = self.attn_hc.mix(hidden_states) + attention_output = attention(block_input) + hidden_states, block_input, injection = self.mlp_hc.combine_and_mix( + hidden_states, attention_output, injection + ) +""" + +import torch +from torch import nn + +from vllm.model_executor.layers.linear import ( + MergedColumnParallelLinear, + ReplicatedLinear, +) +from vllm.model_executor.models.utils import maybe_prefix + +from ..common.hyperconnection import ( + GroupedGemmaRMSNorm, + HyperConnectionConfig, +) +from .ops.hc import ( + grouped_gemma_rmsnorm, + hc_combine, + hc_combine_norm, + hc_gate_mix, + hc_silu, +) + + +# --------------------------------------------------------------------------- +# Gated-residual variant +# --------------------------------------------------------------------------- +class GatedResidual(nn.Module): + """Gated HyperConnection with learnable low-rank mixing and injection. + + ``combine_and_mix()`` runs the pre pipeline (grouped GemmaRMSNorm -> merged + low-rank down+inject GEMM -> silu -> up GEMM -> sigmoid -> gated mean + over the HC streams). When passed a pending block output and an injection, + it fuses their residual combine with the RMSNorm. Final mixers use + ``use_combine=False`` and do not produce a new injection. + + Weights: the norm owns the grouped GemmaRMSNorm affine; the projections + are vLLM Linear modules (merged replicated linear for down+inject), so + GEMM dispatch (e.g. the low-latency skinny GEMM) applies through the + standard quant_method mechanism. + """ + + def __init__( + self, + config: HyperConnectionConfig, + use_combine: bool = True, + prefix: str = "", + ) -> None: + super().__init__() + self.config = config + self.lora_rank = config.hc_lowrank + self.hc_count = config.hc_count + self.hidden_size = config.hidden_size + self.use_combine = use_combine + + norm_size = ( + self.hyper_hidden_size if config.hc_per_branch_norm else config.hidden_size + ) + group_size = config.hidden_size if config.hc_per_branch_norm else None + # Normalize each H-sized HC stream independently while retaining a + # separate affine weight for every element of the HC*H layout. + self.hc_norm = GroupedGemmaRMSNorm( + norm_size, + eps=config.rms_norm_eps, + group_size=group_size, + dtype=config.params_dtype, + ) + + # -- vLLM Linear weights -------------------------------------------- + # The merged skinny-GEMM shape is physically padded to 16 rows for + # alignment and efficient backend dispatch. + self.pad_size = (-(self.lora_rank + self.hc_count)) % 16 if use_combine else 0 + if use_combine: + self.input_mix_weight_down_block_inject = MergedColumnParallelLinear( + self.hyper_hidden_size, + [self.lora_rank, self.hc_count] + + ([self.pad_size] if self.pad_size else []), + bias=False, + params_dtype=config.params_dtype, + quant_config=None, + prefix=maybe_prefix(prefix, "input_mix_weight_down_block_inject"), + return_bias=False, + disable_tp=True, + ) + else: + self.input_mix_weight_down = ReplicatedLinear( + self.hyper_hidden_size, + self.lora_rank, + bias=False, + params_dtype=config.params_dtype, + quant_config=None, + prefix=maybe_prefix(prefix, "input_mix_weight_down"), + return_bias=False, + ) + self.input_mix_weight_up = ReplicatedLinear( + self.lora_rank, + self.hyper_hidden_size, + bias=False, + params_dtype=config.params_dtype, + quant_config=None, + prefix=maybe_prefix(prefix, "input_mix_weight_up"), + return_bias=False, + ) + + def mix( + self, hidden_states: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]: + xn = grouped_gemma_rmsnorm( + hidden_states, + self.hc_norm.weight, + self.config.rms_norm_eps, + self.hc_count, + ) + + if self.use_combine: + # produce injection logits for combine + split_sizes = [self.lora_rank, self.hc_count, self.pad_size] + down_and_injection = self.input_mix_weight_down_block_inject(xn) + lora, injection, _ = down_and_injection.split(split_sizes, dim=-1) + else: + lora = self.input_mix_weight_down(xn) + injection = None + + lora = hc_silu(lora, self.hc_count) + gate = self.input_mix_weight_up(lora) # [M, D] + block_input = hc_gate_mix(xn, gate, self.hc_count) + + return hidden_states, block_input, injection + + def combine_and_mix( + self, + hidden_states: torch.Tensor, + prev_block_output: torch.Tensor, + prev_injection: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]: + """Consume a pending combine, then prepare the next block input. + + ``hidden_states`` is the multi-stream state from before the pending + block's mix. Its combine with ``block_output`` is fused with this + module's input RMSNorm. + """ + hidden_states, xn = hc_combine_norm( + hidden_states, + prev_block_output, + prev_injection, + self.hc_norm.weight, + self.config.rms_norm_eps, + self.hc_count, + ) + + if self.use_combine: + # produce injection logits for combine + split_sizes = [self.lora_rank, self.hc_count, self.pad_size] + down_and_injection = self.input_mix_weight_down_block_inject(xn) + lora, injection, _ = down_and_injection.split(split_sizes, dim=-1) + else: + lora = self.input_mix_weight_down(xn) + injection = None + + lora = hc_silu(lora, self.hc_count) + gate = self.input_mix_weight_up(lora) # [M, D] + block_input = hc_gate_mix(xn, gate, self.hc_count) + + return hidden_states, block_input, injection + + def combine( + self, + hidden_states: torch.Tensor, + block_output: torch.Tensor, + injection: torch.Tensor, + ) -> torch.Tensor: + return hc_combine(hidden_states, block_output, injection, self.hc_count) + + @property + def hyper_hidden_size(self) -> int: + return self.hc_count * self.hidden_size + + +__all__ = [ + "GatedResidual", + "GroupedGemmaRMSNorm", + "HyperConnectionConfig", +] diff --git a/vllm/models/qwen4_exp/amd/indexer_qsa.py b/vllm/models/qwen4_exp/amd/indexer_qsa.py new file mode 100644 index 0000000000..6b7d8f9f93 --- /dev/null +++ b/vllm/models/qwen4_exp/amd/indexer_qsa.py @@ -0,0 +1,321 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Qwen4Exp weight-free QSA indexer.""" + +from __future__ import annotations + +from typing import cast + +import torch +from torch import nn + +from vllm.config import VllmConfig +from vllm.forward_context import get_forward_context +from vllm.model_executor.layers.layernorm import GemmaRMSNorm +from vllm.model_executor.layers.linear import ReplicatedLinear +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding.mrope import triton_mrope +from vllm.transformers_utils.configs.qwen4_exp import ( + Qwen4ExpTextConfig, +) + +from ..common.qsa_cache import ( + QSACompressedKeyCache, + QSAForwardMetadata, + QSAKeyStateCache, + canonical_qsa_rope_positions, +) + + +def apply_qsa_rope( + rotary_emb: nn.Module, + positions: torch.Tensor, + tensor: torch.Tensor, +) -> torch.Tensor: + """Apply the main attention's exact 1D/MRoPE composition to QSA heads.""" + + num_tokens, _, head_dim = tensor.shape + rotary_dim = rotary_emb.rotary_dim + cache = rotary_emb._match_cos_sin_cache_dtype(tensor) # noqa: SLF001 + cos_sin = cache[positions] + cos, sin = cos_sin.chunk(2, dim=-1) + if positions.ndim == 2: + shape = tensor.shape + tensor, _ = triton_mrope( + tensor.reshape(num_tokens, -1), + tensor.new_empty((num_tokens, head_dim)), + cos, + sin, + rotary_emb.mrope_section, + head_dim, + rotary_dim, + rotary_emb.mrope_interleaved, + rotary_emb.is_neox_style, + ) + return tensor.reshape(shape) + + rotated = rotary_emb.apply_rotary_emb( + tensor[..., :rotary_dim], + cos, + sin, + ) + return torch.cat((rotated, tensor[..., rotary_dim:]), dim=-1) + + +def apply_qsa_rmsnorm( + norm: GemmaRMSNorm, + tensor: torch.Tensor, +) -> torch.Tensor: + """Use vLLM's portable RMSNorm implementation on ROCm.""" + + return cast(torch.Tensor, norm(tensor)) + + +class QSAIndexer(nn.Module): + """Replicated Q/K projection plus paged, weight-free QSA selection. + + ``prefix`` must be the checkpoint's indexer prefix, normally + ``model.layers.N.self_attn.indexer``. Consequently the trainable names are + ``index_qk_proj``, ``q_layernorm`` and ``k_layernorm`` under that prefix. + """ + + def __init__( + self, + *, + vllm_config: VllmConfig, + config: Qwen4ExpTextConfig, + layer_id: int, + rotary_emb: nn.Module, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + if vllm_config.cache_config is None: + raise ValueError("QSA requires a paged KV cache") + if vllm_config.model_config.dtype != torch.bfloat16: + raise NotImplementedError("Qwen4Exp QSA currently requires BF16") + + self.layer_id = int(layer_id) + self.index_n_heads = int(config.indexer_n_heads) + self.index_kv_heads = int(config.indexer_kv_heads) + self.index_head_dim = int(config.indexer_head_dim) + self.token_topk = int(config.indexer_budget) + self.compress_ratio = int(config.indexer_compress_ratio) + self.rotary_emb = rotary_emb + self.prefix = prefix + # MTP step 0 selects the target-aligned rows; later steps reuse them + # while continuing to update the QSA side cache. + self.skip_topk = False + + self.index_qk_proj = ReplicatedLinear( + int(config.hidden_size), + (self.index_n_heads + self.index_kv_heads) * self.index_head_dim, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.index_qk_proj" if prefix else "index_qk_proj", + ) + self.q_layernorm = GemmaRMSNorm( + self.index_head_dim, + eps=float(getattr(config, "rms_norm_eps", 1e-6)), + ) + self.k_layernorm = GemmaRMSNorm( + self.index_head_dim, + eps=float(getattr(config, "rms_norm_eps", 1e-6)), + ) + + cache_config = vllm_config.cache_config + cache_prefix = f"{prefix}." if prefix else "" + self.raw_key_cache = QSAKeyStateCache( + head_size=self.index_head_dim, + dtype=torch.bfloat16, + cache_rope_positions=vllm_config.model_config.uses_mrope, + prefix=f"{cache_prefix}raw_key_cache", + cache_config=cache_config, + compress_ratio=self.compress_ratio, + vllm_config=vllm_config, + ) + self.compressed_key_cache = QSACompressedKeyCache( + head_size=self.index_head_dim, + dtype=torch.bfloat16, + compress_ratio=self.compress_ratio, + prefix=f"{cache_prefix}compressed_key_cache", + cache_config=cache_config, + vllm_config=vllm_config, + ) + + @property + def output_width(self) -> int: + return self.token_topk + self.compress_ratio - 1 + + def project_qk( + self, + hidden_states: torch.Tensor, + positions: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Project replicated Q/K, normalize+rotate Q, and preserve raw K.""" + + qk, _ = self.index_qk_proj(hidden_states) + q_raw, token_k = qk.split( + ( + self.index_n_heads * self.index_head_dim, + self.index_kv_heads * self.index_head_dim, + ), + dim=-1, + ) + q = q_raw.reshape(-1, self.index_n_heads, self.index_head_dim) + q = apply_qsa_rmsnorm( + self.q_layernorm, + q.reshape(-1, self.index_head_dim), + ).reshape_as(q) + q = apply_qsa_rope(self.rotary_emb, positions, q) + return q, token_k.reshape(-1, 1, self.index_head_dim) + + def normalize_compressed_keys( + self, + compressed_keys: torch.Tensor, + first_rope_positions: torch.Tensor, + ) -> torch.Tensor: + """Normalize pooled K and apply the first token's exact group position.""" + + keys = compressed_keys.reshape(-1, self.index_head_dim) + keys = apply_qsa_rmsnorm(self.k_layernorm, keys).reshape( + -1, 1, self.index_head_dim + ) + if getattr(self.rotary_emb, "mrope_section", None): + positions = first_rope_positions.transpose(0, 1) + else: + positions = first_rope_positions[:, 0] + return apply_qsa_rope(self.rotary_emb, positions, keys) + + def _metadata( + self, + ) -> tuple[QSAForwardMetadata, QSAForwardMetadata] | None: + metadata = get_forward_context().attn_metadata + if isinstance(metadata, list): + metadata = metadata[0] + if not isinstance(metadata, dict): + return None + raw = cast(QSAForwardMetadata, metadata[self.raw_key_cache.prefix]) + compressed = cast( + QSAForwardMetadata, metadata[self.compressed_key_cache.prefix] + ) + if raw.num_actual_tokens != compressed.num_actual_tokens: + raise RuntimeError("QSA side-cache metadata token counts disagree") + if not raw.logical_positions.is_cuda and ( + not torch.equal(raw.logical_positions, compressed.logical_positions) + ): + raise RuntimeError("QSA side-cache metadata positions disagree") + return raw, compressed + + def _update_and_compress( + self, + token_k: torch.Tensor, + positions: torch.Tensor, + raw_metadata: QSAForwardMetadata, + compressed_metadata: QSAForwardMetadata, + ) -> None: + num_tokens = raw_metadata.num_actual_tokens + raw_key_cache = self.raw_key_cache.key_cache + rope_position_cache = self.raw_key_cache.rope_position_cache + from .ops.qsa import qsa_compress_groups_with_ratio, qsa_store_cache_rows + + if rope_position_cache is None: + position_rows = raw_metadata.logical_positions.view(-1, 1, 1).expand( + -1, 1, 3 + ) + else: + position_rows = canonical_qsa_rope_positions(positions)[:num_tokens].to( + device=raw_key_cache.device + ) + pooled, first_positions = qsa_compress_groups_with_ratio( + token_k[:num_tokens], + position_rows, + raw_key_cache, + raw_metadata.block_table, + raw_metadata.token_to_req, + raw_metadata.query_start_loc, + raw_metadata.logical_positions, + compressed_metadata.slot_mapping, + self.compress_ratio, + rope_position_cache, + ) + normalized = self.normalize_compressed_keys(pooled, first_positions) + qsa_store_cache_rows( + self.compressed_key_cache.kv_cache, + compressed_metadata.slot_mapping, + normalized, + ) + qsa_store_cache_rows( + raw_key_cache, + raw_metadata.slot_mapping, + token_k[:num_tokens], + ) + if rope_position_cache is not None: + qsa_store_cache_rows( + rope_position_cache, + raw_metadata.slot_mapping, + position_rows, + ) + + def _select( + self, + q: torch.Tensor, + metadata: QSAForwardMetadata, + out: torch.Tensor | None, + ) -> torch.Tensor: + from .ops.qsa import qsa_select_paged_tokens + + return qsa_select_paged_tokens( + q, + self.compressed_key_cache.kv_cache, + metadata.block_table, + metadata.token_to_req, + metadata.logical_positions, + metadata.seq_lens, + self.token_topk, + self.compress_ratio, + out, + ) + + def forward( + self, + hidden_states: torch.Tensor, + positions: torch.Tensor, + out: torch.Tensor | None = None, + ) -> torch.Tensor: + """Return fixed-width request-relative token indices padded with ``-1``.""" + + metadata = self._metadata() + if metadata is None: + # Preserve step-0 indices when later MTP steps reuse the buffer. + if self.skip_topk and out is not None: + return out + result = torch.full( + (hidden_states.shape[0], self.output_width), + -1, + dtype=torch.int32, + device=hidden_states.device, + ) + if out is not None: + out.copy_(result) + return out + return result + raw_metadata, compressed_metadata = metadata + num_tokens = raw_metadata.num_actual_tokens + q, token_k = self.project_qk( + hidden_states[:num_tokens], positions[..., :num_tokens] + ) + self._update_and_compress( + token_k, + positions[..., :num_tokens], + raw_metadata, + compressed_metadata, + ) + if self.skip_topk: + if out is None: + raise RuntimeError("QSA top-k reuse requires an output buffer") + return out + return self._select(q, compressed_metadata, out) + + +__all__ = ["QSAIndexer", "apply_qsa_rope"] diff --git a/vllm/models/qwen4_exp/amd/low_latency_gemm.py b/vllm/models/qwen4_exp/amd/low_latency_gemm.py new file mode 100644 index 0000000000..e8c5c2f917 --- /dev/null +++ b/vllm/models/qwen4_exp/amd/low_latency_gemm.py @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Qwen4Exp low-latency GEMM hook for AMD ROCm.""" + +import torch +from torch import nn + + +def enable_qwen4_exp_low_latency_gemm( + module: nn.Module, + dtype: torch.dtype, +) -> None: + """Keep the standard vLLM linear methods on AMD ROCm.""" + + del module, dtype diff --git a/vllm/models/qwen4_exp/amd/model.py b/vllm/models/qwen4_exp/amd/model.py new file mode 100644 index 0000000000..aeb3a14dad --- /dev/null +++ b/vllm/models/qwen4_exp/amd/model.py @@ -0,0 +1,1065 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inference-only Qwen4Exp model.""" + +from collections.abc import Iterable +from itertools import islice + +import torch +from torch import nn + +from vllm.compilation.decorators import support_torch_compile +from vllm.config import VllmConfig +from vllm.distributed import get_pp_group +from vllm.model_executor.layers.fused_moe.utils import ( + is_model_fused_shared_expert_compatible, +) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import ( + QwenGatedDeltaNetAttention, +) +from vllm.model_executor.layers.mamba.mamba_utils import ( + MambaStateCopyFunc, + MambaStateCopyFuncCalculator, + MambaStateCopyFuncsByType, + MambaStateDtypeCalculator, + MambaStateShapeCalculator, +) +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, + VocabParallelEmbedding, +) +from vllm.model_executor.models.interfaces import ( + HasInnerState, + IsHybrid, + MixtureOfExperts, + MultiModalEmbeddings, + SupportsLoRA, + SupportsMRoPE, + SupportsPP, + _require_is_multimodal, +) +from vllm.model_executor.models.qwen3_5 import ( + Qwen3_5ForConditionalGeneration, + Qwen3_5Model, +) +from vllm.model_executor.models.qwen3_next import ( + Qwen3NextAttention, + Qwen3NextMLP, + Qwen3NextSparseMoeBlock, +) +from vllm.model_executor.models.qwen3_vl import ( + Qwen3_VisionTransformer, + Qwen3VLDummyInputsBuilder, + Qwen3VLMultiModalProcessor, + Qwen3VLProcessingInfo, +) +from vllm.model_executor.models.utils import ( + AutoWeightsLoader, + StageMissingLayer, + WeightsMapper, + _merge_multimodal_embeddings, + extract_layer_index, + make_empty_intermediate_tensors_factory, + make_layers, + maybe_fuse_shared_experts, + maybe_prefix, +) +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.inputs import MultiModalFeatureSpec +from vllm.sequence import IntermediateTensors +from vllm.tokenizers.registry import cached_tokenizer_from_config +from vllm.transformers_utils.configs.qwen4_exp import ( + Qwen4ExpTextConfig, +) +from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum +from vllm.v1.kv_cache_interface import MambaSpec + +from ..config import Qwen4ExpConfig +from .hyperconnection import GatedResidual, HyperConnectionConfig +from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm +from .ple_layer import Qwen4ExpPLELayer +from .qsa import Qwen4ExpQSAAttention + + +def without_modelopt_fp4( + quant_config: QuantizationConfig | None, +) -> QuantizationConfig | None: + """Return ``None`` for weights excluded from Qwen4Exp ModelOpt-FP4.""" + + if quant_config is not None and quant_config.get_name() == "modelopt_fp4": + return None + return quant_config + + +def _remap_qsa_cache_scale_name( + name: str, + qsa_layer_ids: frozenset[int], +) -> str: + """Map serialized main-cache scales onto the merged QSA owner. + + Regular attention keeps cache scales below its ``attn`` child. QSA owns + that cache directly, so only QSA layers need the final path component + moved to the owner's persistent ``_k_scale``/``_v_scale`` buffers. + """ + + scale_suffixes = { + "k_proj.k_scale": "_k_scale", + "k_proj.output_scale": "_k_scale", + "attn.k_scale": "_k_scale", + "attn._k_scale": "_k_scale", + "k_scale": "_k_scale", + "_k_scale": "_k_scale", + "v_proj.v_scale": "_v_scale", + "v_proj.output_scale": "_v_scale", + "attn.v_scale": "_v_scale", + "attn._v_scale": "_v_scale", + "v_scale": "_v_scale", + "_v_scale": "_v_scale", + } + for layer_id in qsa_layer_ids: + marker = f"layers.{layer_id}.self_attn." + marker_start = name.find(marker) + if marker_start < 0 or (marker_start > 0 and name[marker_start - 1] != "."): + continue + suffix = name[marker_start + len(marker) :] + mapped_suffix = scale_suffixes.get(suffix) + if mapped_suffix is not None: + return f"{name[: marker_start + len(marker)]}{mapped_suffix}" + return name + + +_QWEN4_EXP_IGNORED_MISSING_SUFFIXES = [ + ".bias", + "_bias", + ".k_scale", + "_k_scale", + ".v_scale", + "_v_scale", + "_weight_scale", + "_input_scale", +] + +# The checkpoint keeps down and injection projections separate; runtime packs +# them into adjacent logical shards of one MergedColumnParallelLinear. +_HC_WEIGHTS_MAPPER = WeightsMapper( + orig_to_new_stacked={ + "hyper_connection.input_mix_weight_down.weight": ( + "hyper_connection.input_mix_weight_down_block_inject.weight", + 0, + ), + "hyper_connection.block_inject_weight.weight": ( + "hyper_connection.input_mix_weight_down_block_inject.weight", + 1, + ), + } +) + + +class Qwen4ExpSparseMoeBlock(Qwen3NextSparseMoeBlock): + """Qwen3Next MoE with Qwen4Exp HC validation.""" + + def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None: + parallel_config = vllm_config.parallel_config + if parallel_config.use_sequence_parallel_moe: + raise NotImplementedError( + "Qwen4Exp HC does not support sequence-parallel MoE" + ) + super().__init__(vllm_config=vllm_config, prefix=prefix) + config = vllm_config.model_config.hf_text_config + self.n_shared_experts = int(config.shared_expert_intermediate_size > 0) + + +class Qwen4ExpDecoderLayer(nn.Module): + def __init__( + self, + vllm_config: VllmConfig, + layer_type: str, + prefix: str = "", + ) -> None: + super().__init__() + config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config + model_config = vllm_config.model_config + cache_config = vllm_config.cache_config + quant_config = vllm_config.quant_config + + self.config = config + self.layer_type = layer_type + self.layer_idx = extract_layer_index(prefix) + if vllm_config.parallel_config.use_sequence_parallel_moe: + raise NotImplementedError( + "Qwen4Exp HC does not support sequence-parallel MoE" + ) + self.ple: Qwen4ExpPLELayer | None = None + ple_layer_ids = config.ple_layer_ids + if (self.layer_idx + 1) in ple_layer_ids: + ple_layer_ids_sorted = sorted(set(ple_layer_ids)) + ple_dense_layer_id_map = { + abs_id: idx for idx, abs_id in enumerate(ple_layer_ids_sorted) + } + ple_dense_layer_id = ple_dense_layer_id_map[self.layer_idx + 1] + self.ple = Qwen4ExpPLELayer( + config, + vllm_config=vllm_config, + layer_idx=self.layer_idx, + ple_dense_layer_id=ple_dense_layer_id, + prefix=f"{prefix}.ple", + ) + + if layer_type == "linear_attention": + self.linear_attn = QwenGatedDeltaNetAttention( + config, + vllm_config=vllm_config, + prefix=f"{prefix}.linear_attn", + gqa_interleaved_layout=False, + ) + elif layer_type == "full_attention": + use_qsa = getattr(config, "indexer_n_heads", None) is not None + if not use_qsa: + self.self_attn = Qwen3NextAttention( + config, + model_config=model_config, + cache_config=cache_config, + quant_config=quant_config, + prefix=f"{prefix}.self_attn", + ) + else: + self.self_attn = Qwen4ExpQSAAttention( + vllm_config=vllm_config, + config=config, + layer_id=self.layer_idx, + quant_config=quant_config, + prefix=f"{prefix}.self_attn", + ) + else: + raise ValueError(f"Invalid layer_type {layer_type}") + + mlp_only_layers = getattr(config, "mlp_only_layers", []) + num_experts = getattr(config, "num_experts", 0) or 0 + absolute_layer_id = self.layer_idx + 1 + is_moe_layer = self.layer_idx not in mlp_only_layers and ( + num_experts > 0 and absolute_layer_id % config.decoder_sparse_step == 0 + ) + if is_moe_layer: + self.mlp = Qwen4ExpSparseMoeBlock( + vllm_config=vllm_config, prefix=f"{prefix}.mlp" + ) + else: + self.mlp = Qwen3NextMLP( + hidden_size=config.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + prefix=f"{prefix}.mlp", + ) + + hc_config = HyperConnectionConfig( + hc_count=config.hc_count, + hidden_size=config.hidden_size, + params_dtype=torch.bfloat16, + hc_lowrank=config.hc_lowrank, + rms_norm_eps=config.rms_norm_eps, + hc_per_branch_norm=True, + ) + self.attn_hyper_connection = GatedResidual( + hc_config, + prefix=maybe_prefix(prefix, "attn_hyper_connection"), + ) + self.mlp_hyper_connection = GatedResidual( + hc_config, + prefix=maybe_prefix(prefix, "mlp_hyper_connection"), + ) + + def forward( + self, + hidden_states: torch.Tensor, + prev_block_output: torch.Tensor | None, + prev_injection: torch.Tensor | None, + positions: torch.Tensor, + *, + input_ids: torch.Tensor | None, + query_start_loc: torch.Tensor | None, + ngram_context: torch.Tensor | None, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + attn_hc = self.attn_hyper_connection + if self.ple is not None: + # PLE adds directly to the multi-stream state, so pending HC state + # must be materialized before the addition. + if prev_block_output is not None and prev_injection is not None: + hidden_states = attn_hc.combine( + hidden_states, prev_block_output, prev_injection + ) + prev_block_output = prev_injection = None + + if input_ids is None or query_start_loc is None or ngram_context is None: + raise RuntimeError("PLE inputs were not prepared") + hidden_states = hidden_states + self.ple( + hidden_states, + input_ids, + query_start_loc, + ngram_context, + ) + + # Fuse a pending combine with this HC module's mix when possible. + if prev_block_output is not None and prev_injection is not None: + hidden_states, block_input, injection = attn_hc.combine_and_mix( + hidden_states, prev_block_output, prev_injection + ) + else: + hidden_states, block_input, injection = attn_hc.mix(hidden_states) + + if self.layer_type == "linear_attention": + attn_out = self.linear_attn(hidden_states=block_input) + elif self.layer_type == "full_attention": + attn_out = self.self_attn( + hidden_states=block_input, + positions=positions, + ) + else: + raise ValueError("Invalid layer_type") + + mlp_hc = self.mlp_hyper_connection + hidden_states, block_input, injection = mlp_hc.combine_and_mix( + hidden_states, attn_out, injection + ) + mlp_out = self.mlp(block_input) + return hidden_states, mlp_out, injection + + +class Qwen4ExpMixtureOfExperts(MixtureOfExperts): + """Expose Qwen4Exp routed experts through vLLM's EPLB protocol.""" + + def set_moe_parameters(self, layers: Iterable[nn.Module]) -> None: + self.moe_layers = [] + self.moe_mlp_layers = [] + example_moe = None + for layer in layers: + if isinstance(layer, Qwen4ExpDecoderLayer) and isinstance( + layer.mlp, Qwen4ExpSparseMoeBlock + ): + example_moe = layer.mlp + self.moe_mlp_layers.append(layer.mlp) + self.moe_layers.append(layer.mlp.experts) + + self.num_moe_layers = len(self.moe_layers) + if example_moe is None: + self.num_expert_groups = 0 + self.num_shared_experts = 0 + self.num_logical_experts = 0 + self.num_physical_experts = 0 + self.num_local_physical_experts = 0 + self.num_routed_experts = 0 + self.num_redundant_experts = 0 + return + + self.num_expert_groups = 1 + self.num_shared_experts = example_moe.n_shared_experts + self.num_logical_experts = example_moe.n_logical_experts + self.num_physical_experts = example_moe.n_physical_experts + self.num_local_physical_experts = example_moe.n_local_physical_experts + self.num_routed_experts = example_moe.n_routed_experts + self.num_redundant_experts = example_moe.n_redundant_experts + + def update_physical_experts_metadata( + self, + num_physical_experts: int, + num_local_physical_experts: int, + ) -> None: + assert self.num_local_physical_experts == num_local_physical_experts + self.num_physical_experts = num_physical_experts + self.num_local_physical_experts = num_local_physical_experts + self.num_redundant_experts = num_physical_experts - self.num_logical_experts + for moe in self.moe_mlp_layers: + moe.n_physical_experts = num_physical_experts + moe.n_local_physical_experts = num_local_physical_experts + moe.n_redundant_experts = self.num_redundant_experts + moe.experts.update_expert_map() + + +@support_torch_compile( + dynamic_arg_dims={ + "input_ids": 0, + "positions": -1, + "intermediate_tensors": 0, + "inputs_embeds": 0, + "query_start_loc": 0, + "ngram_context": 0, + "deepstack_input_embeds": 0, + } +) +class Qwen4ExpModel(nn.Module): + hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _HC_WEIGHTS_MAPPER + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + super().__init__() + config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config + self.config = config + self.num_redundant_experts = ( + vllm_config.parallel_config.eplb_config.num_redundant_experts + ) + self.vocab_size = config.vocab_size + self._qsa_layer_ids = frozenset( + layer_idx + for layer_idx, layer_type in enumerate(config.layer_types) + if layer_type == "full_attention" + and getattr(config, "indexer_n_heads", None) is not None + ) + self.embed_tokens = VocabParallelEmbedding(self.vocab_size, config.hidden_size) + + def get_layer(prefix: str) -> Qwen4ExpDecoderLayer: + layer_idx = extract_layer_index(prefix) + return Qwen4ExpDecoderLayer( + vllm_config, + layer_type=config.layer_types[layer_idx], + prefix=prefix, + ) + + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, get_layer, prefix=f"{prefix}.layers" + ) + self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible( + self.layers, + Qwen4ExpSparseMoeBlock, + "mlp", + ) + intermediate_size = config.hidden_size * config.hc_count + self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory( + ["hidden_states"], intermediate_size + ) + + self.hyper_connection_mixer: GatedResidual | None + if get_pp_group().is_last_rank: + hc_config = HyperConnectionConfig( + hc_count=config.hc_count, + hidden_size=config.hidden_size, + params_dtype=torch.bfloat16, + hc_lowrank=config.hc_lowrank, + rms_norm_eps=config.rms_norm_eps, + hc_per_branch_norm=True, + ) + self.hyper_connection_mixer = GatedResidual( + hc_config, + use_combine=False, + prefix=maybe_prefix(prefix, "hyper_connection_mixer"), + ) + else: + self.hyper_connection_mixer = None + + spec_config = vllm_config.speculative_config + # MTP HC multi-stream outputs: when speculative method=="mtp" and the + # model uses HC with hc_count>1, retain the pre-final-mixer multi-stream + # hidden state [T, hc_count*H] so the MTP drafter can feed a real + # multi-stream backbone hidden on its first step (scheme A). Derived + # purely from config (NOT node identity) so P/D nodes stay consistent. + needs_mtp_hidden = ( + spec_config is not None + and getattr(spec_config, "method", None) == "mtp" + and get_pp_group().is_last_rank + ) + if needs_mtp_hidden: + self._mtp_hidden_buffer = torch.empty( + vllm_config.scheduler_config.max_num_batched_tokens, + config.hc_count * config.hidden_size, + dtype=vllm_config.model_config.dtype, + ) + else: + self._mtp_hidden_buffer = None + + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.embed_tokens(input_ids) + + def forward( + self, + input_ids: torch.Tensor | None, + positions: torch.Tensor, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + query_start_loc: torch.Tensor | None = None, + ngram_context: torch.Tensor | None = None, + deepstack_input_embeds: IntermediateTensors | None = None, + ) -> torch.Tensor | IntermediateTensors: + if get_pp_group().is_first_rank: + if inputs_embeds is not None: + hidden_states = inputs_embeds + else: + if input_ids is None: + raise ValueError("input_ids or inputs_embeds is required") + hidden_states = self.embed_input_ids(input_ids) + hidden_states = hidden_states.repeat(1, self.config.hc_count) + else: + if intermediate_tensors is None: + raise ValueError("pipeline stage requires intermediate tensors") + hidden_states = intermediate_tensors["hidden_states"] + + block_output = None + injection = None + last_layer = None + for layer_idx, layer in islice( + enumerate(self.layers), self.start_layer, self.end_layer + ): + last_layer = layer + hidden_states, block_output, injection = layer( + hidden_states=hidden_states, + prev_block_output=block_output, + prev_injection=injection, + positions=positions, + input_ids=input_ids, + query_start_loc=query_start_loc, + ngram_context=ngram_context, + ) + if deepstack_input_embeds is not None and layer_idx < len( + deepstack_input_embeds + ): + deepstack_embed = deepstack_input_embeds[ + f"deepstack_input_embeds_{layer_idx}" + ] + deepstack_embed = ( + deepstack_embed.unsqueeze(-2) + .expand( + *deepstack_embed.shape[:-1], + self.config.hc_count, + self.config.hidden_size, + ) + .flatten(-2) + ) + # Deepstack is an external addition to the materialized + # multi-stream state and therefore terminates delayed combine. + hidden_states = layer.mlp_hyper_connection.combine( + hidden_states, block_output, injection + ) + block_output = None + injection = None + hidden_states = hidden_states + deepstack_embed + + if not get_pp_group().is_last_rank: + # PP transports one tensor, not the delayed HC tuple. Materialize + # with the HC module that produced the pending injection. + if last_layer is not None and block_output is not None: + hidden_states = last_layer.mlp_hyper_connection.combine( + hidden_states, block_output, injection + ) + return IntermediateTensors({"hidden_states": hidden_states}) + + # The final mixer consumes the last pending combine and returns both + # the sampled single stream and the materialized multi-stream state. + final_mixer = self.hyper_connection_mixer + assert final_mixer is not None + multi_hidden, sample_hidden_states, _ = final_mixer.combine_and_mix( + hidden_states, block_output, injection + ) + if self._mtp_hidden_buffer is not None: + # Capture the pre-final-mixer multi-stream hidden state + # [T, hc_count*H] for the MTP drafter (zero extra compute: + # this tensor is needed by the final mixer regardless). + num_tokens = multi_hidden.shape[0] + self._mtp_hidden_buffer[:num_tokens].copy_(multi_hidden) + return sample_hidden_states + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + weights = ( + ( + _remap_qsa_cache_scale_name(name, self._qsa_layer_ids), + weight, + ) + for name, weight in weights + ) + weights = maybe_fuse_shared_experts( + weights, + enabled=self.is_fused_shared_expert_enabled, + n_routed_experts=getattr(self.config, "num_experts", 0) or 0, + n_shared_experts=1, + ckpt_prefix="mlp.shared_expert", + ) + # Non-persistent PLE state rebuilt in __init__; skip any ckpt + # column for them. + skip_substrs = ( + "hashstats_", + "token_lookup", + "hyper_connection_mixer.block_inject_weight", + ) + mapper = self.hf_to_vllm_mapper | WeightsMapper( + orig_to_new_substr={substr: None for substr in skip_substrs} + ) + # The final HC mixer only exists on the last PP rank; earlier ranks + # must drop its checkpoint weights instead of failing to place them. + ignore_prefixes = ( + None + if self.hyper_connection_mixer is not None + else ["hyper_connection_mixer."] + ) + loader = AutoWeightsLoader( + self, + ignore_unexpected_prefixes=ignore_prefixes, + ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(), + ) + loaded = loader.load_weights( + weights, + mapper=mapper, + ) + return loaded + + +class Qwen4ExpForCausalLM( + nn.Module, + HasInnerState, + SupportsLoRA, + SupportsMRoPE, + SupportsPP, + Qwen4ExpMixtureOfExperts, + IsHybrid, +): + packed_modules_mapping = { + "qkv_proj": ["q_proj", "k_proj", "v_proj"], + "gate_up_proj": ["gate_proj", "up_proj"], + "in_proj_qkvz": ["in_proj_qkv", "in_proj_z"], + "in_proj_ba": ["in_proj_b", "in_proj_a"], + "input_mix_weight_down_block_inject": [ + "input_mix_weight_down", + "block_inject_weight", + "_input_mix_padding", + ], + } + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_prefix={"model.language_model.": "model."} + ) + requires_raw_input_tokens = True + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + super().__init__() + config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config + self.vllm_config = vllm_config + self.model_config = vllm_config.model_config + self.quant_config = vllm_config.quant_config + self.config = config + self.scheduler_config = vllm_config.scheduler_config + if vllm_config.cache_config.mamba_cache_mode == "all": + raise NotImplementedError( + "Qwen4Exp currently does not support 'all' prefix caching, " + "please use '--mamba-cache-mode=align' instead" + ) + self.model = Qwen4ExpModel( + vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") + ) + self.lm_head = ParallelLMHead( + config.vocab_size, + config.hidden_size, + prefix=maybe_prefix(prefix, "lm_head"), + ) + self.logits_processor = LogitsProcessor(config.vocab_size) + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors + ) + self.set_moe_parameters(self.model.layers) + enable_qwen4_exp_low_latency_gemm(self, self.model_config.dtype) + + @staticmethod + def get_model_state_cls(): + from .model_state import Qwen4ExpModelState + + return Qwen4ExpModelState + + def forward( + self, + input_ids: torch.Tensor | None, + positions: torch.Tensor, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + **kwargs: object, + ) -> torch.Tensor | IntermediateTensors: + # Forward kwargs unchanged so the runner's _maybe_add_ngram_kwargs + # path (query_start_loc / ngram_context) reaches Qwen4ExpModel. + return self.model( + input_ids, + positions, + intermediate_tensors, + inputs_embeds, + **kwargs, + ) + + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.model.embed_input_ids(input_ids) + + @classmethod + def get_ple_mamba_state_dtype_from_config( + cls, + vllm_config: VllmConfig, + ) -> tuple[torch.dtype, ...]: + return MambaStateDtypeCalculator.short_conv_state_dtype( + vllm_config.model_config.dtype, + vllm_config.cache_config.mamba_cache_dtype, + ) + + @classmethod + def get_ple_mamba_state_shape_from_config( + cls, vllm_config: VllmConfig + ) -> tuple[tuple[int, int]]: + hf_config = vllm_config.model_config.hf_text_config + conv_kernel_size = hf_config.ple_conv_kernel_size + short_conv_dilation = hf_config.ngram_size + conv_state_len = (conv_kernel_size - 1) * short_conv_dilation + num_spec = ( + vllm_config.speculative_config.num_speculative_tokens + if vllm_config.speculative_config + else 0 + ) + hc_count = hf_config.hc_count + hc_hidden_size = hf_config.hidden_size * hc_count + return MambaStateShapeCalculator.short_conv_state_shape( + tp_world_size=1, + intermediate_size=hc_hidden_size, + conv_kernel=conv_state_len + 1, + num_spec=num_spec, + ) + + @classmethod + def get_gdn_mamba_state_dtype_from_config( + cls, vllm_config: VllmConfig + ) -> tuple[torch.dtype, torch.dtype]: + return MambaStateDtypeCalculator.gated_delta_net_state_dtype( + vllm_config.model_config.dtype, + vllm_config.cache_config.mamba_cache_dtype, + vllm_config.cache_config.mamba_ssm_cache_dtype, + ) + + @classmethod + def get_gdn_mamba_state_shape_from_config( + cls, vllm_config: VllmConfig + ) -> tuple[tuple[int, int], tuple[int, int]]: + parallel_config = vllm_config.parallel_config + hf_config = vllm_config.model_config.hf_text_config + tp_size = parallel_config.tensor_parallel_size + num_spec = ( + vllm_config.speculative_config.num_speculative_tokens + if vllm_config.speculative_config + else 0 + ) + return MambaStateShapeCalculator.gated_delta_net_state_shape( + tp_size, + hf_config.linear_num_key_heads, + hf_config.linear_num_value_heads, + hf_config.linear_key_head_dim, + hf_config.linear_value_head_dim, + hf_config.linear_conv_kernel_dim, + num_spec, + ) + + @classmethod + def get_mamba_state_dtype_from_config( + cls, + vllm_config: VllmConfig, + ) -> tuple[torch.dtype, torch.dtype]: + return cls.get_gdn_mamba_state_dtype_from_config(vllm_config) + + @classmethod + def get_mamba_state_shape_from_config( + cls, vllm_config: VllmConfig + ) -> tuple[tuple[int, int], tuple[int, int]]: + return cls.get_gdn_mamba_state_shape_from_config(vllm_config) + + @classmethod + def get_mamba_state_copy_func( + cls, + ) -> tuple[MambaStateCopyFunc, MambaStateCopyFunc]: + return MambaStateCopyFuncCalculator.gated_delta_net_state_copy_func() + + @classmethod + def get_mamba_state_copy_funcs( + cls, + mamba_types: set[MambaAttentionBackendEnum], + ) -> MambaStateCopyFuncsByType: + copy_funcs_by_type = { + MambaAttentionBackendEnum.GDN_ATTN: cls.get_mamba_state_copy_func(), + MambaAttentionBackendEnum.SHORT_CONV: ( + MambaStateCopyFuncCalculator.short_conv_state_copy_func() + ), + } + missing_types = mamba_types - copy_funcs_by_type.keys() + assert not missing_types, f"missing state copy funcs for {missing_types}" + return { + mamba_type: copy_funcs_by_type[mamba_type] for mamba_type in mamba_types + } + + @classmethod + def get_mamba_specs_from_config( + cls, vllm_config: VllmConfig + ) -> tuple[MambaSpec, ...]: + """Return all MambaSpecs for this model (GDN layers + PLE layer). + + The PLE layer uses a separate short_conv MambaSpec whose page_size_bytes + may exceed the GDN spec; callers should take the maximum. + """ + return ( + MambaSpec( + shapes=cls.get_gdn_mamba_state_shape_from_config(vllm_config), + dtypes=cls.get_gdn_mamba_state_dtype_from_config(vllm_config), + block_size=-1, + ), + MambaSpec( + shapes=cls.get_ple_mamba_state_shape_from_config(vllm_config), + dtypes=cls.get_ple_mamba_state_dtype_from_config(vllm_config), + block_size=-1, + tp_replicated=True, + ), + ) + + def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None: + return self.logits_processor(self.lm_head, hidden_states) + + def get_mtp_target_hidden_states(self) -> torch.Tensor | None: + return self.model._mtp_hidden_buffer + + def get_mrope_input_positions( + self, + input_tokens: list[int], + mm_features: list[MultiModalFeatureSpec], + ) -> tuple[torch.Tensor, int]: + positions = torch.arange(len(input_tokens), dtype=torch.long) + return positions.unsqueeze(0).expand(3, -1), 0 + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + mapper = self.hf_to_vllm_mapper | WeightsMapper( + orig_to_new_substr={"mtp.": None} + ) + loader = AutoWeightsLoader( + self, + ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(), + ) + return loader.load_weights(weights, mapper=mapper) + + +class Qwen4ExpProcessingInfo(Qwen3VLProcessingInfo): + def get_hf_config(self) -> Qwen4ExpConfig: + return self.ctx.get_hf_config(Qwen4ExpConfig) + + +@MULTIMODAL_REGISTRY.register_processor( + Qwen3VLMultiModalProcessor, + info=Qwen4ExpProcessingInfo, + dummy_inputs=Qwen3VLDummyInputsBuilder, +) +class Qwen4ExpForConditionalGeneration( + Qwen3_5ForConditionalGeneration, + HasInnerState, + Qwen4ExpMixtureOfExperts, +): + """Qwen3-VL vision tower backed by the Qwen4Exp language model.""" + + requires_raw_input_tokens = True + + packed_modules_mapping = Qwen3_5ForConditionalGeneration.packed_modules_mapping | { + "input_mix_weight_down_block_inject": [ + "input_mix_weight_down", + "block_inject_weight", + "_input_mix_padding", + ] + } + + @staticmethod + def get_model_state_cls(): + from .model_state import Qwen4ExpModelState + + return Qwen4ExpModelState + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "model") -> None: + nn.Module.__init__(self) + config: Qwen4ExpConfig = vllm_config.model_config.hf_config + quant_config = vllm_config.quant_config + multimodal_config = vllm_config.model_config.multimodal_config + if multimodal_config is None: + raise ValueError( + "Qwen4ExpForConditionalGeneration requires multimodal_config" + ) + + self.config = config + self.model_config = vllm_config.model_config + self.multimodal_config = multimodal_config + self.language_model_only = multimodal_config.language_model_only + if self.language_model_only: + self.use_data_parallel = False + self.is_multimodal_pruning_enabled = False + self.video_pruning_method = None + self.video_pruning_rate = 0.0 + self._tokenizer = None + self.visual = StageMissingLayer("vision_tower") + self._tower_model_names = [] + else: + self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data" + self._init_video_pruning(multimodal_config) + self._tokenizer = cached_tokenizer_from_config(vllm_config.model_config) + + with self._mark_tower_model(vllm_config, {"image", "video"}): + self.visual = Qwen3_VisionTransformer( + config.vision_config, + norm_eps=config.text_config.rms_norm_eps, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "visual"), + ) + + self.use_deepstack = ( + not self.language_model_only + and bool(config.vision_config.deepstack_visual_indexes) + and not isinstance(self.visual, StageMissingLayer) + ) + self.deepstack_num_level = ( + len(config.vision_config.deepstack_visual_indexes) + if self.use_deepstack + else 0 + ) + self.visual_dim = config.vision_config.out_hidden_size + self.multiscale_dim = self.visual_dim * self.deepstack_num_level + + if self.use_deepstack: + self.deepstack_input_embeds = [ + torch.zeros( + vllm_config.scheduler_config.max_num_batched_tokens, + config.text_config.hidden_size, + ) + for _ in range(self.deepstack_num_level) + ] + self.deepstack_input_embeds_num_tokens = 0 + + with self._mark_language_model(vllm_config): + self.language_model = Qwen4ExpForCausalLM( + vllm_config=vllm_config, + prefix=maybe_prefix(prefix, "language_model"), + ) + + self.make_empty_intermediate_tensors = ( + self.language_model.make_empty_intermediate_tensors + ) + if not get_pp_group().is_first_rank and self.use_deepstack: + assert self.language_model.model.start_layer >= len( + config.vision_config.deepstack_visual_indexes + ), ( + "start_layer should be greater than or equal to " + "len(deepstack_visual_indexes)" + ) + self.set_moe_parameters(self.language_model.model.layers) + + def embed_input_ids( + self, + input_ids: torch.Tensor, + multimodal_embeddings: MultiModalEmbeddings | None = None, + *, + is_multimodal: torch.Tensor | None = None, + ) -> torch.Tensor: + inputs_embeds = self._embed_text_input_ids( + input_ids, + self.language_model.embed_input_ids, + is_multimodal=is_multimodal, + ) + if multimodal_embeddings is None or len(multimodal_embeddings) == 0: + return inputs_embeds + if self.language_model_only: + raise ValueError( + "Qwen4Exp language_model_only does not accept multimodal embeddings" + ) + + is_multimodal = _require_is_multimodal(is_multimodal) + if self.use_deepstack: + deepstack_input_embeds, multimodal_embeddings = ( + self._compute_deepstack_embeds( + inputs_embeds=inputs_embeds, + multimodal_embeddings=multimodal_embeddings, + is_multimodal=is_multimodal, + ) + ) + else: + deepstack_input_embeds = None + + inputs_embeds = _merge_multimodal_embeddings( + inputs_embeds=inputs_embeds, + multimodal_embeddings=multimodal_embeddings, + is_multimodal=is_multimodal, + ) + if deepstack_input_embeds is not None: + self._set_deepstack_input_embeds(deepstack_input_embeds) + return inputs_embeds + + def get_mtp_target_hidden_states(self) -> torch.Tensor | None: + return self.language_model.get_mtp_target_hidden_states() + + def forward( + self, + input_ids: torch.Tensor | None, + positions: torch.Tensor, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + **kwargs: object, + ) -> torch.Tensor | IntermediateTensors: + if intermediate_tensors is not None: + inputs_embeds = None + if inputs_embeds is not None and get_pp_group().is_first_rank: + deepstack_input_embeds = self._get_deepstack_input_embeds( + inputs_embeds.size(0) + ) + else: + deepstack_input_embeds = None + + hidden_states = self.language_model.model( + input_ids=input_ids, + positions=positions, + intermediate_tensors=intermediate_tensors, + inputs_embeds=inputs_embeds, + query_start_loc=kwargs.get("query_start_loc"), + ngram_context=kwargs.get("ngram_context"), + deepstack_input_embeds=deepstack_input_embeds, + ) + if inputs_embeds is not None and get_pp_group().is_first_rank: + self._clear_deepstack_input_embeds(inputs_embeds.size(0)) + return hidden_states + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + mapper = self.hf_to_vllm_mapper | WeightsMapper( + orig_to_new_substr={"mtp.": None}, + orig_to_new_prefix={"visual.": None} if self.language_model_only else {}, + ) + loader = AutoWeightsLoader( + self, + ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(), + ) + return loader.load_weights(weights, mapper=mapper) + + @classmethod + def get_mamba_state_dtype_from_config( + cls, + vllm_config: VllmConfig, + ) -> tuple[torch.dtype, torch.dtype]: + return Qwen4ExpForCausalLM.get_mamba_state_dtype_from_config(vllm_config) + + @classmethod + def get_mamba_state_shape_from_config( + cls, + vllm_config: VllmConfig, + ) -> tuple[tuple[int, int], tuple[int, int]]: + return Qwen4ExpForCausalLM.get_mamba_state_shape_from_config(vllm_config) + + @classmethod + def get_mamba_state_copy_func( + cls, + ) -> tuple[MambaStateCopyFunc, MambaStateCopyFunc]: + return Qwen4ExpForCausalLM.get_mamba_state_copy_func() + + @classmethod + def get_mamba_state_copy_funcs( + cls, + mamba_types: set[MambaAttentionBackendEnum], + ) -> MambaStateCopyFuncsByType: + return Qwen4ExpForCausalLM.get_mamba_state_copy_funcs(mamba_types) + + @classmethod + def get_mamba_specs_from_config( + cls, vllm_config: VllmConfig + ) -> tuple[MambaSpec, ...]: + return Qwen4ExpForCausalLM.get_mamba_specs_from_config(vllm_config) + + +__all__ = [ + "Qwen4ExpDecoderLayer", + "Qwen4ExpForCausalLM", + "Qwen4ExpForConditionalGeneration", + "Qwen4ExpMixtureOfExperts", + "Qwen4ExpModel", + "Qwen4ExpSparseMoeBlock", +] diff --git a/vllm/models/qwen4_exp/amd/model_state.py b/vllm/models/qwen4_exp/amd/model_state.py new file mode 100644 index 0000000000..ec2a0fef8b --- /dev/null +++ b/vllm/models/qwen4_exp/amd/model_state.py @@ -0,0 +1,143 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Model-runner state for Qwen4Exp PLE inputs.""" + +from typing import Any + +import torch +import torch.nn as nn + +from vllm.config import VllmConfig +from vllm.v1.worker.gpu.input_batch import InputBatch +from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache +from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState +from vllm.v1.worker.gpu.states import RequestState + + +class Qwen4ExpModelState(MambaHybridModelState): + """Add rollback-safe PLE n-gram context to the model inputs.""" + + def __init__( + self, + vllm_config: VllmConfig, + model: nn.Module, + encoder_cache: EncoderCache | None, + device: torch.device, + ) -> None: + super().__init__(vllm_config, model, encoder_cache, device) + config = self.model_config.hf_text_config + self.uses_ngram_embedding = bool(config.ple_layer_ids) + if not self.uses_ngram_embedding: + self.ngram_context_len = 0 + self.ngram_eos_token_id = 0 + return + + if vllm_config.parallel_config.pipeline_parallel_size > 1: + raise RuntimeError( + "N-gram PLE embedding currently requires " + "pipeline_parallel_size=1 because non-first pipeline ranks do " + "not receive the raw input_ids required by PLE. Please run " + "with PP=1." + ) + + self.ngram_context_len = int(config.ngram_size) - 1 + if self.ngram_context_len <= 0: + raise ValueError("N-gram embedding requires context length >= 1.") + self.ngram_eos_token_id = int(config.eos_token_id) + self.ngram_context = torch.full( + (self.max_num_reqs, self.ngram_context_len), + self.ngram_eos_token_id, + dtype=torch.int32, + device=self.device, + ) + self.ngram_context_offsets = torch.arange( + -self.ngram_context_len, + 0, + dtype=torch.int64, + device=self.device, + ) + self.ple_query_start_loc = torch.zeros( + self.max_num_reqs + 1, + dtype=torch.int32, + device=self.device, + ) + + def _prepare_ngram_context( + self, + input_batch: InputBatch, + req_states: RequestState, + ) -> torch.Tensor: + num_reqs = input_batch.num_reqs + num_reqs_padded = input_batch.num_reqs_after_padding + context = self.ngram_context[:num_reqs_padded] + context.fill_(self.ngram_eos_token_id) + if num_reqs == 0: + return context + + request_indices = input_batch.idx_mapping[:num_reqs].long() + context_end = req_states.num_computed_tokens.gpu[request_indices].long() + token_indices = context_end.unsqueeze(1) + self.ngram_context_offsets + valid_tokens = token_indices >= 0 + token_indices.clamp_min_(0) + context_tokens = req_states.all_token_ids.gpu[ + request_indices.unsqueeze(1), token_indices + ] + context[:num_reqs].copy_( + torch.where( + valid_tokens, + context_tokens, + context_tokens.new_full((), self.ngram_eos_token_id), + ) + ) + return context + + def prepare_inputs( + self, + input_batch: InputBatch, + req_states: RequestState, + ) -> dict[str, Any]: + model_inputs = super().prepare_inputs(input_batch, req_states) + if not self.uses_ngram_embedding: + return model_inputs + + num_reqs_padded = input_batch.num_reqs_after_padding + query_start_loc = self.ple_query_start_loc[: num_reqs_padded + 1] + query_start_loc.copy_(input_batch.query_start_loc[: num_reqs_padded + 1]) + model_inputs.update( + query_start_loc=query_start_loc, + ngram_context=self._prepare_ngram_context(input_batch, req_states), + ) + return model_inputs + + def prepare_dummy_inputs( + self, + num_reqs: int, + num_tokens: int, + ) -> dict[str, Any]: + model_inputs = super().prepare_dummy_inputs(num_reqs, num_tokens) + if not self.uses_ngram_embedding: + return model_inputs + + query_start_loc = self.ple_query_start_loc[: num_reqs + 1] + query_start_loc[0] = 0 + tokens_per_req, num_extra_tokens = divmod(num_tokens, num_reqs) + query_lens = torch.full( + (num_reqs,), + tokens_per_req, + dtype=query_start_loc.dtype, + device=query_start_loc.device, + ) + if num_extra_tokens > 0: + query_lens[-num_extra_tokens:] += 1 + torch.cumsum(query_lens, dim=0, out=query_start_loc[1:]) + + ngram_context = self.ngram_context[:num_reqs] + ngram_context.fill_(self.ngram_eos_token_id) + model_inputs.update( + query_start_loc=query_start_loc, + ngram_context=ngram_context, + ) + return model_inputs + + +__all__ = ["Qwen4ExpModelState"] diff --git a/vllm/models/qwen4_exp/amd/mtp.py b/vllm/models/qwen4_exp/amd/mtp.py new file mode 100644 index 0000000000..5a77448eca --- /dev/null +++ b/vllm/models/qwen4_exp/amd/mtp.py @@ -0,0 +1,462 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inference-only Qwen4Exp MTP (Multi-Token Predictor) model. + +The MTP draft model reuses the Qwen4Exp backbone (PLE/HC/MoE) but: + - drops all multi-modal handling (text-only), + - forces PLE off while keeping the main model's HC stream count, + - fuses the backbone hidden and the new-token embedding via + ``residual_linear_shared`` (fc_embedding + shared fc_hidden) instead of + the ``Linear(2H, H)`` + repeat used by other MTP variants, + - emits TWO hidden streams per step (scheme A): a single stream [T, H] + (final-mixer collapsed, fed to the LM head) and a pre-final-mixer + multi stream [T, hc_count*H] (fed to the next draft step). +""" + +from collections.abc import Iterable + +import regex as re +import torch +from torch import nn + +from vllm.compilation.decorators import support_torch_compile +from vllm.config import VllmConfig, replace, set_current_vllm_config +from vllm.distributed import get_pp_group +from vllm.model_executor.layers.fused_moe.utils import ( + is_model_fused_shared_expert_compatible, +) +from vllm.model_executor.layers.layernorm import GemmaRMSNorm +from vllm.model_executor.layers.linear import ColumnParallelLinear +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, + VocabParallelEmbedding, +) +from vllm.model_executor.model_loader.utils import configure_quant_config +from vllm.model_executor.models.interfaces import SupportsPP +from vllm.model_executor.models.qwen3_5 import Qwen3_5Model +from vllm.model_executor.models.utils import ( + AutoWeightsLoader, + PPMissingLayer, + WeightsMapper, + get_draft_quant_config, + make_empty_intermediate_tensors_factory, + maybe_fuse_shared_experts, + maybe_prefix, +) +from vllm.sequence import IntermediateTensors +from vllm.transformers_utils.configs.qwen4_exp import ( + Qwen4ExpTextConfig, +) + +from .hyperconnection import GatedResidual, HyperConnectionConfig +from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm +from .model import ( + _HC_WEIGHTS_MAPPER, + _QWEN4_EXP_IGNORED_MISSING_SUFFIXES, + Qwen4ExpDecoderLayer, + Qwen4ExpMixtureOfExperts, + Qwen4ExpSparseMoeBlock, +) + + +def _remap_ignored_layers( + ignored_layers: list[str], + mtp_start_layer_idx: int, +) -> list[str]: + remapped: list[str] = [] + for name in ignored_layers: + if name.startswith("mtp."): + new_name = re.sub( + r"(?<=\.layers\.)\d+", + lambda m: str(mtp_start_layer_idx + int(m.group(0))), + name, + ) + remapped.append(new_name) + else: + remapped.append(name) + return remapped + + +def _remap_mtp_weight_name(name: str) -> str | None: + """Map Qwen4Exp checkpoint paths into the standalone draft model.""" + + for checkpoint_prefix in ( + "model.language_model.", + "language_model.", + ): + if name.startswith(checkpoint_prefix): + name = name.removeprefix(checkpoint_prefix) + break + + if name.startswith("embed_tokens."): + name = f"model.{name}" + if name.startswith("model.mtp."): + name = name.removeprefix("model.") + if name.startswith("mtp.shared_head.head."): + return name.replace("mtp.shared_head.head.", "lm_head.", 1) + if name.startswith("model.shared_head.head."): + return name.replace("model.shared_head.head.", "lm_head.", 1) + if name.startswith("shared_head.head."): + return name.replace("shared_head.head.", "lm_head.", 1) + if name.startswith("model.lm_head."): + return name.removeprefix("model.") + if name.startswith("mtp."): + return name.replace("mtp.", "model.", 1) + if name.startswith("model.embed_tokens.") or name.startswith("lm_head."): + return name + return None + + +def _make_draft_vllm_config( + vllm_config: VllmConfig, + mtp_start_layer_idx: int, +) -> VllmConfig: + """Ensure that the draft model config is set in the vLLM config.""" + speculative_config = vllm_config.speculative_config + if speculative_config is None or speculative_config.draft_model_config is None: + raise ValueError("speculative_config.draft_model_config must be set") + + draft_quant_config = get_draft_quant_config(vllm_config) + + # inject packed and ignored modules to the quantization config of draft model + if draft_quant_config is not None: + configure_quant_config(draft_quant_config, Qwen4ExpMTP) + ignored_layers = getattr(draft_quant_config, "ignored_layers", None) + if ignored_layers: + setattr( # noqa: B010 + draft_quant_config, + "ignored_layers", + _remap_ignored_layers(ignored_layers, mtp_start_layer_idx), + ) + exclude_modules = getattr(draft_quant_config, "exclude_modules", None) + if exclude_modules: + setattr( # noqa: B010 + draft_quant_config, + "exclude_modules", + _remap_ignored_layers(exclude_modules, mtp_start_layer_idx), + ) + + draft_vllm_config = replace( + vllm_config, + model_config=speculative_config.draft_model_config, + ) + # VllmConfig post-init derives the target quant config, so restore the + # independently resolved draft quant config after replacement. + draft_vllm_config.quant_config = draft_quant_config + return draft_vllm_config + + +@support_torch_compile( + dynamic_arg_dims={ + "input_ids": 0, + "positions": -1, + "intermediate_tensors": 0, + "inputs_embeds": 0, + "hidden_states": 0, + } +) +class Qwen4ExpMultiTokenPredictor(nn.Module): + hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _HC_WEIGHTS_MAPPER + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + super().__init__() + + model_config = vllm_config.model_config + config: Qwen4ExpTextConfig = model_config.hf_text_config + + self.config = config + self.vocab_size = config.vocab_size + + self.mtp_start_layer_idx = config.num_hidden_layers + self.num_mtp_layers = getattr(config, "mtp_num_hidden_layers", 1) + + self.hidden_size = config.hidden_size + self.hc_count = config.hc_count + + self.embed_tokens = VocabParallelEmbedding(self.vocab_size, self.hidden_size) + draft_vllm_config = _make_draft_vllm_config( + vllm_config, + self.mtp_start_layer_idx, + ) + with set_current_vllm_config(draft_vllm_config, prefix=prefix): + # residual_linear_shared fusion: fc_embedding projects the token + # embedding, fc_hidden (shared across HC branches) projects the + # backbone hidden; the embedding is added as a residual to every + # branch (see mtp_residual_linear_shared.md). + self.fc_embedding = ColumnParallelLinear( + self.hidden_size, + self.hidden_size, + gather_output=True, + bias=False, + return_bias=False, + quant_config=draft_vllm_config.quant_config, + prefix=f"{prefix}.fc_embedding", + ) + self.fc_hidden = ColumnParallelLinear( + self.hidden_size, + self.hidden_size, + gather_output=True, + bias=False, + return_bias=False, + quant_config=draft_vllm_config.quant_config, + prefix=f"{prefix}.fc_hidden", + ) + self.layers = nn.ModuleList( + Qwen4ExpDecoderLayer( + draft_vllm_config, + layer_type="full_attention", + prefix=f"{prefix}.layers.{self.mtp_start_layer_idx + idx}", + ) + for idx in range(self.num_mtp_layers) + ) + self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible( + self.layers, + Qwen4ExpSparseMoeBlock, + "mlp", + ) + + self.pre_fc_norm_embedding = GemmaRMSNorm( + self.hidden_size, eps=config.rms_norm_eps + ) + self.pre_fc_norm_hidden = GemmaRMSNorm( + self.hidden_size * self.hc_count, eps=config.rms_norm_eps + ) + # HC final mixer collapses the multi stream into [T, H] for the LM head. + hc_config = HyperConnectionConfig( + hc_count=config.hc_count, + hidden_size=config.hidden_size, + params_dtype=torch.bfloat16, + hc_lowrank=config.hc_lowrank, + rms_norm_eps=config.rms_norm_eps, + hc_per_branch_norm=True, + ) + self.hyper_connection_mixer = GatedResidual( + hc_config, + use_combine=False, + prefix=maybe_prefix(prefix, "hyper_connection_mixer"), + ) + self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory( + ["hidden_states"], self.hidden_size * self.hc_count + ) + + def _iter_qsa_attentions(self): + """Yield MTP attention modules that own a QSA indexer.""" + + for layer in self.layers: + attention = getattr(layer, "self_attn", None) + if ( + attention is not None + and getattr(attention, "indexer", None) is not None + ): + yield attention + + def set_skip_topk(self, skip: bool) -> None: + """Select on MTP step 0 and reuse its QSA indices on later steps.""" + + for attention in self._iter_qsa_attentions(): + attention.indexer.skip_topk = skip + + def compact_topk_indices(self, row_indices: torch.Tensor) -> None: + """Keep each request's target-aligned step-0 sparse-index row.""" + + num_rows = row_indices.numel() + for attention in self._iter_qsa_attentions(): + buffer = attention.topk_indices_buffer + selected = buffer.index_select(0, row_indices) + buffer[:num_rows].copy_(selected) + + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.embed_tokens(input_ids) + + def forward( + self, + input_ids: torch.Tensor | None, + positions: torch.Tensor, + hidden_states: torch.Tensor | None = None, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + spec_step_idx: int = 0, + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor] | IntermediateTensors: + hc_count = self.hc_count + hidden_size = self.hidden_size + + if get_pp_group().is_first_rank: + assert hidden_states is not None + if inputs_embeds is None: + assert input_ids is not None + inputs_embeds = self.embed_input_ids(input_ids) + # Embedding branch: pre-norm -> fc_embedding -> [T, H]. + inputs_embeds = self.pre_fc_norm_embedding(inputs_embeds) + inputs_embeds = self.fc_embedding(inputs_embeds) + + # Backbone hidden is multi-stream [T, hc_count*H] (scheme A: + # the main model truly emits the pre-final-mixer multi stream + # on the first step; subsequent steps reuse the prior draft + # step's multi stream). + num_tokens = hidden_states.shape[0] + hidden_states = hidden_states.view(num_tokens, hc_count, hidden_size) + hidden_states = self.pre_fc_norm_hidden(hidden_states.flatten(-2)).view( + num_tokens, hc_count, hidden_size + ) + hidden_states = self.fc_hidden(hidden_states) + # Add the embedding residual to every branch, then fold back + # to [T, hc_count*H] (HC outer, HS inner) for the HC decoder. + hidden_states = inputs_embeds.unsqueeze(-2) + hidden_states + hidden_states = hidden_states.flatten(-2) + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + + current_step_idx = spec_step_idx % self.num_mtp_layers + layer = self.layers[current_step_idx] + hidden_states, block_output, injection = layer( + hidden_states=hidden_states, + prev_block_output=None, + prev_injection=None, + positions=positions, + input_ids=None, + query_start_loc=None, + ngram_context=None, + ) + if not get_pp_group().is_last_rank: + # As in the target model, PP carries a materialized tensor rather + # than the delayed hidden/output/injection tuple. + hidden_states = layer.mlp_hyper_connection.combine( + hidden_states, block_output, injection + ) + return IntermediateTensors({"hidden_states": hidden_states}) + + # Last PP rank finalize. Keep both: + # (A) sample_hidden_states [T, H] -> single stream for the LM head + # (B) multi_hidden [T, hc_count*H] -> pre-final-mixer multi stream + # for the next draft step (zero extra compute, just kept). + multi_hidden, sample_hidden_states, _ = ( + self.hyper_connection_mixer.combine_and_mix( + hidden_states, block_output, injection + ) + ) + return sample_hidden_states, multi_hidden + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + weights = maybe_fuse_shared_experts( + weights, + enabled=self.is_fused_shared_expert_enabled, + n_routed_experts=getattr(self.config, "num_experts", 0) or 0, + n_shared_experts=1, + ckpt_prefix="mlp.shared_expert", + ) + mapper = self.hf_to_vllm_mapper | WeightsMapper( + orig_to_new_substr={"hyper_connection_mixer.block_inject_weight": None} + ) + loader = AutoWeightsLoader( + self, + ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(), + ) + return loader.load_weights(weights, mapper=mapper) + + +@support_torch_compile( + dynamic_arg_dims={ + "input_ids": 0, + "positions": -1, + "intermediate_tensors": 0, + "inputs_embeds": 0, + "hidden_states": 0, + } +) +class Qwen4ExpMTP(nn.Module, SupportsPP, Qwen4ExpMixtureOfExperts): + packed_modules_mapping = { + "qkv_proj": ["q_proj", "k_proj", "v_proj"], + "gate_up_proj": ["gate_proj", "up_proj"], + "in_proj_qkvz": ["in_proj_qkv", "in_proj_z"], + "in_proj_ba": ["in_proj_b", "in_proj_a"], + "input_mix_weight_down_block_inject": [ + "input_mix_weight_down", + "block_inject_weight", + "_input_mix_padding", + ], + } + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config + self.vllm_config = vllm_config + cache_config = vllm_config.cache_config + if cache_config.mamba_cache_mode == "all": + raise NotImplementedError( + "Qwen4ExpMTP currently does not support 'all' prefix caching, " + "please use '--mamba-cache-mode=align' instead" + ) + + self.quant_config = vllm_config.quant_config + + super().__init__() + self.config = config + self.model = Qwen4ExpMultiTokenPredictor( + vllm_config=vllm_config, + prefix=maybe_prefix(prefix, "mtp"), + ) + + if get_pp_group().is_last_rank: + if config.tie_word_embeddings: + self.lm_head = self.model.embed_tokens + else: + self.lm_head = ParallelLMHead( + config.vocab_size, + config.hidden_size, + prefix=maybe_prefix(prefix, "lm_head"), + ) + else: + self.lm_head = PPMissingLayer() + + self.logits_processor = LogitsProcessor(config.vocab_size) + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors + ) + self.set_moe_parameters(self.model.layers) + enable_qwen4_exp_low_latency_gemm(self, vllm_config.model_config.dtype) + + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.model.embed_input_ids(input_ids) + + def forward( + self, + input_ids: torch.Tensor | None, + positions: torch.Tensor, + hidden_states: torch.Tensor | None = None, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + spec_step_idx: int = 0, + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor] | IntermediateTensors: + return self.model( + input_ids, + positions, + hidden_states, + intermediate_tensors, + inputs_embeds, + spec_step_idx=spec_step_idx, + ) + + def compute_logits( + self, hidden_states: torch.Tensor, spec_step_idx: int = 0 + ) -> torch.Tensor | None: + return self.logits_processor(self.lm_head, hidden_states) + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + def remap_weight_names(): + for name, weight in weights: + remapped_name = _remap_mtp_weight_name(name) + if remapped_name is not None: + yield remapped_name, weight + + mapper = WeightsMapper( + orig_to_new_substr={"hyper_connection_mixer.block_inject_weight": None} + ) + loader = AutoWeightsLoader( + self, + ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(), + ) + return loader.load_weights(remap_weight_names(), mapper=mapper) + + +__all__ = ["Qwen4ExpMTP", "Qwen4ExpMultiTokenPredictor"] diff --git a/vllm/models/qwen4_exp/amd/ops/__init__.py b/vllm/models/qwen4_exp/amd/ops/__init__.py new file mode 100644 index 0000000000..56c33274ef --- /dev/null +++ b/vllm/models/qwen4_exp/amd/ops/__init__.py @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""AMD ROCm Qwen4Exp kernels; import leaf modules directly.""" diff --git a/vllm/models/qwen4_exp/amd/ops/hc.py b/vllm/models/qwen4_exp/amd/ops/hc.py new file mode 100644 index 0000000000..1faf89b020 --- /dev/null +++ b/vllm/models/qwen4_exp/amd/ops/hc.py @@ -0,0 +1,487 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""AMD ROCm HyperConnection kernels for Qwen4Exp.""" + +import torch + +from vllm.platforms import current_platform +from vllm.triton_utils import tl, triton +from vllm.utils.torch_utils import direct_register_custom_op + + +@triton.jit +def _grouped_gemma_rmsnorm_kernel( + x_ptr, + w_ptr, + y_ptr, + stride_x, + stride_y, + DIM: tl.constexpr, + NUM_GROUPS: tl.constexpr, + W_SHARED: tl.constexpr, + EPS: tl.constexpr, + launch_pdl: tl.constexpr, +) -> None: + GROUP_DIM: tl.constexpr = DIM // NUM_GROUPS + BLOCK_SIZE: tl.constexpr = triton.next_power_of_2(GROUP_DIM) + + pid = tl.program_id(0) + group_id = pid % NUM_GROUPS + row = pid // NUM_GROUPS + + offs_g = tl.arange(0, BLOCK_SIZE) + offsets = group_id * GROUP_DIM + offs_g + mask = offs_g < GROUP_DIM + # A [GROUP_DIM] affine is shared; a [DIM] affine follows the grouped + # checkpoint layout. + w_offs = offs_g if W_SHARED else offsets + + if launch_pdl: + tl.extra.cuda.gdc_wait() + + x = tl.load(x_ptr + row * stride_x + offsets, mask, other=0.0).to(tl.float32) + w = tl.load(w_ptr + w_offs, mask, other=0.0) + + rrms = tl.rsqrt(tl.sum(x * x) / GROUP_DIM + EPS) + # Gemma's (1 + w) affine is written this way to lower to an FMA. + y = x * rrms + y += y * w.to(tl.float32) + + if launch_pdl: + tl.extra.cuda.gdc_launch_dependents() + tl.store(y_ptr + row * stride_y + offsets, y, mask) + + +def _grouped_gemma_rmsnorm( + x: torch.Tensor, weight: torch.Tensor, eps: float, num_groups: int +) -> torch.Tensor: + N, DIM = x.shape + assert x.stride(1) == 1, "grouped Gemma RMSNorm requires unit inner stride" + assert weight.is_contiguous(), "grouped Gemma RMSNorm weight must be contiguous" + assert DIM % num_groups == 0 + group_dim = DIM // num_groups + assert weight.numel() in (group_dim, DIM) + + y = x.new_empty(x.shape) + _grouped_gemma_rmsnorm_kernel[(N * num_groups,)]( + x, + weight, + y, + x.stride(0), + y.stride(0), + DIM, + num_groups, + W_SHARED=weight.numel() == group_dim, + EPS=eps, + launch_pdl=current_platform.is_arch_support_pdl(), + ) + return y + + +@triton.jit +def _hc_silu_kernel( + x_ptr, + y_ptr, + stride_x, + stride_y, + DIM: tl.constexpr, + HC: tl.constexpr, + launch_pdl: tl.constexpr, +) -> None: + BLOCK_SIZE: tl.constexpr = triton.next_power_of_2(DIM) + + row = tl.program_id(0) + offs = tl.arange(0, BLOCK_SIZE) + mask = offs < DIM + + if launch_pdl: + tl.extra.cuda.gdc_wait() + + x = tl.load(x_ptr + row * stride_x + offs, mask).to(tl.float32) / HC + y = x * tl.sigmoid(x) + + if launch_pdl: + tl.extra.cuda.gdc_launch_dependents() + tl.store(y_ptr + row * stride_y + offs, y, mask) + + +def _hc_silu(x: torch.Tensor, hc_count: int) -> torch.Tensor: + num_tokens, DIM = x.shape + assert x.stride(1) == 1 + + output = x.new_empty(x.shape) + _hc_silu_kernel[(num_tokens,)]( + x, + output, + x.stride(0), + output.stride(0), + DIM=DIM, + HC=hc_count, + launch_pdl=current_platform.is_arch_support_pdl(), + ) + return output + + +@triton.jit +def _hc_gate_mix_kernel( + x_ptr, + g_ptr, + y_ptr, + stride_x, + stride_g, + stride_y, + DIM: tl.constexpr, + HC: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + launch_pdl: tl.constexpr, +) -> None: + HC_DIM: tl.constexpr = DIM // HC + + row = tl.program_id(0) + tile_id = tl.program_id(1) + offs_inner = tile_id * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) + mask = offs_inner < HC_DIM + + if launch_pdl: + tl.extra.cuda.gdc_wait() + + # The constexpr loop is unrolled and keeps one stream live at a time. + # Materializing [HC, BLOCK_SIZE] more than doubles latency at large M. + acc = tl.zeros([BLOCK_SIZE], dtype=tl.float32) + for stream in tl.static_range(HC): + offsets = stream * HC_DIM + offs_inner + g = tl.load(g_ptr + row * stride_g + offsets, mask, other=0.0) + x = tl.load(x_ptr + row * stride_x + offsets, mask, other=0.0) + acc += tl.sigmoid(g.to(tl.float32)) * x.to(tl.float32) + acc /= HC + + if launch_pdl: + tl.extra.cuda.gdc_launch_dependents() + tl.store(y_ptr + row * stride_y + offs_inner, acc, mask) + + +def _hc_gate_mix(x: torch.Tensor, gate: torch.Tensor, hc_count: int) -> torch.Tensor: + N, DIM = gate.shape + assert x.shape == gate.shape + assert DIM % hc_count == 0 + assert x.stride(1) == 1 + assert gate.stride(1) == 1 + + HC_DIM = DIM // hc_count + out = x.new_empty(N, HC_DIM) + BLOCK_SIZE = 512 + _hc_gate_mix_kernel[(N, triton.cdiv(HC_DIM, BLOCK_SIZE))]( + x, + gate, + out, + x.stride(0), + gate.stride(0), + out.stride(0), + DIM, + hc_count, + BLOCK_SIZE, + launch_pdl=current_platform.is_arch_support_pdl(), + ) + return out + + +@triton.jit +def _hc_combine_kernel( + block_ptr, + res_ptr, + inj_ptr, + out_ptr, + stride_block, + stride_res, + stride_inj, + stride_out, + HC_DIM: tl.constexpr, + HC: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + launch_pdl: tl.constexpr, +) -> None: + HC_PAD: tl.constexpr = triton.next_power_of_2(HC) + + row = tl.program_id(0) + tile_id = tl.program_id(1) + + offs_inner = tile_id * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) + mask_inner = offs_inner < HC_DIM + offs_hc = tl.arange(0, HC_PAD) + mask_hc = offs_hc < HC + offs = offs_hc[:, None] * HC_DIM + offs_inner[None, :] + mask = mask_hc[:, None] & mask_inner[None, :] + + if launch_pdl: + tl.extra.cuda.gdc_wait() + + inj = tl.load(inj_ptr + row * stride_inj + offs_hc, mask_hc, other=0.0) + block = tl.load(block_ptr + row * stride_block + offs_inner, mask_inner, other=0.0) + res = tl.load(res_ptr + row * stride_res + offs, mask, other=0.0) + + # Keeping HC as a broadcast dimension is faster here than four separate + # residual load/store sequences. + inj = 2.0 * tl.sigmoid(inj.to(tl.float32) / HC) + out = res.to(tl.float32) + block.to(tl.float32)[None, :] * inj[:, None] + + if launch_pdl: + tl.extra.cuda.gdc_launch_dependents() + tl.store(out_ptr + row * stride_out + offs, out, mask=mask) + + +def _hc_combine( + residual: torch.Tensor, + block_output: torch.Tensor, + injection_logits: torch.Tensor, + hc_count: int, +) -> torch.Tensor: + N, DIM = residual.shape + assert DIM % hc_count == 0 + hc_dim = DIM // hc_count + assert block_output.shape == (N, hc_dim) + assert injection_logits.shape == (N, hc_count) + assert residual.stride(1) == 1 + assert block_output.stride(1) == 1 + assert injection_logits.stride(1) == 1 + + out = residual.new_empty(residual.shape) + BLOCK_SIZE = 512 + _hc_combine_kernel[(N, triton.cdiv(hc_dim, BLOCK_SIZE))]( + block_output, + residual, + injection_logits, + out, + block_output.stride(0), + residual.stride(0), + injection_logits.stride(0), + out.stride(0), + hc_dim, + hc_count, + BLOCK_SIZE, + launch_pdl=current_platform.is_arch_support_pdl(), + ) + return out + + +@triton.jit +def _hc_combine_norm_kernel( + block_ptr, + res_ptr, + inj_ptr, + w_ptr, + out_ptr, + y_ptr, + stride_block, + stride_res, + stride_inj, + stride_out, + stride_y, + HC_DIM: tl.constexpr, + HC: tl.constexpr, + W_SHARED: tl.constexpr, + EPS: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + launch_pdl: tl.constexpr, +) -> None: + HC_PAD: tl.constexpr = triton.next_power_of_2(HC) + NUM_TILES: tl.constexpr = triton.cdiv(HC_DIM, BLOCK_SIZE) + NUM_TILES_PAD: tl.constexpr = triton.next_power_of_2(NUM_TILES) + + row = tl.program_id(0) + stream = tl.program_id(1) + offs_hc = tl.arange(0, HC_PAD) + mask_hc = offs_hc < HC + tile_ids = tl.arange(0, NUM_TILES_PAD) + offs_inner = tile_ids[:, None] * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)[None, :] + mask_inner = offs_inner < HC_DIM + offs = stream * HC_DIM + offs_inner + # Shared norm weights repeat across streams; per-branch weights use the + # same flattened HC layout as the residual. + w_offs = offs_inner if W_SHARED else offs + + if launch_pdl: + tl.extra.cuda.gdc_wait() + + # Start the uncached residual load first, then issue the other combine + # loads before consuming any of them. + res = tl.load(res_ptr + row * stride_res + offs, mask_inner, other=0.0) + inj = tl.load(inj_ptr + row * stride_inj + offs_hc, mask_hc, other=0.0) + block = tl.load(block_ptr + row * stride_block + offs_inner, mask_inner, other=0.0) + inj = 2.0 * tl.sigmoid(inj.to(tl.float32) / HC) + inj = tl.sum(tl.where(offs_hc == stream, inj, 0.0)) + # Round the materialized combine result before normalization. This matches + # the unfused combine -> RMSNorm boundary. + out = (res.to(tl.float32) + block.to(tl.float32) * inj).to(out_ptr.dtype.element_ty) + tl.store(out_ptr + row * stride_out + offs, out, mask=mask_inner) + + out = out.to(tl.float32) + # Keep the two-axis reduction: flattening the padded tile is ~40% slower + # at decode sizes. + sum_sq = tl.sum(tl.sum(out * out, axis=1), axis=0) + rrms = tl.rsqrt(sum_sq / HC_DIM + EPS) + + if launch_pdl: + tl.extra.cuda.gdc_launch_dependents() + + # Loading the weight earlier helps decode but keeps the tile live across + # the reduction and regresses larger batches, so defer it to the norm. + w = tl.load(w_ptr + w_offs, mask_inner, other=0.0) + y = out * rrms + y += y * w.to(tl.float32) + tl.store(y_ptr + row * stride_y + offs, y, mask_inner) + + +def _hc_combine_norm( + residual: torch.Tensor, + block_output: torch.Tensor, + injection_logits: torch.Tensor, + norm_weight: torch.Tensor, + eps: float, + hc_count: int, +) -> tuple[torch.Tensor, torch.Tensor]: + N, DIM = residual.shape + assert DIM % hc_count == 0 + hc_dim = DIM // hc_count + assert block_output.shape == (N, hc_dim) + assert injection_logits.shape == (N, hc_count) + assert residual.stride(1) == 1 + assert block_output.stride(1) == 1 + assert injection_logits.stride(1) == 1 + assert norm_weight.is_contiguous() + assert norm_weight.numel() in (hc_dim, DIM) + + out = residual.new_empty(residual.shape) + y = residual.new_empty(residual.shape) + BLOCK_SIZE = 512 + _hc_combine_norm_kernel[(N, hc_count)]( + block_output, + residual, + injection_logits, + norm_weight, + out, + y, + block_output.stride(0), + residual.stride(0), + injection_logits.stride(0), + out.stride(0), + y.stride(0), + hc_dim, + hc_count, + W_SHARED=norm_weight.numel() == hc_dim, + EPS=eps, + BLOCK_SIZE=BLOCK_SIZE, + launch_pdl=current_platform.is_arch_support_pdl(), + ) + return out, y + + +def _same_shape_fake(x: torch.Tensor, *args) -> torch.Tensor: + return x.new_empty(x.shape) + + +def _hc_gate_mix_fake( + x: torch.Tensor, gate: torch.Tensor, hc_count: int +) -> torch.Tensor: + del gate + return x.new_empty((x.shape[0], x.shape[1] // hc_count)) + + +def _hc_combine_fake( + residual: torch.Tensor, + block_output: torch.Tensor, + injection_logits: torch.Tensor, + hc_count: int, +) -> torch.Tensor: + del block_output, injection_logits, hc_count + return residual.new_empty(residual.shape) + + +def _hc_combine_norm_fake( + residual: torch.Tensor, + block_output: torch.Tensor, + injection_logits: torch.Tensor, + norm_weight: torch.Tensor, + eps: float, + hc_count: int, +) -> tuple[torch.Tensor, torch.Tensor]: + del block_output, injection_logits, norm_weight, eps, hc_count + return residual.new_empty(residual.shape), residual.new_empty(residual.shape) + + +direct_register_custom_op( + op_name="qwen4_exp_grouped_gemma_rmsnorm", + op_func=_grouped_gemma_rmsnorm, + fake_impl=_same_shape_fake, +) +direct_register_custom_op( + op_name="qwen4_exp_hc_silu", + op_func=_hc_silu, + fake_impl=_same_shape_fake, +) +direct_register_custom_op( + op_name="qwen4_exp_hc_gate_mix", + op_func=_hc_gate_mix, + fake_impl=_hc_gate_mix_fake, +) +direct_register_custom_op( + op_name="qwen4_exp_hc_combine", + op_func=_hc_combine, + fake_impl=_hc_combine_fake, +) +direct_register_custom_op( + op_name="qwen4_exp_hc_combine_norm", + op_func=_hc_combine_norm, + fake_impl=_hc_combine_norm_fake, +) + + +def grouped_gemma_rmsnorm( + x: torch.Tensor, weight: torch.Tensor, eps: float, num_groups: int +) -> torch.Tensor: + return torch.ops.vllm.qwen4_exp_grouped_gemma_rmsnorm(x, weight, eps, num_groups) + + +def hc_silu(x: torch.Tensor, hc_count: int) -> torch.Tensor: + return torch.ops.vllm.qwen4_exp_hc_silu(x, hc_count) + + +def hc_gate_mix(x: torch.Tensor, gate: torch.Tensor, hc_count: int) -> torch.Tensor: + return torch.ops.vllm.qwen4_exp_hc_gate_mix(x, gate, hc_count) + + +def hc_combine( + residual: torch.Tensor, + block_output: torch.Tensor, + injection_logits: torch.Tensor, + hc_count: int, +) -> torch.Tensor: + return torch.ops.vllm.qwen4_exp_hc_combine( + residual, block_output, injection_logits, hc_count + ) + + +def hc_combine_norm( + residual: torch.Tensor, + block_output: torch.Tensor, + injection_logits: torch.Tensor, + norm_weight: torch.Tensor, + eps: float, + hc_count: int, +) -> tuple[torch.Tensor, torch.Tensor]: + return torch.ops.vllm.qwen4_exp_hc_combine_norm( + residual, + block_output, + injection_logits, + norm_weight, + eps, + hc_count, + ) + + +__all__ = [ + "grouped_gemma_rmsnorm", + "hc_combine", + "hc_combine_norm", + "hc_gate_mix", + "hc_silu", +] diff --git a/vllm/models/qwen4_exp/amd/ops/qsa.py b/vllm/models/qwen4_exp/amd/ops/qsa.py new file mode 100644 index 0000000000..2e45db711d --- /dev/null +++ b/vllm/models/qwen4_exp/amd/ops/qsa.py @@ -0,0 +1,1128 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Triton kernels for the Qwen4Exp weight-free QSA path.""" + +from __future__ import annotations + +import math + +import torch + +from vllm import _custom_ops as ops +from vllm.platforms import current_platform +from vllm.triton_utils import HAS_TRITON, tl, triton + +_LOGITS_WORKSPACE_BYTES = 128 * 1024 * 1024 +_TOPK_WORKSPACE_BYTES = 1024 * 1024 + + +@triton.jit +def _qsa_mqa_paged_kernel( + q_ptr, + k_cache_ptr, + page_table_ptr, + token_to_req_ptr, + query_positions_ptr, + sequence_lengths_ptr, + visible_blocks_ptr, + logits_ptr, + stride_q_row, + stride_q_head, + stride_q_dim, + stride_cache_block, + stride_cache_token, + stride_cache_dim, + stride_table_req, + stride_table_page, + stride_logits_row, + num_rows, + num_columns, + num_pages, + num_requests, + score_divisor, + PAGE_SIZE: tl.constexpr, + PAGE_TABLE_WIDTH: tl.constexpr, + NUM_HEADS: tl.constexpr, + HEAD_DIM: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_D: tl.constexpr, + COMPRESS_RATIO: tl.constexpr, +) -> None: + row = tl.program_id(0) + columns = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N) + dims = tl.arange(0, BLOCK_D) + request = tl.load(token_to_req_ptr + row) + safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1) + query_position = tl.load(query_positions_ptr + row) + sequence_length = tl.load( + sequence_lengths_ptr + safe_request, + mask=(request >= 0) & (request < num_requests), + other=0, + ) + visible = tl.minimum( + (query_position + 1) // COMPRESS_RATIO, + sequence_length // COMPRESS_RATIO, + ) + if tl.program_id(1) == 0: + tl.store(visible_blocks_ptr + row, visible) + logical_page = columns // PAGE_SIZE + page_offset = columns % PAGE_SIZE + valid = ( + (row < num_rows) + & (columns < num_columns) + & (columns < visible) + & (request >= 0) + & (request < num_requests) + & (logical_page < PAGE_TABLE_WIDTH) + ) + safe_logical_page = tl.minimum(logical_page, PAGE_TABLE_WIDTH - 1) + physical_page = tl.load( + page_table_ptr + + safe_request * stride_table_req + + safe_logical_page * stride_table_page, + mask=valid, + other=-1, + ) + valid &= (physical_page >= 0) & (physical_page < num_pages) + # physical_page * block stride can overflow int32 for large caches. + safe_physical_page = tl.maximum(physical_page, 0).to(tl.int64) + score = tl.zeros((BLOCK_N,), dtype=tl.float32) + + for head in tl.static_range(0, NUM_HEADS): + query = tl.load( + q_ptr + row * stride_q_row + head * stride_q_head + dims * stride_q_dim, + mask=dims < HEAD_DIM, + other=0.0, + ).to(tl.float32) + keys = tl.load( + k_cache_ptr + + safe_physical_page[:, None] * stride_cache_block + + page_offset[:, None] * stride_cache_token + + dims[None, :] * stride_cache_dim, + mask=valid[:, None] & (dims[None, :] < HEAD_DIM), + other=0.0, + ).to(tl.float32) + dot = tl.sum(keys * query[None, :], axis=1) + score += tl.maximum(dot, 0.0) + + score /= score_divisor + tl.store( + logits_ptr + row * stride_logits_row + columns, + tl.where(valid, score, -float("inf")), + mask=(row < num_rows) & (columns < num_columns), + ) + + +@triton.jit +def _expand_qsa_indices_kernel( + block_indices_ptr, + query_positions_ptr, + sequence_lengths_ptr, + token_to_req_ptr, + output_ptr, + stride_blocks_row, + stride_blocks_column, + stride_output_row, + stride_output_column, + rows, + num_requests, + BLOCK_TOPK: tl.constexpr, + COMPRESS_RATIO: tl.constexpr, + TOKEN_TOPK: tl.constexpr, + OUTPUT_WIDTH: tl.constexpr, + COLUMN_BLOCK: tl.constexpr, +) -> None: + row = tl.program_id(0) + columns = tl.program_id(1) * COLUMN_BLOCK + tl.arange(0, COLUMN_BLOCK) + query_position = tl.load(query_positions_ptr + row) + request = tl.load(token_to_req_ptr + row) + safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1) + sequence_length = tl.load( + sequence_lengths_ptr + safe_request, + mask=(request >= 0) & (request < num_requests), + other=0, + ) + complete_blocks = tl.minimum( + tl.minimum( + (query_position + 1) // COMPRESS_RATIO, + sequence_length // COMPRESS_RATIO, + ), + BLOCK_TOPK, + ) + expanded_count = complete_blocks * COMPRESS_RATIO + tail_start = ((query_position + 1) // COMPRESS_RATIO) * COMPRESS_RATIO + tail_count = (query_position + 1) - tail_start + + is_expanded = columns < expanded_count + block_rank = columns // COMPRESS_RATIO + offset = columns % COMPRESS_RATIO + safe_rank = tl.minimum(block_rank, BLOCK_TOPK - 1) + block = tl.load( + block_indices_ptr + row * stride_blocks_row + safe_rank * stride_blocks_column, + mask=(row < rows) & is_expanded, + other=-1, + ) + expanded = block * COMPRESS_RATIO + offset + tail_offset = columns - expanded_count + is_tail = ( + (columns >= expanded_count) + & (tail_offset < tail_count) + & (tail_offset < COMPRESS_RATIO - 1) + ) + token = tl.where(is_expanded, expanded, tail_start + tail_offset) + valid = ( + (row < rows) + & (columns < OUTPUT_WIDTH) + & (is_expanded | is_tail) + & (token >= 0) + & (token < sequence_length) + ) + tl.store( + output_ptr + row * stride_output_row + columns * stride_output_column, + tl.where(valid, token, -1), + mask=(row < rows) & (columns < OUTPUT_WIDTH), + ) + + +@triton.jit +def _qsa_sparse_paged_gqa_splitk_kernel( + q_ptr, + k_cache_ptr, + v_cache_ptr, + indices_ptr, + block_table_ptr, + token_to_req_ptr, + partial_output_ptr, + partial_lse_ptr, + output_ptr, + stride_q_row, + stride_q_head, + stride_k_block, + stride_k_token, + stride_k_head, + stride_v_block, + stride_v_token, + stride_v_head, + stride_indices_row, + stride_table_req, + stride_output_row, + stride_output_head, + num_rows, + num_cache_blocks, + num_requests, + TOPK: tl.constexpr, + PAGE_SIZE: tl.constexpr, + PAGE_TABLE_WIDTH: tl.constexpr, + GROUP_SIZE: tl.constexpr, + HEAD_DIM: tl.constexpr, + NUM_QUERY_HEADS: tl.constexpr, + NUM_SPLITS: tl.constexpr, + NUM_TILES: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, +) -> None: + row = tl.program_id(0) + kv_head = tl.program_id(1) + split_id = tl.program_id(2) + request = tl.load(token_to_req_ptr + row) + safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1) + + head_offsets = tl.arange(0, BLOCK_M) + dim_offsets = tl.arange(0, HEAD_DIM) + column_offsets = tl.arange(0, BLOCK_N) + first_head = kv_head * GROUP_SIZE + query = tl.load( + q_ptr + + row * stride_q_row + + (first_head + head_offsets[:, None]) * stride_q_head + + dim_offsets[None, :], + mask=head_offsets[:, None] < GROUP_SIZE, + other=0.0, + ) + + max_value = tl.full((BLOCK_M,), -1.0e20, dtype=tl.float32) + normalizer = tl.zeros((BLOCK_M,), dtype=tl.float32) + accumulator = tl.zeros((BLOCK_M, HEAD_DIM), dtype=tl.float32) + softmax_scale_log2: tl.constexpr = (HEAD_DIM**-0.5) * 1.4426950408889634 + + # Dynamic bounds avoid padded main-loop iterations for uneven splits. + split_tile_start = split_id * NUM_TILES // NUM_SPLITS + split_tile_end = (split_id + 1) * NUM_TILES // NUM_SPLITS + for tile in range(split_tile_start, split_tile_end): + columns = tile * BLOCK_N + column_offsets + logical_token = tl.load( + indices_ptr + row * stride_indices_row + columns, + mask=columns < TOPK, + other=-1, + ) + safe_token = tl.maximum(logical_token, 0) + logical_page = safe_token // PAGE_SIZE + page_offset = safe_token % PAGE_SIZE + valid = ( + (request >= 0) + & (request < num_requests) + & (logical_token >= 0) + & (logical_page < PAGE_TABLE_WIDTH) + ) + physical_page = tl.load( + block_table_ptr + + safe_request * stride_table_req + + tl.minimum(logical_page, PAGE_TABLE_WIDTH - 1), + mask=valid, + other=-1, + ) + valid &= (physical_page >= 0) & (physical_page < num_cache_blocks) + # physical_page * block stride can overflow int32 for large caches. + safe_page = tl.maximum(physical_page, 0).to(tl.int64) + keys = tl.load( + k_cache_ptr + + safe_page[None, :] * stride_k_block + + page_offset[None, :] * stride_k_token + + kv_head * stride_k_head + + dim_offsets[:, None], + mask=valid[None, :], + other=0.0, + ) + values = tl.load( + v_cache_ptr + + safe_page[:, None] * stride_v_block + + page_offset[:, None] * stride_v_token + + kv_head * stride_v_head + + dim_offsets[None, :], + mask=valid[:, None], + other=0.0, + ) + scores = tl.dot(query, keys) + # Scaling scores avoids re-quantizing a scaled query to BF16. + scores *= softmax_scale_log2 + scores = tl.where(valid[None, :], scores, -1.0e20) + next_max = tl.maximum(max_value, tl.max(scores, axis=1)) + alpha = tl.math.exp2(max_value - next_max) + probabilities = tl.where( + valid[None, :], tl.math.exp2(scores - next_max[:, None]), 0.0 + ) + accumulator = tl.dot( + probabilities.to(values.dtype), + values, + acc=accumulator * alpha[:, None], + ) + normalizer = normalizer * alpha + tl.sum(probabilities, axis=1) + max_value = next_max + + has_values = normalizer > 0 + normalized_output = tl.where( + has_values[:, None], + accumulator / tl.maximum(normalizer[:, None], 1.0e-20), + 0.0, + ) + output_mask = head_offsets[:, None] < GROUP_SIZE + if NUM_SPLITS == 1: + tl.store( + output_ptr + + row * stride_output_row + + (first_head + head_offsets[:, None]) * stride_output_head + + dim_offsets[None, :], + normalized_output, + mask=output_mask, + ) + else: + partial_lse = tl.where( + has_values, + max_value + tl.math.log2(tl.maximum(normalizer, 1.0e-20)), + -float("inf"), + ) + tl.store( + partial_output_ptr + + ( + (split_id * num_rows + row) * NUM_QUERY_HEADS + + first_head + + head_offsets[:, None] + ) + * HEAD_DIM + + dim_offsets[None, :], + normalized_output, + mask=output_mask, + ) + tl.store( + partial_lse_ptr + + (split_id * num_rows + row) * NUM_QUERY_HEADS + + first_head + + head_offsets, + partial_lse, + mask=head_offsets < GROUP_SIZE, + ) + + +@triton.jit +def _qsa_merge_splitk_kernel( + partial_output_ptr, + partial_lse_ptr, + output_ptr, + stride_output_row, + stride_output_head, + num_rows, + HEAD_DIM: tl.constexpr, + NUM_QUERY_HEADS: tl.constexpr, + NUM_SPLITS: tl.constexpr, + BLOCK_SPLITS: tl.constexpr, +) -> None: + row = tl.program_id(0) + head = tl.program_id(1) + split_offsets = tl.arange(0, BLOCK_SPLITS) + dim_offsets = tl.arange(0, HEAD_DIM) + split_mask = split_offsets < NUM_SPLITS + lse = tl.load( + partial_lse_ptr + (split_offsets * num_rows + row) * NUM_QUERY_HEADS + head, + mask=split_mask, + other=-float("inf"), + ) + lse_max = tl.max(lse, axis=0) + has_values = lse_max > -float("inf") + shifted = tl.where(split_mask & has_values, lse - lse_max, -float("inf")) + weights = tl.math.exp2(shifted) + denominator = tl.sum(weights, axis=0) + partial_output = tl.load( + partial_output_ptr + + ((split_offsets[:, None] * num_rows + row) * NUM_QUERY_HEADS + head) + * HEAD_DIM + + dim_offsets[None, :], + mask=split_mask[:, None], + other=0.0, + ) + merged = tl.sum(partial_output * weights[:, None], axis=0) + merged = tl.where(denominator > 0, merged / denominator, 0.0) + tl.store( + output_ptr + row * stride_output_row + head * stride_output_head + dim_offsets, + merged, + ) + + +@triton.jit +def _store_qsa_rows_kernel( + cache_ptr, + slots_ptr, + rows_ptr, + stride_cache_block, + stride_cache_token, + stride_cache_dim, + stride_rows_row, + stride_rows_dim, + num_rows, + num_blocks, + PAGE_SIZE: tl.constexpr, + WIDTH: tl.constexpr, + BLOCK_D: tl.constexpr, +) -> None: + row = tl.program_id(0) + dims = tl.arange(0, BLOCK_D) + slot = tl.load(slots_ptr + row) + valid = (row < num_rows) & (slot >= 0) & (slot < num_blocks * PAGE_SIZE) + block = tl.maximum(slot, 0) // PAGE_SIZE + token = tl.maximum(slot, 0) % PAGE_SIZE + values = tl.load( + rows_ptr + row * stride_rows_row + dims * stride_rows_dim, + mask=valid & (dims < WIDTH), + other=0, + ) + tl.store( + cache_ptr + + block * stride_cache_block + + token * stride_cache_token + + dims * stride_cache_dim, + values, + mask=valid & (dims < WIDTH), + ) + + +@triton.jit +def _compress_qsa_groups_kernel( + raw_keys_ptr, # this step's raw key rows, straight from activations + raw_positions_ptr, # this step's per-token positions + compressor_state_cache_ptr, # per-request ring of previous raw keys + rope_cache_ptr, # packed RoPE position tail of the ring + compressor_state_table_ptr, + token_to_req_ptr, + query_start_loc_ptr, + logical_positions_ptr, + compressed_slots_ptr, + pooled_ptr, + first_positions_ptr, + stride_raw_row, + stride_raw_dim, + stride_raw_positions_row, + stride_raw_positions_dim, + stride_compressor_state_block, + stride_compressor_state_token, + stride_compressor_state_dim, + stride_rope_block, + stride_rope_token, + stride_rope_dim, + stride_compressor_state_table_req, + stride_pooled_row, + stride_pooled_dim, + stride_positions_row, + stride_positions_dim, + num_rows, + num_compressor_state_blocks, + num_requests, + COMPRESSOR_STATE_SIZE: tl.constexpr, + COMPRESS_RATIO: tl.constexpr, + HEAD_DIM: tl.constexpr, + BLOCK_D: tl.constexpr, + LOAD_ROPE_POSITIONS: tl.constexpr, +) -> None: + row = tl.program_id(0) + dims = tl.arange(0, BLOCK_D) + request = tl.load(token_to_req_ptr + row) + end_position = tl.load(logical_positions_ptr + row) + compressed_slot = tl.load(compressed_slots_ptr + row) + valid_request = (request >= 0) & (request < num_requests) + safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1) + query_row_start = tl.load( + query_start_loc_ptr + safe_request, mask=valid_request, other=0 + ) + query_row_end = tl.load( + query_start_loc_ptr + safe_request + 1, mask=valid_request, other=0 + ) + chunk_start_position = end_position - (row - query_row_start) + compressor_state_block = tl.load( + compressor_state_table_ptr + safe_request * stride_compressor_state_table_req, + mask=valid_request, + other=-1, + ) + valid_compressor_state_block = (compressor_state_block >= 0) & ( + compressor_state_block < num_compressor_state_blocks + ) + valid_row = ( + (row < num_rows) + & valid_request + & (row >= query_row_start) + & (row < query_row_end) + & (end_position >= COMPRESS_RATIO - 1) + & (compressed_slot >= 0) + ) + accumulator = tl.zeros((BLOCK_D,), dtype=tl.float32) + + # A group can span the compressor-state ring (older members) and this + # step's raw rows (members at positions >= chunk_start_position). + for group_offset in tl.range(0, COMPRESS_RATIO): + position = end_position - (COMPRESS_RATIO - 1 - group_offset) + use_raw = position >= chunk_start_position + raw_row = query_row_start + position - chunk_start_position + raw_values = tl.load( + raw_keys_ptr + raw_row * stride_raw_row + dims * stride_raw_dim, + mask=valid_row + & use_raw + & (raw_row >= query_row_start) + & (raw_row < query_row_end) + & (raw_row < num_rows) + & (dims < HEAD_DIM), + other=0.0, + ).to(tl.float32) + compressor_state_values = tl.load( + compressor_state_cache_ptr + + tl.maximum(compressor_state_block, 0).to(tl.int64) + * stride_compressor_state_block + + (position % COMPRESSOR_STATE_SIZE) * stride_compressor_state_token + + dims * stride_compressor_state_dim, + mask=valid_row + & ~use_raw + & valid_compressor_state_block + & (dims < HEAD_DIM), + other=0.0, + ).to(tl.float32) + accumulator += tl.where(use_raw, raw_values, compressor_state_values) + + tl.store( + pooled_ptr + row * stride_pooled_row + dims * stride_pooled_dim, + accumulator / COMPRESS_RATIO, + mask=(row < num_rows) & (dims < HEAD_DIM), + ) + + position_dims = tl.arange(0, 4) + first_position = end_position - COMPRESS_RATIO + 1 + if LOAD_ROPE_POSITIONS: + first_from_raw = first_position >= chunk_start_position + raw_first_row = query_row_start + first_position - chunk_start_position + raw_position_values = tl.load( + raw_positions_ptr + + raw_first_row * stride_raw_positions_row + + position_dims * stride_raw_positions_dim, + mask=valid_row + & first_from_raw + & (raw_first_row >= query_row_start) + & (raw_first_row < query_row_end) + & (raw_first_row < num_rows) + & (position_dims < 3), + other=0, + ) + compressor_state_position_values = tl.load( + rope_cache_ptr + + tl.maximum(compressor_state_block, 0).to(tl.int64) * stride_rope_block + + (first_position % COMPRESSOR_STATE_SIZE) * stride_rope_token + + position_dims * stride_rope_dim, + mask=valid_row + & ~first_from_raw + & valid_compressor_state_block + & (position_dims < 3), + other=0, + ) + position_values = tl.where( + first_from_raw, + raw_position_values, + compressor_state_position_values, + ) + else: + position_values = tl.where(valid_row, first_position, 0) + tl.store( + first_positions_ptr + + row * stride_positions_row + + position_dims * stride_positions_dim, + position_values, + mask=(row < num_rows) & (position_dims < 3), + ) + + +def _validate_mqa(q: torch.Tensor) -> None: + if q.ndim != 3 or q.shape[1] <= 0 or q.shape[2] <= 0: + raise ValueError("QSA query must be [rows, heads, head_dim]") + + +def qsa_mqa_paged( + q: torch.Tensor, + k_cache: torch.Tensor, + page_table: torch.Tensor, + token_to_req: torch.Tensor, + query_positions: torch.Tensor, + sequence_lengths: torch.Tensor, + compress_ratio: int, + num_columns: int | None = None, + score_scale: float | None = None, +) -> tuple[torch.Tensor, torch.Tensor]: + """Compute QSA scores directly from a paged compressed-key cache.""" + + _validate_mqa(q) + if not q.is_cuda or not HAS_TRITON: + raise RuntimeError("paged QSA scoring requires a GPU and Triton") + if k_cache.ndim != 4 or k_cache.shape[2] != 1: + raise ValueError("QSA cache must be [pages, page_size, 1, head_dim]") + if k_cache.shape[3] != q.shape[2]: + raise ValueError("QSA query and cache dimensions must match") + if page_table.ndim != 2: + raise ValueError("QSA page table must be two-dimensional") + if q.shape[0] and (not all(k_cache.shape[:2]) or not all(page_table.shape)): + raise ValueError("QSA paged scoring cache and page table must be nonempty") + if token_to_req.shape != (q.shape[0],): + raise ValueError("QSA request mapping must match query rows") + if query_positions.shape != (q.shape[0],): + raise ValueError("QSA query positions must match query rows") + if sequence_lengths.shape != (page_table.shape[0],): + raise ValueError("QSA sequence lengths must match page-table requests") + if compress_ratio <= 0: + raise ValueError("QSA compression ratio must be positive") + score_divisor = math.sqrt(q.shape[2]) if score_scale is None else score_scale + if score_divisor <= 0: + raise ValueError("QSA score scale must be positive") + + capacity = page_table.shape[1] * k_cache.shape[1] + columns = capacity if num_columns is None else num_columns + if columns < 0: + raise ValueError("QSA score width must be non-negative") + logits = torch.empty((q.shape[0], columns), dtype=torch.float32, device=q.device) + visible_blocks = torch.empty(q.shape[0], dtype=torch.int32, device=q.device) + if not q.shape[0] or not columns: + return logits, visible_blocks + block_n = 32 + _qsa_mqa_paged_kernel[(q.shape[0], triton.cdiv(columns, block_n))]( + q, + k_cache, + page_table, + token_to_req, + query_positions, + sequence_lengths, + visible_blocks, + logits, + q.stride(0), + q.stride(1), + q.stride(2), + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(3), + page_table.stride(0), + page_table.stride(1), + logits.stride(0), + q.shape[0], + columns, + k_cache.shape[0], + page_table.shape[0], + float(score_divisor), + PAGE_SIZE=k_cache.shape[1], + PAGE_TABLE_WIDTH=page_table.shape[1], + NUM_HEADS=q.shape[1], + HEAD_DIM=q.shape[2], + BLOCK_N=block_n, + BLOCK_D=triton.next_power_of_2(q.shape[2]), + COMPRESS_RATIO=compress_ratio, + num_warps=4, + ) + return logits, visible_blocks + + +def expand_qsa_block_indices_cuda( + block_indices: torch.Tensor, + query_positions: torch.Tensor, + sequence_lengths: torch.Tensor, + token_to_req: torch.Tensor, + compress_ratio: int, + token_topk: int, + out: torch.Tensor | None = None, +) -> torch.Tensor: + """Expand compressed blocks and compact the causal tail of the open group.""" + + if not block_indices.is_cuda or not HAS_TRITON: + raise RuntimeError("QSA index expansion requires a GPU and Triton") + if token_topk % compress_ratio: + raise ValueError("QSA token top-k must be divisible by compression ratio") + block_topk = token_topk // compress_ratio + output_width = token_topk + compress_ratio - 1 + if block_indices.shape != (query_positions.numel(), block_topk): + raise ValueError("QSA compressed top-k has an invalid shape") + if token_to_req.shape != query_positions.shape: + raise ValueError("QSA request mapping must match query positions") + if sequence_lengths.ndim != 1 or not sequence_lengths.shape[0]: + raise ValueError("QSA request sequence lengths must be nonempty") + if out is None: + out = torch.empty( + (block_indices.shape[0], output_width), + dtype=torch.int32, + device=block_indices.device, + ) + elif out.shape != (block_indices.shape[0], output_width): + raise ValueError("QSA expansion output has an invalid shape") + if not block_indices.shape[0]: + return out + column_block = 256 + _expand_qsa_indices_kernel[ + (block_indices.shape[0], triton.cdiv(output_width, column_block)) + ]( + block_indices, + query_positions, + sequence_lengths, + token_to_req, + out, + block_indices.stride(0), + block_indices.stride(1), + out.stride(0), + out.stride(1), + block_indices.shape[0], + sequence_lengths.shape[0], + BLOCK_TOPK=block_topk, + COMPRESS_RATIO=compress_ratio, + TOKEN_TOPK=token_topk, + OUTPUT_WIDTH=output_width, + COLUMN_BLOCK=column_block, + num_warps=4, + ) + return out + + +def qsa_select_paged_tokens( + q: torch.Tensor, + k_cache: torch.Tensor, + page_table: torch.Tensor, + token_to_req: torch.Tensor, + query_positions: torch.Tensor, + sequence_lengths: torch.Tensor, + token_topk: int, + compress_ratio: int, + out: torch.Tensor | None = None, +) -> torch.Tensor: + """Score, select, and expand QSA indices without host synchronization.""" + + rows = q.shape[0] + output_width = token_topk + compress_ratio - 1 + if out is None: + out = torch.empty((rows, output_width), dtype=torch.int32, device=q.device) + if out.shape != (rows, output_width): + raise ValueError("QSA selection output has an invalid shape") + if not rows: + return out + + columns = page_table.shape[1] * k_cache.shape[1] + block_topk = token_topk // compress_ratio + rows_per_chunk = max(1, _LOGITS_WORKSPACE_BYTES // max(columns * 4, 1)) + chunk_rows = min(rows, rows_per_chunk) + blocks_buffer = torch.empty( + (chunk_rows, block_topk), dtype=torch.int32, device=q.device + ) + topk_workspace = torch.empty( + (_TOPK_WORKSPACE_BYTES,), dtype=torch.uint8, device=q.device + ) + for row_start in range(0, rows, rows_per_chunk): + row_end = min(row_start + rows_per_chunk, rows) + row_slice = slice(row_start, row_end) + logits, visible_blocks = qsa_mqa_paged( + q[row_slice], + k_cache, + page_table, + token_to_req[row_slice], + query_positions[row_slice], + sequence_lengths, + compress_ratio, + ) + blocks = blocks_buffer[: row_end - row_start] + use_cooperative_topk = ( + current_platform.is_cuda() + and blocks.shape[0] <= 32 + and logits.stride(0) % 4 == 0 + and current_platform.has_device_capability(90) + and not current_platform.is_device_capability_family(120) + ) + if use_cooperative_topk: + torch.ops._C.cooperative_topk( + logits, + visible_blocks, + blocks, + topk_workspace, + block_topk, + columns, + ) + elif current_platform.is_cuda(): + torch.ops._C.persistent_topk( + logits, + visible_blocks, + blocks, + topk_workspace, + block_topk, + columns, + ) + else: + ops.top_k_per_row_decode( + logits, + 1, + visible_blocks, + blocks, + blocks.shape[0], + logits.stride(0), + logits.stride(1), + block_topk, + ) + expand_qsa_block_indices_cuda( + blocks, + query_positions[row_slice], + sequence_lengths, + token_to_req[row_slice], + compress_ratio, + token_topk, + out[row_slice], + ) + return out + + +def qsa_sparse_paged_attention( + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + logical_indices: torch.Tensor, + block_table: torch.Tensor, + token_to_req: torch.Tensor, + out: torch.Tensor | None = None, +) -> torch.Tensor: + """Run sparse GQA directly over paged BF16 K/V caches.""" + + if not q.is_cuda or not HAS_TRITON: + raise RuntimeError("paged QSA sparse attention requires a GPU and Triton") + if q.ndim != 3 or k_cache.ndim != 4 or v_cache.shape != k_cache.shape: + raise ValueError("QSA sparse attention received invalid Q/K/V shapes") + if logical_indices.ndim != 2 or logical_indices.shape[0] != q.shape[0]: + raise ValueError("QSA indices must have one row per query") + if token_to_req.shape != (q.shape[0],) or block_table.ndim != 2: + raise ValueError("QSA sparse attention metadata has invalid shapes") + if not all(k_cache.shape[:3]) or not all(block_table.shape): + raise ValueError("QSA sparse attention cache and block table must be nonempty") + if logical_indices.shape[1] <= 0: + raise ValueError("QSA sparse attention requires a positive selection width") + if q.shape[2] != k_cache.shape[3] or q.shape[1] % k_cache.shape[2]: + raise ValueError("QSA sparse attention requires valid grouped-query heads") + head_dim = q.shape[2] + assert head_dim >= 16 and (head_dim & (head_dim - 1)) == 0 + assert q.dtype == k_cache.dtype == v_cache.dtype == torch.bfloat16 + assert logical_indices.dtype == block_table.dtype == torch.int32 + assert token_to_req.dtype == torch.int32 + assert q.device == k_cache.device == v_cache.device + assert q.device == logical_indices.device == block_table.device + assert q.device == token_to_req.device + assert q.stride(2) == k_cache.stride(3) == v_cache.stride(3) == 1 + assert logical_indices.stride(1) == block_table.stride(1) == 1 + assert token_to_req.stride(0) == 1 + if out is None: + out = torch.empty_like(q) + if out.shape != q.shape: + raise ValueError("QSA sparse output must match its query") + assert out.dtype == q.dtype and out.device == q.device + assert out.stride(2) == 1 + if not q.shape[0]: + return out + + group_size = q.shape[1] // k_cache.shape[2] + block_m = triton.next_power_of_2(group_size) + base_programs = q.shape[0] * k_cache.shape[2] + small_profile_limit = 8 if block_m <= 8 else 4 + + # Tuned on GB300 for the Qwen-Air TP1, TP2, and TP4 attention shapes. + # Narrow tiles favor decode; wide tiles improve throughput for prefill. + if base_programs <= small_profile_limit: + block_n, target_splits, partial_warps = 16, 64, 4 + elif base_programs < 32: + block_n, target_splits, partial_warps = 16, 32, 4 + elif base_programs <= 256: + block_n, target_splits, partial_warps = 64, 8, 2 + elif base_programs <= 512: + block_n, target_splits, partial_warps = 64, 4, 2 + else: + block_n, target_splits, partial_warps = 64, 1, 2 + # gfx942 and gfx950 have a 64 KiB LDS limit. One software-pipelining + # stage keeps the wide TP4 tile within that shared-memory budget. + partial_stages = 1 if current_platform.is_rocm() else 2 + + num_tiles = triton.cdiv(logical_indices.shape[1], block_n) + # Avoid empty splits when the selection width is smaller than the profile. + max_useful_splits = 1 << (num_tiles.bit_length() - 1) + num_splits = min(max_useful_splits, target_splits) + + # Split=1 writes output directly and compiles out all workspace accesses. + if num_splits == 1: + partial_output = out + partial_lse = out + else: + # FP32 partials preserve accuracy when merging independently normalized + # splits. + partial_output = torch.empty( + (num_splits, *q.shape), dtype=torch.float32, device=q.device + ) + partial_lse = torch.empty( + (num_splits, q.shape[0], q.shape[1]), + dtype=torch.float32, + device=q.device, + ) + + partial_grid = (q.shape[0], k_cache.shape[2], num_splits) + _qsa_sparse_paged_gqa_splitk_kernel[partial_grid]( + q, + k_cache, + v_cache, + logical_indices, + block_table, + token_to_req, + partial_output, + partial_lse, + out, + q.stride(0), + q.stride(1), + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + v_cache.stride(0), + v_cache.stride(1), + v_cache.stride(2), + logical_indices.stride(0), + block_table.stride(0), + out.stride(0), + out.stride(1), + q.shape[0], + k_cache.shape[0], + block_table.shape[0], + TOPK=logical_indices.shape[1], + PAGE_SIZE=k_cache.shape[1], + PAGE_TABLE_WIDTH=block_table.shape[1], + GROUP_SIZE=group_size, + HEAD_DIM=q.shape[2], + NUM_QUERY_HEADS=q.shape[1], + NUM_SPLITS=num_splits, + NUM_TILES=num_tiles, + BLOCK_M=block_m, + BLOCK_N=block_n, + num_warps=partial_warps, + num_stages=partial_stages, + ) + if num_splits == 1: + return out + + _qsa_merge_splitk_kernel[(q.shape[0], q.shape[1])]( + partial_output, + partial_lse, + out, + out.stride(0), + out.stride(1), + q.shape[0], + HEAD_DIM=q.shape[2], + NUM_QUERY_HEADS=q.shape[1], + NUM_SPLITS=num_splits, + BLOCK_SPLITS=triton.next_power_of_2(num_splits), + num_warps=2, + num_stages=1, + ) + return out + + +def qsa_store_cache_rows( + cache: torch.Tensor, + slot_mapping: torch.Tensor, + rows: torch.Tensor, +) -> None: + """Store fixed-width rows in a QSA cache without boolean indexing.""" + + if not cache.is_cuda or not HAS_TRITON: + raise RuntimeError("QSA cache stores require a GPU and Triton") + if cache.ndim != 4 or cache.shape[2] != 1: + raise ValueError("QSA cache must be [pages, page_size, 1, width]") + if not all(cache.shape): + raise ValueError("QSA cache dimensions must be nonzero") + if rows.ndim == 3: + if rows.shape[1] != 1: + raise ValueError("QSA cache rows must have one head") + rows = rows[:, 0] + if rows.shape != (slot_mapping.numel(), cache.shape[3]): + raise ValueError("QSA cache rows and slots have incompatible shapes") + if not rows.shape[0]: + return + _store_qsa_rows_kernel[(rows.shape[0],)]( + cache, + slot_mapping, + rows, + cache.stride(0), + cache.stride(1), + cache.stride(3), + rows.stride(0), + rows.stride(1), + rows.shape[0], + cache.shape[0], + PAGE_SIZE=cache.shape[1], + WIDTH=cache.shape[3], + BLOCK_D=triton.next_power_of_2(cache.shape[3]), + num_warps=4, + ) + + +def qsa_compress_groups_with_ratio( + raw_keys: torch.Tensor, # this step's raw key rows [rows, 1, head_size] + raw_positions: torch.Tensor, # this step's positions [rows, 1, 3] int64 + compressor_state_cache: torch.Tensor, + compressor_state_block_table: torch.Tensor, + token_to_req: torch.Tensor, + query_start_loc: torch.Tensor, + logical_positions: torch.Tensor, + compressed_slots: torch.Tensor, + compress_ratio: int, + rope_cache: torch.Tensor | None = None, +) -> tuple[torch.Tensor, torch.Tensor]: + """Pool completed groups from the compressor-state ring and raw token rows.""" + + if not raw_keys.is_cuda or not HAS_TRITON: + raise RuntimeError("QSA compression requires a GPU and Triton") + rows = token_to_req.numel() + if compress_ratio <= 0: + raise ValueError("QSA compression ratio must be positive") + if raw_keys.ndim != 3 or raw_keys.shape[:2] != (rows, 1): + raise ValueError("QSA raw keys must be [rows, 1, head_size]") + if raw_positions.shape != (rows, 1, 3) or raw_positions.dtype != torch.int64: + raise ValueError("QSA raw positions must be [rows, 1, 3] int64") + if logical_positions.shape != (rows,) or compressed_slots.shape != (rows,): + raise ValueError("QSA compression metadata must match token rows") + if compressor_state_cache.ndim != 4 or compressor_state_cache.shape[2] != 1: + raise ValueError("QSA compressor-state cache has an invalid shape") + if ( + # The ring is wider than one group so speculative rows cannot alias + # onto the committed keys of the group still being collected. + compressor_state_cache.shape[1] < compress_ratio + or compressor_state_cache.shape[3] != raw_keys.shape[2] + or compressor_state_cache.dtype != raw_keys.dtype + ): + raise ValueError( + "QSA compressor-state cache does not match the compression layout" + ) + if ( + compressor_state_block_table.ndim != 2 + or compressor_state_block_table.shape[1] < 1 + ): + raise ValueError( + "QSA compressor-state block table must contain one block per request" + ) + if query_start_loc.ndim != 1 or query_start_loc.shape[0] < 2: + raise ValueError("QSA query starts must contain a terminal offset") + num_requests = query_start_loc.shape[0] - 1 + if compressor_state_block_table.shape[0] < num_requests: + raise ValueError("QSA compressor-state block table has too few request rows") + if rope_cache is not None and ( + rope_cache.ndim != 4 + or rope_cache.shape[:3] != compressor_state_cache.shape[:3] + or rope_cache.shape[3] != 3 + or rope_cache.dtype != torch.int64 + ): + raise ValueError("QSA packed position view has an invalid shape or dtype") + if rows and ( + not all(compressor_state_cache.shape) + or not all(compressor_state_block_table.shape) + ): + raise ValueError("QSA compressor-state cache and block table must be nonempty") + pooled = torch.empty( + (rows, 1, raw_keys.shape[2]), + dtype=raw_keys.dtype, + device=raw_keys.device, + ) + first_positions = torch.empty((rows, 3), dtype=torch.int64, device=raw_keys.device) + if not rows: + return pooled, first_positions + if rope_cache is None: + rope_cache = compressor_state_cache + load_rope_positions = False + else: + load_rope_positions = True + _compress_qsa_groups_kernel[(rows,)]( + raw_keys, + raw_positions, + compressor_state_cache, + rope_cache, + compressor_state_block_table, + token_to_req, + query_start_loc, + logical_positions, + compressed_slots, + pooled, + first_positions, + raw_keys.stride(0), + raw_keys.stride(2), + raw_positions.stride(0), + raw_positions.stride(2), + compressor_state_cache.stride(0), + compressor_state_cache.stride(1), + compressor_state_cache.stride(3), + rope_cache.stride(0), + rope_cache.stride(1), + rope_cache.stride(3), + compressor_state_block_table.stride(0), + pooled.stride(0), + pooled.stride(2), + first_positions.stride(0), + first_positions.stride(1), + rows, + compressor_state_cache.shape[0], + num_requests, + COMPRESSOR_STATE_SIZE=compressor_state_cache.shape[1], + COMPRESS_RATIO=compress_ratio, + HEAD_DIM=raw_keys.shape[2], + LOAD_ROPE_POSITIONS=load_rope_positions, + BLOCK_D=triton.next_power_of_2(raw_keys.shape[2]), + num_warps=4, + ) + return pooled, first_positions + + +__all__ = [ + "expand_qsa_block_indices_cuda", + "qsa_compress_groups_with_ratio", + "qsa_mqa_paged", + "qsa_select_paged_tokens", + "qsa_sparse_paged_attention", + "qsa_store_cache_rows", +] diff --git a/vllm/models/qwen4_exp/amd/ple_layer.py b/vllm/models/qwen4_exp/amd/ple_layer.py new file mode 100644 index 0000000000..69215b1aa4 --- /dev/null +++ b/vllm/models/qwen4_exp/amd/ple_layer.py @@ -0,0 +1,1131 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""GPU-resident Qwen4Exp position-learning enhancement layers.""" + +import math +from collections.abc import Iterable, Sequence + +import torch +import torch.nn.functional as F +from torch import nn + +from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config +from vllm.forward_context import get_forward_context +from vllm.model_executor.layers.linear import ReplicatedLinear +from vllm.model_executor.layers.mamba.abstract import MambaBase +from vllm.model_executor.layers.mamba.mamba_utils import ( + MambaStateDtypeCalculator, + MambaStateShapeCalculator, + is_conv_state_dim_first, +) +from vllm.model_executor.models.utils import AutoWeightsLoader +from vllm.transformers_utils.configs.qwen4_exp import ( + Qwen4ExpTextConfig, +) +from vllm.utils.torch_utils import direct_register_custom_op +from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum +from vllm.v1.attention.backends.short_conv_attn import ( + PleShortConvAttentionBackend, + PleShortConvAttentionMetadata, +) +from vllm.v1.attention.backends.utils import NULL_BLOCK_ID + +from ..common.ple import PLEVocabParallelEmbedding + + +class Qwen4ExpPLEGroupedNorm(nn.Module): + def __init__( + self, + hidden_size: int, + eps: float, + group_size: int | None, + dtype: torch.dtype | None, + ) -> None: + super().__init__() + if group_size is not None and hidden_size % group_size: + raise ValueError( + f"hidden_size ({hidden_size}) must be divisible by " + f"group_size ({group_size})" + ) + self.eps = eps + self.group_size = group_size + self.weight = nn.Parameter(torch.zeros(hidden_size, dtype=dtype)) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + input_dtype = hidden_states.dtype + hidden_states = hidden_states.float() + if self.group_size is None: + variance = hidden_states.square().mean(dim=-1, keepdim=True) + normalized = hidden_states * torch.rsqrt(variance + self.eps) + else: + grouped = hidden_states.unflatten( + -1, (hidden_states.shape[-1] // self.group_size, self.group_size) + ) + variance = grouped.square().mean(dim=-1, keepdim=True) + normalized = (grouped * torch.rsqrt(variance + self.eps)).flatten(-2) + return (normalized * (1.0 + self.weight.float())).to(input_dtype) + + +class Qwen4ExpNGramEmbedding(nn.Module): + _MASK64 = (1 << 64) - 1 + _SPLITMIX_GAMMA = 0x9E3779B97F4A7C15 + _SPLITMIX_M1 = 0xBF58476D1CE4E5B9 + _SPLITMIX_M2 = 0x94D049BB133111EB + _PLE_LAYER_PRIME = 10007 + + @classmethod + def _splitmix64(cls, value: int) -> int: + """Mix an integer into a deterministic unsigned 64-bit value.""" + value = (value + cls._SPLITMIX_GAMMA) & cls._MASK64 + value = ((value ^ (value >> 30)) * cls._SPLITMIX_M1) & cls._MASK64 + value = ((value ^ (value >> 27)) * cls._SPLITMIX_M2) & cls._MASK64 + return (value ^ (value >> 31)) & cls._MASK64 + + @staticmethod + def _is_prime_64(value: int) -> bool: + """Return whether a 64-bit integer is prime.""" + if value < 2: + return False + for prime in (2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37): + if value % prime == 0: + return value == prime + exponent = value - 1 + shifts = 0 + while exponent % 2 == 0: + exponent //= 2 + shifts += 1 + for base in (2, 325, 9375, 28178, 450775, 9780504, 1795265022): + if base % value == 0: + continue + witness = pow(base, exponent, value) + if witness in (1, value - 1): + continue + for _ in range(shifts - 1): + witness = pow(witness, 2, value) + if witness == value - 1: + break + else: + return False + return True + + @classmethod + def _nth_prime_after(cls, start: int, count: int) -> int: + """Return the ``count``-th prime strictly greater than ``start``.""" + prime = int(start) + for _ in range(count): + candidate = prime + 1 + if candidate <= 2: + prime = 2 + continue + if candidate % 2 == 0: + candidate += 1 + while not cls._is_prime_64(candidate): + candidate += 2 + prime = candidate + return prime + + @classmethod + def _make_layer_multipliers( + cls, + *, + ngram_size: int, + unigram_vocab_size: int, + seed: int, + ple_dense_layer_id: int, + ) -> list[int]: + """Build deterministic hash multipliers for one PLE layer.""" + max_multiplier = ((1 << 63) - 1) // unigram_vocab_size + half_bound = max(1, max_multiplier // 2) + base_seed = seed + cls._PLE_LAYER_PRIME * ple_dense_layer_id + multipliers = [] + for index in range(ngram_size): + value = base_seed + cls._SPLITMIX_GAMMA * (index + 1) + multipliers.append(2 * (cls._splitmix64(value) % half_bound) + 1) + return multipliers + + @classmethod + def _make_vocab_layout( + cls, + *, + ngram_vocab_size_base: int, + ngram_heads: int, + ple_dense_layer_id: int, + ) -> tuple[list[int], list[int], int]: + """Build per-head vocabulary sizes, offsets, and total row count.""" + sizes: list[int] = [] + offsets: list[int] = [] + offset = 0 + for local_head in range(ngram_heads): + global_head = ple_dense_layer_id * ngram_heads + local_head + size = cls._nth_prime_after(ngram_vocab_size_base - 1, global_head + 1) + sizes.append(size) + offsets.append(offset) + offset += size + return sizes, offsets, offset + + def __init__( + self, + config: Qwen4ExpTextConfig, + embedding_dim: int, + ple_dense_layer_id: int, + max_total_tokens: int, + max_num_reqs: int, + prefix: str, + layer_name: str, + ) -> None: + super().__init__() + self.embedding_dim = embedding_dim + self.layer_name = layer_name + self.ngram_size = int(config.ngram_size) + self.heads_per_ngram = int(config.heads_per_ngram) + self.ngram_heads = (self.ngram_size - 1) * self.heads_per_ngram + if self.ngram_size < 2: + raise ValueError(f"ngram_size must be >= 2, got {self.ngram_size}") + if self.heads_per_ngram <= 0: + raise ValueError(f"heads_per_ngram must be > 0, got {self.heads_per_ngram}") + if embedding_dim % self.ngram_heads: + raise ValueError( + "ple_embed_dim must be divisible by total ngram heads: " + f"{embedding_dim} % {self.ngram_heads} != 0" + ) + self.head_dim = embedding_dim // self.ngram_heads + self.eos_token_id = int(config.eos_token_id) + self.unigram_vocab_size = int(config.vocab_size) + self.split_ngram_parts = int(getattr(config, "split_ngram_parts", 512)) + if self.split_ngram_parts <= 0: + raise ValueError("split_ngram_parts must be positive") + + multipliers = self._make_layer_multipliers( + ngram_size=self.ngram_size, + unigram_vocab_size=self.unigram_vocab_size, + seed=int(getattr(config, "seed", 1234)), + ple_dense_layer_id=ple_dense_layer_id, + ) + self.register_buffer( + "layer_multipliers", + torch.tensor(multipliers, dtype=torch.long), + persistent=True, + ) + + sizes, offsets, total_vocab_size = self._make_vocab_layout( + ngram_vocab_size_base=int(config.ngram_vocab_size_base), + ngram_heads=self.ngram_heads, + ple_dense_layer_id=ple_dense_layer_id, + ) + self.register_buffer( + "ngram_heads_vocab_sizes", + torch.tensor(sizes, dtype=torch.long), + persistent=True, + ) + self.register_buffer( + "ngram_heads_offsets", + torch.tensor(offsets, dtype=torch.long), + persistent=True, + ) + divisor = int(config.make_ngram_vocab_size_divisible_by) + padded_vocab_size = ((total_vocab_size + divisor - 1) // divisor) * divisor + self.ngram_embedding = PLEVocabParallelEmbedding( + padded_vocab_size, + self.head_dim, + padding_size=divisor, + prefix=f"{prefix}.ngram_embedding", + ) + self.register_buffer( + "positions_buffer", + torch.arange(max_total_tokens, dtype=torch.int64), + persistent=False, + ) + self.register_buffer( + "padded_buffer", + torch.full( + (max_num_reqs, max_total_tokens), + self.eos_token_id, + dtype=torch.int64, + ), + persistent=False, + ) + + @staticmethod + def _shift_precompute( + tokens: torch.Tensor, eos_token_id: int + ) -> tuple[torch.Tensor, torch.Tensor]: + if tokens.dim() != 2: + raise ValueError("tokens must be a 2D tensor") + batch_size, seq_len = tokens.shape + positions = torch.arange(seq_len, device=tokens.device, dtype=torch.int64) + eos_positions = torch.where(tokens == eos_token_id, positions, -1) + previous_eos_inclusive = torch.cummax(eos_positions, dim=1).values + previous_eos = torch.cat( + [ + eos_positions.new_full((batch_size, 1), -1), + previous_eos_inclusive[:, :-1], + ], + dim=1, + ) + return positions, positions.unsqueeze(0) - previous_eos - 1 + + @staticmethod + def _shift_apply( + tokens: torch.Tensor, + positions: torch.Tensor, + position_in_segment: torch.Tensor, + shift: int, + eos_token_id: int, + ) -> torch.Tensor: + if shift == 0: + return tokens + source = positions - shift + gather_indices = source.clamp_min(0).unsqueeze(0).expand(tokens.shape[0], -1) + shifted = tokens.gather(1, gather_indices) + valid = (source.unsqueeze(0) >= 0) & (position_in_segment >= shift) + return torch.where(valid, shifted, tokens.new_full((), eos_token_id)) + + def forward( + self, + input_ids: torch.Tensor, + query_start_loc: torch.Tensor, + ngram_context: torch.Tensor, + ) -> torch.Tensor: + input_ids = input_ids.reshape(-1).long() + query_start_loc = query_start_loc.long() + num_reqs = query_start_loc.numel() - 1 + num_tokens = input_ids.shape[0] + if num_tokens > self.positions_buffer.numel(): + raise ValueError( + f"PLE received {num_tokens} tokens, but its workspace supports " + f"at most {self.positions_buffer.numel()}" + ) + if num_reqs > self.padded_buffer.shape[0]: + raise ValueError( + f"PLE received {num_reqs} requests, but its workspace supports " + f"at most {self.padded_buffer.shape[0]}" + ) + + positions = self.positions_buffer[:num_tokens] + packed = self.padded_buffer[:num_reqs] + packed.fill_(self.eos_token_id) + request_indices = torch.searchsorted(query_start_loc, positions, right=True) - 1 + request_indices.clamp_(max=num_reqs - 1) + columns = (positions - query_start_loc[request_indices]).clamp( + 0, packed.shape[1] - 1 + ) + packed[request_indices, columns] = input_ids + ngram_context = ngram_context[:num_reqs].to( + device=input_ids.device, dtype=torch.long + ) + + context = torch.cat([ngram_context, packed], dim=-1) + positions_2d, position_in_segment = self._shift_precompute( + context, self.eos_token_id + ) + shifted = [context] + for shift in range(1, self.ngram_size): + shifted.append( + self._shift_apply( + context, + positions_2d, + position_in_segment, + shift, + self.eos_token_id, + ) + ) + adjusted_columns = columns + self.ngram_size - 1 + id_blocks = [] + for ngram in range(2, self.ngram_size + 1): + start = (ngram - 2) * self.heads_per_ngram + end = start + self.heads_per_ngram + mixed = shifted[0] * self.layer_multipliers[0] + for index in range(1, ngram): + mixed = torch.bitwise_xor( + mixed, shifted[index] * self.layer_multipliers[index] + ) + sizes = self.ngram_heads_vocab_sizes[start:end] + offsets = self.ngram_heads_offsets[start:end] + ids = torch.remainder(mixed.unsqueeze(-1), sizes) + offsets + id_blocks.append(ids[request_indices, adjusted_columns]) + ngram_ids = torch.cat(id_blocks, dim=-1) + output = ngram_ids.new_empty( + (ngram_ids.shape[0], self.embedding_dim), + dtype=self.ngram_embedding.params_dtype, + ) + torch.ops.vllm.qwen4_exp_amd_ple_ngram_embedding( + ngram_ids, + output, + self.layer_name, + ) + return output + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + """Load hash buffers and checkpoint-split embedding rows.""" + + persistent_buffers = { + "layer_multipliers": self.layer_multipliers, + "ngram_heads_offsets": self.ngram_heads_offsets, + "ngram_heads_vocab_sizes": self.ngram_heads_vocab_sizes, + } + loaded: set[str] = set() + regular_weights: list[tuple[str, torch.Tensor]] = [] + shard_prefix = "ngram_embedding.shard_" + + for name, loaded_weight in weights: + leaf_name = name.rsplit(".", 1)[-1] + if leaf_name.startswith("hashstats_") or leaf_name == "token_lookup": + continue + if name in persistent_buffers: + buffer = persistent_buffers[name] + if buffer.shape != loaded_weight.shape: + raise ValueError( + f"Shape mismatch for {name}: expected " + f"{tuple(buffer.shape)}, got {tuple(loaded_weight.shape)}" + ) + buffer.copy_(loaded_weight.to(device=buffer.device, dtype=buffer.dtype)) + loaded.add(name) + continue + if name.startswith(shard_prefix) and name.endswith(".weight"): + shard_text = name[len(shard_prefix) : -len(".weight")] + if not shard_text.isdigit(): + regular_weights.append((name, loaded_weight)) + continue + shard_index = int(shard_text) + if shard_index >= self.split_ngram_parts: + raise ValueError( + f"PLE embedding shard index {shard_index} exceeds " + f"split_ngram_parts={self.split_ngram_parts}" + ) + embedding = self.ngram_embedding + shard_size = ( + embedding.org_vocab_size + self.split_ngram_parts - 1 + ) // self.split_ngram_parts + checkpoint_start = shard_index * shard_size + expected_rows = max( + 0, + min(shard_size, embedding.org_vocab_size - checkpoint_start), + ) + expected_shape = (expected_rows, embedding.embedding_dim) + if tuple(loaded_weight.shape) != expected_shape: + raise ValueError( + f"Shape mismatch for PLE embedding shard {shard_index}: " + f"expected {expected_shape}, got " + f"{tuple(loaded_weight.shape)}" + ) + embedding.weight.weight_loader( + embedding.weight, + loaded_weight, + checkpoint_start=checkpoint_start, + ) + loaded.add("ngram_embedding.weight") + continue + regular_weights.append((name, loaded_weight)) + + if regular_weights: + loaded.update(AutoWeightsLoader(self).load_weights(regular_weights)) + return loaded + + +class Qwen4ExpPLELayer(nn.Module, MambaBase): + def __init__( + self, + config: Qwen4ExpTextConfig, + vllm_config: VllmConfig, + layer_idx: int = 0, + ple_dense_layer_id: int | None = None, + prefix: str = "", + ) -> None: + super().__init__() + model_config = vllm_config.model_config + cache_config = vllm_config.cache_config + quant_config = vllm_config.quant_config + self.model_config: ModelConfig = model_config + self.cache_config: CacheConfig = cache_config + self.layer_idx = layer_idx + self.ple_dense_layer_id = ( + int(ple_dense_layer_id) + if ple_dense_layer_id is not None + else int(layer_idx) + ) + self.prefix = prefix + self.hidden_size = int(config.hidden_size) + self.hc_count = config.hc_count + self.hc_hidden_size = self.hidden_size * self.hc_count + self.conv_kernel_size = int(config.ple_conv_kernel_size) + self.short_conv_dilation = int(config.ngram_size) + self.conv_state_len = (self.conv_kernel_size - 1) * self.short_conv_dilation + self.num_spec_tokens = vllm_config.num_speculative_tokens + self.activation = "silu" + self.ple_embedding: nn.Module = Qwen4ExpNGramEmbedding( + config, + int(config.ple_embed_dim), + self.ple_dense_layer_id, + vllm_config.scheduler_config.max_num_batched_tokens, + vllm_config.scheduler_config.max_num_seqs, + f"{prefix}.ple_embedding", + prefix, + ) + self.key_proj = ReplicatedLinear( + int(config.ple_embed_dim), + self.hc_hidden_size, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.key_proj", + ) + self.value_proj = ReplicatedLinear( + int(config.ple_embed_dim), + self.hidden_size, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.value_proj", + ) + norm_args = ( + self.hc_hidden_size, + config.rms_norm_eps, + self.hidden_size, + model_config.dtype, + ) + self.norm_key = Qwen4ExpPLEGroupedNorm(*norm_args) + self.norm_query = Qwen4ExpPLEGroupedNorm(*norm_args) + self.norm_conv = Qwen4ExpPLEGroupedNorm(*norm_args) + self.conv1d = nn.Conv1d( + self.hc_hidden_size, + self.hc_hidden_size, + self.conv_kernel_size, + groups=self.hc_hidden_size, + padding=self.conv_state_len, + dilation=self.short_conv_dilation, + bias=False, + dtype=model_config.dtype, + ) + nn.init.zeros_(self.conv1d.weight) + self.conv1d.weight._no_reinit = True + self.kv_cache = (torch.tensor([]),) + compilation_config = get_current_vllm_config().compilation_config + if prefix in compilation_config.static_forward_context: + raise ValueError(f"Duplicate layer name: {prefix}") + compilation_config.static_forward_context[prefix] = self + + @property + def mamba_type(self) -> MambaAttentionBackendEnum: + return MambaAttentionBackendEnum.SHORT_CONV + + @property + def is_kv_cache_tp_replicated(self) -> bool: + return True + + def get_attn_backend(self) -> type[PleShortConvAttentionBackend]: + return PleShortConvAttentionBackend + + def get_state_dtype(self) -> tuple[torch.dtype, ...]: + return MambaStateDtypeCalculator.short_conv_state_dtype( + self.model_config.dtype, self.cache_config.mamba_cache_dtype + ) + + def get_state_shape(self) -> Sequence[tuple[int, ...]]: + return MambaStateShapeCalculator.short_conv_state_shape( + tp_world_size=1, + intermediate_size=self.hc_hidden_size, + conv_kernel=self.conv_state_len + 1, + num_spec=self.num_spec_tokens, + ) + + def _apply_norm( + self, norm: Qwen4ExpPLEGroupedNorm, hidden_states: torch.Tensor + ) -> torch.Tensor: + shape = hidden_states.shape + return norm(hidden_states.flatten(-2)).reshape(shape) + + def _short_conv_fallback(self, inputs: torch.Tensor) -> torch.Tensor: + # Profiling / CUDA graph capture only; conv state is not updated. + inputs_t = inputs.transpose(0, 1).unsqueeze(0) + output = self.conv1d(inputs_t)[..., : inputs_t.size(-1)] + return F.silu(output).squeeze(0).transpose(0, 1) + + def _short_conv_dilated_decode_batched( + self, + x_d: torch.Tensor, + conv_state: torch.Tensor, + conv_weights: torch.Tensor, + state_indices_tensor_d: torch.Tensor, + has_initial_states_d: torch.Tensor | None, + ) -> torch.Tensor: + state_indices = state_indices_tensor_d.to( + device=conv_state.device, dtype=torch.int64 + ) + # FULL cudagraph padded decode rows use NULL_BLOCK_ID. Remap them to + # slot 0 for a safe gather, then zero output and skip write-back. + valid_state = state_indices != NULL_BLOCK_ID + state_indices = torch.where( + valid_state, state_indices, torch.zeros_like(state_indices) + ) + if has_initial_states_d is None: + has_initial_state = valid_state + else: + if has_initial_states_d.numel() < state_indices_tensor_d.numel(): + raise ValueError( + "has_initial_states_d size mismatch: " + f"got {has_initial_states_d.numel()}, " + f"need >= {state_indices_tensor_d.numel()}." + ) + has_initial_state = has_initial_states_d[ + : state_indices_tensor_d.numel() + ].to(device=conv_state.device, dtype=torch.bool) + has_initial_state = has_initial_state & valid_state + + cached_state = conv_state.index_select(0, state_indices) + state = cached_state[..., : self.conv_state_len].to(x_d.dtype) + if self.conv_state_len > 0: + initial_state = torch.where( + has_initial_state.view(-1, 1, 1), + state, + torch.zeros_like(state), + ) + history = torch.cat((initial_state, x_d.unsqueeze(-1)), dim=-1) + else: + history = x_d.unsqueeze(-1) + + conv_output = F.conv1d( + history, + conv_weights.unsqueeze(1).contiguous(), + groups=history.size(1), + dilation=self.short_conv_dilation, + ).squeeze(-1) + output = F.silu(conv_output) + output = output * valid_state.view(-1, 1).to(output.dtype) + + if self.conv_state_len > 0: + next_state = history[..., -self.conv_state_len :] + # Padded rows are remapped to the reserved null slot. Preserve its + # existing value while writing the new states for valid rows. + existing_base_state = cached_state[..., : self.conv_state_len] + safe_next_state = torch.where( + valid_state.view(-1, 1, 1), + next_state.to(conv_state.dtype), + existing_base_state, + ) + cached_state[..., : self.conv_state_len] = safe_next_state + conv_state.index_copy_(0, state_indices, cached_state) + + return output + + def _short_conv_dilated_prefill_batched( + self, + x_p: torch.Tensor, + metadata: PleShortConvAttentionMetadata, + conv_state: torch.Tensor, + conv_weights: torch.Tensor, + state_indices_tensor_p: torch.Tensor, + num_prefills: int, + num_decode_tokens: int, + num_prefill_tokens: int, + ) -> torch.Tensor: + # ``non_spec_query_start_loc`` covers the non-spec (decode + prefill) + # requests and equals ``query_start_loc`` when spec-decode is inactive. + non_spec_query_start_loc = metadata.non_spec_query_start_loc + if non_spec_query_start_loc is None: + raise ValueError("query_start_loc is required for prefill short-conv") + query_start_loc_p = ( + non_spec_query_start_loc[-num_prefills - 1 :] - num_decode_tokens + ) + # The metadata builder guarantees that the prefill query offsets start + # at 0 and end at num_prefill_tokens. Avoid reading those values here, + # since doing so would force a device-to-host synchronization. + has_initial_states_p = metadata.has_initial_states_p + if has_initial_states_p is None: + raise ValueError("has_initial_states_p is required for prefill short-conv") + + output = torch.empty_like(x_p) + q_starts = query_start_loc_p.to(torch.int64) + if state_indices_tensor_p.numel() < num_prefills: + raise ValueError( + "state_indices_tensor_p size mismatch: " + f"got {state_indices_tensor_p.numel()}, " + f"need >= {num_prefills}." + ) + if has_initial_states_p.numel() < num_prefills: + raise ValueError( + "has_initial_states_p size mismatch: " + f"got {has_initial_states_p.numel()}, " + f"need >= {num_prefills}." + ) + if num_prefills == 0 or x_p.numel() == 0: + return output + lengths = q_starts[1:] - q_starts[:-1] + # Use the CPU-computed packing width from the metadata builder instead + # of synchronizing on lengths.max(). + max_len = metadata.max_prefill_query_len + if max_len <= 0: + return output + + hidden_size = x_p.shape[1] + positions = torch.arange( + num_prefill_tokens, device=x_p.device, dtype=torch.int64 + ) + req_indices = torch.searchsorted(q_starts[1:], positions, right=True) + col_indices = positions - q_starts[req_indices] + + packed_tokens = x_p.new_zeros((num_prefills, max_len, hidden_size)) + packed_tokens[req_indices, col_indices] = x_p + packed_tokens = packed_tokens.transpose(1, 2).contiguous() + + state_indices = state_indices_tensor_p[:num_prefills].to( + device=conv_state.device, dtype=torch.int64 + ) + valid_state = state_indices != NULL_BLOCK_ID + state_indices = torch.where( + valid_state, state_indices, torch.zeros_like(state_indices) + ) + has_initial = has_initial_states_p[:num_prefills].to( + device=conv_state.device, dtype=torch.bool + ) + if self.conv_state_len > 0: + if conv_state.shape[0] == 0: + state = conv_state.new_zeros( + (num_prefills, hidden_size, self.conv_state_len), + dtype=x_p.dtype, + ) + else: + state = conv_state.index_select(0, state_indices)[ + ..., : self.conv_state_len + ].to(x_p.dtype) + use_initial_mask = (valid_state & has_initial).view(num_prefills, 1, 1) + initial_state = torch.where( + use_initial_mask, + state, + torch.zeros_like(state), + ) + history = torch.cat((initial_state, packed_tokens), dim=-1) + else: + history = packed_tokens + + conv_output = F.conv1d( + history, + conv_weights.unsqueeze(1).contiguous(), + groups=history.size(1), + dilation=self.short_conv_dilation, + ) + conv_output = F.silu(conv_output).transpose(1, 2).contiguous() + + token_positions = torch.arange(max_len, device=x_p.device, dtype=torch.int64) + valid_tokens = token_positions.view(1, max_len) < lengths.view(num_prefills, 1) + valid_output_mask = valid_tokens & valid_state.to(device=x_p.device).view( + num_prefills, 1 + ) + conv_output.masked_fill_(~valid_output_mask.unsqueeze(-1), 0) + output.copy_(conv_output[req_indices, col_indices]) + + if self.conv_state_len > 0 and conv_state.shape[0] > 0: + state_starts = lengths.to(device=history.device, dtype=torch.int64).view( + num_prefills, 1, 1 + ) + state_offsets = torch.arange( + self.conv_state_len, device=history.device, dtype=torch.int64 + ).view(1, 1, self.conv_state_len) + next_state = history.gather( + dim=2, + index=(state_starts + state_offsets).expand(-1, history.size(1), -1), + ) + # Write back without a host synchronization. Valid, non-empty rows + # receive their new state; padding and zero-length rows keep the + # current cache value. + existing_state = conv_state.index_select(0, state_indices) + existing_base_state = existing_state[..., : self.conv_state_len] + update_mask = valid_state & (lengths.to(device=conv_state.device) > 0) + safe_next_state = torch.where( + update_mask.view(num_prefills, 1, 1), + next_state.to(conv_state.dtype), + existing_base_state, + ) + existing_state[..., : self.conv_state_len] = safe_next_state + conv_state.index_copy_(0, state_indices, existing_state) + return output + + def _short_conv_dilated_spec_batched( + self, + x_spec: torch.Tensor, + conv_state: torch.Tensor, + conv_weights: torch.Tensor, + spec_state_indices_tensor: torch.Tensor, + spec_query_start_loc: torch.Tensor, + num_accepted_tokens: torch.Tensor, + spec_query_len: int, + ) -> torch.Tensor: + """Dilated short-conv for speculative-decode (MTP) requests. + + Each spec request feeds multiple (draft + 1) query tokens. The conv + outputs are computed causally after rolling back the previous draft + state by ``num_accepted_tokens - 1``. The current candidate inputs stay + in the extended cache for the next forward, matching + ``causal_conv1d_update``. + + ``spec_query_len`` (== num_speculative_tokens + 1) is the maximum query + length and is a Python int, so no host synchronization is needed; this + keeps the path safe for full CUDA-graph capture/replay where the buffers + are padded at the request level. + """ + num_reqs = spec_state_indices_tensor.numel() + hidden_size = x_spec.size(-1) + # Use a fixed packing width instead of synchronizing on lengths.max(). + max_len = spec_query_len + # Full CUDA graphs can pad these buffers. Only the first num_reqs + # accepted-token counts belong to actual speculative requests. + num_accepted_tokens = num_accepted_tokens[:num_reqs] + q_starts = spec_query_start_loc[: num_reqs + 1].to(torch.int64) + # Keep the number of real speculative tokens on the device. + total_real_tokens = q_starts[num_reqs] + + state_indices = spec_state_indices_tensor.to( + device=conv_state.device, dtype=torch.int64 + ) + valid_state = state_indices != NULL_BLOCK_ID + state_indices = torch.where( + valid_state, state_indices, torch.zeros_like(state_indices) + ) + positions = torch.arange( + x_spec.size(0), device=x_spec.device, dtype=torch.int64 + ) + # Route graph-padded token rows to the discarded dummy request so that + # they cannot overwrite real packed data. + req_indices = torch.searchsorted(q_starts[1:], positions, right=True) + valid_tokens = (positions < total_real_tokens) & (req_indices < num_reqs) + clamped_req_indices = req_indices.clamp_max(max(num_reqs - 1, 0)) + col_indices = (positions - q_starts[clamped_req_indices]).clamp_(0, max_len - 1) + pack_req_indices = torch.where( + valid_tokens, + clamped_req_indices, + torch.full_like(req_indices, num_reqs), + ) + pack_col_indices = torch.where( + valid_tokens, col_indices, torch.zeros_like(col_indices) + ) + + # The last request row is the dummy sink for graph padding. + packed = x_spec.new_zeros((num_reqs + 1, max_len, hidden_size)) + packed[pack_req_indices, pack_col_indices] = x_spec + packed = packed.transpose(1, 2).contiguous() + + if self.conv_state_len > 0: + cached_state = conv_state.index_select(0, state_indices) + rollback_offsets = num_accepted_tokens.to( + device=conv_state.device, dtype=torch.int64 + ).sub(1) + rollback_offsets = torch.where( + valid_state, + rollback_offsets.clamp_(0, max_len - 1), + torch.zeros_like(rollback_offsets), + ) + state_offsets = torch.arange( + self.conv_state_len, device=conv_state.device, dtype=torch.int64 + ).view(1, 1, self.conv_state_len) + rollback_indices = rollback_offsets.view(-1, 1, 1) + state_offsets + state = cached_state.gather( + 2, rollback_indices.expand(-1, hidden_size, -1) + ).to(x_spec.dtype) + state = torch.where( + valid_state.view(num_reqs, 1, 1), + state, + torch.zeros_like(state), + ) + # Append a zeroed dummy-row state to match the [num_reqs + 1] pack. + dummy_state = state.new_zeros((1, hidden_size, self.conv_state_len)) + state_full = torch.cat((state, dummy_state), dim=0) + history = torch.cat((state_full, packed), dim=-1) + else: + history = packed + + conv_output = F.conv1d( + history, + conv_weights.unsqueeze(1).contiguous(), + groups=history.size(1), + dilation=self.short_conv_dilation, + ) + conv_output = F.silu(conv_output).transpose(1, 2).contiguous() + + output = conv_output[pack_req_indices, pack_col_indices] + output = output * valid_tokens.view(-1, 1).to(output.dtype) + + # Keep all current candidate inputs in the extended state. On the next + # target forward, ``num_accepted_tokens - 1`` selects the rollback + # window before processing the newly scheduled tokens. + if self.conv_state_len > 0: + state_capacity = self.conv_state_len + max_len - 1 + if conv_state.size(-1) < state_capacity: + raise RuntimeError( + "PLE short-conv cache cannot retain speculative tokens: " + f"got {conv_state.size(-1)}, need {state_capacity}." + ) + candidate_state = history[:num_reqs, :, 1 : state_capacity + 1] + query_lengths = q_starts[1:] - q_starts[:-1] + state_positions = torch.arange( + state_capacity, device=history.device, dtype=torch.int64 + ).view(1, 1, state_capacity) + update_lengths = (self.conv_state_len + query_lengths - 1).view( + num_reqs, 1, 1 + ) + update_mask = valid_state.view(num_reqs, 1, 1) & ( + state_positions < update_lengths + ) + existing_state = cached_state[..., :state_capacity] + next_state = torch.where( + update_mask, + candidate_state.to(conv_state.dtype), + existing_state, + ) + cached_state[..., :state_capacity] = next_state + conv_state.index_copy_(0, state_indices, cached_state) + + return output + + def _short_conv_dilated_dispatch( + self, + inputs: torch.Tensor, + metadata: PleShortConvAttentionMetadata, + conv_state: torch.Tensor, + conv_weights: torch.Tensor, + ) -> torch.Tensor: + num_prefills = metadata.num_prefills + num_decodes = metadata.num_decodes + num_decode_tokens = metadata.num_decode_tokens + num_prefill_tokens = metadata.num_prefill_tokens + has_prefill = num_prefills > 0 + has_decode = num_decodes > 0 + has_spec = metadata.spec_sequence_masks is not None + x = inputs[: metadata.num_actual_tokens] + + # Split spec / non-spec tokens. + if has_spec: + if has_prefill or has_decode: + assert metadata.spec_token_indx is not None + assert metadata.non_spec_token_indx is not None + x_spec = x.index_select(0, metadata.spec_token_indx.long()) + x_non_spec = x.index_select(0, metadata.non_spec_token_indx.long()) + else: + x_spec = x + x_non_spec = None + else: + x_spec = None + x_non_spec = x + + spec_output = None + # 1. Run the multi-query speculative-decode part. + if has_spec: + assert metadata.spec_state_indices_tensor is not None + assert metadata.spec_query_start_loc is not None + assert metadata.num_accepted_tokens is not None + spec_output = self._short_conv_dilated_spec_batched( + x_spec=x_spec, + conv_state=conv_state, + conv_weights=conv_weights, + spec_state_indices_tensor=metadata.spec_state_indices_tensor[ + : metadata.num_spec_decodes + ], + spec_query_start_loc=metadata.spec_query_start_loc, + num_accepted_tokens=metadata.num_accepted_tokens, + spec_query_len=metadata.spec_query_len, + ) + + # 2. Run regular decode and prefill requests. + conv_out_non_spec = None + state_indices_tensor = metadata.state_indices_tensor + if x_non_spec is not None: + assert state_indices_tensor is not None + if has_prefill: + state_indices_tensor_d, state_indices_tensor_p = torch.split( + state_indices_tensor, + [num_decodes, num_prefills], + dim=0, + ) + x_d, x_p = torch.split( + x_non_spec, + [num_decode_tokens, num_prefill_tokens], + dim=0, + ) + non_spec_parts: list[torch.Tensor] = [] + if has_decode: + non_spec_parts.append( + self._short_conv_dilated_decode_batched( + x_d=x_d, + conv_state=conv_state, + conv_weights=conv_weights, + state_indices_tensor_d=state_indices_tensor_d, + has_initial_states_d=metadata.has_initial_states_d, + ) + ) + non_spec_parts.append( + self._short_conv_dilated_prefill_batched( + x_p=x_p, + metadata=metadata, + conv_state=conv_state, + conv_weights=conv_weights, + state_indices_tensor_p=state_indices_tensor_p, + num_prefills=num_prefills, + num_decode_tokens=num_decode_tokens, + num_prefill_tokens=num_prefill_tokens, + ) + ) + conv_out_non_spec = torch.vstack(non_spec_parts) + else: + conv_out_non_spec = self._short_conv_dilated_decode_batched( + x_d=x_non_spec, + conv_state=conv_state, + conv_weights=conv_weights, + state_indices_tensor_d=state_indices_tensor[: x_non_spec.size(0)], + has_initial_states_d=metadata.has_initial_states_d, + ) + + # 3. Merge both parts back into the original token order. + if has_spec and conv_out_non_spec is not None: + assert metadata.spec_token_indx is not None + assert metadata.non_spec_token_indx is not None + assert spec_output is not None + output = x.new_empty((metadata.num_actual_tokens, x.size(-1))) + output.index_copy_(0, metadata.spec_token_indx, spec_output) + output.index_copy_(0, metadata.non_spec_token_indx, conv_out_non_spec) + return output + elif has_spec: + assert spec_output is not None + return spec_output + if conv_out_non_spec is None: + return x + return conv_out_non_spec + + def _short_conv(self, inputs: torch.Tensor) -> torch.Tensor: + forward_context = get_forward_context() + attn_metadata = forward_context.attn_metadata + if attn_metadata is None: + return self._short_conv_fallback(inputs) + + if not isinstance(attn_metadata, dict): + raise RuntimeError( + "PLE short-conv expects per-layer attention metadata dict " + f"during inference, got {type(attn_metadata).__name__}." + ) + + layer_attn_metadata = attn_metadata.get(self.prefix) + if layer_attn_metadata is None: + # MRV2 omits Mamba-family metadata during profile warmup. + return self._short_conv_fallback(inputs) + if not isinstance(layer_attn_metadata, PleShortConvAttentionMetadata): + raise TypeError( + "Expected PleShortConvAttentionMetadata for layer " + f"'{self.prefix}', got " + f"{type(layer_attn_metadata).__name__}." + ) + + conv_state = self.kv_cache[0] + if not is_conv_state_dim_first(): + conv_state = conv_state.transpose(-1, -2) + conv_weights = self.conv1d.weight.squeeze(1) + + state_capacity = self.conv_state_len + self.num_spec_tokens + if state_capacity > 0: + if conv_state.size(-1) < state_capacity: + raise RuntimeError( + "PLE short-conv cache is smaller than expected for " + f"dilated convolution: got {conv_state.size(-1)}, " + f"expect at least {state_capacity}." + ) + conv_state = conv_state[..., -state_capacity:] + return self._short_conv_dilated_dispatch( + inputs, + layer_attn_metadata, + conv_state, + conv_weights.to(dtype=inputs.dtype), + ) + + def forward( + self, + hidden_states: torch.Tensor, + input_ids: torch.Tensor, + query_start_loc: torch.Tensor, + ngram_context: torch.Tensor, + ) -> torch.Tensor: + input_ids = input_ids.reshape(-1) + if input_ids.shape[0] != hidden_states.shape[0]: + raise ValueError( + "PLE expects input_ids and hidden_states to have the same " + f"token length, got {input_ids.shape[0]} and " + f"{hidden_states.shape[0]}" + ) + embeddings = self.ple_embedding(input_ids, query_start_loc, ngram_context) + key, _ = self.key_proj(embeddings) + value, _ = self.value_proj(embeddings) + token_count = hidden_states.shape[0] + key = key.reshape(token_count, self.hc_count, self.hidden_size) + query = hidden_states.reshape(token_count, self.hc_count, self.hidden_size) + key = self._apply_norm(self.norm_key, key) + query = self._apply_norm(self.norm_query, query) + gate = (key * query).sum(dim=-1, keepdim=True) / math.sqrt(self.hidden_size) + gate = torch.sigmoid(gate.sign() * gate.abs().clamp_min(1e-6).sqrt()) + gated_value = gate * value.unsqueeze(-2) + normalized = self._apply_norm(self.norm_conv, gated_value).flatten(-2) + conv_output = torch.zeros_like(normalized) + torch.ops.vllm.qwen4_exp_ple_short_conv( + normalized, + conv_output, + self.prefix, + ) + return gated_value.flatten(-2) + conv_output + + +def qwen4_exp_amd_ple_ngram_embedding( + ngram_ids: torch.Tensor, + output: torch.Tensor, + layer_name: str, +) -> None: + """Run the large PLE embedding lookup outside Inductor's FX graph. + + Keeping the embedding weight in ``static_forward_context`` prevents AOT + compile-time autotuning from materializing a synthetic copy of the weight. + """ + layer = get_forward_context().no_compile_layers[layer_name] + if not isinstance(layer, Qwen4ExpPLELayer): + raise TypeError(f"{layer_name} is not a Qwen4Exp PLE owner") + result = layer.ple_embedding.ngram_embedding(ngram_ids).flatten(-2) + output.copy_(result) + + +def qwen4_exp_amd_ple_ngram_embedding_fake( + ngram_ids: torch.Tensor, + output: torch.Tensor, + layer_name: str, +) -> None: + return + + +def qwen4_exp_ple_short_conv( + inputs: torch.Tensor, + output: torch.Tensor, + layer_name: str, +) -> None: + layer = get_forward_context().no_compile_layers[layer_name] + result = layer._short_conv(inputs) + output[: result.shape[0]].copy_(result) + + +def qwen4_exp_ple_short_conv_fake( + inputs: torch.Tensor, + output: torch.Tensor, + layer_name: str, +) -> None: + return + + +direct_register_custom_op( + op_name="qwen4_exp_amd_ple_ngram_embedding", + op_func=qwen4_exp_amd_ple_ngram_embedding, + mutates_args=["output"], + fake_impl=qwen4_exp_amd_ple_ngram_embedding_fake, +) + + +direct_register_custom_op( + op_name="qwen4_exp_ple_short_conv", + op_func=qwen4_exp_ple_short_conv, + mutates_args=["output"], + fake_impl=qwen4_exp_ple_short_conv_fake, +) + + +__all__ = [ + "Qwen4ExpNGramEmbedding", + "Qwen4ExpPLEGroupedNorm", + "Qwen4ExpPLELayer", +] diff --git a/vllm/models/qwen4_exp/amd/qsa.py b/vllm/models/qwen4_exp/amd/qsa.py new file mode 100644 index 0000000000..0e65956e96 --- /dev/null +++ b/vllm/models/qwen4_exp/amd/qsa.py @@ -0,0 +1,474 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""AMD ROCm QSA owner with Triton kernels.""" + +from __future__ import annotations + +from typing import ClassVar, cast + +import torch +from torch import nn + +from vllm.config import VllmConfig +from vllm.config.cache import CacheDType +from vllm.distributed import get_tensor_model_parallel_world_size +from vllm.forward_context import get_forward_context +from vllm.model_executor.layers.attention.attention import ( + set_default_quant_scales, +) +from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase +from vllm.model_executor.layers.layernorm import GemmaRMSNorm +from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.models.qwen3_next import Qwen3NextAttention +from vllm.platforms import current_platform +from vllm.transformers_utils.configs.qwen4_exp import ( + Qwen4ExpTextConfig, +) +from vllm.utils.torch_utils import ( + LayerNameType, + _encode_layer_name, + _resolve_layer_name, + canonicalize_singleton_dim_strides, + direct_register_custom_op, + kv_cache_dtype_str_to_dtype, +) +from vllm.v1.attention.backend import ( + AttentionBackend, + AttentionCGSupport, + AttentionType, + MultipleOf, +) +from vllm.v1.attention.backends.fa_utils import is_flash_attn_varlen_func_available +from vllm.v1.attention.backends.flash_attn import ( + FlashAttentionBackend, + FlashAttentionImpl, + FlashAttentionMetadata, + FlashAttentionMetadataBuilder, +) +from vllm.v1.kv_cache_interface import ( + FullAttentionSpec, + KVCacheSpec, + get_kv_quant_mode, +) + +from ..common.qsa_cache import QSAForwardMetadata +from . import model +from .indexer_qsa import QSAIndexer + + +class Qwen4ExpQSAMetadataBuilder(FlashAttentionMetadataBuilder): + """Flash metadata supporting uniform decode and target-verify graphs.""" + + _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH + + +class Qwen4ExpQSAFlashAttentionBackend(FlashAttentionBackend): + """FullAttentionSpec backend used by the merged QSA owner.""" + + supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16] + supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = ["auto", "bfloat16"] + + @staticmethod + def get_name() -> str: + return "QWEN4_EXP_QSA_TRITON" + + @staticmethod + def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + # QSA consumes manager pages directly and does not use FA4 paged attention. + return [MultipleOf(16)] + + @staticmethod + def get_impl_cls() -> type[Qwen4ExpQSAFlashAttentionImpl]: + return Qwen4ExpQSAFlashAttentionImpl + + @staticmethod + def get_builder_cls() -> type[Qwen4ExpQSAMetadataBuilder]: + return Qwen4ExpQSAMetadataBuilder + + @classmethod + def is_sparse(cls) -> bool: + return True + + @classmethod + def supports_kv_connector(cls) -> bool: + return False + + +class Qwen4ExpQSAFlashAttentionImpl(FlashAttentionImpl): + """Run paged sparse GQA with the QSA Triton kernel.""" + + supports_dcp: bool = False + supports_pcp: bool = False + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + if not is_flash_attn_varlen_func_available(): + raise NotImplementedError("Qwen4Exp QSA requires FlashAttention") + if self.dcp_world_size != 1: + raise NotImplementedError( + "Qwen4Exp QSA does not support decode context parallelism" + ) + if self.kv_cache_dtype not in ("auto", "bfloat16"): + raise NotImplementedError("Qwen4Exp QSA requires a BF16 main KV cache") + self.supports_quant_query_input = False + + def forward_qsa( + self, + layer: torch.nn.Module, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: FlashAttentionMetadata, + output: torch.Tensor, + token_to_req: torch.Tensor, + output_scale: torch.Tensor | None = None, + output_block_scale: torch.Tensor | None = None, + ) -> torch.Tensor: + del key, value + if output_scale is not None or output_block_scale is not None: + raise NotImplementedError("QSA does not support fused output quantization") + if self.alibi_slopes is not None or self.sinks is not None: + raise NotImplementedError("QSA does not support ALiBi or attention sinks") + if self.sliding_window != (-1, -1): + raise NotImplementedError("QSA does not support sliding-window attention") + + num_tokens = attn_metadata.num_actual_tokens + output.zero_() + if num_tokens == 0: + return output + + topk_buffer = getattr(layer, "topk_indices_buffer", None) + if topk_buffer is None: + raise RuntimeError("QSA owner did not provide its top-k buffer") + logical_indices = topk_buffer[:num_tokens] + token_to_req = token_to_req[:num_tokens] + key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1) + key_cache = canonicalize_singleton_dim_strides(key_cache) + value_cache = canonicalize_singleton_dim_strides(value_cache) + if key_cache.dtype != torch.bfloat16 or query.dtype != torch.bfloat16: + raise NotImplementedError("Qwen4Exp QSA requires BF16 Q/K/V") + + from .ops.qsa import qsa_sparse_paged_attention + + qsa_sparse_paged_attention( + query[:num_tokens], + key_cache, + value_cache, + logical_indices, + attn_metadata.block_table, + token_to_req, + output[:num_tokens], + ) + return output + + +class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase): + """Merged Qwen full-attention owner with a QSA index side branch.""" + + supports_dcp = False + + def __init__( + self, + *, + vllm_config: VllmConfig, + config: Qwen4ExpTextConfig, + layer_id: int, + quant_config: QuantizationConfig | None = None, + reduce_results: bool = True, + prefix: str = "", + ) -> None: + nn.Module.__init__(self) + cache_config = vllm_config.cache_config + model_config = vllm_config.model_config + if cache_config is None: + raise ValueError("Qwen4Exp QSA requires a paged KV cache") + if model_config.dtype != torch.bfloat16: + raise NotImplementedError("Qwen4Exp QSA currently requires BF16") + if cache_config.cache_dtype not in ("auto", "bfloat16"): + raise NotImplementedError("Qwen4Exp QSA requires a BF16 main KV cache") + if getattr(quant_config, "kv_cache_scheme", None) is not None: + raise NotImplementedError("Qwen4Exp QSA does not support KV quantization") + parallel_config = vllm_config.parallel_config + if ( + parallel_config.prefill_context_parallel_size > 1 + or parallel_config.decode_context_parallel_size > 1 + ): + raise NotImplementedError( + "Qwen4Exp QSA does not support context parallelism" + ) + if not getattr(config, "is_causal", True): + raise NotImplementedError("Qwen4Exp QSA requires causal decoder attention") + + self.config = config + self.hidden_size = int(config.hidden_size) + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = int(config.num_attention_heads) + if self.total_num_heads % tp_size: + raise ValueError("QSA attention heads must be divisible by TP size") + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = int(config.num_key_value_heads) + if self.total_num_kv_heads >= tp_size: + if self.total_num_kv_heads % tp_size: + raise ValueError("QSA KV heads must be divisible by TP size") + elif tp_size % self.total_num_kv_heads: + raise ValueError("TP size must be divisible by replicated QSA KV heads") + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.head_dim = int(config.head_dim or self.hidden_size // self.num_heads) + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.dual_chunk_attention_config = getattr( + config, "dual_chunk_attention_config", None + ) + if self.dual_chunk_attention_config is not None: + raise NotImplementedError("Qwen4Exp QSA does not support dual-chunk RoPE") + # Qwen4Exp full-attention checkpoints always pack a sigmoid output + # gate next to Q, even when an inherited config default says otherwise. + self.attn_output_gate = True + + self.qkv_proj = QKVParallelLinear( + self.hidden_size, + self.head_dim, + self.total_num_heads * (1 + self.attn_output_gate), + self.total_num_kv_heads, + bias=False, + quant_config=model.without_modelopt_fp4(quant_config), + prefix=f"{prefix}.qkv_proj", + ) + self.o_proj = RowParallelLinear( + self.total_num_heads * self.head_dim, + self.hidden_size, + bias=False, + reduce_results=reduce_results, + quant_config=quant_config, + prefix=f"{prefix}.o_proj", + ) + self.rotary_emb = get_rope( + head_size=self.head_dim, + max_position=config.max_position_embeddings, + rope_parameters=config.rope_parameters, + ) + self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps) + self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps) + + mm_config = model_config.multimodal_config + text_only = mm_config is None or mm_config.language_model_only + self.use_fused_qk_norm_rope_gate = ( + self.attn_output_gate + and getattr(self.rotary_emb, "is_neox_style", False) + and current_platform.is_cuda() + and text_only + ) + + self.layer_name = f"{prefix}.attn" + self.attn_type = AttentionType.DECODER + self.kv_cache_dtype = cache_config.cache_dtype + self.kv_cache_torch_dtype = kv_cache_dtype_str_to_dtype( + self.kv_cache_dtype, model_config + ) + if self.kv_cache_torch_dtype != torch.bfloat16: + raise NotImplementedError("Qwen4Exp QSA requires BF16 cache storage") + self.kv_sharing_target_layer_name = None + self.kv_cache = torch.tensor([]) + set_default_quant_scales(self, register_buffer=True) + + self.attn_backend = Qwen4ExpQSAFlashAttentionBackend + self.impl = Qwen4ExpQSAFlashAttentionImpl( + self.num_heads, + self.head_dim, + self.scaling, + self.num_kv_heads, + None, + None, + self.kv_cache_dtype, + None, + AttentionType.DECODER, + None, + ) + self.indexer = QSAIndexer( + vllm_config=vllm_config, + config=config, + layer_id=layer_id, + rotary_emb=self.rotary_emb, + quant_config=quant_config, + prefix=f"{prefix}.indexer", + ) + max_tokens = vllm_config.scheduler_config.max_num_batched_tokens + self.register_buffer( + "topk_indices_buffer", + torch.empty( + max_tokens, + self.indexer.output_width, + dtype=torch.int32, + ), + persistent=False, + ) + + static_context = vllm_config.compilation_config.static_forward_context + if self.layer_name in static_context: + raise ValueError(f"Duplicate layer name: {self.layer_name}") + static_context[self.layer_name] = self + + def get_attn_backend(self) -> type[AttentionBackend]: + return self.attn_backend + + def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec: + return FullAttentionSpec( + block_size=vllm_config.cache_config.block_size, + num_kv_heads=self.num_kv_heads, + head_size=self.head_dim, + head_size_v=self.head_dim, + dtype=self.kv_cache_torch_dtype, + kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype), + ) + + def _run_qsa( + self, + hidden_states: torch.Tensor, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + output: torch.Tensor, + ) -> None: + metadata = get_forward_context().attn_metadata + if isinstance(metadata, list): + metadata = metadata[0] + if not isinstance(metadata, dict): + output.zero_() + return + main_metadata = cast(FlashAttentionMetadata, metadata[self.layer_name]) + if self.kv_cache.numel() == 0: + raise RuntimeError("QSA main K/V cache is not bound") + + num_tokens = main_metadata.num_actual_tokens + side_metadata = cast( + QSAForwardMetadata, + metadata[self.indexer.raw_key_cache.prefix], + ) + if side_metadata.num_actual_tokens != num_tokens: + raise RuntimeError("QSA main and side metadata token counts disagree") + selected = self.indexer( + hidden_states, + positions, + self.topk_indices_buffer[:num_tokens], + ) + if selected.shape != ( + num_tokens, + self.indexer.output_width, + ): + raise RuntimeError("QSA indexer returned an invalid selection shape") + impl = cast(Qwen4ExpQSAFlashAttentionImpl, self.impl) + impl.do_kv_cache_update( + self, + key, + value, + self.kv_cache, + main_metadata.slot_mapping, + ) + impl.forward_qsa( + self, + query, + key, + value, + self.kv_cache, + main_metadata, + output, + token_to_req=side_metadata.token_to_req, + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v, gate = self._project_qkv_gate(qkv, positions) + num_tokens = hidden_states.shape[0] + query = q.view(num_tokens, self.num_heads, self.head_dim) + key = k.view(num_tokens, self.num_kv_heads, self.head_dim) + value = v.view(num_tokens, self.num_kv_heads, self.head_dim) + attn_output = torch.empty_like(query) + encoded_layer_name = _encode_layer_name(self.layer_name) + if current_platform.opaque_attention_op(): + torch.ops.vllm.qwen4_exp_qsa_with_output( + hidden_states, + positions, + query, + key, + value, + attn_output, + encoded_layer_name, + ) + else: + qwen4_exp_qsa_with_output( + hidden_states, + positions, + query, + key, + value, + attn_output, + encoded_layer_name, + ) + flat_output = attn_output.view(num_tokens, -1) + if gate is not None: + flat_output = flat_output * torch.sigmoid(gate) + output, _ = self.o_proj(flat_output) + return output + + +def qwen4_exp_qsa_with_output( + hidden_states: torch.Tensor, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + output: torch.Tensor, + layer_name: LayerNameType, +) -> None: + """Run the complete QSA state/update/attend transaction.""" + + layer_name = _resolve_layer_name(layer_name) + layer = get_forward_context().no_compile_layers[layer_name] + if not isinstance(layer, Qwen4ExpQSAAttention): + raise TypeError(f"{layer_name} is not a Qwen4Exp QSA owner") + layer._run_qsa( + hidden_states, + positions, + query, + key, + value, + output, + ) + + +def qwen4_exp_qsa_with_output_fake( + hidden_states: torch.Tensor, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + output: torch.Tensor, + layer_name: LayerNameType, +) -> None: + del hidden_states, positions, query, key, value, output, layer_name + + +direct_register_custom_op( + op_name="qwen4_exp_qsa_with_output", + op_func=qwen4_exp_qsa_with_output, + mutates_args=["output"], + fake_impl=qwen4_exp_qsa_with_output_fake, +) + + +__all__ = [ + "QSAIndexer", + "Qwen4ExpQSAAttention", + "Qwen4ExpQSAFlashAttentionBackend", + "Qwen4ExpQSAFlashAttentionImpl", + "qwen4_exp_qsa_with_output", +] diff --git a/vllm/models/qwen4_exp/common/__init__.py b/vllm/models/qwen4_exp/common/__init__.py new file mode 100644 index 0000000000..a89d77b976 --- /dev/null +++ b/vllm/models/qwen4_exp/common/__init__.py @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Common Qwen4Exp model components.""" + +from .hyperconnection import ( + GatedResidual, + GroupedGemmaRMSNorm, + HyperConnectionBase, + HyperConnectionConfig, +) + +__all__ = [ + "GatedResidual", + "GroupedGemmaRMSNorm", + "HyperConnectionBase", + "HyperConnectionConfig", +] diff --git a/vllm/models/qwen4_exp/common/hyperconnection.py b/vllm/models/qwen4_exp/common/hyperconnection.py new file mode 100644 index 0000000000..5f8bde0479 --- /dev/null +++ b/vllm/models/qwen4_exp/common/hyperconnection.py @@ -0,0 +1,250 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""HyperConnection (Gated Residual) utilities. + +Implements the HyperConnection residual scheme proposed in +"HyperConnections" (https://arxiv.org/abs/2409.19606). + +The two concrete variants are: + - ``HyperConnectionBase`` - simple average pooling across hc_count parallel + streams (equivalent to hyperconnection_average). + - ``GatedResidual`` - learnable low-rank gated mixing and injection + (gated_residual). + +Hidden states between layers have shape ``[..., HC*HS]`` with HS inner +(HC outer, HS inner — checkpoint-native layout). The local torch +implementation consumes the hyper input viewed as ``[..., HC, HS]``. + +Typical usage inside a transformer decoder layer:: + + self.attn_hc = GatedResidual(hc_config, role="attn") + self.mlp_hc = GatedResidual(hc_config, role="mlp") + + hidden_states, residual = self.attn_hc.mix(hidden_states) + hidden_states = attention(hidden_states) + hidden_states = self.attn_hc.combine(hidden_states, residual) + + hidden_states, residual = self.mlp_hc.mix(hidden_states) + hidden_states = mlp(hidden_states) + hidden_states = self.mlp_hc.combine(hidden_states, residual) +""" + +from dataclasses import dataclass + +import torch +import torch.nn.functional as F +from torch import nn + + +# --------------------------------------------------------------------------- +# Config +# --------------------------------------------------------------------------- +@dataclass +class HyperConnectionConfig: + """Configuration shared by all HyperConnection variants.""" + + hc_count: int = 4 + hidden_size: int = 64 + params_dtype: torch.dtype = torch.bfloat16 + mtp_hc: bool = False + hc_lowrank: int = 16 + rms_norm_eps: float = 1e-6 + hc_per_branch_norm: bool = False + + +class GroupedGemmaRMSNorm(nn.Module): + def __init__( + self, + hidden_size: int, + eps: float, + group_size: int | None, + dtype: torch.dtype | None, + ) -> None: + super().__init__() + if group_size is not None and hidden_size % group_size: + raise ValueError( + f"hidden_size ({hidden_size}) must be divisible by " + f"group_size ({group_size})" + ) + self.variance_epsilon = eps + self.group_size = group_size + self.weight = nn.Parameter(torch.zeros(hidden_size, dtype=dtype)) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + input_dtype = hidden_states.dtype + hidden_states = hidden_states.float() + if self.group_size is None: + variance = hidden_states.square().mean(dim=-1, keepdim=True) + normalized = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + else: + grouped = hidden_states.unflatten( + -1, (hidden_states.shape[-1] // self.group_size, self.group_size) + ) + variance = grouped.square().mean(dim=-1, keepdim=True) + normalized = ( + grouped * torch.rsqrt(variance + self.variance_epsilon) + ).flatten(-2) + return (normalized * (1.0 + self.weight.float())).to(input_dtype) + + +# --------------------------------------------------------------------------- +# Average-pooling variant +# --------------------------------------------------------------------------- +class HyperConnectionBase(nn.Module): + """Average-pooling HyperConnection (``hyperconnection_average``). + + Splits the incoming ``[..., HC*HS]`` tensor (HC outer, HS inner) into + ``HC`` parallel streams, averages them for the block input, and + broadcasts the block output back to every stream. + """ + + def __init__( + self, + config: HyperConnectionConfig, + layer_idx: int | None = None, + role: str | None = None, + ) -> None: + super().__init__() + self.config = config + self.hc_count = config.hc_count + self.hidden_size = config.hidden_size + self.layer_idx = layer_idx + self.role = role + + @property + def hyper_hidden_size(self) -> int: + return self.hc_count * self.hidden_size + + def mix(self, hyper_input: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """Average the HC streams into a single block input.""" + assert hyper_input.shape[-1] == self.hc_count * self.hidden_size + # [*, HC, HS] — mean over HC (dim=-2). + unflat = hyper_input.unflatten(-1, (self.hc_count, self.hidden_size)) + mixed_input = unflat.mean(dim=-2) + return mixed_input, hyper_input + + def combine( + self, block_output: torch.Tensor, residual: torch.Tensor + ) -> torch.Tensor: + """Broadcast the block output back to every stream.""" + assert residual.shape[-1] == self.hc_count * self.hidden_size + assert block_output.shape[-1] == self.hidden_size + residual_reshaped = residual.unflatten(-1, (self.hc_count, self.hidden_size)) + combined = residual_reshaped + block_output.unsqueeze(-2) + return combined.flatten(-2) + + +# --------------------------------------------------------------------------- +# Gated-residual variant +# --------------------------------------------------------------------------- +class GatedResidual(HyperConnectionBase): + """Gated HyperConnection with learnable low-rank mixing and injection. + + ``mix()`` applies GemmaRMSNorm per HC stream and projects through a + low-rank sigmoid gate to produce a single block input. ``combine()`` + injects the block output back into each stream through a learned + per-stream injection weight. + + This implementation uses only PyTorch operators. Tensor-parallel + collectives are supplied by its caller. + """ + + def __init__( + self, + config: HyperConnectionConfig, + layer_idx: int | None = None, + role: str | None = None, + use_mix: bool = True, + use_combine: bool = True, + ) -> None: + super().__init__(config, layer_idx, role) + norm_size = ( + self.hyper_hidden_size if config.hc_per_branch_norm else config.hidden_size + ) + group_size = config.hidden_size if config.hc_per_branch_norm else None + # Normalize each H-sized HC stream independently while retaining a + # separate affine weight for every element of the HC*H layout. + self.hc_norm = GroupedGemmaRMSNorm( + norm_size, + eps=config.rms_norm_eps, + group_size=group_size, + dtype=config.params_dtype, + ) + + # -- raw Linear weights (checkpoint-compatible) ---------------------- + if use_mix: + self.input_mix_weight_down = nn.Linear( + self.hyper_hidden_size, + config.hc_lowrank, + bias=False, + dtype=config.params_dtype, + ) + self.input_mix_weight_up = nn.Linear( + config.hc_lowrank, + self.hyper_hidden_size, + bias=False, + dtype=config.params_dtype, + ) + if use_combine: + self.block_inject_weight = nn.Linear( + self.hyper_hidden_size, + self.hc_count, + bias=False, + dtype=config.params_dtype, + ) + + def _normalize(self, hyper_input: torch.Tensor) -> torch.Tensor: + if self.config.hc_per_branch_norm: + return self.hc_norm(hyper_input) + return self.hc_norm( + hyper_input.unflatten(-1, (self.hc_count, self.hidden_size)) + ).flatten(-2) + + def mix( + self, hyper_input: torch.Tensor + ) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]: + """Mix: RMSNorm -> low-rank gate -> gated mean.""" + assert hyper_input.shape[-1] == self.hc_count * self.hidden_size + if not hasattr(self, "input_mix_weight_down"): + raise RuntimeError("mix was disabled for this hyper-connection") + hyper_input_normed = self._normalize(hyper_input) + # Gate — original mix order: linear+silu then linear+sigmoid. + gate = F.silu( + F.linear(hyper_input_normed, self.input_mix_weight_down.weight) + / self.hc_count + ) + gate = torch.sigmoid(F.linear(gate, self.input_mix_weight_up.weight)).unflatten( + -1, (self.hc_count, self.hidden_size) + ) + mixed_input = ( + gate * hyper_input_normed.unflatten(-1, (self.hc_count, self.hidden_size)) + ).mean(dim=-2) + return mixed_input.to(hyper_input.dtype), (hyper_input, hyper_input_normed) + + def combine( + self, + block_output: torch.Tensor, + residuals: tuple[torch.Tensor, torch.Tensor], + ) -> torch.Tensor: + if not hasattr(self, "block_inject_weight"): + raise RuntimeError("combine was disabled for this hyper-connection") + hyper_input, hyper_input_normed = residuals + assert hyper_input.shape[-1] == self.hc_count * self.hidden_size + assert block_output.shape[-1] == self.hidden_size + residual = hyper_input.unflatten(-1, (self.hc_count, self.hidden_size)) + # The paired mix keeps its normalized hyper input so combine uses the + # same HC module's injection weight. + injection_weight = 2.0 * torch.sigmoid( + F.linear(hyper_input_normed, self.block_inject_weight.weight) + / self.hc_count + ) + output = residual + block_output.unsqueeze(-2) * injection_weight.unsqueeze(-1) + return output.flatten(-2).to(hyper_input.dtype) + + +__all__ = [ + "GatedResidual", + "GroupedGemmaRMSNorm", + "HyperConnectionBase", + "HyperConnectionConfig", +] diff --git a/vllm/models/qwen4_exp/common/ple.py b/vllm/models/qwen4_exp/common/ple.py new file mode 100644 index 0000000000..bd132685b5 --- /dev/null +++ b/vllm/models/qwen4_exp/common/ple.py @@ -0,0 +1,100 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Common Qwen4Exp PLE helpers.""" + +from dataclasses import dataclass + +import torch + +from vllm.model_executor.layers.vocab_parallel_embedding import ( + VocabParallelEmbedding, +) + + +@dataclass(frozen=True) +class PLEShardOverlap: + """Source and destination slices for one checkpoint embedding shard.""" + + source_start: int + destination_start: int + row_count: int + + +def compute_ple_shard_overlap( + *, + checkpoint_start: int, + checkpoint_rows: int, + tp_start: int, + tp_end: int, +) -> PLEShardOverlap | None: + """Compute the overlap of a checkpoint shard and one TP vocabulary range.""" + + if checkpoint_start < 0 or checkpoint_rows < 0: + raise ValueError("checkpoint shard bounds must be non-negative") + if tp_start < 0 or tp_end < tp_start: + raise ValueError("invalid TP vocabulary range") + checkpoint_end = checkpoint_start + checkpoint_rows + overlap_start = max(checkpoint_start, tp_start) + overlap_end = min(checkpoint_end, tp_end) + if overlap_start >= overlap_end: + return None + return PLEShardOverlap( + source_start=overlap_start - checkpoint_start, + destination_start=overlap_start - tp_start, + row_count=overlap_end - overlap_start, + ) + + +def copy_ple_embedding_shard_( + destination: torch.Tensor, + loaded_weight: torch.Tensor, + *, + checkpoint_start: int, + tp_start: int, + tp_end: int, +) -> int: + """Copy the overlapping rows of a PLE checkpoint shard into a TP table.""" + + if destination.ndim == 0 or loaded_weight.ndim != destination.ndim: + raise ValueError("destination and loaded weight must have matching ranks") + if destination.shape[1:] != loaded_weight.shape[1:]: + raise ValueError( + "embedding shard dimensions do not match: " + f"{tuple(destination.shape[1:])} != {tuple(loaded_weight.shape[1:])}" + ) + if destination.shape[0] < tp_end - tp_start: + raise ValueError("destination does not cover the requested TP range") + overlap = compute_ple_shard_overlap( + checkpoint_start=checkpoint_start, + checkpoint_rows=loaded_weight.shape[0], + tp_start=tp_start, + tp_end=tp_end, + ) + if overlap is None: + return 0 + source = loaded_weight.narrow(0, overlap.source_start, overlap.row_count) + target = destination.narrow(0, overlap.destination_start, overlap.row_count) + with torch.no_grad(): + target.copy_(source.to(device=target.device, dtype=target.dtype)) + return overlap.row_count + + +class PLEVocabParallelEmbedding(VocabParallelEmbedding): + """Vocab-parallel embedding that accepts checkpoint row shards.""" + + def weight_loader( + self, + param: torch.Tensor, + loaded_weight: torch.Tensor, + checkpoint_start: int | None = None, + ) -> None: + if checkpoint_start is None: + super().weight_loader(param, loaded_weight) + return + copy_ple_embedding_shard_( + param, + loaded_weight, + checkpoint_start=checkpoint_start, + tp_start=self.shard_indices.org_vocab_start_index, + tp_end=self.shard_indices.org_vocab_end_index, + ) diff --git a/vllm/models/qwen4_exp/common/qsa_cache.py b/vllm/models/qwen4_exp/common/qsa_cache.py new file mode 100644 index 0000000000..cffb961def --- /dev/null +++ b/vllm/models/qwen4_exp/common/qsa_cache.py @@ -0,0 +1,807 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Paged side-cache ownership and metadata for Qwen4Exp QSA. + +Each QSA layer keeps a fixed circular buffer of raw index keys (the +compressor state) and one compressed key. MRoPE models pack exact three-axis +positions beside the raw keys; text models derive group positions from +logical positions. The compressor state uses one block per request, while + the compressed owner uses ``MLAAttentionSpec.tokens_per_state`` so its block +table follows the main KV-cache lifecycle. Their physical tensor storage is +shared by the generic cache-layout planner. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from functools import cache +from typing import ClassVar + +import torch +from torch import nn + +from vllm.config import CacheConfig, VllmConfig +from vllm.config.cache import CacheDType +from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase +from vllm.platforms import current_platform +from vllm.triton_utils import HAS_TRITON, tl, triton +from vllm.utils.math_utils import cdiv +from vllm.v1.attention.backend import ( + AttentionBackend, + AttentionCGSupport, + AttentionMetadata, + AttentionMetadataBuilder, + CommonAttentionMetadata, +) +from vllm.v1.attention.backends.utils import PAD_SLOT_ID +from vllm.v1.kv_cache_interface import ( + AttentionSpec, + CircularBufferSpec, + KVCacheLayout, + KVCacheSpec, + MLAAttentionSpec, +) + + +def canonical_qsa_rope_positions(positions: torch.Tensor) -> torch.Tensor: + """Return exact per-token positions as ``[tokens, 1, 3]`` int64 rows.""" + + if positions.ndim == 1: + positions = positions.unsqueeze(0).expand(3, -1) + elif positions.ndim != 2 or positions.shape[0] not in (1, 3): + raise ValueError("QSA RoPE positions must be [tokens] or [1|3, tokens]") + if positions.shape[0] == 1: + positions = positions.expand(3, -1) + return positions.transpose(0, 1).unsqueeze(1).to(torch.int64) + + +def _logical_positions( + query_start_loc: torch.Tensor, + seq_lens: torch.Tensor, + token_to_req: torch.Tensor, + num_tokens: int, +) -> torch.Tensor: + if num_tokens == 0: + return seq_lens.new_empty((0,), dtype=torch.int64) + arange = torch.arange(num_tokens, device=query_start_loc.device) + requests = token_to_req[:num_tokens].long() + query_lens = torch.diff(query_start_loc) + within_query = arange - query_start_loc.index_select(0, requests) + return ( + seq_lens.index_select(0, requests).long() + - query_lens.index_select(0, requests).long() + + within_query.long() + ) + + +def _logical_to_physical_qsa_slots( + block_table: torch.Tensor, + request_indices: torch.Tensor, + logical_positions: torch.Tensor, + block_size: int, +) -> torch.Tensor: + if block_size <= 0: + raise ValueError("QSA cache block size must be positive") + if block_table.ndim != 2: + raise ValueError("QSA block table must be two-dimensional") + if request_indices.shape != logical_positions.shape: + request_indices = torch.broadcast_to(request_indices, logical_positions.shape) + + requests = request_indices.to(device=block_table.device, dtype=torch.long) + positions = logical_positions.to(device=block_table.device, dtype=torch.long) + valid = (requests >= 0) & (requests < block_table.shape[0]) & (positions >= 0) + logical_blocks = torch.div( + positions.clamp_min(0), block_size, rounding_mode="floor" + ) + valid &= logical_blocks < block_table.shape[1] + safe_requests = requests.clamp(0, max(block_table.shape[0] - 1, 0)) + safe_blocks = logical_blocks.clamp(0, max(block_table.shape[1] - 1, 0)) + if not all(block_table.shape): + return torch.full_like(positions, PAD_SLOT_ID) + physical_blocks = block_table[safe_requests, safe_blocks].long() + valid &= physical_blocks >= 0 + slots = physical_blocks * block_size + positions.remainder(block_size) + return torch.where(valid, slots, PAD_SLOT_ID) + + +def circular_qsa_slot_mapping( + block_table: torch.Tensor, + token_to_req: torch.Tensor, + logical_positions: torch.Tensor, + compressor_state_size: int, + query_start_loc: torch.Tensor | None = None, + out: torch.Tensor | None = None, +) -> torch.Tensor: + """Map each request to its fixed physical block as a circular token ring.""" + + if compressor_state_size <= 0: + raise ValueError("QSA circular buffer size must be positive") + if block_table.ndim != 2: + raise ValueError("QSA block table must be two-dimensional") + + requests = token_to_req.to(device=block_table.device, dtype=torch.long) + positions = logical_positions.to(device=block_table.device, dtype=torch.long) + if not all(block_table.shape): + slots = torch.full_like(positions, PAD_SLOT_ID) + else: + valid = (requests >= 0) & (requests < block_table.shape[0]) & (positions >= 0) + safe_requests = requests.clamp(0, block_table.shape[0] - 1) + physical_blocks = block_table[safe_requests, 0].long() + valid &= physical_blocks >= 0 + slots = physical_blocks * compressor_state_size + positions.remainder( + compressor_state_size + ) + slots = torch.where(valid, slots, PAD_SLOT_ID) + + if query_start_loc is not None: + if query_start_loc.ndim != 1 or query_start_loc.shape[0] < 2: + raise ValueError("QSA query starts must contain a terminal offset") + query_start_loc = query_start_loc.to(block_table.device) + num_requests = query_start_loc.shape[0] - 1 + safe_requests = requests.clamp(0, num_requests - 1) + request_ends = query_start_loc.index_select(0, safe_requests + 1) + rows = torch.arange(slots.numel(), device=slots.device) + keep = ( + (requests >= 0) + & (requests < num_requests) + & (rows + compressor_state_size >= request_ends) + ) + slots = torch.where(keep, slots, PAD_SLOT_ID) + + slots = slots.to(torch.int64) + if out is not None: + out.fill_(PAD_SLOT_ID) + out[: slots.numel()].copy_(slots) + return out[: slots.numel()] + return slots + + +def compressed_qsa_slot_mapping( + block_table: torch.Tensor, + token_to_req: torch.Tensor, + logical_positions: torch.Tensor, + storage_block_size: int, + compress_ratio: int, + out: torch.Tensor | None = None, +) -> torch.Tensor: + """Build boundary-only slots for an ``MLAAttentionSpec`` QSA cache.""" + + if storage_block_size <= 0 or compress_ratio <= 0: + raise ValueError("QSA block size and compression ratio must be positive") + compressed_positions = torch.div( + logical_positions.clamp_min(0), compress_ratio, rounding_mode="floor" + ) + slots = _logical_to_physical_qsa_slots( + block_table, + token_to_req, + compressed_positions, + storage_block_size, + ) + valid = (logical_positions >= 0) & ( + (logical_positions + 1).remainder(compress_ratio) == 0 + ) + slots = torch.where(valid, slots, PAD_SLOT_ID).to(torch.int64) + if out is not None: + out.fill_(PAD_SLOT_ID) + out[: slots.numel()].copy_(slots) + return out[: slots.numel()] + return slots + + +@cache +def _metadata_launch_pdl() -> bool: + return current_platform.is_arch_support_pdl() + + +@triton.jit( + do_not_specialize=[ + "num_reqs", + "num_mapped_tokens", + "num_tokens", + "max_num_work", + "num_search_steps", + "work_search_steps", + ] +) +def _build_qsa_metadata_kernel( + query_start_loc_ptr, + seq_lens_ptr, + common_slot_mapping_ptr, + block_table_ptr, + token_to_req_ptr, + logical_positions_ptr, + slot_mapping_ptr, + k_work_metadata_ptr, + block_table_stride_0: tl.constexpr, + block_table_stride_1: tl.constexpr, + num_reqs, + num_mapped_tokens, + num_tokens, + max_num_work, + num_search_steps, + work_search_steps, + storage_block_size: tl.constexpr, + compress_ratio: tl.constexpr, + circular_buffer_size: tl.constexpr, + num_block_table_columns: tl.constexpr, + launch_pdl: tl.constexpr, + TOKEN_BLOCK_SIZE: tl.constexpr, + REQUEST_SCAN_SIZE: tl.constexpr, + WORK_BLOCK_SIZE: tl.constexpr, +): + if launch_pdl: + tl.extra.cuda.gdc_wait() + + pid = tl.program_id(0) + token_idx = pid * TOKEN_BLOCK_SIZE + tl.arange(0, TOKEN_BLOCK_SIZE) + store_mask = token_idx < num_tokens + mapped = token_idx < num_mapped_tokens + search_token_idx = tl.minimum(token_idx, num_mapped_tokens - 1) + request_idx = tl.zeros((TOKEN_BLOCK_SIZE,), tl.int32) + # Find the last query start at or before each token. The dynamic loop avoids + # compiling a kernel variant for every ceil(log2(num_reqs)). + for step in tl.range(0, num_search_steps): + candidate = request_idx + (1 << (num_search_steps - step - 1)) + valid_candidate = candidate < num_reqs + candidate_start = tl.load( + query_start_loc_ptr + candidate, + mask=valid_candidate, + other=num_mapped_tokens + 1, + ) + advance = valid_candidate & (candidate_start <= search_token_idx) + request_idx = tl.where(advance, candidate, request_idx) + query_start = tl.load(query_start_loc_ptr + request_idx, mask=mapped, other=0) + query_end = tl.load(query_start_loc_ptr + request_idx + 1, mask=mapped, other=0) + seq_len = tl.load(seq_lens_ptr + request_idx, mask=mapped, other=0) + logical_position = seq_len - (query_end - query_start) + token_idx - query_start + logical_position = tl.where(mapped, logical_position, -1) + tl.store( + token_to_req_ptr + token_idx, + tl.where(mapped, request_idx, 0), + mask=store_mask, + ) + tl.store( + logical_positions_ptr + token_idx, + logical_position, + mask=store_mask, + ) + + # circular_buffer_size is constexpr, so each builder instance compiles out + # the other QSA cache owner's slot-mapping rule. + if circular_buffer_size > 0: + valid = ( + mapped + & (logical_position >= 0) + & (token_idx + circular_buffer_size >= query_end) + & (num_block_table_columns > 0) + ) + physical_block = tl.load( + block_table_ptr + request_idx * block_table_stride_0, + mask=valid, + other=-1, + ) + valid &= physical_block >= 0 + slot = physical_block * circular_buffer_size + ( + logical_position % circular_buffer_size + ) + elif compress_ratio != 1: + compressed_position = tl.maximum(logical_position, 0) // compress_ratio + logical_block = compressed_position // storage_block_size + valid = ( + mapped + & (logical_position >= 0) + & ((logical_position + 1) % compress_ratio == 0) + & (logical_block < num_block_table_columns) + ) + physical_block = tl.load( + block_table_ptr + + request_idx * block_table_stride_0 + + logical_block * block_table_stride_1, + mask=valid, + other=-1, + ) + valid &= physical_block >= 0 + valid &= ( + tl.load(common_slot_mapping_ptr + token_idx, mask=mapped, other=-1) >= 0 + ) + slot = physical_block * storage_block_size + ( + compressed_position % storage_block_size + ) + if (circular_buffer_size > 0) or (compress_ratio != 1): + tl.store( + slot_mapping_ptr + token_idx, + tl.where(valid, slot, -1), + mask=store_mask, + ) + work_tile_start = pid * WORK_BLOCK_SIZE + has_work_tile = work_tile_start < max_num_work + if k_work_metadata_ptr is not None and has_work_tile: + # Every work CTA builds the request prefix in registers. Recomputing this + # small vector lets CTAs write disjoint work tiles without a grid barrier. + requests = tl.arange(0, REQUEST_SCAN_SIZE) + valid_request = requests < num_reqs + request_query_start = tl.load( + query_start_loc_ptr + requests, mask=valid_request, other=0 + ) + request_query_end = tl.load( + query_start_loc_ptr + requests + 1, mask=valid_request, other=0 + ) + request_seq_len = tl.load(seq_lens_ptr + requests, mask=valid_request, other=0) + request_query_len = request_query_end - request_query_start + chunk_start = request_seq_len - request_query_len + num_groups = request_seq_len // compress_ratio - chunk_start // compress_ratio + # Nonempty requests need one item even without a completed compression + # group because work item zero also commits the current raw-K suffix. + work_counts = tl.where(request_query_len > 0, tl.maximum(num_groups, 1), 0) + work_ends = tl.cumsum(tl.where(valid_request, work_counts, 0), axis=0) + total_work = tl.sum(tl.where(valid_request, work_counts, 0), axis=0) + work_offsets = tl.arange(0, WORK_BLOCK_SIZE) + work = work_tile_start + work_offsets + in_bounds = work < max_num_work + active = in_bounds & (work < total_work) + request = tl.zeros((WORK_BLOCK_SIZE,), dtype=tl.int32) + # Request zero starts at zero for every active work item. Descending + # steps find the last request starting at or before this item. + for step_idx in tl.range(0, work_search_steps): + step = 1 << (work_search_steps - step_idx - 1) + candidate = request + step + valid_candidate = candidate < num_reqs + candidate_start = tl.gather(work_ends, candidate - 1, 0) + advance = active & valid_candidate & (candidate_start <= work) + request = tl.where(advance, candidate, request) + + if launch_pdl: + # Let the dependent grid start launch setup while these CTAs finish + # stores; its gdc_wait still orders access to the completed metadata. + tl.extra.cuda.gdc_launch_dependents() + + owner_work_start = tl.gather(work_ends, tl.maximum(request - 1, 0), 0) + owner_work_start = tl.where(request == 0, 0, owner_work_start) + work_in_request = work - owner_work_start + tl.store( + k_work_metadata_ptr + work * 2, + tl.where(active, request, -1), + mask=in_bounds, + ) + tl.store( + k_work_metadata_ptr + work * 2 + 1, + tl.where(active, work_in_request, -1), + mask=in_bounds, + ) + + else: + if launch_pdl: + # A dependent grid launches only after every CTA has signaled. + tl.extra.cuda.gdc_launch_dependents() + + +def build_qsa_metadata_triton( + common_attn_metadata: CommonAttentionMetadata, + token_to_req_buffer: torch.Tensor, + logical_positions_buffer: torch.Tensor, + slot_mapping_buffer: torch.Tensor, + *, + storage_block_size: int, + compress_ratio: int, + circular_buffer_size: int = 0, + k_work_metadata_buffer: torch.Tensor | None = None, + request_capacity: int | None = None, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Build QSA side-cache and optional pre-indexer work metadata.""" + num_tokens = common_attn_metadata.num_actual_tokens + num_mapped_tokens = int(common_attn_metadata.query_start_loc_cpu[-1]) + token_to_req = token_to_req_buffer[:num_tokens] + logical_positions = logical_positions_buffer[:num_tokens] + slot_mapping = slot_mapping_buffer[:num_tokens] + num_reqs = common_attn_metadata.query_start_loc.shape[0] - 1 + assert num_reqs > 0 + + if k_work_metadata_buffer is not None: + if request_capacity is None: + request_capacity = num_reqs + assert request_capacity >= num_reqs + # Pad for tl.arange while keeping the scan width stable across live batches. + request_scan_size = 1 << int(math.ceil(math.log2(request_capacity))) + max_num_work = k_work_metadata_buffer.shape[0] + else: + request_scan_size = 1 + max_num_work = 0 + + if num_tokens == 0 and k_work_metadata_buffer is None: + return token_to_req, logical_positions, slot_mapping + + block_table = common_attn_metadata.block_table_tensor + num_search_steps = int(math.ceil(math.log2(num_reqs))) + work_search_steps = int(math.ceil(math.log2(num_reqs))) + # The same grid covers token tiles and, for the compressed cache, work tiles. + num_token_blocks = cdiv(num_tokens, 128) + num_work_blocks = ( + cdiv(max_num_work, 256) if k_work_metadata_buffer is not None else 0 + ) + _build_qsa_metadata_kernel[(max(num_token_blocks, num_work_blocks, 1),)]( + common_attn_metadata.query_start_loc, + common_attn_metadata.seq_lens, + common_attn_metadata.slot_mapping, + block_table, + token_to_req, + logical_positions, + slot_mapping, + k_work_metadata_buffer, + block_table.stride(0), + block_table.stride(1), + num_reqs, + num_mapped_tokens, + num_tokens, + max_num_work, + num_search_steps, + work_search_steps, + storage_block_size, + compress_ratio, + circular_buffer_size, + block_table.shape[1], + launch_pdl=_metadata_launch_pdl(), + TOKEN_BLOCK_SIZE=128, + REQUEST_SCAN_SIZE=request_scan_size, + WORK_BLOCK_SIZE=256, + num_warps=4, + ) + if circular_buffer_size == 0 and compress_ratio == 1: + slot_mapping = common_attn_metadata.slot_mapping[:num_tokens] + return token_to_req, logical_positions, slot_mapping + + +def _build_qsa_metadata_torch( + common_attn_metadata: CommonAttentionMetadata, + token_to_req_buffer: torch.Tensor, + logical_positions_buffer: torch.Tensor, + slot_mapping_buffer: torch.Tensor, + *, + storage_block_size: int, + compress_ratio: int, + circular_buffer_size: int = 0, + k_work_metadata_buffer: torch.Tensor | None = None, + request_capacity: int | None = None, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + del request_capacity + num_tokens = common_attn_metadata.num_actual_tokens + num_mapped_tokens = int(common_attn_metadata.query_start_loc_cpu[-1]) + logical_positions = logical_positions_buffer[:num_tokens] + + token_to_req = common_attn_metadata.token_to_req_indices(token_to_req_buffer)[ + :num_tokens + ] + logical_positions[:num_mapped_tokens].copy_( + _logical_positions( + common_attn_metadata.query_start_loc, + common_attn_metadata.seq_lens, + token_to_req[:num_mapped_tokens], + num_mapped_tokens, + ) + ) + if num_mapped_tokens < num_tokens: + logical_positions[num_mapped_tokens:].fill_(-1) + if circular_buffer_size > 0: + slot_mapping = circular_qsa_slot_mapping( + common_attn_metadata.block_table_tensor, + token_to_req, + logical_positions, + circular_buffer_size, + query_start_loc=common_attn_metadata.query_start_loc, + out=slot_mapping_buffer, + ) + elif compress_ratio == 1: + slot_mapping = common_attn_metadata.slot_mapping[:num_tokens] + else: + slot_mapping = compressed_qsa_slot_mapping( + common_attn_metadata.block_table_tensor, + token_to_req, + logical_positions, + storage_block_size, + compress_ratio, + slot_mapping_buffer, + ) + slot_mapping.masked_fill_( + common_attn_metadata.slot_mapping[:num_tokens] < 0, -1 + ) + if k_work_metadata_buffer is not None: + query_lens = ( + common_attn_metadata.query_start_loc[1:] + - common_attn_metadata.query_start_loc[:-1] + ) + chunk_starts = common_attn_metadata.seq_lens - query_lens + num_work_per_request = ( + common_attn_metadata.seq_lens // compress_ratio + - chunk_starts // compress_ratio + ) + num_work_per_request = torch.where( + query_lens > 0, num_work_per_request.clamp_min(1), 0 + ) + k_start_loc = torch.empty( + query_lens.shape[0] + 1, + dtype=torch.int32, + device=query_lens.device, + ) + k_start_loc[0] = 0 + torch.cumsum(num_work_per_request, 0, out=k_start_loc[1:]) + work = torch.arange( + k_work_metadata_buffer.shape[0], + device=k_work_metadata_buffer.device, + ) + requests = torch.searchsorted(k_start_loc[1:], work, right=True) + active = work < k_start_loc[-1] + work_in_request = ( + work - k_start_loc[requests.clamp_max(query_lens.shape[0] - 1)] + ) + k_work_metadata_buffer[:, 0].copy_( + torch.where(active, requests, -1).to(torch.int32) + ) + k_work_metadata_buffer[:, 1].copy_( + torch.where(active, work_in_request, -1).to(torch.int32) + ) + return token_to_req, logical_positions, slot_mapping + + +# Resolve the fallback outside the per-step metadata hot path. +build_qsa_metadata = ( + build_qsa_metadata_triton if HAS_TRITON else _build_qsa_metadata_torch +) + + +@dataclass +class QSAForwardMetadata(AttentionMetadata): + """Common per-forward metadata for one QSA side cache.""" + + block_table: torch.Tensor + slot_mapping: torch.Tensor + seq_lens: torch.Tensor + query_start_loc: torch.Tensor + token_to_req: torch.Tensor + logical_positions: torch.Tensor + k_work_metadata: torch.Tensor + num_actual_tokens: int + storage_block_size: int + compress_ratio: int + + +class QSAMetadataBuilder(AttentionMetadataBuilder[QSAForwardMetadata]): + """Build QSA metadata from vLLM's cache-group-specific common metadata.""" + + _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH + + def __init__( + self, + kv_cache_spec: AttentionSpec, + layer_names: list[str], + vllm_config: VllmConfig, + device: torch.device, + ) -> None: + super().__init__(kv_cache_spec, layer_names, vllm_config, device) + self.is_circular_buffer = isinstance(kv_cache_spec, CircularBufferSpec) + if isinstance(kv_cache_spec, MLAAttentionSpec): + compress_ratio = kv_cache_spec.tokens_per_state + assert isinstance(compress_ratio, int), ( + "QSA compression requires an integer tokens_per_state" + ) + self.compress_ratio = compress_ratio + else: + self.compress_ratio = 1 + self.storage_block_size = kv_cache_spec.num_states + max_tokens = vllm_config.scheduler_config.max_num_batched_tokens + self.token_to_req_buffer = torch.empty( + max_tokens, dtype=torch.int32, device=device + ) + self.slot_mapping_buffer = torch.empty( + max_tokens, dtype=torch.int64, device=device + ) + self.logical_positions_buffer = torch.empty( + max_tokens, dtype=torch.int64, device=device + ) + max_requests = vllm_config.scheduler_config.max_num_seqs + self.request_capacity = max_requests + if not self.is_circular_buffer and self.compress_ratio != 1: + max_k_work = ( + max_tokens + (self.compress_ratio - 1) * max_requests + ) // self.compress_ratio + self.k_work_metadata_buffer = torch.empty( + max_k_work, 2, dtype=torch.int32, device=device + ) + else: + self.k_work_metadata_buffer = torch.empty( + 0, 2, dtype=torch.int32, device=device + ) + + def build( + self, + common_prefix_len: int, + common_attn_metadata: CommonAttentionMetadata, + fast_build: bool = False, + ) -> QSAForwardMetadata: + del common_prefix_len, fast_build + num_tokens = common_attn_metadata.num_actual_tokens + build_k_work = not self.is_circular_buffer and self.compress_ratio != 1 + k_work_metadata = self.k_work_metadata_buffer + request_capacity = None + if build_k_work: + num_requests = common_attn_metadata.query_start_loc.shape[0] - 1 + request_capacity = self.request_capacity + max_num_work = ( + num_tokens + (self.compress_ratio - 1) * num_requests + ) // self.compress_ratio + k_work_metadata = self.k_work_metadata_buffer[:max_num_work] + token_to_req, logical_positions, slot_mapping = build_qsa_metadata( + common_attn_metadata, + self.token_to_req_buffer, + self.logical_positions_buffer, + self.slot_mapping_buffer, + storage_block_size=self.storage_block_size, + compress_ratio=self.compress_ratio, + circular_buffer_size=( + self.kv_cache_spec.block_size if self.is_circular_buffer else 0 + ), + k_work_metadata_buffer=k_work_metadata if build_k_work else None, + request_capacity=request_capacity, + ) + return QSAForwardMetadata( + block_table=common_attn_metadata.block_table_tensor, + slot_mapping=slot_mapping, + seq_lens=common_attn_metadata.seq_lens, + query_start_loc=common_attn_metadata.query_start_loc, + token_to_req=token_to_req, + logical_positions=logical_positions, + k_work_metadata=k_work_metadata, + num_actual_tokens=num_tokens, + storage_block_size=self.storage_block_size, + compress_ratio=self.compress_ratio, + ) + + +class QSAStateBackend(AttentionBackend): + """Key-only dummy backend for out-of-band BF16 QSA side-cache operations.""" + + supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16] + supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = ["auto", "bfloat16"] + + @staticmethod + def get_name() -> str: + return "QWEN4_EXP_EXP_QSA_STATE" + + @staticmethod + def get_impl_cls(): + raise NotImplementedError( + "QSA state caches run out-of-band and have no attention impl" + ) + + @staticmethod + def get_builder_cls() -> type[QSAMetadataBuilder]: + return QSAMetadataBuilder + + @classmethod + def supported_kv_cache_layouts(cls) -> tuple[KVCacheLayout, ...]: + # QSA pages are packed beside the main KV pages within each block. + return (KVCacheLayout.BLNHC, KVCacheLayout.BLHNC) + + +class _QSAStateCache(nn.Module, AttentionLayerBase): + supports_dcp = False + + def __init__( + self, + *, + head_size: int, + dtype: torch.dtype, + cache_config: CacheConfig, + prefix: str, + vllm_config: VllmConfig, + compress_ratio: int = 1, + ) -> None: + super().__init__() + if head_size <= 0: + raise ValueError("QSA cache head size must be positive") + if compress_ratio <= 0: + raise ValueError("QSA compression ratio must be positive") + if cache_config.block_size % compress_ratio: + raise ValueError( + "QSA cache block size must be divisible by the compression ratio" + ) + self.head_size = head_size + self.dtype = dtype + self.cache_config = cache_config + self.prefix = prefix + self.compress_ratio = compress_ratio + self.kv_cache = torch.tensor([]) + + static_context = vllm_config.compilation_config.static_forward_context + if prefix in static_context: + raise ValueError(f"Duplicate layer name: {prefix}") + static_context[prefix] = self + + def forward(self) -> None: ... + + def bind_kv_cache(self, kv_cache: torch.Tensor) -> None: + """Adapt the unified [B, H, N, C] view to QSA's [B, N, H, C].""" + if kv_cache.ndim != 4 or kv_cache.shape[1] != 1: + raise ValueError("QSA state cache must be [blocks, 1, states, width]") + if kv_cache.dtype != torch.bfloat16 or kv_cache.shape[3] != self.head_size: + raise ValueError("QSA state cache does not match its packed BF16 spec") + super().bind_kv_cache(kv_cache.transpose(1, 2)) + + def get_attn_backend(self) -> type[AttentionBackend]: + return QSAStateBackend + + +class QSAKeyStateCache(_QSAStateCache): + """Raw BF16 key, optionally followed by exact int64 MRoPE positions.""" + + _BF16_PER_INT64 = 4 + _NUM_ROPE_AXES = 3 + + def __init__(self, *, cache_rope_positions: bool = False, **kwargs) -> None: + key_head_size = int(kwargs.pop("head_size")) + self.key_head_size = key_head_size + self.cache_rope_positions = bool(cache_rope_positions) + self.rope_position_offset = ( + (key_head_size + self._BF16_PER_INT64 - 1) // self._BF16_PER_INT64 + ) * self._BF16_PER_INT64 + storage_head_size = key_head_size + if self.cache_rope_positions: + storage_head_size = self.rope_position_offset + ( + self._NUM_ROPE_AXES * self._BF16_PER_INT64 + ) + super().__init__(head_size=storage_head_size, **kwargs) + + def bind_kv_cache(self, kv_cache: torch.Tensor) -> None: + super().bind_kv_cache(kv_cache) + qsa_cache = self.kv_cache + self.key_cache = qsa_cache[..., : self.key_head_size] + if self.cache_rope_positions: + position_tail = qsa_cache[..., self.rope_position_offset :] + self.rope_position_cache = position_tail.view(torch.int64) + else: + self.rope_position_cache = None + + def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec: + # Hold the open group's committed keys plus every row a speculative + # step stores before acceptance is known, rounded up to whole groups so + # the ring divides the attention block size (it joins the LCM that sets + # the scheduler block size). Anything narrower lets a rejected draft row + # overwrite a committed key the next step needs to close the group. + span = self.compress_ratio + vllm_config.num_speculative_tokens + capacity = self.compress_ratio * cdiv(span, self.compress_ratio) + assert self.cache_config.block_size % capacity == 0, ( + f"QSA ring capacity {capacity} must divide the attention block " + f"size {self.cache_config.block_size}" + ) + return CircularBufferSpec( + block_size=capacity, + num_kv_heads=1, + head_size=self.head_size, + head_size_v=0, + dtype=self.dtype, + ) + + +class QSACompressedKeyCache(_QSAStateCache): + """Normalized, group-first-RoPE BF16 key at one row per complete group.""" + + def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec: + del vllm_config + return MLAAttentionSpec( + block_size=self.cache_config.block_size, + num_kv_heads=1, + head_size=self.head_size, + dtype=self.dtype, + tokens_per_state=self.compress_ratio, + ) + + +__all__ = [ + "QSACompressedKeyCache", + "QSAForwardMetadata", + "QSAKeyStateCache", + "QSAMetadataBuilder", + "QSAStateBackend", + "canonical_qsa_rope_positions", + "circular_qsa_slot_mapping", + "compressed_qsa_slot_mapping", +] diff --git a/vllm/models/qwen4_exp/config.py b/vllm/models/qwen4_exp/config.py new file mode 100644 index 0000000000..a8eedab83f --- /dev/null +++ b/vllm/models/qwen4_exp/config.py @@ -0,0 +1,252 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Qwen4Exp model configuration.""" + +from typing import Any, ClassVar, cast + +from transformers import PretrainedConfig +from transformers.models.qwen3_vl.configuration_qwen3_vl import ( + Qwen3VLVisionConfig, +) + +from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig + +_QSA_CONFIG_FIELDS = ( + "indexer_n_heads", + "indexer_kv_heads", + "indexer_head_dim", + "indexer_budget", + "indexer_compress_ratio", +) + + +class Qwen4ExpVisionConfig(Qwen3VLVisionConfig): + model_type = "qwen4_exp" + base_config_key = "vision_config" + + +class Qwen4ExpTextConfig(Qwen3NextConfig): + model_type = "qwen4_exp_text" + base_config_key = "text_config" + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + hc_count: int = 4, + hc_lowrank: int = 320, + ple_layer_ids: list[int] | None = None, + ple_embed_dim: int | None = None, + ple_conv_kernel_size: int = 4, + ngram_size: int = 3, + heads_per_ngram: int = 8, + ngram_vocab_size_base: int = 20_000_000, + make_ngram_vocab_size_divisible_by: int = 128, + output_gate_type: str = "sigmoid", + rope_parameters: dict[str, Any] | None = None, + layer_types: list[str] | None = None, + **kwargs: Any, + ) -> None: + if hc_count <= 1: + raise ValueError(f"Qwen4Exp requires hc_count > 1, got {hc_count}.") + + if rope_parameters is not None: + if kwargs.get("rope_scaling") is None: + kwargs["rope_scaling"] = rope_parameters + if kwargs.get("rope_theta") is None and "rope_theta" in rope_parameters: + kwargs["rope_theta"] = rope_parameters["rope_theta"] + if ( + kwargs.get("partial_rotary_factor") is None + and "partial_rotary_factor" in rope_parameters + ): + kwargs["partial_rotary_factor"] = rope_parameters[ + "partial_rotary_factor" + ] + + rope_scaling = kwargs.get("rope_scaling") + rope_theta = kwargs.get("rope_theta", 10_000.0) + super().__init__(layer_types=layer_types, **kwargs) + + normalized_rope_parameters = self.rope_parameters + self.rope_scaling = ( + rope_scaling or rope_parameters or normalized_rope_parameters + ) + self.rope_parameters = rope_parameters or normalized_rope_parameters + self.rope_theta = rope_theta + + self.hc_count = hc_count + self.hc_lowrank = hc_lowrank + self.ple_layer_ids = ple_layer_ids or [] + self.ple_embed_dim = ( + self.hidden_size if ple_embed_dim is None else ple_embed_dim + ) + self.ple_conv_kernel_size = ple_conv_kernel_size + self.ngram_size = ngram_size + self.heads_per_ngram = heads_per_ngram + self.ngram_vocab_size_base = ngram_vocab_size_base + self.make_ngram_vocab_size_divisible_by = make_ngram_vocab_size_divisible_by + self.output_gate_type = output_gate_type + + self._validate_ple_config() + self._validate_ple_layer_ids() + self._validate_qsa_config() + + def _validate_ple_config(self) -> None: + if self.hc_lowrank <= 0: + raise ValueError(f"hc_lowrank must be positive, got {self.hc_lowrank}") + if self.ngram_size < 2: + raise ValueError(f"ngram_size must be >= 2, got {self.ngram_size}") + if self.heads_per_ngram <= 0: + raise ValueError( + f"heads_per_ngram must be positive, got {self.heads_per_ngram}" + ) + if self.ple_embed_dim <= 0: + raise ValueError( + f"ple_embed_dim must be positive, got {self.ple_embed_dim}" + ) + ngram_heads = (self.ngram_size - 1) * self.heads_per_ngram + if self.ple_embed_dim % ngram_heads: + raise ValueError( + "ple_embed_dim must be divisible by total ngram heads: " + f"{self.ple_embed_dim} % {ngram_heads} != 0" + ) + if self.ple_conv_kernel_size <= 0: + raise ValueError( + "ple_conv_kernel_size must be positive, got " + f"{self.ple_conv_kernel_size}" + ) + if self.ngram_vocab_size_base <= 0: + raise ValueError("ngram_vocab_size_base must be positive") + if self.make_ngram_vocab_size_divisible_by <= 0: + raise ValueError("make_ngram_vocab_size_divisible_by must be positive") + + def _validate_ple_layer_ids(self) -> None: + invalid = [ + layer_id + for layer_id in self.ple_layer_ids + if not 1 <= int(layer_id) <= self.num_hidden_layers + ] + if invalid: + raise ValueError( + "ple_layer_ids are 1-based and must refer to an existing layer; " + f"got {invalid} for {self.num_hidden_layers} layers" + ) + + def _validate_qsa_config(self) -> None: + configured = {name: getattr(self, name, None) for name in _QSA_CONFIG_FIELDS} + if all(value is None for value in configured.values()): + return + + missing = [name for name, value in configured.items() if value is None] + if missing: + raise ValueError(f"QSA config is missing required fields: {missing}") + + values = {name: int(cast(int, value)) for name, value in configured.items()} + if any(value <= 0 for value in values.values()): + raise ValueError(f"QSA config values must be positive: {values}") + if values["indexer_kv_heads"] != 1: + raise ValueError("the QSA MQA operators require indexer_kv_heads=1") + if values["indexer_budget"] % values["indexer_compress_ratio"] != 0: + raise ValueError( + "indexer_budget must be divisible by indexer_compress_ratio" + ) + block_topk = values["indexer_budget"] // values["indexer_compress_ratio"] + if block_topk not in (512, 2048): + raise ValueError( + "QSA requires indexer_budget / indexer_compress_ratio " + f"to be 512 or 2048, got {block_topk}" + ) + rotary_dim = int(self.head_dim * self.partial_rotary_factor) + if rotary_dim > values["indexer_head_dim"]: + raise ValueError( + "QSA indexer_head_dim must cover the attention rotary " + f"dimension, got {values['indexer_head_dim']} < {rotary_dim}" + ) + + @property + def layers_block_type(self) -> list[str]: + return [ + "attention" if layer_type == "full_attention" else layer_type + for layer_type in self.layer_types + ] + + @property + def short_conv_layer_ids(self) -> list[int]: + if not self.ple_layer_ids: + return [] + return sorted({int(layer_id) - 1 for layer_id in self.ple_layer_ids}) + + @property + def short_conv_state_shape(self) -> tuple[int, int] | None: + if not self.short_conv_layer_ids: + return None + ple_state_len = (self.ple_conv_kernel_size - 1) * self.ngram_size + ple_channels = self.hidden_size * self.hc_count + return ple_channels, ple_state_len + + @property + def ngram_context_len(self) -> int: + if not self.ple_layer_ids: + return 0 + return max(int(self.ngram_size) - 1, 0) + + +class Qwen4ExpConfig(PretrainedConfig): + model_type = "qwen4_exp" + sub_configs: ClassVar[dict[str, type[PretrainedConfig]]] = { + "vision_config": Qwen4ExpVisionConfig, + "text_config": Qwen4ExpTextConfig, + } + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + text_config: Qwen4ExpTextConfig | dict[str, Any] | None = None, + vision_config: Qwen4ExpVisionConfig | dict[str, Any] | None = None, + image_token_id: int = 248056, + video_token_id: int = 248057, + vision_start_token_id: int = 248053, + vision_end_token_id: int = 248054, + tie_word_embeddings: bool = False, + rope_parameters: dict[str, Any] | None = None, + **kwargs: Any, + ) -> None: + if text_config is not None: + kwargs.pop("split_ngram_parts", None) + + text_kwargs = ( + dict(kwargs) + if text_config is None + and "hidden_size" in kwargs + and "num_hidden_layers" in kwargs + else {} + ) + + if isinstance(vision_config, dict): + self.vision_config = self.sub_configs["vision_config"](**vision_config) + elif vision_config is None: + self.vision_config = self.sub_configs["vision_config"]() + else: + self.vision_config = vision_config + + if isinstance(text_config, dict): + self.text_config = self.sub_configs["text_config"](**text_config) + elif text_config is None: + self.text_config = self.sub_configs["text_config"](**text_kwargs) + else: + self.text_config = text_config + + self.image_token_id = image_token_id + self.video_token_id = video_token_id + self.vision_start_token_id = vision_start_token_id + self.vision_end_token_id = vision_end_token_id + self.rope_parameters = rope_parameters or getattr( + self.text_config, "rope_parameters", {} + ) + super().__init__(**kwargs, tie_word_embeddings=tie_word_embeddings) + + +__all__ = [ + "Qwen4ExpConfig", + "Qwen4ExpTextConfig", + "Qwen4ExpVisionConfig", +] diff --git a/vllm/models/qwen4_exp/nvidia/__init__.py b/vllm/models/qwen4_exp/nvidia/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/vllm/models/qwen4_exp/nvidia/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm/models/qwen4_exp/nvidia/hyperconnection.py b/vllm/models/qwen4_exp/nvidia/hyperconnection.py new file mode 100644 index 0000000000..8ca503d214 --- /dev/null +++ b/vllm/models/qwen4_exp/nvidia/hyperconnection.py @@ -0,0 +1,205 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""HyperConnection (Gated Residual) utilities — NVIDIA model variant. + +Implements the HyperConnection residual scheme proposed in +"HyperConnections" (https://arxiv.org/abs/2409.19606). This NVIDIA variant +delays each HC combine to the following HC mix boundary. HC glue kernels, +including fused combine+RMSNorm, live in ``ops/hc.py``; projections remain +standard vLLM Linear modules. + +Hidden states between layers have shape ``[..., HC*HS]`` with HS inner +(HC outer, HS inner — checkpoint-native layout). + +Typical usage inside a transformer decoder layer:: + + self.attn_hc = GatedResidual(hc_config) + + hidden_states, block_input, injection = self.attn_hc.mix(hidden_states) + attention_output = attention(block_input) + hidden_states, block_input, injection = self.mlp_hc.combine_and_mix( + hidden_states, attention_output, injection + ) +""" + +import torch +from torch import nn + +from vllm.model_executor.layers.linear import ( + MergedColumnParallelLinear, + ReplicatedLinear, +) +from vllm.model_executor.models.utils import maybe_prefix + +from ..common.hyperconnection import ( + GroupedGemmaRMSNorm, + HyperConnectionConfig, +) +from .ops.hc import ( + grouped_gemma_rmsnorm, + hc_combine, + hc_combine_norm, + hc_gate_mix, + hc_silu, +) + + +# --------------------------------------------------------------------------- +# Gated-residual variant +# --------------------------------------------------------------------------- +class GatedResidual(nn.Module): + """Gated HyperConnection with learnable low-rank mixing and injection. + + ``combine_and_mix()`` runs the pre pipeline (grouped GemmaRMSNorm -> merged + low-rank down+inject GEMM -> silu -> up GEMM -> sigmoid -> gated mean + over the HC streams). When passed a pending block output and an injection, + it fuses their residual combine with the RMSNorm. Final mixers use + ``use_combine=False`` and do not produce a new injection. + + Weights: the norm owns the grouped GemmaRMSNorm affine; the projections + are vLLM Linear modules (merged replicated linear for down+inject), so + GEMM dispatch (e.g. the low-latency skinny GEMM) applies through the + standard quant_method mechanism. + """ + + def __init__( + self, + config: HyperConnectionConfig, + use_combine: bool = True, + prefix: str = "", + ) -> None: + super().__init__() + self.config = config + self.lora_rank = config.hc_lowrank + self.hc_count = config.hc_count + self.hidden_size = config.hidden_size + self.use_combine = use_combine + + norm_size = ( + self.hyper_hidden_size if config.hc_per_branch_norm else config.hidden_size + ) + group_size = config.hidden_size if config.hc_per_branch_norm else None + # Normalize each H-sized HC stream independently while retaining a + # separate affine weight for every element of the HC*H layout. + self.hc_norm = GroupedGemmaRMSNorm( + norm_size, + eps=config.rms_norm_eps, + group_size=group_size, + dtype=config.params_dtype, + ) + + # -- vLLM Linear weights -------------------------------------------- + # The merged skinny-GEMM shape is physically padded to 16 rows to ensure + # good alignment and performant implementation chosen by CuBLAS heuristics. + self.pad_size = (-(self.lora_rank + self.hc_count)) % 16 if use_combine else 0 + if use_combine: + self.input_mix_weight_down_block_inject = MergedColumnParallelLinear( + self.hyper_hidden_size, + [self.lora_rank, self.hc_count] + + ([self.pad_size] if self.pad_size else []), + bias=False, + params_dtype=config.params_dtype, + quant_config=None, + prefix=maybe_prefix(prefix, "input_mix_weight_down_block_inject"), + return_bias=False, + disable_tp=True, + ) + else: + self.input_mix_weight_down = ReplicatedLinear( + self.hyper_hidden_size, + self.lora_rank, + bias=False, + params_dtype=config.params_dtype, + quant_config=None, + prefix=maybe_prefix(prefix, "input_mix_weight_down"), + return_bias=False, + ) + self.input_mix_weight_up = ReplicatedLinear( + self.lora_rank, + self.hyper_hidden_size, + bias=False, + params_dtype=config.params_dtype, + quant_config=None, + prefix=maybe_prefix(prefix, "input_mix_weight_up"), + return_bias=False, + ) + + def mix( + self, hidden_states: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]: + xn = grouped_gemma_rmsnorm( + hidden_states, + self.hc_norm.weight, + self.config.rms_norm_eps, + self.hc_count, + ) + + if self.use_combine: + # produce injection logits for combine + split_sizes = [self.lora_rank, self.hc_count, self.pad_size] + down_and_injection = self.input_mix_weight_down_block_inject(xn) + lora, injection, _ = down_and_injection.split(split_sizes, dim=-1) + else: + lora = self.input_mix_weight_down(xn) + injection = None + + lora = hc_silu(lora, self.hc_count) + gate = self.input_mix_weight_up(lora) # [M, D] + block_input = hc_gate_mix(xn, gate, self.hc_count) + + return hidden_states, block_input, injection + + def combine_and_mix( + self, + hidden_states: torch.Tensor, + prev_block_output: torch.Tensor, + prev_injection: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]: + """Consume a pending combine, then prepare the next block input. + + ``hidden_states`` is the multi-stream state from before the pending + block's mix. Its combine with ``block_output`` is fused with this + module's input RMSNorm. + """ + hidden_states, xn = hc_combine_norm( + hidden_states, + prev_block_output, + prev_injection, + self.hc_norm.weight, + self.config.rms_norm_eps, + self.hc_count, + ) + + if self.use_combine: + # produce injection logits for combine + split_sizes = [self.lora_rank, self.hc_count, self.pad_size] + down_and_injection = self.input_mix_weight_down_block_inject(xn) + lora, injection, _ = down_and_injection.split(split_sizes, dim=-1) + else: + lora = self.input_mix_weight_down(xn) + injection = None + + lora = hc_silu(lora, self.hc_count) + gate = self.input_mix_weight_up(lora) # [M, D] + block_input = hc_gate_mix(xn, gate, self.hc_count) + + return hidden_states, block_input, injection + + def combine( + self, + hidden_states: torch.Tensor, + block_output: torch.Tensor, + injection: torch.Tensor, + ) -> torch.Tensor: + return hc_combine(hidden_states, block_output, injection, self.hc_count) + + @property + def hyper_hidden_size(self) -> int: + return self.hc_count * self.hidden_size + + +__all__ = [ + "GatedResidual", + "GroupedGemmaRMSNorm", + "HyperConnectionConfig", +] diff --git a/vllm/models/qwen4_exp/nvidia/indexer_qsa.py b/vllm/models/qwen4_exp/nvidia/indexer_qsa.py new file mode 100644 index 0000000000..1e6c308718 --- /dev/null +++ b/vllm/models/qwen4_exp/nvidia/indexer_qsa.py @@ -0,0 +1,356 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Qwen4Exp weight-free QSA indexer.""" + +from __future__ import annotations + +from typing import cast + +import torch +from torch import nn + +from vllm.config import VllmConfig +from vllm.forward_context import get_forward_context +from vllm.model_executor.layers.layernorm import GemmaRMSNorm +from vllm.model_executor.layers.linear import ReplicatedLinear +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding.mrope import triton_mrope +from vllm.transformers_utils.configs.qwen4_exp import ( + Qwen4ExpTextConfig, +) + +from ..common.qsa_cache import ( + QSACompressedKeyCache, + QSAForwardMetadata, + QSAKeyStateCache, + canonical_qsa_rope_positions, +) +from .ops.qsa_pre_indexer import qsa_pre_indexer + + +def apply_qsa_rope( + rotary_emb: nn.Module, + positions: torch.Tensor, + tensor: torch.Tensor, +) -> torch.Tensor: + """Apply the main attention's exact 1D/MRoPE composition to QSA heads.""" + + num_tokens, _, head_dim = tensor.shape + rotary_dim = rotary_emb.rotary_dim + cache = rotary_emb._match_cos_sin_cache_dtype(tensor) # noqa: SLF001 + cos_sin = cache[positions] + cos, sin = cos_sin.chunk(2, dim=-1) + if positions.ndim == 2: + shape = tensor.shape + tensor, _ = triton_mrope( + tensor.reshape(num_tokens, -1), + tensor.new_empty((num_tokens, head_dim)), + cos, + sin, + rotary_emb.mrope_section, + head_dim, + rotary_dim, + rotary_emb.mrope_interleaved, + rotary_emb.is_neox_style, + ) + return tensor.reshape(shape) + + rotated = rotary_emb.apply_rotary_emb.forward_cuda( + tensor[..., :rotary_dim], + cos, + sin, + ) + return torch.cat((rotated, tensor[..., rotary_dim:]), dim=-1) + + +def _supports_fused_pre_indexer( + rotary_emb: nn.Module, + head_dim: int, + num_kv_heads: int, + compress_ratio: int, +) -> bool: + rotary_dim = int(rotary_emb.rotary_dim) + mrope_section = getattr(rotary_emb, "mrope_section", None) + return ( + bool(getattr(rotary_emb, "is_neox_style", False)) + and ( + not mrope_section + or ( + len(mrope_section) == 3 + and sum(mrope_section) == rotary_dim // 2 + and bool(getattr(rotary_emb, "mrope_interleaved", False)) + ) + ) + and head_dim == 128 + and rotary_dim == 64 + and num_kv_heads == 1 + and compress_ratio > 1 + and compress_ratio & (compress_ratio - 1) == 0 + ) + + +class QSAIndexer(nn.Module): + """Replicated Q/K projection plus paged, weight-free QSA selection. + + ``prefix`` must be the checkpoint's indexer prefix, normally + ``model.layers.N.self_attn.indexer``. Consequently the trainable names are + ``index_qk_proj``, ``q_layernorm`` and ``k_layernorm`` under that prefix. + """ + + def __init__( + self, + *, + vllm_config: VllmConfig, + config: Qwen4ExpTextConfig, + layer_id: int, + rotary_emb: nn.Module, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + if vllm_config.cache_config is None: + raise ValueError("QSA requires a paged KV cache") + if vllm_config.model_config.dtype != torch.bfloat16: + raise NotImplementedError("Qwen4Exp QSA currently requires BF16") + + self.layer_id = int(layer_id) + self.index_n_heads = int(config.indexer_n_heads) + self.index_kv_heads = int(config.indexer_kv_heads) + self.index_head_dim = int(config.indexer_head_dim) + self.token_topk = int(config.indexer_budget) + self.compress_ratio = int(config.indexer_compress_ratio) + self.rotary_emb = rotary_emb + self.use_fused_pre_indexer = _supports_fused_pre_indexer( + rotary_emb, + self.index_head_dim, + self.index_kv_heads, + self.compress_ratio, + ) + self.prefix = prefix + # MTP step 0 selects the target-aligned rows; later steps reuse them + # while continuing to update the QSA side cache. + self.skip_topk = False + + self.index_qk_proj = ReplicatedLinear( + int(config.hidden_size), + (self.index_n_heads + self.index_kv_heads) * self.index_head_dim, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.index_qk_proj" if prefix else "index_qk_proj", + ) + self.q_layernorm = GemmaRMSNorm( + self.index_head_dim, + eps=float(getattr(config, "rms_norm_eps", 1e-6)), + ) + self.k_layernorm = GemmaRMSNorm( + self.index_head_dim, + eps=float(getattr(config, "rms_norm_eps", 1e-6)), + ) + + cache_config = vllm_config.cache_config + cache_prefix = f"{prefix}." if prefix else "" + self.raw_key_cache = QSAKeyStateCache( + head_size=self.index_head_dim, + dtype=torch.bfloat16, + cache_rope_positions=vllm_config.model_config.uses_mrope, + prefix=f"{cache_prefix}raw_key_cache", + cache_config=cache_config, + compress_ratio=self.compress_ratio, + vllm_config=vllm_config, + ) + self.compressed_key_cache = QSACompressedKeyCache( + head_size=self.index_head_dim, + dtype=torch.bfloat16, + compress_ratio=self.compress_ratio, + prefix=f"{cache_prefix}compressed_key_cache", + cache_config=cache_config, + vllm_config=vllm_config, + ) + + @property + def output_width(self) -> int: + return self.token_topk + self.compress_ratio - 1 + + def _metadata( + self, + ) -> tuple[QSAForwardMetadata, QSAForwardMetadata] | None: + metadata = get_forward_context().attn_metadata + if isinstance(metadata, list): + metadata = metadata[0] + if not isinstance(metadata, dict): + return None + raw = cast(QSAForwardMetadata, metadata[self.raw_key_cache.prefix]) + compressed = cast( + QSAForwardMetadata, metadata[self.compressed_key_cache.prefix] + ) + if raw.num_actual_tokens != compressed.num_actual_tokens: + raise RuntimeError("QSA side-cache metadata token counts disagree") + if not raw.logical_positions.is_cuda and ( + not torch.equal(raw.logical_positions, compressed.logical_positions) + ): + raise RuntimeError("QSA side-cache metadata positions disagree") + return raw, compressed + + def forward( + self, + hidden_states: torch.Tensor, + positions: torch.Tensor, + out: torch.Tensor | None = None, + ) -> torch.Tensor: + """Return fixed-width request-relative token indices padded with ``-1``.""" + + metadata = self._metadata() + if metadata is None: + # Preserve step-0 indices when later MTP steps reuse the buffer. + if self.skip_topk and out is not None: + return out + result = torch.full( + (hidden_states.shape[0], self.output_width), + -1, + dtype=torch.int32, + device=hidden_states.device, + ) + if out is not None: + out.copy_(result) + return out + return result + + from .ops.qsa import ( + qsa_compress_groups_with_ratio, + qsa_select_paged_tokens, + qsa_store_cache_rows, + ) + + raw_metadata, compressed_metadata = metadata + num_tokens = raw_metadata.num_actual_tokens + hidden_states = hidden_states[:num_tokens] + positions = positions[..., :num_tokens] + + # Q/K projection + projected_qk, _ = self.index_qk_proj(hidden_states) + projected_q, raw_keys = projected_qk.split( + ( + self.index_n_heads * self.index_head_dim, + self.index_kv_heads * self.index_head_dim, + ), + dim=-1, + ) + raw_key_state_cache = self.raw_key_cache + compressed_key_cache = self.compressed_key_cache.kv_cache + + if self.use_fused_pre_indexer: + q = projected_q.new_empty( + num_tokens, + self.index_n_heads, + self.index_head_dim, + ) + qsa_pre_indexer( + projected_q, + raw_keys, + positions, + self.rotary_emb.cos_sin_cache, + self.q_layernorm.weight, + self.k_layernorm.weight, + self.q_layernorm.variance_epsilon, + q, + raw_key_state_cache.kv_cache, + raw_metadata.slot_mapping, + raw_metadata.block_table, + raw_metadata.query_start_loc, + raw_metadata.logical_positions, + compressed_key_cache, + compressed_metadata.slot_mapping, + compressed_metadata.k_work_metadata, + compress_ratio=self.compress_ratio, + mrope_section=getattr(self.rotary_emb, "mrope_section", None), + rope_pos_offset=( + raw_key_state_cache.rope_position_offset + if raw_key_state_cache.rope_position_cache is not None + else None + ), + ) + else: + # Unfused reference path + from flashinfer.norm import gemma_rmsnorm + + q = projected_q.reshape(-1, self.index_n_heads, self.index_head_dim) + q = gemma_rmsnorm( + q.reshape(-1, self.index_head_dim), + self.q_layernorm.weight, + self.q_layernorm.variance_epsilon, + ).reshape_as(q) + q = apply_qsa_rope(self.rotary_emb, positions, q) + + raw_key_cache = raw_key_state_cache.key_cache + rope_position_cache = raw_key_state_cache.rope_position_cache + if rope_position_cache is None: + position_rows = raw_metadata.logical_positions.view(-1, 1, 1).expand( + -1, 1, 3 + ) + else: + position_rows = canonical_qsa_rope_positions(positions).to( + device=raw_key_cache.device + ) + pooled, first_positions = qsa_compress_groups_with_ratio( + raw_keys.reshape(-1, 1, self.index_head_dim), + position_rows, + raw_key_cache, + raw_metadata.block_table, + raw_metadata.token_to_req, + raw_metadata.query_start_loc, + raw_metadata.logical_positions, + compressed_metadata.slot_mapping, + self.compress_ratio, + rope_position_cache, + ) + compressed_keys = gemma_rmsnorm( + pooled.reshape(-1, self.index_head_dim), + self.k_layernorm.weight, + self.k_layernorm.variance_epsilon, + ).reshape(-1, 1, self.index_head_dim) + if getattr(self.rotary_emb, "mrope_section", None): + first_positions = first_positions.transpose(0, 1) + else: + first_positions = first_positions[:, 0] + compressed_keys = apply_qsa_rope( + self.rotary_emb, + first_positions, + compressed_keys, + ) + qsa_store_cache_rows( + compressed_key_cache, + compressed_metadata.slot_mapping, + compressed_keys, + ) + qsa_store_cache_rows( + raw_key_cache, + raw_metadata.slot_mapping, + raw_keys, + ) + if rope_position_cache is not None: + qsa_store_cache_rows( + rope_position_cache, + raw_metadata.slot_mapping, + position_rows, + ) + + if self.skip_topk: + if out is None: + raise RuntimeError("QSA top-k reuse requires an output buffer") + return out + + # Score compressed keys, select blocks, then expand them to token indices. + return qsa_select_paged_tokens( + q, + compressed_key_cache, + compressed_metadata.block_table, + compressed_metadata.token_to_req, + compressed_metadata.logical_positions, + compressed_metadata.seq_lens, + self.token_topk, + self.compress_ratio, + out, + ) + + +__all__ = ["QSAIndexer", "apply_qsa_rope"] diff --git a/vllm/models/qwen4_exp/nvidia/low_latency_gemm.py b/vllm/models/qwen4_exp/nvidia/low_latency_gemm.py new file mode 100644 index 0000000000..43b7a76bc5 --- /dev/null +++ b/vllm/models/qwen4_exp/nvidia/low_latency_gemm.py @@ -0,0 +1,185 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Qwen4Exp decode GEMM selection on Blackwell. + +Dispatch follows Kimi-K3 and uses the local ``(N, K)`` shape and token count. +Plans contain measured CUDA graph capture sizes; other token counts use the +standard linear implementation. +""" + +import torch +from torch import nn + +import vllm.envs as envs +from vllm.model_executor.kernels.linear.cute_dsl.skinny_gemm import ( + SkinnyGemmConfig, + shape_dynamic_skinny_gemm, +) +from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, + UnquantizedEmbeddingMethod, +) +from vllm.platforms import current_platform +from vllm.utils.torch_utils import direct_register_custom_op + +QWEN4_EXP_GEMM_PLANS: dict[tuple[int, int], dict[int, SkinnyGemmConfig]] = { + # GDN fused QKVZ projection, TP=4. + (4096, 2560): { + 1: SkinnyGemmConfig(1, 64, 4, k_unroll=4), + 2: SkinnyGemmConfig(2, 64, 4, k_unroll=4), + }, + # GDN and QSA output projections, TP=4. + (2560, 1536): { + 1: SkinnyGemmConfig(1, 128, 2, k_unroll=2, vector_width=4), + 2: SkinnyGemmConfig(2, 128, 2, k_unroll=2, vector_width=4), + 4: SkinnyGemmConfig(4, 64, 2, k_unroll=2), + }, + # GDN fused B/A projection, TP=4. + (24, 2560): { + 1: SkinnyGemmConfig(1, 128, 2, k_unroll=4, vector_width=4), + 2: SkinnyGemmConfig(2, 128, 2, k_unroll=4, vector_width=4), + 4: SkinnyGemmConfig(4, 128, 2, k_unroll=4, vector_width=4), + 8: SkinnyGemmConfig(8, 128, 1, k_unroll=4, vector_width=4), + 16: SkinnyGemmConfig(16, 128, 1, k_unroll=4, vector_width=4), + }, + # QSA fused QKV/gate projection, TP=4. + (3584, 2560): { + 1: SkinnyGemmConfig(1, 128, 4, k_unroll=4, vector_width=4), + 2: SkinnyGemmConfig(2, 64, 2, k_unroll=2), + 4: SkinnyGemmConfig(4, 64, 2, k_unroll=2), + }, + # QSA indexer Q/K projection, replicated in a TP=4 deployment. + (640, 2560): { + 1: SkinnyGemmConfig(1, 128, 1, k_unroll=4, vector_width=4), + 2: SkinnyGemmConfig(2, 128, 1, k_unroll=4, vector_width=4), + 4: SkinnyGemmConfig(4, 128, 1, k_unroll=4, vector_width=4), + 8: SkinnyGemmConfig(8, 128, 1, k_unroll=4, vector_width=4), + }, + # Shared-expert fused gate/up projection, TP=4. + (320, 2560): { + 1: SkinnyGemmConfig(1, 128, 2, k_unroll=4, vector_width=4), + 2: SkinnyGemmConfig(2, 128, 2, k_unroll=4, vector_width=4), + 4: SkinnyGemmConfig(4, 128, 2, k_unroll=4, vector_width=4), + 8: SkinnyGemmConfig(8, 64, 1, k_unroll=4), + 16: SkinnyGemmConfig(16, 128, 2, k_unroll=4, vector_width=4), + }, + # LM head, TP=4. + (62080, 2560): { + 1: SkinnyGemmConfig(1, 64, 4, k_unroll=2), + 2: SkinnyGemmConfig(2, 32, 4, k_unroll=2), + }, + # HC merged down/injection projection, replicated in a TP=4 deployment. + (336, 10240): { + 1: SkinnyGemmConfig(1, 128, 1, static_k=10240), + 2: SkinnyGemmConfig(2, 128, 1, static_k=10240), + 4: SkinnyGemmConfig(4, 128, 2, static_k=10240), + 8: SkinnyGemmConfig(8, 128, 1, k_unroll=4), + }, +} + + +def _is_sm103() -> bool: + return current_platform.is_device_capability((10, 3)) + + +def _is_packed_row_major(tensor: torch.Tensor) -> bool: + return tensor.dim() == 2 and tensor.stride() == (tensor.shape[1], 1) + + +def _runtime_ok(x: torch.Tensor, weight: torch.Tensor) -> bool: + return ( + not envs.VLLM_BATCH_INVARIANT + and _is_packed_row_major(x) + and _is_packed_row_major(weight) + and x.dtype == torch.bfloat16 + and weight.dtype == torch.bfloat16 + and x.is_cuda + and weight.is_cuda + and x.device == weight.device + and x.shape[1] == weight.shape[1] + ) + + +class _Qwen4ExpLowLatencyApply: + def apply( + self, + layer: nn.Module, + x: torch.Tensor, + bias: torch.Tensor | None = None, + ) -> torch.Tensor: + if bias is None and not envs.VLLM_BATCH_INVARIANT: + return torch.ops.vllm.qwen4_exp_low_latency_gemm(x, layer.weight) + return super().apply(layer, x, bias) # type: ignore[misc] + + +class Qwen4ExpLowLatencyLinearMethod(_Qwen4ExpLowLatencyApply, UnquantizedLinearMethod): + pass + + +class Qwen4ExpLowLatencyEmbeddingMethod( + _Qwen4ExpLowLatencyApply, UnquantizedEmbeddingMethod +): + pass + + +def _qwen4_exp_low_latency_gemm(x: torch.Tensor, weight: torch.Tensor) -> torch.Tensor: + plan = QWEN4_EXP_GEMM_PLANS.get((weight.shape[0], weight.shape[1])) + config = None if plan is None else plan.get(x.shape[0]) + if ( + config is not None + and _runtime_ok(x, weight) + and shape_dynamic_skinny_gemm.is_available() + ): + return shape_dynamic_skinny_gemm(x, weight, config) + return torch.nn.functional.linear(x, weight) + + +def _qwen4_exp_low_latency_gemm_fake( + x: torch.Tensor, weight: torch.Tensor +) -> torch.Tensor: + return x.new_empty((*x.shape[:-1], weight.shape[0])) + + +direct_register_custom_op( + op_name="qwen4_exp_low_latency_gemm", + op_func=_qwen4_exp_low_latency_gemm, + fake_impl=_qwen4_exp_low_latency_gemm_fake, +) + + +def enable_qwen4_exp_low_latency_gemm( + module: nn.Module, + dtype: torch.dtype, +) -> None: + if dtype != torch.bfloat16 or not _is_sm103(): + return + if not shape_dynamic_skinny_gemm.is_available(): + return + + warmup_configs: set[SkinnyGemmConfig] = set() + for child in module.modules(): + is_linear = ( + isinstance(child, LinearBase) + and type(child.quant_method) is UnquantizedLinearMethod + ) + is_head = ( + isinstance(child, ParallelLMHead) + and type(child.quant_method) is UnquantizedEmbeddingMethod + ) + if not (is_linear or is_head): + continue + weight = getattr(child, "weight", None) + if weight is None or weight.dim() != 2: + continue + plan = QWEN4_EXP_GEMM_PLANS.get((weight.shape[0], weight.shape[1])) + if plan is None: + continue + if is_linear: + child.quant_method = Qwen4ExpLowLatencyLinearMethod() + else: + child.quant_method = Qwen4ExpLowLatencyEmbeddingMethod() + warmup_configs.update(plan.values()) + + if warmup_configs: + shape_dynamic_skinny_gemm.request_warmup_configs(dtype, warmup_configs) diff --git a/vllm/models/qwen4_exp/nvidia/model.py b/vllm/models/qwen4_exp/nvidia/model.py new file mode 100644 index 0000000000..aeb3a14dad --- /dev/null +++ b/vllm/models/qwen4_exp/nvidia/model.py @@ -0,0 +1,1065 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inference-only Qwen4Exp model.""" + +from collections.abc import Iterable +from itertools import islice + +import torch +from torch import nn + +from vllm.compilation.decorators import support_torch_compile +from vllm.config import VllmConfig +from vllm.distributed import get_pp_group +from vllm.model_executor.layers.fused_moe.utils import ( + is_model_fused_shared_expert_compatible, +) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import ( + QwenGatedDeltaNetAttention, +) +from vllm.model_executor.layers.mamba.mamba_utils import ( + MambaStateCopyFunc, + MambaStateCopyFuncCalculator, + MambaStateCopyFuncsByType, + MambaStateDtypeCalculator, + MambaStateShapeCalculator, +) +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, + VocabParallelEmbedding, +) +from vllm.model_executor.models.interfaces import ( + HasInnerState, + IsHybrid, + MixtureOfExperts, + MultiModalEmbeddings, + SupportsLoRA, + SupportsMRoPE, + SupportsPP, + _require_is_multimodal, +) +from vllm.model_executor.models.qwen3_5 import ( + Qwen3_5ForConditionalGeneration, + Qwen3_5Model, +) +from vllm.model_executor.models.qwen3_next import ( + Qwen3NextAttention, + Qwen3NextMLP, + Qwen3NextSparseMoeBlock, +) +from vllm.model_executor.models.qwen3_vl import ( + Qwen3_VisionTransformer, + Qwen3VLDummyInputsBuilder, + Qwen3VLMultiModalProcessor, + Qwen3VLProcessingInfo, +) +from vllm.model_executor.models.utils import ( + AutoWeightsLoader, + StageMissingLayer, + WeightsMapper, + _merge_multimodal_embeddings, + extract_layer_index, + make_empty_intermediate_tensors_factory, + make_layers, + maybe_fuse_shared_experts, + maybe_prefix, +) +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.inputs import MultiModalFeatureSpec +from vllm.sequence import IntermediateTensors +from vllm.tokenizers.registry import cached_tokenizer_from_config +from vllm.transformers_utils.configs.qwen4_exp import ( + Qwen4ExpTextConfig, +) +from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum +from vllm.v1.kv_cache_interface import MambaSpec + +from ..config import Qwen4ExpConfig +from .hyperconnection import GatedResidual, HyperConnectionConfig +from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm +from .ple_layer import Qwen4ExpPLELayer +from .qsa import Qwen4ExpQSAAttention + + +def without_modelopt_fp4( + quant_config: QuantizationConfig | None, +) -> QuantizationConfig | None: + """Return ``None`` for weights excluded from Qwen4Exp ModelOpt-FP4.""" + + if quant_config is not None and quant_config.get_name() == "modelopt_fp4": + return None + return quant_config + + +def _remap_qsa_cache_scale_name( + name: str, + qsa_layer_ids: frozenset[int], +) -> str: + """Map serialized main-cache scales onto the merged QSA owner. + + Regular attention keeps cache scales below its ``attn`` child. QSA owns + that cache directly, so only QSA layers need the final path component + moved to the owner's persistent ``_k_scale``/``_v_scale`` buffers. + """ + + scale_suffixes = { + "k_proj.k_scale": "_k_scale", + "k_proj.output_scale": "_k_scale", + "attn.k_scale": "_k_scale", + "attn._k_scale": "_k_scale", + "k_scale": "_k_scale", + "_k_scale": "_k_scale", + "v_proj.v_scale": "_v_scale", + "v_proj.output_scale": "_v_scale", + "attn.v_scale": "_v_scale", + "attn._v_scale": "_v_scale", + "v_scale": "_v_scale", + "_v_scale": "_v_scale", + } + for layer_id in qsa_layer_ids: + marker = f"layers.{layer_id}.self_attn." + marker_start = name.find(marker) + if marker_start < 0 or (marker_start > 0 and name[marker_start - 1] != "."): + continue + suffix = name[marker_start + len(marker) :] + mapped_suffix = scale_suffixes.get(suffix) + if mapped_suffix is not None: + return f"{name[: marker_start + len(marker)]}{mapped_suffix}" + return name + + +_QWEN4_EXP_IGNORED_MISSING_SUFFIXES = [ + ".bias", + "_bias", + ".k_scale", + "_k_scale", + ".v_scale", + "_v_scale", + "_weight_scale", + "_input_scale", +] + +# The checkpoint keeps down and injection projections separate; runtime packs +# them into adjacent logical shards of one MergedColumnParallelLinear. +_HC_WEIGHTS_MAPPER = WeightsMapper( + orig_to_new_stacked={ + "hyper_connection.input_mix_weight_down.weight": ( + "hyper_connection.input_mix_weight_down_block_inject.weight", + 0, + ), + "hyper_connection.block_inject_weight.weight": ( + "hyper_connection.input_mix_weight_down_block_inject.weight", + 1, + ), + } +) + + +class Qwen4ExpSparseMoeBlock(Qwen3NextSparseMoeBlock): + """Qwen3Next MoE with Qwen4Exp HC validation.""" + + def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None: + parallel_config = vllm_config.parallel_config + if parallel_config.use_sequence_parallel_moe: + raise NotImplementedError( + "Qwen4Exp HC does not support sequence-parallel MoE" + ) + super().__init__(vllm_config=vllm_config, prefix=prefix) + config = vllm_config.model_config.hf_text_config + self.n_shared_experts = int(config.shared_expert_intermediate_size > 0) + + +class Qwen4ExpDecoderLayer(nn.Module): + def __init__( + self, + vllm_config: VllmConfig, + layer_type: str, + prefix: str = "", + ) -> None: + super().__init__() + config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config + model_config = vllm_config.model_config + cache_config = vllm_config.cache_config + quant_config = vllm_config.quant_config + + self.config = config + self.layer_type = layer_type + self.layer_idx = extract_layer_index(prefix) + if vllm_config.parallel_config.use_sequence_parallel_moe: + raise NotImplementedError( + "Qwen4Exp HC does not support sequence-parallel MoE" + ) + self.ple: Qwen4ExpPLELayer | None = None + ple_layer_ids = config.ple_layer_ids + if (self.layer_idx + 1) in ple_layer_ids: + ple_layer_ids_sorted = sorted(set(ple_layer_ids)) + ple_dense_layer_id_map = { + abs_id: idx for idx, abs_id in enumerate(ple_layer_ids_sorted) + } + ple_dense_layer_id = ple_dense_layer_id_map[self.layer_idx + 1] + self.ple = Qwen4ExpPLELayer( + config, + vllm_config=vllm_config, + layer_idx=self.layer_idx, + ple_dense_layer_id=ple_dense_layer_id, + prefix=f"{prefix}.ple", + ) + + if layer_type == "linear_attention": + self.linear_attn = QwenGatedDeltaNetAttention( + config, + vllm_config=vllm_config, + prefix=f"{prefix}.linear_attn", + gqa_interleaved_layout=False, + ) + elif layer_type == "full_attention": + use_qsa = getattr(config, "indexer_n_heads", None) is not None + if not use_qsa: + self.self_attn = Qwen3NextAttention( + config, + model_config=model_config, + cache_config=cache_config, + quant_config=quant_config, + prefix=f"{prefix}.self_attn", + ) + else: + self.self_attn = Qwen4ExpQSAAttention( + vllm_config=vllm_config, + config=config, + layer_id=self.layer_idx, + quant_config=quant_config, + prefix=f"{prefix}.self_attn", + ) + else: + raise ValueError(f"Invalid layer_type {layer_type}") + + mlp_only_layers = getattr(config, "mlp_only_layers", []) + num_experts = getattr(config, "num_experts", 0) or 0 + absolute_layer_id = self.layer_idx + 1 + is_moe_layer = self.layer_idx not in mlp_only_layers and ( + num_experts > 0 and absolute_layer_id % config.decoder_sparse_step == 0 + ) + if is_moe_layer: + self.mlp = Qwen4ExpSparseMoeBlock( + vllm_config=vllm_config, prefix=f"{prefix}.mlp" + ) + else: + self.mlp = Qwen3NextMLP( + hidden_size=config.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + prefix=f"{prefix}.mlp", + ) + + hc_config = HyperConnectionConfig( + hc_count=config.hc_count, + hidden_size=config.hidden_size, + params_dtype=torch.bfloat16, + hc_lowrank=config.hc_lowrank, + rms_norm_eps=config.rms_norm_eps, + hc_per_branch_norm=True, + ) + self.attn_hyper_connection = GatedResidual( + hc_config, + prefix=maybe_prefix(prefix, "attn_hyper_connection"), + ) + self.mlp_hyper_connection = GatedResidual( + hc_config, + prefix=maybe_prefix(prefix, "mlp_hyper_connection"), + ) + + def forward( + self, + hidden_states: torch.Tensor, + prev_block_output: torch.Tensor | None, + prev_injection: torch.Tensor | None, + positions: torch.Tensor, + *, + input_ids: torch.Tensor | None, + query_start_loc: torch.Tensor | None, + ngram_context: torch.Tensor | None, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + attn_hc = self.attn_hyper_connection + if self.ple is not None: + # PLE adds directly to the multi-stream state, so pending HC state + # must be materialized before the addition. + if prev_block_output is not None and prev_injection is not None: + hidden_states = attn_hc.combine( + hidden_states, prev_block_output, prev_injection + ) + prev_block_output = prev_injection = None + + if input_ids is None or query_start_loc is None or ngram_context is None: + raise RuntimeError("PLE inputs were not prepared") + hidden_states = hidden_states + self.ple( + hidden_states, + input_ids, + query_start_loc, + ngram_context, + ) + + # Fuse a pending combine with this HC module's mix when possible. + if prev_block_output is not None and prev_injection is not None: + hidden_states, block_input, injection = attn_hc.combine_and_mix( + hidden_states, prev_block_output, prev_injection + ) + else: + hidden_states, block_input, injection = attn_hc.mix(hidden_states) + + if self.layer_type == "linear_attention": + attn_out = self.linear_attn(hidden_states=block_input) + elif self.layer_type == "full_attention": + attn_out = self.self_attn( + hidden_states=block_input, + positions=positions, + ) + else: + raise ValueError("Invalid layer_type") + + mlp_hc = self.mlp_hyper_connection + hidden_states, block_input, injection = mlp_hc.combine_and_mix( + hidden_states, attn_out, injection + ) + mlp_out = self.mlp(block_input) + return hidden_states, mlp_out, injection + + +class Qwen4ExpMixtureOfExperts(MixtureOfExperts): + """Expose Qwen4Exp routed experts through vLLM's EPLB protocol.""" + + def set_moe_parameters(self, layers: Iterable[nn.Module]) -> None: + self.moe_layers = [] + self.moe_mlp_layers = [] + example_moe = None + for layer in layers: + if isinstance(layer, Qwen4ExpDecoderLayer) and isinstance( + layer.mlp, Qwen4ExpSparseMoeBlock + ): + example_moe = layer.mlp + self.moe_mlp_layers.append(layer.mlp) + self.moe_layers.append(layer.mlp.experts) + + self.num_moe_layers = len(self.moe_layers) + if example_moe is None: + self.num_expert_groups = 0 + self.num_shared_experts = 0 + self.num_logical_experts = 0 + self.num_physical_experts = 0 + self.num_local_physical_experts = 0 + self.num_routed_experts = 0 + self.num_redundant_experts = 0 + return + + self.num_expert_groups = 1 + self.num_shared_experts = example_moe.n_shared_experts + self.num_logical_experts = example_moe.n_logical_experts + self.num_physical_experts = example_moe.n_physical_experts + self.num_local_physical_experts = example_moe.n_local_physical_experts + self.num_routed_experts = example_moe.n_routed_experts + self.num_redundant_experts = example_moe.n_redundant_experts + + def update_physical_experts_metadata( + self, + num_physical_experts: int, + num_local_physical_experts: int, + ) -> None: + assert self.num_local_physical_experts == num_local_physical_experts + self.num_physical_experts = num_physical_experts + self.num_local_physical_experts = num_local_physical_experts + self.num_redundant_experts = num_physical_experts - self.num_logical_experts + for moe in self.moe_mlp_layers: + moe.n_physical_experts = num_physical_experts + moe.n_local_physical_experts = num_local_physical_experts + moe.n_redundant_experts = self.num_redundant_experts + moe.experts.update_expert_map() + + +@support_torch_compile( + dynamic_arg_dims={ + "input_ids": 0, + "positions": -1, + "intermediate_tensors": 0, + "inputs_embeds": 0, + "query_start_loc": 0, + "ngram_context": 0, + "deepstack_input_embeds": 0, + } +) +class Qwen4ExpModel(nn.Module): + hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _HC_WEIGHTS_MAPPER + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + super().__init__() + config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config + self.config = config + self.num_redundant_experts = ( + vllm_config.parallel_config.eplb_config.num_redundant_experts + ) + self.vocab_size = config.vocab_size + self._qsa_layer_ids = frozenset( + layer_idx + for layer_idx, layer_type in enumerate(config.layer_types) + if layer_type == "full_attention" + and getattr(config, "indexer_n_heads", None) is not None + ) + self.embed_tokens = VocabParallelEmbedding(self.vocab_size, config.hidden_size) + + def get_layer(prefix: str) -> Qwen4ExpDecoderLayer: + layer_idx = extract_layer_index(prefix) + return Qwen4ExpDecoderLayer( + vllm_config, + layer_type=config.layer_types[layer_idx], + prefix=prefix, + ) + + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, get_layer, prefix=f"{prefix}.layers" + ) + self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible( + self.layers, + Qwen4ExpSparseMoeBlock, + "mlp", + ) + intermediate_size = config.hidden_size * config.hc_count + self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory( + ["hidden_states"], intermediate_size + ) + + self.hyper_connection_mixer: GatedResidual | None + if get_pp_group().is_last_rank: + hc_config = HyperConnectionConfig( + hc_count=config.hc_count, + hidden_size=config.hidden_size, + params_dtype=torch.bfloat16, + hc_lowrank=config.hc_lowrank, + rms_norm_eps=config.rms_norm_eps, + hc_per_branch_norm=True, + ) + self.hyper_connection_mixer = GatedResidual( + hc_config, + use_combine=False, + prefix=maybe_prefix(prefix, "hyper_connection_mixer"), + ) + else: + self.hyper_connection_mixer = None + + spec_config = vllm_config.speculative_config + # MTP HC multi-stream outputs: when speculative method=="mtp" and the + # model uses HC with hc_count>1, retain the pre-final-mixer multi-stream + # hidden state [T, hc_count*H] so the MTP drafter can feed a real + # multi-stream backbone hidden on its first step (scheme A). Derived + # purely from config (NOT node identity) so P/D nodes stay consistent. + needs_mtp_hidden = ( + spec_config is not None + and getattr(spec_config, "method", None) == "mtp" + and get_pp_group().is_last_rank + ) + if needs_mtp_hidden: + self._mtp_hidden_buffer = torch.empty( + vllm_config.scheduler_config.max_num_batched_tokens, + config.hc_count * config.hidden_size, + dtype=vllm_config.model_config.dtype, + ) + else: + self._mtp_hidden_buffer = None + + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.embed_tokens(input_ids) + + def forward( + self, + input_ids: torch.Tensor | None, + positions: torch.Tensor, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + query_start_loc: torch.Tensor | None = None, + ngram_context: torch.Tensor | None = None, + deepstack_input_embeds: IntermediateTensors | None = None, + ) -> torch.Tensor | IntermediateTensors: + if get_pp_group().is_first_rank: + if inputs_embeds is not None: + hidden_states = inputs_embeds + else: + if input_ids is None: + raise ValueError("input_ids or inputs_embeds is required") + hidden_states = self.embed_input_ids(input_ids) + hidden_states = hidden_states.repeat(1, self.config.hc_count) + else: + if intermediate_tensors is None: + raise ValueError("pipeline stage requires intermediate tensors") + hidden_states = intermediate_tensors["hidden_states"] + + block_output = None + injection = None + last_layer = None + for layer_idx, layer in islice( + enumerate(self.layers), self.start_layer, self.end_layer + ): + last_layer = layer + hidden_states, block_output, injection = layer( + hidden_states=hidden_states, + prev_block_output=block_output, + prev_injection=injection, + positions=positions, + input_ids=input_ids, + query_start_loc=query_start_loc, + ngram_context=ngram_context, + ) + if deepstack_input_embeds is not None and layer_idx < len( + deepstack_input_embeds + ): + deepstack_embed = deepstack_input_embeds[ + f"deepstack_input_embeds_{layer_idx}" + ] + deepstack_embed = ( + deepstack_embed.unsqueeze(-2) + .expand( + *deepstack_embed.shape[:-1], + self.config.hc_count, + self.config.hidden_size, + ) + .flatten(-2) + ) + # Deepstack is an external addition to the materialized + # multi-stream state and therefore terminates delayed combine. + hidden_states = layer.mlp_hyper_connection.combine( + hidden_states, block_output, injection + ) + block_output = None + injection = None + hidden_states = hidden_states + deepstack_embed + + if not get_pp_group().is_last_rank: + # PP transports one tensor, not the delayed HC tuple. Materialize + # with the HC module that produced the pending injection. + if last_layer is not None and block_output is not None: + hidden_states = last_layer.mlp_hyper_connection.combine( + hidden_states, block_output, injection + ) + return IntermediateTensors({"hidden_states": hidden_states}) + + # The final mixer consumes the last pending combine and returns both + # the sampled single stream and the materialized multi-stream state. + final_mixer = self.hyper_connection_mixer + assert final_mixer is not None + multi_hidden, sample_hidden_states, _ = final_mixer.combine_and_mix( + hidden_states, block_output, injection + ) + if self._mtp_hidden_buffer is not None: + # Capture the pre-final-mixer multi-stream hidden state + # [T, hc_count*H] for the MTP drafter (zero extra compute: + # this tensor is needed by the final mixer regardless). + num_tokens = multi_hidden.shape[0] + self._mtp_hidden_buffer[:num_tokens].copy_(multi_hidden) + return sample_hidden_states + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + weights = ( + ( + _remap_qsa_cache_scale_name(name, self._qsa_layer_ids), + weight, + ) + for name, weight in weights + ) + weights = maybe_fuse_shared_experts( + weights, + enabled=self.is_fused_shared_expert_enabled, + n_routed_experts=getattr(self.config, "num_experts", 0) or 0, + n_shared_experts=1, + ckpt_prefix="mlp.shared_expert", + ) + # Non-persistent PLE state rebuilt in __init__; skip any ckpt + # column for them. + skip_substrs = ( + "hashstats_", + "token_lookup", + "hyper_connection_mixer.block_inject_weight", + ) + mapper = self.hf_to_vllm_mapper | WeightsMapper( + orig_to_new_substr={substr: None for substr in skip_substrs} + ) + # The final HC mixer only exists on the last PP rank; earlier ranks + # must drop its checkpoint weights instead of failing to place them. + ignore_prefixes = ( + None + if self.hyper_connection_mixer is not None + else ["hyper_connection_mixer."] + ) + loader = AutoWeightsLoader( + self, + ignore_unexpected_prefixes=ignore_prefixes, + ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(), + ) + loaded = loader.load_weights( + weights, + mapper=mapper, + ) + return loaded + + +class Qwen4ExpForCausalLM( + nn.Module, + HasInnerState, + SupportsLoRA, + SupportsMRoPE, + SupportsPP, + Qwen4ExpMixtureOfExperts, + IsHybrid, +): + packed_modules_mapping = { + "qkv_proj": ["q_proj", "k_proj", "v_proj"], + "gate_up_proj": ["gate_proj", "up_proj"], + "in_proj_qkvz": ["in_proj_qkv", "in_proj_z"], + "in_proj_ba": ["in_proj_b", "in_proj_a"], + "input_mix_weight_down_block_inject": [ + "input_mix_weight_down", + "block_inject_weight", + "_input_mix_padding", + ], + } + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_prefix={"model.language_model.": "model."} + ) + requires_raw_input_tokens = True + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + super().__init__() + config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config + self.vllm_config = vllm_config + self.model_config = vllm_config.model_config + self.quant_config = vllm_config.quant_config + self.config = config + self.scheduler_config = vllm_config.scheduler_config + if vllm_config.cache_config.mamba_cache_mode == "all": + raise NotImplementedError( + "Qwen4Exp currently does not support 'all' prefix caching, " + "please use '--mamba-cache-mode=align' instead" + ) + self.model = Qwen4ExpModel( + vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") + ) + self.lm_head = ParallelLMHead( + config.vocab_size, + config.hidden_size, + prefix=maybe_prefix(prefix, "lm_head"), + ) + self.logits_processor = LogitsProcessor(config.vocab_size) + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors + ) + self.set_moe_parameters(self.model.layers) + enable_qwen4_exp_low_latency_gemm(self, self.model_config.dtype) + + @staticmethod + def get_model_state_cls(): + from .model_state import Qwen4ExpModelState + + return Qwen4ExpModelState + + def forward( + self, + input_ids: torch.Tensor | None, + positions: torch.Tensor, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + **kwargs: object, + ) -> torch.Tensor | IntermediateTensors: + # Forward kwargs unchanged so the runner's _maybe_add_ngram_kwargs + # path (query_start_loc / ngram_context) reaches Qwen4ExpModel. + return self.model( + input_ids, + positions, + intermediate_tensors, + inputs_embeds, + **kwargs, + ) + + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.model.embed_input_ids(input_ids) + + @classmethod + def get_ple_mamba_state_dtype_from_config( + cls, + vllm_config: VllmConfig, + ) -> tuple[torch.dtype, ...]: + return MambaStateDtypeCalculator.short_conv_state_dtype( + vllm_config.model_config.dtype, + vllm_config.cache_config.mamba_cache_dtype, + ) + + @classmethod + def get_ple_mamba_state_shape_from_config( + cls, vllm_config: VllmConfig + ) -> tuple[tuple[int, int]]: + hf_config = vllm_config.model_config.hf_text_config + conv_kernel_size = hf_config.ple_conv_kernel_size + short_conv_dilation = hf_config.ngram_size + conv_state_len = (conv_kernel_size - 1) * short_conv_dilation + num_spec = ( + vllm_config.speculative_config.num_speculative_tokens + if vllm_config.speculative_config + else 0 + ) + hc_count = hf_config.hc_count + hc_hidden_size = hf_config.hidden_size * hc_count + return MambaStateShapeCalculator.short_conv_state_shape( + tp_world_size=1, + intermediate_size=hc_hidden_size, + conv_kernel=conv_state_len + 1, + num_spec=num_spec, + ) + + @classmethod + def get_gdn_mamba_state_dtype_from_config( + cls, vllm_config: VllmConfig + ) -> tuple[torch.dtype, torch.dtype]: + return MambaStateDtypeCalculator.gated_delta_net_state_dtype( + vllm_config.model_config.dtype, + vllm_config.cache_config.mamba_cache_dtype, + vllm_config.cache_config.mamba_ssm_cache_dtype, + ) + + @classmethod + def get_gdn_mamba_state_shape_from_config( + cls, vllm_config: VllmConfig + ) -> tuple[tuple[int, int], tuple[int, int]]: + parallel_config = vllm_config.parallel_config + hf_config = vllm_config.model_config.hf_text_config + tp_size = parallel_config.tensor_parallel_size + num_spec = ( + vllm_config.speculative_config.num_speculative_tokens + if vllm_config.speculative_config + else 0 + ) + return MambaStateShapeCalculator.gated_delta_net_state_shape( + tp_size, + hf_config.linear_num_key_heads, + hf_config.linear_num_value_heads, + hf_config.linear_key_head_dim, + hf_config.linear_value_head_dim, + hf_config.linear_conv_kernel_dim, + num_spec, + ) + + @classmethod + def get_mamba_state_dtype_from_config( + cls, + vllm_config: VllmConfig, + ) -> tuple[torch.dtype, torch.dtype]: + return cls.get_gdn_mamba_state_dtype_from_config(vllm_config) + + @classmethod + def get_mamba_state_shape_from_config( + cls, vllm_config: VllmConfig + ) -> tuple[tuple[int, int], tuple[int, int]]: + return cls.get_gdn_mamba_state_shape_from_config(vllm_config) + + @classmethod + def get_mamba_state_copy_func( + cls, + ) -> tuple[MambaStateCopyFunc, MambaStateCopyFunc]: + return MambaStateCopyFuncCalculator.gated_delta_net_state_copy_func() + + @classmethod + def get_mamba_state_copy_funcs( + cls, + mamba_types: set[MambaAttentionBackendEnum], + ) -> MambaStateCopyFuncsByType: + copy_funcs_by_type = { + MambaAttentionBackendEnum.GDN_ATTN: cls.get_mamba_state_copy_func(), + MambaAttentionBackendEnum.SHORT_CONV: ( + MambaStateCopyFuncCalculator.short_conv_state_copy_func() + ), + } + missing_types = mamba_types - copy_funcs_by_type.keys() + assert not missing_types, f"missing state copy funcs for {missing_types}" + return { + mamba_type: copy_funcs_by_type[mamba_type] for mamba_type in mamba_types + } + + @classmethod + def get_mamba_specs_from_config( + cls, vllm_config: VllmConfig + ) -> tuple[MambaSpec, ...]: + """Return all MambaSpecs for this model (GDN layers + PLE layer). + + The PLE layer uses a separate short_conv MambaSpec whose page_size_bytes + may exceed the GDN spec; callers should take the maximum. + """ + return ( + MambaSpec( + shapes=cls.get_gdn_mamba_state_shape_from_config(vllm_config), + dtypes=cls.get_gdn_mamba_state_dtype_from_config(vllm_config), + block_size=-1, + ), + MambaSpec( + shapes=cls.get_ple_mamba_state_shape_from_config(vllm_config), + dtypes=cls.get_ple_mamba_state_dtype_from_config(vllm_config), + block_size=-1, + tp_replicated=True, + ), + ) + + def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None: + return self.logits_processor(self.lm_head, hidden_states) + + def get_mtp_target_hidden_states(self) -> torch.Tensor | None: + return self.model._mtp_hidden_buffer + + def get_mrope_input_positions( + self, + input_tokens: list[int], + mm_features: list[MultiModalFeatureSpec], + ) -> tuple[torch.Tensor, int]: + positions = torch.arange(len(input_tokens), dtype=torch.long) + return positions.unsqueeze(0).expand(3, -1), 0 + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + mapper = self.hf_to_vllm_mapper | WeightsMapper( + orig_to_new_substr={"mtp.": None} + ) + loader = AutoWeightsLoader( + self, + ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(), + ) + return loader.load_weights(weights, mapper=mapper) + + +class Qwen4ExpProcessingInfo(Qwen3VLProcessingInfo): + def get_hf_config(self) -> Qwen4ExpConfig: + return self.ctx.get_hf_config(Qwen4ExpConfig) + + +@MULTIMODAL_REGISTRY.register_processor( + Qwen3VLMultiModalProcessor, + info=Qwen4ExpProcessingInfo, + dummy_inputs=Qwen3VLDummyInputsBuilder, +) +class Qwen4ExpForConditionalGeneration( + Qwen3_5ForConditionalGeneration, + HasInnerState, + Qwen4ExpMixtureOfExperts, +): + """Qwen3-VL vision tower backed by the Qwen4Exp language model.""" + + requires_raw_input_tokens = True + + packed_modules_mapping = Qwen3_5ForConditionalGeneration.packed_modules_mapping | { + "input_mix_weight_down_block_inject": [ + "input_mix_weight_down", + "block_inject_weight", + "_input_mix_padding", + ] + } + + @staticmethod + def get_model_state_cls(): + from .model_state import Qwen4ExpModelState + + return Qwen4ExpModelState + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "model") -> None: + nn.Module.__init__(self) + config: Qwen4ExpConfig = vllm_config.model_config.hf_config + quant_config = vllm_config.quant_config + multimodal_config = vllm_config.model_config.multimodal_config + if multimodal_config is None: + raise ValueError( + "Qwen4ExpForConditionalGeneration requires multimodal_config" + ) + + self.config = config + self.model_config = vllm_config.model_config + self.multimodal_config = multimodal_config + self.language_model_only = multimodal_config.language_model_only + if self.language_model_only: + self.use_data_parallel = False + self.is_multimodal_pruning_enabled = False + self.video_pruning_method = None + self.video_pruning_rate = 0.0 + self._tokenizer = None + self.visual = StageMissingLayer("vision_tower") + self._tower_model_names = [] + else: + self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data" + self._init_video_pruning(multimodal_config) + self._tokenizer = cached_tokenizer_from_config(vllm_config.model_config) + + with self._mark_tower_model(vllm_config, {"image", "video"}): + self.visual = Qwen3_VisionTransformer( + config.vision_config, + norm_eps=config.text_config.rms_norm_eps, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "visual"), + ) + + self.use_deepstack = ( + not self.language_model_only + and bool(config.vision_config.deepstack_visual_indexes) + and not isinstance(self.visual, StageMissingLayer) + ) + self.deepstack_num_level = ( + len(config.vision_config.deepstack_visual_indexes) + if self.use_deepstack + else 0 + ) + self.visual_dim = config.vision_config.out_hidden_size + self.multiscale_dim = self.visual_dim * self.deepstack_num_level + + if self.use_deepstack: + self.deepstack_input_embeds = [ + torch.zeros( + vllm_config.scheduler_config.max_num_batched_tokens, + config.text_config.hidden_size, + ) + for _ in range(self.deepstack_num_level) + ] + self.deepstack_input_embeds_num_tokens = 0 + + with self._mark_language_model(vllm_config): + self.language_model = Qwen4ExpForCausalLM( + vllm_config=vllm_config, + prefix=maybe_prefix(prefix, "language_model"), + ) + + self.make_empty_intermediate_tensors = ( + self.language_model.make_empty_intermediate_tensors + ) + if not get_pp_group().is_first_rank and self.use_deepstack: + assert self.language_model.model.start_layer >= len( + config.vision_config.deepstack_visual_indexes + ), ( + "start_layer should be greater than or equal to " + "len(deepstack_visual_indexes)" + ) + self.set_moe_parameters(self.language_model.model.layers) + + def embed_input_ids( + self, + input_ids: torch.Tensor, + multimodal_embeddings: MultiModalEmbeddings | None = None, + *, + is_multimodal: torch.Tensor | None = None, + ) -> torch.Tensor: + inputs_embeds = self._embed_text_input_ids( + input_ids, + self.language_model.embed_input_ids, + is_multimodal=is_multimodal, + ) + if multimodal_embeddings is None or len(multimodal_embeddings) == 0: + return inputs_embeds + if self.language_model_only: + raise ValueError( + "Qwen4Exp language_model_only does not accept multimodal embeddings" + ) + + is_multimodal = _require_is_multimodal(is_multimodal) + if self.use_deepstack: + deepstack_input_embeds, multimodal_embeddings = ( + self._compute_deepstack_embeds( + inputs_embeds=inputs_embeds, + multimodal_embeddings=multimodal_embeddings, + is_multimodal=is_multimodal, + ) + ) + else: + deepstack_input_embeds = None + + inputs_embeds = _merge_multimodal_embeddings( + inputs_embeds=inputs_embeds, + multimodal_embeddings=multimodal_embeddings, + is_multimodal=is_multimodal, + ) + if deepstack_input_embeds is not None: + self._set_deepstack_input_embeds(deepstack_input_embeds) + return inputs_embeds + + def get_mtp_target_hidden_states(self) -> torch.Tensor | None: + return self.language_model.get_mtp_target_hidden_states() + + def forward( + self, + input_ids: torch.Tensor | None, + positions: torch.Tensor, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + **kwargs: object, + ) -> torch.Tensor | IntermediateTensors: + if intermediate_tensors is not None: + inputs_embeds = None + if inputs_embeds is not None and get_pp_group().is_first_rank: + deepstack_input_embeds = self._get_deepstack_input_embeds( + inputs_embeds.size(0) + ) + else: + deepstack_input_embeds = None + + hidden_states = self.language_model.model( + input_ids=input_ids, + positions=positions, + intermediate_tensors=intermediate_tensors, + inputs_embeds=inputs_embeds, + query_start_loc=kwargs.get("query_start_loc"), + ngram_context=kwargs.get("ngram_context"), + deepstack_input_embeds=deepstack_input_embeds, + ) + if inputs_embeds is not None and get_pp_group().is_first_rank: + self._clear_deepstack_input_embeds(inputs_embeds.size(0)) + return hidden_states + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + mapper = self.hf_to_vllm_mapper | WeightsMapper( + orig_to_new_substr={"mtp.": None}, + orig_to_new_prefix={"visual.": None} if self.language_model_only else {}, + ) + loader = AutoWeightsLoader( + self, + ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(), + ) + return loader.load_weights(weights, mapper=mapper) + + @classmethod + def get_mamba_state_dtype_from_config( + cls, + vllm_config: VllmConfig, + ) -> tuple[torch.dtype, torch.dtype]: + return Qwen4ExpForCausalLM.get_mamba_state_dtype_from_config(vllm_config) + + @classmethod + def get_mamba_state_shape_from_config( + cls, + vllm_config: VllmConfig, + ) -> tuple[tuple[int, int], tuple[int, int]]: + return Qwen4ExpForCausalLM.get_mamba_state_shape_from_config(vllm_config) + + @classmethod + def get_mamba_state_copy_func( + cls, + ) -> tuple[MambaStateCopyFunc, MambaStateCopyFunc]: + return Qwen4ExpForCausalLM.get_mamba_state_copy_func() + + @classmethod + def get_mamba_state_copy_funcs( + cls, + mamba_types: set[MambaAttentionBackendEnum], + ) -> MambaStateCopyFuncsByType: + return Qwen4ExpForCausalLM.get_mamba_state_copy_funcs(mamba_types) + + @classmethod + def get_mamba_specs_from_config( + cls, vllm_config: VllmConfig + ) -> tuple[MambaSpec, ...]: + return Qwen4ExpForCausalLM.get_mamba_specs_from_config(vllm_config) + + +__all__ = [ + "Qwen4ExpDecoderLayer", + "Qwen4ExpForCausalLM", + "Qwen4ExpForConditionalGeneration", + "Qwen4ExpMixtureOfExperts", + "Qwen4ExpModel", + "Qwen4ExpSparseMoeBlock", +] diff --git a/vllm/models/qwen4_exp/nvidia/model_state.py b/vllm/models/qwen4_exp/nvidia/model_state.py new file mode 100644 index 0000000000..ec2a0fef8b --- /dev/null +++ b/vllm/models/qwen4_exp/nvidia/model_state.py @@ -0,0 +1,143 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Model-runner state for Qwen4Exp PLE inputs.""" + +from typing import Any + +import torch +import torch.nn as nn + +from vllm.config import VllmConfig +from vllm.v1.worker.gpu.input_batch import InputBatch +from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache +from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState +from vllm.v1.worker.gpu.states import RequestState + + +class Qwen4ExpModelState(MambaHybridModelState): + """Add rollback-safe PLE n-gram context to the model inputs.""" + + def __init__( + self, + vllm_config: VllmConfig, + model: nn.Module, + encoder_cache: EncoderCache | None, + device: torch.device, + ) -> None: + super().__init__(vllm_config, model, encoder_cache, device) + config = self.model_config.hf_text_config + self.uses_ngram_embedding = bool(config.ple_layer_ids) + if not self.uses_ngram_embedding: + self.ngram_context_len = 0 + self.ngram_eos_token_id = 0 + return + + if vllm_config.parallel_config.pipeline_parallel_size > 1: + raise RuntimeError( + "N-gram PLE embedding currently requires " + "pipeline_parallel_size=1 because non-first pipeline ranks do " + "not receive the raw input_ids required by PLE. Please run " + "with PP=1." + ) + + self.ngram_context_len = int(config.ngram_size) - 1 + if self.ngram_context_len <= 0: + raise ValueError("N-gram embedding requires context length >= 1.") + self.ngram_eos_token_id = int(config.eos_token_id) + self.ngram_context = torch.full( + (self.max_num_reqs, self.ngram_context_len), + self.ngram_eos_token_id, + dtype=torch.int32, + device=self.device, + ) + self.ngram_context_offsets = torch.arange( + -self.ngram_context_len, + 0, + dtype=torch.int64, + device=self.device, + ) + self.ple_query_start_loc = torch.zeros( + self.max_num_reqs + 1, + dtype=torch.int32, + device=self.device, + ) + + def _prepare_ngram_context( + self, + input_batch: InputBatch, + req_states: RequestState, + ) -> torch.Tensor: + num_reqs = input_batch.num_reqs + num_reqs_padded = input_batch.num_reqs_after_padding + context = self.ngram_context[:num_reqs_padded] + context.fill_(self.ngram_eos_token_id) + if num_reqs == 0: + return context + + request_indices = input_batch.idx_mapping[:num_reqs].long() + context_end = req_states.num_computed_tokens.gpu[request_indices].long() + token_indices = context_end.unsqueeze(1) + self.ngram_context_offsets + valid_tokens = token_indices >= 0 + token_indices.clamp_min_(0) + context_tokens = req_states.all_token_ids.gpu[ + request_indices.unsqueeze(1), token_indices + ] + context[:num_reqs].copy_( + torch.where( + valid_tokens, + context_tokens, + context_tokens.new_full((), self.ngram_eos_token_id), + ) + ) + return context + + def prepare_inputs( + self, + input_batch: InputBatch, + req_states: RequestState, + ) -> dict[str, Any]: + model_inputs = super().prepare_inputs(input_batch, req_states) + if not self.uses_ngram_embedding: + return model_inputs + + num_reqs_padded = input_batch.num_reqs_after_padding + query_start_loc = self.ple_query_start_loc[: num_reqs_padded + 1] + query_start_loc.copy_(input_batch.query_start_loc[: num_reqs_padded + 1]) + model_inputs.update( + query_start_loc=query_start_loc, + ngram_context=self._prepare_ngram_context(input_batch, req_states), + ) + return model_inputs + + def prepare_dummy_inputs( + self, + num_reqs: int, + num_tokens: int, + ) -> dict[str, Any]: + model_inputs = super().prepare_dummy_inputs(num_reqs, num_tokens) + if not self.uses_ngram_embedding: + return model_inputs + + query_start_loc = self.ple_query_start_loc[: num_reqs + 1] + query_start_loc[0] = 0 + tokens_per_req, num_extra_tokens = divmod(num_tokens, num_reqs) + query_lens = torch.full( + (num_reqs,), + tokens_per_req, + dtype=query_start_loc.dtype, + device=query_start_loc.device, + ) + if num_extra_tokens > 0: + query_lens[-num_extra_tokens:] += 1 + torch.cumsum(query_lens, dim=0, out=query_start_loc[1:]) + + ngram_context = self.ngram_context[:num_reqs] + ngram_context.fill_(self.ngram_eos_token_id) + model_inputs.update( + query_start_loc=query_start_loc, + ngram_context=ngram_context, + ) + return model_inputs + + +__all__ = ["Qwen4ExpModelState"] diff --git a/vllm/models/qwen4_exp/nvidia/mtp.py b/vllm/models/qwen4_exp/nvidia/mtp.py new file mode 100644 index 0000000000..5a77448eca --- /dev/null +++ b/vllm/models/qwen4_exp/nvidia/mtp.py @@ -0,0 +1,462 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inference-only Qwen4Exp MTP (Multi-Token Predictor) model. + +The MTP draft model reuses the Qwen4Exp backbone (PLE/HC/MoE) but: + - drops all multi-modal handling (text-only), + - forces PLE off while keeping the main model's HC stream count, + - fuses the backbone hidden and the new-token embedding via + ``residual_linear_shared`` (fc_embedding + shared fc_hidden) instead of + the ``Linear(2H, H)`` + repeat used by other MTP variants, + - emits TWO hidden streams per step (scheme A): a single stream [T, H] + (final-mixer collapsed, fed to the LM head) and a pre-final-mixer + multi stream [T, hc_count*H] (fed to the next draft step). +""" + +from collections.abc import Iterable + +import regex as re +import torch +from torch import nn + +from vllm.compilation.decorators import support_torch_compile +from vllm.config import VllmConfig, replace, set_current_vllm_config +from vllm.distributed import get_pp_group +from vllm.model_executor.layers.fused_moe.utils import ( + is_model_fused_shared_expert_compatible, +) +from vllm.model_executor.layers.layernorm import GemmaRMSNorm +from vllm.model_executor.layers.linear import ColumnParallelLinear +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, + VocabParallelEmbedding, +) +from vllm.model_executor.model_loader.utils import configure_quant_config +from vllm.model_executor.models.interfaces import SupportsPP +from vllm.model_executor.models.qwen3_5 import Qwen3_5Model +from vllm.model_executor.models.utils import ( + AutoWeightsLoader, + PPMissingLayer, + WeightsMapper, + get_draft_quant_config, + make_empty_intermediate_tensors_factory, + maybe_fuse_shared_experts, + maybe_prefix, +) +from vllm.sequence import IntermediateTensors +from vllm.transformers_utils.configs.qwen4_exp import ( + Qwen4ExpTextConfig, +) + +from .hyperconnection import GatedResidual, HyperConnectionConfig +from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm +from .model import ( + _HC_WEIGHTS_MAPPER, + _QWEN4_EXP_IGNORED_MISSING_SUFFIXES, + Qwen4ExpDecoderLayer, + Qwen4ExpMixtureOfExperts, + Qwen4ExpSparseMoeBlock, +) + + +def _remap_ignored_layers( + ignored_layers: list[str], + mtp_start_layer_idx: int, +) -> list[str]: + remapped: list[str] = [] + for name in ignored_layers: + if name.startswith("mtp."): + new_name = re.sub( + r"(?<=\.layers\.)\d+", + lambda m: str(mtp_start_layer_idx + int(m.group(0))), + name, + ) + remapped.append(new_name) + else: + remapped.append(name) + return remapped + + +def _remap_mtp_weight_name(name: str) -> str | None: + """Map Qwen4Exp checkpoint paths into the standalone draft model.""" + + for checkpoint_prefix in ( + "model.language_model.", + "language_model.", + ): + if name.startswith(checkpoint_prefix): + name = name.removeprefix(checkpoint_prefix) + break + + if name.startswith("embed_tokens."): + name = f"model.{name}" + if name.startswith("model.mtp."): + name = name.removeprefix("model.") + if name.startswith("mtp.shared_head.head."): + return name.replace("mtp.shared_head.head.", "lm_head.", 1) + if name.startswith("model.shared_head.head."): + return name.replace("model.shared_head.head.", "lm_head.", 1) + if name.startswith("shared_head.head."): + return name.replace("shared_head.head.", "lm_head.", 1) + if name.startswith("model.lm_head."): + return name.removeprefix("model.") + if name.startswith("mtp."): + return name.replace("mtp.", "model.", 1) + if name.startswith("model.embed_tokens.") or name.startswith("lm_head."): + return name + return None + + +def _make_draft_vllm_config( + vllm_config: VllmConfig, + mtp_start_layer_idx: int, +) -> VllmConfig: + """Ensure that the draft model config is set in the vLLM config.""" + speculative_config = vllm_config.speculative_config + if speculative_config is None or speculative_config.draft_model_config is None: + raise ValueError("speculative_config.draft_model_config must be set") + + draft_quant_config = get_draft_quant_config(vllm_config) + + # inject packed and ignored modules to the quantization config of draft model + if draft_quant_config is not None: + configure_quant_config(draft_quant_config, Qwen4ExpMTP) + ignored_layers = getattr(draft_quant_config, "ignored_layers", None) + if ignored_layers: + setattr( # noqa: B010 + draft_quant_config, + "ignored_layers", + _remap_ignored_layers(ignored_layers, mtp_start_layer_idx), + ) + exclude_modules = getattr(draft_quant_config, "exclude_modules", None) + if exclude_modules: + setattr( # noqa: B010 + draft_quant_config, + "exclude_modules", + _remap_ignored_layers(exclude_modules, mtp_start_layer_idx), + ) + + draft_vllm_config = replace( + vllm_config, + model_config=speculative_config.draft_model_config, + ) + # VllmConfig post-init derives the target quant config, so restore the + # independently resolved draft quant config after replacement. + draft_vllm_config.quant_config = draft_quant_config + return draft_vllm_config + + +@support_torch_compile( + dynamic_arg_dims={ + "input_ids": 0, + "positions": -1, + "intermediate_tensors": 0, + "inputs_embeds": 0, + "hidden_states": 0, + } +) +class Qwen4ExpMultiTokenPredictor(nn.Module): + hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _HC_WEIGHTS_MAPPER + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + super().__init__() + + model_config = vllm_config.model_config + config: Qwen4ExpTextConfig = model_config.hf_text_config + + self.config = config + self.vocab_size = config.vocab_size + + self.mtp_start_layer_idx = config.num_hidden_layers + self.num_mtp_layers = getattr(config, "mtp_num_hidden_layers", 1) + + self.hidden_size = config.hidden_size + self.hc_count = config.hc_count + + self.embed_tokens = VocabParallelEmbedding(self.vocab_size, self.hidden_size) + draft_vllm_config = _make_draft_vllm_config( + vllm_config, + self.mtp_start_layer_idx, + ) + with set_current_vllm_config(draft_vllm_config, prefix=prefix): + # residual_linear_shared fusion: fc_embedding projects the token + # embedding, fc_hidden (shared across HC branches) projects the + # backbone hidden; the embedding is added as a residual to every + # branch (see mtp_residual_linear_shared.md). + self.fc_embedding = ColumnParallelLinear( + self.hidden_size, + self.hidden_size, + gather_output=True, + bias=False, + return_bias=False, + quant_config=draft_vllm_config.quant_config, + prefix=f"{prefix}.fc_embedding", + ) + self.fc_hidden = ColumnParallelLinear( + self.hidden_size, + self.hidden_size, + gather_output=True, + bias=False, + return_bias=False, + quant_config=draft_vllm_config.quant_config, + prefix=f"{prefix}.fc_hidden", + ) + self.layers = nn.ModuleList( + Qwen4ExpDecoderLayer( + draft_vllm_config, + layer_type="full_attention", + prefix=f"{prefix}.layers.{self.mtp_start_layer_idx + idx}", + ) + for idx in range(self.num_mtp_layers) + ) + self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible( + self.layers, + Qwen4ExpSparseMoeBlock, + "mlp", + ) + + self.pre_fc_norm_embedding = GemmaRMSNorm( + self.hidden_size, eps=config.rms_norm_eps + ) + self.pre_fc_norm_hidden = GemmaRMSNorm( + self.hidden_size * self.hc_count, eps=config.rms_norm_eps + ) + # HC final mixer collapses the multi stream into [T, H] for the LM head. + hc_config = HyperConnectionConfig( + hc_count=config.hc_count, + hidden_size=config.hidden_size, + params_dtype=torch.bfloat16, + hc_lowrank=config.hc_lowrank, + rms_norm_eps=config.rms_norm_eps, + hc_per_branch_norm=True, + ) + self.hyper_connection_mixer = GatedResidual( + hc_config, + use_combine=False, + prefix=maybe_prefix(prefix, "hyper_connection_mixer"), + ) + self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory( + ["hidden_states"], self.hidden_size * self.hc_count + ) + + def _iter_qsa_attentions(self): + """Yield MTP attention modules that own a QSA indexer.""" + + for layer in self.layers: + attention = getattr(layer, "self_attn", None) + if ( + attention is not None + and getattr(attention, "indexer", None) is not None + ): + yield attention + + def set_skip_topk(self, skip: bool) -> None: + """Select on MTP step 0 and reuse its QSA indices on later steps.""" + + for attention in self._iter_qsa_attentions(): + attention.indexer.skip_topk = skip + + def compact_topk_indices(self, row_indices: torch.Tensor) -> None: + """Keep each request's target-aligned step-0 sparse-index row.""" + + num_rows = row_indices.numel() + for attention in self._iter_qsa_attentions(): + buffer = attention.topk_indices_buffer + selected = buffer.index_select(0, row_indices) + buffer[:num_rows].copy_(selected) + + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.embed_tokens(input_ids) + + def forward( + self, + input_ids: torch.Tensor | None, + positions: torch.Tensor, + hidden_states: torch.Tensor | None = None, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + spec_step_idx: int = 0, + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor] | IntermediateTensors: + hc_count = self.hc_count + hidden_size = self.hidden_size + + if get_pp_group().is_first_rank: + assert hidden_states is not None + if inputs_embeds is None: + assert input_ids is not None + inputs_embeds = self.embed_input_ids(input_ids) + # Embedding branch: pre-norm -> fc_embedding -> [T, H]. + inputs_embeds = self.pre_fc_norm_embedding(inputs_embeds) + inputs_embeds = self.fc_embedding(inputs_embeds) + + # Backbone hidden is multi-stream [T, hc_count*H] (scheme A: + # the main model truly emits the pre-final-mixer multi stream + # on the first step; subsequent steps reuse the prior draft + # step's multi stream). + num_tokens = hidden_states.shape[0] + hidden_states = hidden_states.view(num_tokens, hc_count, hidden_size) + hidden_states = self.pre_fc_norm_hidden(hidden_states.flatten(-2)).view( + num_tokens, hc_count, hidden_size + ) + hidden_states = self.fc_hidden(hidden_states) + # Add the embedding residual to every branch, then fold back + # to [T, hc_count*H] (HC outer, HS inner) for the HC decoder. + hidden_states = inputs_embeds.unsqueeze(-2) + hidden_states + hidden_states = hidden_states.flatten(-2) + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + + current_step_idx = spec_step_idx % self.num_mtp_layers + layer = self.layers[current_step_idx] + hidden_states, block_output, injection = layer( + hidden_states=hidden_states, + prev_block_output=None, + prev_injection=None, + positions=positions, + input_ids=None, + query_start_loc=None, + ngram_context=None, + ) + if not get_pp_group().is_last_rank: + # As in the target model, PP carries a materialized tensor rather + # than the delayed hidden/output/injection tuple. + hidden_states = layer.mlp_hyper_connection.combine( + hidden_states, block_output, injection + ) + return IntermediateTensors({"hidden_states": hidden_states}) + + # Last PP rank finalize. Keep both: + # (A) sample_hidden_states [T, H] -> single stream for the LM head + # (B) multi_hidden [T, hc_count*H] -> pre-final-mixer multi stream + # for the next draft step (zero extra compute, just kept). + multi_hidden, sample_hidden_states, _ = ( + self.hyper_connection_mixer.combine_and_mix( + hidden_states, block_output, injection + ) + ) + return sample_hidden_states, multi_hidden + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + weights = maybe_fuse_shared_experts( + weights, + enabled=self.is_fused_shared_expert_enabled, + n_routed_experts=getattr(self.config, "num_experts", 0) or 0, + n_shared_experts=1, + ckpt_prefix="mlp.shared_expert", + ) + mapper = self.hf_to_vllm_mapper | WeightsMapper( + orig_to_new_substr={"hyper_connection_mixer.block_inject_weight": None} + ) + loader = AutoWeightsLoader( + self, + ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(), + ) + return loader.load_weights(weights, mapper=mapper) + + +@support_torch_compile( + dynamic_arg_dims={ + "input_ids": 0, + "positions": -1, + "intermediate_tensors": 0, + "inputs_embeds": 0, + "hidden_states": 0, + } +) +class Qwen4ExpMTP(nn.Module, SupportsPP, Qwen4ExpMixtureOfExperts): + packed_modules_mapping = { + "qkv_proj": ["q_proj", "k_proj", "v_proj"], + "gate_up_proj": ["gate_proj", "up_proj"], + "in_proj_qkvz": ["in_proj_qkv", "in_proj_z"], + "in_proj_ba": ["in_proj_b", "in_proj_a"], + "input_mix_weight_down_block_inject": [ + "input_mix_weight_down", + "block_inject_weight", + "_input_mix_padding", + ], + } + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config + self.vllm_config = vllm_config + cache_config = vllm_config.cache_config + if cache_config.mamba_cache_mode == "all": + raise NotImplementedError( + "Qwen4ExpMTP currently does not support 'all' prefix caching, " + "please use '--mamba-cache-mode=align' instead" + ) + + self.quant_config = vllm_config.quant_config + + super().__init__() + self.config = config + self.model = Qwen4ExpMultiTokenPredictor( + vllm_config=vllm_config, + prefix=maybe_prefix(prefix, "mtp"), + ) + + if get_pp_group().is_last_rank: + if config.tie_word_embeddings: + self.lm_head = self.model.embed_tokens + else: + self.lm_head = ParallelLMHead( + config.vocab_size, + config.hidden_size, + prefix=maybe_prefix(prefix, "lm_head"), + ) + else: + self.lm_head = PPMissingLayer() + + self.logits_processor = LogitsProcessor(config.vocab_size) + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors + ) + self.set_moe_parameters(self.model.layers) + enable_qwen4_exp_low_latency_gemm(self, vllm_config.model_config.dtype) + + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.model.embed_input_ids(input_ids) + + def forward( + self, + input_ids: torch.Tensor | None, + positions: torch.Tensor, + hidden_states: torch.Tensor | None = None, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + spec_step_idx: int = 0, + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor] | IntermediateTensors: + return self.model( + input_ids, + positions, + hidden_states, + intermediate_tensors, + inputs_embeds, + spec_step_idx=spec_step_idx, + ) + + def compute_logits( + self, hidden_states: torch.Tensor, spec_step_idx: int = 0 + ) -> torch.Tensor | None: + return self.logits_processor(self.lm_head, hidden_states) + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + def remap_weight_names(): + for name, weight in weights: + remapped_name = _remap_mtp_weight_name(name) + if remapped_name is not None: + yield remapped_name, weight + + mapper = WeightsMapper( + orig_to_new_substr={"hyper_connection_mixer.block_inject_weight": None} + ) + loader = AutoWeightsLoader( + self, + ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(), + ) + return loader.load_weights(remap_weight_names(), mapper=mapper) + + +__all__ = ["Qwen4ExpMTP", "Qwen4ExpMultiTokenPredictor"] diff --git a/vllm/models/qwen4_exp/nvidia/ops/__init__.py b/vllm/models/qwen4_exp/nvidia/ops/__init__.py new file mode 100644 index 0000000000..779c57191d --- /dev/null +++ b/vllm/models/qwen4_exp/nvidia/ops/__init__.py @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""NVIDIA-only Qwen4Exp kernels; import leaf modules directly.""" diff --git a/vllm/models/qwen4_exp/nvidia/ops/hc.py b/vllm/models/qwen4_exp/nvidia/ops/hc.py new file mode 100644 index 0000000000..a3487b916e --- /dev/null +++ b/vllm/models/qwen4_exp/nvidia/ops/hc.py @@ -0,0 +1,487 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""NVIDIA HyperConnection kernels for Qwen4Exp.""" + +import torch + +from vllm.platforms import current_platform +from vllm.triton_utils import tl, triton +from vllm.utils.torch_utils import direct_register_custom_op + + +@triton.jit +def _grouped_gemma_rmsnorm_kernel( + x_ptr, + w_ptr, + y_ptr, + stride_x, + stride_y, + DIM: tl.constexpr, + NUM_GROUPS: tl.constexpr, + W_SHARED: tl.constexpr, + EPS: tl.constexpr, + launch_pdl: tl.constexpr, +) -> None: + GROUP_DIM: tl.constexpr = DIM // NUM_GROUPS + BLOCK_SIZE: tl.constexpr = triton.next_power_of_2(GROUP_DIM) + + pid = tl.program_id(0) + group_id = pid % NUM_GROUPS + row = pid // NUM_GROUPS + + offs_g = tl.arange(0, BLOCK_SIZE) + offsets = group_id * GROUP_DIM + offs_g + mask = offs_g < GROUP_DIM + # A [GROUP_DIM] affine is shared; a [DIM] affine follows the grouped + # checkpoint layout. + w_offs = offs_g if W_SHARED else offsets + + if launch_pdl: + tl.extra.cuda.gdc_wait() + + x = tl.load(x_ptr + row * stride_x + offsets, mask, other=0.0).to(tl.float32) + w = tl.load(w_ptr + w_offs, mask, other=0.0) + + rrms = tl.rsqrt(tl.sum(x * x) / GROUP_DIM + EPS) + # Gemma's (1 + w) affine is written this way to lower to an FMA. + y = x * rrms + y += y * w.to(tl.float32) + + if launch_pdl: + tl.extra.cuda.gdc_launch_dependents() + tl.store(y_ptr + row * stride_y + offsets, y, mask) + + +def _grouped_gemma_rmsnorm( + x: torch.Tensor, weight: torch.Tensor, eps: float, num_groups: int +) -> torch.Tensor: + N, DIM = x.shape + assert x.stride(1) == 1, "grouped Gemma RMSNorm requires unit inner stride" + assert weight.is_contiguous(), "grouped Gemma RMSNorm weight must be contiguous" + assert DIM % num_groups == 0 + group_dim = DIM // num_groups + assert weight.numel() in (group_dim, DIM) + + y = x.new_empty(x.shape) + _grouped_gemma_rmsnorm_kernel[(N * num_groups,)]( + x, + weight, + y, + x.stride(0), + y.stride(0), + DIM, + num_groups, + W_SHARED=weight.numel() == group_dim, + EPS=eps, + launch_pdl=current_platform.is_arch_support_pdl(), + ) + return y + + +@triton.jit +def _hc_silu_kernel( + x_ptr, + y_ptr, + stride_x, + stride_y, + DIM: tl.constexpr, + HC: tl.constexpr, + launch_pdl: tl.constexpr, +) -> None: + BLOCK_SIZE: tl.constexpr = triton.next_power_of_2(DIM) + + row = tl.program_id(0) + offs = tl.arange(0, BLOCK_SIZE) + mask = offs < DIM + + if launch_pdl: + tl.extra.cuda.gdc_wait() + + x = tl.load(x_ptr + row * stride_x + offs, mask).to(tl.float32) / HC + y = x * tl.sigmoid(x) + + if launch_pdl: + tl.extra.cuda.gdc_launch_dependents() + tl.store(y_ptr + row * stride_y + offs, y, mask) + + +def _hc_silu(x: torch.Tensor, hc_count: int) -> torch.Tensor: + num_tokens, DIM = x.shape + assert x.stride(1) == 1 + + output = x.new_empty(x.shape) + _hc_silu_kernel[(num_tokens,)]( + x, + output, + x.stride(0), + output.stride(0), + DIM=DIM, + HC=hc_count, + launch_pdl=current_platform.is_arch_support_pdl(), + ) + return output + + +@triton.jit +def _hc_gate_mix_kernel( + x_ptr, + g_ptr, + y_ptr, + stride_x, + stride_g, + stride_y, + DIM: tl.constexpr, + HC: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + launch_pdl: tl.constexpr, +) -> None: + HC_DIM: tl.constexpr = DIM // HC + + row = tl.program_id(0) + tile_id = tl.program_id(1) + offs_inner = tile_id * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) + mask = offs_inner < HC_DIM + + if launch_pdl: + tl.extra.cuda.gdc_wait() + + # The constexpr loop is unrolled and keeps one stream live at a time. + # Materializing [HC, BLOCK_SIZE] more than doubles latency at large M. + acc = tl.zeros([BLOCK_SIZE], dtype=tl.float32) + for stream in tl.static_range(HC): + offsets = stream * HC_DIM + offs_inner + g = tl.load(g_ptr + row * stride_g + offsets, mask, other=0.0) + x = tl.load(x_ptr + row * stride_x + offsets, mask, other=0.0) + acc += tl.sigmoid(g.to(tl.float32)) * x.to(tl.float32) + acc /= HC + + if launch_pdl: + tl.extra.cuda.gdc_launch_dependents() + tl.store(y_ptr + row * stride_y + offs_inner, acc, mask) + + +def _hc_gate_mix(x: torch.Tensor, gate: torch.Tensor, hc_count: int) -> torch.Tensor: + N, DIM = gate.shape + assert x.shape == gate.shape + assert DIM % hc_count == 0 + assert x.stride(1) == 1 + assert gate.stride(1) == 1 + + HC_DIM = DIM // hc_count + out = x.new_empty(N, HC_DIM) + BLOCK_SIZE = 512 + _hc_gate_mix_kernel[(N, triton.cdiv(HC_DIM, BLOCK_SIZE))]( + x, + gate, + out, + x.stride(0), + gate.stride(0), + out.stride(0), + DIM, + hc_count, + BLOCK_SIZE, + launch_pdl=current_platform.is_arch_support_pdl(), + ) + return out + + +@triton.jit +def _hc_combine_kernel( + block_ptr, + res_ptr, + inj_ptr, + out_ptr, + stride_block, + stride_res, + stride_inj, + stride_out, + HC_DIM: tl.constexpr, + HC: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + launch_pdl: tl.constexpr, +) -> None: + HC_PAD: tl.constexpr = triton.next_power_of_2(HC) + + row = tl.program_id(0) + tile_id = tl.program_id(1) + + offs_inner = tile_id * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) + mask_inner = offs_inner < HC_DIM + offs_hc = tl.arange(0, HC_PAD) + mask_hc = offs_hc < HC + offs = offs_hc[:, None] * HC_DIM + offs_inner[None, :] + mask = mask_hc[:, None] & mask_inner[None, :] + + if launch_pdl: + tl.extra.cuda.gdc_wait() + + inj = tl.load(inj_ptr + row * stride_inj + offs_hc, mask_hc, other=0.0) + block = tl.load(block_ptr + row * stride_block + offs_inner, mask_inner, other=0.0) + res = tl.load(res_ptr + row * stride_res + offs, mask, other=0.0) + + # Keeping HC as a broadcast dimension is faster here than four separate + # residual load/store sequences. + inj = 2.0 * tl.sigmoid(inj.to(tl.float32) / HC) + out = res.to(tl.float32) + block.to(tl.float32)[None, :] * inj[:, None] + + if launch_pdl: + tl.extra.cuda.gdc_launch_dependents() + tl.store(out_ptr + row * stride_out + offs, out, mask=mask) + + +def _hc_combine( + residual: torch.Tensor, + block_output: torch.Tensor, + injection_logits: torch.Tensor, + hc_count: int, +) -> torch.Tensor: + N, DIM = residual.shape + assert DIM % hc_count == 0 + hc_dim = DIM // hc_count + assert block_output.shape == (N, hc_dim) + assert injection_logits.shape == (N, hc_count) + assert residual.stride(1) == 1 + assert block_output.stride(1) == 1 + assert injection_logits.stride(1) == 1 + + out = residual.new_empty(residual.shape) + BLOCK_SIZE = 512 + _hc_combine_kernel[(N, triton.cdiv(hc_dim, BLOCK_SIZE))]( + block_output, + residual, + injection_logits, + out, + block_output.stride(0), + residual.stride(0), + injection_logits.stride(0), + out.stride(0), + hc_dim, + hc_count, + BLOCK_SIZE, + launch_pdl=current_platform.is_arch_support_pdl(), + ) + return out + + +@triton.jit +def _hc_combine_norm_kernel( + block_ptr, + res_ptr, + inj_ptr, + w_ptr, + out_ptr, + y_ptr, + stride_block, + stride_res, + stride_inj, + stride_out, + stride_y, + HC_DIM: tl.constexpr, + HC: tl.constexpr, + W_SHARED: tl.constexpr, + EPS: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + launch_pdl: tl.constexpr, +) -> None: + HC_PAD: tl.constexpr = triton.next_power_of_2(HC) + NUM_TILES: tl.constexpr = triton.cdiv(HC_DIM, BLOCK_SIZE) + NUM_TILES_PAD: tl.constexpr = triton.next_power_of_2(NUM_TILES) + + row = tl.program_id(0) + stream = tl.program_id(1) + offs_hc = tl.arange(0, HC_PAD) + mask_hc = offs_hc < HC + tile_ids = tl.arange(0, NUM_TILES_PAD) + offs_inner = tile_ids[:, None] * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)[None, :] + mask_inner = offs_inner < HC_DIM + offs = stream * HC_DIM + offs_inner + # Shared norm weights repeat across streams; per-branch weights use the + # same flattened HC layout as the residual. + w_offs = offs_inner if W_SHARED else offs + + if launch_pdl: + tl.extra.cuda.gdc_wait() + + # Start the uncached residual load first, then issue the other combine + # loads before consuming any of them. + res = tl.load(res_ptr + row * stride_res + offs, mask_inner, other=0.0) + inj = tl.load(inj_ptr + row * stride_inj + offs_hc, mask_hc, other=0.0) + block = tl.load(block_ptr + row * stride_block + offs_inner, mask_inner, other=0.0) + inj = 2.0 * tl.sigmoid(inj.to(tl.float32) / HC) + inj = tl.sum(tl.where(offs_hc == stream, inj, 0.0)) + # Round the materialized combine result before normalization. This matches + # the unfused combine -> RMSNorm boundary. + out = (res.to(tl.float32) + block.to(tl.float32) * inj).to(out_ptr.dtype.element_ty) + tl.store(out_ptr + row * stride_out + offs, out, mask=mask_inner) + + out = out.to(tl.float32) + # Keep the two-axis reduction: flattening the padded tile is ~40% slower + # at decode sizes. + sum_sq = tl.sum(tl.sum(out * out, axis=1), axis=0) + rrms = tl.rsqrt(sum_sq / HC_DIM + EPS) + + if launch_pdl: + tl.extra.cuda.gdc_launch_dependents() + + # Loading the weight earlier helps decode but keeps the tile live across + # the reduction and regresses larger batches, so defer it to the norm. + w = tl.load(w_ptr + w_offs, mask_inner, other=0.0) + y = out * rrms + y += y * w.to(tl.float32) + tl.store(y_ptr + row * stride_y + offs, y, mask_inner) + + +def _hc_combine_norm( + residual: torch.Tensor, + block_output: torch.Tensor, + injection_logits: torch.Tensor, + norm_weight: torch.Tensor, + eps: float, + hc_count: int, +) -> tuple[torch.Tensor, torch.Tensor]: + N, DIM = residual.shape + assert DIM % hc_count == 0 + hc_dim = DIM // hc_count + assert block_output.shape == (N, hc_dim) + assert injection_logits.shape == (N, hc_count) + assert residual.stride(1) == 1 + assert block_output.stride(1) == 1 + assert injection_logits.stride(1) == 1 + assert norm_weight.is_contiguous() + assert norm_weight.numel() in (hc_dim, DIM) + + out = residual.new_empty(residual.shape) + y = residual.new_empty(residual.shape) + BLOCK_SIZE = 512 + _hc_combine_norm_kernel[(N, hc_count)]( + block_output, + residual, + injection_logits, + norm_weight, + out, + y, + block_output.stride(0), + residual.stride(0), + injection_logits.stride(0), + out.stride(0), + y.stride(0), + hc_dim, + hc_count, + W_SHARED=norm_weight.numel() == hc_dim, + EPS=eps, + BLOCK_SIZE=BLOCK_SIZE, + launch_pdl=current_platform.is_arch_support_pdl(), + ) + return out, y + + +def _same_shape_fake(x: torch.Tensor, *args) -> torch.Tensor: + return x.new_empty(x.shape) + + +def _hc_gate_mix_fake( + x: torch.Tensor, gate: torch.Tensor, hc_count: int +) -> torch.Tensor: + del gate + return x.new_empty((x.shape[0], x.shape[1] // hc_count)) + + +def _hc_combine_fake( + residual: torch.Tensor, + block_output: torch.Tensor, + injection_logits: torch.Tensor, + hc_count: int, +) -> torch.Tensor: + del block_output, injection_logits, hc_count + return residual.new_empty(residual.shape) + + +def _hc_combine_norm_fake( + residual: torch.Tensor, + block_output: torch.Tensor, + injection_logits: torch.Tensor, + norm_weight: torch.Tensor, + eps: float, + hc_count: int, +) -> tuple[torch.Tensor, torch.Tensor]: + del block_output, injection_logits, norm_weight, eps, hc_count + return residual.new_empty(residual.shape), residual.new_empty(residual.shape) + + +direct_register_custom_op( + op_name="qwen4_exp_grouped_gemma_rmsnorm", + op_func=_grouped_gemma_rmsnorm, + fake_impl=_same_shape_fake, +) +direct_register_custom_op( + op_name="qwen4_exp_hc_silu", + op_func=_hc_silu, + fake_impl=_same_shape_fake, +) +direct_register_custom_op( + op_name="qwen4_exp_hc_gate_mix", + op_func=_hc_gate_mix, + fake_impl=_hc_gate_mix_fake, +) +direct_register_custom_op( + op_name="qwen4_exp_hc_combine", + op_func=_hc_combine, + fake_impl=_hc_combine_fake, +) +direct_register_custom_op( + op_name="qwen4_exp_hc_combine_norm", + op_func=_hc_combine_norm, + fake_impl=_hc_combine_norm_fake, +) + + +def grouped_gemma_rmsnorm( + x: torch.Tensor, weight: torch.Tensor, eps: float, num_groups: int +) -> torch.Tensor: + return torch.ops.vllm.qwen4_exp_grouped_gemma_rmsnorm(x, weight, eps, num_groups) + + +def hc_silu(x: torch.Tensor, hc_count: int) -> torch.Tensor: + return torch.ops.vllm.qwen4_exp_hc_silu(x, hc_count) + + +def hc_gate_mix(x: torch.Tensor, gate: torch.Tensor, hc_count: int) -> torch.Tensor: + return torch.ops.vllm.qwen4_exp_hc_gate_mix(x, gate, hc_count) + + +def hc_combine( + residual: torch.Tensor, + block_output: torch.Tensor, + injection_logits: torch.Tensor, + hc_count: int, +) -> torch.Tensor: + return torch.ops.vllm.qwen4_exp_hc_combine( + residual, block_output, injection_logits, hc_count + ) + + +def hc_combine_norm( + residual: torch.Tensor, + block_output: torch.Tensor, + injection_logits: torch.Tensor, + norm_weight: torch.Tensor, + eps: float, + hc_count: int, +) -> tuple[torch.Tensor, torch.Tensor]: + return torch.ops.vllm.qwen4_exp_hc_combine_norm( + residual, + block_output, + injection_logits, + norm_weight, + eps, + hc_count, + ) + + +__all__ = [ + "grouped_gemma_rmsnorm", + "hc_combine", + "hc_combine_norm", + "hc_gate_mix", + "hc_silu", +] diff --git a/vllm/models/qwen4_exp/nvidia/ops/qsa.py b/vllm/models/qwen4_exp/nvidia/ops/qsa.py new file mode 100644 index 0000000000..d6aa06f3a3 --- /dev/null +++ b/vllm/models/qwen4_exp/nvidia/ops/qsa.py @@ -0,0 +1,1115 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Triton kernels for the Qwen4Exp weight-free QSA path.""" + +from __future__ import annotations + +import math + +import torch + +from vllm.platforms import current_platform +from vllm.triton_utils import HAS_TRITON, tl, triton + +_LOGITS_WORKSPACE_BYTES = 128 * 1024 * 1024 +_TOPK_WORKSPACE_BYTES = 1024 * 1024 + + +@triton.jit +def _qsa_mqa_paged_kernel( + q_ptr, + k_cache_ptr, + page_table_ptr, + token_to_req_ptr, + query_positions_ptr, + sequence_lengths_ptr, + visible_blocks_ptr, + logits_ptr, + stride_q_row, + stride_q_head, + stride_q_dim, + stride_cache_block, + stride_cache_token, + stride_cache_dim, + stride_table_req, + stride_table_page, + stride_logits_row, + num_rows, + num_columns, + num_pages, + num_requests, + score_divisor, + PAGE_SIZE: tl.constexpr, + PAGE_TABLE_WIDTH: tl.constexpr, + NUM_HEADS: tl.constexpr, + HEAD_DIM: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_D: tl.constexpr, + TILES_PER_PROG: tl.constexpr, + STAGES: tl.constexpr, + MAX_N: tl.constexpr, + COMPRESS_RATIO: tl.constexpr, +) -> None: + row = tl.program_id(0) + dims = tl.arange(0, BLOCK_D) + heads = tl.arange(0, MAX_N) + request = tl.load(token_to_req_ptr + row) + safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1) + query_position = tl.load(query_positions_ptr + row) + sequence_length = tl.load( + sequence_lengths_ptr + safe_request, + mask=(request >= 0) & (request < num_requests), + other=0, + ) + visible = tl.minimum( + (query_position + 1) // COMPRESS_RATIO, + sequence_length // COMPRESS_RATIO, + ) + if tl.program_id(1) == 0: + tl.store(visible_blocks_ptr + row, visible) + tile_start = tl.program_id(1) * TILES_PER_PROG + # Top-k is bounded by visible_blocks, so columns beyond it need no value. + if tile_start * BLOCK_N >= visible: + return + tile_end = tl.minimum(tile_start + TILES_PER_PROG, tl.cdiv(visible, BLOCK_N)) + tile_end = tl.minimum(tile_end, tl.cdiv(num_columns, BLOCK_N)) + + # Pad the small head axis to a tensor-core-compatible N dimension. + query = tl.load( + q_ptr + + row * stride_q_row + + heads[None, :] * stride_q_head + + dims[:, None] * stride_q_dim, + mask=(heads[None, :] < NUM_HEADS) & (dims[:, None] < HEAD_DIM), + other=0.0, + ) + column_offsets = tl.arange(0, BLOCK_N) + for tile in tl.range(tile_start, tile_end, num_stages=STAGES): + columns = tile * BLOCK_N + column_offsets + live = columns < visible + logical_page = tl.minimum(columns // PAGE_SIZE, PAGE_TABLE_WIDTH - 1) + page_offset = columns % PAGE_SIZE + physical_page = tl.load( + page_table_ptr + + safe_request * stride_table_req + + logical_page * stride_table_page, + mask=live, + other=-1, + ) + page_valid = live & (physical_page >= 0) & (physical_page < num_pages) + # physical_page * block stride can overflow int32 for large caches. + safe_physical_page = tl.maximum(physical_page, 0).to(tl.int64) + keys = tl.load( + k_cache_ptr + + safe_physical_page[:, None] * stride_cache_block + + page_offset[:, None] * stride_cache_token + + dims[None, :] * stride_cache_dim, + mask=page_valid[:, None] & (dims[None, :] < HEAD_DIM), + other=0.0, + eviction_policy="evict_first", + ) + scores = tl.dot(keys, query, out_dtype=tl.float32) + scores = tl.where(heads[None, :] < NUM_HEADS, tl.maximum(scores, 0.0), 0.0) + score = tl.sum(scores, axis=1) / score_divisor + tl.store( + logits_ptr + row * stride_logits_row + columns, + tl.where(page_valid, score, -float("inf")), + mask=live & (columns < num_columns), + ) + + +@triton.jit +def _expand_qsa_indices_kernel( + block_indices_ptr, + query_positions_ptr, + sequence_lengths_ptr, + token_to_req_ptr, + output_ptr, + stride_blocks_row, + stride_blocks_column, + stride_output_row, + stride_output_column, + rows, + num_requests, + BLOCK_TOPK: tl.constexpr, + COMPRESS_RATIO: tl.constexpr, + TOKEN_TOPK: tl.constexpr, + OUTPUT_WIDTH: tl.constexpr, + COLUMN_BLOCK: tl.constexpr, +) -> None: + row = tl.program_id(0) + columns = tl.program_id(1) * COLUMN_BLOCK + tl.arange(0, COLUMN_BLOCK) + query_position = tl.load(query_positions_ptr + row) + request = tl.load(token_to_req_ptr + row) + safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1) + sequence_length = tl.load( + sequence_lengths_ptr + safe_request, + mask=(request >= 0) & (request < num_requests), + other=0, + ) + complete_blocks = tl.minimum( + tl.minimum( + (query_position + 1) // COMPRESS_RATIO, + sequence_length // COMPRESS_RATIO, + ), + BLOCK_TOPK, + ) + expanded_count = complete_blocks * COMPRESS_RATIO + tail_start = ((query_position + 1) // COMPRESS_RATIO) * COMPRESS_RATIO + tail_count = (query_position + 1) - tail_start + + is_expanded = columns < expanded_count + block_rank = columns // COMPRESS_RATIO + offset = columns % COMPRESS_RATIO + safe_rank = tl.minimum(block_rank, BLOCK_TOPK - 1) + block = tl.load( + block_indices_ptr + row * stride_blocks_row + safe_rank * stride_blocks_column, + mask=(row < rows) & is_expanded, + other=-1, + ) + expanded = block * COMPRESS_RATIO + offset + tail_offset = columns - expanded_count + is_tail = ( + (columns >= expanded_count) + & (tail_offset < tail_count) + & (tail_offset < COMPRESS_RATIO - 1) + ) + token = tl.where(is_expanded, expanded, tail_start + tail_offset) + valid = ( + (row < rows) + & (columns < OUTPUT_WIDTH) + & (is_expanded | is_tail) + & (token >= 0) + & (token < sequence_length) + ) + tl.store( + output_ptr + row * stride_output_row + columns * stride_output_column, + tl.where(valid, token, -1), + mask=(row < rows) & (columns < OUTPUT_WIDTH), + ) + + +@triton.jit +def _qsa_sparse_paged_gqa_splitk_kernel( + q_ptr, + k_cache_ptr, + v_cache_ptr, + indices_ptr, + block_table_ptr, + token_to_req_ptr, + partial_output_ptr, + partial_lse_ptr, + output_ptr, + stride_q_row, + stride_q_head, + stride_k_block, + stride_k_token, + stride_k_head, + stride_v_block, + stride_v_token, + stride_v_head, + stride_indices_row, + stride_table_req, + stride_output_row, + stride_output_head, + num_rows, + num_cache_blocks, + num_requests, + TOPK: tl.constexpr, + PAGE_SIZE: tl.constexpr, + PAGE_TABLE_WIDTH: tl.constexpr, + GROUP_SIZE: tl.constexpr, + HEAD_DIM: tl.constexpr, + NUM_QUERY_HEADS: tl.constexpr, + NUM_SPLITS: tl.constexpr, + NUM_TILES: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, +) -> None: + row = tl.program_id(0) + kv_head = tl.program_id(1) + split_id = tl.program_id(2) + request = tl.load(token_to_req_ptr + row) + safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1) + + head_offsets = tl.arange(0, BLOCK_M) + dim_offsets = tl.arange(0, HEAD_DIM) + column_offsets = tl.arange(0, BLOCK_N) + first_head = kv_head * GROUP_SIZE + query = tl.load( + q_ptr + + row * stride_q_row + + (first_head + head_offsets[:, None]) * stride_q_head + + dim_offsets[None, :], + mask=head_offsets[:, None] < GROUP_SIZE, + other=0.0, + ) + + max_value = tl.full((BLOCK_M,), -1.0e20, dtype=tl.float32) + normalizer = tl.zeros((BLOCK_M,), dtype=tl.float32) + accumulator = tl.zeros((BLOCK_M, HEAD_DIM), dtype=tl.float32) + softmax_scale_log2: tl.constexpr = (HEAD_DIM**-0.5) * 1.4426950408889634 + + # Dynamic bounds avoid padded main-loop iterations for uneven splits. + split_tile_start = split_id * NUM_TILES // NUM_SPLITS + split_tile_end = (split_id + 1) * NUM_TILES // NUM_SPLITS + for tile in range(split_tile_start, split_tile_end): + columns = tile * BLOCK_N + column_offsets + logical_token = tl.load( + indices_ptr + row * stride_indices_row + columns, + mask=columns < TOPK, + other=-1, + ) + safe_token = tl.maximum(logical_token, 0) + logical_page = safe_token // PAGE_SIZE + page_offset = safe_token % PAGE_SIZE + valid = ( + (request >= 0) + & (request < num_requests) + & (logical_token >= 0) + & (logical_page < PAGE_TABLE_WIDTH) + ) + physical_page = tl.load( + block_table_ptr + + safe_request * stride_table_req + + tl.minimum(logical_page, PAGE_TABLE_WIDTH - 1), + mask=valid, + other=-1, + ) + valid &= (physical_page >= 0) & (physical_page < num_cache_blocks) + # physical_page * block stride can overflow int32 for large caches. + safe_page = tl.maximum(physical_page, 0).to(tl.int64) + keys = tl.load( + k_cache_ptr + + safe_page[None, :] * stride_k_block + + page_offset[None, :] * stride_k_token + + kv_head * stride_k_head + + dim_offsets[:, None], + mask=valid[None, :], + other=0.0, + ) + values = tl.load( + v_cache_ptr + + safe_page[:, None] * stride_v_block + + page_offset[:, None] * stride_v_token + + kv_head * stride_v_head + + dim_offsets[None, :], + mask=valid[:, None], + other=0.0, + ) + scores = tl.dot(query, keys) + # Scaling scores avoids re-quantizing a scaled query to BF16. + scores *= softmax_scale_log2 + scores = tl.where(valid[None, :], scores, -1.0e20) + next_max = tl.maximum(max_value, tl.max(scores, axis=1)) + alpha = tl.math.exp2(max_value - next_max) + probabilities = tl.where( + valid[None, :], tl.math.exp2(scores - next_max[:, None]), 0.0 + ) + accumulator = tl.dot( + probabilities.to(values.dtype), + values, + acc=accumulator * alpha[:, None], + ) + normalizer = normalizer * alpha + tl.sum(probabilities, axis=1) + max_value = next_max + + has_values = normalizer > 0 + normalized_output = tl.where( + has_values[:, None], + accumulator / tl.maximum(normalizer[:, None], 1.0e-20), + 0.0, + ) + output_mask = head_offsets[:, None] < GROUP_SIZE + if NUM_SPLITS == 1: + tl.store( + output_ptr + + row * stride_output_row + + (first_head + head_offsets[:, None]) * stride_output_head + + dim_offsets[None, :], + normalized_output, + mask=output_mask, + ) + else: + partial_lse = tl.where( + has_values, + max_value + tl.math.log2(tl.maximum(normalizer, 1.0e-20)), + -float("inf"), + ) + tl.store( + partial_output_ptr + + ( + (split_id * num_rows + row) * NUM_QUERY_HEADS + + first_head + + head_offsets[:, None] + ) + * HEAD_DIM + + dim_offsets[None, :], + normalized_output, + mask=output_mask, + ) + tl.store( + partial_lse_ptr + + (split_id * num_rows + row) * NUM_QUERY_HEADS + + first_head + + head_offsets, + partial_lse, + mask=head_offsets < GROUP_SIZE, + ) + + +@triton.jit +def _qsa_merge_splitk_kernel( + partial_output_ptr, + partial_lse_ptr, + output_ptr, + stride_output_row, + stride_output_head, + num_rows, + HEAD_DIM: tl.constexpr, + NUM_QUERY_HEADS: tl.constexpr, + NUM_SPLITS: tl.constexpr, + BLOCK_SPLITS: tl.constexpr, +) -> None: + row = tl.program_id(0) + head = tl.program_id(1) + split_offsets = tl.arange(0, BLOCK_SPLITS) + dim_offsets = tl.arange(0, HEAD_DIM) + split_mask = split_offsets < NUM_SPLITS + lse = tl.load( + partial_lse_ptr + (split_offsets * num_rows + row) * NUM_QUERY_HEADS + head, + mask=split_mask, + other=-float("inf"), + ) + lse_max = tl.max(lse, axis=0) + has_values = lse_max > -float("inf") + shifted = tl.where(split_mask & has_values, lse - lse_max, -float("inf")) + weights = tl.math.exp2(shifted) + denominator = tl.sum(weights, axis=0) + partial_output = tl.load( + partial_output_ptr + + ((split_offsets[:, None] * num_rows + row) * NUM_QUERY_HEADS + head) + * HEAD_DIM + + dim_offsets[None, :], + mask=split_mask[:, None], + other=0.0, + ) + merged = tl.sum(partial_output * weights[:, None], axis=0) + merged = tl.where(denominator > 0, merged / denominator, 0.0) + tl.store( + output_ptr + row * stride_output_row + head * stride_output_head + dim_offsets, + merged, + ) + + +@triton.jit +def _store_qsa_rows_kernel( + cache_ptr, + slots_ptr, + rows_ptr, + stride_cache_block, + stride_cache_token, + stride_cache_dim, + stride_rows_row, + stride_rows_dim, + num_rows, + num_blocks, + PAGE_SIZE: tl.constexpr, + WIDTH: tl.constexpr, + BLOCK_D: tl.constexpr, +) -> None: + row = tl.program_id(0) + dims = tl.arange(0, BLOCK_D) + slot = tl.load(slots_ptr + row) + valid = (row < num_rows) & (slot >= 0) & (slot < num_blocks * PAGE_SIZE) + block = tl.maximum(slot, 0) // PAGE_SIZE + token = tl.maximum(slot, 0) % PAGE_SIZE + values = tl.load( + rows_ptr + row * stride_rows_row + dims * stride_rows_dim, + mask=valid & (dims < WIDTH), + other=0, + ) + tl.store( + cache_ptr + + block * stride_cache_block + + token * stride_cache_token + + dims * stride_cache_dim, + values, + mask=valid & (dims < WIDTH), + ) + + +@triton.jit +def _compress_qsa_groups_kernel( + raw_keys_ptr, # this step's raw key rows, straight from activations + raw_positions_ptr, # this step's per-token positions + compressor_state_cache_ptr, # per-request ring of previous raw keys + rope_cache_ptr, # packed RoPE position tail of the ring + compressor_state_table_ptr, + token_to_req_ptr, + query_start_loc_ptr, + logical_positions_ptr, + compressed_slots_ptr, + pooled_ptr, + first_positions_ptr, + stride_raw_row, + stride_raw_dim, + stride_raw_positions_row, + stride_raw_positions_dim, + stride_compressor_state_block, + stride_compressor_state_token, + stride_compressor_state_dim, + stride_rope_block, + stride_rope_token, + stride_rope_dim, + stride_compressor_state_table_req, + stride_pooled_row, + stride_pooled_dim, + stride_positions_row, + stride_positions_dim, + num_rows, + num_compressor_state_blocks, + num_requests, + COMPRESSOR_STATE_SIZE: tl.constexpr, + COMPRESS_RATIO: tl.constexpr, + HEAD_DIM: tl.constexpr, + BLOCK_D: tl.constexpr, + LOAD_ROPE_POSITIONS: tl.constexpr, +) -> None: + row = tl.program_id(0) + dims = tl.arange(0, BLOCK_D) + request = tl.load(token_to_req_ptr + row) + end_position = tl.load(logical_positions_ptr + row) + compressed_slot = tl.load(compressed_slots_ptr + row) + valid_request = (request >= 0) & (request < num_requests) + safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1) + query_row_start = tl.load( + query_start_loc_ptr + safe_request, mask=valid_request, other=0 + ) + query_row_end = tl.load( + query_start_loc_ptr + safe_request + 1, mask=valid_request, other=0 + ) + chunk_start_position = end_position - (row - query_row_start) + compressor_state_block = tl.load( + compressor_state_table_ptr + safe_request * stride_compressor_state_table_req, + mask=valid_request, + other=-1, + ) + valid_compressor_state_block = (compressor_state_block >= 0) & ( + compressor_state_block < num_compressor_state_blocks + ) + valid_row = ( + (row < num_rows) + & valid_request + & (row >= query_row_start) + & (row < query_row_end) + & (end_position >= COMPRESS_RATIO - 1) + & (compressed_slot >= 0) + ) + accumulator = tl.zeros((BLOCK_D,), dtype=tl.float32) + + # A group can span the compressor-state ring (older members) and this + # step's raw rows (members at positions >= chunk_start_position). + for group_offset in tl.range(0, COMPRESS_RATIO): + position = end_position - (COMPRESS_RATIO - 1 - group_offset) + use_raw = position >= chunk_start_position + raw_row = query_row_start + position - chunk_start_position + raw_values = tl.load( + raw_keys_ptr + raw_row * stride_raw_row + dims * stride_raw_dim, + mask=valid_row + & use_raw + & (raw_row >= query_row_start) + & (raw_row < query_row_end) + & (raw_row < num_rows) + & (dims < HEAD_DIM), + other=0.0, + ).to(tl.float32) + compressor_state_values = tl.load( + compressor_state_cache_ptr + + tl.maximum(compressor_state_block, 0).to(tl.int64) + * stride_compressor_state_block + + (position % COMPRESSOR_STATE_SIZE) * stride_compressor_state_token + + dims * stride_compressor_state_dim, + mask=valid_row + & ~use_raw + & valid_compressor_state_block + & (dims < HEAD_DIM), + other=0.0, + ).to(tl.float32) + accumulator += tl.where(use_raw, raw_values, compressor_state_values) + + tl.store( + pooled_ptr + row * stride_pooled_row + dims * stride_pooled_dim, + accumulator / COMPRESS_RATIO, + mask=(row < num_rows) & (dims < HEAD_DIM), + ) + + position_dims = tl.arange(0, 4) + first_position = end_position - COMPRESS_RATIO + 1 + if LOAD_ROPE_POSITIONS: + first_from_raw = first_position >= chunk_start_position + raw_first_row = query_row_start + first_position - chunk_start_position + raw_position_values = tl.load( + raw_positions_ptr + + raw_first_row * stride_raw_positions_row + + position_dims * stride_raw_positions_dim, + mask=valid_row + & first_from_raw + & (raw_first_row >= query_row_start) + & (raw_first_row < query_row_end) + & (raw_first_row < num_rows) + & (position_dims < 3), + other=0, + ) + compressor_state_position_values = tl.load( + rope_cache_ptr + + tl.maximum(compressor_state_block, 0).to(tl.int64) * stride_rope_block + + (first_position % COMPRESSOR_STATE_SIZE) * stride_rope_token + + position_dims * stride_rope_dim, + mask=valid_row + & ~first_from_raw + & valid_compressor_state_block + & (position_dims < 3), + other=0, + ) + position_values = tl.where( + first_from_raw, + raw_position_values, + compressor_state_position_values, + ) + else: + position_values = tl.where(valid_row, first_position, 0) + tl.store( + first_positions_ptr + + row * stride_positions_row + + position_dims * stride_positions_dim, + position_values, + mask=(row < num_rows) & (position_dims < 3), + ) + + +def _validate_mqa(q: torch.Tensor) -> None: + if q.ndim != 3 or q.shape[1] <= 0 or q.shape[2] <= 0: + raise ValueError("QSA query must be [rows, heads, head_dim]") + + +def qsa_mqa_paged( + q: torch.Tensor, + k_cache: torch.Tensor, + page_table: torch.Tensor, + token_to_req: torch.Tensor, + query_positions: torch.Tensor, + sequence_lengths: torch.Tensor, + compress_ratio: int, + num_columns: int | None = None, + score_scale: float | None = None, +) -> tuple[torch.Tensor, torch.Tensor]: + """Compute QSA scores directly from a paged compressed-key cache.""" + + _validate_mqa(q) + if not q.is_cuda or not HAS_TRITON: + raise RuntimeError("paged QSA scoring requires CUDA and Triton") + if k_cache.ndim != 4 or k_cache.shape[2] != 1: + raise ValueError("QSA cache must be [pages, page_size, 1, head_dim]") + if k_cache.shape[3] != q.shape[2]: + raise ValueError("QSA query and cache dimensions must match") + if page_table.ndim != 2: + raise ValueError("QSA page table must be two-dimensional") + if q.shape[0] and (not all(k_cache.shape[:2]) or not all(page_table.shape)): + raise ValueError("QSA paged scoring cache and page table must be nonempty") + if token_to_req.shape != (q.shape[0],): + raise ValueError("QSA request mapping must match query rows") + if query_positions.shape != (q.shape[0],): + raise ValueError("QSA query positions must match query rows") + if sequence_lengths.shape != (page_table.shape[0],): + raise ValueError("QSA sequence lengths must match page-table requests") + if compress_ratio <= 0: + raise ValueError("QSA compression ratio must be positive") + score_divisor = math.sqrt(q.shape[2]) if score_scale is None else score_scale + if score_divisor <= 0: + raise ValueError("QSA score scale must be positive") + + capacity = page_table.shape[1] * k_cache.shape[1] + columns = capacity if num_columns is None else num_columns + if columns < 0: + raise ValueError("QSA score width must be non-negative") + logits = torch.empty((q.shape[0], columns), dtype=torch.float32, device=q.device) + visible_blocks = torch.empty(q.shape[0], dtype=torch.int32, device=q.device) + if not q.shape[0] or not columns: + return logits, visible_blocks + BLOCK_N = 64 + BLOCK_D = max(16, triton.next_power_of_2(q.shape[2])) + MAX_N = max(16, triton.next_power_of_2(q.shape[1])) + # Tuned on GB300: larger row batches provide enough parallelism to reuse Q. + tiles_per_program = 1 if q.shape[0] <= 32 else 8 + _qsa_mqa_paged_kernel[ + (q.shape[0], triton.cdiv(columns, BLOCK_N * tiles_per_program)) + ]( + q, + k_cache, + page_table, + token_to_req, + query_positions, + sequence_lengths, + visible_blocks, + logits, + q.stride(0), + q.stride(1), + q.stride(2), + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(3), + page_table.stride(0), + page_table.stride(1), + logits.stride(0), + q.shape[0], + columns, + k_cache.shape[0], + page_table.shape[0], + float(score_divisor), + PAGE_SIZE=k_cache.shape[1], + PAGE_TABLE_WIDTH=page_table.shape[1], + NUM_HEADS=q.shape[1], + HEAD_DIM=q.shape[2], + BLOCK_N=BLOCK_N, + BLOCK_D=BLOCK_D, + TILES_PER_PROG=tiles_per_program, + STAGES=2, + MAX_N=MAX_N, + COMPRESS_RATIO=compress_ratio, + num_warps=2, + ) + return logits, visible_blocks + + +def expand_qsa_block_indices_cuda( + block_indices: torch.Tensor, + query_positions: torch.Tensor, + sequence_lengths: torch.Tensor, + token_to_req: torch.Tensor, + compress_ratio: int, + token_topk: int, + out: torch.Tensor | None = None, +) -> torch.Tensor: + """Expand compressed blocks and compact the causal tail of the open group.""" + + if not block_indices.is_cuda or not HAS_TRITON: + raise RuntimeError("QSA CUDA expansion requires Triton") + if token_topk % compress_ratio: + raise ValueError("QSA token top-k must be divisible by compression ratio") + block_topk = token_topk // compress_ratio + output_width = token_topk + compress_ratio - 1 + if block_indices.shape != (query_positions.numel(), block_topk): + raise ValueError("QSA compressed top-k has an invalid shape") + if token_to_req.shape != query_positions.shape: + raise ValueError("QSA request mapping must match query positions") + if sequence_lengths.ndim != 1 or not sequence_lengths.shape[0]: + raise ValueError("QSA request sequence lengths must be nonempty") + if out is None: + out = torch.empty( + (block_indices.shape[0], output_width), + dtype=torch.int32, + device=block_indices.device, + ) + elif out.shape != (block_indices.shape[0], output_width): + raise ValueError("QSA expansion output has an invalid shape") + if not block_indices.shape[0]: + return out + column_block = 256 + _expand_qsa_indices_kernel[ + (block_indices.shape[0], triton.cdiv(output_width, column_block)) + ]( + block_indices, + query_positions, + sequence_lengths, + token_to_req, + out, + block_indices.stride(0), + block_indices.stride(1), + out.stride(0), + out.stride(1), + block_indices.shape[0], + sequence_lengths.shape[0], + BLOCK_TOPK=block_topk, + COMPRESS_RATIO=compress_ratio, + TOKEN_TOPK=token_topk, + OUTPUT_WIDTH=output_width, + COLUMN_BLOCK=column_block, + num_warps=4, + ) + return out + + +def qsa_select_paged_tokens( + q: torch.Tensor, + k_cache: torch.Tensor, + page_table: torch.Tensor, + token_to_req: torch.Tensor, + query_positions: torch.Tensor, + sequence_lengths: torch.Tensor, + token_topk: int, + compress_ratio: int, + out: torch.Tensor | None = None, +) -> torch.Tensor: + """Score, select, and expand QSA indices without host synchronization.""" + + rows = q.shape[0] + output_width = token_topk + compress_ratio - 1 + if out is None: + out = torch.empty((rows, output_width), dtype=torch.int32, device=q.device) + if out.shape != (rows, output_width): + raise ValueError("QSA selection output has an invalid shape") + if not rows: + return out + + columns = page_table.shape[1] * k_cache.shape[1] + block_topk = token_topk // compress_ratio + rows_per_chunk = max(1, _LOGITS_WORKSPACE_BYTES // max(columns * 4, 1)) + chunk_rows = min(rows, rows_per_chunk) + blocks_buffer = torch.empty( + (chunk_rows, block_topk), dtype=torch.int32, device=q.device + ) + topk_workspace = torch.empty( + (_TOPK_WORKSPACE_BYTES,), dtype=torch.uint8, device=q.device + ) + for row_start in range(0, rows, rows_per_chunk): + row_end = min(row_start + rows_per_chunk, rows) + row_slice = slice(row_start, row_end) + logits, visible_blocks = qsa_mqa_paged( + q[row_slice], + k_cache, + page_table, + token_to_req[row_slice], + query_positions[row_slice], + sequence_lengths, + compress_ratio, + ) + blocks = blocks_buffer[: row_end - row_start] + use_cooperative_topk = ( + blocks.shape[0] <= 32 + and logits.stride(0) % 4 == 0 + and current_platform.has_device_capability(90) + and not current_platform.is_device_capability_family(120) + ) + topk_op = ( + torch.ops._C.cooperative_topk + if use_cooperative_topk + else torch.ops._C.persistent_topk + ) + topk_op(logits, visible_blocks, blocks, topk_workspace, block_topk, columns) + expand_qsa_block_indices_cuda( + blocks, + query_positions[row_slice], + sequence_lengths, + token_to_req[row_slice], + compress_ratio, + token_topk, + out[row_slice], + ) + return out + + +def qsa_sparse_paged_attention( + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + logical_indices: torch.Tensor, + block_table: torch.Tensor, + token_to_req: torch.Tensor, + out: torch.Tensor | None = None, +) -> torch.Tensor: + """Run sparse GQA directly over paged BF16 K/V caches.""" + + if not q.is_cuda or not HAS_TRITON: + raise RuntimeError("paged QSA sparse attention requires CUDA and Triton") + if q.ndim != 3 or k_cache.ndim != 4 or v_cache.shape != k_cache.shape: + raise ValueError("QSA sparse attention received invalid Q/K/V shapes") + if logical_indices.ndim != 2 or logical_indices.shape[0] != q.shape[0]: + raise ValueError("QSA indices must have one row per query") + if token_to_req.shape != (q.shape[0],) or block_table.ndim != 2: + raise ValueError("QSA sparse attention metadata has invalid shapes") + if not all(k_cache.shape[:3]) or not all(block_table.shape): + raise ValueError("QSA sparse attention cache and block table must be nonempty") + if logical_indices.shape[1] <= 0: + raise ValueError("QSA sparse attention requires a positive selection width") + if q.shape[2] != k_cache.shape[3] or q.shape[1] % k_cache.shape[2]: + raise ValueError("QSA sparse attention requires valid grouped-query heads") + head_dim = q.shape[2] + assert head_dim >= 16 and (head_dim & (head_dim - 1)) == 0 + assert q.dtype == k_cache.dtype == v_cache.dtype == torch.bfloat16 + assert logical_indices.dtype == block_table.dtype == torch.int32 + assert token_to_req.dtype == torch.int32 + assert q.device == k_cache.device == v_cache.device + assert q.device == logical_indices.device == block_table.device + assert q.device == token_to_req.device + assert q.stride(2) == k_cache.stride(3) == v_cache.stride(3) == 1 + assert logical_indices.stride(1) == block_table.stride(1) == 1 + assert token_to_req.stride(0) == 1 + if out is None: + out = torch.empty_like(q) + if out.shape != q.shape: + raise ValueError("QSA sparse output must match its query") + assert out.dtype == q.dtype and out.device == q.device + assert out.stride(2) == 1 + if not q.shape[0]: + return out + + group_size = q.shape[1] // k_cache.shape[2] + block_m = triton.next_power_of_2(group_size) + base_programs = q.shape[0] * k_cache.shape[2] + small_profile_limit = 8 if block_m <= 8 else 4 + + # Tuned on GB300 for the Qwen-Air TP1, TP2, and TP4 attention shapes. + # Narrow tiles favor decode; wide tiles improve throughput for prefill. + if base_programs <= small_profile_limit: + block_n, target_splits, partial_warps = 16, 64, 4 + elif base_programs < 32: + block_n, target_splits, partial_warps = 16, 32, 4 + elif base_programs <= 256: + block_n, target_splits, partial_warps = 64, 8, 2 + elif base_programs <= 512: + block_n, target_splits, partial_warps = 64, 4, 2 + else: + block_n, target_splits, partial_warps = 64, 1, 2 + + num_tiles = triton.cdiv(logical_indices.shape[1], block_n) + # Avoid empty splits when the selection width is smaller than the profile. + max_useful_splits = 1 << (num_tiles.bit_length() - 1) + num_splits = min(max_useful_splits, target_splits) + + # Split=1 writes output directly and compiles out all workspace accesses. + if num_splits == 1: + partial_output = out + partial_lse = out + else: + # FP32 partials preserve accuracy when merging independently normalized + # splits. + partial_output = torch.empty( + (num_splits, *q.shape), dtype=torch.float32, device=q.device + ) + partial_lse = torch.empty( + (num_splits, q.shape[0], q.shape[1]), + dtype=torch.float32, + device=q.device, + ) + + partial_grid = (q.shape[0], k_cache.shape[2], num_splits) + _qsa_sparse_paged_gqa_splitk_kernel[partial_grid]( + q, + k_cache, + v_cache, + logical_indices, + block_table, + token_to_req, + partial_output, + partial_lse, + out, + q.stride(0), + q.stride(1), + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + v_cache.stride(0), + v_cache.stride(1), + v_cache.stride(2), + logical_indices.stride(0), + block_table.stride(0), + out.stride(0), + out.stride(1), + q.shape[0], + k_cache.shape[0], + block_table.shape[0], + TOPK=logical_indices.shape[1], + PAGE_SIZE=k_cache.shape[1], + PAGE_TABLE_WIDTH=block_table.shape[1], + GROUP_SIZE=group_size, + HEAD_DIM=q.shape[2], + NUM_QUERY_HEADS=q.shape[1], + NUM_SPLITS=num_splits, + NUM_TILES=num_tiles, + BLOCK_M=block_m, + BLOCK_N=block_n, + num_warps=partial_warps, + num_stages=2, + ) + if num_splits == 1: + return out + + _qsa_merge_splitk_kernel[(q.shape[0], q.shape[1])]( + partial_output, + partial_lse, + out, + out.stride(0), + out.stride(1), + q.shape[0], + HEAD_DIM=q.shape[2], + NUM_QUERY_HEADS=q.shape[1], + NUM_SPLITS=num_splits, + BLOCK_SPLITS=triton.next_power_of_2(num_splits), + num_warps=2, + num_stages=1, + ) + return out + + +def qsa_store_cache_rows( + cache: torch.Tensor, + slot_mapping: torch.Tensor, + rows: torch.Tensor, +) -> None: + """Store fixed-width rows in a QSA cache without boolean indexing.""" + + if not cache.is_cuda or not HAS_TRITON: + raise RuntimeError("QSA CUDA cache stores require Triton") + if cache.ndim != 4 or cache.shape[2] != 1: + raise ValueError("QSA cache must be [pages, page_size, 1, width]") + if not all(cache.shape): + raise ValueError("QSA cache dimensions must be nonzero") + if rows.ndim == 3: + if rows.shape[1] != 1: + raise ValueError("QSA cache rows must have one head") + rows = rows[:, 0] + if rows.shape != (slot_mapping.numel(), cache.shape[3]): + raise ValueError("QSA cache rows and slots have incompatible shapes") + if not rows.shape[0]: + return + _store_qsa_rows_kernel[(rows.shape[0],)]( + cache, + slot_mapping, + rows, + cache.stride(0), + cache.stride(1), + cache.stride(3), + rows.stride(0), + rows.stride(1), + rows.shape[0], + cache.shape[0], + PAGE_SIZE=cache.shape[1], + WIDTH=cache.shape[3], + BLOCK_D=triton.next_power_of_2(cache.shape[3]), + num_warps=4, + ) + + +def qsa_compress_groups_with_ratio( + raw_keys: torch.Tensor, # this step's raw key rows [rows, 1, head_size] + raw_positions: torch.Tensor, # this step's positions [rows, 1, 3] int64 + compressor_state_cache: torch.Tensor, + compressor_state_block_table: torch.Tensor, + token_to_req: torch.Tensor, + query_start_loc: torch.Tensor, + logical_positions: torch.Tensor, + compressed_slots: torch.Tensor, + compress_ratio: int, + rope_cache: torch.Tensor | None = None, +) -> tuple[torch.Tensor, torch.Tensor]: + """Pool completed groups from the compressor-state ring and raw token rows.""" + + if not raw_keys.is_cuda or not HAS_TRITON: + raise RuntimeError("QSA CUDA compression requires Triton") + rows = token_to_req.numel() + if compress_ratio <= 0: + raise ValueError("QSA compression ratio must be positive") + if raw_keys.ndim != 3 or raw_keys.shape[:2] != (rows, 1): + raise ValueError("QSA raw keys must be [rows, 1, head_size]") + if raw_positions.shape != (rows, 1, 3) or raw_positions.dtype != torch.int64: + raise ValueError("QSA raw positions must be [rows, 1, 3] int64") + if logical_positions.shape != (rows,) or compressed_slots.shape != (rows,): + raise ValueError("QSA compression metadata must match token rows") + if compressor_state_cache.ndim != 4 or compressor_state_cache.shape[2] != 1: + raise ValueError("QSA compressor-state cache has an invalid shape") + if ( + # The ring is wider than one group so speculative rows cannot alias + # onto the committed keys of the group still being collected. + compressor_state_cache.shape[1] < compress_ratio + or compressor_state_cache.shape[3] != raw_keys.shape[2] + or compressor_state_cache.dtype != raw_keys.dtype + ): + raise ValueError( + "QSA compressor-state cache does not match the compression layout" + ) + if ( + compressor_state_block_table.ndim != 2 + or compressor_state_block_table.shape[1] < 1 + ): + raise ValueError( + "QSA compressor-state block table must contain one block per request" + ) + if query_start_loc.ndim != 1 or query_start_loc.shape[0] < 2: + raise ValueError("QSA query starts must contain a terminal offset") + num_requests = query_start_loc.shape[0] - 1 + if compressor_state_block_table.shape[0] < num_requests: + raise ValueError("QSA compressor-state block table has too few request rows") + if rope_cache is not None and ( + rope_cache.ndim != 4 + or rope_cache.shape[:3] != compressor_state_cache.shape[:3] + or rope_cache.shape[3] != 3 + or rope_cache.dtype != torch.int64 + ): + raise ValueError("QSA packed position view has an invalid shape or dtype") + if rows and ( + not all(compressor_state_cache.shape) + or not all(compressor_state_block_table.shape) + ): + raise ValueError("QSA compressor-state cache and block table must be nonempty") + pooled = torch.empty( + (rows, 1, raw_keys.shape[2]), + dtype=raw_keys.dtype, + device=raw_keys.device, + ) + first_positions = torch.empty((rows, 3), dtype=torch.int64, device=raw_keys.device) + if not rows: + return pooled, first_positions + if rope_cache is None: + rope_cache = compressor_state_cache + load_rope_positions = False + else: + load_rope_positions = True + _compress_qsa_groups_kernel[(rows,)]( + raw_keys, + raw_positions, + compressor_state_cache, + rope_cache, + compressor_state_block_table, + token_to_req, + query_start_loc, + logical_positions, + compressed_slots, + pooled, + first_positions, + raw_keys.stride(0), + raw_keys.stride(2), + raw_positions.stride(0), + raw_positions.stride(2), + compressor_state_cache.stride(0), + compressor_state_cache.stride(1), + compressor_state_cache.stride(3), + rope_cache.stride(0), + rope_cache.stride(1), + rope_cache.stride(3), + compressor_state_block_table.stride(0), + pooled.stride(0), + pooled.stride(2), + first_positions.stride(0), + first_positions.stride(1), + rows, + compressor_state_cache.shape[0], + num_requests, + COMPRESSOR_STATE_SIZE=compressor_state_cache.shape[1], + COMPRESS_RATIO=compress_ratio, + HEAD_DIM=raw_keys.shape[2], + LOAD_ROPE_POSITIONS=load_rope_positions, + BLOCK_D=triton.next_power_of_2(raw_keys.shape[2]), + num_warps=4, + ) + return pooled, first_positions + + +__all__ = [ + "expand_qsa_block_indices_cuda", + "qsa_compress_groups_with_ratio", + "qsa_mqa_paged", + "qsa_select_paged_tokens", + "qsa_sparse_paged_attention", + "qsa_store_cache_rows", +] diff --git a/vllm/models/qwen4_exp/nvidia/ops/qsa_pre_indexer.py b/vllm/models/qwen4_exp/nvidia/ops/qsa_pre_indexer.py new file mode 100644 index 0000000000..d996f3977a --- /dev/null +++ b/vllm/models/qwen4_exp/nvidia/ops/qsa_pre_indexer.py @@ -0,0 +1,508 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Fused QSA pre-indexer kernel for Qwen4Exp.""" + +import torch + +from vllm.triton_utils import tl, triton + + +@triton.jit +def _norm_rope( + x, + pos_t, + pos_h, + pos_w, + cos_sin_ptr, + cos_sin_stride, + norm_weight_ptr, + eps, + IS_MROPE: tl.constexpr, + MROPE_H: tl.constexpr, + MROPE_W: tl.constexpr, +): + """Apply Gemma RMSNorm and selected-axis NeoX RoPE to register rows.""" + TILE_T: tl.constexpr = x.shape[0] + TILE_H: tl.constexpr = x.shape[1] + D: tl.constexpr = x.shape[2] + ROWS: tl.constexpr = TILE_T * TILE_H + HALF: tl.constexpr = D // 2 + QUARTER: tl.constexpr = D // 4 + pairs = tl.arange(0, QUARTER) + if IS_MROPE: + # Qwen interleaves temporal, height, and width rotary pairs. Each axis + # still indexes the same position-major cos/sin table. + h_mask = ((pairs % 3) == 1) & (pairs <= 3 * MROPE_H) + w_mask = ((pairs % 3) == 2) & (pairs <= 3 * MROPE_W) + t_mask = ~(h_mask | w_mask) + base = cos_sin_ptr + pairs[None, :] + pos_rows = (pos_t, pos_h, pos_w) + axis_masks = (t_mask, h_mask, w_mask) + cos = tl.zeros((TILE_T, QUARTER), dtype=cos_sin_ptr.dtype.element_ty) + sin = tl.zeros((TILE_T, QUARTER), dtype=cos_sin_ptr.dtype.element_ty) + for axis in tl.static_range(3): + cos += tl.load( + base + pos_rows[axis][:, None] * cos_sin_stride, + mask=axis_masks[axis][None, :], + other=0, + ) + sin += tl.load( + base + pos_rows[axis][:, None] * cos_sin_stride + QUARTER, + mask=axis_masks[axis][None, :], + other=0, + ) + else: + cos = tl.load(cos_sin_ptr + pos_t[:, None] * cos_sin_stride + pairs[None, :]) + sin = tl.load( + cos_sin_ptr + pos_t[:, None] * cos_sin_stride + QUARTER + pairs[None, :] + ) + + cos = tl.reshape( + tl.broadcast_to(cos[:, None, :], (TILE_T, TILE_H, QUARTER)), + (ROWS, QUARTER), + ) + sin = tl.reshape( + tl.broadcast_to(sin[:, None, :], (TILE_T, TILE_H, QUARTER)), + (ROWS, QUARTER), + ) + x = tl.reshape(x, (ROWS, D)).to(tl.float32) + weight = tl.load(norm_weight_ptr + tl.arange(0, D)).to(tl.float32) + 1.0 + rrms = tl.rsqrt(tl.sum(x * x, axis=1) / D + eps) + y = (x * rrms[:, None] * weight[None, :]).to(cos.dtype) + rotated, passthrough = tl.split( + tl.permute(tl.reshape(y, (ROWS, 2, HALF)), (0, 2, 1)) + ) + r0, r1 = tl.split(tl.permute(tl.reshape(rotated, (ROWS, 2, QUARTER)), (0, 2, 1))) + out0 = r0 * cos - r1 * sin + out1 = r1 * cos + r0 * sin + rotated = tl.reshape(tl.permute(tl.join(out0, out1), (0, 2, 1)), (ROWS, HALF)) + result = tl.reshape(tl.permute(tl.join(rotated, passthrough), (0, 2, 1)), (ROWS, D)) + return tl.reshape(result, (TILE_T, TILE_H, D)) + + +@triton.jit( + do_not_specialize=[ + "num_tokens", + "num_state_blocks", + "num_compressed_blocks", + "num_k_work", + ] +) +def _qsa_pre_indexer_kernel( + q_ptr, + q_stride_token, + k_ptr, + k_stride_token, + pos_ptr, + pos_stride_axis, + pos_stride_token, + cos_sin_ptr, + q_norm_weight_ptr, + k_norm_weight_ptr, + eps, + q_out_ptr, + q_out_stride_token, + q_out_stride_head, + state_cache_ptr, + state_cache_stride_block, + state_cache_stride_token, + state_slots_ptr, + state_table_ptr, + state_table_stride_req, + query_start_loc_ptr, + logical_positions_ptr, + compressed_slots_ptr, + k_work_metadata_ptr, + compressed_cache_ptr, + compressed_cache_stride_block, + compressed_cache_stride_token, + num_tokens, + num_state_blocks, + num_compressed_blocks, + num_k_work, + HQ: tl.constexpr, + D: tl.constexpr, + TILE_T_Q: tl.constexpr, + TILE_H_Q: tl.constexpr, + COMPRESS_RATIO: tl.constexpr, + STATE_SIZE: tl.constexpr, + COMP_PAGE_SIZE: tl.constexpr, + IS_2D_POSITIONS: tl.constexpr, + IS_K_MROPE: tl.constexpr, + CACHE_HAS_ROPE_POS: tl.constexpr, + MROPE_H: tl.constexpr, + MROPE_W: tl.constexpr, +): + pid = tl.program_id(0) + # K work occupies the first programs; the remaining programs tile Q. This + # keeps both paths in one launch while leaving their register shapes + # independent. + if pid >= num_k_work: + q_pid = pid - num_k_work + num_head_tiles: tl.constexpr = tl.cdiv(HQ, TILE_H_Q) + token_tile = q_pid // num_head_tiles + head_tile = q_pid % num_head_tiles + tokens = token_tile * TILE_T_Q + tl.arange(0, TILE_T_Q) + heads = head_tile * TILE_H_Q + tl.arange(0, TILE_H_Q) + valid_tokens = tokens < num_tokens + valid_heads = heads < HQ + dims = tl.arange(0, D) + mask = valid_tokens[:, None, None] & valid_heads[None, :, None] + x = tl.load( + q_ptr + + tokens[:, None, None] * q_stride_token + + heads[None, :, None] * D + + dims[None, None, :], + mask=mask, + other=0.0, + ) + pos_t = tl.load(pos_ptr + tokens * pos_stride_token, mask=valid_tokens, other=0) + if IS_2D_POSITIONS: + pos_h = tl.load( + pos_ptr + pos_stride_axis + tokens * pos_stride_token, + mask=valid_tokens, + other=0, + ) + pos_w = tl.load( + pos_ptr + 2 * pos_stride_axis + tokens * pos_stride_token, + mask=valid_tokens, + other=0, + ) + else: + pos_h = pos_t + pos_w = pos_t + y = _norm_rope( + x, + pos_t, + pos_h, + pos_w, + cos_sin_ptr, + D // 2, + q_norm_weight_ptr, + eps, + IS_2D_POSITIONS, + MROPE_H, + MROPE_W, + ) + tl.store( + q_out_ptr + + tokens[:, None, None] * q_out_stride_token + + heads[None, :, None] * q_out_stride_head + + dims[None, None, :], + y, + mask=mask, + ) + + if pid < num_k_work: + # One work item owns one completed compression group. Work item zero + # additionally commits the request's current raw-K suffix below. + work_metadata = tl.load(k_work_metadata_ptr + pid * 2 + tl.arange(0, 2)) + request, work_in_request = tl.split(work_metadata) + if request < 0: + return + + query_start = tl.load(query_start_loc_ptr + request) + query_end = tl.load(query_start_loc_ptr + request + 1) + query_len = query_end - query_start + chunk_end = tl.load(logical_positions_ptr + query_end - 1) + chunk_start = chunk_end - query_len + 1 + num_groups = (chunk_end + 1) // COMPRESS_RATIO - chunk_start // COMPRESS_RATIO + dims = tl.arange(0, D) + + if work_in_request < num_groups: + first_boundary = ( + (chunk_start + COMPRESS_RATIO) // COMPRESS_RATIO + ) * COMPRESS_RATIO - 1 + end_position = first_boundary + work_in_request * COMPRESS_RATIO + boundary_token = query_start + end_position - chunk_start + valid_token = ( + (boundary_token >= query_start) + & (boundary_token < query_end) + & (boundary_token < num_tokens) + ) + compressed_slot = tl.load( + compressed_slots_ptr + boundary_token, + mask=valid_token, + other=-1, + ) + valid = ( + valid_token + & (compressed_slot >= 0) + & (compressed_slot < num_compressed_blocks * COMP_PAGE_SIZE) + ) + state_block = tl.load(state_table_ptr + request * state_table_stride_req) + state_block_valid = (state_block >= 0) & (state_block < num_state_blocks) + safe_state_block = tl.maximum(state_block, 0).to(tl.int64) + group_offsets = tl.arange(0, COMPRESS_RATIO) + source_positions = end_position - (COMPRESS_RATIO - 1) + group_offsets + source_in_chunk = source_positions >= chunk_start + source_tokens = query_start + source_positions - chunk_start + source_tokens_valid = ( + (source_tokens >= query_start) + & (source_tokens < query_end) + & (source_tokens < num_tokens) + ) + current_base = ( + k_ptr + tl.maximum(source_tokens, 0)[:, None] * k_stride_token + ) + cached_base = ( + state_cache_ptr + + safe_state_block * state_cache_stride_block + + (source_positions % STATE_SIZE)[:, None] * state_cache_stride_token + ) + # Only the first completed group can cross the chunk boundary. Select + # historical rows from the ring without issuing two masked loads. + source_base = tl.where(source_in_chunk[:, None], current_base, cached_base) + # Pointer selection obscures alignment from Triton's analysis. + source_base = tl.multiple_of(source_base, (8, 8)) + source_valid = tl.where( + source_in_chunk, source_tokens_valid, state_block_valid + ) + source = tl.load( + source_base + dims[None, :], + mask=valid & source_valid[:, None], + other=0.0, + ).to(tl.float32) + # Match the unfused path's BF16 pooled tensor before RMSNorm. + pooled = ( + (tl.sum(source, axis=0) / COMPRESS_RATIO).to(tl.bfloat16).to(tl.float32) + ) + + first_position = end_position - (COMPRESS_RATIO - 1) + if CACHE_HAS_ROPE_POS: + # RoPE uses the first token in the pooled group. Its exact MRoPE + # coordinates may live in this chunk or the raw-state ring. + first_in_chunk = first_position >= chunk_start + first_token = query_start + first_position - chunk_start + first_token_valid = ( + (first_token >= query_start) + & (first_token < query_end) + & (first_token < num_tokens) + ) + safe_first_token = tl.maximum(first_token, 0) + load_current_position = first_in_chunk & first_token_valid + first_pos_t = tl.load( + pos_ptr + safe_first_token * pos_stride_token, + mask=load_current_position, + other=0, + ) + if IS_2D_POSITIONS: + first_pos_h = tl.load( + pos_ptr + pos_stride_axis + safe_first_token * pos_stride_token, + mask=load_current_position, + other=0, + ) + first_pos_w = tl.load( + pos_ptr + + 2 * pos_stride_axis + + safe_first_token * pos_stride_token, + mask=load_current_position, + other=0, + ) + else: + first_pos_h = first_pos_t + first_pos_w = first_pos_t + tail = ( + state_cache_ptr + + safe_state_block * state_cache_stride_block + + (first_position % STATE_SIZE) * state_cache_stride_token + + D + ).to(tl.pointer_type(tl.int64)) + load_cached_position = ~first_in_chunk & state_block_valid + cached_pos_t = tl.load(tail, mask=load_cached_position, other=0) + cached_pos_h = tl.load(tail + 1, mask=load_cached_position, other=0) + cached_pos_w = tl.load(tail + 2, mask=load_cached_position, other=0) + pos_t = tl.where(first_in_chunk, first_pos_t.to(tl.int64), cached_pos_t) + pos_h = tl.where(first_in_chunk, first_pos_h.to(tl.int64), cached_pos_h) + pos_w = tl.where(first_in_chunk, first_pos_w.to(tl.int64), cached_pos_w) + else: + pos_t = first_position + pos_h = first_position + pos_w = first_position + y = _norm_rope( + tl.reshape(pooled, (1, 1, D)), + pos_t + tl.arange(0, 1), + pos_h + tl.arange(0, 1), + pos_w + tl.arange(0, 1), + cos_sin_ptr, + D // 2, + k_norm_weight_ptr, + eps, + IS_K_MROPE, + MROPE_H, + MROPE_W, + ) + compressed_block = (compressed_slot // COMP_PAGE_SIZE).to(tl.int64) + compressed_row = compressed_slot % COMP_PAGE_SIZE + tl.store( + compressed_cache_ptr + + compressed_block * compressed_cache_stride_block + + compressed_row * compressed_cache_stride_token + + dims, + tl.reshape(y, (D,)), + mask=valid, + ) + + if work_in_request == 0: + # This CTA may have just read historical rows from the circular buffer. + # Keep every lane past those loads before overwriting the same ring. + tl.debug_barrier() + # One CTA per request commits only the suffix retained by the ring. + num_state_rows = tl.minimum(query_len, STATE_SIZE) + for state_offset in tl.range(0, num_state_rows): + token = query_end - num_state_rows + state_offset + valid_token = ( + (token >= query_start) & (token < query_end) & (token < num_tokens) + ) + slot = tl.load(state_slots_ptr + token, mask=valid_token, other=-1) + valid_slot = ( + valid_token & (slot >= 0) & (slot < num_state_blocks * STATE_SIZE) + ) + safe_slot = tl.maximum(slot, 0) + state_row = ( + state_cache_ptr + + (safe_slot // STATE_SIZE).to(tl.int64) * state_cache_stride_block + + (safe_slot % STATE_SIZE) * state_cache_stride_token + ) + k = tl.load( + k_ptr + tl.maximum(token, 0) * k_stride_token + dims, + mask=valid_slot, + other=0.0, + ) + tl.store(state_row + dims, k, mask=valid_slot) + if CACHE_HAS_ROPE_POS: + pos_t = tl.load( + pos_ptr + tl.maximum(token, 0) * pos_stride_token, + mask=valid_slot, + other=0, + ) + if IS_2D_POSITIONS: + pos_h = tl.load( + pos_ptr + + pos_stride_axis + + tl.maximum(token, 0) * pos_stride_token, + mask=valid_slot, + other=0, + ) + pos_w = tl.load( + pos_ptr + + 2 * pos_stride_axis + + tl.maximum(token, 0) * pos_stride_token, + mask=valid_slot, + other=0, + ) + else: + pos_h = pos_t + pos_w = pos_t + tail = (state_row + D).to(tl.pointer_type(tl.int64)) + tl.store(tail, pos_t.to(tl.int64), mask=valid_slot) + tl.store(tail + 1, pos_h.to(tl.int64), mask=valid_slot) + tl.store(tail + 2, pos_w.to(tl.int64), mask=valid_slot) + + +def qsa_pre_indexer( + q: torch.Tensor, + k: torch.Tensor, + positions: torch.Tensor, + cos_sin_cache: torch.Tensor, + q_norm_weight: torch.Tensor, + k_norm_weight: torch.Tensor, + eps: float, + q_out: torch.Tensor, + state_cache: torch.Tensor, + state_slots: torch.Tensor, + state_block_table: torch.Tensor, + query_start_loc: torch.Tensor, + logical_positions: torch.Tensor, + compressed_cache: torch.Tensor, + compressed_slots: torch.Tensor, + k_work_metadata: torch.Tensor, + *, + compress_ratio: int, + mrope_section: tuple[int, int, int] | None, + rope_pos_offset: int | None, +) -> None: + """Normalize Q, compress K, then update the circular raw state.""" + num_tokens = q.shape[0] + if num_tokens == 0: + return + num_q_heads, head_dim = q_out.shape[1:] + assert cos_sin_cache.shape[-1] * 2 == head_dim + assert q.shape == (num_tokens, num_q_heads * head_dim) + assert k.shape == (num_tokens, head_dim) + assert q.stride(-1) == 1 + assert k.stride(-1) == 1 + assert q_out.stride(-1) == 1 + assert cos_sin_cache.is_contiguous() + assert state_cache.stride(-1) == 1 + assert compressed_cache.stride(-1) == 1 + assert k_work_metadata.ndim == 2 and k_work_metadata.shape[1] == 2 + is_2d_positions = positions.ndim == 2 + is_k_mrope = bool(mrope_section) + cache_has_rope_pos = rope_pos_offset is not None + assert rope_pos_offset is None or rope_pos_offset == head_dim + if is_2d_positions: + assert positions.shape == (3, num_tokens) + assert is_k_mrope + pos_stride_axis, pos_stride_token = positions.stride() + else: + assert positions.shape == (num_tokens,) + pos_stride_axis, pos_stride_token = 0, positions.stride(0) + section = mrope_section if mrope_section is not None else (0, 0, 0) + assert len(section) == 3 + + if num_tokens <= 4096: + TILE_T_Q, TILE_H_Q = 2, 2 + else: + TILE_T_Q, TILE_H_Q = 2, 4 + num_k_work = k_work_metadata.shape[0] + num_q_work = triton.cdiv(num_tokens, TILE_T_Q) * triton.cdiv(num_q_heads, TILE_H_Q) + _qsa_pre_indexer_kernel[(num_k_work + num_q_work,)]( + q, + q.stride(0), + k, + k.stride(0), + positions, + pos_stride_axis, + pos_stride_token, + cos_sin_cache, + q_norm_weight, + k_norm_weight, + eps, + q_out, + q_out.stride(0), + q_out.stride(1), + state_cache, + state_cache.stride(0), + state_cache.stride(1), + state_slots, + state_block_table, + state_block_table.stride(0), + query_start_loc, + logical_positions, + compressed_slots, + k_work_metadata, + compressed_cache, + compressed_cache.stride(0), + compressed_cache.stride(1), + num_tokens, + state_cache.shape[0], + compressed_cache.shape[0], + num_k_work, + HQ=num_q_heads, + D=head_dim, + TILE_T_Q=TILE_T_Q, + TILE_H_Q=TILE_H_Q, + COMPRESS_RATIO=compress_ratio, + STATE_SIZE=state_cache.shape[1], + COMP_PAGE_SIZE=compressed_cache.shape[1], + IS_2D_POSITIONS=is_2d_positions, + IS_K_MROPE=is_k_mrope, + CACHE_HAS_ROPE_POS=cache_has_rope_pos, + MROPE_H=section[1], + MROPE_W=section[2], + num_warps=1, + ) + + +__all__ = ["qsa_pre_indexer"] diff --git a/vllm/models/qwen4_exp/nvidia/ple_layer.py b/vllm/models/qwen4_exp/nvidia/ple_layer.py new file mode 100644 index 0000000000..09e8e32499 --- /dev/null +++ b/vllm/models/qwen4_exp/nvidia/ple_layer.py @@ -0,0 +1,1256 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""GPU-resident Qwen4Exp position-learning enhancement layers.""" + +import math +from collections.abc import Iterable, Sequence + +import torch +import torch.nn.functional as F +from torch import nn + +from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config +from vllm.forward_context import get_forward_context +from vllm.model_executor.layers.linear import ReplicatedLinear +from vllm.model_executor.layers.mamba.abstract import MambaBase +from vllm.model_executor.layers.mamba.mamba_utils import ( + MambaStateDtypeCalculator, + MambaStateShapeCalculator, + is_conv_state_dim_first, +) +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig, + QuantizeMethodBase, +) +from vllm.model_executor.layers.quantization.fp8 import Fp8Config +from vllm.model_executor.layers.quantization.utils.fp8_utils import ( + create_fp8_scale_parameter, + create_fp8_weight_parameter, + is_fp8, +) +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + is_layer_skipped, +) +from vllm.model_executor.models.utils import AutoWeightsLoader +from vllm.model_executor.parameter import PerTensorScaleParameter +from vllm.transformers_utils.configs.qwen4_exp import ( + Qwen4ExpTextConfig, +) +from vllm.utils.torch_utils import direct_register_custom_op +from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum +from vllm.v1.attention.backends.short_conv_attn import ( + PleShortConvAttentionBackend, + PleShortConvAttentionMetadata, +) +from vllm.v1.attention.backends.utils import NULL_BLOCK_ID + +from ..common.ple import PLEVocabParallelEmbedding + + +class Qwen4ExpPLEGroupedNorm(nn.Module): + def __init__( + self, + hidden_size: int, + eps: float, + group_size: int | None, + dtype: torch.dtype | None, + ) -> None: + super().__init__() + if group_size is not None and hidden_size % group_size: + raise ValueError( + f"hidden_size ({hidden_size}) must be divisible by " + f"group_size ({group_size})" + ) + self.eps = eps + self.group_size = group_size + self.weight = nn.Parameter(torch.zeros(hidden_size, dtype=dtype)) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + input_dtype = hidden_states.dtype + hidden_states = hidden_states.float() + if self.group_size is None: + variance = hidden_states.square().mean(dim=-1, keepdim=True) + normalized = hidden_states * torch.rsqrt(variance + self.eps) + else: + grouped = hidden_states.unflatten( + -1, (hidden_states.shape[-1] // self.group_size, self.group_size) + ) + variance = grouped.square().mean(dim=-1, keepdim=True) + normalized = (grouped * torch.rsqrt(variance + self.eps)).flatten(-2) + return (normalized * (1.0 + self.weight.float())).to(input_dtype) + + +class Qwen4ExpPLEFp8EmbeddingMethod(QuantizeMethodBase): + """FP8 PLE embedding with one global checkpoint scale.""" + + def create_weights( + self, + layer: nn.Module, + input_size_per_partition: int, + output_partition_sizes: list[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ) -> None: + del input_size, output_size, params_dtype + weight_loader = extra_weight_attrs.get("weight_loader") + weight = create_fp8_weight_parameter( + sum(output_partition_sizes), input_size_per_partition, weight_loader + ) + layer.register_parameter("weight", weight) + + weight_scale = create_fp8_scale_parameter( + PerTensorScaleParameter, + output_partition_sizes, + input_size_per_partition, + None, + weight_loader, + scale_dtype=torch.bfloat16, + ) + layer.register_parameter("weight_scale", weight_scale) + + def apply( + self, + layer: nn.Module, + x: torch.Tensor, + bias: torch.Tensor | None = None, + ) -> torch.Tensor: + raise NotImplementedError("PLE FP8 weights only support embedding lookup") + + def embedding(self, layer: nn.Module, input_: torch.Tensor) -> torch.Tensor: + return F.embedding(input_, layer.weight) + + +def _get_ple_embedding_quant_method( + quant_config: QuantizationConfig | None, + prefix: str, +) -> QuantizeMethodBase | None: + """Select global-scale FP8 only for quantized PLE checkpoint shards.""" + + if not isinstance(quant_config, Fp8Config): + return None + if not quant_config.is_checkpoint_fp8_serialized: + return None + + ignored_layers = quant_config.ignored_layers + if is_layer_skipped( + prefix, + ignored_layers, + quant_config.packed_modules_mapping, + match_mode=quant_config.ignored_layers_match_mode, + ): + return None + # PLE checkpoint shards form one runtime embedding parameter. + shard_prefix = f"{prefix}.shard_" + if any(name.startswith(shard_prefix) for name in ignored_layers): + return None + return Qwen4ExpPLEFp8EmbeddingMethod() + + +class Qwen4ExpNGramEmbedding(nn.Module): + _MASK64 = (1 << 64) - 1 + _SPLITMIX_GAMMA = 0x9E3779B97F4A7C15 + _SPLITMIX_M1 = 0xBF58476D1CE4E5B9 + _SPLITMIX_M2 = 0x94D049BB133111EB + _PLE_LAYER_PRIME = 10007 + + @classmethod + def _splitmix64(cls, value: int) -> int: + """Mix an integer into a deterministic unsigned 64-bit value.""" + value = (value + cls._SPLITMIX_GAMMA) & cls._MASK64 + value = ((value ^ (value >> 30)) * cls._SPLITMIX_M1) & cls._MASK64 + value = ((value ^ (value >> 27)) * cls._SPLITMIX_M2) & cls._MASK64 + return (value ^ (value >> 31)) & cls._MASK64 + + @staticmethod + def _is_prime_64(value: int) -> bool: + """Return whether a 64-bit integer is prime.""" + if value < 2: + return False + for prime in (2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37): + if value % prime == 0: + return value == prime + exponent = value - 1 + shifts = 0 + while exponent % 2 == 0: + exponent //= 2 + shifts += 1 + for base in (2, 325, 9375, 28178, 450775, 9780504, 1795265022): + if base % value == 0: + continue + witness = pow(base, exponent, value) + if witness in (1, value - 1): + continue + for _ in range(shifts - 1): + witness = pow(witness, 2, value) + if witness == value - 1: + break + else: + return False + return True + + @classmethod + def _nth_prime_after(cls, start: int, count: int) -> int: + """Return the ``count``-th prime strictly greater than ``start``.""" + prime = int(start) + for _ in range(count): + candidate = prime + 1 + if candidate <= 2: + prime = 2 + continue + if candidate % 2 == 0: + candidate += 1 + while not cls._is_prime_64(candidate): + candidate += 2 + prime = candidate + return prime + + @classmethod + def _make_layer_multipliers( + cls, + *, + ngram_size: int, + unigram_vocab_size: int, + seed: int, + ple_dense_layer_id: int, + ) -> list[int]: + """Build deterministic hash multipliers for one PLE layer.""" + max_multiplier = ((1 << 63) - 1) // unigram_vocab_size + half_bound = max(1, max_multiplier // 2) + base_seed = seed + cls._PLE_LAYER_PRIME * ple_dense_layer_id + multipliers = [] + for index in range(ngram_size): + value = base_seed + cls._SPLITMIX_GAMMA * (index + 1) + multipliers.append(2 * (cls._splitmix64(value) % half_bound) + 1) + return multipliers + + @classmethod + def _make_vocab_layout( + cls, + *, + ngram_vocab_size_base: int, + ngram_heads: int, + ple_dense_layer_id: int, + ) -> tuple[list[int], list[int], int]: + """Build per-head vocabulary sizes, offsets, and total row count.""" + sizes: list[int] = [] + offsets: list[int] = [] + offset = 0 + for local_head in range(ngram_heads): + global_head = ple_dense_layer_id * ngram_heads + local_head + size = cls._nth_prime_after(ngram_vocab_size_base - 1, global_head + 1) + sizes.append(size) + offsets.append(offset) + offset += size + return sizes, offsets, offset + + def __init__( + self, + config: Qwen4ExpTextConfig, + embedding_dim: int, + ple_dense_layer_id: int, + max_total_tokens: int, + max_num_reqs: int, + prefix: str, + layer_name: str, + quant_config: QuantizationConfig | None = None, + params_dtype: torch.dtype | None = None, + ) -> None: + super().__init__() + self.layer_name = layer_name + self.embedding_dim = embedding_dim + self.ngram_size = int(config.ngram_size) + self.heads_per_ngram = int(config.heads_per_ngram) + self.ngram_heads = (self.ngram_size - 1) * self.heads_per_ngram + if self.ngram_size < 2: + raise ValueError(f"ngram_size must be >= 2, got {self.ngram_size}") + if self.heads_per_ngram <= 0: + raise ValueError(f"heads_per_ngram must be > 0, got {self.heads_per_ngram}") + if embedding_dim % self.ngram_heads: + raise ValueError( + "ple_embed_dim must be divisible by total ngram heads: " + f"{embedding_dim} % {self.ngram_heads} != 0" + ) + self.head_dim = embedding_dim // self.ngram_heads + self.eos_token_id = int(config.eos_token_id) + self.unigram_vocab_size = int(config.vocab_size) + self.split_ngram_parts = int(getattr(config, "split_ngram_parts", 512)) + if self.split_ngram_parts <= 0: + raise ValueError("split_ngram_parts must be positive") + + multipliers = self._make_layer_multipliers( + ngram_size=self.ngram_size, + unigram_vocab_size=self.unigram_vocab_size, + seed=int(getattr(config, "seed", 1234)), + ple_dense_layer_id=ple_dense_layer_id, + ) + self.register_buffer( + "layer_multipliers", + torch.tensor(multipliers, dtype=torch.long), + persistent=True, + ) + + sizes, offsets, total_vocab_size = self._make_vocab_layout( + ngram_vocab_size_base=int(config.ngram_vocab_size_base), + ngram_heads=self.ngram_heads, + ple_dense_layer_id=ple_dense_layer_id, + ) + self.register_buffer( + "ngram_heads_vocab_sizes", + torch.tensor(sizes, dtype=torch.long), + persistent=True, + ) + self.register_buffer( + "ngram_heads_offsets", + torch.tensor(offsets, dtype=torch.long), + persistent=True, + ) + divisor = int(config.make_ngram_vocab_size_divisible_by) + padded_vocab_size = ((total_vocab_size + divisor - 1) // divisor) * divisor + self.ngram_embedding = PLEVocabParallelEmbedding( + padded_vocab_size, + self.head_dim, + params_dtype=params_dtype, + padding_size=divisor, + prefix=f"{prefix}.ngram_embedding", + quant_method=_get_ple_embedding_quant_method( + quant_config, f"{prefix}.ngram_embedding" + ), + ) + self.register_buffer( + "positions_buffer", + torch.arange(max_total_tokens, dtype=torch.int64), + persistent=False, + ) + self.register_buffer( + "padded_buffer", + torch.full( + (max_num_reqs, max_total_tokens), + self.eos_token_id, + dtype=torch.int64, + ), + persistent=False, + ) + + @staticmethod + def _shift_precompute( + tokens: torch.Tensor, eos_token_id: int + ) -> tuple[torch.Tensor, torch.Tensor]: + if tokens.dim() != 2: + raise ValueError("tokens must be a 2D tensor") + batch_size, seq_len = tokens.shape + positions = torch.arange(seq_len, device=tokens.device, dtype=torch.int64) + eos_positions = torch.where(tokens == eos_token_id, positions, -1) + previous_eos_inclusive = torch.cummax(eos_positions, dim=1).values + previous_eos = torch.cat( + [ + eos_positions.new_full((batch_size, 1), -1), + previous_eos_inclusive[:, :-1], + ], + dim=1, + ) + return positions, positions.unsqueeze(0) - previous_eos - 1 + + @staticmethod + def _shift_apply( + tokens: torch.Tensor, + positions: torch.Tensor, + position_in_segment: torch.Tensor, + shift: int, + eos_token_id: int, + ) -> torch.Tensor: + if shift == 0: + return tokens + source = positions - shift + gather_indices = source.clamp_min(0).unsqueeze(0).expand(tokens.shape[0], -1) + shifted = tokens.gather(1, gather_indices) + valid = (source.unsqueeze(0) >= 0) & (position_in_segment >= shift) + return torch.where(valid, shifted, tokens.new_full((), eos_token_id)) + + def compute_ngram_ids( + self, + input_ids: torch.Tensor, + query_start_loc: torch.Tensor, + ngram_context: torch.Tensor, + ) -> torch.Tensor: + """Compute n-gram embedding indices for the current request layout.""" + input_ids = input_ids.reshape(-1).long() + query_start_loc = query_start_loc.long() + num_reqs = query_start_loc.numel() - 1 + num_tokens = input_ids.shape[0] + if num_tokens > self.positions_buffer.numel(): + raise ValueError( + f"PLE received {num_tokens} tokens, but its workspace supports " + f"at most {self.positions_buffer.numel()}" + ) + if num_reqs > self.padded_buffer.shape[0]: + raise ValueError( + f"PLE received {num_reqs} requests, but its workspace supports " + f"at most {self.padded_buffer.shape[0]}" + ) + + positions = self.positions_buffer[:num_tokens] + packed = self.padded_buffer[:num_reqs, :num_tokens] + packed.fill_(self.eos_token_id) + request_indices = torch.searchsorted(query_start_loc, positions, right=True) - 1 + request_indices.clamp_(max=num_reqs - 1) + columns = (positions - query_start_loc[request_indices]).clamp( + 0, packed.shape[1] - 1 + ) + packed[request_indices, columns] = input_ids + ngram_context = ngram_context[:num_reqs].to( + device=input_ids.device, dtype=torch.long + ) + + context = torch.cat([ngram_context, packed], dim=-1) + positions_2d, position_in_segment = self._shift_precompute( + context, self.eos_token_id + ) + shifted = [context] + for shift in range(1, self.ngram_size): + shifted.append( + self._shift_apply( + context, + positions_2d, + position_in_segment, + shift, + self.eos_token_id, + ) + ) + adjusted_columns = columns + self.ngram_size - 1 + id_blocks = [] + for ngram in range(2, self.ngram_size + 1): + start = (ngram - 2) * self.heads_per_ngram + end = start + self.heads_per_ngram + mixed = shifted[0] * self.layer_multipliers[0] + for index in range(1, ngram): + mixed = torch.bitwise_xor( + mixed, shifted[index] * self.layer_multipliers[index] + ) + sizes = self.ngram_heads_vocab_sizes[start:end] + offsets = self.ngram_heads_offsets[start:end] + ids = torch.remainder(mixed.unsqueeze(-1), sizes) + offsets + id_blocks.append(ids[request_indices, adjusted_columns]) + return torch.cat(id_blocks, dim=-1) + + def forward( + self, + input_ids: torch.Tensor, + query_start_loc: torch.Tensor, + ngram_context: torch.Tensor, + ) -> torch.Tensor: + ngram_ids = input_ids.new_empty( + (input_ids.shape[0], self.ngram_heads), + dtype=torch.long, + ) + # Keep num_reqs-dependent ID generation outside PIECEWISE CUDA graphs, + # which dispatch only on the padded token count. + torch.ops.vllm.qwen4_exp_compute_ple_ngram_ids( + input_ids, + query_start_loc, + ngram_context, + ngram_ids, + self.layer_name, + ) + return self.ngram_embedding(ngram_ids).flatten(-2) + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + """Load hash buffers and checkpoint-split embedding rows.""" + + persistent_buffers = { + "layer_multipliers": self.layer_multipliers, + "ngram_heads_offsets": self.ngram_heads_offsets, + "ngram_heads_vocab_sizes": self.ngram_heads_vocab_sizes, + } + loaded: set[str] = set() + regular_weights: list[tuple[str, torch.Tensor]] = [] + shard_prefix = "ngram_embedding.shard_" + + for name, loaded_weight in weights: + leaf_name = name.rsplit(".", 1)[-1] + if leaf_name.startswith("hashstats_") or leaf_name == "token_lookup": + continue + if name in persistent_buffers: + buffer = persistent_buffers[name] + if buffer.shape != loaded_weight.shape: + raise ValueError( + f"Shape mismatch for {name}: expected " + f"{tuple(buffer.shape)}, got {tuple(loaded_weight.shape)}" + ) + buffer.copy_(loaded_weight.to(device=buffer.device, dtype=buffer.dtype)) + loaded.add(name) + continue + if name.startswith(shard_prefix) and name.endswith(".weight"): + shard_text = name[len(shard_prefix) : -len(".weight")] + if not shard_text.isdigit(): + regular_weights.append((name, loaded_weight)) + continue + shard_index = int(shard_text) + if shard_index >= self.split_ngram_parts: + raise ValueError( + f"PLE embedding shard index {shard_index} exceeds " + f"split_ngram_parts={self.split_ngram_parts}" + ) + embedding = self.ngram_embedding + shard_size = ( + embedding.org_vocab_size + self.split_ngram_parts - 1 + ) // self.split_ngram_parts + checkpoint_start = shard_index * shard_size + expected_rows = max( + 0, + min(shard_size, embedding.org_vocab_size - checkpoint_start), + ) + expected_shape = (expected_rows, embedding.embedding_dim) + if tuple(loaded_weight.shape) != expected_shape: + raise ValueError( + f"Shape mismatch for PLE embedding shard {shard_index}: " + f"expected {expected_shape}, got " + f"{tuple(loaded_weight.shape)}" + ) + embedding.weight.weight_loader( + embedding.weight, + loaded_weight, + checkpoint_start=checkpoint_start, + ) + loaded.add("ngram_embedding.weight") + continue + regular_weights.append((name, loaded_weight)) + + if regular_weights: + loaded.update(AutoWeightsLoader(self).load_weights(regular_weights)) + return loaded + + +class Qwen4ExpPLELayer(nn.Module, MambaBase): + def __init__( + self, + config: Qwen4ExpTextConfig, + vllm_config: VllmConfig, + layer_idx: int = 0, + ple_dense_layer_id: int | None = None, + prefix: str = "", + ) -> None: + super().__init__() + model_config = vllm_config.model_config + cache_config = vllm_config.cache_config + quant_config = vllm_config.quant_config + self.model_config: ModelConfig = model_config + self.cache_config: CacheConfig = cache_config + self.layer_idx = layer_idx + self.ple_dense_layer_id = ( + int(ple_dense_layer_id) + if ple_dense_layer_id is not None + else int(layer_idx) + ) + self.prefix = prefix + self.hidden_size = int(config.hidden_size) + self.hc_count = config.hc_count + self.hc_hidden_size = self.hidden_size * self.hc_count + self.conv_kernel_size = int(config.ple_conv_kernel_size) + self.short_conv_dilation = int(config.ngram_size) + self.conv_state_len = (self.conv_kernel_size - 1) * self.short_conv_dilation + self.num_spec_tokens = vllm_config.num_speculative_tokens + self.activation = "silu" + self.ple_embedding = Qwen4ExpNGramEmbedding( + config, + int(config.ple_embed_dim), + self.ple_dense_layer_id, + vllm_config.scheduler_config.max_num_batched_tokens, + vllm_config.scheduler_config.max_num_seqs, + prefix=f"{prefix}.ple_embedding", + layer_name=prefix, + quant_config=quant_config, + params_dtype=model_config.dtype, + ) + self.key_proj = ReplicatedLinear( + int(config.ple_embed_dim), + self.hc_hidden_size, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.key_proj", + ) + self.value_proj = ReplicatedLinear( + int(config.ple_embed_dim), + self.hidden_size, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.value_proj", + ) + norm_args = ( + self.hc_hidden_size, + config.rms_norm_eps, + self.hidden_size, + model_config.dtype, + ) + self.norm_key = Qwen4ExpPLEGroupedNorm(*norm_args) + self.norm_query = Qwen4ExpPLEGroupedNorm(*norm_args) + self.norm_conv = Qwen4ExpPLEGroupedNorm(*norm_args) + self.conv1d = nn.Conv1d( + self.hc_hidden_size, + self.hc_hidden_size, + self.conv_kernel_size, + groups=self.hc_hidden_size, + padding=self.conv_state_len, + dilation=self.short_conv_dilation, + bias=False, + dtype=model_config.dtype, + ) + nn.init.zeros_(self.conv1d.weight) + self.conv1d.weight._no_reinit = True + self.kv_cache = (torch.tensor([]),) + compilation_config = get_current_vllm_config().compilation_config + if prefix in compilation_config.static_forward_context: + raise ValueError(f"Duplicate layer name: {prefix}") + compilation_config.static_forward_context[prefix] = self + + def _get_embedding_weight_scale(self) -> torch.Tensor | None: + embedding = getattr(self.ple_embedding, "ngram_embedding", None) + return getattr(embedding, "weight_scale", None) + + def _dequantize_embeddings( + self, + embeddings: torch.Tensor, + output_dtype: torch.dtype, + ) -> torch.Tensor: + """Dequantize PLE lookup output.""" + + if not is_fp8(embeddings): + return embeddings + weight_scale = self._get_embedding_weight_scale() + if weight_scale is None: + raise RuntimeError("FP8 PLE embedding is missing its global scale") + if weight_scale.device != embeddings.device: + raise RuntimeError("FP8 PLE embedding scale must be on the output device") + return embeddings.to(output_dtype) * weight_scale.to(output_dtype) + + @property + def mamba_type(self) -> MambaAttentionBackendEnum: + return MambaAttentionBackendEnum.SHORT_CONV + + @property + def is_kv_cache_tp_replicated(self) -> bool: + return True + + def get_attn_backend(self) -> type[PleShortConvAttentionBackend]: + return PleShortConvAttentionBackend + + def get_state_dtype(self) -> tuple[torch.dtype, ...]: + return MambaStateDtypeCalculator.short_conv_state_dtype( + self.model_config.dtype, self.cache_config.mamba_cache_dtype + ) + + def get_state_shape(self) -> Sequence[tuple[int, ...]]: + return MambaStateShapeCalculator.short_conv_state_shape( + tp_world_size=1, + intermediate_size=self.hc_hidden_size, + conv_kernel=self.conv_state_len + 1, + num_spec=self.num_spec_tokens, + ) + + def _apply_norm( + self, norm: Qwen4ExpPLEGroupedNorm, hidden_states: torch.Tensor + ) -> torch.Tensor: + shape = hidden_states.shape + return norm(hidden_states.flatten(-2)).reshape(shape) + + def _short_conv_fallback(self, inputs: torch.Tensor) -> torch.Tensor: + # Profiling / CUDA graph capture only; conv state is not updated. + inputs_t = inputs.transpose(0, 1).unsqueeze(0) + output = self.conv1d(inputs_t)[..., : inputs_t.size(-1)] + return F.silu(output).squeeze(0).transpose(0, 1) + + def _short_conv_dilated_decode_batched( + self, + x_d: torch.Tensor, + conv_state: torch.Tensor, + conv_weights: torch.Tensor, + state_indices_tensor_d: torch.Tensor, + has_initial_states_d: torch.Tensor | None, + ) -> torch.Tensor: + state_indices = state_indices_tensor_d.to( + device=conv_state.device, dtype=torch.int64 + ) + # FULL cudagraph padded decode rows use NULL_BLOCK_ID. Remap them to + # slot 0 for a safe gather, then zero output and skip write-back. + valid_state = state_indices != NULL_BLOCK_ID + state_indices = torch.where( + valid_state, state_indices, torch.zeros_like(state_indices) + ) + if has_initial_states_d is None: + has_initial_state = valid_state + else: + if has_initial_states_d.numel() < state_indices_tensor_d.numel(): + raise ValueError( + "has_initial_states_d size mismatch: " + f"got {has_initial_states_d.numel()}, " + f"need >= {state_indices_tensor_d.numel()}." + ) + has_initial_state = has_initial_states_d[ + : state_indices_tensor_d.numel() + ].to(device=conv_state.device, dtype=torch.bool) + has_initial_state = has_initial_state & valid_state + + cached_state = conv_state.index_select(0, state_indices) + state = cached_state[..., : self.conv_state_len].to(x_d.dtype) + if self.conv_state_len > 0: + initial_state = torch.where( + has_initial_state.view(-1, 1, 1), + state, + torch.zeros_like(state), + ) + history = torch.cat((initial_state, x_d.unsqueeze(-1)), dim=-1) + else: + history = x_d.unsqueeze(-1) + + conv_output = F.conv1d( + history, + conv_weights.unsqueeze(1).contiguous(), + groups=history.size(1), + dilation=self.short_conv_dilation, + ).squeeze(-1) + output = F.silu(conv_output) + output = output * valid_state.view(-1, 1).to(output.dtype) + + if self.conv_state_len > 0: + next_state = history[..., -self.conv_state_len :] + # Padded rows are remapped to the reserved null slot. Preserve its + # existing value while writing the new states for valid rows. + existing_base_state = cached_state[..., : self.conv_state_len] + safe_next_state = torch.where( + valid_state.view(-1, 1, 1), + next_state.to(conv_state.dtype), + existing_base_state, + ) + cached_state[..., : self.conv_state_len] = safe_next_state + conv_state.index_copy_(0, state_indices, cached_state) + + return output + + def _short_conv_dilated_prefill_batched( + self, + x_p: torch.Tensor, + metadata: PleShortConvAttentionMetadata, + conv_state: torch.Tensor, + conv_weights: torch.Tensor, + state_indices_tensor_p: torch.Tensor, + num_prefills: int, + num_decode_tokens: int, + num_prefill_tokens: int, + ) -> torch.Tensor: + # ``non_spec_query_start_loc`` covers the non-spec (decode + prefill) + # requests and equals ``query_start_loc`` when spec-decode is inactive. + non_spec_query_start_loc = metadata.non_spec_query_start_loc + if non_spec_query_start_loc is None: + raise ValueError("query_start_loc is required for prefill short-conv") + query_start_loc_p = ( + non_spec_query_start_loc[-num_prefills - 1 :] - num_decode_tokens + ) + # The metadata builder guarantees that the prefill query offsets start + # at 0 and end at num_prefill_tokens. Avoid reading those values here, + # since doing so would force a device-to-host synchronization. + has_initial_states_p = metadata.has_initial_states_p + if has_initial_states_p is None: + raise ValueError("has_initial_states_p is required for prefill short-conv") + + output = torch.empty_like(x_p) + q_starts = query_start_loc_p.to(torch.int64) + if state_indices_tensor_p.numel() < num_prefills: + raise ValueError( + "state_indices_tensor_p size mismatch: " + f"got {state_indices_tensor_p.numel()}, " + f"need >= {num_prefills}." + ) + if has_initial_states_p.numel() < num_prefills: + raise ValueError( + "has_initial_states_p size mismatch: " + f"got {has_initial_states_p.numel()}, " + f"need >= {num_prefills}." + ) + if num_prefills == 0 or x_p.numel() == 0: + return output + lengths = q_starts[1:] - q_starts[:-1] + # Use the CPU-computed packing width from the metadata builder instead + # of synchronizing on lengths.max(). + max_len = metadata.max_prefill_query_len + if max_len <= 0: + return output + + hidden_size = x_p.shape[1] + positions = torch.arange( + num_prefill_tokens, device=x_p.device, dtype=torch.int64 + ) + req_indices = torch.searchsorted(q_starts[1:], positions, right=True) + col_indices = positions - q_starts[req_indices] + + packed_tokens = x_p.new_zeros((num_prefills, max_len, hidden_size)) + packed_tokens[req_indices, col_indices] = x_p + packed_tokens = packed_tokens.transpose(1, 2).contiguous() + + state_indices = state_indices_tensor_p[:num_prefills].to( + device=conv_state.device, dtype=torch.int64 + ) + valid_state = state_indices != NULL_BLOCK_ID + state_indices = torch.where( + valid_state, state_indices, torch.zeros_like(state_indices) + ) + has_initial = has_initial_states_p[:num_prefills].to( + device=conv_state.device, dtype=torch.bool + ) + if self.conv_state_len > 0: + if conv_state.shape[0] == 0: + state = conv_state.new_zeros( + (num_prefills, hidden_size, self.conv_state_len), + dtype=x_p.dtype, + ) + else: + state = conv_state.index_select(0, state_indices)[ + ..., : self.conv_state_len + ].to(x_p.dtype) + use_initial_mask = (valid_state & has_initial).view(num_prefills, 1, 1) + initial_state = torch.where( + use_initial_mask, + state, + torch.zeros_like(state), + ) + history = torch.cat((initial_state, packed_tokens), dim=-1) + else: + history = packed_tokens + + conv_output = F.conv1d( + history, + conv_weights.unsqueeze(1).contiguous(), + groups=history.size(1), + dilation=self.short_conv_dilation, + ) + conv_output = F.silu(conv_output).transpose(1, 2).contiguous() + + token_positions = torch.arange(max_len, device=x_p.device, dtype=torch.int64) + valid_tokens = token_positions.view(1, max_len) < lengths.view(num_prefills, 1) + valid_output_mask = valid_tokens & valid_state.to(device=x_p.device).view( + num_prefills, 1 + ) + conv_output.masked_fill_(~valid_output_mask.unsqueeze(-1), 0) + output.copy_(conv_output[req_indices, col_indices]) + + if self.conv_state_len > 0 and conv_state.shape[0] > 0: + state_starts = lengths.to(device=history.device, dtype=torch.int64).view( + num_prefills, 1, 1 + ) + state_offsets = torch.arange( + self.conv_state_len, device=history.device, dtype=torch.int64 + ).view(1, 1, self.conv_state_len) + next_state = history.gather( + dim=2, + index=(state_starts + state_offsets).expand(-1, history.size(1), -1), + ) + # Write back without a host synchronization. Valid, non-empty rows + # receive their new state; padding and zero-length rows keep the + # current cache value. + existing_state = conv_state.index_select(0, state_indices) + existing_base_state = existing_state[..., : self.conv_state_len] + update_mask = valid_state & (lengths.to(device=conv_state.device) > 0) + safe_next_state = torch.where( + update_mask.view(num_prefills, 1, 1), + next_state.to(conv_state.dtype), + existing_base_state, + ) + existing_state[..., : self.conv_state_len] = safe_next_state + conv_state.index_copy_(0, state_indices, existing_state) + return output + + def _short_conv_dilated_spec_batched( + self, + x_spec: torch.Tensor, + conv_state: torch.Tensor, + conv_weights: torch.Tensor, + spec_state_indices_tensor: torch.Tensor, + spec_query_start_loc: torch.Tensor, + num_accepted_tokens: torch.Tensor, + spec_query_len: int, + ) -> torch.Tensor: + """Dilated short-conv for speculative-decode (MTP) requests. + + Each spec request feeds multiple (draft + 1) query tokens. The conv + outputs are computed causally after rolling back the previous draft + state by ``num_accepted_tokens - 1``. The current candidate inputs stay + in the extended cache for the next forward, matching + ``causal_conv1d_update``. + + ``spec_query_len`` (== num_speculative_tokens + 1) is the maximum query + length and is a Python int, so no host synchronization is needed; this + keeps the path safe for full CUDA-graph capture/replay where the buffers + are padded at the request level. + """ + num_reqs = spec_state_indices_tensor.numel() + hidden_size = x_spec.size(-1) + # Use a fixed packing width instead of synchronizing on lengths.max(). + max_len = spec_query_len + # Full CUDA graphs can pad these buffers. Only the first num_reqs + # accepted-token counts belong to actual speculative requests. + num_accepted_tokens = num_accepted_tokens[:num_reqs] + q_starts = spec_query_start_loc[: num_reqs + 1].to(torch.int64) + # Keep the number of real speculative tokens on the device. + total_real_tokens = q_starts[num_reqs] + + state_indices = spec_state_indices_tensor.to( + device=conv_state.device, dtype=torch.int64 + ) + valid_state = state_indices != NULL_BLOCK_ID + state_indices = torch.where( + valid_state, state_indices, torch.zeros_like(state_indices) + ) + positions = torch.arange( + x_spec.size(0), device=x_spec.device, dtype=torch.int64 + ) + # Route graph-padded token rows to the discarded dummy request so that + # they cannot overwrite real packed data. + req_indices = torch.searchsorted(q_starts[1:], positions, right=True) + valid_tokens = (positions < total_real_tokens) & (req_indices < num_reqs) + clamped_req_indices = req_indices.clamp_max(max(num_reqs - 1, 0)) + col_indices = (positions - q_starts[clamped_req_indices]).clamp_(0, max_len - 1) + pack_req_indices = torch.where( + valid_tokens, + clamped_req_indices, + torch.full_like(req_indices, num_reqs), + ) + pack_col_indices = torch.where( + valid_tokens, col_indices, torch.zeros_like(col_indices) + ) + + # The last request row is the dummy sink for graph padding. + packed = x_spec.new_zeros((num_reqs + 1, max_len, hidden_size)) + packed[pack_req_indices, pack_col_indices] = x_spec + packed = packed.transpose(1, 2).contiguous() + + if self.conv_state_len > 0: + cached_state = conv_state.index_select(0, state_indices) + rollback_offsets = num_accepted_tokens.to( + device=conv_state.device, dtype=torch.int64 + ).sub(1) + rollback_offsets = torch.where( + valid_state, + rollback_offsets.clamp_(0, max_len - 1), + torch.zeros_like(rollback_offsets), + ) + state_offsets = torch.arange( + self.conv_state_len, device=conv_state.device, dtype=torch.int64 + ).view(1, 1, self.conv_state_len) + rollback_indices = rollback_offsets.view(-1, 1, 1) + state_offsets + state = cached_state.gather( + 2, rollback_indices.expand(-1, hidden_size, -1) + ).to(x_spec.dtype) + state = torch.where( + valid_state.view(num_reqs, 1, 1), + state, + torch.zeros_like(state), + ) + # Append a zeroed dummy-row state to match the [num_reqs + 1] pack. + dummy_state = state.new_zeros((1, hidden_size, self.conv_state_len)) + state_full = torch.cat((state, dummy_state), dim=0) + history = torch.cat((state_full, packed), dim=-1) + else: + history = packed + + conv_output = F.conv1d( + history, + conv_weights.unsqueeze(1).contiguous(), + groups=history.size(1), + dilation=self.short_conv_dilation, + ) + conv_output = F.silu(conv_output).transpose(1, 2).contiguous() + + output = conv_output[pack_req_indices, pack_col_indices] + output = output * valid_tokens.view(-1, 1).to(output.dtype) + + # Keep all current candidate inputs in the extended state. On the next + # target forward, ``num_accepted_tokens - 1`` selects the rollback + # window before processing the newly scheduled tokens. + if self.conv_state_len > 0: + state_capacity = self.conv_state_len + max_len - 1 + if conv_state.size(-1) < state_capacity: + raise RuntimeError( + "PLE short-conv cache cannot retain speculative tokens: " + f"got {conv_state.size(-1)}, need {state_capacity}." + ) + candidate_state = history[:num_reqs, :, 1 : state_capacity + 1] + query_lengths = q_starts[1:] - q_starts[:-1] + state_positions = torch.arange( + state_capacity, device=history.device, dtype=torch.int64 + ).view(1, 1, state_capacity) + update_lengths = (self.conv_state_len + query_lengths - 1).view( + num_reqs, 1, 1 + ) + update_mask = valid_state.view(num_reqs, 1, 1) & ( + state_positions < update_lengths + ) + existing_state = cached_state[..., :state_capacity] + next_state = torch.where( + update_mask, + candidate_state.to(conv_state.dtype), + existing_state, + ) + cached_state[..., :state_capacity] = next_state + conv_state.index_copy_(0, state_indices, cached_state) + + return output + + def _short_conv_dilated_dispatch( + self, + inputs: torch.Tensor, + metadata: PleShortConvAttentionMetadata, + conv_state: torch.Tensor, + conv_weights: torch.Tensor, + ) -> torch.Tensor: + num_prefills = metadata.num_prefills + num_decodes = metadata.num_decodes + num_decode_tokens = metadata.num_decode_tokens + num_prefill_tokens = metadata.num_prefill_tokens + has_prefill = num_prefills > 0 + has_decode = num_decodes > 0 + has_spec = metadata.spec_sequence_masks is not None + x = inputs[: metadata.num_actual_tokens] + + # Split spec / non-spec tokens. + if has_spec: + if has_prefill or has_decode: + assert metadata.spec_token_indx is not None + assert metadata.non_spec_token_indx is not None + x_spec = x.index_select(0, metadata.spec_token_indx.long()) + x_non_spec = x.index_select(0, metadata.non_spec_token_indx.long()) + else: + x_spec = x + x_non_spec = None + else: + x_spec = None + x_non_spec = x + + spec_output = None + # 1. Run the multi-query speculative-decode part. + if has_spec: + assert metadata.spec_state_indices_tensor is not None + assert metadata.spec_query_start_loc is not None + assert metadata.num_accepted_tokens is not None + spec_output = self._short_conv_dilated_spec_batched( + x_spec=x_spec, + conv_state=conv_state, + conv_weights=conv_weights, + spec_state_indices_tensor=metadata.spec_state_indices_tensor[ + : metadata.num_spec_decodes + ], + spec_query_start_loc=metadata.spec_query_start_loc, + num_accepted_tokens=metadata.num_accepted_tokens, + spec_query_len=metadata.spec_query_len, + ) + + # 2. Run regular decode and prefill requests. + conv_out_non_spec = None + state_indices_tensor = metadata.state_indices_tensor + if x_non_spec is not None: + assert state_indices_tensor is not None + if has_prefill: + state_indices_tensor_d, state_indices_tensor_p = torch.split( + state_indices_tensor, + [num_decodes, num_prefills], + dim=0, + ) + x_d, x_p = torch.split( + x_non_spec, + [num_decode_tokens, num_prefill_tokens], + dim=0, + ) + non_spec_parts: list[torch.Tensor] = [] + if has_decode: + non_spec_parts.append( + self._short_conv_dilated_decode_batched( + x_d=x_d, + conv_state=conv_state, + conv_weights=conv_weights, + state_indices_tensor_d=state_indices_tensor_d, + has_initial_states_d=metadata.has_initial_states_d, + ) + ) + non_spec_parts.append( + self._short_conv_dilated_prefill_batched( + x_p=x_p, + metadata=metadata, + conv_state=conv_state, + conv_weights=conv_weights, + state_indices_tensor_p=state_indices_tensor_p, + num_prefills=num_prefills, + num_decode_tokens=num_decode_tokens, + num_prefill_tokens=num_prefill_tokens, + ) + ) + conv_out_non_spec = torch.vstack(non_spec_parts) + else: + conv_out_non_spec = self._short_conv_dilated_decode_batched( + x_d=x_non_spec, + conv_state=conv_state, + conv_weights=conv_weights, + state_indices_tensor_d=state_indices_tensor[: x_non_spec.size(0)], + has_initial_states_d=metadata.has_initial_states_d, + ) + + # 3. Merge both parts back into the original token order. + if has_spec and conv_out_non_spec is not None: + assert metadata.spec_token_indx is not None + assert metadata.non_spec_token_indx is not None + assert spec_output is not None + output = x.new_empty((metadata.num_actual_tokens, x.size(-1))) + output.index_copy_(0, metadata.spec_token_indx, spec_output) + output.index_copy_(0, metadata.non_spec_token_indx, conv_out_non_spec) + return output + elif has_spec: + assert spec_output is not None + return spec_output + if conv_out_non_spec is None: + return x + return conv_out_non_spec + + def _short_conv(self, inputs: torch.Tensor) -> torch.Tensor: + forward_context = get_forward_context() + attn_metadata = forward_context.attn_metadata + if attn_metadata is None: + return self._short_conv_fallback(inputs) + + if not isinstance(attn_metadata, dict): + raise RuntimeError( + "PLE short-conv expects per-layer attention metadata dict " + f"during inference, got {type(attn_metadata).__name__}." + ) + + layer_attn_metadata = attn_metadata.get(self.prefix) + if layer_attn_metadata is None: + # MRV2 omits Mamba-family metadata during profile warmup. + return self._short_conv_fallback(inputs) + if not isinstance(layer_attn_metadata, PleShortConvAttentionMetadata): + raise TypeError( + "Expected PleShortConvAttentionMetadata for layer " + f"'{self.prefix}', got " + f"{type(layer_attn_metadata).__name__}." + ) + + conv_state = self.kv_cache[0] + if not is_conv_state_dim_first(): + conv_state = conv_state.transpose(-1, -2) + conv_weights = self.conv1d.weight.squeeze(1) + + state_capacity = self.conv_state_len + self.num_spec_tokens + if state_capacity > 0: + if conv_state.size(-1) < state_capacity: + raise RuntimeError( + "PLE short-conv cache is smaller than expected for " + f"dilated convolution: got {conv_state.size(-1)}, " + f"expect at least {state_capacity}." + ) + conv_state = conv_state[..., -state_capacity:] + return self._short_conv_dilated_dispatch( + inputs, + layer_attn_metadata, + conv_state, + conv_weights.to(dtype=inputs.dtype), + ) + + def forward( + self, + hidden_states: torch.Tensor, + input_ids: torch.Tensor, + query_start_loc: torch.Tensor, + ngram_context: torch.Tensor, + ) -> torch.Tensor: + input_ids = input_ids.reshape(-1) + if input_ids.shape[0] != hidden_states.shape[0]: + raise ValueError( + "PLE expects input_ids and hidden_states to have the same " + f"token length, got {input_ids.shape[0]} and " + f"{hidden_states.shape[0]}" + ) + embeddings = self.ple_embedding(input_ids, query_start_loc, ngram_context) + embeddings = self._dequantize_embeddings(embeddings, hidden_states.dtype) + key, _ = self.key_proj(embeddings) + value, _ = self.value_proj(embeddings) + token_count = hidden_states.shape[0] + key = key.reshape(token_count, self.hc_count, self.hidden_size) + query = hidden_states.reshape(token_count, self.hc_count, self.hidden_size) + key = self._apply_norm(self.norm_key, key) + query = self._apply_norm(self.norm_query, query) + gate = (key * query).sum(dim=-1, keepdim=True) / math.sqrt(self.hidden_size) + gate = torch.sigmoid(gate.sign() * gate.abs().clamp_min(1e-6).sqrt()) + gated_value = gate * value.unsqueeze(-2) + normalized = self._apply_norm(self.norm_conv, gated_value).flatten(-2) + conv_output = torch.zeros_like(normalized) + torch.ops.vllm.qwen4_exp_ple_short_conv( + normalized, + conv_output, + self.prefix, + ) + return gated_value.flatten(-2) + conv_output + + +def qwen4_exp_compute_ple_ngram_ids( + input_ids: torch.Tensor, + query_start_loc: torch.Tensor, + ngram_context: torch.Tensor, + output: torch.Tensor, + layer_name: str, +) -> None: + """Compute request-dependent PLE n-gram IDs outside piecewise graphs.""" + layer = get_forward_context().no_compile_layers[layer_name] + ngram_ids = layer.ple_embedding.compute_ngram_ids( + input_ids, + query_start_loc, + ngram_context, + ) + output.copy_(ngram_ids) + + +def qwen4_exp_compute_ple_ngram_ids_fake( + input_ids: torch.Tensor, + query_start_loc: torch.Tensor, + ngram_context: torch.Tensor, + output: torch.Tensor, + layer_name: str, +) -> None: + return + + +def qwen4_exp_ple_short_conv( + inputs: torch.Tensor, + output: torch.Tensor, + layer_name: str, +) -> None: + layer = get_forward_context().no_compile_layers[layer_name] + result = layer._short_conv(inputs) + output[: result.shape[0]].copy_(result) + + +def qwen4_exp_ple_short_conv_fake( + inputs: torch.Tensor, + output: torch.Tensor, + layer_name: str, +) -> None: + return + + +direct_register_custom_op( + op_name="qwen4_exp_compute_ple_ngram_ids", + op_func=qwen4_exp_compute_ple_ngram_ids, + mutates_args=["output"], + fake_impl=qwen4_exp_compute_ple_ngram_ids_fake, +) + + +direct_register_custom_op( + op_name="qwen4_exp_ple_short_conv", + op_func=qwen4_exp_ple_short_conv, + mutates_args=["output"], + fake_impl=qwen4_exp_ple_short_conv_fake, +) + + +__all__ = [ + "Qwen4ExpNGramEmbedding", + "Qwen4ExpPLEGroupedNorm", + "Qwen4ExpPLELayer", +] diff --git a/vllm/models/qwen4_exp/nvidia/qsa.py b/vllm/models/qwen4_exp/nvidia/qsa.py new file mode 100644 index 0000000000..064339dc53 --- /dev/null +++ b/vllm/models/qwen4_exp/nvidia/qsa.py @@ -0,0 +1,487 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""NVIDIA QSA owner with Triton kernels.""" + +from __future__ import annotations + +from typing import ClassVar, cast + +import torch +from torch import nn + +from vllm.config import VllmConfig +from vllm.config.cache import CacheDType +from vllm.distributed import get_tensor_model_parallel_world_size +from vllm.forward_context import get_forward_context +from vllm.model_executor.layers.attention.attention import ( + set_default_quant_scales, +) +from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase +from vllm.model_executor.layers.layernorm import GemmaRMSNorm +from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding, get_rope +from vllm.model_executor.models.qwen3_next import Qwen3NextAttention +from vllm.platforms import current_platform +from vllm.transformers_utils.configs.qwen4_exp import ( + Qwen4ExpTextConfig, +) +from vllm.utils.torch_utils import ( + LayerNameType, + _encode_layer_name, + _resolve_layer_name, + canonicalize_singleton_dim_strides, + direct_register_custom_op, + kv_cache_dtype_str_to_dtype, +) +from vllm.v1.attention.backend import ( + AttentionBackend, + AttentionCGSupport, + AttentionType, + MultipleOf, +) +from vllm.v1.attention.backends.fa_utils import is_flash_attn_varlen_func_available +from vllm.v1.attention.backends.flash_attn import ( + FlashAttentionBackend, + FlashAttentionImpl, + FlashAttentionMetadata, + FlashAttentionMetadataBuilder, +) +from vllm.v1.kv_cache_interface import ( + FullAttentionSpec, + KVCacheSpec, + get_kv_quant_mode, +) + +from ..common.qsa_cache import QSAForwardMetadata +from . import model +from .indexer_qsa import QSAIndexer + + +class Qwen4ExpQSAMetadataBuilder(FlashAttentionMetadataBuilder): + """Flash metadata supporting uniform decode and target-verify graphs.""" + + _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH + + +class Qwen4ExpQSAFlashAttentionBackend(FlashAttentionBackend): + """FullAttentionSpec backend used by the merged QSA owner.""" + + supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16] + supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = ["auto", "bfloat16"] + + @staticmethod + def get_name() -> str: + return "QWEN4_EXP_QSA_TRITON" + + @staticmethod + def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + # QSA consumes manager pages directly and does not use FA4 paged attention. + return [MultipleOf(16)] + + @staticmethod + def get_impl_cls() -> type[Qwen4ExpQSAFlashAttentionImpl]: + return Qwen4ExpQSAFlashAttentionImpl + + @staticmethod + def get_builder_cls() -> type[Qwen4ExpQSAMetadataBuilder]: + return Qwen4ExpQSAMetadataBuilder + + @classmethod + def is_sparse(cls) -> bool: + return True + + @classmethod + def supports_kv_connector(cls) -> bool: + return False + + +class Qwen4ExpQSAFlashAttentionImpl(FlashAttentionImpl): + """Run paged sparse GQA with the QSA Triton kernel.""" + + supports_dcp: bool = False + supports_pcp: bool = False + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + if not is_flash_attn_varlen_func_available(): + raise NotImplementedError("Qwen4Exp QSA requires FlashAttention") + if self.dcp_world_size != 1: + raise NotImplementedError( + "Qwen4Exp QSA does not support decode context parallelism" + ) + if self.kv_cache_dtype not in ("auto", "bfloat16"): + raise NotImplementedError("Qwen4Exp QSA requires a BF16 main KV cache") + self.supports_quant_query_input = False + + def forward_qsa( + self, + layer: torch.nn.Module, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: FlashAttentionMetadata, + output: torch.Tensor, + token_to_req: torch.Tensor, + output_scale: torch.Tensor | None = None, + output_block_scale: torch.Tensor | None = None, + ) -> torch.Tensor: + del key, value + if output_scale is not None or output_block_scale is not None: + raise NotImplementedError("QSA does not support fused output quantization") + if self.alibi_slopes is not None or self.sinks is not None: + raise NotImplementedError("QSA does not support ALiBi or attention sinks") + if self.sliding_window != (-1, -1): + raise NotImplementedError("QSA does not support sliding-window attention") + + num_tokens = attn_metadata.num_actual_tokens + output.zero_() + if num_tokens == 0: + return output + + topk_buffer = getattr(layer, "topk_indices_buffer", None) + if topk_buffer is None: + raise RuntimeError("QSA owner did not provide its top-k buffer") + logical_indices = topk_buffer[:num_tokens] + token_to_req = token_to_req[:num_tokens] + key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1) + key_cache = canonicalize_singleton_dim_strides(key_cache) + value_cache = canonicalize_singleton_dim_strides(value_cache) + if key_cache.dtype != torch.bfloat16 or query.dtype != torch.bfloat16: + raise NotImplementedError("Qwen4Exp QSA requires BF16 Q/K/V") + + from .ops.qsa import qsa_sparse_paged_attention + + qsa_sparse_paged_attention( + query[:num_tokens], + key_cache, + value_cache, + logical_indices, + attn_metadata.block_table, + token_to_req, + output[:num_tokens], + ) + return output + + +class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase): + """Merged Qwen full-attention owner with a QSA index side branch.""" + + supports_dcp = False + + def __init__( + self, + *, + vllm_config: VllmConfig, + config: Qwen4ExpTextConfig, + layer_id: int, + quant_config: QuantizationConfig | None = None, + reduce_results: bool = True, + prefix: str = "", + ) -> None: + nn.Module.__init__(self) + cache_config = vllm_config.cache_config + model_config = vllm_config.model_config + if cache_config is None: + raise ValueError("Qwen4Exp QSA requires a paged KV cache") + if model_config.dtype != torch.bfloat16: + raise NotImplementedError("Qwen4Exp QSA currently requires BF16") + if cache_config.cache_dtype not in ("auto", "bfloat16"): + raise NotImplementedError("Qwen4Exp QSA requires a BF16 main KV cache") + if getattr(quant_config, "kv_cache_scheme", None) is not None: + raise NotImplementedError("Qwen4Exp QSA does not support KV quantization") + parallel_config = vllm_config.parallel_config + if ( + parallel_config.prefill_context_parallel_size > 1 + or parallel_config.decode_context_parallel_size > 1 + ): + raise NotImplementedError( + "Qwen4Exp QSA does not support context parallelism" + ) + if not getattr(config, "is_causal", True): + raise NotImplementedError("Qwen4Exp QSA requires causal decoder attention") + + self.config = config + self.hidden_size = int(config.hidden_size) + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = int(config.num_attention_heads) + if self.total_num_heads % tp_size: + raise ValueError("QSA attention heads must be divisible by TP size") + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = int(config.num_key_value_heads) + if self.total_num_kv_heads >= tp_size: + if self.total_num_kv_heads % tp_size: + raise ValueError("QSA KV heads must be divisible by TP size") + elif tp_size % self.total_num_kv_heads: + raise ValueError("TP size must be divisible by replicated QSA KV heads") + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.head_dim = int(config.head_dim or self.hidden_size // self.num_heads) + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.dual_chunk_attention_config = getattr( + config, "dual_chunk_attention_config", None + ) + if self.dual_chunk_attention_config is not None: + raise NotImplementedError("Qwen4Exp QSA does not support dual-chunk RoPE") + # Qwen4Exp full-attention checkpoints always pack a sigmoid output + # gate next to Q, even when an inherited config default says otherwise. + self.attn_output_gate = True + + self.qkv_proj = QKVParallelLinear( + self.hidden_size, + self.head_dim, + self.total_num_heads * (1 + self.attn_output_gate), + self.total_num_kv_heads, + bias=False, + quant_config=model.without_modelopt_fp4(quant_config), + prefix=f"{prefix}.qkv_proj", + ) + self.o_proj = RowParallelLinear( + self.total_num_heads * self.head_dim, + self.hidden_size, + bias=False, + reduce_results=reduce_results, + quant_config=quant_config, + prefix=f"{prefix}.o_proj", + ) + self.rotary_emb = get_rope( + head_size=self.head_dim, + max_position=config.max_position_embeddings, + rope_parameters=config.rope_parameters, + ) + self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps) + self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps) + + mm_config = model_config.multimodal_config + text_only = mm_config is None or mm_config.language_model_only + mrope_section = getattr(self.rotary_emb, "mrope_section", None) + supports_mrope = bool( + type(self.rotary_emb) is MRotaryEmbedding + and mrope_section + and len(mrope_section) == 3 + and sum(mrope_section) == self.rotary_emb.rotary_dim // 2 + and getattr(self.rotary_emb, "mrope_interleaved", False) + ) + supports_dtype = getattr(self.rotary_emb, "dtype", None) in ( + torch.float16, + torch.bfloat16, + ) + self.use_fused_qk_norm_rope_gate = ( + self.attn_output_gate + and getattr(self.rotary_emb, "is_neox_style", False) + and current_platform.is_cuda() + and supports_dtype + and (text_only or supports_mrope) + ) + + self.layer_name = f"{prefix}.attn" + self.attn_type = AttentionType.DECODER + self.kv_cache_dtype = cache_config.cache_dtype + self.kv_cache_torch_dtype = kv_cache_dtype_str_to_dtype( + self.kv_cache_dtype, model_config + ) + if self.kv_cache_torch_dtype != torch.bfloat16: + raise NotImplementedError("Qwen4Exp QSA requires BF16 cache storage") + self.kv_sharing_target_layer_name = None + self.kv_cache = torch.tensor([]) + set_default_quant_scales(self, register_buffer=True) + + self.attn_backend = Qwen4ExpQSAFlashAttentionBackend + self.impl = Qwen4ExpQSAFlashAttentionImpl( + self.num_heads, + self.head_dim, + self.scaling, + self.num_kv_heads, + None, + None, + self.kv_cache_dtype, + None, + AttentionType.DECODER, + None, + ) + self.indexer = QSAIndexer( + vllm_config=vllm_config, + config=config, + layer_id=layer_id, + rotary_emb=self.rotary_emb, + quant_config=quant_config, + prefix=f"{prefix}.indexer", + ) + max_tokens = vllm_config.scheduler_config.max_num_batched_tokens + self.register_buffer( + "topk_indices_buffer", + torch.empty( + max_tokens, + self.indexer.output_width, + dtype=torch.int32, + ), + persistent=False, + ) + + static_context = vllm_config.compilation_config.static_forward_context + if self.layer_name in static_context: + raise ValueError(f"Duplicate layer name: {self.layer_name}") + static_context[self.layer_name] = self + + def get_attn_backend(self) -> type[AttentionBackend]: + return self.attn_backend + + def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec: + return FullAttentionSpec( + block_size=vllm_config.cache_config.block_size, + num_kv_heads=self.num_kv_heads, + head_size=self.head_dim, + head_size_v=self.head_dim, + dtype=self.kv_cache_torch_dtype, + kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype), + ) + + def _run_qsa( + self, + hidden_states: torch.Tensor, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + output: torch.Tensor, + ) -> None: + metadata = get_forward_context().attn_metadata + if isinstance(metadata, list): + metadata = metadata[0] + if not isinstance(metadata, dict): + output.zero_() + return + main_metadata = cast(FlashAttentionMetadata, metadata[self.layer_name]) + if self.kv_cache.numel() == 0: + raise RuntimeError("QSA main K/V cache is not bound") + + num_tokens = main_metadata.num_actual_tokens + side_metadata = cast( + QSAForwardMetadata, + metadata[self.indexer.raw_key_cache.prefix], + ) + if side_metadata.num_actual_tokens != num_tokens: + raise RuntimeError("QSA main and side metadata token counts disagree") + selected = self.indexer( + hidden_states, + positions, + self.topk_indices_buffer[:num_tokens], + ) + if selected.shape != ( + num_tokens, + self.indexer.output_width, + ): + raise RuntimeError("QSA indexer returned an invalid selection shape") + impl = cast(Qwen4ExpQSAFlashAttentionImpl, self.impl) + impl.do_kv_cache_update( + self, + key, + value, + self.kv_cache, + main_metadata.slot_mapping, + ) + impl.forward_qsa( + self, + query, + key, + value, + self.kv_cache, + main_metadata, + output, + token_to_req=side_metadata.token_to_req, + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v, gate = self._project_qkv_gate(qkv, positions) + num_tokens = hidden_states.shape[0] + query = q.view(num_tokens, self.num_heads, self.head_dim) + key = k.view(num_tokens, self.num_kv_heads, self.head_dim) + value = v.view(num_tokens, self.num_kv_heads, self.head_dim) + attn_output = torch.empty_like(query) + encoded_layer_name = _encode_layer_name(self.layer_name) + if current_platform.opaque_attention_op(): + torch.ops.vllm.qwen4_exp_qsa_with_output( + hidden_states, + positions, + query, + key, + value, + attn_output, + encoded_layer_name, + ) + else: + qwen4_exp_qsa_with_output( + hidden_states, + positions, + query, + key, + value, + attn_output, + encoded_layer_name, + ) + flat_output = attn_output.view(num_tokens, -1) + if gate is not None: + flat_output = flat_output * torch.sigmoid(gate) + output, _ = self.o_proj(flat_output) + return output + + +def qwen4_exp_qsa_with_output( + hidden_states: torch.Tensor, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + output: torch.Tensor, + layer_name: LayerNameType, +) -> None: + """Run the complete QSA state/update/attend transaction.""" + + layer_name = _resolve_layer_name(layer_name) + layer = get_forward_context().no_compile_layers[layer_name] + if not isinstance(layer, Qwen4ExpQSAAttention): + raise TypeError(f"{layer_name} is not a Qwen4Exp QSA owner") + layer._run_qsa( + hidden_states, + positions, + query, + key, + value, + output, + ) + + +def qwen4_exp_qsa_with_output_fake( + hidden_states: torch.Tensor, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + output: torch.Tensor, + layer_name: LayerNameType, +) -> None: + del hidden_states, positions, query, key, value, output, layer_name + + +direct_register_custom_op( + op_name="qwen4_exp_qsa_with_output", + op_func=qwen4_exp_qsa_with_output, + mutates_args=["output"], + fake_impl=qwen4_exp_qsa_with_output_fake, +) + + +__all__ = [ + "QSAIndexer", + "Qwen4ExpQSAAttention", + "Qwen4ExpQSAFlashAttentionBackend", + "Qwen4ExpQSAFlashAttentionImpl", + "qwen4_exp_qsa_with_output", +] diff --git a/vllm/platforms/interface.py b/vllm/platforms/interface.py index 5c041f1632..d7a31323ba 100644 --- a/vllm/platforms/interface.py +++ b/vllm/platforms/interface.py @@ -855,11 +855,18 @@ class Platform: model_config.architecture, model_config=model_config, ) - mamba_page_size = MambaSpec( - shapes=model_cls.get_mamba_state_shape_from_config(vllm_config), - dtypes=model_cls.get_mamba_state_dtype_from_config(vllm_config), - block_size=-1, - ).page_size_bytes + # Qwen4Exp has multiple Mamba state layouts with different sizes. + if hasattr(model_cls, "get_mamba_specs_from_config"): + mamba_page_size = max( + spec.page_size_bytes + for spec in model_cls.get_mamba_specs_from_config(vllm_config) + ) + else: + mamba_page_size = MambaSpec( + shapes=model_cls.get_mamba_state_shape_from_config(vllm_config), + dtypes=model_cls.get_mamba_state_dtype_from_config(vllm_config), + block_size=-1, + ).page_size_bytes if mamba_page_size == 0: return diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index 0ed71feb22..b5b0ccd8f0 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -125,6 +125,8 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict( qianfan_ocr="QianfanOCRConfig", qwen3_asr="Qwen3ASRConfig", qwen3_next="Qwen3NextConfig", + qwen4_exp="Qwen4ExpConfig", + qwen4_exp_text="Qwen4ExpTextConfig", qwen3_5="Qwen3_5Config", qwen3_5_text="Qwen3_5TextConfig", qwen3_5_moe="Qwen3_5MoeConfig", diff --git a/vllm/transformers_utils/configs/__init__.py b/vllm/transformers_utils/configs/__init__.py index cfc6d1c3bf..257b0e7792 100644 --- a/vllm/transformers_utils/configs/__init__.py +++ b/vllm/transformers_utils/configs/__init__.py @@ -91,6 +91,9 @@ _CLASS_TO_MODULE: dict[str, str] = { "QianfanOCRVisionConfig": "vllm.transformers_utils.configs.qianfan_ocr", "Qwen3ASRConfig": "vllm.transformers_utils.configs.qwen3_asr", "Qwen3NextConfig": "vllm.transformers_utils.configs.qwen3_next", + "Qwen4ExpConfig": "vllm.transformers_utils.configs.qwen4_exp", + "Qwen4ExpTextConfig": "vllm.transformers_utils.configs.qwen4_exp", + "Qwen4ExpVisionConfig": "vllm.transformers_utils.configs.qwen4_exp", "Qwen3_5Config": "vllm.transformers_utils.configs.qwen3_5", "Qwen3_5TextConfig": "vllm.transformers_utils.configs.qwen3_5", "Qwen3_5MoeConfig": "vllm.transformers_utils.configs.qwen3_5_moe", @@ -176,6 +179,9 @@ __all__ = [ "QianfanOCRVisionConfig", "Qwen3ASRConfig", "Qwen3NextConfig", + "Qwen4ExpConfig", + "Qwen4ExpTextConfig", + "Qwen4ExpVisionConfig", "Qwen3_5Config", "Qwen3_5TextConfig", "Qwen3_5MoeConfig", diff --git a/vllm/transformers_utils/configs/qwen4_exp.py b/vllm/transformers_utils/configs/qwen4_exp.py new file mode 100644 index 0000000000..ed4c6cca8e --- /dev/null +++ b/vllm/transformers_utils/configs/qwen4_exp.py @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Exports for the package-owned Qwen4Exp configs.""" + +from vllm.models.qwen4_exp.config import ( + Qwen4ExpConfig, + Qwen4ExpTextConfig, + Qwen4ExpVisionConfig, +) + +__all__ = [ + "Qwen4ExpConfig", + "Qwen4ExpTextConfig", + "Qwen4ExpVisionConfig", +] diff --git a/vllm/transformers_utils/model_arch_config_convertor.py b/vllm/transformers_utils/model_arch_config_convertor.py index 01613911f0..d0e3d7c0dc 100644 --- a/vllm/transformers_utils/model_arch_config_convertor.py +++ b/vllm/transformers_utils/model_arch_config_convertor.py @@ -642,6 +642,15 @@ class Qwen3NextMTPModelArchConfigConvertor(ModelArchConfigConvertorBase): return getattr(self.hf_text_config, "num_nextn_predict_layers", 0) +class Qwen4ExpMTPModelArchConfigConvertor(ModelArchConfigConvertorBase): + def get_num_hidden_layers(self) -> int: + return getattr( + self.hf_text_config, + "mtp_num_hidden_layers", + getattr(self.hf_text_config, "num_nextn_predict_layers", 0), + ) + + class BailingHybridMTPModelArchConfigConvertor(ModelArchConfigConvertorBase): def get_num_hidden_layers(self) -> int: return getattr(self.hf_text_config, "num_nextn_predict_layers", 0) @@ -791,6 +800,7 @@ MODEL_ARCH_CONFIG_CONVERTORS = { "pangu_ultra_moe_mtp": PanguUltraMoeMTPModelArchConfigConvertor, "qwen3_5_mtp": Qwen3_5MTPModelArchConfigConvertor, "qwen3_next_mtp": Qwen3NextMTPModelArchConfigConvertor, + "qwen4_exp_mtp": Qwen4ExpMTPModelArchConfigConvertor, "RefinedWeb": FalconModelArchConfigConvertor, "RefinedWebModel": FalconModelArchConfigConvertor, "step3p5_mtp": Step3p5MTPModelArchConfigConvertor, diff --git a/vllm/v1/attention/backends/short_conv_attn.py b/vllm/v1/attention/backends/short_conv_attn.py index 9c85ec5efb..3e888e69b7 100644 --- a/vllm/v1/attention/backends/short_conv_attn.py +++ b/vllm/v1/attention/backends/short_conv_attn.py @@ -1,12 +1,27 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from dataclasses import dataclass +from dataclasses import dataclass, replace +from typing import Any -from vllm.v1.attention.backend import AttentionBackend +import torch + +from vllm.config import VllmConfig +from vllm.utils.torch_utils import async_tensor_h2d +from vllm.v1.attention.backend import ( + AttentionBackend, + AttentionCGSupport, + CommonAttentionMetadata, +) from vllm.v1.attention.backends.mamba_attn import ( BaseMambaAttentionMetadata, BaseMambaAttentionMetadataBuilder, ) +from vllm.v1.attention.backends.utils import ( + NULL_BLOCK_ID, + compute_causal_conv1d_metadata, + mamba_get_block_table_tensor, +) +from vllm.v1.kv_cache_interface import MambaSpec class ShortConvAttentionBackend(AttentionBackend): @@ -32,3 +47,513 @@ class ShortConvAttentionMetadataBuilder( BaseMambaAttentionMetadataBuilder[ShortConvAttentionMetadata] ): metadata_cls = ShortConvAttentionMetadata + + +@dataclass +class PleShortConvAttentionMetadata(ShortConvAttentionMetadata): + # Number of speculative-decode (multi-query / MTP) requests and the total + # number of tokens they contribute. These are 0 when spec-decode is off. + num_spec_decodes: int = 0 + num_spec_decode_tokens: int = 0 + num_actual_tokens: int = 0 + + # Max query length among spec-decode requests (== num_speculative_tokens + 1). + # Used as ``max_query_len`` for the varlen spec causal_conv1d_update. + spec_query_len: int = 1 + + # Max query length among the non-spec *prefill* requests, precomputed + # CPU-side in the builder. The dilated PLE short-conv uses it to size its + # packing buffer without a device->host sync (``lengths.max().item()``). + # 0 when there are no prefill requests. + max_prefill_query_len: int = 0 + query_start_loc: torch.Tensor | None = None + + # ``state_indices_tensor`` keeps the historical (non-spec) layout used by + # all existing short-conv consumers: the conv-state slot for each regular + # decode followed by each prefill request. When spec-decode is active this + # only covers the non-spec requests. + state_indices_tensor: torch.Tensor | None = None + has_initial_states_d: torch.Tensor | None = None + + # ``non_spec_query_start_loc`` is the varlen cumulative token offset over + # the non-spec requests only (decodes then prefills). It equals + # ``query_start_loc`` when there are no spec-decode requests. + non_spec_query_start_loc: torch.Tensor | None = None + + # Speculative-decode (MTP) conv metadata. Only column 0 of the block table + # is needed for the convolution state, so these tensors are 1-D over the + # spec-decode requests. + spec_query_start_loc: torch.Tensor | None = None # [num_spec_decodes + 1] + spec_state_indices_tensor: torch.Tensor | None = None # [num_spec_decodes] + spec_sequence_masks: torch.Tensor | None = None # [batch] + spec_token_indx: torch.Tensor | None = None + non_spec_token_indx: torch.Tensor | None = None + num_decode_draft_tokens_cpu: torch.Tensor | None = None + + +class PleShortConvAttentionBackend(ShortConvAttentionBackend): + @staticmethod + def get_name() -> str: + return "PLE_SHORT_CONV_ATTN" + + @staticmethod + def get_builder_cls() -> type["PleShortConvAttentionMetadataBuilder"]: + return PleShortConvAttentionMetadataBuilder + + +class PleShortConvAttentionMetadataBuilder(ShortConvAttentionMetadataBuilder): + metadata_cls = PleShortConvAttentionMetadata + # Spec-decode requires a uniform (multi-token) decode batch for full + # CUDA graph capture, matching the GDN backend. + _cudagraph_support = AttentionCGSupport.UNIFORM_BATCH + reorder_batch_threshold: int = 1 + supports_update_block_table = False + + def __init__( + self, + kv_cache_spec: MambaSpec, + layer_names: list[str], + vllm_config: VllmConfig, + device: torch.device, + ) -> None: + super().__init__(kv_cache_spec, layer_names, vllm_config, device) + self.num_spec = self.num_spec_tokens + self.use_full_cuda_graph = ( + self.compilation_config.cudagraph_mode.has_full_cudagraphs() + ) + + max_num_seqs = vllm_config.scheduler_config.max_num_seqs + max_capture_size = self.compilation_config.max_cudagraph_capture_size + self.decode_cudagraph_max_bs = max_num_seqs + self.decode_cudagraph_max_tokens = max_num_seqs * (self.num_spec + 1) + if max_capture_size is not None: + self.decode_cudagraph_max_bs = min( + self.decode_cudagraph_max_bs, max_capture_size + ) + self.decode_cudagraph_max_tokens = min( + self.decode_cudagraph_max_tokens, max_capture_size + ) + + # Persistent buffers reused during full CUDA graph capture and replay. + self.spec_state_indices_tensor = torch.empty( + (self.decode_cudagraph_max_bs,), dtype=torch.int32, device=device + ) + self.spec_sequence_masks = torch.empty( + (self.decode_cudagraph_max_bs,), dtype=torch.bool, device=device + ) + self.spec_token_indx = torch.empty( + (self.decode_cudagraph_max_tokens,), dtype=torch.int32, device=device + ) + self.non_spec_token_indx = torch.empty( + (self.decode_cudagraph_max_tokens,), dtype=torch.int32, device=device + ) + self.spec_query_start_loc = torch.empty( + (self.decode_cudagraph_max_bs + 1,), dtype=torch.int32, device=device + ) + self.num_accepted_tokens = torch.empty( + (self.decode_cudagraph_max_bs,), dtype=torch.int32, device=device + ) + self.has_initial_states_d = torch.empty( + (self.decode_cudagraph_max_bs,), dtype=torch.bool, device=device + ) + + def _build_non_spec_metadata( + self, + common_prefix_len: int, + common_attn_metadata: CommonAttentionMetadata, + fast_build: bool, + num_decode_draft_tokens_cpu: torch.Tensor | None, + **kwargs: Any, + ) -> PleShortConvAttentionMetadata: + metadata = super().build( + common_prefix_len, + common_attn_metadata, + fast_build, + num_accepted_tokens=None, + **kwargs, + ) + assert isinstance(metadata, PleShortConvAttentionMetadata) + + state_indices_d = metadata.state_indices_tensor_d + if state_indices_d is not None and state_indices_d.dim() > 1: + state_indices_d = state_indices_d[:, 0] + state_indices_p = metadata.state_indices_tensor_p + if metadata.num_prefills == 0: + assert state_indices_d is not None + # BaseMambaAttentionMetadataBuilder pads decode state indices into + # a persistent tensor for full CUDA graphs. Keep those rows so the + # PLE decode receives one cache slot per graph-padded token. + state_indices_tensor = state_indices_d + elif metadata.num_decodes == 0: + assert state_indices_p is not None + state_indices_tensor = state_indices_p[: metadata.num_prefills] + else: + assert state_indices_d is not None + assert state_indices_p is not None + state_indices_tensor = torch.cat( + (state_indices_d, state_indices_p[: metadata.num_prefills]) + ) + + has_initial_states_d = None + if metadata.num_decodes > 0: + num_computed_tokens = common_attn_metadata.compute_num_computed_tokens() + has_initial_states_d = num_computed_tokens[: metadata.num_decodes] > 0 + if ( + self.use_full_cuda_graph + and metadata.num_prefills == 0 + and metadata.num_decodes <= self.decode_cudagraph_max_bs + ): + assert state_indices_d is not None + # Prepare tensors for CUDA graph replay. Padded rows have no + # initial state and use NULL_BLOCK_ID in state_indices_d. + num_decode_rows = state_indices_d.numel() + self.has_initial_states_d[: metadata.num_decodes].copy_( + has_initial_states_d, non_blocking=True + ) + self.has_initial_states_d[metadata.num_decodes : num_decode_rows].fill_( + False + ) + has_initial_states_d = self.has_initial_states_d[:num_decode_rows] + + max_prefill_query_len = 0 + if metadata.num_prefills > 0: + query_lens_cpu = torch.diff(common_attn_metadata.query_start_loc_cpu) + max_prefill_query_len = int( + query_lens_cpu[ + metadata.num_decodes : ( + metadata.num_decodes + metadata.num_prefills + ) + ] + .max() + .item() + ) + + return replace( + metadata, + num_actual_tokens=common_attn_metadata.num_actual_tokens, + spec_query_len=self.num_spec + 1, + max_prefill_query_len=max_prefill_query_len, + query_start_loc=common_attn_metadata.query_start_loc, + state_indices_tensor=state_indices_tensor, + has_initial_states_d=has_initial_states_d, + non_spec_query_start_loc=common_attn_metadata.query_start_loc, + num_decode_draft_tokens_cpu=num_decode_draft_tokens_cpu, + ) + + def build( # type: ignore[override] + self, + common_prefix_len: int, + common_attn_metadata: CommonAttentionMetadata, + fast_build: bool = False, + *, + num_accepted_tokens: torch.Tensor | None = None, + num_decode_draft_tokens_cpu: torch.Tensor | None = None, + **kwargs: Any, + ) -> PleShortConvAttentionMetadata: + m = common_attn_metadata + spec_sequence_masks_cpu: torch.Tensor | None = None + # Detect speculative-decode requests. We use -1 to mark prefill and + # plain-decode requests, so any value >= 0 is a (multi-query) + # spec-decode request. + if self.use_spec_decode and num_decode_draft_tokens_cpu is not None: + candidate_mask = num_decode_draft_tokens_cpu[: m.num_reqs] >= 0 + if bool(candidate_mask.any().item()): + spec_sequence_masks_cpu = candidate_mask + + if spec_sequence_masks_cpu is None: + return self._build_non_spec_metadata( + common_prefix_len, + common_attn_metadata, + fast_build, + num_decode_draft_tokens_cpu, + **kwargs, + ) + + del common_prefix_len, fast_build, kwargs + query_start_loc = m.query_start_loc + query_start_loc_cpu = m.query_start_loc_cpu + query_lens_cpu = torch.diff(query_start_loc_cpu) + block_table_tensor = mamba_get_block_table_tensor( + m.block_table_tensor, + m.seq_lens, + self.kv_cache_spec, + self.vllm_config.cache_config.mamba_cache_mode, + ) + + if query_start_loc.device.type == "cpu": + spec_sequence_masks = spec_sequence_masks_cpu + else: + spec_sequence_masks = async_tensor_h2d( + spec_sequence_masks_cpu, device=query_start_loc.device + ) + + # For causal_conv1d (non-spec prefill Triton kernel metadata). + nums_dict = None + batch_ptr = None + token_chunk_offset_ptr = None + has_initial_states_p = None + has_initial_states_d = None + num_computed_tokens_p = None + # Original request indices of the non-spec requests, ordered + # [decodes, prefills]. Used to gather per-request data consistently. + non_spec_req_idx_cpu: torch.Tensor | None = None + + query_lens = torch.diff(query_start_loc) + # Per-request classification by mask, NOT by position. With + # spec-decode, the front decode group can contain both spec-decode + # requests and plain non-spec single-token decodes. Spec requests are + # therefore not guaranteed to occupy the first num_spec_decodes slots. + non_spec_mask_cpu = ~spec_sequence_masks_cpu + decode_mask_cpu = non_spec_mask_cpu & (query_lens_cpu == 1) + prefill_mask_cpu = non_spec_mask_cpu & (query_lens_cpu > 1) + + num_spec_decodes = int(spec_sequence_masks_cpu.sum().item()) + num_decodes = int(decode_mask_cpu.sum().item()) + num_prefills = int(prefill_mask_cpu.sum().item()) + num_decode_tokens = num_decodes + num_prefill_tokens = int(query_lens_cpu[prefill_mask_cpu].sum().item()) + num_spec_decode_tokens = int( + query_lens_cpu[spec_sequence_masks_cpu].sum().item() + ) + # Max prefill query length, CPU-side (no device-to-host sync) for the + # PLE dilated short-conv packing buffer. + max_prefill_query_len = ( + int(query_lens_cpu[prefill_mask_cpu].max().item()) + if num_prefills > 0 + else 0 + ) + + # Original request indices grouped as + # [spec | non-spec decode | non-spec prefill]; each group keeps the + # original (already reordered) relative order via a stable nonzero. + spec_req_idx_cpu = spec_sequence_masks_cpu.nonzero(as_tuple=True)[0] + decode_req_idx_cpu = decode_mask_cpu.nonzero(as_tuple=True)[0] + prefill_req_idx_cpu = prefill_mask_cpu.nonzero(as_tuple=True)[0] + non_spec_req_idx_cpu = torch.cat((decode_req_idx_cpu, prefill_req_idx_cpu)) + spec_req_idx = spec_req_idx_cpu.to(query_start_loc.device) + non_spec_req_idx = non_spec_req_idx_cpu.to(query_start_loc.device) + + if num_decodes == 0 and num_prefills == 0: + # Pure speculative-decode batch: all real tokens are spec tokens. + spec_token_indx = torch.arange( + num_spec_decode_tokens, + dtype=torch.int32, + device=query_start_loc.device, + ) + non_spec_token_indx = torch.empty( + 0, dtype=torch.int32, device=query_start_loc.device + ) + spec_state_indices_tensor = block_table_tensor[spec_req_idx, 0] + non_spec_state_indices_tensor = None + spec_query_start_loc = query_start_loc[: num_spec_decodes + 1] + non_spec_query_start_loc = None + non_spec_query_start_loc_cpu = None + else: + # Mixed batch: build a per-token group key consistent with the + # request grouping above (spec=0 | decode=1 | prefill=2) and a + # stable sort, so tokens of each request stay contiguous and in + # request order. This yields spec tokens first, then the non-spec + # [decode, prefill] tokens. + req_group = torch.full( + (m.num_reqs,), + 2, + dtype=torch.int64, + device=query_start_loc.device, + ) + req_group[spec_req_idx] = 0 + req_group[decode_req_idx_cpu.to(query_start_loc.device)] = 1 + token_group = torch.repeat_interleave(req_group, query_lens) + token_perm = torch.argsort(token_group, stable=True) + spec_token_indx = token_perm[:num_spec_decode_tokens] + non_spec_token_indx = token_perm[num_spec_decode_tokens:] + + spec_state_indices_tensor = block_table_tensor[spec_req_idx, 0] + non_spec_state_indices_tensor = block_table_tensor[non_spec_req_idx, 0] + spec_query_start_loc = torch.zeros( + num_spec_decodes + 1, + dtype=torch.int32, + device=query_start_loc.device, + ) + torch.cumsum(query_lens[spec_req_idx], dim=0, out=spec_query_start_loc[1:]) + non_spec_query_start_loc = torch.zeros( + num_decodes + num_prefills + 1, + dtype=torch.int32, + device=query_start_loc.device, + ) + torch.cumsum( + query_lens[non_spec_req_idx], + dim=0, + out=non_spec_query_start_loc[1:], + ) + non_spec_query_start_loc_cpu = torch.zeros( + num_decodes + num_prefills + 1, dtype=torch.int32 + ) + torch.cumsum( + query_lens_cpu[non_spec_req_idx_cpu], + dim=0, + out=non_spec_query_start_loc_cpu[1:], + ) + + assert num_accepted_tokens is not None + # Accepted-token counts must follow the same request order as the + # speculative state indices. + num_accepted_tokens = num_accepted_tokens[ + spec_req_idx_cpu.to(num_accepted_tokens.device) + ] + + # Compute the conv-state slots for the non-spec decode/prefill split, + # plus the initial-state masks and Triton causal_conv1d metadata. + if non_spec_state_indices_tensor is None: + state_indices_tensor = block_table_tensor[:0, 0] + else: + state_indices_tensor = non_spec_state_indices_tensor + + # Build the regular decode/prefill state metadata inherited from the + # generic short-conv metadata contract. + query_start_loc_p = None + query_start_loc_d = None + state_indices_tensor_p = None + state_indices_tensor_d = None + if num_decodes > 0 or num_prefills > 0: + num_computed_tokens = m.compute_num_computed_tokens() + if non_spec_req_idx_cpu is not None: + non_spec_req_idx = non_spec_req_idx_cpu.to(num_computed_tokens.device) + num_computed_tokens = num_computed_tokens[non_spec_req_idx] + + state_indices_tensor_d = state_indices_tensor[:num_decodes] + state_indices_tensor_p = state_indices_tensor[ + num_decodes : num_decodes + num_prefills + ] + if num_decodes > 0: + has_initial_states_d = num_computed_tokens[:num_decodes] > 0 + assert non_spec_query_start_loc is not None + query_start_loc_d = non_spec_query_start_loc[: num_decodes + 1] + if num_prefills > 0: + num_computed_tokens_p = num_computed_tokens[ + num_decodes : num_decodes + num_prefills + ] + has_initial_states_p = num_computed_tokens_p > 0 + assert non_spec_query_start_loc is not None + assert non_spec_query_start_loc_cpu is not None + query_start_loc_p = ( + non_spec_query_start_loc[num_decodes:] - num_decode_tokens + ) + query_start_loc_p_cpu = ( + non_spec_query_start_loc_cpu[num_decodes:] - num_decode_tokens + ) + if query_start_loc.device.type != "cpu": + nums_dict, batch_ptr, token_chunk_offset_ptr = ( + compute_causal_conv1d_metadata( + query_start_loc_p_cpu, + device=query_start_loc.device, + ) + ) + + # Prepare persistent tensors for CUDA graph capture and replay. + # ``m.num_actual_tokens`` is already padded by the model runner. + # Request-level buffers use ``m.num_reqs`` while token-level buffers + # use their independently bounded token count. + batch_size = m.num_reqs + if ( + self.use_full_cuda_graph + and num_prefills == 0 + and num_decodes == 0 + and spec_sequence_masks is not None + and num_spec_decodes <= self.decode_cudagraph_max_bs + and num_spec_decode_tokens <= self.decode_cudagraph_max_tokens + ): + assert spec_state_indices_tensor is not None + self.spec_state_indices_tensor[:num_spec_decodes].copy_( + spec_state_indices_tensor, non_blocking=True + ) + spec_state_indices_tensor = self.spec_state_indices_tensor[:batch_size] + spec_state_indices_tensor[num_spec_decodes:].fill_(NULL_BLOCK_ID) + + self.spec_sequence_masks[:batch_size].copy_( + spec_sequence_masks[:batch_size], non_blocking=True + ) + spec_sequence_masks = self.spec_sequence_masks[:batch_size] + + assert spec_query_start_loc is not None + self.spec_query_start_loc[: num_spec_decodes + 1].copy_( + spec_query_start_loc, non_blocking=True + ) + spec_num_query_tokens = spec_query_start_loc[-1] + spec_query_start_loc = self.spec_query_start_loc[: batch_size + 1] + spec_query_start_loc[num_spec_decodes + 1 :].fill_(spec_num_query_tokens) + + assert num_accepted_tokens is not None + self.num_accepted_tokens[:num_spec_decodes].copy_( + num_accepted_tokens, non_blocking=True + ) + num_accepted_tokens = self.num_accepted_tokens[:batch_size] + num_accepted_tokens[num_spec_decodes:].fill_(1) + + return PleShortConvAttentionMetadata( + num_prefills=num_prefills, + num_prefill_tokens=num_prefill_tokens, + num_decodes=num_decodes, + num_decode_tokens=num_decode_tokens, + num_reqs=m.num_reqs, + num_spec_decodes=num_spec_decodes, + num_spec_decode_tokens=num_spec_decode_tokens, + num_actual_tokens=m.num_actual_tokens, + spec_query_len=self.num_spec + 1, + max_prefill_query_len=max_prefill_query_len, + query_start_loc=query_start_loc, + state_indices_tensor=state_indices_tensor, + has_initial_states_p=has_initial_states_p, + has_initial_states_d=has_initial_states_d, + non_spec_query_start_loc=non_spec_query_start_loc, + spec_query_start_loc=spec_query_start_loc, + spec_state_indices_tensor=spec_state_indices_tensor, + spec_sequence_masks=spec_sequence_masks, + spec_token_indx=spec_token_indx, + non_spec_token_indx=non_spec_token_indx, + num_accepted_tokens=num_accepted_tokens, + num_decode_draft_tokens_cpu=num_decode_draft_tokens_cpu, + nums_dict=nums_dict, + batch_ptr=batch_ptr, + token_chunk_offset_ptr=token_chunk_offset_ptr, + query_start_loc_p=query_start_loc_p, + query_start_loc_d=query_start_loc_d, + state_indices_tensor_p=state_indices_tensor_p, + state_indices_tensor_d=state_indices_tensor_d, + num_computed_tokens_p=num_computed_tokens_p, + block_idx_last_scheduled_token=None, + block_idx_first_scheduled_token_p=None, + block_idx_last_computed_token=None, + block_idx_last_scheduled_token_prev_step=None, + seq_lens=m.seq_lens, + ) + + def build_for_cudagraph_capture( + self, common_attn_metadata: CommonAttentionMetadata + ) -> PleShortConvAttentionMetadata: + """Build metadata for full CUDA graph capture. + + Currently, only decode is supported for full CUDA graphs with + short-conv. + """ + m = common_attn_metadata + assert ( + m.num_reqs <= self.decode_cudagraph_max_bs + and m.num_actual_tokens <= self.decode_cudagraph_max_tokens + ), ( + "ShortConv only supports decode-only full CUDAGraph capture. " + f"Make sure batch size ({m.num_reqs}) <= " + f"cudagraph capture size ({self.decode_cudagraph_max_bs}) and " + f"number of tokens ({m.num_actual_tokens}) <= " + f"token capture size ({self.decode_cudagraph_max_tokens})." + ) + + if self.use_spec_decode: + num_accepted_tokens = torch.diff(m.query_start_loc) + num_decode_draft_tokens_cpu = (num_accepted_tokens - 1).cpu() + return self.build( + 0, + m, + num_accepted_tokens=num_accepted_tokens, + num_decode_draft_tokens_cpu=num_decode_draft_tokens_cpu, + ) + return self.build(0, m) diff --git a/vllm/v1/core/kv_cache_coordinator.py b/vllm/v1/core/kv_cache_coordinator.py index c097542f89..3d5adb9014 100644 --- a/vllm/v1/core/kv_cache_coordinator.py +++ b/vllm/v1/core/kv_cache_coordinator.py @@ -599,7 +599,11 @@ class HybridKVCacheCoordinator(KVCacheCoordinator): self.hash_block_size = hash_block_size self.dcp_world_size = dcp_world_size group_block_sizes = [ - manager.block_size for manager in self.single_type_managers + manager.block_size + for manager, group in zip( + self.single_type_managers, kv_cache_config.kv_cache_groups + ) + if group.kv_cache_spec.prefix_cacheable ] assert all( block_size % hash_block_size == 0 for block_size in group_block_sizes @@ -638,8 +642,11 @@ class HybridKVCacheCoordinator(KVCacheCoordinator): if self.enable_partial_hash_hits: unsupported_partial_hit_managers = { type(manager).__name__ - for manager in self.single_type_managers - if not manager.supports_fine_grained_hash_lookup + for manager, group in zip( + self.single_type_managers, kv_cache_config.kv_cache_groups + ) + if group.kv_cache_spec.prefix_cacheable + and not manager.supports_fine_grained_hash_lookup and manager.block_size != hash_block_size } if unsupported_partial_hit_managers: @@ -668,6 +675,8 @@ class HybridKVCacheCoordinator(KVCacheCoordinator): """ self.attention_groups: list[SpecGroup] = [] for i, g in enumerate(self.kv_cache_config.kv_cache_groups): + if not g.kv_cache_spec.prefix_cacheable: + continue manager_cls = self.single_type_managers[i].__class__ spec = g.kv_cache_spec use_eagle = i in self.eagle_group_ids @@ -687,8 +696,8 @@ class HybridKVCacheCoordinator(KVCacheCoordinator): SpecGroup(spec, [i], manager_cls, use_eagle) ) - assert len(self.attention_groups) > 1, ( - "HybridKVCacheCoordinator requires at least two attention groups." + assert self.attention_groups, ( + "HybridKVCacheCoordinator requires at least one cacheable group." ) # Put full attention first: its efficient left-to-right scan provides diff --git a/vllm/v1/core/kv_cache_manager.py b/vllm/v1/core/kv_cache_manager.py index 613aab73c1..ef9f8da7be 100644 --- a/vllm/v1/core/kv_cache_manager.py +++ b/vllm/v1/core/kv_cache_manager.py @@ -779,6 +779,14 @@ class KVCacheManager: self.kv_cache_config.kv_cache_groups, strict=True, ): + if not group.kv_cache_spec.prefix_cacheable: + # Scratch groups (e.g. the CSA compressor ring) hold a fixed + # block covering no token range, so a lookup result never + # carries computed blocks for them and their block size does + # not divide the endpoint. + assert not group_blocks + truncated.append([]) + continue assert num_computed_tokens % manager.block_size == 0 num_blocks = num_computed_tokens // manager.block_size if isinstance(group.kv_cache_spec, MambaSpec): diff --git a/vllm/v1/core/kv_cache_utils.py b/vllm/v1/core/kv_cache_utils.py index d578b4300a..45d8e8532e 100644 --- a/vllm/v1/core/kv_cache_utils.py +++ b/vllm/v1/core/kv_cache_utils.py @@ -16,7 +16,7 @@ from vllm import envs from vllm.config import VllmConfig from vllm.logger import init_logger from vllm.utils.hashing import xxhash, xxhash_cbor -from vllm.utils.math_utils import cdiv, round_up +from vllm.utils.math_utils import cdiv from vllm.utils.mem_utils import format_gib from vllm.utils.torch_utils import get_dtype_size from vllm.v1.kv_cache_interface import ( @@ -717,21 +717,52 @@ def resolve_kv_cache_block_sizes( # scheduler block size. Read the mode from the resolved group spec because # its block size may have been updated independently of cache_config. if any( - isinstance(g.kv_cache_spec, MambaSpec) - and g.kv_cache_spec.mamba_cache_mode != "align" - for g in groups + isinstance(spec, MambaSpec) and spec.mamba_cache_mode != "align" + for group in groups + for spec in iter_layer_specs(group.kv_cache_spec) ): return scheduler_block_size, scheduler_block_size + hashing_sizes = [ + block_size + for group, block_size in zip(groups, group_block_sizes) + if group.kv_cache_spec.prefix_cacheable + ] or group_block_sizes requested = cache_config.prefix_match_unit - hash_block_size = ( - requested if requested is not None else math.gcd(*group_block_sizes) - ) - if any(bs % hash_block_size != 0 for bs in group_block_sizes): + hash_block_size = requested if requested is not None else math.gcd(*hashing_sizes) + if any(bs % hash_block_size != 0 for bs in hashing_sizes): raise ValueError( - f"Invalid prefix_match_unit={hash_block_size}; all KV cache group " - f"block sizes must be divisible by prefix_match_unit. " - f"Got group block sizes={group_block_sizes}." + f"Invalid prefix_match_unit={hash_block_size}; prefix-cacheable " + "KV cache group block sizes must be divisible by prefix_match_unit. " + f"Got group block sizes={group_block_sizes}, " + f"prefix-cacheable={hashing_sizes}." + ) + prefix_alignments = { + spec.tokens_per_state + for group in groups + for spec in iter_layer_specs(group.kv_cache_spec) + if spec.prefix_cacheable + and isinstance(spec.tokens_per_state, int) + and spec.tokens_per_state > 1 + } + has_partial_mamba_group = any( + isinstance(spec, MambaSpec) + and spec.mamba_cache_mode == "align" + and ( + (dcp == 1 and block_size > hash_block_size) + or (dcp > 1 and block_size >= hash_block_size) + ) + for group, block_size in zip(groups, group_block_sizes) + for spec in iter_layer_specs(group.kv_cache_spec) + ) + cache_hit_alignment = ( + hash_block_size if has_partial_mamba_group else scheduler_block_size + ) + if any(cache_hit_alignment % alignment for alignment in prefix_alignments): + raise ValueError( + f"Invalid prefix_match_unit={hash_block_size}; prefix-cache boundaries " + "must align with each spec's per-state compression. " + f"Got alignments={sorted(prefix_alignments)}." ) return scheduler_block_size, hash_block_size @@ -1577,49 +1608,6 @@ def unify_hybrid_kv_cache_specs(kv_cache_spec: dict[str, KVCacheSpec]): kv_cache_spec.update(_promote_local_kv_cache_specs(kv_cache_spec)) -def group_and_unify_kv_cache_specs( - kv_cache_spec: dict[str, KVCacheSpec], -) -> list[UniformTypeKVCacheSpecs] | None: - """ - Group the KV cache specs and unify each group into one UniformTypeKVCacheSpecs. - Currently, this is only used for DeepseekV4. - """ - if not any( - isinstance(spec, SlidingWindowMLASpec) for spec in kv_cache_spec.values() - ): - return None - - # SlidingWindowMLASpec models with uniform page sizes don't need tuple packing. - page_sizes = {spec.page_size_bytes for spec in kv_cache_spec.values()} - if len(page_sizes) <= 1: - return None - - mla_specs: dict[str, KVCacheSpec] = {} - grouped_swa_mla_specs: dict[tuple[int, int], dict[str, KVCacheSpec]] = defaultdict( - dict - ) - # NOTE: Here we group SWA layers by (block_size, sliding_window), which separates - # SWA layers, C4I+C4A layers, and C128A layers into three different groups. It can - # be fragile with only block_size and sliding_window as keys, but fine for now. - for name, spec in kv_cache_spec.items(): - if isinstance(spec, SlidingWindowMLASpec): - grouped_swa_mla_specs[(spec.block_size, spec.sliding_window)][name] = spec - elif isinstance(spec, MLAAttentionSpec): - mla_specs[name] = spec - - assert len(mla_specs) > 0 - mla_uniform_spec = UniformTypeKVCacheSpecs.from_specs(mla_specs) - assert mla_uniform_spec is not None - - swa_uniform_specs: list[UniformTypeKVCacheSpecs] = [] - for spec_dict in grouped_swa_mla_specs.values(): - uniform_spec = UniformTypeKVCacheSpecs.from_specs(spec_dict) - assert uniform_spec is not None - swa_uniform_specs.append(uniform_spec) - - return [mla_uniform_spec, *swa_uniform_specs] - - def _approximate_gcd(values: Sequence[int], *, lower_bound: int | None = None) -> int: """Pick a chunk size that minimizes total upward padding. @@ -1655,91 +1643,145 @@ def _approximate_gcd(values: Sequence[int], *, lower_bound: int | None = None) - return best_d -def _get_kv_cache_groups_uniform_groups( - grouped_specs: list[UniformTypeKVCacheSpecs], -) -> list[KVCacheGroupSpec]: +def _get_packed_kv_cache_groups( + vllm_config: VllmConfig, + kv_cache_spec: dict[str, KVCacheSpec], +) -> list[KVCacheGroupSpec] | None: + """Group mixed-page-size layers for contiguous block-outermost packing. + + Greedily buckets layers into uniform-type specs. Buckets with equal layer + counts per page size are treated as a repeating layer pattern (one layer + per page size) and split into groups covering the same number of pattern + repeats (picked by ``_approximate_gcd`` to minimize padding), so all + groups pack into the same per-block layout. Mamba buckets are additionally + split to fit the block the attention buckets already need. + Returns None when the layout is not block-outermost or all layers already + share one page size. """ - Generate the KV cache groups from the grouped specs. - """ - assert len(grouped_specs) > 0 and all( - isinstance(spec, UniformTypeKVCacheSpecs) for spec in grouped_specs + layout = vllm_config.cache_config.get_resolved_kv_cache_layout() + page_sizes = {spec.page_size_bytes for spec in kv_cache_spec.values()} + if not layout.is_block_outermost or len(page_sizes) <= 1: + return None + + buckets: list[dict[str, KVCacheSpec]] = [] + for name, spec in kv_cache_spec.items(): + for bucket in buckets: + candidate = {**bucket, name: spec} + if UniformTypeKVCacheSpecs.is_uniform_type(candidate): + bucket[name] = spec + break + else: + buckets.append({name: spec}) + + bucketed = [] + for bucket in buckets: + uniform_spec = UniformTypeKVCacheSpecs.from_specs(bucket) + assert uniform_spec is not None + page_size_layers: dict[int, list[str]] = defaultdict(list) + for layer_name, layer_spec in bucket.items(): + page_size_layers[layer_spec.page_size_bytes].append(layer_name) + # Only 1:1 patterns (one layer of each page size per repeat) are + # supported; counts sharing a gcd > 1 (e.g. 2:1) could in principle + # repeat too, but such buckets are emitted whole instead. + balanced = len(set(map(len, page_size_layers.values()))) == 1 + bucketed.append((uniform_spec, page_size_layers, balanced)) + + # Balanced buckets that mix page sizes must stay whole, so the largest one + # sets a floor on the repeats per group; larger single-size buckets are + # split down toward it. No such bucket means nothing needs packing. + min_repeats_per_group = max( + ( + spec.get_max_layers_per_page_size() + for spec, page_size_layers, balanced in bucketed + if balanced and len(page_size_layers) > 1 + ), + default=0, ) - # For now, we restrict the first grouped_spec to be UniformTypeKVCacheSpecs - # containing only MLAAttentionSpec. - full_mla_spec = grouped_specs[0] - assert all( - isinstance(spec, MLAAttentionSpec) - for spec in full_mla_spec.kv_cache_specs.values() - ) - full_mla_group = KVCacheGroupSpec( - layer_names=list(full_mla_spec.kv_cache_specs.keys()), - kv_cache_spec=full_mla_spec, + repeats_per_group = ( + _approximate_gcd( + [ + spec.get_max_layers_per_page_size() + for spec, _, balanced in bucketed + if balanced + ], + lower_bound=min_repeats_per_group, + ) + if min_repeats_per_group + else None ) - # We define a layer tuple as a group of layers with different page sizes, and - # one UniformTypeKVCacheSpecs contains a list of layer tuples. - # For example, if we have 11 C4 layers and 10 C128 layers, we can define a layer - # tuple as [C4I, C4A, C128], and the full_mla_group will contain "11" layer tuples. - # The other uniform KV cache specs will be similarly partitioned into layer tuples. - # Say we have 21 SWA layers, all with the same page size, then we will have "21" - # layer tuples. - num_layer_tuples_per_group: list[int] = [ - g_spec.get_num_layer_tuples() for g_spec in grouped_specs - ] - # Choose `num_layer_tuples` to minimize total padding across groups. - num_layer_tuples = _approximate_gcd( - num_layer_tuples_per_group, lower_bound=num_layer_tuples_per_group[0] - ) - # Round up to the nearest multiple of `num_layer_tuples` (i.e., padding) - num_layer_tuples_per_group = [ - round_up(x, num_layer_tuples) for x in num_layer_tuples_per_group - ] + def num_groups_for(spec: UniformTypeKVCacheSpecs, balanced: bool) -> int: + if balanced and repeats_per_group is not None: + return cdiv(spec.get_max_layers_per_page_size(), repeats_per_group) + return 1 - swa_mla_specs = grouped_specs[1:] - assert all( - isinstance(spec, SlidingWindowMLASpec) - for group in swa_mla_specs - for spec in group.kv_cache_specs.values() + def widest_group_bytes(page_size_layers: dict[int, list[str]], n: int) -> int: + """Page bytes of the largest of the n groups a bucket splits into.""" + return sum( + cdiv(len(names), n) * page for page, names in page_size_layers.items() + ) + + # Bytes a block must hold however the mamba buckets end up split: a mamba + # bucket can go down to one state per group, every other bucket's split is + # already fixed by the repeat pattern. + anchor_bytes = max( + ( + widest_group_bytes( + page_size_layers, + len(spec.kv_cache_specs) + if isinstance(spec.first_spec, MambaSpec) + else num_groups_for(spec, balanced), + ) + for spec, page_size_layers, balanced in bucketed + ), + default=0, ) - # Split each SWA UniformKV group into smaller groups to align their - # numbers of layer tuples. The packed block planner overlays groups, so - # their page sizes do not need to match. - swa_mla_groups = [] - for sm_spec in swa_mla_specs: - layers_per_size: dict[int, list[str]] = defaultdict(list) + groups = [] + for spec, page_size_layers, balanced in bucketed: + num_groups = num_groups_for(spec, balanced) + # `_align_hybrid_block_size` pads a mamba state up to one attention + # page, so cap a mamba group at the states a block already fits rather + # than let it widen the block. + if anchor_bytes and isinstance(spec.first_spec, MambaSpec): + states_per_block = max(anchor_bytes // spec.first_spec.page_size_bytes, 1) + num_groups = max( + num_groups, cdiv(len(spec.kv_cache_specs), states_per_block) + ) + if num_groups == 1: + groups.append(KVCacheGroupSpec(list(spec.kv_cache_specs), spec)) + continue - for layer_name, layer_spec in sm_spec.kv_cache_specs.items(): - layers_per_size[layer_spec.page_size_bytes].append(layer_name) - # NOTE(yifan): for now, inside a UniformKV group, each page_size should - # have the same number of layers. This also means we don't need to pad layers - # inside a partial-full layer tuple. - assert len(set(len(layers) for layers in layers_per_size.values())) == 1 - num_layers_per_size = len(next(iter(layers_per_size.values()))) - - # Split layers inside each UniformKV group for aligned #(layers). - # See `_get_kv_cache_groups_uniform_page_size` for more details. - num_tuple_groups = cdiv(num_layers_per_size, num_layer_tuples) - layer_tuples = list(zip(*layers_per_size.values())) - for i in range(num_tuple_groups): - group_layer_tuples = layer_tuples[i::num_tuple_groups] - # Flatten tuples and build dict for from_specs + pattern_repeats = list(zip(*page_size_layers.values())) + for i in range(num_groups): group_layer_names = [ - name for layer_tuple in group_layer_tuples for name in layer_tuple + name for repeat in pattern_repeats[i::num_groups] for name in repeat ] group_layer_specs = { - name: sm_spec.kv_cache_specs[name] for name in group_layer_names + name: spec.kv_cache_specs[name] for name in group_layer_names } - sub_sm_spec = UniformTypeKVCacheSpecs.from_specs(group_layer_specs) - assert sub_sm_spec is not None - swa_mla_groups.append( - KVCacheGroupSpec( - layer_names=group_layer_names, - kv_cache_spec=sub_sm_spec, - ) - ) + group_spec = UniformTypeKVCacheSpecs.from_specs(group_layer_specs) + assert group_spec is not None + groups.append(KVCacheGroupSpec(group_layer_names, group_spec)) - return [full_mla_group, *swa_mla_groups] + _annotate_eagle_groups( + vllm_config, + kv_cache_spec, + groups, + use_deepseek_v4_fallback=_is_deepseek_v4_eagle(vllm_config), + ) + _warn_if_unannotated_eagle_mamba(vllm_config, groups) + return groups + + +def _is_deepseek_v4_eagle(vllm_config: VllmConfig) -> bool: + spec_config = vllm_config.speculative_config + if spec_config is None or not spec_config.use_eagle(): + return False + model_config = vllm_config.model_config + return ( + model_config is not None and model_config.hf_config.model_type == "deepseek_v4" + ) def _annotate_eagle_groups( @@ -1763,9 +1805,10 @@ def _annotate_eagle_groups( the target's own decoder layer and so carries no spec marker. Its draft attention layer is always the last registered layer, so flag whichever group holds it. This rule is only valid where the groups partition - exactly the layers of ``kv_cache_spec``, which is true on the - group_and_unify path and not in general; other callers must leave - ``use_deepseek_v4_fallback`` False. + exactly the layers of ``kv_cache_spec``, which is true on the packed + grouping path and not in general; other callers must leave + ``use_deepseek_v4_fallback`` False. The caller gates this fallback on + the configured model type. FIXME(yifan): avoid/generalize this hacky check. Args: @@ -1773,25 +1816,21 @@ def _annotate_eagle_groups( kv_cache_spec: The kv cache spec of each attention layer, in layer registration order. Only read by rule 2. kv_cache_groups: Groups to annotate in place. - use_deepseek_v4_fallback: Enable rule 2. Only the group_and_unify path - may set this. + use_deepseek_v4_fallback: Enable rule 2 for a DeepseekV4 packed group. """ spec_config = vllm_config.speculative_config if spec_config is None or not spec_config.use_eagle(): return for group in kv_cache_groups: - if getattr(group.kv_cache_spec, "non_causal_multi_token_decode", False): + if any( + getattr(spec, "non_causal_multi_token_decode", False) + for spec in iter_layer_specs(group.kv_cache_spec) + ): group.is_eagle_group = True if not use_deepseek_v4_fallback: return - # Detection uses the merged MLA spec's model_version. - if not any( - getattr(spec, "model_version", None) == "deepseek_v4" - for spec in kv_cache_spec.values() - ): - return last_layer = next(reversed(kv_cache_spec)) for group in kv_cache_groups: if last_layer in group.layer_names: @@ -1822,7 +1861,10 @@ def _warn_if_unannotated_eagle_mamba( mamba_groups = [ idx for idx, group in enumerate(kv_cache_groups) - if isinstance(group.kv_cache_spec, MambaSpec) + if any( + isinstance(spec, MambaSpec) + for spec in iter_layer_specs(group.kv_cache_spec) + ) ] if not mamba_groups: return @@ -1846,7 +1888,8 @@ def _largest_divisor_at_most(value: int, limit: int) -> int: def get_kv_cache_groups( - vllm_config: VllmConfig, kv_cache_spec: dict[str, KVCacheSpec] + vllm_config: VllmConfig, + kv_cache_spec: dict[str, KVCacheSpec], ) -> list[KVCacheGroupSpec]: """ Split the layers in the model into groups with the same KV cache spec. @@ -1876,22 +1919,8 @@ def get_kv_cache_groups( # full attention, or all layers are sliding window attention with the # same window size). Put all layers into one group. return _get_kv_cache_groups_uniform_type(uniform_spec) - elif grouped_specs := group_and_unify_kv_cache_specs(kv_cache_spec): - # DeepseekV4 case: All layers need the same number of token slots, - # yet some layers are full attention while others are sliding window - # attention in different sizes. Need to group layers into multiple - # UniformTypeKVCacheSpecs. - kv_cache_groups = _get_kv_cache_groups_uniform_groups(grouped_specs) - _annotate_eagle_groups( - vllm_config, - kv_cache_spec, - kv_cache_groups, - use_deepseek_v4_fallback=True, - ) - return kv_cache_groups - - # Pull HiddenStateCacheSpec layers out before the general multi-group - # path so they don't affect page-size unification or grouping. + # Hidden-state layers use their own block table and must not be absorbed + # into a compatible attention bucket. hidden_specs = { k: v for k, v in kv_cache_spec.items() if isinstance(v, HiddenStateCacheSpec) } @@ -1901,6 +1930,14 @@ def get_kv_cache_groups( if not isinstance(v, HiddenStateCacheSpec) } + if packed_groups := _get_packed_kv_cache_groups(vllm_config, filtered_spec): + # Block-outermost blocks are strided by the widest group, so hidden + # groups need no page alignment. + packed_groups += [ + KVCacheGroupSpec([name], spec) for name, spec in hidden_specs.items() + ] + return packed_groups + # Prefer preserving each layer's cache semantics. If physical pages cannot # be unified, try a supported allocation-only fallback before failing. try: diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index ecea90a25c..83ed59799e 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -1008,6 +1008,18 @@ class Scheduler(SchedulerInterface): ) if num_new_tokens == 0: break + if ( + pad_spec_decode + and num_new_tokens != 1 + self.num_spec_tokens + ): + # Alignment clipped the placeholder rows. The split + # aligns prefill chunks, but the padded tail rows are + # speculative positions, not prefill tokens. A padded + # request must keep all 1 + num_spec rows or the + # sampler's row count stops matching its query rows, + # so drop the padding instead of shortening it. + num_new_tokens = 1 + pad_spec_decode = False # Schedule encoder inputs. if request.has_encoder_inputs: @@ -1170,6 +1182,7 @@ class Scheduler(SchedulerInterface): request.status = RequestStatus.RUNNING request.num_computed_tokens = num_computed_tokens if pad_spec_decode: + assert num_new_tokens == 1 + self.num_spec_tokens scheduled_spec_decode_tokens[request_id] = [ -1 ] * self.num_spec_tokens diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index b08f6f3c38..f3036eb051 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -18,6 +18,7 @@ from vllm.v1.core.kv_cache_utils import ( from vllm.v1.kv_cache_interface import ( AttentionSpec, ChunkedLocalAttentionSpec, + CircularBufferSpec, CrossAttentionSpec, FullAttentionSpec, HiddenStateCacheSpec, @@ -83,8 +84,10 @@ class SingleTypeKVCacheManager(ABC): self._max_admission_blocks_per_request = max_admission_blocks_per_request # Record newly allocated block ids only when worker-side zeroing will # consume them and this manager holds a spec type that gets zeroed. - self._record_new_block_ids = needs_kv_cache_zeroing and isinstance( - kv_cache_spec, AttentionSpec + self._record_new_block_ids = ( + needs_kv_cache_zeroing + and isinstance(kv_cache_spec, AttentionSpec) + and not isinstance(kv_cache_spec, CircularBufferSpec) ) self.new_block_ids: list[int] = [] @@ -1109,6 +1112,93 @@ class SlidingWindowManager(SingleTypeKVCacheManager): return 0 +class CircularBufferManager(FullAttentionManager): + """Claims the ring's single block per request; prefix caching disabled.""" + + supports_fine_grained_hash_lookup: ClassVar[bool] = False + + def _claim_ring_block(self, request_id: str) -> list[KVCacheBlock]: + req_blocks = self.req_to_blocks[request_id] + if req_blocks: + return [] + new_blocks = self.block_pool.get_new_blocks(1) + req_blocks.extend(new_blocks) + if self._record_new_block_ids: + self.new_block_ids.extend(block.block_id for block in new_blocks) + return new_blocks + + def get_num_blocks_to_allocate( + self, + request_id: str, + num_tokens: int, + new_computed_blocks: Sequence[KVCacheBlock], + total_computed_tokens: int, + num_local_computed_tokens: int, + num_tokens_main_model: int, + apply_admission_cap: bool = False, + ) -> int: + return 0 if self.req_to_blocks.get(request_id) else 1 + + def allocate_new_blocks( + self, request_id: str, num_tokens: int, num_tokens_main_model: int + ) -> list[KVCacheBlock]: + return self._claim_ring_block(request_id) + + def allocate_external_computed_blocks( + self, + request_id: str, + num_local_computed_tokens: int, + num_external_computed_tokens: int, + ) -> None: + self._claim_ring_block(request_id) + + @classmethod + def find_longest_cache_hit( + cls, + block_hashes: BlockHashList, + max_length: int, + kv_cache_group_ids: list[int], + block_pool: BlockPool, + kv_cache_spec: KVCacheSpec, + drop_eagle_block: bool, + alignment_tokens: int, + dcp_world_size: int = 1, + pcp_world_size: int = 1, + ) -> tuple[tuple[list[KVCacheBlock], ...], int]: + return tuple([] for _ in kv_cache_group_ids), 0 + + def cache_blocks( + self, + request: Request, + num_tokens: int, + retention_interval: int | None = None, + ) -> None: + return + + def add_local_computed_blocks( + self, + request_id: str, + new_computed_blocks: Sequence[KVCacheBlock], + num_local_computed_tokens: int, + num_external_computed_tokens: int, + ) -> None: + return + + def remove_skipped_blocks( + self, + request_id: str, + processed_computed_tokens: int, + num_prompt_tokens: int | None = None, + ) -> None: + return + + def get_num_common_prefix_blocks(self, running_request_id: str) -> int: + return 0 + + def get_num_skipped_tokens(self, num_computed_tokens: int) -> int: + return 0 + + class ChunkedLocalAttentionManager(SingleTypeKVCacheManager): def __init__(self, kv_cache_spec: ChunkedLocalAttentionSpec, **kwargs) -> None: super().__init__(kv_cache_spec, **kwargs) @@ -1953,6 +2043,11 @@ def register_all_kvcache_specs(vllm_config): SlidingWindowManager, uniform_type_base_spec=SlidingWindowSpec, ) + KVCacheSpecRegistry.register( + CircularBufferSpec, + CircularBufferManager, + uniform_type_base_spec=CircularBufferSpec, + ) KVCacheSpecRegistry.register( SlidingWindowMLASpec, SlidingWindowManager, diff --git a/vllm/v1/kv_cache_interface.py b/vllm/v1/kv_cache_interface.py index 79c0055edc..1007c48a21 100644 --- a/vllm/v1/kv_cache_interface.py +++ b/vllm/v1/kv_cache_interface.py @@ -150,6 +150,10 @@ class KVCacheSpec: # number of tokens in a block block_size: int + @property + def prefix_cacheable(self) -> bool: + return True + @property def num_heads(self) -> int: raise NotImplementedError @@ -745,6 +749,38 @@ class SlidingWindowSpec(AttentionSpec): ) +@dataclass(frozen=True, kw_only=True) +class CircularBufferSpec(AttentionSpec): + """One block per request holding the raw keys of the token group that + is still being compressed. + + ``block_size`` is the ring capacity. It must exceed the compression ratio + by the speculative lookahead: a speculative step stores all of its rows, + drafts included, before acceptance is known, while the next step still + reads the open group's committed keys from the ring. + """ + + def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int: + # The ring occupies one block per request for its whole lifetime. + del vllm_config + return self.page_size_bytes + + def max_num_blocks_per_req(self, vllm_config: VllmConfig, max_len: int) -> int: + del vllm_config, max_len + return 1 + + def is_uniform_with_collection( + self, kv_cache_specs: dict[str, KVCacheSpec] + ) -> bool: + return all( + isinstance(spec, CircularBufferSpec) for spec in kv_cache_specs.values() + ) + + @property + def prefix_cacheable(self) -> bool: + return False + + @dataclass(frozen=True, kw_only=True) class SlidingWindowMLASpec(SlidingWindowSpec): """Sliding window attention with MLA cache format.""" @@ -814,7 +850,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec): @dataclass(frozen=True) class MambaSpec(KVCacheSpec): shapes: tuple[tuple[int, ...], ...] - dtypes: tuple[torch.dtype] + dtypes: tuple[torch.dtype, ...] page_size_padded: int | None = None mamba_type: MambaAttentionBackendEnum = MambaAttentionBackendEnum.MAMBA2 mamba_cache_mode: str = "none" @@ -822,6 +858,9 @@ class MambaSpec(KVCacheSpec): num_prefill_checkpoint_blocks: int = 0 num_heads: int = 1 tokens_per_state: int = -1 + # False: the state is sharded across TP ranks (e.g. GDN). True: every TP + # rank holds the full state (e.g. the replicated PLE conv state). + tp_replicated: bool = False @property def state_content_size_bytes(self) -> int: @@ -873,6 +912,8 @@ class MambaSpec(KVCacheSpec): isinstance(spec, MambaSpec) and spec.num_speculative_blocks == self.num_speculative_blocks and spec.num_prefill_checkpoint_blocks == self.num_prefill_checkpoint_blocks + and spec.page_size_bytes == self.page_size_bytes + and spec.tp_replicated == self.tp_replicated for spec in kv_cache_specs.values() ) @@ -963,6 +1004,15 @@ class UniformTypeKVCacheSpecs(KVCacheSpec): kv_cache_specs: dict[str, KVCacheSpec] + @property + def prefix_cacheable(self) -> bool: + return all(spec.prefix_cacheable for spec in self.kv_cache_specs.values()) + + @property + def first_spec(self) -> KVCacheSpec: + """Return the first spec in the group.""" + return next(iter(self.kv_cache_specs.values())) + @property def page_size_bytes(self) -> int: return sum(spec.page_size_bytes for spec in self.kv_cache_specs.values()) @@ -1015,8 +1065,9 @@ class UniformTypeKVCacheSpecs(KVCacheSpec): else: return None - # NOTE: below util functions are only used by DeepseekV4 for now. - def get_num_layer_tuples(self) -> int: + def get_max_layers_per_page_size(self) -> int: + """Max number of layers sharing a page size. For a balanced bucket + this equals the number of repetitions of the layer pattern.""" return Counter( spec.page_size_bytes for spec in self.kv_cache_specs.values() ).most_common(1)[0][1] @@ -1212,7 +1263,11 @@ class KVCacheConfig: @property def has_mamba_layers(self) -> bool: - return any(isinstance(g.kv_cache_spec, MambaSpec) for g in self.kv_cache_groups) + return any( + isinstance(spec, MambaSpec) + for group in self.kv_cache_groups + for spec in iter_layer_specs(group.kv_cache_spec) + ) @property def has_mixed_precision_kv_cache(self) -> bool: diff --git a/vllm/v1/worker/gpu/block_table.py b/vllm/v1/worker/gpu/block_table.py index 5e09383ee6..c01895e635 100644 --- a/vllm/v1/worker/gpu/block_table.py +++ b/vllm/v1/worker/gpu/block_table.py @@ -26,6 +26,7 @@ class BlockTables: cp_size: int = 1, cp_rank: int = 0, cp_interleave: int = 1, + slot_mapping_enabled: list[bool] | None = None, ): self.block_sizes = block_sizes self.kernel_block_sizes = kernel_block_sizes @@ -39,6 +40,10 @@ class BlockTables: self.num_kv_cache_groups = len(self.block_sizes) assert len(max_num_blocks_per_group) == self.num_kv_cache_groups + if slot_mapping_enabled is None: + slot_mapping_enabled = [True] * self.num_kv_cache_groups + assert len(slot_mapping_enabled) == self.num_kv_cache_groups + self._slot_mapping_enabled = slot_mapping_enabled self.blocks_per_kv_block = [ bs // kbs for bs, kbs in zip(block_sizes, kernel_block_sizes) @@ -100,6 +105,9 @@ class BlockTables: self.kernel_block_sizes_tensor = torch.tensor( self.kernel_block_sizes, dtype=torch.int32, device=self.device ) + self.slot_mapping_enabled = torch.tensor( + self._slot_mapping_enabled, dtype=torch.bool, device=self.device + ) self.input_block_table_ptrs = self._make_ptr_tensor(self.input_block_tables) def append_block_ids( @@ -202,6 +210,7 @@ class BlockTables: self.block_table_strides, self.block_sizes_tensor, self.kernel_block_sizes_tensor, + self.slot_mapping_enabled, slot_mappings, slot_mappings.stride(0), self.cp_rank, @@ -274,6 +283,7 @@ def _compute_slot_mappings_kernel( block_table_strides, # [num_kv_cache_groups] block_sizes, # [num_kv_cache_groups] kernel_block_sizes, # [num_kv_cache_groups] + slot_mapping_enabled, # [num_kv_cache_groups] slot_mappings_ptr, # [num_kv_cache_groups, max_num_tokens] slot_mappings_stride, cp_rank, @@ -302,6 +312,7 @@ def _compute_slot_mappings_kernel( block_table_stride = tl.load(block_table_strides + group_id) kv_block_size = tl.load(block_sizes + group_id) kernel_block_size = tl.load(kernel_block_sizes + group_id) + mapping_enabled = tl.load(slot_mapping_enabled + group_id) req_state_idx = tl.load(idx_mapping + batch_idx) start_idx = tl.load(query_start_loc + batch_idx) @@ -325,7 +336,9 @@ def _compute_slot_mappings_kernel( local_offsets = rounds * CP_INTERLEAVE + remainder local_positions = virtual_block_indices * kv_block_size + local_offsets - block_indices = local_positions // kernel_block_size + block_indices = tl.where( + mapping_enabled, local_positions // kernel_block_size, 0 + ) block_offsets = local_positions % kernel_block_size block_numbers = tl.load( block_table_ptr + req_state_idx * block_table_stride + block_indices, @@ -336,4 +349,5 @@ def _compute_slot_mappings_kernel( if CP_SIZE != 1: slot_ids = tl.where(is_local, slot_ids, PAD_ID) + slot_ids = tl.where(mapping_enabled, slot_ids, PAD_ID) tl.store(slot_mapping_ptr + offset, slot_ids, mask=offset < end_idx) diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index cc2ac442c7..0ee05328c9 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -64,7 +64,12 @@ from vllm.tasks import SupportedTask from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput -from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec +from vllm.v1.kv_cache_interface import ( + CircularBufferSpec, + KVCacheConfig, + MambaSpec, + UniformTypeKVCacheSpecs, +) from vllm.v1.outputs import ( DraftTokenIds, ECConnectorOutput, @@ -548,9 +553,14 @@ class GPUModelRunner(LoRAModelRunnerMixin): block_sizes = [] max_num_blocks_per_group = [] + slot_mapping_enabled = [] for kv_cache_group in kv_cache_config.kv_cache_groups: spec = kv_cache_group.kv_cache_spec block_sizes.append(spec.block_size) + layer_spec = ( + spec.first_spec if isinstance(spec, UniformTypeKVCacheSpecs) else spec + ) + slot_mapping_enabled.append(not isinstance(layer_spec, CircularBufferSpec)) # Let each cache type account for CP. Attention KV is DCP-sharded, # while Mamba/GDN recurrent state is replicated across DCP ranks. max_num_blocks = spec.max_num_blocks_per_req( @@ -558,7 +568,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): ) # Preserve each cache type's alignment requirements after applying # its topology-aware block-table width. - if isinstance(spec, MambaSpec): + if isinstance(layer_spec, (MambaSpec, CircularBufferSpec)): max_num_blocks = get_block_table_width( max_num_blocks, spec.block_size, token_alignment=None ) @@ -605,6 +615,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): max_num_blocks_per_group=max_num_blocks_per_group, device=self.device, kernel_block_sizes=self.kernel_block_sizes, + slot_mapping_enabled=slot_mapping_enabled, cp_size=self.dcp_size, cp_rank=self.dcp_rank, cp_interleave=self.cp_interleave, @@ -1192,6 +1203,10 @@ class GPUModelRunner(LoRAModelRunnerMixin): total_num_draft_tokens = int(num_draft_tokens_per_req.sum()) total_num_logits = num_reqs * num_bonus_tokens + total_num_draft_tokens num_logits = num_draft_tokens_per_req + num_bonus_tokens + # combine_sampled_and_draft_tokens places a request's logits rows + # at [query_end - num_logits, query_end). Fewer query rows than + # that would silently select the preceding request's hidden states. + assert (num_scheduled_tokens_np >= num_logits).all() cu_num_logits_np = np.empty(num_reqs + 1, dtype=np.int32) cu_num_logits_np[0] = 0 np.cumsum(num_logits, out=cu_num_logits_np[1:]) diff --git a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py index c8ccde34fe..6d07c1588c 100644 --- a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py +++ b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py @@ -9,10 +9,14 @@ import torch.nn as nn from vllm.config import VllmConfig from vllm.config.compilation import CUDAGraphMode +from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFuncsByType from vllm.triton_utils import tl, triton from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadataBuilder from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadataBuilder -from vllm.v1.attention.backends.short_conv_attn import ShortConvAttentionMetadataBuilder +from vllm.v1.attention.backends.short_conv_attn import ( + PleShortConvAttentionMetadataBuilder, + ShortConvAttentionMetadataBuilder, +) from vllm.v1.core.sched.output import NewRequestData from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec from vllm.v1.utils import CpuGpuBuffer @@ -24,7 +28,10 @@ from vllm.v1.worker.gpu.model_states.interface import ModelSpecificAttnMetadata from vllm.v1.worker.gpu.model_states.recoverssm import RecoverSSMState from vllm.v1.worker.mamba_utils import ( MambaSpecDecodeGPUContext, + get_mamba_group_ids, + get_mamba_groups, preprocess_mamba_align_fused_kernel, + validate_mamba_state_copy_funcs, ) from vllm.v1.worker.utils import AttentionGroup @@ -53,6 +60,7 @@ class MambaHybridAttnMetadata(ModelSpecificAttnMetadata): Mamba2AttentionMetadataBuilder, GDNAttentionMetadataBuilder, ShortConvAttentionMetadataBuilder, + PleShortConvAttentionMetadataBuilder, ), ): return {} @@ -102,6 +110,7 @@ class MambaHybridModelState(DefaultModelState): self._mamba_ctx: MambaSpecDecodeGPUContext | None = None self._mamba_group_ids: list[int] = [] self._mamba_spec: MambaSpec | None = None + self._mamba_state_copy_funcs: MambaStateCopyFuncsByType | None = None def add_request(self, req_index: int, new_req_data: NewRequestData) -> None: super().add_request(req_index, new_req_data) @@ -117,17 +126,16 @@ class MambaHybridModelState(DefaultModelState): self, kv_cache_config: KVCacheConfig ) -> tuple[list[int], MambaSpec]: if self._mamba_spec is None: - group_ids: list[int] = [] - specs: list[MambaSpec] = [] - for i, group in enumerate(kv_cache_config.kv_cache_groups): - spec = group.kv_cache_spec - if isinstance(spec, MambaSpec): - group_ids.append(i) - specs.append(spec) - assert specs, "no mamba layers in the model" - assert all(specs[0] == s for s in specs) - self._mamba_group_ids = group_ids - self._mamba_spec = specs[0] + mamba_groups = get_mamba_groups(kv_cache_config) + mamba_spec = next(iter(mamba_groups)) + assert all( + spec.block_size == mamba_spec.block_size + and spec.num_speculative_blocks == mamba_spec.num_speculative_blocks + and spec.mamba_cache_mode == mamba_spec.mamba_cache_mode + for spec in mamba_groups + ), "all mamba groups must share cache scheduling parameters" + self._mamba_group_ids = get_mamba_group_ids(mamba_groups) + self._mamba_spec = mamba_spec return self._mamba_group_ids, self._mamba_spec def _ensure_align_ctx( @@ -136,8 +144,14 @@ class MambaHybridModelState(DefaultModelState): mamba_group_ids: list[int], block_tables: tuple[torch.Tensor, ...], ) -> MambaSpecDecodeGPUContext: + if self._mamba_state_copy_funcs is None: + mamba_groups = get_mamba_groups(kv_cache_config) + mamba_types = {spec.mamba_type for spec in mamba_groups} + copy_funcs = self.model.get_mamba_state_copy_funcs(mamba_types) + validate_mamba_state_copy_funcs(mamba_groups, copy_funcs) + self._mamba_state_copy_funcs = copy_funcs + copy_funcs = self._mamba_state_copy_funcs if self._mamba_ctx is None: - copy_funcs = self.model.get_mamba_state_copy_func() # Both SD and DS conv layouts support a >0 spec-decode shift: the # fused pre-copy kernel (``_copy_mamba_state_block``) applies the # ``token_bias = num_accepted - 1`` window shift per conv layout @@ -146,7 +160,7 @@ class MambaHybridModelState(DefaultModelState): self._mamba_ctx = MambaSpecDecodeGPUContext.create( max_num_reqs=self.max_num_reqs, kv_cache_config=kv_cache_config, - num_state_types=len(copy_funcs), + copy_funcs=copy_funcs, device=self.device, make_buffer=lambda n, dtype: CpuGpuBuffer( n, dtype=dtype, device=self.device @@ -161,7 +175,7 @@ class MambaHybridModelState(DefaultModelState): ctx.initialize_from_forward_context( kv_cache_config, forward_context, - self.model.get_mamba_state_copy_func(), + copy_funcs, [block_tables[gid] for gid in mamba_group_ids], ) return ctx diff --git a/vllm/v1/worker/gpu/warmup.py b/vllm/v1/worker/gpu/warmup.py index e3c5e11e03..f99c924f26 100644 --- a/vllm/v1/worker/gpu/warmup.py +++ b/vllm/v1/worker/gpu/warmup.py @@ -18,7 +18,13 @@ from vllm.v1.core.sched.output import ( NewRequestData, SchedulerOutput, ) -from vllm.v1.kv_cache_interface import CrossAttentionSpec, KVCacheSpec, MambaSpec +from vllm.v1.kv_cache_interface import ( + CircularBufferSpec, + CrossAttentionSpec, + KVCacheSpec, + MambaSpec, + UniformTypeKVCacheSpecs, +) from vllm.v1.request import Request from vllm.v1.worker.gpu.model_runner import GPUModelRunner @@ -39,6 +45,11 @@ def _reserved_block_count( `KVCacheManager.allocate_slots` reserves: the token range plus `num_lookahead_tokens`, where the speculator writes the KV of its drafts. """ + if isinstance(kvcache_spec, UniformTypeKVCacheSpecs): + kvcache_spec = kvcache_spec.first_spec + if isinstance(kvcache_spec, CircularBufferSpec): + # Circular caches keep one physical ring block for the request lifetime. + return 1 if isinstance(kvcache_spec, CrossAttentionSpec): # Cross-attention blocks cover the encoder sequence only. return cdiv(max_encoder_len, kvcache_spec.block_size) diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index e6e63e1e98..e6ba7bca77 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -66,6 +66,7 @@ from vllm.model_executor.layers.fused_moe.routed_experts_capturer import ( RoutedExpertsCapturer, bind_routed_experts_capturer, ) +from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFuncsByType from vllm.model_executor.layers.mamba.ops.ssu_dispatch import ( initialize_mamba_ssu_backend, ) @@ -997,6 +998,7 @@ class GPUModelRunner( self.kv_connector_output: KVConnectorOutput | None = None self.mamba_state_idx: dict[str, int] = {} self._mamba_bufs: mamba_utils.MambaBuffers | None = None + self._mamba_state_copy_funcs: MambaStateCopyFuncsByType | None = None self.mamba_prev_last_scheduled_idx: CpuGpuBuffer | None = None if self.cache_config.mamba_cache_mode == "all" and self.num_spec_tokens > 0: self.mamba_prev_last_scheduled_idx = self._make_buffer( @@ -1053,6 +1055,15 @@ class GPUModelRunner( with_numpy=numpy, ) + def _get_mamba_state_copy_funcs(self) -> MambaStateCopyFuncsByType: + if self._mamba_state_copy_funcs is None: + mamba_groups = mamba_utils.get_mamba_groups(self.kv_cache_config) + mamba_types = {spec.mamba_type for spec in mamba_groups} + copy_funcs = self.model.get_mamba_state_copy_funcs(mamba_types) + mamba_utils.validate_mamba_state_copy_funcs(mamba_groups, copy_funcs) + self._mamba_state_copy_funcs = copy_funcs + return self._mamba_state_copy_funcs + def _get_mamba_bufs(self) -> mamba_utils.MambaBuffers: # Only reachable on the ``mamba_cache_mode == "align"`` path. # The postprocess sub-object is additionally gated on spec @@ -1062,7 +1073,7 @@ class GPUModelRunner( self._mamba_bufs = mamba_utils.MambaBuffers.create( max_num_reqs=self.max_num_reqs, kv_cache_config=self.kv_cache_config, - copy_funcs=self.model.get_mamba_state_copy_func(), + copy_funcs=self._get_mamba_state_copy_funcs(), make_buffer=self._make_buffer, device=self.device, with_postprocess_align=( @@ -1599,7 +1610,7 @@ class GPUModelRunner( input_batch=self.input_batch, kv_cache_config=self.kv_cache_config, forward_context=self.compilation_config.static_forward_context, - mamba_state_copy_funcs=self.model.get_mamba_state_copy_func(), + mamba_state_copy_funcs=self._get_mamba_state_copy_funcs(), ) assert self.num_accepted_tokens_event is not None @@ -4404,7 +4415,7 @@ class GPUModelRunner( self.input_batch, self.requests, self.compilation_config.static_forward_context, - self.model.get_mamba_state_copy_func(), + self._get_mamba_state_copy_funcs(), mamba_bufs.preprocess, align_ctx=mamba_bufs.postprocess_align, ) @@ -7488,6 +7499,7 @@ class GPUModelRunner( kv_cache_config = deepcopy(kv_cache_config) self.kv_cache_config = kv_cache_config self._mamba_bufs = None + self._mamba_state_copy_funcs = None self.may_add_encoder_only_layers_to_kv_cache_config() self.maybe_add_kv_sharing_layers_to_kv_cache_groups(kv_cache_config) self.initialize_attn_backend(kv_cache_config, is_profiling=is_profiling) diff --git a/vllm/v1/worker/mamba_utils.py b/vllm/v1/worker/mamba_utils.py index c222ee8a5c..53e46a9cf9 100644 --- a/vllm/v1/worker/mamba_utils.py +++ b/vllm/v1/worker/mamba_utils.py @@ -10,7 +10,7 @@ import torch from vllm.config import CacheConfig from vllm.logger import init_logger from vllm.model_executor.layers.mamba.mamba_utils import ( - MambaStateCopyFunc, + MambaStateCopyFuncsByType, get_conv_copy_spec, get_temporal_copy_spec, is_conv_state_dim_first, @@ -19,7 +19,12 @@ from vllm.triton_utils import tl, triton from vllm.utils.gpu_sync_debug import gpu_sync_allowed from vllm.utils.math_utils import cdiv from vllm.v1.core.sched.output import SchedulerOutput -from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec +from vllm.v1.kv_cache_interface import ( + KVCacheConfig, + KVCacheGroupSpec, + MambaSpec, + UniformTypeKVCacheSpecs, +) from vllm.v1.utils import CpuGpuBuffer from vllm.v1.worker.gpu_input_batch import CachedRequestState from vllm.v1.worker.lora_model_runner_mixin import GPUInputBatch @@ -374,8 +379,8 @@ def postprocess_mamba_fused_kernel( # block table. block_table_ptrs_ptr, block_table_stride_req: tl.int64, # stride between requests (in elements) - # Mamba state metadata (per-layer, per-state-type) - # These are 1D arrays indexed by (layer_idx * num_state_types + state_type_idx) + # Mamba state metadata, flattened across all layer/state tensors. Different + # Mamba groups may contribute different numbers of state tensors. state_base_addrs_ptr, # base address of each state tensor state_block_strides_ptr, # bytes per block for each state state_elem_sizes_ptr, # element size for each state @@ -415,14 +420,13 @@ def postprocess_mamba_fused_kernel( Fused GPU kernel for postprocess_mamba that computes decisions AND performs mamba state copies without any CPU-GPU synchronization. - Grid: (num_reqs, num_layers * num_state_types [, TEMPORAL_TILES]) + Grid: (num_reqs, num_states [, TEMPORAL_TILES]) - program_id(0) = request/batch index - program_id(1) = state_idx (flattened index into layer/state_type metadata) - program_id(2) = temporal-copy tile index (0 when TEMPORAL_TILES == 1) - Note: num_layers and num_state_types are not passed as kernel parameters - because the kernel indexes directly into pre-flattened metadata arrays - using program_id(1). The grid dimensions encode the total state count. + The kernel indexes directly into pre-flattened metadata arrays using + program_id(1). The grid dimensions encode the total state count. """ batch_idx = tl.program_id(0) state_idx = tl.program_id(1) @@ -581,8 +585,8 @@ def precopy_mamba_align_fused_kernel( copy specs), but driven by the GPU-resident src columns so it needs no CPU-GPU sync (async-scheduling safe). - Grid: (num_reqs, num_layers * num_state_types [, TEMPORAL_TILES]). V2 passes - a batch-to-state idx_mapping; V1 already stores the staged arrays in batch + Grid: (num_reqs, num_states [, TEMPORAL_TILES]). V2 passes a + batch-to-state idx_mapping; V1 already stores the staged arrays in batch order and uses HAS_IDX_MAPPING=False. """ batch_idx = tl.program_id(0) @@ -663,17 +667,56 @@ def batch_memcpy(src_ptrs, dst_ptrs, sizes): batch_memcpy_kernel[grid](src_ptrs, dst_ptrs, sizes, BLOCK_SIZE=BLOCK_SIZE) -def get_mamba_groups(kv_cache_config: KVCacheConfig) -> tuple[list[int], MambaSpec]: - mamba_group_ids: list[int] = [] - mamba_specs: list[MambaSpec] = [] - for i in range(len(kv_cache_config.kv_cache_groups)): - kv_cache_spec = kv_cache_config.kv_cache_groups[i].kv_cache_spec - if isinstance(kv_cache_spec, MambaSpec): - mamba_group_ids.append(i) - mamba_specs.append(kv_cache_spec) - assert len(mamba_group_ids) > 0, "no mamba layers in the model" - assert all(mamba_specs[0] == spec for spec in mamba_specs) - return mamba_group_ids, mamba_specs[0] +def _get_mamba_spec_for_layer( + kv_cache_group: KVCacheGroupSpec, layer_name: str +) -> MambaSpec: + kv_cache_spec = kv_cache_group.kv_cache_spec + if isinstance(kv_cache_spec, UniformTypeKVCacheSpecs): + kv_cache_spec = kv_cache_spec.kv_cache_specs[layer_name] + assert isinstance(kv_cache_spec, MambaSpec) + return kv_cache_spec + + +def get_mamba_group_ids(mamba_groups: dict[MambaSpec, list[int]]) -> list[int]: + """Return the sorted group ids of all mamba groups, without duplicates. + + A group whose layers use different ``MambaSpec``s is listed under one key + per spec, so the same group id can appear more than once. + """ + return sorted( + {group_id for group_ids in mamba_groups.values() for group_id in group_ids} + ) + + +def get_mamba_groups(kv_cache_config: KVCacheConfig) -> dict[MambaSpec, list[int]]: + """Return a mapping from each distinct MambaSpec to its sorted group_ids.""" + mamba_groups: dict[MambaSpec, set[int]] = {} + for i, kv_cache_group in enumerate(kv_cache_config.kv_cache_groups): + kv_cache_spec = kv_cache_group.kv_cache_spec + if isinstance(kv_cache_spec, UniformTypeKVCacheSpecs): + kv_cache_spec = kv_cache_spec.first_spec + if not isinstance(kv_cache_spec, MambaSpec): + continue + for layer_name in kv_cache_group.layer_names: + mamba_spec = _get_mamba_spec_for_layer(kv_cache_group, layer_name) + mamba_groups.setdefault(mamba_spec, set()).add(i) + assert len(mamba_groups) > 0, "no mamba layers in the model" + return {spec: sorted(ids) for spec, ids in mamba_groups.items()} + + +def validate_mamba_state_copy_funcs( + mamba_groups: dict[MambaSpec, list[int]], + copy_funcs: MambaStateCopyFuncsByType, +) -> None: + for mamba_spec in mamba_groups: + assert mamba_spec.mamba_type in copy_funcs, ( + f"missing state copy funcs for {mamba_spec.mamba_type}" + ) + state_copy_funcs = copy_funcs[mamba_spec.mamba_type] + assert len(state_copy_funcs) == len(mamba_spec.shapes), ( + f"{mamba_spec.mamba_type} declares {len(mamba_spec.shapes)} states, " + f"but provides {len(state_copy_funcs)} state copy funcs" + ) @dataclasses.dataclass @@ -690,14 +733,29 @@ class MambaCopyBuffers: cls, max_num_reqs: int, kv_cache_config: KVCacheConfig, - copy_funcs: tuple[MambaStateCopyFunc, ...], + copy_funcs: MambaStateCopyFuncsByType, make_buffer: Callable[..., CpuGpuBuffer], ) -> "MambaCopyBuffers": - mamba_group_ids, mamba_spec = get_mamba_groups(kv_cache_config) + mamba_groups = get_mamba_groups(kv_cache_config) + mamba_spec = next(iter(mamba_groups)) + assert all( + spec.block_size == mamba_spec.block_size + and spec.num_speculative_blocks == mamba_spec.num_speculative_blocks + and spec.mamba_cache_mode == mamba_spec.mamba_cache_mode + for spec in mamba_groups + ), "all mamba groups must share cache scheduling parameters" + mamba_group_ids = get_mamba_group_ids(mamba_groups) entries_per_req = sum( - len(kv_cache_config.kv_cache_groups[gid].layer_names) + len( + copy_funcs[ + _get_mamba_spec_for_layer( + kv_cache_config.kv_cache_groups[gid], layer_name + ).mamba_type + ] + ) for gid in mamba_group_ids - ) * len(copy_funcs) + for layer_name in kv_cache_config.kv_cache_groups[gid].layer_names + ) n = max_num_reqs * entries_per_req return cls( @@ -725,7 +783,9 @@ class MambaSpecDecodeGPUContext: window with offset-based copies), 0 for temporal states (full block copies). """ - # Per-state metadata tensors (shape: [num_layers * num_state_types]) + # Per-state metadata tensors (shape: [num_states]). Mamba groups may have + # different numbers of state types (for example, GDN has two while PLE + # short-conv has one). # These are populated from forward_context during the first forward pass state_base_addrs: torch.Tensor # int64: base address of each state tensor state_block_strides: torch.Tensor # int64: bytes per block @@ -739,8 +799,7 @@ class MambaSpecDecodeGPUContext: # Configuration block_size: int - num_layers: int - num_state_types: int + num_states: int mamba_group_ids: list[int] num_groups: int @@ -776,19 +835,37 @@ class MambaSpecDecodeGPUContext: cls, max_num_reqs: int, kv_cache_config: KVCacheConfig, - num_state_types: int, + copy_funcs: MambaStateCopyFuncsByType, device: torch.device, make_buffer: Callable[..., CpuGpuBuffer], ) -> "MambaSpecDecodeGPUContext": """Create context with allocated buffers (metadata populated later).""" - mamba_group_ids, mamba_spec = get_mamba_groups(kv_cache_config) + mamba_groups = get_mamba_groups(kv_cache_config) + mamba_group_ids = get_mamba_group_ids(mamba_groups) + mamba_spec = next(iter(mamba_groups)) + assert all( + spec.block_size == mamba_spec.block_size + and spec.num_speculative_blocks == mamba_spec.num_speculative_blocks + and spec.mamba_cache_mode == mamba_spec.mamba_cache_mode + for spec in mamba_groups + ), "all mamba groups must share cache scheduling parameters" + copy_funcs_by_spec = { + spec: copy_funcs[spec.mamba_type] for spec in mamba_groups + } - # Count total layers across all mamba groups - num_layers = sum( - len(kv_cache_config.kv_cache_groups[gid].layer_names) + # Count physical state tensors across all Mamba groups. Different + # groups may expose different state specs. + total_states = sum( + len( + copy_funcs_by_spec[ + _get_mamba_spec_for_layer( + kv_cache_config.kv_cache_groups[gid], layer_name + ) + ] + ) for gid in mamba_group_ids + for layer_name in kv_cache_config.kv_cache_groups[gid].layer_names ) - total_states = num_layers * num_state_types return cls( state_base_addrs=torch.zeros( @@ -816,8 +893,7 @@ class MambaSpecDecodeGPUContext: total_states, dtype=torch.int64, device=device ), block_size=mamba_spec.block_size, - num_layers=num_layers, - num_state_types=num_state_types, + num_states=total_states, mamba_group_ids=mamba_group_ids, num_groups=len(mamba_group_ids), num_accepted_tokens_out=torch.zeros( @@ -848,7 +924,7 @@ class MambaSpecDecodeGPUContext: self, kv_cache_config: KVCacheConfig, forward_context: dict[str, Any], - mamba_state_copy_funcs: tuple[MambaStateCopyFunc, ...], + mamba_state_copy_funcs: MambaStateCopyFuncsByType, block_tables: list[torch.Tensor], ) -> None: """ @@ -879,8 +955,8 @@ class MambaSpecDecodeGPUContext: forward_context: Dictionary mapping layer names to attention objects, populated after the model is loaded. Each attention object must have a `kv_cache` attribute containing the list of state tensors. - mamba_state_copy_funcs: Tuple of copy functions (one per state type) - used to determine whether each state is a conv or temporal state. + mamba_state_copy_funcs: Mapping from MambaAttentionBackendEnum to + copy functions. block_tables: per-mamba-group persistent block-table tensors, in the same order as `mamba_group_ids`. Their `data_ptr()` / `stride(0)` are captured once for the kernel to index into. @@ -900,22 +976,24 @@ class MambaSpecDecodeGPUContext: self, kv_cache_config: KVCacheConfig, forward_context: dict[str, Any], - mamba_state_copy_funcs: tuple[MambaStateCopyFunc, ...], + mamba_state_copy_funcs: MambaStateCopyFuncsByType, block_tables: list[torch.Tensor], ) -> None: idx = 0 for group_local_idx, mamba_group_id in enumerate(self.mamba_group_ids): - layer_names = kv_cache_config.kv_cache_groups[mamba_group_id].layer_names + kv_cache_group = kv_cache_config.kv_cache_groups[mamba_group_id] + layer_names = kv_cache_group.layer_names for layer_name in layer_names: + mamba_spec = _get_mamba_spec_for_layer(kv_cache_group, layer_name) + state_copy_funcs = mamba_state_copy_funcs[mamba_spec.mamba_type] attention = forward_context[layer_name] kv_caches: list[torch.Tensor] = attention.kv_cache - - if len(kv_caches) < self.num_state_types: + if len(kv_caches) < len(state_copy_funcs): raise ValueError( - f"Expected at least {self.num_state_types} Mamba state " + f"Expected at least {len(state_copy_funcs)} Mamba state " f"tensors, got {len(kv_caches)}" ) - for state_type_idx, copy_func in enumerate(mamba_state_copy_funcs): + for state_type_idx, copy_func in enumerate(state_copy_funcs): state = kv_caches[state_type_idx] # Base address self.state_base_addrs[idx] = _reinterpret_u64_as_i64( @@ -994,6 +1072,8 @@ class MambaSpecDecodeGPUContext: self.state_group_indices[idx] = group_local_idx idx += 1 + assert idx == self.num_states + # Cache per-group block-table base addresses and per-request stride. # `block_tables[i]` is the persistent 2D int32 block-table tensor for # `mamba_group_ids[i]`; `data_ptr()` / `stride(0)` are stable for the @@ -1079,7 +1159,7 @@ class MambaSpecDecodeGPUContext: num_accepted_tokens_gpu[:num_reqs] ) - total_states = self.num_layers * self.num_state_types + total_states = self.num_states grid = (num_reqs, total_states, _TEMPORAL_TILES) postprocess_mamba_fused_kernel[grid]( @@ -1128,7 +1208,7 @@ class MambaSpecDecodeGPUContext: """ if num_reqs == 0 or not self.is_initialized: return - total_states = self.num_layers * self.num_state_types + total_states = self.num_states grid = (num_reqs, total_states, _TEMPORAL_TILES) precopy_mamba_align_fused_kernel[grid]( state_idx_gpu, @@ -1178,7 +1258,7 @@ class MambaSpecDecodeGPUContext: num_accepted_tokens_snapshot = self.num_accepted_tokens_out num_accepted_tokens_snapshot.copy_(num_accepted_tokens_gpu) - total_states = self.num_layers * self.num_state_types + total_states = self.num_states grid = (num_reqs, total_states, _TEMPORAL_TILES) postprocess_mamba_fused_kernel[grid]( num_accepted_tokens_snapshot, @@ -1226,7 +1306,7 @@ class MambaBuffers: cls, max_num_reqs: int, kv_cache_config: KVCacheConfig, - copy_funcs: tuple[MambaStateCopyFunc, ...], + copy_funcs: MambaStateCopyFuncsByType, make_buffer: Callable[..., CpuGpuBuffer], device: torch.device, with_postprocess_align: bool, @@ -1239,7 +1319,7 @@ class MambaBuffers: MambaSpecDecodeGPUContext.create( max_num_reqs=max_num_reqs, kv_cache_config=kv_cache_config, - num_state_types=len(copy_funcs), + copy_funcs=copy_funcs, device=device, make_buffer=make_buffer, ) @@ -1252,7 +1332,7 @@ class MambaBuffers: def collect_mamba_copy_meta( copy_bufs: MambaCopyBuffers, kv_cache_config: KVCacheConfig, - mamba_state_copy_funcs: tuple[MambaStateCopyFunc, ...], + mamba_state_copy_funcs: MambaStateCopyFuncsByType, mamba_group_ids: list[int], src_block_idx: int, dest_block_idx: int, @@ -1271,11 +1351,14 @@ def collect_mamba_copy_meta( for mamba_group_id in mamba_group_ids: block_ids = req_state.block_ids[mamba_group_id] dest_block_id = block_ids[dest_block_idx] - layer_names = kv_cache_config.kv_cache_groups[mamba_group_id].layer_names + kv_cache_group = kv_cache_config.kv_cache_groups[mamba_group_id] + layer_names = kv_cache_group.layer_names for layer_name in layer_names: + mamba_spec = _get_mamba_spec_for_layer(kv_cache_group, layer_name) + state_copy_funcs = mamba_state_copy_funcs[mamba_spec.mamba_type] attention = forward_context[layer_name] kv_caches: list[torch.Tensor] = attention.kv_cache - for state, state_copy_func in zip(kv_caches, mamba_state_copy_funcs): + for state, state_copy_func in zip(kv_caches, state_copy_funcs): copy_spec = state_copy_func( state, block_ids, src_block_idx, accept_token_bias + 1 ) @@ -1355,7 +1438,7 @@ def preprocess_mamba( input_batch: GPUInputBatch, requests: dict[str, CachedRequestState], forward_context: dict[str, Any], - mamba_state_copy_funcs: tuple[MambaStateCopyFunc, ...], + mamba_state_copy_funcs: MambaStateCopyFuncsByType, copy_bufs: MambaCopyBuffers, align_ctx: MambaSpecDecodeGPUContext | None = None, ): @@ -1470,8 +1553,10 @@ def postprocess_mamba_all( """ if num_spec_tokens <= 0: return - _, mamba_spec = get_mamba_groups(kv_cache_config) - block_size = mamba_spec.block_size + mamba_groups = get_mamba_groups(kv_cache_config) + block_sizes = {mamba_spec.block_size for mamba_spec in mamba_groups} + assert len(block_sizes) == 1, "all mamba groups must share block_size" + block_size = next(iter(block_sizes)) full_decode_len = 1 + num_spec_tokens scheduled = scheduler_output.num_scheduled_tokens for req_id in input_batch.req_ids[:num_reqs]: @@ -1508,7 +1593,7 @@ def postprocess_mamba_align_gpu( input_batch: GPUInputBatch, kv_cache_config: KVCacheConfig, forward_context: dict[str, Any], - mamba_state_copy_funcs: tuple[MambaStateCopyFunc, ...], + mamba_state_copy_funcs: MambaStateCopyFuncsByType, ) -> None: """GPU-side mamba postprocess for spec decode + hybrid + align mode.