[Model] Support Qwen3.8-Flash-Next (#53896)
Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com> Signed-off-by: andyluo7 <andy.luo@amd.com> Signed-off-by: lishunyang12 <lishunyang03@gmail.com> Signed-off-by: Harry Huang <huanghaoyan.hhy@alibaba-inc.com> Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: KungYork <firelikest@163.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: andyluo7 <43718156+andyluo7@users.noreply.github.com> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: andyluo7 <andy.luo@amd.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: lishunyang12 <lishunyang03@gmail.com> Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
zjy0516
KungYork
Thien Tran
Yifan Qiao
andyluo7
Isotr0py
andyluo7
OpenAI Codex
lishunyang12
Vadim Gimpelson
Isotr0py
Lucas Wilkinson
Claude Opus 5
OpenAI Codex
parent
687db59744
commit
e126687a9a
@@ -331,6 +331,42 @@ steps:
|
||||
- uv pip install --system 'gpt-oss[eval]==0.0.5'
|
||||
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-b200.txt
|
||||
|
||||
- label: ":nvidia: (B200) Qwen3.8-Flash-Next-FP8 Accuracy Eval"
|
||||
key: accuracy-eval-qwen3-8-flash-next-fp8-b200
|
||||
timeout_in_minutes: 60
|
||||
device: b200-k8s
|
||||
optional: true
|
||||
num_devices: 4
|
||||
source_file_dependencies:
|
||||
- csrc/libtorch_stable/gdn/
|
||||
- tests/evals/qwen4_exp/
|
||||
- vllm/model_executor/layers/mamba/
|
||||
- vllm/models/qwen4_exp/
|
||||
- vllm/transformers_utils/configs/qwen4_exp.py
|
||||
- vllm/v1/attention/backends/short_conv_attn.py
|
||||
- vllm/v1/spec_decode/qwen4_exp.py
|
||||
commands:
|
||||
- uv pip install --system 'evalscope==1.10.0'
|
||||
- pytest -s -v evals/qwen4_exp/test_accuracy.py --config-list-file=configs/models-b200.txt
|
||||
|
||||
- label: ":nvidia: (H200) Qwen3.8-Flash-Next-FP8 Accuracy Eval"
|
||||
key: accuracy-eval-qwen3-8-flash-next-fp8-h200
|
||||
timeout_in_minutes: 60
|
||||
device: h200
|
||||
optional: true
|
||||
num_devices: 4
|
||||
source_file_dependencies:
|
||||
- csrc/libtorch_stable/gdn/
|
||||
- tests/evals/qwen4_exp/
|
||||
- vllm/model_executor/layers/mamba/
|
||||
- vllm/models/qwen4_exp/
|
||||
- vllm/transformers_utils/configs/qwen4_exp.py
|
||||
- vllm/v1/attention/backends/short_conv_attn.py
|
||||
- vllm/v1/spec_decode/qwen4_exp.py
|
||||
commands:
|
||||
- uv pip install --system 'evalscope==1.10.0'
|
||||
- pytest -s -v evals/qwen4_exp/test_accuracy.py --config-list-file=configs/models-h200.txt
|
||||
|
||||
- label: ":nvidia: (DGX) Spark GPQA Eval (GPT-OSS)"
|
||||
key: gpqa-eval-gpt-oss-spark
|
||||
timeout_in_minutes: 35
|
||||
|
||||
@@ -86,6 +86,43 @@ steps:
|
||||
# The native NVIDIA Kimi K3 kernels require the SM100 family.
|
||||
- pytest -v -s models/kimi_k3 kernels/attention/test_kimi_k3_mla_fused_epilogue.py kernels/test_bf16_skinny_gemm.py
|
||||
|
||||
- label: ":nvidia: (H200) Qwen4 Exp"
|
||||
key: qwen4-exp-unit-tests
|
||||
timeout_in_minutes: 40
|
||||
device: h200_35gb
|
||||
source_file_dependencies:
|
||||
- vllm/models/qwen4_exp/
|
||||
- vllm/transformers_utils/configs/qwen4_exp.py
|
||||
- vllm/config/compilation.py
|
||||
- vllm/v1/attention/backends/short_conv_attn.py
|
||||
- tests/models/qwen4_exp/
|
||||
commands:
|
||||
- pytest -v -s models/qwen4_exp/test_hc_ops.py models/qwen4_exp/test_qsa_pre_indexer.py models/qwen4_exp/test_qsa_reference.py
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI300) Qwen4 Exp"
|
||||
dind: false
|
||||
device: mi300_1
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
commands:
|
||||
- pytest -v -s models/qwen4_exp/test_qsa_amd.py
|
||||
|
||||
- label: ":computer: (CPU) Qwen4 Exp"
|
||||
key: qwen4-exp-unit-tests-cpu
|
||||
depends_on:
|
||||
- image-build-cpu
|
||||
timeout_in_minutes: 20
|
||||
device: cpu-small
|
||||
source_file_dependencies:
|
||||
- vllm/models/qwen4_exp/
|
||||
- vllm/transformers_utils/configs/qwen4_exp.py
|
||||
- vllm/config/compilation.py
|
||||
- vllm/v1/attention/backends/short_conv_attn.py
|
||||
- tests/models/qwen4_exp/
|
||||
commands:
|
||||
- pytest -v -s models/qwen4_exp/test_config.py models/qwen4_exp/test_ple.py
|
||||
|
||||
- label: ":computer: (CPU) Basic Models Other"
|
||||
key: basic-models-test-other-cpu
|
||||
depends_on:
|
||||
|
||||
@@ -11,6 +11,7 @@ steps:
|
||||
- vllm/distributed/
|
||||
- vllm/inputs/
|
||||
- vllm/model_executor/
|
||||
- vllm/models/qwen4_exp/
|
||||
- vllm/platforms/
|
||||
- vllm/sampling_params.py
|
||||
- vllm/transformers_utils/
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
*/
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <cuda_bf16.h>
|
||||
#include <cuda_fp16.h>
|
||||
#include <cuda_runtime.h>
|
||||
@@ -146,7 +147,7 @@ __device__ __forceinline__ Sum2 warp_reduce_sum_pair(float x, float y) {
|
||||
return {x, y};
|
||||
}
|
||||
|
||||
template <typename StateT, int ValueHeadsPerKeyHead>
|
||||
template <typename StateT, int ValueHeadsPerKeyHead, bool SigmoidGate>
|
||||
__global__ __launch_bounds__(kThreads, 2) void gdn_decode_post_conv_mtp_kernel(
|
||||
const __nv_bfloat16* __restrict__ mixed_qkv,
|
||||
const __nv_bfloat16* __restrict__ a, const __nv_bfloat16* __restrict__ b,
|
||||
@@ -354,9 +355,11 @@ __global__ __launch_bounds__(kThreads, 2) void gdn_decode_post_conv_mtp_kernel(
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 4; ++i) {
|
||||
const int value = lane + i * 32;
|
||||
const float gate = silu_fast(__bfloat162float(
|
||||
const float gate_input = __bfloat162float(
|
||||
output_gate[static_cast<int64_t>(token) * strides.gate_row +
|
||||
value_head * kDimV + value]));
|
||||
value_head * kDimV + value]);
|
||||
const float gate =
|
||||
SigmoidGate ? sigmoid_fast(gate_input) : silu_fast(gate_input);
|
||||
const float weight =
|
||||
norm_weight_is_bf16
|
||||
? __bfloat162float(
|
||||
@@ -370,7 +373,7 @@ __global__ __launch_bounds__(kThreads, 2) void gdn_decode_post_conv_mtp_kernel(
|
||||
}
|
||||
}
|
||||
|
||||
template <typename StateT, int ValueHeadsPerKeyHead>
|
||||
template <typename StateT, int ValueHeadsPerKeyHead, bool SigmoidGate>
|
||||
void launch_gdn_decode_post_conv_mtp(
|
||||
torch::stable::Tensor const& mixed_qkv, torch::stable::Tensor const& a_log,
|
||||
torch::stable::Tensor const& dt_bias,
|
||||
@@ -395,7 +398,7 @@ void launch_gdn_decode_post_conv_mtp(
|
||||
get_current_cuda_stream(mixed_qkv.get_device_index());
|
||||
const int num_requests = static_cast<int>(state_indices.size(0));
|
||||
const dim3 grid(num_requests, num_value_heads);
|
||||
gdn_decode_post_conv_mtp_kernel<StateT, ValueHeadsPerKeyHead>
|
||||
gdn_decode_post_conv_mtp_kernel<StateT, ValueHeadsPerKeyHead, SigmoidGate>
|
||||
<<<grid, kThreads, 0, stream>>>(
|
||||
static_cast<const __nv_bfloat16*>(mixed_qkv.data_ptr()), a, b,
|
||||
static_cast<const float*>(a_log.data_ptr()), dt_bias.data_ptr(),
|
||||
@@ -425,7 +428,7 @@ void fused_gdn_decode_post_conv_mtp(
|
||||
torch::stable::Tensor const& num_accepted_tokens,
|
||||
torch::stable::Tensor& state, torch::stable::Tensor const& output_gate,
|
||||
torch::stable::Tensor const& norm_weight, torch::stable::Tensor& out,
|
||||
double scale, double norm_eps) {
|
||||
double scale, double norm_eps, const std::string& output_gate_activation) {
|
||||
using torch::headeronly::ScalarType;
|
||||
|
||||
STD_TORCH_CHECK(
|
||||
@@ -466,6 +469,9 @@ void fused_gdn_decode_post_conv_mtp(
|
||||
"norm_weight must be a CUDA float32 or bfloat16 tensor");
|
||||
STD_TORCH_CHECK(out.is_cuda() && out.scalar_type() == ScalarType::BFloat16,
|
||||
"out must be a CUDA bfloat16 tensor");
|
||||
STD_TORCH_CHECK(
|
||||
output_gate_activation == "silu" || output_gate_activation == "sigmoid",
|
||||
"output_gate_activation must be 'silu' or 'sigmoid'");
|
||||
|
||||
STD_TORCH_CHECK(mixed_qkv.dim() == 2,
|
||||
"mixed_qkv must have shape [L, 2 * H * 128 + HV * 128]");
|
||||
@@ -547,8 +553,9 @@ void fused_gdn_decode_post_conv_mtp(
|
||||
const auto* b_ptr = static_cast<const __nv_bfloat16*>(b.data_ptr());
|
||||
const auto* output_gate_ptr =
|
||||
static_cast<const __nv_bfloat16*>(output_gate.data_ptr());
|
||||
const auto launch = [&]<typename StateT, int ValueHeadsPerKeyHead>() {
|
||||
launch_gdn_decode_post_conv_mtp<StateT, ValueHeadsPerKeyHead>(
|
||||
const auto launch = [&]<typename StateT, int ValueHeadsPerKeyHead,
|
||||
bool SigmoidGate>() {
|
||||
launch_gdn_decode_post_conv_mtp<StateT, ValueHeadsPerKeyHead, SigmoidGate>(
|
||||
mixed_qkv, a_log, dt_bias, state_indices, cu_seqlens,
|
||||
num_accepted_tokens, state, norm_weight, out, a_ptr, b_ptr,
|
||||
output_gate_ptr, num_key_heads, num_value_heads, scale, norm_eps,
|
||||
@@ -556,9 +563,18 @@ void fused_gdn_decode_post_conv_mtp(
|
||||
};
|
||||
const auto dispatch_state_type = [&]<int ValueHeadsPerKeyHead>() {
|
||||
if (state_scalar_type == ScalarType::Float) {
|
||||
launch.template operator()<float, ValueHeadsPerKeyHead>();
|
||||
if (output_gate_activation == "sigmoid") {
|
||||
launch.template operator()<float, ValueHeadsPerKeyHead, true>();
|
||||
} else {
|
||||
launch.template operator()<float, ValueHeadsPerKeyHead, false>();
|
||||
}
|
||||
} else {
|
||||
launch.template operator()<__nv_bfloat16, ValueHeadsPerKeyHead>();
|
||||
if (output_gate_activation == "sigmoid") {
|
||||
launch.template operator()<__nv_bfloat16, ValueHeadsPerKeyHead, true>();
|
||||
} else {
|
||||
launch
|
||||
.template operator()<__nv_bfloat16, ValueHeadsPerKeyHead, false>();
|
||||
}
|
||||
}
|
||||
};
|
||||
switch (value_heads_per_key_head) {
|
||||
|
||||
@@ -405,7 +405,7 @@ void fused_gdn_decode_post_conv_mtp(
|
||||
torch::stable::Tensor const& num_accepted_tokens,
|
||||
torch::stable::Tensor& state, torch::stable::Tensor const& output_gate,
|
||||
torch::stable::Tensor const& norm_weight, torch::stable::Tensor& out,
|
||||
double scale, double norm_eps);
|
||||
double scale, double norm_eps, const std::string& output_gate_activation);
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
@@ -535,7 +535,8 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
"Tensor mixed_qkv, Tensor a, Tensor b, Tensor A_log, Tensor dt_bias, "
|
||||
"Tensor state_indices, Tensor cu_seqlens, Tensor num_accepted_tokens, "
|
||||
"Tensor! state, Tensor output_gate, Tensor norm_weight, Tensor! out, "
|
||||
"float scale, float norm_eps=1e-5) -> ()");
|
||||
"float scale, float norm_eps=1e-5, "
|
||||
"str output_gate_activation='silu') -> ()");
|
||||
#endif
|
||||
|
||||
#ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES
|
||||
|
||||
@@ -7,6 +7,7 @@ from enum import Enum
|
||||
import pytest
|
||||
|
||||
from vllm.config.cache import CacheConfig
|
||||
from vllm.config.scheduler import SchedulerConfig
|
||||
from vllm.config.utils import get_hash_factors, hash_factors, normalize_value
|
||||
|
||||
# Helpers
|
||||
@@ -216,6 +217,24 @@ def test_cache_config_hash_ignores_kv_cache_sizing_knobs():
|
||||
assert CacheConfig(gpu_memory_utilization=0.5).compute_hash() == base_hash
|
||||
|
||||
|
||||
def test_scheduler_config_hash_includes_max_num_seqs():
|
||||
"""Per-request workspace sizes must invalidate compiled graphs."""
|
||||
base_hash = SchedulerConfig(
|
||||
max_model_len=8192,
|
||||
is_encoder_decoder=False,
|
||||
max_num_batched_tokens=8192,
|
||||
max_num_seqs=128,
|
||||
).compute_hash()
|
||||
larger_batch_hash = SchedulerConfig(
|
||||
max_model_len=8192,
|
||||
is_encoder_decoder=False,
|
||||
max_num_batched_tokens=8192,
|
||||
max_num_seqs=1024,
|
||||
).compute_hash()
|
||||
|
||||
assert larger_batch_hash != base_hash
|
||||
|
||||
|
||||
def test_cache_config_hash_ignores_prefix_cache_retention_interval():
|
||||
base_hash = CacheConfig().compute_hash()
|
||||
assert CacheConfig(prefix_cache_retention_interval=64).compute_hash() == base_hash
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for SpeculativeConfig.compose_draft_hf_overrides.
|
||||
"""Tests for draft config overrides used by SpeculativeConfig.
|
||||
|
||||
Callable ``hf_overrides`` on the target model config (e.g. the
|
||||
``dummy_hf_overrides`` shrink used by ``tests/models/test_initialization.py``)
|
||||
@@ -12,10 +12,12 @@ when the target itself is shrunk — which is what kept spec-decode archs like
|
||||
"""
|
||||
|
||||
import functools
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.config.speculative import SpeculativeConfig
|
||||
|
||||
|
||||
@@ -134,3 +136,52 @@ def test_composed_override_is_picklable():
|
||||
|
||||
out = composed(_make_hf_config())
|
||||
assert out.num_hidden_layers == 1
|
||||
|
||||
|
||||
def _make_mtp_speculative_config(
|
||||
override: bool | None,
|
||||
checkpoint_value: bool,
|
||||
) -> SpeculativeConfig:
|
||||
draft_hf_config = _make_hf_config(
|
||||
architectures=["Qwen4ExpMTP"],
|
||||
model_type="qwen4_exp_mtp",
|
||||
n_predict=1,
|
||||
index_share_for_mtp_iteration=checkpoint_value,
|
||||
)
|
||||
draft_model_config = MagicMock(
|
||||
model="draft",
|
||||
hf_config=draft_hf_config,
|
||||
architectures=draft_hf_config.architectures,
|
||||
max_model_len=128,
|
||||
)
|
||||
target_model_config = MagicMock(
|
||||
model="target",
|
||||
max_model_len=128,
|
||||
quantization=None,
|
||||
hf_overrides={},
|
||||
)
|
||||
|
||||
with patch("vllm.config.speculative.ModelConfig", return_value=draft_model_config):
|
||||
return SpeculativeConfig(
|
||||
model="draft",
|
||||
method="mtp",
|
||||
num_speculative_tokens=1,
|
||||
index_share_for_mtp_iteration=override,
|
||||
target_model_config=target_model_config,
|
||||
target_parallel_config=ParallelConfig(),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize(
|
||||
("override", "checkpoint_value", "expected"),
|
||||
[(None, True, True), (False, True, False), (True, False, True)],
|
||||
)
|
||||
def test_mtp_index_share_override(
|
||||
override: bool | None, checkpoint_value: bool, expected: bool
|
||||
):
|
||||
speculative_config = _make_mtp_speculative_config(override, checkpoint_value)
|
||||
assert (
|
||||
speculative_config.draft_model_config.hf_config.index_share_for_mtp_iteration
|
||||
is expected
|
||||
)
|
||||
|
||||
@@ -24,6 +24,29 @@ for i, v in enumerate(test_sizes):
|
||||
test_sizes[i] -= v % 8
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("dtype", "expected"),
|
||||
[
|
||||
(torch.float32, True),
|
||||
(torch.float16, True),
|
||||
(torch.bfloat16, True),
|
||||
(torch.int8, False),
|
||||
(torch.float8_e4m3fn, False),
|
||||
],
|
||||
)
|
||||
def test_custom_allreduce_filters_dtype(
|
||||
dtype: torch.dtype,
|
||||
expected: bool,
|
||||
) -> None:
|
||||
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
|
||||
communicator.disabled = False
|
||||
communicator.world_size = 2
|
||||
communicator.max_size = 1024
|
||||
communicator._ptr = 0
|
||||
|
||||
assert communicator.should_custom_ar(torch.empty(16, dtype=dtype)) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("major", "local_multicast", "expected"),
|
||||
[
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
# Qwen4Exp accuracy evaluation
|
||||
|
||||
This suite starts a Qwen3.8-Flash-Next-FP8 OpenAI-compatible server once and
|
||||
uses EvalScope to evaluate GSM8K and AIME25.
|
||||
|
||||
```bash
|
||||
# B200
|
||||
pytest -s -v tests/evals/qwen4_exp/test_accuracy.py \
|
||||
--config-list-file=configs/models-b200.txt
|
||||
|
||||
# H200
|
||||
pytest -s -v tests/evals/qwen4_exp/test_accuracy.py \
|
||||
--config-list-file=configs/models-h200.txt
|
||||
```
|
||||
@@ -0,0 +1,3 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
model_name: "Qwen/Qwen3.8-Flash-Next-FP8"
|
||||
datasets:
|
||||
gsm8k:
|
||||
metric_threshold: 0.98
|
||||
tolerance: 0.02
|
||||
aime25:
|
||||
metric_threshold: 0.90
|
||||
tolerance: 0.05
|
||||
eval_batch_size: 32
|
||||
startup_max_wait_seconds: 1800
|
||||
generation_config:
|
||||
seed: 1236
|
||||
do_sample: true
|
||||
temperature: 0.7
|
||||
top_p: 0.8
|
||||
top_k: 20
|
||||
repetition_penalty: 1.0
|
||||
presence_penalty: 1.5
|
||||
max_tokens: 32768
|
||||
stream: true
|
||||
extra_body:
|
||||
chat_template_kwargs:
|
||||
enable_thinking: true
|
||||
server_args: "--tensor-parallel-size 4"
|
||||
@@ -0,0 +1,2 @@
|
||||
# B200 configuration for Qwen3.8-Flash-Next-FP8 accuracy evaluation
|
||||
Qwen3.8-Flash-Next-FP8.yaml
|
||||
@@ -0,0 +1,2 @@
|
||||
# H200 configuration for Qwen3.8-Flash-Next-FP8 accuracy evaluation
|
||||
Qwen3.8-Flash-Next-FP8.yaml
|
||||
@@ -0,0 +1,42 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def pytest_addoption(parser):
|
||||
parser.addoption(
|
||||
"--config-list-file",
|
||||
required=True,
|
||||
help="File containing Qwen4Exp evaluation config files",
|
||||
)
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc):
|
||||
if "config_filename" not in metafunc.fixturenames:
|
||||
return
|
||||
|
||||
config_list_path = Path(metafunc.config.getoption("--config-list-file"))
|
||||
if not config_list_path.is_absolute():
|
||||
test_relative_path = Path(__file__).parent / config_list_path
|
||||
config_list_path = (
|
||||
test_relative_path
|
||||
if test_relative_path.exists()
|
||||
else Path.cwd() / config_list_path
|
||||
)
|
||||
|
||||
config_dir = config_list_path.parent
|
||||
config_files = [
|
||||
config_dir / line.strip()
|
||||
for line in config_list_path.read_text(encoding="utf-8").splitlines()
|
||||
if line.strip() and not line.lstrip().startswith("#")
|
||||
]
|
||||
missing = [path for path in config_files if not path.exists()]
|
||||
if missing:
|
||||
raise FileNotFoundError(f"Missing evaluation configs: {missing}")
|
||||
|
||||
metafunc.parametrize(
|
||||
"config_filename",
|
||||
config_files,
|
||||
ids=[config_file.stem for config_file in config_files],
|
||||
)
|
||||
@@ -0,0 +1,68 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""GSM8K and AIME25 accuracy evaluation for Qwen3.8-Flash-Next-FP8."""
|
||||
|
||||
import shlex
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
|
||||
def run_evalscope(
|
||||
eval_config: dict[str, Any], base_url: str, work_dir: Path
|
||||
) -> dict[str, float]:
|
||||
from evalscope.run import run_task
|
||||
|
||||
reports = run_task(
|
||||
{
|
||||
"model": eval_config["model_name"],
|
||||
"api_url": base_url,
|
||||
"api_key": "EMPTY_TOKEN",
|
||||
"datasets": list(eval_config["datasets"]),
|
||||
"eval_batch_size": eval_config.get("eval_batch_size", 32),
|
||||
"generation_config": eval_config["generation_config"],
|
||||
"work_dir": str(work_dir),
|
||||
"no_timestamp": True,
|
||||
}
|
||||
)
|
||||
if not isinstance(reports, dict):
|
||||
raise TypeError(f"Expected EvalScope reports to be a dict, got {type(reports)}")
|
||||
|
||||
return {dataset: float(report.score) for dataset, report in reports.items()}
|
||||
|
||||
|
||||
def test_qwen4_exp_accuracy(config_filename: Path, tmp_path: Path):
|
||||
eval_config = yaml.safe_load(config_filename.read_text(encoding="utf-8"))
|
||||
server_args = shlex.split(eval_config.get("server_args", ""))
|
||||
server_args.extend(["--trust-remote-code", "--disable-uvicorn-access-log"])
|
||||
|
||||
model_name = eval_config["model_name"]
|
||||
print(f"Starting Qwen4Exp evaluation for model: {model_name}")
|
||||
print(f"Datasets: {', '.join(eval_config['datasets'])}")
|
||||
print(f"Server args: {' '.join(server_args)}")
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
model_name,
|
||||
server_args,
|
||||
env_dict=eval_config.get("env"),
|
||||
max_wait_seconds=eval_config.get("startup_max_wait_seconds", 1800),
|
||||
) as remote_server:
|
||||
scores = run_evalscope(eval_config, remote_server.url_for("v1"), tmp_path)
|
||||
|
||||
for dataset, metric_config in eval_config["datasets"].items():
|
||||
score = scores[dataset]
|
||||
threshold = metric_config["metric_threshold"]
|
||||
tolerance = metric_config.get("tolerance", 0.0)
|
||||
minimum_score = threshold - tolerance
|
||||
|
||||
print(
|
||||
f"{dataset}: measured={score:.4f}, expected={threshold:.4f}, "
|
||||
f"tolerance={tolerance:.4f}"
|
||||
)
|
||||
assert score >= minimum_score, (
|
||||
f"{dataset} score too low: {score:.4f} < {threshold:.4f} - "
|
||||
f"{tolerance:.4f} = {minimum_score:.4f}"
|
||||
)
|
||||
@@ -57,13 +57,13 @@ EPS = 1e-6
|
||||
|
||||
|
||||
class _TestGatedNorm:
|
||||
def __init__(self, weight: torch.Tensor) -> None:
|
||||
def __init__(self, weight: torch.Tensor, activation: str) -> None:
|
||||
self.weight = weight
|
||||
self.bias = None
|
||||
self.eps = EPS
|
||||
self.group_size = None
|
||||
self.norm_before_gate = True
|
||||
self.activation = "silu"
|
||||
self.activation = activation
|
||||
|
||||
def __call__(self, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor:
|
||||
return rmsnorm_fn(
|
||||
@@ -99,6 +99,7 @@ def _build_layer(
|
||||
dt_bias: torch.Tensor,
|
||||
conv_weight: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
output_gate_activation: str,
|
||||
):
|
||||
layer = types.SimpleNamespace(
|
||||
prefix=PREFIX,
|
||||
@@ -116,7 +117,7 @@ def _build_layer(
|
||||
dt_bias=dt_bias,
|
||||
conv1d=types.SimpleNamespace(weight=conv_weight, bias=None),
|
||||
kv_cache=(conv_state, ssm_state),
|
||||
norm=_TestGatedNorm(norm_weight),
|
||||
norm=_TestGatedNorm(norm_weight, output_gate_activation),
|
||||
layer_norm_epsilon=EPS,
|
||||
gdn_decode_kernel="cuda",
|
||||
)
|
||||
@@ -250,12 +251,14 @@ def test_fused_forward_uses_packed_entrypoint() -> None:
|
||||
pytest.param([128], [1], [-1], 0, id="pure-decode"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("output_gate_activation", ["silu", "sigmoid"])
|
||||
@torch.inference_mode()
|
||||
def test_fused_model_path_matches_reference(
|
||||
seq_lens: list[int],
|
||||
query_lens: list[int],
|
||||
draft_tokens: list[int],
|
||||
expected_fused_calls: int,
|
||||
output_gate_activation: str,
|
||||
) -> None:
|
||||
"""Fused MTP and its mixed/prefill/decode fallbacks match the reference."""
|
||||
torch.manual_seed(1)
|
||||
@@ -334,6 +337,7 @@ def test_fused_model_path_matches_reference(
|
||||
dt_bias,
|
||||
conv_weight,
|
||||
norm_weight,
|
||||
output_gate_activation,
|
||||
)
|
||||
reference_out = torch.zeros_like(output_gate)
|
||||
with patch.object(
|
||||
@@ -355,6 +359,7 @@ def test_fused_model_path_matches_reference(
|
||||
dt_bias,
|
||||
conv_weight,
|
||||
norm_weight,
|
||||
output_gate_activation,
|
||||
)
|
||||
context.no_compile_layers = {PREFIX: fused_layer}
|
||||
fused_out = torch.zeros_like(output_gate)
|
||||
|
||||
@@ -366,7 +366,7 @@ def test_preprocess_fused_align_matches_scalar_bookkeeping(monkeypatch, token_bi
|
||||
input_batch=scalar_case[1],
|
||||
requests=scalar_case[2],
|
||||
forward_context={},
|
||||
mamba_state_copy_funcs=(),
|
||||
mamba_state_copy_funcs={},
|
||||
copy_bufs=scalar_copy_bufs,
|
||||
)
|
||||
|
||||
@@ -384,7 +384,7 @@ def test_preprocess_fused_align_matches_scalar_bookkeeping(monkeypatch, token_bi
|
||||
input_batch=fused_case[1],
|
||||
requests=fused_case[2],
|
||||
forward_context={},
|
||||
mamba_state_copy_funcs=(),
|
||||
mamba_state_copy_funcs={},
|
||||
copy_bufs=fused_copy_bufs,
|
||||
align_ctx=ctx,
|
||||
)
|
||||
|
||||
@@ -351,3 +351,107 @@ def test_w8a8_block_fp8_deep_gemm_fused_moe(M, N, K, E, topk, seed, monkeypatch)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
torch.testing.assert_close(out, ref_out, atol=0.035, rtol=0.035)
|
||||
|
||||
|
||||
def _dequant_block_fp8(
|
||||
w_fp8: torch.Tensor, w_s: torch.Tensor, block_shape: list[int]
|
||||
) -> torch.Tensor:
|
||||
"""Dequantize a [N, K] fp8 weight with [Nb, Kb] block scales to fp32."""
|
||||
n, k = w_fp8.shape
|
||||
bn, bk = block_shape
|
||||
s = w_s.repeat_interleave(bn, dim=0).repeat_interleave(bk, dim=1)
|
||||
return w_fp8.to(torch.float32) * s[:n, :k]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_rank", [0, 1, 2, 3])
|
||||
@torch.inference_mode()
|
||||
def test_w8a8_block_fp8_fused_moe_refined_block_scales(tp_rank, workspace_init):
|
||||
"""TP-misaligned blockwise FP8 MoE (e.g. Qwen4Exp, intermediate
|
||||
640 at TP=4 -> 160 per rank, not divisible by the checkpoint's 128 block).
|
||||
|
||||
The per-rank scale grid is refined from 128 to 32 (lossless: each 32-block
|
||||
lies inside one global 128-block) so the Triton kernel can consume exact
|
||||
per-shard scales. Simulates one TP rank's shard and checks:
|
||||
1. the refined+sharded scales dequantize exactly like the global scales;
|
||||
2. the Triton fused MoE kernel with block_shape=[32, 32] matches the
|
||||
native-torch blockwise reference on the same shard.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
dtype = torch.bfloat16
|
||||
M, E, topk = 8, 4, 2
|
||||
hidden = 256
|
||||
inter_full = 640
|
||||
tp_size = 4
|
||||
n_shard = inter_full // tp_size # 160
|
||||
ckpt_block = [128, 128]
|
||||
refined_block = [32, 32]
|
||||
factor = ckpt_block[0] // refined_block[0] # 4
|
||||
|
||||
a = torch.randn((M, hidden), dtype=dtype) / 10
|
||||
score = torch.randn((M, E), dtype=dtype)
|
||||
|
||||
(_, w1_full, w1_s_full, _), (_, w2_full, w2_s_full, _) = make_test_weights(
|
||||
E,
|
||||
inter_full,
|
||||
hidden,
|
||||
dtype,
|
||||
torch.float8_e4m3fn,
|
||||
per_out_ch_quant=False,
|
||||
block_shape=ckpt_block,
|
||||
)
|
||||
assert w1_s_full is not None
|
||||
assert w2_s_full is not None
|
||||
|
||||
# TP shard the weights (exact slicing, no scale involvement).
|
||||
lo, hi = tp_rank * n_shard, (tp_rank + 1) * n_shard
|
||||
w1 = torch.cat(
|
||||
[w1_full[:, lo:hi], w1_full[:, inter_full + lo : inter_full + hi]], dim=1
|
||||
)
|
||||
w2 = w2_full[:, :, lo:hi]
|
||||
|
||||
# Refine the global 128-block scales to 32 blocks, then take the shard's
|
||||
# slice -- mirrors the Fp8MoEMethod/RoutedExperts loading path.
|
||||
w1_s32 = w1_s_full.repeat_interleave(factor, dim=-2).repeat_interleave(
|
||||
factor, dim=-1
|
||||
)
|
||||
nb = n_shard // refined_block[0] # 5 local 32-blocks per projection
|
||||
gate_hi = w1_s32.shape[1] // 2
|
||||
w1_s = torch.cat(
|
||||
[
|
||||
w1_s32[:, tp_rank * nb : (tp_rank + 1) * nb],
|
||||
w1_s32[:, gate_hi + tp_rank * nb : gate_hi + (tp_rank + 1) * nb],
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
w2_s32 = w2_s_full.repeat_interleave(factor, dim=-2).repeat_interleave(
|
||||
factor, dim=-1
|
||||
)
|
||||
w2_s = w2_s32[:, :, tp_rank * nb : (tp_rank + 1) * nb]
|
||||
|
||||
# The refined per-shard scales must reproduce the global-scale dequant
|
||||
# exactly (dequant full-width, then slice to the shard).
|
||||
for e in range(E):
|
||||
d32 = _dequant_block_fp8(w1[e, :n_shard], w1_s[e, :nb], refined_block)
|
||||
d128 = _dequant_block_fp8(w1_full[e], w1_s_full[e], ckpt_block)[lo:hi]
|
||||
assert torch.equal(d32, d128)
|
||||
d32 = _dequant_block_fp8(w2[e], w2_s[e], refined_block)
|
||||
d128 = _dequant_block_fp8(w2_full[e], w2_s_full[e], ckpt_block)[:, lo:hi]
|
||||
assert torch.equal(d32, d128)
|
||||
|
||||
quant_config = fp8_w8a8_moe_quant_config(
|
||||
w1_scale=w1_s,
|
||||
w2_scale=w2_s,
|
||||
block_shape=refined_block,
|
||||
)
|
||||
|
||||
topk_weights, topk_ids, _ = fused_topk(a, score.float(), topk, False)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
ref_out = torch_w8a8_block_fp8_moe(
|
||||
a, w1, w2, w1_s, w2_s, topk_weights, topk_ids, refined_block
|
||||
)
|
||||
out = fused_experts(
|
||||
a, w1, w2, topk_weights, topk_ids, quant_config=quant_config
|
||||
)
|
||||
|
||||
torch.testing.assert_close(out, ref_out, atol=0.035, rtol=0.035)
|
||||
|
||||
@@ -28,8 +28,9 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import (
|
||||
TrtLlmBf16ExpertsModular,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
convert_moe_weights_to_flashinfer_trtllm_block_layout,
|
||||
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
UnquantizedMoeBackend,
|
||||
convert_to_unquantized_kernel_format,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
|
||||
@@ -46,9 +47,8 @@ if pytest and (
|
||||
)
|
||||
|
||||
# (m, n, k) = (tokens, intermediate_size_per_partition, hidden_dim).
|
||||
# Covers larger NvFP4-like shapes while keeping BF16's FlashInfer TRTLLM
|
||||
# intermediate-size multiple-of-128 requirement.
|
||||
MNK_FACTORS = [
|
||||
(2, 160, 2560),
|
||||
(2, 1024, 1024),
|
||||
(64, 2048, 1536),
|
||||
(64, 1024, 4096),
|
||||
@@ -97,11 +97,15 @@ def test_trtllm_bf16_moe_modular_no_graph(
|
||||
max_num_tokens=next_power_of_2(m),
|
||||
)
|
||||
|
||||
trtllm_w1, trtllm_w2 = convert_moe_weights_to_flashinfer_trtllm_block_layout(
|
||||
{},
|
||||
trtllm_w1, trtllm_w2 = convert_to_unquantized_kernel_format(
|
||||
UnquantizedMoeBackend.FLASHINFER_TRTLLM,
|
||||
moe_config,
|
||||
w1,
|
||||
w2,
|
||||
)
|
||||
expected_n = (n + 127) // 128 * 128
|
||||
assert moe_config.intermediate_size_per_partition == expected_n
|
||||
assert trtllm_w2.numel() == e * k * expected_n
|
||||
|
||||
trtllm_experts = mk.FusedMoEKernel(
|
||||
maybe_make_prepare_finalize(
|
||||
|
||||
@@ -267,6 +267,7 @@ def test_fused_post_conv_l0():
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("output_gate_activation", ["silu", "sigmoid"])
|
||||
@torch.inference_mode()
|
||||
def test_fused_gdn_decode_post_conv_mtp_head_ratios(
|
||||
head_ratio: int,
|
||||
@@ -274,6 +275,7 @@ def test_fused_gdn_decode_post_conv_mtp_head_ratios(
|
||||
query_lengths: tuple[int, ...],
|
||||
state_dtype: torch.dtype,
|
||||
norm_dtype: torch.dtype,
|
||||
output_gate_activation: str,
|
||||
) -> None:
|
||||
if torch.cuda.get_device_capability() < (8, 0):
|
||||
pytest.skip("fused GDN decode MTP requires compute capability 8.0+")
|
||||
@@ -357,7 +359,7 @@ def test_fused_gdn_decode_post_conv_mtp_head_ratios(
|
||||
z=output_gate,
|
||||
eps=eps,
|
||||
norm_before_gate=True,
|
||||
activation="silu",
|
||||
activation=output_gate_activation,
|
||||
)
|
||||
actual = ops.fused_gdn_decode_post_conv_mtp(
|
||||
mixed_qkv=mixed_qkv,
|
||||
@@ -374,6 +376,7 @@ def test_fused_gdn_decode_post_conv_mtp_head_ratios(
|
||||
out=torch.empty_like(output_gate),
|
||||
scale=scale,
|
||||
norm_eps=eps,
|
||||
output_gate_activation=output_gate_activation,
|
||||
)
|
||||
|
||||
output_error = (actual.float() - expected.float()).norm()
|
||||
|
||||
@@ -18,8 +18,12 @@ from vllm.model_executor.layers.fused_moe.layer import determine_expert_counts
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
|
||||
from vllm.model_executor.layers.quantization.utils.config_utils import (
|
||||
get_quark_ocp_mx_group_size,
|
||||
is_shared_expert_quant_fse_compatible,
|
||||
)
|
||||
from vllm.model_executor.models.qwen3_next import (
|
||||
_should_replicate_misaligned_shared_expert,
|
||||
)
|
||||
from vllm.model_executor.models.utils import PPMissingLayer
|
||||
from vllm.models.deepseek_v4 import quant_config as deepseek_v4_quant_config
|
||||
from vllm.models.minimax_m3.amd import model as minimax_m3_model
|
||||
@@ -605,6 +609,82 @@ def test_quark_shared_expert_fse_compatibility(
|
||||
)
|
||||
|
||||
|
||||
def test_quark_ocp_mx_group_size_for_layer() -> None:
|
||||
quant_config = QuarkConfig({**_QUARK_FSE_CONFIG, "exclude": []})
|
||||
assert (
|
||||
get_quark_ocp_mx_group_size(
|
||||
quant_config,
|
||||
"model.layers.0.mlp.shared_expert.down_proj",
|
||||
)
|
||||
== 32
|
||||
)
|
||||
|
||||
quant_config.quant_config["exclude"] = ["model.layers.0.mlp.shared_expert_gate"]
|
||||
assert (
|
||||
get_quark_ocp_mx_group_size(
|
||||
quant_config,
|
||||
"model.layers.0.mlp.shared_expert.down_proj",
|
||||
)
|
||||
== 32
|
||||
)
|
||||
|
||||
quant_config.quant_config["exclude"] = [
|
||||
"model.layers.0.mlp.shared_expert.down_proj"
|
||||
]
|
||||
assert (
|
||||
get_quark_ocp_mx_group_size(
|
||||
quant_config,
|
||||
"model.layers.0.mlp.shared_expert.down_proj",
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [1, 2, 4])
|
||||
def test_quark_ocp_mx_shared_expert_uses_supported_tp_shards(tp_size: int) -> None:
|
||||
assert not _should_replicate_misaligned_shared_expert(
|
||||
intermediate_size=640,
|
||||
tp_size=tp_size,
|
||||
group_size=32,
|
||||
enable_expert_parallel=False,
|
||||
is_sequence_parallel=False,
|
||||
)
|
||||
|
||||
|
||||
def test_quark_ocp_mx_shared_expert_rejects_plain_tp8() -> None:
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="TP size 8 produces a partition of 80",
|
||||
):
|
||||
_should_replicate_misaligned_shared_expert(
|
||||
intermediate_size=640,
|
||||
tp_size=8,
|
||||
group_size=32,
|
||||
enable_expert_parallel=False,
|
||||
is_sequence_parallel=False,
|
||||
)
|
||||
|
||||
|
||||
def test_quark_ocp_mx_shared_expert_is_replicated_for_tep8() -> None:
|
||||
assert _should_replicate_misaligned_shared_expert(
|
||||
intermediate_size=640,
|
||||
tp_size=8,
|
||||
group_size=32,
|
||||
enable_expert_parallel=True,
|
||||
is_sequence_parallel=False,
|
||||
)
|
||||
|
||||
|
||||
def test_quark_ocp_mx_shared_expert_is_replicated_when_not_tp_divisible() -> None:
|
||||
assert _should_replicate_misaligned_shared_expert(
|
||||
intermediate_size=672,
|
||||
tp_size=8,
|
||||
group_size=32,
|
||||
enable_expert_parallel=True,
|
||||
is_sequence_parallel=False,
|
||||
)
|
||||
|
||||
|
||||
def test_quark_shared_expert_fse_requires_matching_layer_quant_configs() -> None:
|
||||
global_quant_config = {"weight": {"dtype": "fp4"}}
|
||||
quant_config = QuarkConfig(
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,200 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.speculative import SpeculativeConfig
|
||||
from vllm.model_executor.models.config import (
|
||||
Qwen3_5ForConditionalGenerationConfig,
|
||||
Qwen4ExpForConditionalGenerationConfig,
|
||||
)
|
||||
from vllm.models.qwen4_exp.config import (
|
||||
Qwen4ExpConfig,
|
||||
Qwen4ExpTextConfig,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia.model_state import Qwen4ExpModelState
|
||||
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
|
||||
|
||||
|
||||
def _text_config(**kwargs) -> Qwen4ExpTextConfig:
|
||||
values = {
|
||||
"vocab_size": 64,
|
||||
"hidden_size": 16,
|
||||
"intermediate_size": 32,
|
||||
"num_hidden_layers": 2,
|
||||
"num_attention_heads": 2,
|
||||
"num_key_value_heads": 1,
|
||||
"head_dim": 8,
|
||||
"layer_types": ["linear_attention", "full_attention"],
|
||||
"linear_num_key_heads": 2,
|
||||
"linear_num_value_heads": 2,
|
||||
"linear_key_head_dim": 8,
|
||||
"linear_value_head_dim": 8,
|
||||
"num_experts": 0,
|
||||
"hc_count": 2,
|
||||
"hc_lowrank": 4,
|
||||
"ple_layer_ids": [1],
|
||||
"mtp_num_hidden_layers": 1,
|
||||
"mtp": {"hybrid": True},
|
||||
}
|
||||
values.update(kwargs)
|
||||
return Qwen4ExpTextConfig(**values)
|
||||
|
||||
|
||||
def test_qwen4_exp_mtp_returns_sample_and_multi_streams() -> None:
|
||||
from vllm.models.qwen4_exp.nvidia.mtp import (
|
||||
Qwen4ExpMultiTokenPredictor,
|
||||
)
|
||||
|
||||
model = object.__new__(Qwen4ExpMultiTokenPredictor)
|
||||
torch.nn.Module.__init__(model)
|
||||
model.hc_count = 2
|
||||
model.hidden_size = 4
|
||||
model.num_mtp_layers = 1
|
||||
model.layers = [
|
||||
lambda **kwargs: (
|
||||
kwargs["hidden_states"],
|
||||
kwargs["hidden_states"],
|
||||
torch.zeros(kwargs["hidden_states"].shape[0], 2),
|
||||
),
|
||||
]
|
||||
model.hyper_connection_mixer = SimpleNamespace(
|
||||
combine_and_mix=lambda hidden_states, block_output, injection: (
|
||||
hidden_states,
|
||||
hidden_states.unflatten(-1, (2, 4)).mean(-2),
|
||||
None,
|
||||
),
|
||||
)
|
||||
multi_hidden = torch.arange(16, dtype=torch.float32).reshape(2, 8)
|
||||
pp_group = SimpleNamespace(is_first_rank=False, is_last_rank=True)
|
||||
|
||||
with patch(
|
||||
"vllm.models.qwen4_exp.nvidia.mtp.get_pp_group",
|
||||
return_value=pp_group,
|
||||
):
|
||||
sample_hidden, returned_multi_hidden = model.forward(
|
||||
input_ids=None,
|
||||
positions=torch.arange(2),
|
||||
intermediate_tensors={"hidden_states": multi_hidden},
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
sample_hidden,
|
||||
multi_hidden.unflatten(-1, (2, 4)).mean(dim=-2),
|
||||
)
|
||||
assert returned_multi_hidden is multi_hidden
|
||||
|
||||
|
||||
@pytest.mark.parametrize("wrapped_config", [False, True])
|
||||
def test_qwen4_exp_mtp_override_sets_draft_config(
|
||||
wrapped_config: bool,
|
||||
) -> None:
|
||||
text_config = _text_config(
|
||||
architectures=["Qwen4ExpForCausalLM"],
|
||||
index_share_for_mtp_iteration=True,
|
||||
)
|
||||
config = (
|
||||
Qwen4ExpConfig(
|
||||
architectures=["Qwen4ExpForConditionalGeneration"],
|
||||
text_config=text_config,
|
||||
)
|
||||
if wrapped_config
|
||||
else text_config
|
||||
)
|
||||
|
||||
draft_config = SpeculativeConfig.hf_config_override(config)
|
||||
|
||||
assert draft_config.index_share_for_mtp_iteration is True
|
||||
assert draft_config.model_type == "qwen4_exp_mtp"
|
||||
assert draft_config.architectures == ["Qwen4ExpMTP"]
|
||||
assert draft_config.hc_mult == 2
|
||||
assert draft_config.n_predict == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ple_layer_ids", [[1], []])
|
||||
def test_qwen4_exp_rejects_pipeline_parallel_only_with_ple(ple_layer_ids) -> None:
|
||||
"""PLE needs raw input_ids, which non-first pipeline ranks never see. The
|
||||
rest of the architecture is PP-capable, so the refusal must be conditional
|
||||
-- and must land before the engine spends time loading weights."""
|
||||
vllm_config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
hf_text_config=_text_config(ple_layer_ids=ple_layer_ids),
|
||||
multimodal_config=None,
|
||||
),
|
||||
parallel_config=SimpleNamespace(
|
||||
pipeline_parallel_size=2, enable_dbo=False, ubatch_size=1
|
||||
),
|
||||
speculative_config=None,
|
||||
)
|
||||
with patch.object(
|
||||
Qwen3_5ForConditionalGenerationConfig, "verify_and_update_config"
|
||||
):
|
||||
if ple_layer_ids:
|
||||
with pytest.raises(NotImplementedError, match="pipeline_parallel_size=1"):
|
||||
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(
|
||||
vllm_config
|
||||
)
|
||||
else:
|
||||
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(vllm_config)
|
||||
|
||||
|
||||
def test_qwen4_exp_model_state_prepares_ngram_context() -> None:
|
||||
model_state = object.__new__(Qwen4ExpModelState)
|
||||
model_state.uses_ngram_embedding = True
|
||||
model_state.ngram_context_len = 3
|
||||
model_state.ngram_eos_token_id = 99
|
||||
model_state.ngram_context = torch.empty((4, 3), dtype=torch.int32)
|
||||
model_state.ngram_context_offsets = torch.arange(-3, 0, dtype=torch.int64)
|
||||
model_state.ple_query_start_loc = torch.empty(5, dtype=torch.int32)
|
||||
|
||||
input_batch = SimpleNamespace(
|
||||
num_reqs=2,
|
||||
num_reqs_after_padding=3,
|
||||
idx_mapping=torch.tensor([1, 0]),
|
||||
query_start_loc=torch.tensor([0, 2, 3, 3], dtype=torch.int32),
|
||||
)
|
||||
req_states = SimpleNamespace(
|
||||
num_computed_tokens=SimpleNamespace(gpu=torch.tensor([3, 1])),
|
||||
all_token_ids=SimpleNamespace(
|
||||
gpu=torch.tensor([[1, 2, 3, 4], [20, 21, 22, 23]], dtype=torch.int32)
|
||||
),
|
||||
)
|
||||
|
||||
with patch.object(MambaHybridModelState, "prepare_inputs", return_value={}):
|
||||
model_inputs = model_state.prepare_inputs(input_batch, req_states)
|
||||
|
||||
torch.testing.assert_close(
|
||||
model_inputs["query_start_loc"],
|
||||
torch.tensor([0, 2, 3, 3], dtype=torch.int32),
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
model_inputs["ngram_context"],
|
||||
torch.tensor([[99, 99, 20], [1, 2, 3], [99, 99, 99]], dtype=torch.int32),
|
||||
)
|
||||
|
||||
|
||||
def test_qwen4_exp_model_state_prepares_stable_dummy_ngram_inputs() -> None:
|
||||
model_state = object.__new__(Qwen4ExpModelState)
|
||||
model_state.uses_ngram_embedding = True
|
||||
model_state.ngram_eos_token_id = 99
|
||||
model_state.ngram_context = torch.empty((4, 3), dtype=torch.int32)
|
||||
model_state.ple_query_start_loc = torch.empty(5, dtype=torch.int32)
|
||||
|
||||
with patch.object(MambaHybridModelState, "prepare_dummy_inputs", return_value={}):
|
||||
first = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=8)
|
||||
query_start_loc_ptr = first["query_start_loc"].data_ptr()
|
||||
ngram_context_ptr = first["ngram_context"].data_ptr()
|
||||
second = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=8)
|
||||
|
||||
torch.testing.assert_close(
|
||||
second["query_start_loc"], torch.tensor([0, 2, 5, 8], dtype=torch.int32)
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
second["ngram_context"], torch.full((3, 3), 99, dtype=torch.int32)
|
||||
)
|
||||
assert second["query_start_loc"].data_ptr() == query_start_loc_ptr
|
||||
assert second["ngram_context"].data_ptr() == ngram_context_ptr
|
||||
@@ -0,0 +1,94 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.qwen4_exp.nvidia.ops.hc import (
|
||||
grouped_gemma_rmsnorm,
|
||||
hc_combine,
|
||||
hc_combine_norm,
|
||||
hc_gate_mix,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda() or not HAS_TRITON,
|
||||
reason="HC kernels require CUDA and Triton",
|
||||
)
|
||||
|
||||
HC = 4
|
||||
HIDDEN_SIZE = 2560
|
||||
HYPER_HIDDEN_SIZE = HC * HIDDEN_SIZE
|
||||
EPS = 1e-6
|
||||
|
||||
|
||||
def test_grouped_gemma_rmsnorm() -> None:
|
||||
torch.manual_seed(0)
|
||||
x = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
actual = grouped_gemma_rmsnorm(x, weight, EPS, HC)
|
||||
|
||||
grouped = x.float().unflatten(-1, (HC, HIDDEN_SIZE))
|
||||
variance = grouped.square().mean(-1, keepdim=True)
|
||||
expected = grouped * torch.rsqrt(variance + EPS)
|
||||
expected = expected.flatten(-2) * (1.0 + weight.float())
|
||||
torch.testing.assert_close(actual, expected.to(torch.bfloat16))
|
||||
|
||||
|
||||
def test_hc_gate_mix() -> None:
|
||||
torch.manual_seed(0)
|
||||
x = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
gate = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
actual = hc_gate_mix(x, gate, HC)
|
||||
expected = (
|
||||
torch.sigmoid(gate.float().unflatten(-1, (HC, HIDDEN_SIZE)))
|
||||
* x.float().unflatten(-1, (HC, HIDDEN_SIZE))
|
||||
).mean(-2)
|
||||
|
||||
torch.testing.assert_close(actual, expected.to(torch.bfloat16))
|
||||
|
||||
|
||||
def test_hc_combine() -> None:
|
||||
torch.manual_seed(0)
|
||||
block_output = torch.randn(2, HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
residual = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
injection = torch.randn(2, HC, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
actual = hc_combine(residual, block_output, injection, HC)
|
||||
injection_weight = 2.0 * torch.sigmoid(injection.float() / HC)
|
||||
expected = residual.float().unflatten(-1, (HC, HIDDEN_SIZE))
|
||||
expected = expected + block_output.float().unsqueeze(
|
||||
-2
|
||||
) * injection_weight.unsqueeze(-1)
|
||||
|
||||
torch.testing.assert_close(actual, expected.flatten(-2).to(torch.bfloat16))
|
||||
|
||||
|
||||
def test_hc_combine_norm() -> None:
|
||||
torch.manual_seed(0)
|
||||
block_output = torch.randn(2, HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
residual = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
injection = torch.randn(2, HC, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
actual, actual_norm = hc_combine_norm(
|
||||
residual, block_output, injection, weight, EPS, HC
|
||||
)
|
||||
|
||||
injection_weight = 2.0 * torch.sigmoid(injection.float() / HC)
|
||||
expected = residual.float().unflatten(-1, (HC, HIDDEN_SIZE))
|
||||
expected = expected + block_output.float().unsqueeze(
|
||||
-2
|
||||
) * injection_weight.unsqueeze(-1)
|
||||
expected = expected.flatten(-2).to(residual.dtype)
|
||||
grouped = expected.float().unflatten(-1, (HC, HIDDEN_SIZE))
|
||||
variance = grouped.square().mean(-1, keepdim=True)
|
||||
expected_norm = grouped * torch.rsqrt(variance + EPS)
|
||||
expected_norm = expected_norm.flatten(-2) * (1.0 + weight.float())
|
||||
|
||||
torch.testing.assert_close(actual, expected)
|
||||
torch.testing.assert_close(actual_norm, expected_norm.to(torch.bfloat16))
|
||||
@@ -0,0 +1,401 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from functools import partial
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.nn import functional as F
|
||||
|
||||
import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module
|
||||
import vllm.model_executor.parameter as parameter_module
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.models.qwen4_exp.common.ple import (
|
||||
PLEShardOverlap,
|
||||
compute_ple_shard_overlap,
|
||||
copy_ple_embedding_shard_,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia import ple_layer as ple_layer_module
|
||||
from vllm.models.qwen4_exp.nvidia.ple_layer import (
|
||||
Qwen4ExpNGramEmbedding,
|
||||
Qwen4ExpPLEFp8EmbeddingMethod,
|
||||
Qwen4ExpPLELayer,
|
||||
_get_ple_embedding_quant_method,
|
||||
)
|
||||
|
||||
|
||||
def _make_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding:
|
||||
module = Qwen4ExpNGramEmbedding.__new__(Qwen4ExpNGramEmbedding)
|
||||
nn.Module.__init__(module)
|
||||
module.split_ngram_parts = 2
|
||||
module.register_buffer("layer_multipliers", torch.zeros(1, dtype=torch.long))
|
||||
module.register_buffer("ngram_heads_offsets", torch.zeros(1, dtype=torch.long))
|
||||
module.register_buffer("ngram_heads_vocab_sizes", torch.zeros(1, dtype=torch.long))
|
||||
module.ngram_embedding = SimpleNamespace(
|
||||
org_vocab_size=8,
|
||||
embedding_dim=2,
|
||||
weight=nn.Parameter(torch.full((4, 2), -1.0)),
|
||||
shard_indices=SimpleNamespace(
|
||||
org_vocab_start_index=2,
|
||||
org_vocab_end_index=6,
|
||||
),
|
||||
)
|
||||
_set_test_embedding_weight_loader(module.ngram_embedding)
|
||||
return module
|
||||
|
||||
|
||||
def _set_test_embedding_weight_loader(embedding) -> None:
|
||||
embedding.weight.weight_loader = partial(
|
||||
copy_ple_embedding_shard_,
|
||||
tp_start=embedding.shard_indices.org_vocab_start_index,
|
||||
tp_end=embedding.shard_indices.org_vocab_end_index,
|
||||
)
|
||||
|
||||
|
||||
def _make_fp8_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding:
|
||||
module = _make_ngram_embedding_for_load_test()
|
||||
embedding = nn.Module()
|
||||
embedding.org_vocab_size = 8
|
||||
embedding.embedding_dim = 2
|
||||
embedding.shard_indices = SimpleNamespace(
|
||||
org_vocab_start_index=2,
|
||||
org_vocab_end_index=6,
|
||||
)
|
||||
embedding.register_parameter(
|
||||
"weight",
|
||||
nn.Parameter(
|
||||
torch.full((4, 2), -1.0).to(torch.float8_e4m3fn),
|
||||
requires_grad=False,
|
||||
),
|
||||
)
|
||||
embedding.register_parameter(
|
||||
"weight_scale",
|
||||
nn.Parameter(torch.zeros(1, dtype=torch.bfloat16), requires_grad=False),
|
||||
)
|
||||
_set_test_embedding_weight_loader(embedding)
|
||||
module.ngram_embedding = embedding
|
||||
return module
|
||||
|
||||
|
||||
def test_ple_shard_overlap_and_copy() -> None:
|
||||
overlap = compute_ple_shard_overlap(
|
||||
checkpoint_start=2, checkpoint_rows=5, tp_start=4, tp_end=8
|
||||
)
|
||||
assert overlap == PLEShardOverlap(source_start=2, destination_start=0, row_count=3)
|
||||
|
||||
destination = torch.full((4, 2), -1.0)
|
||||
loaded = torch.arange(10, dtype=torch.float64).reshape(5, 2)
|
||||
copied = copy_ple_embedding_shard_(
|
||||
destination,
|
||||
loaded,
|
||||
checkpoint_start=2,
|
||||
tp_start=4,
|
||||
tp_end=8,
|
||||
)
|
||||
|
||||
assert copied == 3
|
||||
torch.testing.assert_close(destination[:3], loaded[2:5].float())
|
||||
torch.testing.assert_close(destination[3], torch.tensor([-1.0, -1.0]))
|
||||
|
||||
|
||||
def test_ple_shard_copy_is_a_noop_without_overlap() -> None:
|
||||
destination = torch.ones(4, 2)
|
||||
copied = copy_ple_embedding_shard_(
|
||||
destination,
|
||||
torch.zeros(2, 2),
|
||||
checkpoint_start=10,
|
||||
tp_start=4,
|
||||
tp_end=8,
|
||||
)
|
||||
|
||||
assert copied == 0
|
||||
assert torch.equal(destination, torch.ones_like(destination))
|
||||
|
||||
|
||||
def test_ngram_embedding_loads_shards_and_ignores_legacy_token_lookup() -> None:
|
||||
module = _make_ngram_embedding_for_load_test()
|
||||
shard_0 = torch.arange(8, dtype=torch.float32).reshape(4, 2)
|
||||
shard_1 = torch.arange(8, 16, dtype=torch.float32).reshape(4, 2)
|
||||
|
||||
loaded = module.load_weights(
|
||||
[
|
||||
("ngram_embedding.shard_0.weight", shard_0),
|
||||
("ngram_embedding.shard_1.weight", shard_1),
|
||||
("token_lookup", torch.tensor([2, 1, 0])),
|
||||
]
|
||||
)
|
||||
|
||||
assert loaded == {"ngram_embedding.weight"}
|
||||
torch.testing.assert_close(
|
||||
module.ngram_embedding.weight,
|
||||
torch.cat((shard_0[2:4], shard_1[0:2])),
|
||||
)
|
||||
|
||||
|
||||
def test_ngram_embedding_loads_fp8_shards_and_global_scale() -> None:
|
||||
module = _make_fp8_ngram_embedding_for_load_test()
|
||||
shard_0 = torch.arange(8, dtype=torch.float32).reshape(4, 2).to(torch.float8_e4m3fn)
|
||||
shard_1 = (
|
||||
torch.arange(8, 16, dtype=torch.float32).reshape(4, 2).to(torch.float8_e4m3fn)
|
||||
)
|
||||
weight_scale = torch.tensor([0.25], dtype=torch.bfloat16)
|
||||
|
||||
loaded = module.load_weights(
|
||||
[
|
||||
("ngram_embedding.shard_0.weight", shard_0),
|
||||
("ngram_embedding.shard_1.weight", shard_1),
|
||||
("ngram_embedding.weight_scale", weight_scale),
|
||||
]
|
||||
)
|
||||
|
||||
assert loaded == {"ngram_embedding.weight", "ngram_embedding.weight_scale"}
|
||||
assert module.ngram_embedding.weight.dtype == torch.float8_e4m3fn
|
||||
assert torch.equal(
|
||||
module.ngram_embedding.weight.float(),
|
||||
torch.cat((shard_0[2:4], shard_1[0:2])).float(),
|
||||
)
|
||||
assert torch.equal(module.ngram_embedding.weight_scale, weight_scale)
|
||||
|
||||
|
||||
def _make_fp8_embedding_layer(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> embedding_module.VocabParallelEmbedding:
|
||||
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
embedding_module, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
)
|
||||
monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
parameter_module, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
)
|
||||
method = Qwen4ExpPLEFp8EmbeddingMethod()
|
||||
layer = embedding_module.VocabParallelEmbedding(
|
||||
3,
|
||||
2,
|
||||
params_dtype=torch.bfloat16,
|
||||
padding_size=1,
|
||||
quant_method=method,
|
||||
)
|
||||
weight = torch.tensor([[1.0, 2.0], [4.0, 8.0], [16.0, 32.0]])
|
||||
layer.weight.data.copy_(weight.to(torch.float8_e4m3fn))
|
||||
layer.weight_scale.data.copy_(torch.tensor([0.25], dtype=torch.bfloat16))
|
||||
return layer
|
||||
|
||||
|
||||
def test_ple_fp8_embedding_dequantizes_in_ple_layer(monkeypatch) -> None:
|
||||
layer = _make_fp8_embedding_layer(monkeypatch)
|
||||
quantized_output = layer(torch.tensor([2, 0]))
|
||||
ple_layer = Qwen4ExpPLELayer.__new__(Qwen4ExpPLELayer)
|
||||
nn.Module.__init__(ple_layer)
|
||||
ple_layer.ple_embedding = nn.Module()
|
||||
ple_layer.ple_embedding.ngram_embedding = layer
|
||||
|
||||
output = ple_layer._dequantize_embeddings(
|
||||
quantized_output,
|
||||
torch.bfloat16,
|
||||
)
|
||||
|
||||
assert layer.weight.dtype == torch.float8_e4m3fn
|
||||
assert layer.weight_scale.dtype == torch.bfloat16
|
||||
assert quantized_output.dtype == torch.float8_e4m3fn
|
||||
assert output.dtype == torch.bfloat16
|
||||
weight = torch.tensor([[1.0, 2.0], [4.0, 8.0], [16.0, 32.0]])
|
||||
torch.testing.assert_close(output, (weight[[2, 0]] * 0.25).bfloat16())
|
||||
|
||||
|
||||
def test_ple_fp8_embedding_uses_int8_for_tp_reduce(monkeypatch) -> None:
|
||||
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
embedding_module, "get_tensor_model_parallel_world_size", lambda: 2
|
||||
)
|
||||
monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
parameter_module, "get_tensor_model_parallel_world_size", lambda: 2
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
embedding_module,
|
||||
"get_masked_input_and_mask",
|
||||
lambda *args: (
|
||||
torch.tensor([0, 0]),
|
||||
torch.tensor([False, True]),
|
||||
),
|
||||
)
|
||||
reduced_dtypes = []
|
||||
|
||||
def all_reduce(tensor: torch.Tensor) -> torch.Tensor:
|
||||
reduced_dtypes.append(tensor.dtype)
|
||||
return tensor.clone()
|
||||
|
||||
monkeypatch.setattr(
|
||||
embedding_module,
|
||||
"tensor_model_parallel_all_reduce",
|
||||
all_reduce,
|
||||
)
|
||||
layer = embedding_module.VocabParallelEmbedding(
|
||||
4,
|
||||
2,
|
||||
params_dtype=torch.bfloat16,
|
||||
padding_size=1,
|
||||
quant_method=Qwen4ExpPLEFp8EmbeddingMethod(),
|
||||
)
|
||||
layer.weight.data.copy_(
|
||||
torch.tensor([[1.0, 2.0], [4.0, 8.0]]).to(torch.float8_e4m3fn)
|
||||
)
|
||||
|
||||
output = layer(torch.tensor([0, 2]))
|
||||
|
||||
assert reduced_dtypes == [torch.int8]
|
||||
assert output.dtype == torch.float8_e4m3fn
|
||||
torch.testing.assert_close(output[0].float(), layer.weight[0].float())
|
||||
assert torch.count_nonzero(output[1].float()) == 0
|
||||
|
||||
|
||||
def test_ple_fp8_embedding_respects_checkpoint_shard_exclusions() -> None:
|
||||
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
|
||||
quant_config = Fp8Config(
|
||||
is_checkpoint_fp8_serialized=True,
|
||||
ignored_layers=[],
|
||||
weight_block_size=[128, 128],
|
||||
)
|
||||
assert isinstance(
|
||||
_get_ple_embedding_quant_method(quant_config, prefix),
|
||||
Qwen4ExpPLEFp8EmbeddingMethod,
|
||||
)
|
||||
|
||||
quant_config.ignored_layers = [f"{prefix}.shard_0"]
|
||||
assert _get_ple_embedding_quant_method(quant_config, prefix) is None
|
||||
|
||||
|
||||
def test_ple_ngram_ids_custom_op_uses_current_request_layout(monkeypatch) -> None:
|
||||
class RuntimeNGramEmbedding(nn.Module):
|
||||
def compute_ngram_ids(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
ngram_context: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
del input_ids, ngram_context
|
||||
num_reqs = query_start_loc.numel() - 1
|
||||
return torch.full((4, 2), num_reqs, dtype=torch.long)
|
||||
|
||||
layer = Qwen4ExpPLELayer.__new__(Qwen4ExpPLELayer)
|
||||
nn.Module.__init__(layer)
|
||||
layer.ple_embedding = RuntimeNGramEmbedding()
|
||||
monkeypatch.setattr(
|
||||
ple_layer_module,
|
||||
"get_forward_context",
|
||||
lambda: SimpleNamespace(no_compile_layers={"ple": layer}),
|
||||
)
|
||||
input_ids = torch.arange(4)
|
||||
ngram_context = torch.zeros(2, 2, dtype=torch.long)
|
||||
output = torch.empty(4, 2, dtype=torch.long)
|
||||
|
||||
ple_layer_module.qwen4_exp_compute_ple_ngram_ids(
|
||||
input_ids,
|
||||
torch.tensor([0, 4]),
|
||||
ngram_context,
|
||||
output,
|
||||
"ple",
|
||||
)
|
||||
assert torch.equal(output, torch.ones_like(output))
|
||||
|
||||
ple_layer_module.qwen4_exp_compute_ple_ngram_ids(
|
||||
input_ids,
|
||||
torch.tensor([0, 2, 4]),
|
||||
ngram_context,
|
||||
output,
|
||||
"ple",
|
||||
)
|
||||
assert torch.equal(output, torch.full_like(output, 2))
|
||||
|
||||
|
||||
def test_dilated_ple_spec_state_rolls_back_before_next_forward() -> None:
|
||||
module = Qwen4ExpPLELayer.__new__(Qwen4ExpPLELayer)
|
||||
nn.Module.__init__(module)
|
||||
module.conv_state_len = 6
|
||||
module.short_conv_dilation = 2
|
||||
|
||||
conv_weights = torch.tensor([[0.25, -0.5, 0.75, 1.0]])
|
||||
conv_state = torch.zeros(2, 1, 9)
|
||||
conv_state[1] = torch.arange(1, 10, dtype=torch.float32).reshape(1, 9)
|
||||
first_inputs = torch.tensor([[10.0], [20.0], [30.0], [40.0]])
|
||||
initial_state = conv_state[1:].clone()
|
||||
first_history = torch.cat(
|
||||
(initial_state[..., : module.conv_state_len], first_inputs.T.unsqueeze(0)),
|
||||
dim=-1,
|
||||
)
|
||||
|
||||
graph_padded_inputs = F.pad(first_inputs, (0, 0, 0, 4))
|
||||
first_output = module._short_conv_dilated_spec_batched(
|
||||
graph_padded_inputs,
|
||||
conv_state,
|
||||
conv_weights,
|
||||
torch.tensor([1, 0]),
|
||||
torch.tensor([0, 4, 4]),
|
||||
torch.tensor([1, 0]),
|
||||
spec_query_len=4,
|
||||
)
|
||||
|
||||
expected_first_output = F.silu(
|
||||
F.conv1d(
|
||||
first_history,
|
||||
conv_weights.unsqueeze(1),
|
||||
groups=1,
|
||||
dilation=module.short_conv_dilation,
|
||||
)
|
||||
).transpose(1, 2)[0]
|
||||
expected_first_state = first_history[..., 1:10]
|
||||
torch.testing.assert_close(first_output[:4], expected_first_output)
|
||||
assert torch.count_nonzero(first_output[4:]) == 0
|
||||
assert torch.count_nonzero(conv_state[0]) == 0
|
||||
torch.testing.assert_close(conv_state[1:], expected_first_state)
|
||||
|
||||
second_inputs = torch.tensor([[50.0], [60.0]])
|
||||
rollback_state = expected_first_state[..., 1:7]
|
||||
padded_second_inputs = F.pad(second_inputs.T.unsqueeze(0), (0, 2))
|
||||
second_history = torch.cat((rollback_state, padded_second_inputs), dim=-1)
|
||||
expected_second_state = expected_first_state.clone()
|
||||
expected_second_state[..., :7] = second_history[..., 1:8]
|
||||
|
||||
second_output = module._short_conv_dilated_spec_batched(
|
||||
second_inputs,
|
||||
conv_state,
|
||||
conv_weights,
|
||||
torch.tensor([1]),
|
||||
torch.tensor([0, 2]),
|
||||
torch.tensor([2]),
|
||||
spec_query_len=4,
|
||||
)
|
||||
|
||||
expected_second_output = F.silu(
|
||||
F.conv1d(
|
||||
second_history,
|
||||
conv_weights.unsqueeze(1),
|
||||
groups=1,
|
||||
dilation=module.short_conv_dilation,
|
||||
)
|
||||
).transpose(1, 2)[0, :2]
|
||||
torch.testing.assert_close(second_output, expected_second_output)
|
||||
torch.testing.assert_close(conv_state[1:], expected_second_state)
|
||||
|
||||
|
||||
def test_ple_short_conv_uses_fallback_when_profile_metadata_is_omitted(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
module = Qwen4ExpPLELayer.__new__(Qwen4ExpPLELayer)
|
||||
nn.Module.__init__(module)
|
||||
module.prefix = "model.layers.1.ple"
|
||||
inputs = torch.arange(6, dtype=torch.float32).reshape(3, 2)
|
||||
expected = inputs + 1
|
||||
monkeypatch.setattr(module, "_short_conv_fallback", lambda _: expected)
|
||||
monkeypatch.setattr(
|
||||
ple_layer_module,
|
||||
"get_forward_context",
|
||||
lambda: SimpleNamespace(attn_metadata={"model.layers.0.self_attn": object()}),
|
||||
)
|
||||
|
||||
output = module._short_conv(inputs)
|
||||
|
||||
assert output is expected
|
||||
@@ -0,0 +1,297 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
|
||||
from vllm.models.qwen4_exp.amd import (
|
||||
model as _qwen4_exp_model, # noqa: F401
|
||||
)
|
||||
from vllm.models.qwen4_exp.amd import ple_layer as ple_layer_module
|
||||
from vllm.models.qwen4_exp.amd.indexer_qsa import (
|
||||
apply_qsa_rmsnorm,
|
||||
apply_qsa_rope,
|
||||
)
|
||||
from vllm.models.qwen4_exp.amd.ops import qsa as qsa_ops
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="AMD QSA requires ROCm",
|
||||
)
|
||||
|
||||
requires_qsa_kernels = pytest.mark.skipif(
|
||||
not HAS_TRITON,
|
||||
reason="AMD QSA kernels require Triton",
|
||||
)
|
||||
|
||||
|
||||
def test_ple_ngram_embedding_custom_op_uses_resident_weight(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
layer_name = "model.layers.0.ple"
|
||||
layer = ple_layer_module.Qwen4ExpPLELayer.__new__(ple_layer_module.Qwen4ExpPLELayer)
|
||||
torch.nn.Module.__init__(layer)
|
||||
layer.ple_embedding = torch.nn.Module()
|
||||
layer.ple_embedding.ngram_embedding = torch.nn.Embedding(8, 3)
|
||||
context = SimpleNamespace(no_compile_layers={layer_name: layer})
|
||||
monkeypatch.setattr(ple_layer_module, "get_forward_context", lambda: context)
|
||||
|
||||
ngram_ids = torch.tensor([[0, 1], [2, 3]])
|
||||
output = torch.empty(2, 6)
|
||||
ple_layer_module.qwen4_exp_amd_ple_ngram_embedding(
|
||||
ngram_ids,
|
||||
output,
|
||||
layer_name,
|
||||
)
|
||||
|
||||
expected = layer.ple_embedding.ngram_embedding(ngram_ids).flatten(-2)
|
||||
torch.testing.assert_close(output, expected)
|
||||
|
||||
|
||||
def _qsa_sparse_paged_attention_reference(
|
||||
q: torch.Tensor,
|
||||
k_cache: torch.Tensor,
|
||||
v_cache: torch.Tensor,
|
||||
logical_indices: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
token_to_req: torch.Tensor,
|
||||
softmax_scale: float,
|
||||
) -> torch.Tensor:
|
||||
output = torch.zeros_like(q)
|
||||
repeats = q.shape[1] // k_cache.shape[2]
|
||||
page_size = k_cache.shape[1]
|
||||
for row in range(q.shape[0]):
|
||||
logical = logical_indices[row]
|
||||
logical = logical[logical >= 0].long()
|
||||
if not logical.numel():
|
||||
continue
|
||||
request = token_to_req[row].long()
|
||||
pages = block_table[request, logical // page_size].long()
|
||||
offsets = logical % page_size
|
||||
keys = k_cache[pages, offsets].repeat_interleave(repeats, dim=1)
|
||||
values = v_cache[pages, offsets].repeat_interleave(repeats, dim=1)
|
||||
scores = torch.einsum("hd,khd->hk", q[row].float(), keys.float())
|
||||
probabilities = torch.softmax(scores * softmax_scale, dim=-1)
|
||||
output[row] = torch.einsum("hk,khd->hd", probabilities, values.float()).to(
|
||||
q.dtype
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
def test_qsa_rope_uses_platform_dispatch() -> None:
|
||||
tensor = torch.arange(16, dtype=torch.float32).reshape(2, 2, 4)
|
||||
positions = torch.tensor([0, 1])
|
||||
calls = []
|
||||
|
||||
def apply_rotary_emb(
|
||||
rotary_input: torch.Tensor,
|
||||
cos: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
calls.append((rotary_input, cos, sin))
|
||||
return rotary_input + 1
|
||||
|
||||
rotary_emb = SimpleNamespace(
|
||||
rotary_dim=2,
|
||||
apply_rotary_emb=apply_rotary_emb,
|
||||
_match_cos_sin_cache_dtype=lambda _: torch.zeros(2, 4),
|
||||
)
|
||||
|
||||
output = apply_qsa_rope(rotary_emb, positions, tensor)
|
||||
|
||||
assert len(calls) == 1
|
||||
torch.testing.assert_close(output[..., :2], tensor[..., :2] + 1)
|
||||
torch.testing.assert_close(output[..., 2:], tensor[..., 2:])
|
||||
|
||||
|
||||
def test_qsa_rmsnorm_uses_portable_implementation(default_vllm_config) -> None:
|
||||
norm = GemmaRMSNorm(4, eps=1e-6)
|
||||
norm.weight.data.copy_(torch.tensor([0.1, -0.2, 0.3, -0.4]))
|
||||
tensor = torch.arange(8, dtype=torch.float32).reshape(2, 4)
|
||||
output = apply_qsa_rmsnorm(norm, tensor)
|
||||
|
||||
torch.testing.assert_close(output, norm.forward_native(tensor))
|
||||
|
||||
|
||||
def test_qsa_selection_uses_portable_topk_on_rocm(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
rows = 2
|
||||
token_topk = 8
|
||||
compress_ratio = 4
|
||||
block_topk = token_topk // compress_ratio
|
||||
blocks = torch.empty((rows, block_topk), dtype=torch.int32)
|
||||
visible_blocks = torch.tensor([4, 6], dtype=torch.int32)
|
||||
logits = torch.empty((rows, 8), dtype=torch.float32)
|
||||
selection_output = torch.empty(
|
||||
(rows, token_topk + compress_ratio - 1), dtype=torch.int32
|
||||
)
|
||||
topk_call: dict[str, Any] = {}
|
||||
|
||||
monkeypatch.setattr(
|
||||
qsa_ops,
|
||||
"qsa_mqa_paged",
|
||||
lambda *args, **kwargs: (logits, visible_blocks),
|
||||
)
|
||||
monkeypatch.setattr(qsa_ops.current_platform, "is_cuda", lambda: False)
|
||||
|
||||
def top_k_per_row_decode(
|
||||
topk_logits,
|
||||
next_n,
|
||||
seq_lens,
|
||||
raw_topk_indices,
|
||||
num_rows,
|
||||
stride0,
|
||||
stride1,
|
||||
topk_tokens,
|
||||
) -> None:
|
||||
topk_call.update(
|
||||
logits=topk_logits,
|
||||
next_n=next_n,
|
||||
seq_lens=seq_lens,
|
||||
raw_topk_indices=raw_topk_indices,
|
||||
num_rows=num_rows,
|
||||
strides=(stride0, stride1),
|
||||
topk_tokens=topk_tokens,
|
||||
)
|
||||
raw_topk_indices.zero_()
|
||||
|
||||
def expand_qsa_block_indices(*args) -> None:
|
||||
args[-1].fill_(-1)
|
||||
|
||||
monkeypatch.setattr(qsa_ops.ops, "top_k_per_row_decode", top_k_per_row_decode)
|
||||
monkeypatch.setattr(
|
||||
qsa_ops,
|
||||
"expand_qsa_block_indices_cuda",
|
||||
expand_qsa_block_indices,
|
||||
)
|
||||
|
||||
output = qsa_ops.qsa_select_paged_tokens(
|
||||
torch.empty((rows, 1, 1)),
|
||||
torch.empty((1, 4, 1, 1)),
|
||||
torch.empty((1, 2), dtype=torch.int32),
|
||||
torch.zeros(rows, dtype=torch.int32),
|
||||
torch.arange(rows, dtype=torch.int32),
|
||||
torch.tensor([8], dtype=torch.int32),
|
||||
token_topk,
|
||||
compress_ratio,
|
||||
selection_output,
|
||||
)
|
||||
|
||||
assert output is selection_output
|
||||
assert torch.all(output == -1)
|
||||
assert topk_call["logits"] is logits
|
||||
assert topk_call["next_n"] == 1
|
||||
assert topk_call["seq_lens"] is visible_blocks
|
||||
assert topk_call["raw_topk_indices"].shape == blocks.shape
|
||||
assert topk_call["num_rows"] == rows
|
||||
assert topk_call["strides"] == (logits.stride(0), logits.stride(1))
|
||||
assert topk_call["topk_tokens"] == block_topk
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
@pytest.mark.parametrize(
|
||||
("num_rows", "num_query_heads", "num_kv_heads", "page_size"),
|
||||
[
|
||||
pytest.param(1, 24, 2, 1792, id="tp1_split64"),
|
||||
pytest.param(16, 12, 1, 1792, id="tp2_split32"),
|
||||
pytest.param(32, 6, 1, 1024, id="tp4_split8"),
|
||||
pytest.param(257, 6, 1, 1024, id="tp4_split4"),
|
||||
pytest.param(513, 6, 1, 1024, id="tp4_split1"),
|
||||
],
|
||||
)
|
||||
def test_qsa_sparse_paged_attention_matches_reference(
|
||||
num_rows: int,
|
||||
num_query_heads: int,
|
||||
num_kv_heads: int,
|
||||
page_size: int,
|
||||
) -> None:
|
||||
torch.manual_seed(2)
|
||||
head_dim = 256
|
||||
num_requests = 2
|
||||
num_selected_pages = 64
|
||||
num_pages_per_request = num_selected_pages + 1
|
||||
num_cache_blocks = num_requests * num_pages_per_request
|
||||
indexer_budget = 2048
|
||||
indexer_compress_ratio = 4
|
||||
selection_width = indexer_budget + indexer_compress_ratio - 1
|
||||
q = torch.randn(
|
||||
num_rows, num_query_heads, head_dim, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
kv_cache = torch.randn(
|
||||
num_cache_blocks,
|
||||
page_size,
|
||||
num_kv_heads,
|
||||
2 * head_dim,
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
k_cache, v_cache = kv_cache.split(head_dim, dim=-1)
|
||||
block_table = (
|
||||
torch.randperm(num_cache_blocks, device="cuda")
|
||||
.reshape(num_requests, num_pages_per_request)
|
||||
.to(torch.int32)
|
||||
)
|
||||
rows_per_request = math.ceil(num_rows / num_requests)
|
||||
row_indices = torch.arange(num_rows, device="cuda", dtype=torch.int32)
|
||||
token_to_req = row_indices // rows_per_request
|
||||
request_row_counts = torch.tensor(
|
||||
[rows_per_request, num_rows - rows_per_request],
|
||||
device="cuda",
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
context_length = num_pages_per_request * page_size - 1
|
||||
block_topk = indexer_budget // indexer_compress_ratio
|
||||
compressed_blocks_per_page = page_size // indexer_compress_ratio
|
||||
selection = torch.arange(block_topk, device="cuda")
|
||||
selected_pages = selection % num_selected_pages
|
||||
selected_offsets = selection // num_selected_pages
|
||||
row_shifts = 2 * row_indices.unsqueeze(1)
|
||||
selected_offsets = (selected_offsets + row_shifts) % compressed_blocks_per_page
|
||||
block_indices = (selected_pages * compressed_blocks_per_page + selected_offsets).to(
|
||||
torch.int32
|
||||
)
|
||||
rows_within_request = row_indices % rows_per_request
|
||||
query_positions = (
|
||||
context_length - request_row_counts[token_to_req.long()] + rows_within_request
|
||||
).to(torch.int64)
|
||||
sequence_lengths = torch.full(
|
||||
(num_requests,), context_length, device="cuda", dtype=torch.int32
|
||||
)
|
||||
logical_indices = qsa_ops.expand_qsa_block_indices_cuda(
|
||||
block_indices,
|
||||
query_positions,
|
||||
sequence_lengths,
|
||||
token_to_req,
|
||||
indexer_compress_ratio,
|
||||
indexer_budget,
|
||||
)
|
||||
assert logical_indices.shape == (num_rows, selection_width)
|
||||
|
||||
actual = qsa_ops.qsa_sparse_paged_attention(
|
||||
q,
|
||||
k_cache,
|
||||
v_cache,
|
||||
logical_indices,
|
||||
block_table,
|
||||
token_to_req,
|
||||
)
|
||||
expected = _qsa_sparse_paged_attention_reference(
|
||||
q,
|
||||
k_cache,
|
||||
v_cache,
|
||||
logical_indices,
|
||||
block_table,
|
||||
token_to_req,
|
||||
q.shape[-1] ** -0.5,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, rtol=2e-2, atol=2e-2)
|
||||
@@ -0,0 +1,297 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Correctness tests for the fused QSA pre-indexer."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.qwen4_exp.common.qsa_cache import (
|
||||
canonical_qsa_rope_positions,
|
||||
circular_qsa_slot_mapping,
|
||||
compressed_qsa_slot_mapping,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia.indexer_qsa import apply_qsa_rope
|
||||
from vllm.models.qwen4_exp.nvidia.ops.qsa import (
|
||||
qsa_compress_groups_with_ratio,
|
||||
qsa_store_cache_rows,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia.ops.qsa_pre_indexer import (
|
||||
qsa_pre_indexer,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
|
||||
requires_qsa_kernels = pytest.mark.skipif(
|
||||
not current_platform.is_cuda() or not HAS_TRITON,
|
||||
reason="QSA kernels require CUDA and Triton",
|
||||
)
|
||||
|
||||
HQ, D = 4, 128
|
||||
CR = 4
|
||||
MROPE_SECTION = (11, 11, 10)
|
||||
EPS = 1e-6
|
||||
BLOCK_SIZE = 16
|
||||
COMP_PAGE = BLOCK_SIZE // CR
|
||||
ROPE_POS_OFFSET = D
|
||||
RTOL = 1.6e-2
|
||||
ATOL = 1e-2
|
||||
MIXED_BATCH = ([260, 259, 138], [1, 1, 37], [8, 8, 8])
|
||||
|
||||
|
||||
def _make_block_table(block_counts):
|
||||
num_blocks = sum(block_counts)
|
||||
table = torch.full((len(block_counts), max(block_counts)), -1, dtype=torch.int32)
|
||||
physical_blocks = torch.randperm(num_blocks)
|
||||
offset = 0
|
||||
for request, count in enumerate(block_counts):
|
||||
table[request, :count] = physical_blocks[offset : offset + count]
|
||||
offset += count
|
||||
return table, num_blocks
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
@pytest.mark.parametrize(
|
||||
"mrope,is_2d_positions,cache_rope_positions,state_size,seq_lens,query_lens,history_lens",
|
||||
[
|
||||
pytest.param(True, True, True, 4, *MIXED_BATCH, id="mrope"),
|
||||
pytest.param(False, False, False, 4, *MIXED_BATCH, id="text"),
|
||||
pytest.param(True, False, True, 8, *MIXED_BATCH, id="mrope-model-1d"),
|
||||
pytest.param(True, True, False, 4, *MIXED_BATCH, id="mrope-no-position-cache"),
|
||||
pytest.param(
|
||||
True,
|
||||
False,
|
||||
False,
|
||||
8,
|
||||
*MIXED_BATCH,
|
||||
id="mrope-model-1d-no-position-cache",
|
||||
),
|
||||
pytest.param(
|
||||
False, False, True, 4, *MIXED_BATCH, id="text-with-position-cache"
|
||||
),
|
||||
pytest.param(True, True, True, 4, [37], [37], [0], id="fresh"),
|
||||
pytest.param(True, True, True, 4, [4097], [4097], [0], id="tiled"),
|
||||
],
|
||||
)
|
||||
def test_qsa_fused_pre_indexer_matches_unfused(
|
||||
mrope,
|
||||
is_2d_positions,
|
||||
cache_rope_positions,
|
||||
state_size,
|
||||
seq_lens,
|
||||
query_lens,
|
||||
history_lens,
|
||||
) -> None:
|
||||
from flashinfer.norm import gemma_rmsnorm
|
||||
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
|
||||
device = "cuda"
|
||||
rope_params = {
|
||||
"partial_rotary_factor": 0.25,
|
||||
"rope_theta": 10000000,
|
||||
"rope_type": "default",
|
||||
}
|
||||
if mrope:
|
||||
rope_params["mrope_interleaved"] = True
|
||||
rope_params["mrope_section"] = list(MROPE_SECTION)
|
||||
with torch.device(device):
|
||||
rope = get_rope(
|
||||
head_size=256,
|
||||
max_position=32768,
|
||||
rope_parameters=rope_params,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
token_to_req = torch.cat(
|
||||
[
|
||||
torch.full((length,), request, dtype=torch.int32)
|
||||
for request, length in enumerate(query_lens)
|
||||
]
|
||||
).to(device)
|
||||
logical_positions = torch.cat(
|
||||
[
|
||||
torch.arange(seq_len - query_len, seq_len, dtype=torch.int64)
|
||||
for seq_len, query_len in zip(seq_lens, query_lens)
|
||||
]
|
||||
).to(device)
|
||||
num_tokens = logical_positions.numel()
|
||||
query_start_loc = torch.tensor(
|
||||
[0, *torch.tensor(query_lens).cumsum(0).tolist()], dtype=torch.int32
|
||||
).to(device)
|
||||
positions = (
|
||||
torch.stack(
|
||||
[
|
||||
logical_positions,
|
||||
logical_positions // 7 + 3,
|
||||
logical_positions // 13 + 11,
|
||||
]
|
||||
)
|
||||
if is_2d_positions
|
||||
else logical_positions
|
||||
)
|
||||
position_rows = (
|
||||
canonical_qsa_rope_positions(positions)
|
||||
if cache_rope_positions
|
||||
else logical_positions.view(-1, 1, 1).expand(-1, 1, 3)
|
||||
)
|
||||
|
||||
compressed_block_counts = [
|
||||
(seq_len // CR + COMP_PAGE - 1) // COMP_PAGE for seq_len in seq_lens
|
||||
]
|
||||
raw_block_table, num_raw_blocks = _make_block_table([1] * len(seq_lens))
|
||||
compressed_block_table, num_compressed_blocks = _make_block_table(
|
||||
compressed_block_counts
|
||||
)
|
||||
raw_block_table = raw_block_table.to(device)
|
||||
raw_slots = circular_qsa_slot_mapping(
|
||||
raw_block_table,
|
||||
token_to_req,
|
||||
logical_positions,
|
||||
state_size,
|
||||
query_start_loc,
|
||||
)
|
||||
compressed_slots = compressed_qsa_slot_mapping(
|
||||
compressed_block_table.to(device),
|
||||
token_to_req,
|
||||
logical_positions,
|
||||
COMP_PAGE,
|
||||
CR,
|
||||
)
|
||||
group_counts = torch.tensor(
|
||||
[
|
||||
seq_len // CR - (seq_len - query_len) // CR
|
||||
for seq_len, query_len in zip(seq_lens, query_lens)
|
||||
],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
k_work_counts = torch.maximum(group_counts, torch.ones_like(group_counts))
|
||||
k_start_loc = torch.cat([k_work_counts.new_zeros(1), k_work_counts.cumsum(0)])
|
||||
work_requests = torch.repeat_interleave(
|
||||
torch.arange(len(query_lens), dtype=torch.int32, device=device),
|
||||
k_work_counts,
|
||||
)
|
||||
local_work = torch.arange(
|
||||
int(k_start_loc[-1]), dtype=torch.int32, device=device
|
||||
) - torch.repeat_interleave(k_start_loc[:-1], k_work_counts)
|
||||
max_k_work = (num_tokens + (CR - 1) * len(query_lens)) // CR
|
||||
k_work_metadata = torch.full((max_k_work, 2), -1, dtype=torch.int32, device=device)
|
||||
k_work_metadata[: work_requests.numel()] = torch.stack(
|
||||
(work_requests, local_work), dim=1
|
||||
)
|
||||
|
||||
raw_width = D + 12 if cache_rope_positions else D
|
||||
# Match vLLM's padded-page cache layout: rows are contiguous, while physical
|
||||
# blocks have a larger stride than their logical contents.
|
||||
raw_page_elements = state_size * raw_width
|
||||
fused_raw_storage = torch.zeros(
|
||||
num_raw_blocks,
|
||||
raw_page_elements + 16,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
fused_raw = torch.as_strided(
|
||||
fused_raw_storage,
|
||||
(num_raw_blocks, state_size, 1, raw_width),
|
||||
(raw_page_elements + 16, raw_width, raw_width, 1),
|
||||
)
|
||||
for request, history_len in enumerate(history_lens):
|
||||
history_end = seq_lens[request] - query_lens[request]
|
||||
for position in range(history_end - history_len, history_end):
|
||||
block = int(raw_block_table[request, 0])
|
||||
row = fused_raw[block, position % state_size, 0]
|
||||
row[:D] = torch.randn(D, dtype=torch.bfloat16, device=device)
|
||||
if cache_rope_positions:
|
||||
row[ROPE_POS_OFFSET:].view(torch.int64).copy_(
|
||||
torch.tensor(
|
||||
[position, position // 7 + 3, position // 13 + 11],
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
)
|
||||
)
|
||||
unfused_raw = fused_raw.clone()
|
||||
compressed_page_elements = COMP_PAGE * D
|
||||
fused_compressed_storage = torch.zeros(
|
||||
num_compressed_blocks,
|
||||
compressed_page_elements + 16,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
fused_compressed = torch.as_strided(
|
||||
fused_compressed_storage,
|
||||
(num_compressed_blocks, COMP_PAGE, 1, D),
|
||||
(compressed_page_elements + 16, D, D, 1),
|
||||
)
|
||||
unfused_compressed = fused_compressed.clone()
|
||||
|
||||
projected_qk = torch.randn(
|
||||
num_tokens, (HQ + 1) * D, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
q_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
|
||||
k_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
|
||||
|
||||
fused_query = torch.empty(num_tokens, HQ, D, dtype=torch.bfloat16, device=device)
|
||||
qsa_pre_indexer(
|
||||
projected_qk[:, : HQ * D],
|
||||
projected_qk[:, HQ * D :],
|
||||
positions,
|
||||
rope.cos_sin_cache,
|
||||
q_weight,
|
||||
k_weight,
|
||||
EPS,
|
||||
fused_query,
|
||||
fused_raw,
|
||||
raw_slots,
|
||||
raw_block_table,
|
||||
query_start_loc,
|
||||
logical_positions,
|
||||
fused_compressed,
|
||||
compressed_slots,
|
||||
k_work_metadata,
|
||||
compress_ratio=CR,
|
||||
mrope_section=MROPE_SECTION if mrope else None,
|
||||
rope_pos_offset=ROPE_POS_OFFSET if cache_rope_positions else None,
|
||||
)
|
||||
|
||||
unfused_query = projected_qk[:, : HQ * D].reshape(num_tokens, HQ, D)
|
||||
unfused_query = gemma_rmsnorm(
|
||||
unfused_query.reshape(-1, D), q_weight, EPS
|
||||
).reshape_as(unfused_query)
|
||||
unfused_query = apply_qsa_rope(rope, positions, unfused_query)
|
||||
|
||||
raw_keys = unfused_raw[..., :D]
|
||||
rope_positions = (
|
||||
unfused_raw[..., ROPE_POS_OFFSET:].view(torch.int64)
|
||||
if cache_rope_positions
|
||||
else None
|
||||
)
|
||||
pooled, first_positions = qsa_compress_groups_with_ratio(
|
||||
projected_qk[:, HQ * D :].reshape(-1, 1, D),
|
||||
position_rows,
|
||||
raw_keys,
|
||||
raw_block_table,
|
||||
token_to_req,
|
||||
query_start_loc,
|
||||
logical_positions,
|
||||
compressed_slots,
|
||||
CR,
|
||||
rope_positions,
|
||||
)
|
||||
compressed_rows = gemma_rmsnorm(pooled.reshape(-1, D), k_weight, EPS).reshape(
|
||||
-1, 1, D
|
||||
)
|
||||
group_positions = (
|
||||
first_positions.transpose(0, 1) if mrope else first_positions[:, 0]
|
||||
)
|
||||
compressed_rows = apply_qsa_rope(rope, group_positions, compressed_rows)
|
||||
qsa_store_cache_rows(unfused_compressed, compressed_slots, compressed_rows)
|
||||
qsa_store_cache_rows(raw_keys, raw_slots, projected_qk[:, HQ * D :])
|
||||
if rope_positions is not None:
|
||||
qsa_store_cache_rows(rope_positions, raw_slots, position_rows)
|
||||
|
||||
torch.testing.assert_close(fused_query, unfused_query, rtol=RTOL, atol=ATOL)
|
||||
assert torch.equal(fused_raw.view(torch.int16), unfused_raw.view(torch.int16))
|
||||
torch.testing.assert_close(
|
||||
fused_compressed, unfused_compressed, rtol=RTOL, atol=ATOL
|
||||
)
|
||||
@@ -0,0 +1,973 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.qwen4_exp.common import qsa_cache
|
||||
from vllm.models.qwen4_exp.common.qsa_cache import QSAMetadataBuilder
|
||||
from vllm.models.qwen4_exp.nvidia import indexer_qsa
|
||||
from vllm.models.qwen4_exp.nvidia import (
|
||||
model as _qwen4_exp_model, # noqa: F401
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia.ops import qsa as qsa_ops
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
|
||||
requires_qsa_kernels = pytest.mark.skipif(
|
||||
not current_platform.is_cuda() or not HAS_TRITON,
|
||||
reason="QSA kernels require CUDA and Triton",
|
||||
)
|
||||
|
||||
|
||||
def test_qsa_mtp_index_share_updates_cache_but_skips_selection(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
rows = torch.tensor([[3, 1, -1], [5, 2, 0]], dtype=torch.int32)
|
||||
raw_metadata = SimpleNamespace(
|
||||
num_actual_tokens=2,
|
||||
slot_mapping=torch.arange(2),
|
||||
block_table=torch.empty(0),
|
||||
query_start_loc=torch.arange(3),
|
||||
logical_positions=torch.arange(2),
|
||||
)
|
||||
compressed_metadata = SimpleNamespace(
|
||||
num_actual_tokens=2,
|
||||
slot_mapping=torch.arange(2),
|
||||
k_work_metadata=torch.empty(0),
|
||||
)
|
||||
updates = []
|
||||
selections = []
|
||||
indexer = SimpleNamespace(
|
||||
skip_topk=True,
|
||||
_metadata=lambda: (raw_metadata, compressed_metadata),
|
||||
index_qk_proj=lambda hidden: (torch.zeros(2, 2), None),
|
||||
index_n_heads=1,
|
||||
index_kv_heads=1,
|
||||
index_head_dim=1,
|
||||
raw_key_cache=SimpleNamespace(
|
||||
kv_cache=torch.empty(0),
|
||||
rope_position_cache=None,
|
||||
rope_position_offset=0,
|
||||
),
|
||||
compressed_key_cache=SimpleNamespace(kv_cache=torch.empty(0)),
|
||||
use_fused_pre_indexer=True,
|
||||
rotary_emb=SimpleNamespace(cos_sin_cache=torch.empty(0)),
|
||||
q_layernorm=SimpleNamespace(weight=torch.ones(1), variance_epsilon=1e-6),
|
||||
k_layernorm=SimpleNamespace(weight=torch.ones(1)),
|
||||
compress_ratio=2,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
indexer_qsa,
|
||||
"qsa_pre_indexer",
|
||||
lambda *args, **kwargs: updates.append((args, kwargs)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
qsa_ops,
|
||||
"qsa_select_paged_tokens",
|
||||
lambda *args, **kwargs: selections.append((args, kwargs)),
|
||||
)
|
||||
|
||||
actual = indexer_qsa.QSAIndexer.forward(
|
||||
indexer,
|
||||
torch.zeros(2, 4),
|
||||
torch.tensor([7, 8]),
|
||||
rows,
|
||||
)
|
||||
|
||||
assert actual is rows
|
||||
assert len(updates) == 1
|
||||
assert not selections
|
||||
|
||||
|
||||
def _qsa_mqa_paged_reference(
|
||||
q: torch.Tensor,
|
||||
k_cache: torch.Tensor,
|
||||
page_table: torch.Tensor,
|
||||
token_to_req: torch.Tensor,
|
||||
visible_lengths: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
pages = page_table.index_select(0, token_to_req.long()).long()
|
||||
keys = k_cache[pages, :, 0, :].flatten(1, 2)
|
||||
scores = torch.einsum("rhd,rnd->rnh", q.float(), keys.float())
|
||||
logits = torch.relu(scores).sum(dim=-1) / math.sqrt(q.shape[-1])
|
||||
positions = torch.arange(keys.shape[1], device=q.device).unsqueeze(0)
|
||||
return logits.masked_fill(positions >= visible_lengths.unsqueeze(1), -torch.inf)
|
||||
|
||||
|
||||
def _qsa_relative_topk_reference(
|
||||
logits: torch.Tensor,
|
||||
row_starts: torch.Tensor,
|
||||
row_ends: torch.Tensor,
|
||||
topk: int,
|
||||
) -> torch.Tensor:
|
||||
output = torch.full(
|
||||
(logits.shape[0], topk), -1, dtype=torch.int32, device=logits.device
|
||||
)
|
||||
for row in range(logits.shape[0]):
|
||||
start = int(row_starts[row].item())
|
||||
length = int((row_ends[row] - row_starts[row]).item())
|
||||
width = min(length, topk)
|
||||
if width:
|
||||
output[row, :width] = torch.topk(
|
||||
logits[row, start : start + length], width
|
||||
).indices.to(torch.int32)
|
||||
return output
|
||||
|
||||
|
||||
def _expand_qsa_indices_reference(
|
||||
block_indices: torch.Tensor,
|
||||
query_positions: torch.Tensor,
|
||||
sequence_lengths: torch.Tensor,
|
||||
compress_ratio: int,
|
||||
token_topk: int,
|
||||
) -> torch.Tensor:
|
||||
rows = block_indices.shape[0]
|
||||
block_topk = token_topk // compress_ratio
|
||||
output_width = token_topk + compress_ratio - 1
|
||||
offsets = torch.arange(compress_ratio, device=block_indices.device)
|
||||
blocks = block_indices.long()
|
||||
expanded = blocks.unsqueeze(-1) * compress_ratio + offsets
|
||||
expanded = torch.where(
|
||||
blocks.unsqueeze(-1) >= 0, expanded, torch.full_like(expanded, -1)
|
||||
).reshape(rows, block_topk * compress_ratio)
|
||||
expanded = expanded[:, :token_topk]
|
||||
expanded = torch.where(
|
||||
(expanded >= 0) & (expanded < sequence_lengths.unsqueeze(1)),
|
||||
expanded,
|
||||
torch.full_like(expanded, -1),
|
||||
)
|
||||
|
||||
tail_offsets = torch.arange(compress_ratio - 1, device=block_indices.device)
|
||||
visible_tokens = query_positions + 1
|
||||
tail_start = visible_tokens // compress_ratio * compress_ratio
|
||||
tail = tail_start.unsqueeze(1) + tail_offsets.unsqueeze(0)
|
||||
tail_count = (visible_tokens - tail_start).unsqueeze(1)
|
||||
tail_valid = (tail_offsets.unsqueeze(0) < tail_count) & (
|
||||
tail < sequence_lengths.unsqueeze(1)
|
||||
)
|
||||
tail = torch.where(tail_valid, tail, torch.full_like(tail, -1))
|
||||
|
||||
result = torch.cat((expanded, tail), dim=1)
|
||||
order = torch.arange(output_width, device=result.device).expand(rows, -1)
|
||||
sort_key = torch.where(result >= 0, order, order + output_width)
|
||||
return result.gather(1, torch.argsort(sort_key, dim=1, stable=True)).to(torch.int32)
|
||||
|
||||
|
||||
def _qsa_select_paged_tokens_reference(
|
||||
q: torch.Tensor,
|
||||
k_cache: torch.Tensor,
|
||||
page_table: torch.Tensor,
|
||||
token_to_req: torch.Tensor,
|
||||
query_positions: torch.Tensor,
|
||||
sequence_lengths: torch.Tensor,
|
||||
token_topk: int,
|
||||
compress_ratio: int,
|
||||
) -> torch.Tensor:
|
||||
row_sequence_lengths = sequence_lengths.index_select(0, token_to_req.long())
|
||||
visible_blocks = torch.minimum(
|
||||
(query_positions + 1) // compress_ratio,
|
||||
row_sequence_lengths // compress_ratio,
|
||||
).to(torch.int32)
|
||||
logits = _qsa_mqa_paged_reference(
|
||||
q,
|
||||
k_cache,
|
||||
page_table,
|
||||
token_to_req,
|
||||
visible_blocks,
|
||||
)
|
||||
starts = torch.zeros_like(visible_blocks)
|
||||
blocks = _qsa_relative_topk_reference(
|
||||
logits,
|
||||
starts,
|
||||
visible_blocks,
|
||||
token_topk // compress_ratio,
|
||||
)
|
||||
return _expand_qsa_indices_reference(
|
||||
blocks,
|
||||
query_positions,
|
||||
row_sequence_lengths,
|
||||
compress_ratio,
|
||||
token_topk,
|
||||
)
|
||||
|
||||
|
||||
def _qsa_sparse_paged_attention_reference(
|
||||
q: torch.Tensor,
|
||||
k_cache: torch.Tensor,
|
||||
v_cache: torch.Tensor,
|
||||
logical_indices: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
token_to_req: torch.Tensor,
|
||||
softmax_scale: float,
|
||||
) -> torch.Tensor:
|
||||
output = torch.zeros_like(q)
|
||||
repeats = q.shape[1] // k_cache.shape[2]
|
||||
page_size = k_cache.shape[1]
|
||||
for row in range(q.shape[0]):
|
||||
logical = logical_indices[row]
|
||||
logical = logical[logical >= 0].long()
|
||||
if not logical.numel():
|
||||
continue
|
||||
request = token_to_req[row].long()
|
||||
pages = block_table[request, logical // page_size].long()
|
||||
offsets = logical % page_size
|
||||
keys = k_cache[pages, offsets].repeat_interleave(repeats, dim=1)
|
||||
values = v_cache[pages, offsets].repeat_interleave(repeats, dim=1)
|
||||
scores = torch.einsum("hd,khd->hk", q[row].float(), keys.float())
|
||||
probabilities = torch.softmax(scores * softmax_scale, dim=-1)
|
||||
output[row] = torch.einsum("hk,khd->hd", probabilities, values.float()).to(
|
||||
q.dtype
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
def test_qsa_side_metadata_marks_cudagraph_padding_inert() -> None:
|
||||
device = torch.device("cuda")
|
||||
builder = QSAMetadataBuilder.__new__(QSAMetadataBuilder)
|
||||
builder.compress_ratio = 1
|
||||
builder.is_circular_buffer = False
|
||||
builder.storage_block_size = 64
|
||||
builder.token_to_req_buffer = torch.empty(16, dtype=torch.int32, device=device)
|
||||
builder.slot_mapping_buffer = torch.empty(16, dtype=torch.int64, device=device)
|
||||
builder.logical_positions_buffer = torch.empty(16, dtype=torch.int64, device=device)
|
||||
builder.k_work_metadata_buffer = torch.empty(0, 2, dtype=torch.int32, device=device)
|
||||
query_start_loc = torch.tensor([0, 4, 8, 12, 12], dtype=torch.int32, device=device)
|
||||
token_to_req = torch.tensor([0] * 4 + [1] * 4 + [2] * 4 + [0] * 4, device=device)
|
||||
common = SimpleNamespace(
|
||||
num_actual_tokens=16,
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc.cpu(),
|
||||
seq_lens=torch.tensor([68, 68, 68, 0], dtype=torch.int32, device=device),
|
||||
slot_mapping=torch.tensor(list(range(12)) + [-1] * 4, device=device),
|
||||
block_table_tensor=torch.empty((4, 0), dtype=torch.int32, device=device),
|
||||
token_to_req_indices=lambda buffer: buffer.copy_(token_to_req),
|
||||
)
|
||||
|
||||
metadata = builder.build(0, common)
|
||||
|
||||
assert metadata.logical_positions.tolist() == [
|
||||
64,
|
||||
65,
|
||||
66,
|
||||
67,
|
||||
64,
|
||||
65,
|
||||
66,
|
||||
67,
|
||||
64,
|
||||
65,
|
||||
66,
|
||||
67,
|
||||
-1,
|
||||
-1,
|
||||
-1,
|
||||
-1,
|
||||
]
|
||||
assert metadata.slot_mapping.tolist() == list(range(12)) + [-1] * 4
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
def test_qsa_circular_buffer_metadata_keeps_only_each_requests_suffix() -> None:
|
||||
device = torch.device("cuda")
|
||||
builder = QSAMetadataBuilder.__new__(QSAMetadataBuilder)
|
||||
builder.compress_ratio = 4
|
||||
builder.is_circular_buffer = True
|
||||
builder.kv_cache_spec = SimpleNamespace(block_size=4)
|
||||
builder.storage_block_size = 4
|
||||
builder.token_to_req_buffer = torch.empty(16, dtype=torch.int32, device=device)
|
||||
builder.slot_mapping_buffer = torch.empty(16, dtype=torch.int64, device=device)
|
||||
builder.logical_positions_buffer = torch.empty(16, dtype=torch.int64, device=device)
|
||||
builder.k_work_metadata_buffer = torch.empty(0, 2, dtype=torch.int32, device=device)
|
||||
query_start_loc = torch.tensor([0, 7, 13, 13], dtype=torch.int32, device=device)
|
||||
token_to_req = torch.tensor([0] * 7 + [1] * 6 + [0] * 3, device=device)
|
||||
block_table = torch.tensor([[1], [0], [2]], dtype=torch.int32, device=device)
|
||||
common = SimpleNamespace(
|
||||
num_actual_tokens=16,
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc.cpu(),
|
||||
seq_lens=torch.tensor([9, 11, 0], dtype=torch.int32, device=device),
|
||||
slot_mapping=torch.full((16,), -1, dtype=torch.int64, device=device),
|
||||
block_table_tensor=block_table,
|
||||
token_to_req_indices=lambda buffer: buffer.copy_(token_to_req),
|
||||
)
|
||||
|
||||
metadata = builder.build(0, common)
|
||||
expected = [
|
||||
-1,
|
||||
-1,
|
||||
-1,
|
||||
5,
|
||||
6,
|
||||
7,
|
||||
4,
|
||||
-1,
|
||||
-1,
|
||||
3,
|
||||
0,
|
||||
1,
|
||||
2,
|
||||
-1,
|
||||
-1,
|
||||
-1,
|
||||
]
|
||||
|
||||
assert metadata.slot_mapping.tolist() == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("chunk_start", list(range(8)))
|
||||
def test_qsa_circular_buffer_survives_one_speculative_step(chunk_start: int) -> None:
|
||||
"""A speculative step must not overwrite the open group's committed keys.
|
||||
|
||||
The step stores every row it computes, drafts included, before acceptance
|
||||
is known, while the next step still reads the earlier members of the group
|
||||
being compressed from the ring. A ring sized at the compression ratio makes
|
||||
those rows alias, so a rejected draft silently replaces a committed key.
|
||||
"""
|
||||
compress_ratio = 4
|
||||
num_spec = 3
|
||||
capacity = compress_ratio * -(-(compress_ratio + num_spec) // compress_ratio)
|
||||
query_len = num_spec + 1
|
||||
|
||||
slots = qsa_cache.circular_qsa_slot_mapping(
|
||||
torch.tensor([[0]], dtype=torch.int32),
|
||||
torch.zeros(query_len, dtype=torch.int32),
|
||||
torch.arange(chunk_start, chunk_start + query_len),
|
||||
capacity,
|
||||
query_start_loc=torch.tensor([0, query_len], dtype=torch.int32),
|
||||
)
|
||||
|
||||
committed = torch.arange(chunk_start - chunk_start % compress_ratio, chunk_start)
|
||||
assert set(slots.tolist()).isdisjoint((committed % capacity).tolist())
|
||||
|
||||
|
||||
def _qsa_key_cache(block_size: int, compress_ratio: int) -> qsa_cache.QSAKeyStateCache:
|
||||
return qsa_cache.QSAKeyStateCache(
|
||||
head_size=64,
|
||||
dtype=torch.bfloat16,
|
||||
cache_config=SimpleNamespace(block_size=block_size),
|
||||
prefix=f"raw.{block_size}.{compress_ratio}",
|
||||
vllm_config=SimpleNamespace(
|
||||
compilation_config=SimpleNamespace(static_forward_context={})
|
||||
),
|
||||
compress_ratio=compress_ratio,
|
||||
)
|
||||
|
||||
|
||||
def test_qsa_state_caches_adapt_the_unified_logical_layout() -> None:
|
||||
raw_cache = _qsa_key_cache(block_size=32, compress_ratio=4)
|
||||
compressed_cache = qsa_cache.QSACompressedKeyCache(
|
||||
head_size=64,
|
||||
dtype=torch.bfloat16,
|
||||
cache_config=SimpleNamespace(block_size=32),
|
||||
prefix="compressed.bind",
|
||||
vllm_config=SimpleNamespace(
|
||||
compilation_config=SimpleNamespace(static_forward_context={})
|
||||
),
|
||||
compress_ratio=4,
|
||||
)
|
||||
raw_view = torch.empty(2, 1, 8, 64, dtype=torch.bfloat16)
|
||||
compressed_view = torch.empty(2, 1, 8, 64, dtype=torch.bfloat16)
|
||||
|
||||
raw_cache.bind_kv_cache(raw_view)
|
||||
compressed_cache.bind_kv_cache(compressed_view)
|
||||
|
||||
assert raw_cache.kv_cache.shape == (2, 8, 1, 64)
|
||||
assert compressed_cache.kv_cache.shape == (2, 8, 1, 64)
|
||||
assert raw_cache.kv_cache.data_ptr() == raw_view.data_ptr()
|
||||
assert compressed_cache.kv_cache.data_ptr() == compressed_view.data_ptr()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("compress_ratio", "num_spec", "expected"),
|
||||
[(4, 0, 4), (4, 1, 8), (4, 3, 8), (4, 4, 8), (4, 5, 12), (2, 3, 6)],
|
||||
)
|
||||
def test_qsa_ring_capacity_covers_one_speculative_step(
|
||||
compress_ratio: int, num_spec: int, expected: int
|
||||
) -> None:
|
||||
"""Capacity spans the open group plus one speculative step, in whole groups."""
|
||||
spec = _qsa_key_cache(
|
||||
block_size=48, compress_ratio=compress_ratio
|
||||
).get_kv_cache_spec(SimpleNamespace(num_speculative_tokens=num_spec))
|
||||
assert spec.block_size == expected
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
def test_qsa_compressed_metadata_keeps_dummy_slots_inert() -> None:
|
||||
device = torch.device("cuda")
|
||||
builder = QSAMetadataBuilder.__new__(QSAMetadataBuilder)
|
||||
builder.compress_ratio = 4
|
||||
builder.is_circular_buffer = False
|
||||
builder.storage_block_size = 16
|
||||
builder.token_to_req_buffer = torch.empty(8, dtype=torch.int32, device=device)
|
||||
builder.slot_mapping_buffer = torch.empty(8, dtype=torch.int64, device=device)
|
||||
builder.logical_positions_buffer = torch.empty(8, dtype=torch.int64, device=device)
|
||||
# Simulate max_num_seqs exceeding the three live requests below.
|
||||
builder.request_capacity = 8
|
||||
builder.k_work_metadata_buffer = torch.empty(4, 2, dtype=torch.int32, device=device)
|
||||
query_start_loc = torch.tensor([0, 3, 3, 8], dtype=torch.int32, device=device)
|
||||
token_to_req = torch.tensor(
|
||||
[0, 0, 0, 2, 2, 2, 2, 2], dtype=torch.int32, device=device
|
||||
)
|
||||
common = SimpleNamespace(
|
||||
num_actual_tokens=8,
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc.cpu(),
|
||||
seq_lens=torch.tensor([7, 0, 12], dtype=torch.int32, device=device),
|
||||
slot_mapping=torch.full((8,), -1, dtype=torch.int64, device=device),
|
||||
block_table_tensor=torch.zeros((3, 1), dtype=torch.int32, device=device),
|
||||
token_to_req_indices=lambda buffer: buffer.copy_(token_to_req),
|
||||
)
|
||||
|
||||
metadata = builder.build(0, common)
|
||||
|
||||
assert metadata.slot_mapping.tolist() == [-1] * 8
|
||||
assert metadata.k_work_metadata.tolist() == [[0, 0], [2, 0], [2, 1], [-1, -1]]
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
@pytest.mark.parametrize("compress_ratio", [1, 4])
|
||||
@pytest.mark.parametrize("num_reqs", [2, 3, 4, 7, 8, 9])
|
||||
def test_qsa_triton_metadata_matches_pytorch(
|
||||
compress_ratio: int, num_reqs: int
|
||||
) -> None:
|
||||
device = torch.device("cuda")
|
||||
num_tokens = 8
|
||||
query_start_loc = torch.tensor(
|
||||
[0, 3, *([3] * (num_reqs - 2)), 8], dtype=torch.int32, device=device
|
||||
)
|
||||
token_to_req = torch.tensor(
|
||||
[0, 0, 0, *([num_reqs - 1] * 5)],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
block_table_rows = torch.tensor(
|
||||
[
|
||||
[4, -1, 8, -1, 12, -1],
|
||||
[1, -1, 2, -1, 3, -1],
|
||||
[7, -1, 9, -1, 11, -1],
|
||||
],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
block_table_storage = block_table_rows[
|
||||
torch.arange(num_reqs, device=device) % block_table_rows.shape[0]
|
||||
]
|
||||
seq_lens = torch.zeros(num_reqs, dtype=torch.int32, device=device)
|
||||
seq_lens[0] = 10
|
||||
seq_lens[-1] = 20
|
||||
common = SimpleNamespace(
|
||||
num_actual_tokens=num_tokens,
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc.cpu(),
|
||||
seq_lens=seq_lens,
|
||||
slot_mapping=torch.tensor(
|
||||
[0, 1, -1, 3, 4, -1, -1, -1], dtype=torch.int64, device=device
|
||||
),
|
||||
block_table_tensor=block_table_storage[:, ::2],
|
||||
token_to_req_indices=lambda buffer: buffer.copy_(token_to_req),
|
||||
)
|
||||
|
||||
def make_buffers() -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
return (
|
||||
torch.empty(num_tokens, dtype=torch.int32, device=device),
|
||||
torch.empty(num_tokens, dtype=torch.int64, device=device),
|
||||
torch.empty(num_tokens, dtype=torch.int64, device=device),
|
||||
)
|
||||
|
||||
max_num_work = (
|
||||
(num_tokens + (compress_ratio - 1) * num_reqs) // compress_ratio
|
||||
if compress_ratio != 1
|
||||
else 0
|
||||
)
|
||||
actual_k_work = (
|
||||
torch.empty(max_num_work, 2, dtype=torch.int32, device=device)
|
||||
if max_num_work
|
||||
else None
|
||||
)
|
||||
actual_buffers = make_buffers()
|
||||
actual = qsa_cache.build_qsa_metadata_triton(
|
||||
common,
|
||||
*actual_buffers,
|
||||
storage_block_size=2,
|
||||
compress_ratio=compress_ratio,
|
||||
k_work_metadata_buffer=actual_k_work,
|
||||
request_capacity=num_reqs,
|
||||
)
|
||||
|
||||
expected_k_work = (
|
||||
torch.empty_like(actual_k_work) if actual_k_work is not None else None
|
||||
)
|
||||
expected_buffers = make_buffers()
|
||||
expected = qsa_cache._build_qsa_metadata_torch(
|
||||
common,
|
||||
*expected_buffers,
|
||||
storage_block_size=2,
|
||||
compress_ratio=compress_ratio,
|
||||
k_work_metadata_buffer=expected_k_work,
|
||||
request_capacity=num_reqs,
|
||||
)
|
||||
|
||||
for actual_tensor, expected_tensor in zip(actual, expected):
|
||||
torch.testing.assert_close(actual_tensor, expected_tensor)
|
||||
if actual_k_work is not None:
|
||||
torch.testing.assert_close(actual_k_work, expected_k_work)
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
def test_qsa_fused_metadata_matches_pytorch_for_large_padded_prefill() -> None:
|
||||
device = torch.device("cuda")
|
||||
num_mapped_tokens = 4096
|
||||
num_tokens = 4224
|
||||
query_start_loc = torch.tensor(
|
||||
[0, num_mapped_tokens], dtype=torch.int32, device=device
|
||||
)
|
||||
common = SimpleNamespace(
|
||||
num_actual_tokens=num_tokens,
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc.cpu(),
|
||||
seq_lens=torch.tensor(
|
||||
[num_mapped_tokens + 32], dtype=torch.int32, device=device
|
||||
),
|
||||
block_table_tensor=torch.arange(256, dtype=torch.int32, device=device)[None],
|
||||
slot_mapping=torch.tensor(
|
||||
[0] * num_mapped_tokens + [-1] * (num_tokens - num_mapped_tokens),
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
),
|
||||
token_to_req_indices=lambda buffer: buffer.zero_(),
|
||||
)
|
||||
|
||||
def make_buffers() -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
return (
|
||||
torch.empty(num_tokens, dtype=torch.int32, device=device),
|
||||
torch.empty(num_tokens, dtype=torch.int64, device=device),
|
||||
torch.empty(num_tokens, dtype=torch.int64, device=device),
|
||||
)
|
||||
|
||||
max_num_work = (num_tokens + 3) // 4
|
||||
actual_k_work = torch.empty(max_num_work, 2, dtype=torch.int32, device=device)
|
||||
expected_k_work = torch.empty_like(actual_k_work)
|
||||
actual = qsa_cache.build_qsa_metadata_triton(
|
||||
common,
|
||||
*make_buffers(),
|
||||
storage_block_size=8,
|
||||
compress_ratio=4,
|
||||
k_work_metadata_buffer=actual_k_work,
|
||||
)
|
||||
expected = qsa_cache._build_qsa_metadata_torch(
|
||||
common,
|
||||
*make_buffers(),
|
||||
storage_block_size=8,
|
||||
compress_ratio=4,
|
||||
k_work_metadata_buffer=expected_k_work,
|
||||
)
|
||||
|
||||
for actual_tensor, expected_tensor in zip(actual, expected):
|
||||
torch.testing.assert_close(actual_tensor, expected_tensor)
|
||||
torch.testing.assert_close(actual_k_work, expected_k_work)
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
@pytest.mark.parametrize(
|
||||
"num_rows",
|
||||
[
|
||||
pytest.param(3, id="one_tile_per_program"),
|
||||
pytest.param(33, id="looped_tiles"),
|
||||
],
|
||||
)
|
||||
def test_qsa_mqa_paged_matches_test_reference(num_rows: int) -> None:
|
||||
torch.manual_seed(1)
|
||||
head_dim = 128
|
||||
q = torch.randn(num_rows, 4, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
cache = torch.randn(40, 4, 1, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
page_table = torch.randperm(40, device="cuda", dtype=torch.int32).reshape(2, 20)
|
||||
request_rows = [(num_rows + 1) // 2, num_rows // 2]
|
||||
token_to_req = torch.repeat_interleave(
|
||||
torch.arange(2, device="cuda", dtype=torch.int32),
|
||||
torch.tensor(request_rows, device="cuda"),
|
||||
)
|
||||
sequence_length_values = [320, 264]
|
||||
sequence_lengths = torch.tensor(
|
||||
sequence_length_values, device="cuda", dtype=torch.int32
|
||||
)
|
||||
query_positions = torch.cat(
|
||||
[
|
||||
torch.arange(length - rows, length, device="cuda", dtype=torch.int32)
|
||||
for rows, length in zip(request_rows, sequence_length_values, strict=True)
|
||||
]
|
||||
)
|
||||
compress_ratio = 4
|
||||
visible_lengths = (query_positions + 1) // compress_ratio
|
||||
|
||||
actual, actual_visible_blocks = qsa_ops.qsa_mqa_paged(
|
||||
q,
|
||||
cache,
|
||||
page_table,
|
||||
token_to_req,
|
||||
query_positions,
|
||||
sequence_lengths,
|
||||
compress_ratio=compress_ratio,
|
||||
)
|
||||
expected = _qsa_mqa_paged_reference(
|
||||
q, cache, page_table, token_to_req, visible_lengths
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual_visible_blocks, visible_lengths)
|
||||
# Top-k is bounded by visible_blocks; later columns are intentionally unwritten.
|
||||
columns = torch.arange(actual.shape[1], device=actual.device)
|
||||
visible = columns[None, :] < actual_visible_blocks[:, None]
|
||||
torch.testing.assert_close(actual[visible], expected[visible], rtol=1e-3, atol=1e-3)
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
def test_qsa_block_expansion_matches_test_reference() -> None:
|
||||
blocks = torch.tensor([[0, -1], [1, 0]], device="cuda", dtype=torch.int32)
|
||||
query_positions = torch.tensor([5, 10], device="cuda")
|
||||
sequence_lengths = torch.tensor([6, 11], device="cuda")
|
||||
token_to_req = torch.tensor([0, 1], device="cuda", dtype=torch.int32)
|
||||
|
||||
actual = qsa_ops.expand_qsa_block_indices_cuda(
|
||||
blocks,
|
||||
query_positions,
|
||||
sequence_lengths,
|
||||
token_to_req,
|
||||
compress_ratio=4,
|
||||
token_topk=8,
|
||||
)
|
||||
expected = _expand_qsa_indices_reference(
|
||||
blocks,
|
||||
query_positions,
|
||||
sequence_lengths,
|
||||
compress_ratio=4,
|
||||
token_topk=8,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
@pytest.mark.parametrize(
|
||||
("num_rows", "num_query_heads", "num_kv_heads", "page_size"),
|
||||
[
|
||||
# Kernel-visible pages with --block-size 256 and hybrid-cache alignment.
|
||||
pytest.param(1, 24, 2, 1792, id="tp1_split64"),
|
||||
pytest.param(16, 12, 1, 1792, id="tp2_split32"),
|
||||
pytest.param(32, 6, 1, 1024, id="tp4_split8"),
|
||||
pytest.param(257, 6, 1, 1024, id="tp4_split4"),
|
||||
pytest.param(513, 6, 1, 1024, id="tp4_split1"),
|
||||
],
|
||||
)
|
||||
def test_qsa_sparse_paged_attention_matches_test_reference(
|
||||
num_rows: int,
|
||||
num_query_heads: int,
|
||||
num_kv_heads: int,
|
||||
page_size: int,
|
||||
) -> None:
|
||||
torch.manual_seed(2)
|
||||
head_dim = 256
|
||||
num_requests = 2
|
||||
num_selected_pages = 64
|
||||
# Keep the newest page outside the synthetic top-k as causal headroom.
|
||||
num_pages_per_request = num_selected_pages + 1
|
||||
num_cache_blocks = num_requests * num_pages_per_request
|
||||
indexer_budget = 2048
|
||||
indexer_compress_ratio = 4
|
||||
selection_width = indexer_budget + indexer_compress_ratio - 1
|
||||
q = torch.randn(
|
||||
num_rows, num_query_heads, head_dim, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
kv_cache = torch.randn(
|
||||
num_cache_blocks,
|
||||
page_size,
|
||||
num_kv_heads,
|
||||
2 * head_dim,
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
k_cache, v_cache = kv_cache.split(head_dim, dim=-1)
|
||||
block_table = (
|
||||
torch.randperm(num_cache_blocks, device="cuda")
|
||||
.reshape(num_requests, num_pages_per_request)
|
||||
.to(torch.int32)
|
||||
)
|
||||
rows_per_request = math.ceil(num_rows / num_requests)
|
||||
row_indices = torch.arange(num_rows, device="cuda", dtype=torch.int32)
|
||||
token_to_req = row_indices // rows_per_request
|
||||
request_row_counts = torch.tensor(
|
||||
[rows_per_request, num_rows - rows_per_request],
|
||||
device="cuda",
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
context_length = num_pages_per_request * page_size - 1
|
||||
block_topk = indexer_budget // indexer_compress_ratio
|
||||
compressed_blocks_per_page = page_size // indexer_compress_ratio
|
||||
selection = torch.arange(block_topk, device="cuda")
|
||||
selected_pages = selection % num_selected_pages
|
||||
selected_offsets = selection // num_selected_pages
|
||||
row_shifts = 2 * row_indices.unsqueeze(1)
|
||||
# Eight blocks per page; adjacent rows overlap by six of those eight.
|
||||
selected_offsets = (selected_offsets + row_shifts) % compressed_blocks_per_page
|
||||
block_indices = (selected_pages * compressed_blocks_per_page + selected_offsets).to(
|
||||
torch.int32
|
||||
)
|
||||
rows_within_request = row_indices % rows_per_request
|
||||
query_positions = (
|
||||
context_length - request_row_counts[token_to_req.long()] + rows_within_request
|
||||
).to(torch.int64)
|
||||
sequence_lengths = torch.full(
|
||||
(num_requests,), context_length, device="cuda", dtype=torch.int32
|
||||
)
|
||||
logical_indices = qsa_ops.expand_qsa_block_indices_cuda(
|
||||
block_indices,
|
||||
query_positions,
|
||||
sequence_lengths,
|
||||
token_to_req,
|
||||
indexer_compress_ratio,
|
||||
indexer_budget,
|
||||
)
|
||||
assert logical_indices.shape == (num_rows, selection_width)
|
||||
scale = q.shape[-1] ** -0.5
|
||||
|
||||
actual = qsa_ops.qsa_sparse_paged_attention(
|
||||
q,
|
||||
k_cache,
|
||||
v_cache,
|
||||
logical_indices,
|
||||
block_table,
|
||||
token_to_req,
|
||||
)
|
||||
expected = _qsa_sparse_paged_attention_reference(
|
||||
q,
|
||||
k_cache,
|
||||
v_cache,
|
||||
logical_indices,
|
||||
block_table,
|
||||
token_to_req,
|
||||
scale,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, rtol=2e-2, atol=2e-2)
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
def test_qsa_selection_chunks_workspace_and_matches_test_reference(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
workspace_init,
|
||||
) -> None:
|
||||
rows, keys, heads, head_dim = 65, 640, 4, 16
|
||||
token_topk, compress_ratio = 2048, 4
|
||||
torch.manual_seed(3)
|
||||
q = torch.randn(rows, heads, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
cache = torch.randn(40, 16, 1, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
page_table = torch.randperm(40, device="cuda", dtype=torch.int32).unsqueeze(0)
|
||||
token_to_req = torch.zeros(rows, device="cuda", dtype=torch.int32)
|
||||
query_positions = torch.full((rows,), 2559, device="cuda", dtype=torch.int32)
|
||||
sequence_lengths = torch.tensor([2560], device="cuda", dtype=torch.int32)
|
||||
monkeypatch.setattr(qsa_ops, "_LOGITS_WORKSPACE_BYTES", 32 * keys * 4)
|
||||
original_score = qsa_ops.qsa_mqa_paged
|
||||
scored_row_counts = []
|
||||
|
||||
def record_score(query: torch.Tensor, *args, **kwargs):
|
||||
scored_row_counts.append(query.shape[0])
|
||||
return original_score(query, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(qsa_ops, "qsa_mqa_paged", record_score)
|
||||
|
||||
actual = qsa_ops.qsa_select_paged_tokens(
|
||||
q,
|
||||
cache,
|
||||
page_table,
|
||||
token_to_req,
|
||||
query_positions,
|
||||
sequence_lengths,
|
||||
token_topk,
|
||||
compress_ratio,
|
||||
)
|
||||
expected = _qsa_select_paged_tokens_reference(
|
||||
q,
|
||||
cache,
|
||||
page_table,
|
||||
token_to_req,
|
||||
query_positions,
|
||||
sequence_lengths,
|
||||
token_topk,
|
||||
compress_ratio,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual.sort().values, expected.sort().values)
|
||||
assert scored_row_counts == [32, 32, 1]
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
def test_qsa_selection_handles_no_complete_compressed_blocks(workspace_init) -> None:
|
||||
q = torch.zeros(2, 4, 8, device="cuda", dtype=torch.bfloat16)
|
||||
cache = torch.zeros(1, 16, 1, 8, device="cuda", dtype=torch.bfloat16)
|
||||
page_table = torch.zeros(1, 1, device="cuda", dtype=torch.int32)
|
||||
token_to_req = torch.zeros(2, device="cuda", dtype=torch.int32)
|
||||
query_positions = torch.tensor([1, 2], device="cuda", dtype=torch.int32)
|
||||
sequence_lengths = torch.tensor([3], device="cuda", dtype=torch.int32)
|
||||
|
||||
selected = qsa_ops.qsa_select_paged_tokens(
|
||||
q,
|
||||
cache,
|
||||
page_table,
|
||||
token_to_req,
|
||||
query_positions,
|
||||
sequence_lengths,
|
||||
token_topk=2048,
|
||||
compress_ratio=4,
|
||||
)
|
||||
|
||||
assert selected[0, :2].tolist() == [0, 1]
|
||||
assert selected[1, :3].tolist() == [0, 1, 2]
|
||||
assert torch.all(selected[0, 2:] == -1)
|
||||
assert torch.all(selected[1, 3:] == -1)
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
def test_qsa_streaming_compression_and_compressor_state_store_match_reference() -> None:
|
||||
head_dim = 8
|
||||
current_pairs = [
|
||||
*((0, position) for position in range(2, 9)),
|
||||
*((1, position) for position in range(5, 11)),
|
||||
]
|
||||
|
||||
def key_row(request: int, position: int) -> torch.Tensor:
|
||||
return (
|
||||
torch.arange(head_dim, dtype=torch.float32) + request * 1000 + position * 10
|
||||
)
|
||||
|
||||
def position_row(request: int, position: int) -> torch.Tensor:
|
||||
return torch.tensor(
|
||||
[
|
||||
request * 1000 + position,
|
||||
request * 1000 + position + 100,
|
||||
request * 1000 + position + 200,
|
||||
],
|
||||
dtype=torch.int64,
|
||||
)
|
||||
|
||||
raw_keys = (
|
||||
torch.stack([key_row(request, position) for request, position in current_pairs])
|
||||
.unsqueeze(1)
|
||||
.to(device="cuda", dtype=torch.bfloat16)
|
||||
)
|
||||
raw_positions = (
|
||||
torch.stack(
|
||||
[position_row(request, position) for request, position in current_pairs]
|
||||
)
|
||||
.unsqueeze(1)
|
||||
.to(device="cuda")
|
||||
)
|
||||
token_to_req = torch.tensor(
|
||||
[request for request, _ in current_pairs],
|
||||
dtype=torch.int32,
|
||||
device="cuda",
|
||||
)
|
||||
logical_positions = torch.tensor(
|
||||
[position for _, position in current_pairs],
|
||||
dtype=torch.int64,
|
||||
device="cuda",
|
||||
)
|
||||
query_start_loc = torch.tensor([0, 7, 13], dtype=torch.int32, device="cuda")
|
||||
compressor_state_block_table = torch.tensor(
|
||||
[[1], [0]], dtype=torch.int32, device="cuda"
|
||||
)
|
||||
compressor_state_cache = torch.zeros(
|
||||
2, 4, 1, head_dim, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
rope_cache = torch.zeros(2, 4, 1, 3, dtype=torch.int64, device="cuda")
|
||||
for request, position, block in ((0, 0, 1), (0, 1, 1), (1, 4, 0)):
|
||||
compressor_state_cache[block, position % 4, 0] = key_row(request, position).to(
|
||||
device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
rope_cache[block, position % 4, 0] = position_row(request, position).to("cuda")
|
||||
|
||||
compressed_slots = torch.full(
|
||||
(len(current_pairs),), -1, dtype=torch.int64, device="cuda"
|
||||
)
|
||||
valid_rows = torch.tensor([1, 5, 9], dtype=torch.int64, device="cuda")
|
||||
compressed_slots[valid_rows] = torch.arange(3, device="cuda")
|
||||
pooled, first_positions = qsa_ops.qsa_compress_groups_with_ratio(
|
||||
raw_keys,
|
||||
raw_positions,
|
||||
compressor_state_cache,
|
||||
compressor_state_block_table,
|
||||
token_to_req,
|
||||
query_start_loc,
|
||||
logical_positions,
|
||||
compressed_slots,
|
||||
compress_ratio=4,
|
||||
rope_cache=rope_cache,
|
||||
)
|
||||
pooled_without_rope, scalar_first_positions = (
|
||||
qsa_ops.qsa_compress_groups_with_ratio(
|
||||
raw_keys,
|
||||
raw_positions,
|
||||
compressor_state_cache,
|
||||
compressor_state_block_table,
|
||||
token_to_req,
|
||||
query_start_loc,
|
||||
logical_positions,
|
||||
compressed_slots,
|
||||
compress_ratio=4,
|
||||
)
|
||||
)
|
||||
|
||||
groups = [
|
||||
[(0, position) for position in range(0, 4)],
|
||||
[(0, position) for position in range(4, 8)],
|
||||
[(1, position) for position in range(4, 8)],
|
||||
]
|
||||
expected_pooled = (
|
||||
torch.stack(
|
||||
[
|
||||
torch.stack([key_row(*pair) for pair in group]).mean(dim=0)
|
||||
for group in groups
|
||||
]
|
||||
)
|
||||
.unsqueeze(1)
|
||||
.to(device="cuda", dtype=torch.bfloat16)
|
||||
)
|
||||
expected_positions = torch.stack(
|
||||
[position_row(0, 0), position_row(0, 4), position_row(1, 4)]
|
||||
).to("cuda")
|
||||
expected_scalar_positions = torch.tensor(
|
||||
[[0, 0, 0], [4, 4, 4], [4, 4, 4]],
|
||||
dtype=torch.int64,
|
||||
device="cuda",
|
||||
)
|
||||
|
||||
torch.testing.assert_close(pooled[valid_rows], expected_pooled)
|
||||
torch.testing.assert_close(pooled_without_rope[valid_rows], expected_pooled)
|
||||
torch.testing.assert_close(first_positions[valid_rows], expected_positions)
|
||||
torch.testing.assert_close(
|
||||
scalar_first_positions[valid_rows], expected_scalar_positions
|
||||
)
|
||||
|
||||
compressor_state_slots = torch.tensor(
|
||||
[-1, -1, -1, 5, 6, 7, 4, -1, -1, 3, 0, 1, 2],
|
||||
dtype=torch.int64,
|
||||
device="cuda",
|
||||
)
|
||||
qsa_ops.qsa_store_cache_rows(
|
||||
compressor_state_cache, compressor_state_slots, raw_keys
|
||||
)
|
||||
qsa_ops.qsa_store_cache_rows(rope_cache, compressor_state_slots, raw_positions)
|
||||
for request, positions, block in ((0, range(5, 9), 1), (1, range(7, 11), 0)):
|
||||
for position in positions:
|
||||
torch.testing.assert_close(
|
||||
compressor_state_cache[block, position % 4, 0],
|
||||
key_row(request, position).to(device="cuda", dtype=torch.bfloat16),
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
rope_cache[block, position % 4, 0],
|
||||
position_row(request, position).to("cuda"),
|
||||
)
|
||||
@@ -508,6 +508,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
extras={"native-prefix": "imdatta0/small_qwen3_5_20b"},
|
||||
max_model_len=4096,
|
||||
),
|
||||
"Qwen4ExpForCausalLM": _HfExamplesInfo("", is_available_online=False),
|
||||
"MellumForCausalLM": _HfExamplesInfo("JetBrains/Mellum2-12B-A2.5B-Base"),
|
||||
"Qwen3NextForCausalLM": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-Next-80B-A3B-Instruct",
|
||||
@@ -1348,6 +1349,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"Qwen/Qwen3.5-35B-A3B",
|
||||
max_model_len=4096,
|
||||
),
|
||||
"Qwen4ExpForConditionalGeneration": _HfExamplesInfo(
|
||||
"",
|
||||
is_available_online=False,
|
||||
),
|
||||
"Qwen3OmniMoeForConditionalGeneration": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
max_model_len=4096,
|
||||
@@ -1797,6 +1802,7 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
"Qwen/Qwen3.5-35B-A3B",
|
||||
speculative_model="Qwen/Qwen3.5-35B-A3B",
|
||||
),
|
||||
"Qwen4ExpMTP": _HfExamplesInfo("", is_available_online=False),
|
||||
"Step3p5MTP": _HfExamplesInfo(
|
||||
"stepfun-ai/Step-3.5-Flash",
|
||||
speculative_model="stepfun-ai/Step-3.5-Flash",
|
||||
|
||||
@@ -6,6 +6,7 @@ Run `pytest tests/quantization/test_fp8.py --forked`.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
@@ -31,6 +32,10 @@ from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import (
|
||||
Fp8PerTensorOnlineLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils import flashinfer_utils
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
prepare_fp8_moe_layer_for_fi,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
process_fp8_input_tensor_strategy_moe,
|
||||
)
|
||||
@@ -50,6 +55,45 @@ MODELS = [
|
||||
]
|
||||
|
||||
|
||||
def test_prepare_gated_trtllm_fp8_moe_weights_pads_each_projection(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
flashinfer_utils,
|
||||
"rotate_weights_for_fi_trtllm_fp8_per_tensor_moe",
|
||||
lambda *args: None,
|
||||
)
|
||||
intermediate = 17
|
||||
padded_intermediate = 32
|
||||
hidden_size = 4
|
||||
gate = torch.ones((1, intermediate, hidden_size), dtype=torch.float8_e4m3fn)
|
||||
up = torch.full_like(gate, 2)
|
||||
w13 = torch.cat((gate, up), dim=1)
|
||||
w2 = torch.ones((1, hidden_size, intermediate), dtype=torch.float8_e4m3fn)
|
||||
layer = SimpleNamespace(
|
||||
activation=SimpleNamespace(is_gated=True),
|
||||
moe_config=SimpleNamespace(
|
||||
is_act_and_mul=True,
|
||||
intermediate_size_per_partition=intermediate,
|
||||
),
|
||||
)
|
||||
|
||||
padded_w31, _, _, _ = prepare_fp8_moe_layer_for_fi(
|
||||
layer,
|
||||
w13,
|
||||
w2,
|
||||
w13_scale=torch.ones(1),
|
||||
w13_input_scale=torch.ones(1),
|
||||
w2_scale=torch.ones(1),
|
||||
w2_input_scale=torch.ones(1),
|
||||
is_trtllm=True,
|
||||
)
|
||||
|
||||
expected = w13.new_zeros((1, 2 * padded_intermediate, hidden_size))
|
||||
expected[:, :intermediate] = up
|
||||
expected[:, padded_intermediate : padded_intermediate + intermediate] = gate
|
||||
assert layer.moe_config.intermediate_size_per_partition == padded_intermediate
|
||||
assert torch.equal(padded_w31, expected)
|
||||
|
||||
|
||||
def test_static_fp8_moe_input_scales_remain_scalar() -> None:
|
||||
a1_scale, a2_scale = process_fp8_input_tensor_strategy_moe(
|
||||
torch.tensor([0.25, 0.5]),
|
||||
|
||||
@@ -16,6 +16,7 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
Mxfp4MoeBackend,
|
||||
_requires_qwen38_tep8_emulation,
|
||||
select_mxfp4_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
@@ -72,6 +73,91 @@ def _make_w4a4_moe_config(moe_backend: str = "auto") -> FusedMoEConfig:
|
||||
)
|
||||
|
||||
|
||||
def _make_qwen38_tep8_moe_config() -> FusedMoEConfig:
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
|
||||
return FusedMoEConfig(
|
||||
num_experts=512,
|
||||
experts_per_token=9,
|
||||
hidden_dim=2560,
|
||||
intermediate_size=640,
|
||||
num_local_experts=64,
|
||||
num_logical_experts=512,
|
||||
moe_parallel_config=FusedMoEParallelConfig(
|
||||
tp_size=1,
|
||||
tp_rank=0,
|
||||
pcp_size=1,
|
||||
pcp_rank=0,
|
||||
dp_size=1,
|
||||
dp_rank=0,
|
||||
ep_size=8,
|
||||
ep_rank=0,
|
||||
sp_size=1,
|
||||
use_ep=True,
|
||||
all2all_backend="allgather_reducescatter",
|
||||
enable_eplb=False,
|
||||
),
|
||||
activation=MoEActivation.SILU,
|
||||
in_dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
routing_method=RoutingMethodType.Renormalize,
|
||||
)
|
||||
|
||||
|
||||
def test_qwen38_tep8_requires_emulation_only_on_gfx950(monkeypatch):
|
||||
import vllm.model_executor.layers.fused_moe.oracle.mxfp4 as mxfp4_oracle
|
||||
|
||||
config = _make_qwen38_tep8_moe_config()
|
||||
monkeypatch.setattr(current_platform, "is_rocm", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: True)
|
||||
|
||||
assert _requires_qwen38_tep8_emulation(config, kMxfp4Dynamic)
|
||||
|
||||
config.moe_parallel_config.ep_size = 4
|
||||
config.num_local_experts = 128
|
||||
assert not _requires_qwen38_tep8_emulation(config, kMxfp4Dynamic)
|
||||
|
||||
config = _make_qwen38_tep8_moe_config()
|
||||
monkeypatch.setattr(mxfp4_oracle.current_platform, "is_rocm", lambda: False)
|
||||
assert not _requires_qwen38_tep8_emulation(config, kMxfp4Dynamic)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"requested_backend,expected_backend",
|
||||
[
|
||||
("auto", Mxfp4MoeBackend.EMULATION),
|
||||
("aiter", Mxfp4MoeBackend.AITER_MXFP4_MXFP4),
|
||||
],
|
||||
)
|
||||
def test_qwen38_tep8_auto_fallback_respects_explicit_backend(
|
||||
requested_backend,
|
||||
expected_backend,
|
||||
monkeypatch,
|
||||
):
|
||||
import vllm.model_executor.layers.fused_moe.oracle.mxfp4 as mxfp4_oracle
|
||||
|
||||
class SupportedExperts:
|
||||
@staticmethod
|
||||
def is_supported_config(*args, **kwargs):
|
||||
return True, None
|
||||
|
||||
config = _make_qwen38_tep8_moe_config()
|
||||
config.moe_backend = requested_backend
|
||||
monkeypatch.setattr(current_platform, "is_rocm", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: True)
|
||||
monkeypatch.setattr(mxfp4_oracle, "_user_moe_activation_override", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
mxfp4_oracle, "backend_to_kernel_cls", lambda backend: [SupportedExperts]
|
||||
)
|
||||
|
||||
backend, experts_cls = select_mxfp4_moe_backend(
|
||||
config, activation_key=kMxfp4Dynamic
|
||||
)
|
||||
|
||||
assert backend == expected_backend
|
||||
assert experts_cls is SupportedExperts
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mxfp4_oracle_config():
|
||||
"""Stub the config the oracle reads (``model_config.quantization_config``)
|
||||
|
||||
@@ -604,3 +604,24 @@ def test_online_mxfp4_dense_matches_quark(
|
||||
keys=("weight", "weight_scale"),
|
||||
packed_weight_keys=("weight",),
|
||||
)
|
||||
|
||||
|
||||
def test_quark_ocp_mx_dense_rejects_unaligned_input_partition():
|
||||
scheme = object.__new__(QuarkOCP_MX)
|
||||
layer = torch.nn.Module()
|
||||
layer.prefix = "model.layers.0.mlp.shared_expert.down_proj"
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match=(
|
||||
"input size per partition of 80, which must be divisible by "
|
||||
"the OCP MX group size 32"
|
||||
),
|
||||
):
|
||||
scheme.create_weights(
|
||||
layer=layer,
|
||||
output_partition_sizes=[2560],
|
||||
input_size_per_partition=80,
|
||||
params_dtype=torch.bfloat16,
|
||||
weight_loader=lambda *_: None,
|
||||
)
|
||||
|
||||
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
|
||||
prepare_nvfp4_moe_layer_for_fi_or_cutlass,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
align_fp4_moe_weights_for_fi,
|
||||
align_trtllm_fp4_moe_hidden_dim_for_fi,
|
||||
)
|
||||
|
||||
@@ -107,3 +108,53 @@ def test_align_trtllm_fp4_moe_hidden_dim_pads_to_256_multiple():
|
||||
assert torch.count_nonzero(out_w13_scale[:, :, hidden_dim // 16 :]) == 0
|
||||
assert torch.count_nonzero(out_w2[:, hidden_dim:, :]) == 0
|
||||
assert torch.count_nonzero(out_w2_scale[:, hidden_dim:, :]) == 0
|
||||
|
||||
|
||||
def test_align_trtllm_fp4_moe_intermediate_pads_gate_and_up_separately():
|
||||
# Qwen3.8-Flash-Next has intermediate_size=640. At TP4 each rank owns
|
||||
# 160 values, which TRTLLM pads to the next 64-value boundary (192).
|
||||
intermediate = 160
|
||||
padded_intermediate = 192
|
||||
hidden_dim = 32
|
||||
|
||||
gate = torch.ones((1, intermediate, hidden_dim // 2), dtype=torch.uint8)
|
||||
up = torch.full_like(gate, 2)
|
||||
w13 = torch.cat((gate, up), dim=1)
|
||||
gate_scale = torch.full((1, intermediate, hidden_dim // 16), 3, dtype=torch.uint8)
|
||||
up_scale = torch.full_like(gate_scale, 4)
|
||||
w13_scale = torch.cat((gate_scale, up_scale), dim=1)
|
||||
w2 = torch.full((1, hidden_dim, intermediate // 2), 5, dtype=torch.uint8)
|
||||
w2_scale = torch.full((1, hidden_dim, intermediate // 16), 6, dtype=torch.uint8)
|
||||
|
||||
out_w13, out_w13_scale, out_w2, out_w2_scale, out_intermediate = (
|
||||
align_fp4_moe_weights_for_fi(
|
||||
w13,
|
||||
w13_scale,
|
||||
w2,
|
||||
w2_scale,
|
||||
is_act_and_mul=True,
|
||||
min_alignment=64,
|
||||
)
|
||||
)
|
||||
|
||||
assert out_intermediate == padded_intermediate
|
||||
torch.testing.assert_close(out_w13[:, :intermediate], gate)
|
||||
torch.testing.assert_close(
|
||||
out_w13[:, padded_intermediate : padded_intermediate + intermediate], up
|
||||
)
|
||||
torch.testing.assert_close(out_w13_scale[:, :intermediate], gate_scale)
|
||||
torch.testing.assert_close(
|
||||
out_w13_scale[:, padded_intermediate : padded_intermediate + intermediate],
|
||||
up_scale,
|
||||
)
|
||||
|
||||
assert torch.count_nonzero(out_w13[:, intermediate:padded_intermediate]) == 0
|
||||
assert torch.count_nonzero(out_w13[:, padded_intermediate + intermediate :]) == 0
|
||||
assert torch.count_nonzero(out_w13_scale[:, intermediate:padded_intermediate]) == 0
|
||||
assert (
|
||||
torch.count_nonzero(out_w13_scale[:, padded_intermediate + intermediate :]) == 0
|
||||
)
|
||||
torch.testing.assert_close(out_w2[:, :, : intermediate // 2], w2)
|
||||
torch.testing.assert_close(out_w2_scale[:, :, : intermediate // 16], w2_scale)
|
||||
assert torch.count_nonzero(out_w2[:, :, intermediate // 2 :]) == 0
|
||||
assert torch.count_nonzero(out_w2_scale[:, :, intermediate // 16 :]) == 0
|
||||
|
||||
@@ -9,23 +9,34 @@ block dim (a contiguous region per layer) or inside it (all layers' pages within
|
||||
block); the allocation is the same either way.
|
||||
"""
|
||||
|
||||
from dataclasses import replace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import CacheConfig
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheManager
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
_get_kv_cache_bytes_per_block,
|
||||
_get_packed_kv_cache_groups,
|
||||
_pool_bytes_per_block,
|
||||
generate_scheduler_kv_cache_config,
|
||||
get_kv_cache_config_from_groups,
|
||||
get_kv_cache_groups,
|
||||
resolve_kv_cache_block_sizes,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheLayout,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowMLASpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
iter_layer_specs,
|
||||
)
|
||||
from vllm.v1.worker.utils import allocate_kv_cache
|
||||
|
||||
@@ -86,6 +97,307 @@ def _bind(config, layout: str):
|
||||
return allocate_kv_cache(config, torch.device("cpu"), KVCacheLayout[layout], None)
|
||||
|
||||
|
||||
MAIN_KV_PAGE_BYTES = 2_048
|
||||
COMPRESSED_PAGE_BYTES = 128
|
||||
NUM_CACHE_TUPLES = 3
|
||||
BYTES_PER_BLOCK = NUM_CACHE_TUPLES * (MAIN_KV_PAGE_BYTES + COMPRESSED_PAGE_BYTES)
|
||||
|
||||
|
||||
def _main_kv_name(layer_index: int) -> str:
|
||||
return f"model.layers.{layer_index}.self_attn"
|
||||
|
||||
|
||||
def _compressed_name(layer_index: int) -> str:
|
||||
return f"model.layers.{layer_index}.self_attn.indexer.compressed_key_cache"
|
||||
|
||||
|
||||
def _compressor_state_name(layer_index: int) -> str:
|
||||
return f"model.layers.{layer_index}.self_attn.indexer.raw_key_cache"
|
||||
|
||||
|
||||
def _make_csa_linear_specs(
|
||||
num_mamba: int = 7,
|
||||
num_tuples: int = NUM_CACHE_TUPLES,
|
||||
) -> dict[str, KVCacheSpec]:
|
||||
specs: dict[str, KVCacheSpec] = {}
|
||||
for layer_index in range(num_mamba):
|
||||
specs[f"model.layers.{layer_index}.linear_attn"] = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((32,),),
|
||||
dtypes=(torch.bfloat16,),
|
||||
)
|
||||
specs["model.layers.0.ple"] = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((24,),),
|
||||
dtypes=(torch.bfloat16,),
|
||||
tp_replicated=True,
|
||||
)
|
||||
for layer_index in range(num_tuples):
|
||||
specs[_main_kv_name(layer_index)] = FullAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=2,
|
||||
head_size=16,
|
||||
head_size_v=16,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
specs[_compressed_name(layer_index)] = MLAAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=16,
|
||||
dtype=torch.bfloat16,
|
||||
tokens_per_state=4,
|
||||
)
|
||||
specs[_compressor_state_name(layer_index)] = CircularBufferSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=1,
|
||||
head_size=8,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
return specs
|
||||
|
||||
|
||||
def _shared_layout_config():
|
||||
config = _mock_vllm_config("BLNHC")
|
||||
config.scheduler_config.disable_hybrid_kv_cache_manager = False
|
||||
config.speculative_config = None
|
||||
config.model_config.max_model_len = 16
|
||||
config.model_config.get_total_num_hidden_layers.return_value = 64
|
||||
config.model_config.get_total_num_kv_heads.return_value = 2
|
||||
config.model_config.get_num_kv_heads.return_value = 2
|
||||
config.parallel_config.pipeline_parallel_size = 1
|
||||
config.parallel_config.decode_context_parallel_size = 1
|
||||
config.cache_config.block_size = 16
|
||||
config.cache_config.enable_prefix_caching = False
|
||||
config.cache_config.prefix_match_unit = None
|
||||
config.cache_config.mamba_cache_mode = "none"
|
||||
return config
|
||||
|
||||
|
||||
class TestCSALinearGrouping:
|
||||
"""A CSA + linear-attention model (sparse attention with a compressor ring,
|
||||
plus sharded GDN and one TP-replicated PLE state) goes through the generic
|
||||
packed-group path; no model-specific branch is involved."""
|
||||
|
||||
@staticmethod
|
||||
def _mamba_groups(groups):
|
||||
"""(group, per-layer mamba spec) for every group holding mamba state."""
|
||||
out = []
|
||||
for group in groups:
|
||||
spec = next(iter(iter_layer_specs(group.kv_cache_spec)))
|
||||
if isinstance(spec, MambaSpec):
|
||||
out.append((group, spec))
|
||||
return out
|
||||
|
||||
def test_replicated_state_gets_its_own_group(self):
|
||||
"""The PLE state is TP-replicated and a different size, so it must not
|
||||
share a manager group with the sharded GDN states (the NIXL worker
|
||||
requires exactly one single-layer replicated group)."""
|
||||
groups = get_kv_cache_groups(_shared_layout_config(), _make_csa_linear_specs())
|
||||
|
||||
mamba_groups = self._mamba_groups(groups)
|
||||
replicated = [g for g, spec in mamba_groups if spec.tp_replicated]
|
||||
assert len(replicated) == 1
|
||||
assert replicated[0].layer_names == ["model.layers.0.ple"]
|
||||
sharded = [g for g, spec in mamba_groups if not spec.tp_replicated]
|
||||
assert sharded, "GDN states must keep their own groups"
|
||||
assert sorted(n for g in sharded for n in g.layer_names) == sorted(
|
||||
f"model.layers.{i}.linear_attn" for i in range(7)
|
||||
)
|
||||
|
||||
def test_roles_land_in_separate_groups(self):
|
||||
groups = get_kv_cache_groups(_shared_layout_config(), _make_csa_linear_specs())
|
||||
|
||||
owner = next(g for g in groups if _main_kv_name(0) in g.layer_names)
|
||||
assert sorted(owner.layer_names) == sorted(
|
||||
[
|
||||
*(_main_kv_name(i) for i in range(NUM_CACHE_TUPLES)),
|
||||
*(_compressed_name(i) for i in range(NUM_CACHE_TUPLES)),
|
||||
]
|
||||
)
|
||||
assert owner.kv_cache_spec.prefix_cacheable
|
||||
|
||||
scratch = next(g for g in groups if _compressor_state_name(0) in g.layer_names)
|
||||
assert scratch.layer_names == [
|
||||
_compressor_state_name(i) for i in range(NUM_CACHE_TUPLES)
|
||||
]
|
||||
assert not scratch.kv_cache_spec.prefix_cacheable
|
||||
|
||||
def test_every_group_fits_one_packed_block(self):
|
||||
config = _shared_layout_config()
|
||||
groups = get_kv_cache_groups(config, _make_csa_linear_specs())
|
||||
bytes_per_block = _get_kv_cache_bytes_per_block(groups)
|
||||
|
||||
pages = _pages(groups)
|
||||
for group in groups:
|
||||
assert sum(pages[n] for n in group.layer_names) <= bytes_per_block
|
||||
|
||||
kv_cache_config = get_kv_cache_config_from_groups(
|
||||
config, groups, available_memory=bytes_per_block * 32
|
||||
)
|
||||
assert kv_cache_config.num_blocks == 32
|
||||
# Groups overlay from byte 0 of each block, so a layer never addresses
|
||||
# past the block it belongs to.
|
||||
for tensor in kv_cache_config.kv_cache_tensors:
|
||||
assert tensor.block_stride == bytes_per_block
|
||||
assert tensor.offset < bytes_per_block
|
||||
|
||||
def test_scratch_group_survives_computed_block_truncation(self):
|
||||
"""The scratch group contributes no computed blocks, so truncating a
|
||||
lookup result must skip it: its block size is the ring capacity, which
|
||||
neither divides the hit length nor bounds the (empty) block list."""
|
||||
config = _shared_layout_config()
|
||||
config.cache_config.enable_prefix_caching = True
|
||||
groups = get_kv_cache_groups(config, _make_csa_linear_specs())
|
||||
kv_cache_config = get_kv_cache_config_from_groups(
|
||||
config, groups, available_memory=BYTES_PER_BLOCK * 64
|
||||
)
|
||||
scheduler_config = generate_scheduler_kv_cache_config([kv_cache_config])
|
||||
manager = KVCacheManager(
|
||||
scheduler_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=16,
|
||||
scheduler_block_size=16,
|
||||
)
|
||||
blocks = manager.create_kv_cache_blocks(
|
||||
tuple(
|
||||
manager.block_pool.get_new_blocks(3)
|
||||
if group.kv_cache_spec.prefix_cacheable
|
||||
else []
|
||||
for group in scheduler_config.kv_cache_groups
|
||||
)
|
||||
)
|
||||
truncated = manager.truncate_computed_blocks(blocks, 48)
|
||||
scratch_index = next(
|
||||
i
|
||||
for i, group in enumerate(scheduler_config.kv_cache_groups)
|
||||
if not group.kv_cache_spec.prefix_cacheable
|
||||
)
|
||||
assert truncated.blocks[scratch_index] == []
|
||||
|
||||
def test_prefix_hits_respect_compression_alignment(self):
|
||||
config = _shared_layout_config()
|
||||
config.cache_config.enable_prefix_caching = True
|
||||
config.cache_config.prefix_match_unit = 16
|
||||
config.cache_config.mamba_cache_mode = "align"
|
||||
specs = {
|
||||
name: replace(spec, mamba_cache_mode="align")
|
||||
if isinstance(spec, MambaSpec)
|
||||
else spec
|
||||
for name, spec in _make_csa_linear_specs(num_tuples=1).items()
|
||||
}
|
||||
groups = get_kv_cache_groups(config, specs)
|
||||
kv_cache_config = get_kv_cache_config_from_groups(
|
||||
config, groups, available_memory=8 * MAIN_KV_PAGE_BYTES
|
||||
)
|
||||
assert resolve_kv_cache_block_sizes(kv_cache_config, config) == (16, 16)
|
||||
|
||||
# 2 tokens is not a multiple of the compression ratio 4: a prefix hit
|
||||
# could land inside a partially filled compressed state.
|
||||
config.cache_config.prefix_match_unit = 2
|
||||
with pytest.raises(ValueError, match="per-state compression"):
|
||||
resolve_kv_cache_block_sizes(kv_cache_config, config)
|
||||
|
||||
def test_scratch_ring_does_not_drag_hash_granularity(self):
|
||||
config = _shared_layout_config()
|
||||
config.cache_config.enable_prefix_caching = True
|
||||
config.cache_config.mamba_cache_mode = "align"
|
||||
specs = {
|
||||
name: replace(spec, mamba_cache_mode="align")
|
||||
if isinstance(spec, MambaSpec)
|
||||
else spec
|
||||
for name, spec in _make_csa_linear_specs(num_tuples=1).items()
|
||||
}
|
||||
groups = get_kv_cache_groups(config, specs)
|
||||
kv_cache_config = get_kv_cache_config_from_groups(
|
||||
config, groups, available_memory=8 * MAIN_KV_PAGE_BYTES
|
||||
)
|
||||
# The hash granularity is the GCD over prefix-cacheable groups only;
|
||||
# the 4-token scratch ring is excluded (it would drag it to 4).
|
||||
assert resolve_kv_cache_block_sizes(kv_cache_config, config) == (16, 16)
|
||||
|
||||
def test_compressed_attention_hashes_can_be_finer_than_cache_hits(self):
|
||||
config = _shared_layout_config()
|
||||
config.cache_config.enable_prefix_caching = True
|
||||
specs = {
|
||||
"compressed.4": MLAAttentionSpec(
|
||||
block_size=256,
|
||||
num_kv_heads=1,
|
||||
head_size=16,
|
||||
dtype=torch.bfloat16,
|
||||
tokens_per_state=4,
|
||||
),
|
||||
"compressed.128": MLAAttentionSpec(
|
||||
block_size=256,
|
||||
num_kv_heads=1,
|
||||
head_size=16,
|
||||
dtype=torch.bfloat16,
|
||||
tokens_per_state=128,
|
||||
),
|
||||
"compressor_state.4": SlidingWindowMLASpec(
|
||||
block_size=4,
|
||||
num_kv_heads=1,
|
||||
head_size=8,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=8,
|
||||
),
|
||||
}
|
||||
groups = get_kv_cache_groups(config, specs)
|
||||
kv_cache_config = get_kv_cache_config_from_groups(
|
||||
config, groups, available_memory=8 * MAIN_KV_PAGE_BYTES
|
||||
)
|
||||
|
||||
# Hashes are computed every 4 tokens, but without an align-mode Mamba
|
||||
# group cache hits remain on the 256-token scheduler boundary.
|
||||
assert resolve_kv_cache_block_sizes(kv_cache_config, config) == (256, 4)
|
||||
|
||||
@pytest.mark.parametrize("wide", ["unbalanced_attention", "unsplittable_state"])
|
||||
def test_mamba_split_measures_the_block_the_other_groups_already_force(self, wide):
|
||||
"""Whatever fixes the block stride -- a bucket with unequal layer counts
|
||||
per page size, which is emitted whole, or a single-layer state that
|
||||
cannot be split at all -- the mamba layers must be sized against it.
|
||||
Sizing them against a narrower bucket splits them past what the block
|
||||
already fits, spending a pool block per extra group for no saving."""
|
||||
config = _mock_vllm_config("BLNHC")
|
||||
config.speculative_config = None
|
||||
specs = {}
|
||||
if wide == "unbalanced_attention":
|
||||
for i in range(3):
|
||||
specs[f"wide.{i}"] = _mla(1024)
|
||||
specs["wide.odd"] = _mla(800)
|
||||
else:
|
||||
specs["wide.state"] = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((51_200,),),
|
||||
dtypes=(torch.bfloat16,),
|
||||
tp_replicated=True,
|
||||
)
|
||||
# Mixed and balanced, but far narrower than the bucket above.
|
||||
for i in range(10):
|
||||
specs[f"narrow.a.{i}"] = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=8, dtype=torch.uint8
|
||||
)
|
||||
specs[f"narrow.b.{i}"] = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=16, dtype=torch.uint8
|
||||
)
|
||||
for i in range(100):
|
||||
specs[f"gdn.{i}"] = MambaSpec(
|
||||
block_size=16, shapes=((512,),), dtypes=(torch.bfloat16,)
|
||||
)
|
||||
|
||||
groups = _get_packed_kv_cache_groups(config, specs)
|
||||
gdn = [g for g in groups if g.layer_names[0].startswith("gdn.")]
|
||||
|
||||
assert _get_kv_cache_bytes_per_block(groups) == sum(
|
||||
specs[name].page_size_bytes for name in specs if name.startswith("wide.")
|
||||
)
|
||||
# That block holds every GDN state at once, so the repeat pattern alone
|
||||
# decides the split; the cap must not add groups on top of it.
|
||||
assert len(gdn) == 10
|
||||
|
||||
|
||||
class TestDensePacking:
|
||||
def test_bytes_per_block_is_largest_group(self):
|
||||
groups, g1, g2 = _mixed_page_groups()
|
||||
|
||||
@@ -12,7 +12,7 @@ import pytest
|
||||
import torch
|
||||
|
||||
import vllm.v1.core.kv_cache_utils as kv_cache_utils
|
||||
from vllm.config import ModelConfig, SchedulerConfig, VllmConfig
|
||||
from vllm.config import CacheConfig, ModelConfig, SchedulerConfig, VllmConfig
|
||||
from vllm.config.kv_events import KVEventsConfig
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.multimodal.inputs import (
|
||||
@@ -37,7 +37,6 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
get_kv_cache_groups,
|
||||
get_max_concurrency_for_kv_cache_config,
|
||||
get_request_block_hasher,
|
||||
group_and_unify_kv_cache_specs,
|
||||
hash_block_tokens,
|
||||
init_none_hash,
|
||||
is_kv_cache_spec_uniform,
|
||||
@@ -2226,38 +2225,6 @@ def new_swa_mla_spec(head_size=576, sliding_window=128, model_version=None):
|
||||
)
|
||||
|
||||
|
||||
def test_group_and_unify_kv_cache_specs_no_swa_mla_returns_none():
|
||||
# Without any SlidingWindowMLASpec the function does not apply.
|
||||
specs = {"mla.0": new_mla_spec(), "mla.1": new_mla_spec()}
|
||||
assert group_and_unify_kv_cache_specs(specs) is None
|
||||
|
||||
|
||||
def test_group_and_unify_kv_cache_specs_uniform_page_size_returns_none():
|
||||
# A non-DeepseekV4 model that mixes full MLA and sliding-window MLA layers
|
||||
# with a uniform page size must not fall into the DeepseekV4 tuple-packing
|
||||
# path; it should defer to the generic uniform-page-size grouping instead.
|
||||
mla_spec = new_mla_spec()
|
||||
swa_spec = new_swa_mla_spec()
|
||||
assert mla_spec.page_size_bytes == swa_spec.page_size_bytes
|
||||
specs = {"mla.0": mla_spec, "mla.1": new_mla_spec(), "swa.0": swa_spec}
|
||||
assert group_and_unify_kv_cache_specs(specs) is None
|
||||
|
||||
|
||||
def test_group_and_unify_kv_cache_specs_mixed_page_size_groups():
|
||||
# DeepseekV4-style: differing page sizes across MLA and sliding-window MLA
|
||||
# layers do require tuple packing, so grouping must still be produced.
|
||||
mla_spec = new_mla_spec()
|
||||
swa_spec = new_swa_mla_spec(head_size=1024)
|
||||
assert mla_spec.page_size_bytes != swa_spec.page_size_bytes
|
||||
specs = {"mla.0": mla_spec, "mla.1": new_mla_spec(), "swa.0": swa_spec}
|
||||
grouped = group_and_unify_kv_cache_specs(specs)
|
||||
assert grouped is not None
|
||||
# One MLA group plus one sliding-window MLA group.
|
||||
assert len(grouped) == 2
|
||||
layer_names = {name for g in grouped for name in g.kv_cache_specs}
|
||||
assert layer_names == {"mla.0", "mla.1", "swa.0"}
|
||||
|
||||
|
||||
def new_indexer_mla_spec(block_size=16):
|
||||
# Sparse-attention indexer k_cache: an MLAAttentionSpec with a much smaller
|
||||
# page size than the main MLA attention (uint8, small head), so their pages
|
||||
@@ -2270,10 +2237,30 @@ def new_indexer_mla_spec(block_size=16):
|
||||
)
|
||||
|
||||
|
||||
def test_mixed_page_size_groups_use_spec_compatibility():
|
||||
specs = {}
|
||||
for i in range(3):
|
||||
specs[f"mla.{i}"] = new_mla_spec()
|
||||
specs[f"indexer.{i}"] = new_indexer_mla_spec()
|
||||
specs.update({f"swa.{i}": new_swa_mla_spec(head_size=1024) for i in range(5)})
|
||||
|
||||
config = _grouping_config()
|
||||
config.cache_config = CacheConfig()
|
||||
config.cache_config.kv_cache_layout = "BLNHC"
|
||||
groups = get_kv_cache_groups(config, specs)
|
||||
|
||||
assert len(groups) == 3
|
||||
assert {name for group in groups for name in group.layer_names} == set(specs)
|
||||
assert sorted(len(group.layer_names) for group in groups) == [2, 3, 6]
|
||||
|
||||
|
||||
def _grouping_config():
|
||||
cache_config = CacheConfig()
|
||||
cache_config.kv_cache_layout = "LBNHC"
|
||||
return SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
|
||||
speculative_config=None,
|
||||
cache_config=cache_config,
|
||||
)
|
||||
|
||||
|
||||
@@ -3281,10 +3268,16 @@ def test_iter_layer_specs_returns_group_members():
|
||||
assert list(iter_layer_specs(wrapped)) == [full, mla]
|
||||
|
||||
|
||||
def _spec_decode_grouping_config(method="dspark"):
|
||||
def _spec_decode_grouping_config(method="dspark", model_type=None):
|
||||
"""Grouping config with an EAGLE-family speculative method enabled."""
|
||||
return SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
|
||||
cache_config=SimpleNamespace(
|
||||
get_resolved_kv_cache_layout=lambda: SimpleNamespace(
|
||||
is_block_outermost=True
|
||||
)
|
||||
),
|
||||
model_config=SimpleNamespace(hf_config=SimpleNamespace(model_type=model_type)),
|
||||
speculative_config=SimpleNamespace(
|
||||
method=method,
|
||||
use_eagle=lambda: True,
|
||||
@@ -3322,13 +3315,12 @@ def test_draft_group_annotated_on_hybrid_general_path():
|
||||
|
||||
flagged = [g for g in groups if g.is_eagle_group]
|
||||
assert len(flagged) == 1
|
||||
assert flagged[0].layer_names == ["draft.attn.0"]
|
||||
assert "draft.attn.0" in flagged[0].layer_names
|
||||
|
||||
|
||||
def test_mamba_groups_never_flagged_even_when_draft_shares_a_group():
|
||||
# MLAAttentionSpec.merge ORs non_causal_multi_token_decode rather than
|
||||
# requiring equality, so a draft layer can share a group with target
|
||||
# layers; that combined group still holds volatile draft KV and must be
|
||||
# Packed uniform-type groups can contain distinct target and draft layer
|
||||
# specs; the combined group still holds volatile draft KV and must be
|
||||
# flagged. What must never happen is a Mamba group being flagged: that
|
||||
# widens its lookup window to two consecutive chunks, which align-mode
|
||||
# checkpointing never produces, zeroing every lookup.
|
||||
@@ -3347,10 +3339,8 @@ def test_mamba_groups_never_flagged_even_when_draft_shares_a_group():
|
||||
def test_draft_group_not_annotated_without_spec_decode():
|
||||
# The marker alone must not flag anything; the eagle semantics only apply
|
||||
# when a speculative method is actually enabled.
|
||||
config = SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
|
||||
speculative_config=None,
|
||||
)
|
||||
config = _spec_decode_grouping_config()
|
||||
config.speculative_config = None
|
||||
groups = get_kv_cache_groups(config, _hybrid_specs_with_draft(draft=True))
|
||||
|
||||
assert not any(g.is_eagle_group for g in groups)
|
||||
@@ -3396,12 +3386,13 @@ def _deepseek_v4_specs(model_version="deepseek_v4"):
|
||||
}
|
||||
|
||||
|
||||
def test_deepseek_v4_draft_group_annotated_on_group_and_unify_path():
|
||||
def test_deepseek_v4_draft_group_annotated_on_packed_path():
|
||||
# DeepseekV4's MTP block reuses the target's decoder layer, so its spec
|
||||
# carries no draft marker and only the positional rule can find it. This
|
||||
# pins the pre-existing behaviour that the unified annotator must preserve.
|
||||
groups = get_kv_cache_groups(
|
||||
_spec_decode_grouping_config(method="mtp"), _deepseek_v4_specs()
|
||||
_spec_decode_grouping_config(method="mtp", model_type="deepseek_v4"),
|
||||
_deepseek_v4_specs(model_version=None),
|
||||
)
|
||||
|
||||
flagged = [g for g in groups if g.is_eagle_group]
|
||||
@@ -3409,13 +3400,13 @@ def test_deepseek_v4_draft_group_annotated_on_group_and_unify_path():
|
||||
assert "model.layers.3.self_attn.attn" in flagged[0].layer_names
|
||||
|
||||
|
||||
def test_deepseek_v4_annotation_requires_model_version():
|
||||
def test_deepseek_v4_annotation_requires_model_type():
|
||||
# The positional rule is only sound for DeepseekV4, where the draft layer
|
||||
# is known to be registered last. Without that model gate nothing may be
|
||||
# flagged, however the grouping happens to fall out.
|
||||
groups = get_kv_cache_groups(
|
||||
_spec_decode_grouping_config(method="mtp"),
|
||||
_deepseek_v4_specs(model_version=None),
|
||||
_spec_decode_grouping_config(method="mtp", model_type="other"),
|
||||
_deepseek_v4_specs(),
|
||||
)
|
||||
|
||||
assert not any(g.is_eagle_group for g in groups)
|
||||
|
||||
@@ -1761,6 +1761,52 @@ def test_spec_decode_padding_skipped_for_diffusion():
|
||||
assert r2.request_id not in out.scheduled_spec_decode_tokens
|
||||
|
||||
|
||||
@pytest.mark.parametrize("aligned_tokens", [1, 2, 3])
|
||||
def test_spec_decode_padding_dropped_when_recurrent_alignment_clips(
|
||||
aligned_tokens: int,
|
||||
):
|
||||
"""A clipped padded request must drop its padding, not shorten it.
|
||||
|
||||
The sampler derives a request's row window from its draft count, so a
|
||||
padded request that keeps fewer than 1 + num_spec query rows would select
|
||||
the preceding request's hidden states.
|
||||
"""
|
||||
num_spec = 3
|
||||
scheduler = create_scheduler(
|
||||
num_speculative_tokens=num_spec,
|
||||
enable_prefix_caching=True,
|
||||
block_size=16,
|
||||
)
|
||||
r1, r2 = create_requests(
|
||||
num_requests=2, num_tokens=33, same_prompt=True, max_tokens=16
|
||||
)
|
||||
|
||||
scheduler.add_request(r1)
|
||||
out = scheduler.schedule()
|
||||
assert out.num_scheduled_tokens[r1.request_id] == 33
|
||||
_model_output(scheduler, out, [[100]])
|
||||
scheduler.update_draft_token_ids(DraftTokenIds([r1.request_id], [[1, 2, 3]]))
|
||||
|
||||
# Model the recurrent align split that clips a padded 1+3 prompt tail at
|
||||
# the next cache boundary. The real QSA block size produces all three cases
|
||||
# depending on prompt length modulo four.
|
||||
scheduler.need_mamba_block_aligned_split = True
|
||||
original_split = scheduler._mamba_block_aligned_split
|
||||
|
||||
def align_prompt_tail(request, num_new_tokens, *args):
|
||||
if request is r2:
|
||||
return aligned_tokens
|
||||
return original_split(request, num_new_tokens, *args)
|
||||
|
||||
scheduler._mamba_block_aligned_split = align_prompt_tail
|
||||
scheduler.add_request(r2)
|
||||
out = scheduler.schedule()
|
||||
|
||||
assert out.scheduled_spec_decode_tokens[r1.request_id] == [1, 2, 3]
|
||||
assert out.num_scheduled_tokens[r2.request_id] == 1
|
||||
assert r2.request_id not in out.scheduled_spec_decode_tokens
|
||||
|
||||
|
||||
def test_spec_decode_padding_skipped_with_prefill_in_batch():
|
||||
"""Padding is skipped when the batch contains a prefill chunk: the batch is
|
||||
already mixed/non-uniform, so padding a new decode request buys nothing.
|
||||
|
||||
@@ -14,6 +14,7 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
)
|
||||
from vllm.v1.core.single_type_kv_cache_manager import (
|
||||
ChunkedLocalAttentionManager,
|
||||
CircularBufferManager,
|
||||
FullAttentionManager,
|
||||
MambaManager,
|
||||
RSWAManager,
|
||||
@@ -21,6 +22,7 @@ from vllm.v1.core.single_type_kv_cache_manager import (
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
ChunkedLocalAttentionSpec,
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
MambaSpec,
|
||||
RSWASpec,
|
||||
@@ -136,6 +138,67 @@ def get_chunked_local_attention_manager(
|
||||
)
|
||||
|
||||
|
||||
def test_circular_buffer_allocates_one_block_for_the_request_lifetime():
|
||||
block_size = 4
|
||||
spec = CircularBufferSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=8,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
block_pool = BlockPool(
|
||||
num_gpu_blocks=10, enable_caching=True, hash_block_size=block_size
|
||||
)
|
||||
manager = CircularBufferManager(
|
||||
spec,
|
||||
block_pool=block_pool,
|
||||
enable_caching=True,
|
||||
kv_cache_group_id=0,
|
||||
scheduler_block_size=block_size,
|
||||
needs_kv_cache_zeroing=True,
|
||||
max_admission_blocks_per_request=1,
|
||||
)
|
||||
request_id = "request"
|
||||
|
||||
assert (
|
||||
manager.get_num_blocks_to_allocate(
|
||||
request_id,
|
||||
num_tokens=128,
|
||||
new_computed_blocks=[block_pool.blocks[1]],
|
||||
total_computed_tokens=0,
|
||||
num_local_computed_tokens=0,
|
||||
num_tokens_main_model=128,
|
||||
)
|
||||
== 1
|
||||
)
|
||||
blocks = manager.allocate_new_blocks(
|
||||
request_id, num_tokens=128, num_tokens_main_model=128
|
||||
)
|
||||
assert len(blocks) == 1
|
||||
assert not manager.records_new_block_ids
|
||||
assert manager.take_new_block_ids() == []
|
||||
|
||||
for num_tokens in (256, 1024):
|
||||
assert (
|
||||
manager.get_num_blocks_to_allocate(
|
||||
request_id,
|
||||
num_tokens,
|
||||
[],
|
||||
total_computed_tokens=num_tokens - 1,
|
||||
num_local_computed_tokens=0,
|
||||
num_tokens_main_model=num_tokens,
|
||||
)
|
||||
== 0
|
||||
)
|
||||
assert manager.allocate_new_blocks(request_id, num_tokens, num_tokens) == []
|
||||
|
||||
manager.cache_blocks(request_id, 1024)
|
||||
manager.remove_skipped_blocks(request_id, 1024)
|
||||
assert manager.get_num_common_prefix_blocks(request_id) == 0
|
||||
assert manager.get_num_skipped_tokens(1024) == 0
|
||||
assert manager.req_to_blocks[request_id] == blocks
|
||||
|
||||
|
||||
def test_sliding_window_records_new_blocks_for_zeroing():
|
||||
block_size = 2
|
||||
spec = SlidingWindowSpec(
|
||||
|
||||
@@ -234,7 +234,8 @@ def get_fake_execute_model_fn(original_execute_model_fn: Callable):
|
||||
iter(scheduler_output.num_scheduled_tokens.values())
|
||||
)
|
||||
assert num_scheduled_tokens == cur_step_action.num_scheduled_tokens
|
||||
mamba_group_ids, mamba_spec = get_mamba_groups(self.kv_cache_config)
|
||||
mamba_groups = get_mamba_groups(self.kv_cache_config)
|
||||
mamba_spec, mamba_group_ids = next(iter(mamba_groups.items()))
|
||||
mamba_group_id = mamba_group_ids[0]
|
||||
mamba_layer_name = self.kv_cache_config.kv_cache_groups[
|
||||
mamba_group_id
|
||||
@@ -321,7 +322,8 @@ def get_fake_process_mamba_fn(
|
||||
assert len(copy_info[0]) == len(copy_info[1]) == len(copy_info[2]) == 0
|
||||
else:
|
||||
assert len(copy_info[0]) == len(copy_info[1]) == len(copy_info[2]) == 2
|
||||
mamba_group_ids, mamba_spec = get_mamba_groups(kv_cache_config)
|
||||
mamba_groups = get_mamba_groups(kv_cache_config)
|
||||
mamba_spec, mamba_group_ids = next(iter(mamba_groups.items()))
|
||||
mamba_group_id = mamba_group_ids[0]
|
||||
mamba_layer_name = kv_cache_config.kv_cache_groups[
|
||||
mamba_group_id
|
||||
@@ -936,12 +938,12 @@ def _run_mamba_prefix_cache_mrv1(
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
@create_new_process_for_each_test("spawn")
|
||||
def test_mamba_prefix_cache_mrv1(monkeypatch: pytest.MonkeyPatch):
|
||||
_run_mamba_prefix_cache_mrv1(monkeypatch, async_scheduling=False)
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
@create_new_process_for_each_test("spawn")
|
||||
def test_mamba_prefix_cache_mrv1_async(monkeypatch: pytest.MonkeyPatch):
|
||||
_run_mamba_prefix_cache_mrv1(monkeypatch, async_scheduling=True)
|
||||
|
||||
@@ -975,11 +977,12 @@ def _run_mamba_prefix_cache_mrv2(
|
||||
forward_context = (
|
||||
model_state.vllm_config.compilation_config.static_forward_context
|
||||
)
|
||||
group_ids, _ = get_mamba_groups(kv_cache_config)
|
||||
for group_id in group_ids:
|
||||
block_table = block_tables[group_id]
|
||||
for layer_name in kv_cache_config.kv_cache_groups[group_id].layer_names:
|
||||
yield forward_context[layer_name].kv_cache[-1], block_table
|
||||
mamba_groups = get_mamba_groups(kv_cache_config)
|
||||
for group_ids in mamba_groups.values():
|
||||
for group_id in group_ids:
|
||||
block_table = block_tables[group_id]
|
||||
for layer_name in kv_cache_config.kv_cache_groups[group_id].layer_names:
|
||||
yield forward_context[layer_name].kv_cache[-1], block_table
|
||||
|
||||
def temporal_block(temporal_state, block_table, col):
|
||||
# Resolving the block id for assertions is a deliberate D2H.
|
||||
|
||||
@@ -15,6 +15,8 @@ from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
HiddenStateCacheSpec,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheLayout,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowMLASpec,
|
||||
)
|
||||
@@ -99,12 +101,12 @@ def test_get_block_size_falls_back_to_cache_config_when_no_kv_cache_config():
|
||||
assert block_size == 16
|
||||
|
||||
|
||||
# ---- MLA-verifier absorption ------------------------------------------------
|
||||
# ---- Packed MLA grouping ----------------------------------------------------
|
||||
|
||||
|
||||
def test_find_group_id_errors_clearly_when_absorbed_by_mla_swa_verifier():
|
||||
# HiddenStateCacheSpec subclasses MLAAttentionSpec, so an MLA + sliding-
|
||||
# window MLA verifier absorbs it into the MLA group instead of isolating it.
|
||||
def test_hidden_state_group_isolated_from_packed_mla_groups():
|
||||
# HiddenStateCacheSpec subclasses MLAAttentionSpec, but grouping must pull
|
||||
# it out before packing compatible MLA cache specs.
|
||||
dt = torch.bfloat16
|
||||
spec = {
|
||||
"layers.0.mla": MLAAttentionSpec(
|
||||
@@ -116,11 +118,54 @@ def test_find_group_id_errors_clearly_when_absorbed_by_mla_swa_verifier():
|
||||
"cache_only_layers.61": _hidden(64),
|
||||
}
|
||||
vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(
|
||||
get_resolved_kv_cache_layout=lambda: KVCacheLayout.BLHNC
|
||||
),
|
||||
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
|
||||
speculative_config=None,
|
||||
)
|
||||
groups = get_kv_cache_groups(vllm_config, spec)
|
||||
assert not any(isinstance(g.kv_cache_spec, HiddenStateCacheSpec) for g in groups)
|
||||
hidden_group_ids = [
|
||||
i
|
||||
for i, group in enumerate(groups)
|
||||
if isinstance(group.kv_cache_spec, HiddenStateCacheSpec)
|
||||
]
|
||||
assert len(hidden_group_ids) == 1
|
||||
cfg = SimpleNamespace(kv_cache_groups=groups)
|
||||
with pytest.raises(ValueError, match="MLA verifiers are unsupported"):
|
||||
ExampleHiddenStatesConnector._find_cache_kv_group_id(cfg)
|
||||
assert (
|
||||
ExampleHiddenStatesConnector._find_cache_kv_group_id(cfg) == hidden_group_ids[0]
|
||||
)
|
||||
|
||||
|
||||
def test_hidden_state_group_isolated_from_packed_mixed_page_groups():
|
||||
# Packed grouping keeps groups with unequal page sizes (blocks are strided
|
||||
# by the widest group), so the hidden group is appended without padding.
|
||||
hidden = _hidden(16)
|
||||
spec = {
|
||||
"layers.0.attn": _full(16),
|
||||
"layers.1.mamba": MambaSpec(
|
||||
block_size=16, shapes=((1024,),), dtypes=(torch.float32,)
|
||||
),
|
||||
"cache_only_layers.61": hidden,
|
||||
}
|
||||
vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(
|
||||
get_resolved_kv_cache_layout=lambda: KVCacheLayout.BLHNC
|
||||
),
|
||||
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
|
||||
speculative_config=None,
|
||||
)
|
||||
groups = get_kv_cache_groups(vllm_config, spec)
|
||||
page_sizes = {g.kv_cache_spec.page_size_bytes for g in groups}
|
||||
assert len(page_sizes) > 1, "spec must exercise the packed grouping path"
|
||||
hidden_group_ids = [
|
||||
i
|
||||
for i, group in enumerate(groups)
|
||||
if isinstance(group.kv_cache_spec, HiddenStateCacheSpec)
|
||||
]
|
||||
assert len(hidden_group_ids) == 1
|
||||
assert groups[hidden_group_ids[0]].kv_cache_spec == hidden
|
||||
cfg = SimpleNamespace(kv_cache_groups=groups)
|
||||
assert (
|
||||
ExampleHiddenStatesConnector._find_cache_kv_group_id(cfg) == hidden_group_ids[0]
|
||||
)
|
||||
|
||||
@@ -865,6 +865,10 @@ def _make_mock_worker_for_desc_ids(
|
||||
worker._group_spec_types = group_spec_types
|
||||
worker.block_len_per_layer = block_len_per_layer or [100]
|
||||
worker._conv_decomp = None
|
||||
worker._is_csa_linear = False
|
||||
worker._ssm_region_indices = [0] if has_mamba else []
|
||||
worker._ple_group_index = None
|
||||
worker._ple_region_index = None
|
||||
if has_mamba:
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import ( # noqa: E501
|
||||
MambaConvSplitInfo,
|
||||
@@ -1823,6 +1827,7 @@ def test_push_write_hybrid_mla_replicates_attention():
|
||||
worker.shutdown = lambda: None # skeleton worker: silence __del__
|
||||
worker.use_mla = True
|
||||
worker._has_mamba = True
|
||||
worker._is_csa_linear = False
|
||||
worker._group_spec_types = (MLAAttentionSpec, MambaSpec)
|
||||
worker.transfer_topo = MagicMock()
|
||||
worker.transfer_topo.tp_ratio.return_value = -2
|
||||
|
||||
@@ -645,3 +645,302 @@ def test_mismatched_mla_kernel_page_rejected_for_mla_hybrid():
|
||||
meta_r.block_lens = [x // 2 for x in worker.block_len_per_layer]
|
||||
with pytest.raises((AssertionError, RuntimeError)):
|
||||
worker.add_remote_agent(meta_r, remote_tp_rank=0, remote_tp_size=2)
|
||||
|
||||
|
||||
def _make_csa_linear_ple_worker(scratch_aliases: str = "compressed"):
|
||||
"""``scratch_aliases`` selects which pages the compressor ring overlays."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker as bw
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
|
||||
NixlConnectorWorker,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
|
||||
main_kv_specs = {
|
||||
f"main_kv.{index}": FullAttentionSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=1,
|
||||
head_size=16,
|
||||
dtype=torch.float16,
|
||||
)
|
||||
for index in range(2)
|
||||
}
|
||||
compressed_specs = {
|
||||
f"compressed.{index}": MLAAttentionSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=1,
|
||||
head_size=64,
|
||||
dtype=torch.float16,
|
||||
tokens_per_state=2,
|
||||
)
|
||||
for index in range(2)
|
||||
}
|
||||
compressor_state_specs = {
|
||||
f"compressor_state.{index}": CircularBufferSpec(
|
||||
block_size=2,
|
||||
num_kv_heads=1,
|
||||
head_size=32,
|
||||
dtype=torch.float16,
|
||||
)
|
||||
for index in range(2)
|
||||
}
|
||||
sharded_mamba_spec = MambaSpec(
|
||||
block_size=1,
|
||||
shapes=((8, 3), (1, 4, 4)),
|
||||
dtypes=(torch.float16, torch.float32),
|
||||
page_size_padded=256,
|
||||
mamba_type=MambaAttentionBackendEnum.GDN_ATTN,
|
||||
)
|
||||
ple_spec = MambaSpec(
|
||||
block_size=1,
|
||||
shapes=((6, 3),),
|
||||
dtypes=(torch.float16,),
|
||||
page_size_padded=256,
|
||||
mamba_type=MambaAttentionBackendEnum.SHORT_CONV,
|
||||
tp_replicated=True,
|
||||
)
|
||||
compressed_sparse_specs = {
|
||||
name: spec
|
||||
for index in range(2)
|
||||
for name, spec in (
|
||||
(f"main_kv.{index}", main_kv_specs[f"main_kv.{index}"]),
|
||||
(f"compressed.{index}", compressed_specs[f"compressed.{index}"]),
|
||||
)
|
||||
}
|
||||
|
||||
def group(specs):
|
||||
uniform = UniformTypeKVCacheSpecs.from_specs(specs)
|
||||
assert uniform is not None
|
||||
return KVCacheGroupSpec(list(specs), uniform)
|
||||
|
||||
tensor_regions: tuple[tuple[str, ...], ...]
|
||||
if scratch_aliases == "compressed":
|
||||
tensor_regions = (
|
||||
("main_kv.0", "mamba.sharded", "mamba.ple"),
|
||||
("main_kv.1",),
|
||||
("compressed.0", "compressor_state.0"),
|
||||
("compressed.1", "compressor_state.1"),
|
||||
)
|
||||
else:
|
||||
assert scratch_aliases == "main_kv"
|
||||
tensor_regions = (
|
||||
("main_kv.0", "mamba.sharded", "mamba.ple", "compressor_state.0"),
|
||||
("main_kv.1", "compressor_state.1"),
|
||||
("compressed.0",),
|
||||
("compressed.1",),
|
||||
)
|
||||
region_size = 512
|
||||
page_size = 256
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=2,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=len(tensor_regions) * region_size,
|
||||
layers=[layer_name],
|
||||
layer_stride=region_size,
|
||||
block_stride=page_size,
|
||||
offset=region_index * region_size,
|
||||
)
|
||||
for region_index, layer_names in enumerate(tensor_regions)
|
||||
for layer_name in layer_names
|
||||
],
|
||||
kv_cache_groups=[
|
||||
group(compressed_sparse_specs),
|
||||
group(compressor_state_specs),
|
||||
KVCacheGroupSpec(["mamba.sharded"], sharded_mamba_spec),
|
||||
KVCacheGroupSpec(["mamba.ple"], ple_spec),
|
||||
],
|
||||
)
|
||||
|
||||
vllm_config = create_vllm_config(block_size=4)
|
||||
vllm_config.cache_config.enable_prefix_caching = False
|
||||
vllm_config.kv_transfer_config.kv_buffer_device = "cuda"
|
||||
fake_backend = MagicMock()
|
||||
fake_backend.get_supported_kernel_block_sizes.return_value = [4]
|
||||
fake_backend.get_name.return_value = "TEST_ATTN"
|
||||
fake_backend.full_cls_name.return_value = "test.AttentionBackend"
|
||||
fake_platform = MagicMock()
|
||||
fake_platform.device_type = "cuda"
|
||||
fake_platform.get_nixl_memory_type.return_value = "VRAM"
|
||||
|
||||
with (
|
||||
patch.object(bw, "NixlWrapper", _RecordingNixl),
|
||||
patch.object(bw, "get_tensor_model_parallel_rank", return_value=0),
|
||||
patch.object(bw, "get_tensor_model_parallel_world_size", return_value=1),
|
||||
patch.object(bw, "get_current_attn_backends", return_value=[fake_backend]),
|
||||
patch.object(bw, "current_platform", fake_platform),
|
||||
patch(
|
||||
"vllm.model_executor.layers.mamba.mamba_utils.get_conv_state_layout",
|
||||
return_value="DS",
|
||||
),
|
||||
set_current_vllm_config(vllm_config),
|
||||
):
|
||||
worker = NixlConnectorWorker(vllm_config, "local-engine", kv_cache_config)
|
||||
tensors = [torch.zeros((2, 256), dtype=torch.uint8) for _ in range(4)]
|
||||
worker.register_kv_caches(
|
||||
{
|
||||
layer_name: tensors[region_index]
|
||||
for region_index, layer_names in enumerate(tensor_regions)
|
||||
for layer_name in layer_names
|
||||
}
|
||||
)
|
||||
return worker
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_csa_linear_registration_discovers_shared_regions():
|
||||
worker = _make_csa_linear_ple_worker()
|
||||
|
||||
assert worker._ssm_region_indices == [0]
|
||||
assert worker._ple_group_index == 3
|
||||
assert worker._ple_region_index == 0
|
||||
assert worker._region_is_mla == [False, False, True, True]
|
||||
assert worker._scratch_region_indices == [2, 3]
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_csa_linear_scratch_descs_follow_the_pages_they_overlay():
|
||||
"""The scratch ring is addressed by the regions it actually registered in,
|
||||
not by inferring that it must sit on the MLA pages: overlaying the main KV
|
||||
pages instead moves its descriptors with it."""
|
||||
worker = _make_csa_linear_ple_worker(scratch_aliases="main_kv")
|
||||
|
||||
# Same MLA regions as before, but the ring now lives in the non-MLA ones.
|
||||
assert worker._region_is_mla == [False, False, True, True]
|
||||
assert worker._scratch_region_indices == [0, 1]
|
||||
|
||||
desc_ids = worker._compute_desc_ids(
|
||||
block_ids=([1], [0], [1], [1]),
|
||||
dst_num_blocks=2,
|
||||
block_size_ratio=None,
|
||||
physical_blocks_per_logical=1,
|
||||
)
|
||||
# Scratch group (block 0) now maps to regions 0 and 1 -> descs 0 and 2,
|
||||
# where the compressed-page placement gave 4 and 6.
|
||||
assert desc_ids.tolist()[4:6] == [0, 2]
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_csa_linear_selects_compressed_sparse_compressor_state_and_mamba_regions():
|
||||
worker = _make_csa_linear_ple_worker()
|
||||
|
||||
desc_ids = worker._compute_desc_ids(
|
||||
block_ids=([1], [0], [1], [1]),
|
||||
dst_num_blocks=2,
|
||||
block_size_ratio=None,
|
||||
physical_blocks_per_logical=1,
|
||||
)
|
||||
assert desc_ids.tolist() == [
|
||||
1,
|
||||
3,
|
||||
5,
|
||||
7,
|
||||
4,
|
||||
6,
|
||||
9,
|
||||
11,
|
||||
13,
|
||||
15,
|
||||
17,
|
||||
]
|
||||
|
||||
mamba_only = worker._compute_desc_ids(
|
||||
block_ids=([], [], [1], [1]),
|
||||
dst_num_blocks=2,
|
||||
block_size_ratio=None,
|
||||
physical_blocks_per_logical=1,
|
||||
)
|
||||
assert mamba_only.tolist() == [9, 11, 13, 15, 17]
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_csa_linear_ple_descriptor_is_not_split():
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
|
||||
TPMapping,
|
||||
)
|
||||
|
||||
worker = _make_csa_linear_ple_worker()
|
||||
bases = [0x10000, 0x20000, 0x30000, 0x40000]
|
||||
attention = worker._build_fa_local(bases, block_size_ratio=1)
|
||||
mamba = worker._build_mamba_local(bases)
|
||||
assert attention.shape == (8, 3)
|
||||
assert mamba.shape == (10, 3)
|
||||
assert mamba[-2:, 0].tolist() == [0x10000, 0x10100]
|
||||
assert mamba[-2:, 1].tolist() == [256, 256]
|
||||
|
||||
mapping = TPMapping(
|
||||
source_ranks_per_group=((0,), (0,), (0, 1), (0, 1)),
|
||||
all_source_ranks=(0, 1),
|
||||
rank_to_attention_slot={0: 0, 1: 0},
|
||||
rank_offset_factor=0,
|
||||
)
|
||||
splits = list(
|
||||
worker._build_local_splits_from_plan(
|
||||
mapping,
|
||||
np.concatenate([attention, mamba]),
|
||||
num_fa_descs=len(attention),
|
||||
)
|
||||
)
|
||||
assert len(splits) == 2
|
||||
assert [row[1] for row in splits[0][-2:]] == [256, 256]
|
||||
assert [row[1] for row in splits[1][-2:]] == [256, 256]
|
||||
assert splits[0][-3][1] * 2 == mamba[-3, 1]
|
||||
assert splits[1][-3][1] * 2 == mamba[-3, 1]
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_csa_linear_remote_ple_is_copied_whole():
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
NixlAgentMetadata,
|
||||
)
|
||||
|
||||
worker = _make_csa_linear_ple_worker()
|
||||
metadata = NixlAgentMetadata(
|
||||
engine_id="remote",
|
||||
agent_metadata=b"agent",
|
||||
kv_caches_base_addr=[
|
||||
0x10000,
|
||||
0x20000,
|
||||
0x30000,
|
||||
0x40000,
|
||||
],
|
||||
device_id=0,
|
||||
num_blocks=2,
|
||||
block_lens=[256] * 4,
|
||||
block_strides=[256] * 4,
|
||||
kv_cache_layout="HND",
|
||||
block_size=4,
|
||||
ssm_sizes=(24, 32),
|
||||
attn_backend_name="test",
|
||||
physical_blocks_per_logical_kv_block=1,
|
||||
)
|
||||
descriptors = worker._build_mamba_remote(
|
||||
metadata,
|
||||
tp_ratio=-2,
|
||||
transfer_info=SimpleNamespace(remote_physical_blocks_per_logical=1),
|
||||
)
|
||||
assert descriptors.shape == (10, 3)
|
||||
assert descriptors[-2:, 0].tolist() == [0x10000, 0x10100]
|
||||
assert descriptors[-2:, 1].tolist() == [256, 256]
|
||||
|
||||
metadata.block_lens[0] = 128
|
||||
with pytest.raises(ValueError, match="PLE pages require identical"):
|
||||
worker._build_mamba_remote(
|
||||
metadata,
|
||||
tp_ratio=-2,
|
||||
transfer_info=SimpleNamespace(remote_physical_blocks_per_logical=1),
|
||||
)
|
||||
|
||||
@@ -359,6 +359,7 @@ class _StubWriterWorker(NixlPushConnectorWorker):
|
||||
w._physical_blocks_per_logical_kv_block = 1
|
||||
# Single non-hybrid attention group, matching the stub block id lists.
|
||||
w._has_mamba = False
|
||||
w._is_csa_linear = False
|
||||
w._group_spec_types = (FullAttentionSpec,)
|
||||
w._engine_ttl = 0.0
|
||||
w._engine_last_active = {}
|
||||
@@ -1198,6 +1199,75 @@ class TestPushWriterMlaReplication:
|
||||
# engine thread never sees a partial set and double-frees the request.
|
||||
assert sorted(w._sending_transfers["p-req"]) == [1000, 1001]
|
||||
|
||||
def test_csa_linear_writes_every_group_to_each_target(self):
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
RemoteMeta,
|
||||
ReqMeta,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
|
||||
TPMapping,
|
||||
)
|
||||
|
||||
engine_id = "decode-engine"
|
||||
w = _StubWriterWorker.fresh()
|
||||
w.world_size = 2
|
||||
w.use_mla = False
|
||||
w._has_mamba = True
|
||||
w._is_csa_linear = True
|
||||
w.nixl_wrapper = MagicMock()
|
||||
w.transfer_topo = MagicMock()
|
||||
w.transfer_topo.get_engine_info.return_value = SimpleNamespace(
|
||||
remote_tp_size=4,
|
||||
remote_block_size=16,
|
||||
remote_physical_blocks_per_logical=1,
|
||||
)
|
||||
w.transfer_topo.tp_ratio.return_value = -2
|
||||
w.transfer_topo.handshake_target_ranks.return_value = (0, 1)
|
||||
w.tp_mappings = {
|
||||
engine_id: TPMapping(
|
||||
source_ranks_per_group=((0,), (0,), (0, 1), (0, 1)),
|
||||
all_source_ranks=(0, 1),
|
||||
rank_to_attention_slot={0: 0, 1: 0},
|
||||
rank_offset_factor=0,
|
||||
)
|
||||
}
|
||||
w._logical_to_kernel_block_ids = lambda block_ids, ratio: block_ids
|
||||
w.dst_xfer_side_handles = {engine_id: {0: 1000, 1: 1001}}
|
||||
w.src_xfer_handles_by_tp_ratio = {(-2, 16): [2000, 2001]}
|
||||
|
||||
writes = []
|
||||
|
||||
def _fake_xfer(**kwargs):
|
||||
writes.append(kwargs)
|
||||
return 3000 + kwargs["read_spec"].remote_rank
|
||||
|
||||
w._xfer_blocks = _fake_xfer
|
||||
block_ids = ([10], [20], [30], [40])
|
||||
meta = ReqMeta(
|
||||
local_block_ids=block_ids,
|
||||
local_physical_block_ids=block_ids,
|
||||
tp_size=2,
|
||||
remote=RemoteMeta(
|
||||
block_ids=([50], [60], [70], [80]),
|
||||
host="",
|
||||
port=0,
|
||||
engine_id=engine_id,
|
||||
request_id="d-req",
|
||||
),
|
||||
)
|
||||
|
||||
w._xfer_blocks_for_req(req_id="p-req", meta=meta)
|
||||
|
||||
assert [write["read_spec"].remote_rank for write in writes] == [0, 1]
|
||||
assert [write["read_spec"].local_block_ids for write in writes] == [
|
||||
list(block_ids),
|
||||
list(block_ids),
|
||||
]
|
||||
assert [write["local_xfer_side_handle"] for write in writes] == [2000, 2001]
|
||||
assert sorted(w._sending_transfers["p-req"]) == [3000, 3001]
|
||||
|
||||
|
||||
class TestPushPrefixCaching:
|
||||
"""Partial prefix-cache hit on D: D preallocates only its *uncomputed*
|
||||
|
||||
@@ -154,6 +154,9 @@ def _make_mock_worker_for_splits(group_spec_types):
|
||||
worker.block_len_per_layer = []
|
||||
worker.num_regions = 0
|
||||
worker._region_is_mla = []
|
||||
worker._conv_decomp = SimpleNamespace(local_conv_offsets=())
|
||||
worker._ssm_region_indices = [0] if MambaSpec in group_spec_types else []
|
||||
worker._ple_region_index = None
|
||||
return worker
|
||||
|
||||
|
||||
@@ -282,3 +285,31 @@ class TestMambaPlanSplitHandles:
|
||||
|
||||
with pytest.raises(AssertionError, match="Head-sharded"):
|
||||
list(worker._build_local_splits_from_plan(plan, src_blocks_data, 2, 2))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("total_kv_heads", "local_tp", "remote_tp", "compatible"),
|
||||
[
|
||||
# 2 heads: TP1 packs both heads into a page, TP >= 2 packs one.
|
||||
(2, 1, 2, False),
|
||||
(2, 2, 1, False),
|
||||
(2, 1, 1, True),
|
||||
(2, 2, 4, True),
|
||||
(2, 4, 2, True),
|
||||
# 4 heads: the boundary moves with the head count.
|
||||
(4, 2, 4, False),
|
||||
(4, 2, 2, True),
|
||||
(4, 4, 8, True),
|
||||
],
|
||||
)
|
||||
def test_csa_linear_tp_layout_boundary(total_kv_heads, local_tp, remote_tp, compatible):
|
||||
worker = object.__new__(NixlConnectorWorker)
|
||||
worker.world_size = local_tp
|
||||
worker._is_csa_linear = True
|
||||
worker.transfer_topo = SimpleNamespace(total_num_kv_heads=total_kv_heads)
|
||||
|
||||
if compatible:
|
||||
worker._validate_csa_linear_tp_layout(remote_tp)
|
||||
else:
|
||||
with pytest.raises(ValueError, match="KV-head sharding boundary"):
|
||||
worker._validate_csa_linear_tp_layout(remote_tp)
|
||||
|
||||
@@ -104,6 +104,47 @@ def _make_speculator(
|
||||
return speculator
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("hc_mult", "expected"), [(None, 64), (4, 256)])
|
||||
def test_speculator_uses_draft_model_hidden_size(monkeypatch, hc_mult, expected):
|
||||
# Qwen4Exp targets expose multi-stream HC residuals to the drafter.
|
||||
monkeypatch.setattr(base_spec_module, "_target_feeds_hc_residual", lambda _: True)
|
||||
hf_config = SimpleNamespace()
|
||||
if hc_mult is not None:
|
||||
hf_config.hc_mult = hc_mult
|
||||
draft_model_config = SimpleNamespace(
|
||||
hf_config=hf_config,
|
||||
get_hidden_size=lambda: 64,
|
||||
get_vocab_size=lambda: 32,
|
||||
)
|
||||
speculative_config = SimpleNamespace(
|
||||
method="mtp",
|
||||
num_speculative_tokens=3,
|
||||
draft_model_config=draft_model_config,
|
||||
use_local_argmax_reduction=False,
|
||||
draft_sample_method="greedy",
|
||||
)
|
||||
vllm_config = SimpleNamespace(
|
||||
speculative_config=speculative_config,
|
||||
scheduler_config=SimpleNamespace(
|
||||
max_num_seqs=2,
|
||||
max_num_batched_tokens=8,
|
||||
),
|
||||
model_config=SimpleNamespace(
|
||||
max_model_len=32,
|
||||
dtype=torch.float32,
|
||||
use_fp64_gumbel=False,
|
||||
),
|
||||
parallel_config=SimpleNamespace(
|
||||
data_parallel_size=1,
|
||||
data_parallel_rank=0,
|
||||
),
|
||||
)
|
||||
|
||||
speculator = _TestSpeculator(vllm_config, torch.device("cpu"))
|
||||
|
||||
assert speculator.hidden_size == expected
|
||||
|
||||
|
||||
def test_mm_support_configured_after_model_load(monkeypatch):
|
||||
target_model_config = object()
|
||||
draft_model_config = object()
|
||||
@@ -197,9 +238,11 @@ def test_load_model_disables_mm_support_for_text_only_drafter(monkeypatch):
|
||||
|
||||
assert not speculator.supports_mm_inputs
|
||||
assert warning_messages == [
|
||||
"Draft model _TextOnlyDraftModel does not support external multimodal "
|
||||
"embeddings. Embeddings from the target model will not be passed to the "
|
||||
"drafter; using text-only draft inputs instead."
|
||||
(
|
||||
"Draft model _TextOnlyDraftModel does not support external multimodal "
|
||||
"embeddings. Embeddings from the target model will not be passed to the "
|
||||
"drafter; using text-only draft inputs instead."
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -132,6 +132,42 @@ def test_block_tables_apply_staged_writes_single_group():
|
||||
)
|
||||
|
||||
|
||||
def test_block_tables_skip_custom_slot_mapping_groups():
|
||||
device = torch.device("cuda")
|
||||
block_tables = BlockTables(
|
||||
block_sizes=[8, 262144],
|
||||
max_num_reqs=1,
|
||||
max_num_batched_tokens=4,
|
||||
max_num_blocks_per_group=[1, 1],
|
||||
device=device,
|
||||
kernel_block_sizes=[8, 262144],
|
||||
slot_mapping_enabled=[False, True],
|
||||
)
|
||||
block_tables.append_block_ids(
|
||||
req_index=0,
|
||||
new_block_ids=([7], [12]),
|
||||
overwrite=True,
|
||||
)
|
||||
block_tables.apply_staged_writes()
|
||||
|
||||
idx_mapping = torch.tensor([0], dtype=torch.int32, device=device)
|
||||
query_start_loc = torch.tensor([0, 2], dtype=torch.int32, device=device)
|
||||
positions = torch.tensor([153797, 165757], dtype=torch.int64, device=device)
|
||||
slot_mappings = block_tables.compute_slot_mappings(
|
||||
idx_mapping,
|
||||
query_start_loc,
|
||||
positions,
|
||||
num_tokens_padded=2,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
assert slot_mappings[0].tolist() == [-1, -1]
|
||||
assert slot_mappings[1].tolist() == [
|
||||
12 * 262144 + 153797,
|
||||
12 * 262144 + 165757,
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cp_rank", range(4))
|
||||
def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
|
||||
"""DCP interleave is expressed in logical-block token coordinates."""
|
||||
|
||||
@@ -8,15 +8,107 @@ import torch
|
||||
|
||||
import vllm.v1.worker.gpu.model_runner as model_runner_module
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
|
||||
|
||||
|
||||
def test_qsa_circular_group_uses_custom_slot_mapping(monkeypatch):
|
||||
runner = GPUModelRunner.__new__(GPUModelRunner)
|
||||
runner.max_model_len = 262144
|
||||
runner.is_encoder_decoder = False
|
||||
runner.dcp_size = 1
|
||||
runner.dcp_rank = 0
|
||||
runner.cp_interleave = 1
|
||||
runner.cache_config = SimpleNamespace(enable_prefix_caching=True)
|
||||
parallel_config = SimpleNamespace(
|
||||
decode_context_parallel_size=1,
|
||||
cp_kv_cache_interleave_size=1,
|
||||
)
|
||||
runner.parallel_config = parallel_config
|
||||
runner.vllm_config = SimpleNamespace(
|
||||
parallel_config=parallel_config,
|
||||
cache_config=SimpleNamespace(mamba_cache_mode="none"),
|
||||
)
|
||||
runner.model_state = SimpleNamespace(
|
||||
get_additional_cg_support=lambda: (),
|
||||
num_new_sampled_tokens_per_step=1,
|
||||
)
|
||||
runner.speculator = None
|
||||
runner.req_states = []
|
||||
runner.input_buffers = SimpleNamespace(query_start_loc=None)
|
||||
runner.vocab_size = 1
|
||||
runner.max_num_reqs = 1
|
||||
runner.max_num_tokens = 2
|
||||
runner.device = torch.device("cuda")
|
||||
|
||||
raw_spec = CircularBufferSpec(
|
||||
block_size=8,
|
||||
num_kv_heads=1,
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
compressed_spec = FullAttentionSpec(
|
||||
block_size=262144,
|
||||
num_kv_heads=1,
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=1,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
layer_names=["raw"],
|
||||
kv_cache_spec=UniformTypeKVCacheSpecs(
|
||||
block_size=8,
|
||||
kv_cache_specs={"raw": raw_spec},
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(layer_names=["compressed"], kv_cache_spec=compressed_spec),
|
||||
],
|
||||
)
|
||||
|
||||
class FakeAttnCGSupport:
|
||||
def narrow(self, *args):
|
||||
return self
|
||||
|
||||
attn_cg_support = FakeAttnCGSupport()
|
||||
monkeypatch.setattr(
|
||||
model_runner_module,
|
||||
"init_attn_backend",
|
||||
lambda *args: ([], attn_cg_support, [8, 262144]),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
model_runner_module,
|
||||
"maybe_create_adaptive_verification_manager",
|
||||
lambda **kwargs: None,
|
||||
)
|
||||
|
||||
captured = {}
|
||||
|
||||
class BlockTablesCaptured(Exception):
|
||||
pass
|
||||
|
||||
def capture_block_tables(**kwargs):
|
||||
captured.update(kwargs)
|
||||
raise BlockTablesCaptured
|
||||
|
||||
monkeypatch.setattr(model_runner_module, "BlockTables", capture_block_tables)
|
||||
|
||||
with pytest.raises(BlockTablesCaptured):
|
||||
runner.initialize_kv_cache(kv_cache_config)
|
||||
|
||||
assert captured["max_num_blocks_per_group"] == [1, 1]
|
||||
assert captured["slot_mapping_enabled"] == [False, True]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mamba_cache_mode", "num_speculative_blocks", "expected"),
|
||||
[
|
||||
|
||||
@@ -20,6 +20,7 @@ from vllm.config.vllm import VllmConfig
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
@@ -53,6 +54,18 @@ def _attention_group() -> KVCacheGroupSpec:
|
||||
)
|
||||
|
||||
|
||||
def _circular_group() -> KVCacheGroupSpec:
|
||||
return KVCacheGroupSpec(
|
||||
["circular"],
|
||||
CircularBufferSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _mamba_group(mamba_cache_mode: str) -> KVCacheGroupSpec:
|
||||
# Name carries the mode so several groups can coexist in one config.
|
||||
return KVCacheGroupSpec(
|
||||
@@ -212,8 +225,7 @@ def test_warmup_reserves_mamba_speculative_blocks(mamba_cache_mode):
|
||||
|
||||
|
||||
def _hybrid_kv_cache_config(num_blocks: int) -> KVCacheConfig:
|
||||
"""Full attention plus one Mamba group per cache mode, so a single manager
|
||||
exercises every branch `_reserved_block_count` has.
|
||||
"""Attention, circular, and Mamba groups exercise every reservation branch.
|
||||
|
||||
"none" and "all" reach the same branch of both `_reserved_block_count` and
|
||||
`MambaManager`, which tests only for "align". They are still both listed:
|
||||
@@ -226,6 +238,7 @@ def _hybrid_kv_cache_config(num_blocks: int) -> KVCacheConfig:
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
_attention_group(),
|
||||
_circular_group(),
|
||||
_mamba_group("none"),
|
||||
_mamba_group("all"),
|
||||
_mamba_group("align"),
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
@@ -10,11 +9,20 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateCopyFunc,
|
||||
MambaStateCopyFuncsByType,
|
||||
get_conv_copy_spec,
|
||||
get_temporal_copy_spec,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheGroupSpec, MambaSpec
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
|
||||
from vllm.v1.worker.mamba_utils import (
|
||||
MambaCopyBuffers,
|
||||
MambaSpecDecodeGPUContext,
|
||||
@@ -22,17 +30,21 @@ from vllm.v1.worker.mamba_utils import (
|
||||
batch_memcpy,
|
||||
collect_mamba_copy_meta,
|
||||
do_mamba_copy_block,
|
||||
get_mamba_groups,
|
||||
preprocess_mamba,
|
||||
stage_postprocess_inputs_to_gpu,
|
||||
)
|
||||
|
||||
MambaStateCopyFunc = Callable[..., Any]
|
||||
|
||||
# Conv + temporal copy specs, in the order the tests' MambaSpec shapes expect.
|
||||
_COPY_FUNCS: tuple[MambaStateCopyFunc, ...] = (
|
||||
_DEFAULT_COPY_FUNCS: tuple[MambaStateCopyFunc, ...] = (
|
||||
get_conv_copy_spec,
|
||||
get_temporal_copy_spec,
|
||||
)
|
||||
_COPY_FUNCS: MambaStateCopyFuncsByType = {
|
||||
MambaAttentionBackendEnum.MAMBA2: _DEFAULT_COPY_FUNCS,
|
||||
MambaAttentionBackendEnum.GDN_ATTN: _DEFAULT_COPY_FUNCS,
|
||||
MambaAttentionBackendEnum.SHORT_CONV: (get_conv_copy_spec,),
|
||||
}
|
||||
|
||||
|
||||
def postprocess_mamba(
|
||||
@@ -41,7 +53,7 @@ def postprocess_mamba(
|
||||
input_batch: Any,
|
||||
requests: dict[str, Any],
|
||||
forward_context: dict[str, Any],
|
||||
mamba_state_copy_funcs: tuple[MambaStateCopyFunc, ...],
|
||||
mamba_state_copy_funcs: MambaStateCopyFuncsByType,
|
||||
copy_bufs: "MambaCopyBuffers",
|
||||
):
|
||||
"""CPU reference for the align-mode postprocess.
|
||||
@@ -147,7 +159,7 @@ def test_resumed_req_ids_cleared_from_mamba_state_idx():
|
||||
input_batch,
|
||||
{}, # requests
|
||||
{}, # forward_context
|
||||
(), # mamba_state_copy_funcs
|
||||
{}, # mamba_state_copy_funcs
|
||||
copy_bufs,
|
||||
)
|
||||
|
||||
@@ -430,7 +442,10 @@ def _make_requests(
|
||||
|
||||
|
||||
def _make_copy_bufs(
|
||||
cfg: _TestConfig, kv_cache_config: KVCacheConfig, device: torch.device
|
||||
cfg: _TestConfig,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
device: torch.device,
|
||||
copy_funcs: MambaStateCopyFuncsByType = _COPY_FUNCS,
|
||||
) -> MambaCopyBuffers:
|
||||
"""Create MambaCopyBuffers for the Python path."""
|
||||
|
||||
@@ -440,13 +455,16 @@ def _make_copy_bufs(
|
||||
return MambaCopyBuffers.create(
|
||||
max_num_reqs=cfg.max_num_reqs,
|
||||
kv_cache_config=kv_cache_config,
|
||||
copy_funcs=(get_conv_copy_spec, get_temporal_copy_spec),
|
||||
copy_funcs=copy_funcs,
|
||||
make_buffer=make_buffer,
|
||||
)
|
||||
|
||||
|
||||
def _make_gpu_ctx(
|
||||
cfg: _TestConfig, kv_cache_config: KVCacheConfig, device: torch.device
|
||||
cfg: _TestConfig,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
device: torch.device,
|
||||
copy_funcs: MambaStateCopyFuncsByType = _COPY_FUNCS,
|
||||
) -> MambaSpecDecodeGPUContext:
|
||||
"""Create MambaSpecDecodeGPUContext for the GPU path."""
|
||||
|
||||
@@ -456,12 +474,170 @@ def _make_gpu_ctx(
|
||||
return MambaSpecDecodeGPUContext.create(
|
||||
max_num_reqs=cfg.max_num_reqs,
|
||||
kv_cache_config=kv_cache_config,
|
||||
num_state_types=2,
|
||||
copy_funcs=copy_funcs,
|
||||
device=device,
|
||||
make_buffer=make_buffer,
|
||||
)
|
||||
|
||||
|
||||
def test_mamba_groups_support_different_state_specs():
|
||||
gdn_spec = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((4, 4), (2, 4, 4)),
|
||||
dtypes=(torch.float16, torch.float32),
|
||||
mamba_type=MambaAttentionBackendEnum.GDN_ATTN,
|
||||
mamba_cache_mode="align",
|
||||
num_speculative_blocks=3,
|
||||
)
|
||||
short_conv_spec = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((12, 8),),
|
||||
dtypes=(torch.float16,),
|
||||
mamba_type=MambaAttentionBackendEnum.SHORT_CONV,
|
||||
mamba_cache_mode="align",
|
||||
num_speculative_blocks=3,
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=8,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["gdn.0", "gdn.1"], gdn_spec),
|
||||
KVCacheGroupSpec(["ple.0"], short_conv_spec),
|
||||
],
|
||||
)
|
||||
|
||||
mamba_groups = get_mamba_groups(kv_cache_config)
|
||||
assert mamba_groups == {gdn_spec: [0], short_conv_spec: [1]}
|
||||
|
||||
model_state = object.__new__(MambaHybridModelState)
|
||||
model_state._mamba_group_ids = []
|
||||
model_state._mamba_spec = None
|
||||
group_ids, representative_spec = model_state._get_mamba_group_info(kv_cache_config)
|
||||
assert group_ids == [0, 1]
|
||||
assert representative_spec is gdn_spec
|
||||
|
||||
cfg = _TestConfig()
|
||||
copy_bufs = _make_copy_bufs(cfg, kv_cache_config, torch.device("cpu"))
|
||||
assert copy_bufs.src_ptrs.cpu.numel() == cfg.max_num_reqs * 5
|
||||
assert copy_bufs.dst_ptrs.cpu.numel() == cfg.max_num_reqs * 5
|
||||
assert copy_bufs.sizes.cpu.numel() == cfg.max_num_reqs * 5
|
||||
|
||||
ctx = _make_gpu_ctx(cfg, kv_cache_config, torch.device("cpu"))
|
||||
assert ctx.num_states == 5
|
||||
|
||||
forward_context = {}
|
||||
gdn_spec_layer_names = ["gdn.0", "gdn.1"]
|
||||
for layer_name in gdn_spec_layer_names:
|
||||
attention = MagicMock()
|
||||
attention.kv_cache = (
|
||||
torch.empty(8, 4, 4, dtype=torch.float16),
|
||||
torch.empty(8, 2, 4, 4, dtype=torch.float32),
|
||||
)
|
||||
forward_context[layer_name] = attention
|
||||
ple_attention = MagicMock()
|
||||
ple_attention.kv_cache = (torch.empty(8, 12, 8, dtype=torch.float16),)
|
||||
forward_context["ple.0"] = ple_attention
|
||||
assert gdn_spec_layer_names == kv_cache_config.kv_cache_groups[0].layer_names
|
||||
|
||||
block_tables = [
|
||||
torch.zeros(4, 8, dtype=torch.int32),
|
||||
torch.zeros(4, 8, dtype=torch.int32),
|
||||
]
|
||||
ctx.initialize_from_forward_context(
|
||||
kv_cache_config,
|
||||
forward_context,
|
||||
_COPY_FUNCS,
|
||||
block_tables,
|
||||
)
|
||||
assert ctx.is_initialized
|
||||
assert ctx.state_group_indices.tolist() == [0, 0, 0, 0, 1]
|
||||
assert ctx.state_conv_widths.tolist() == [4, 0, 4, 0, 12]
|
||||
|
||||
|
||||
def test_mamba_groups_support_mixed_specs_in_uniform_group():
|
||||
gdn_spec = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((4, 4), (2, 4, 4)),
|
||||
dtypes=(torch.float16, torch.float32),
|
||||
mamba_type=MambaAttentionBackendEnum.GDN_ATTN,
|
||||
mamba_cache_mode="align",
|
||||
num_speculative_blocks=3,
|
||||
)
|
||||
short_conv_spec = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((12, 8),),
|
||||
dtypes=(torch.float16,),
|
||||
mamba_type=MambaAttentionBackendEnum.SHORT_CONV,
|
||||
mamba_cache_mode="align",
|
||||
num_speculative_blocks=3,
|
||||
)
|
||||
layer_specs = {
|
||||
"gdn.0": gdn_spec,
|
||||
"gdn.1": gdn_spec,
|
||||
"ple.0": short_conv_spec,
|
||||
}
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=8,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
list(layer_specs),
|
||||
UniformTypeKVCacheSpecs(
|
||||
block_size=16,
|
||||
kv_cache_specs=layer_specs,
|
||||
),
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assert kv_cache_config.has_mamba_layers
|
||||
assert get_mamba_groups(kv_cache_config) == {
|
||||
gdn_spec: [0],
|
||||
short_conv_spec: [0],
|
||||
}
|
||||
|
||||
# The group is listed once per spec, so its id must not be repeated.
|
||||
model_state = object.__new__(MambaHybridModelState)
|
||||
model_state._mamba_group_ids = []
|
||||
model_state._mamba_spec = None
|
||||
group_ids, representative_spec = model_state._get_mamba_group_info(kv_cache_config)
|
||||
assert group_ids == [0]
|
||||
assert representative_spec is gdn_spec
|
||||
|
||||
cfg = _TestConfig()
|
||||
copy_bufs = _make_copy_bufs(cfg, kv_cache_config, torch.device("cpu"))
|
||||
assert copy_bufs.mamba_group_ids == [0]
|
||||
assert copy_bufs.src_ptrs.cpu.numel() == cfg.max_num_reqs * 5
|
||||
|
||||
ctx = _make_gpu_ctx(cfg, kv_cache_config, torch.device("cpu"))
|
||||
assert ctx.mamba_group_ids == [0]
|
||||
assert ctx.num_states == 5
|
||||
|
||||
forward_context = {
|
||||
"gdn.0": MagicMock(
|
||||
kv_cache=(
|
||||
torch.empty(8, 4, 4, dtype=torch.float16),
|
||||
torch.empty(8, 2, 4, 4, dtype=torch.float32),
|
||||
)
|
||||
),
|
||||
"gdn.1": MagicMock(
|
||||
kv_cache=(
|
||||
torch.empty(8, 4, 4, dtype=torch.float16),
|
||||
torch.empty(8, 2, 4, 4, dtype=torch.float32),
|
||||
)
|
||||
),
|
||||
"ple.0": MagicMock(kv_cache=(torch.empty(8, 12, 8, dtype=torch.float16),)),
|
||||
}
|
||||
ctx.initialize_from_forward_context(
|
||||
kv_cache_config,
|
||||
forward_context,
|
||||
_COPY_FUNCS,
|
||||
[torch.zeros(4, 8, dtype=torch.int32)],
|
||||
)
|
||||
assert ctx.state_group_indices.tolist() == [0, 0, 0, 0, 0]
|
||||
assert ctx.state_conv_widths.tolist() == [4, 0, 4, 0, 12]
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# stage_postprocess_inputs_to_gpu: single-pass staging into pinned views
|
||||
# -----------------------------------------------------------------------------
|
||||
@@ -620,7 +796,7 @@ def _run_gpu_postprocess(
|
||||
*,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
forward_context: dict[str, Any],
|
||||
copy_funcs: tuple,
|
||||
copy_funcs: MambaStateCopyFuncsByType,
|
||||
block_table: torch.Tensor,
|
||||
req_ids: list[str],
|
||||
num_accepted_tokens: list[int],
|
||||
|
||||
@@ -2817,6 +2817,7 @@ def fused_gdn_decode_post_conv_mtp(
|
||||
out: torch.Tensor | None = None,
|
||||
scale: float = 128**-0.5,
|
||||
norm_eps: float = 1e-5,
|
||||
output_gate_activation: str = "silu",
|
||||
) -> torch.Tensor:
|
||||
if out is None:
|
||||
out = torch.empty_like(output_gate)
|
||||
@@ -2835,6 +2836,7 @@ def fused_gdn_decode_post_conv_mtp(
|
||||
out,
|
||||
scale,
|
||||
norm_eps,
|
||||
output_gate_activation,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@@ -768,6 +768,9 @@ class CompilationConfig:
|
||||
"vllm::mamba_mixer2",
|
||||
"vllm::mamba_mixer",
|
||||
"vllm::short_conv",
|
||||
"vllm::qwen4_exp_compute_ple_ngram_ids",
|
||||
"vllm::qwen4_exp_ple_short_conv",
|
||||
"vllm::qwen4_exp_qsa_with_output",
|
||||
"vllm::linear_attention",
|
||||
"vllm::qwen_gdn_attention_core",
|
||||
"vllm::qwen_gdn_attention_core_fused_norm_packed",
|
||||
|
||||
@@ -215,6 +215,10 @@ class SchedulerConfig:
|
||||
# https://github.com/vllm-project/vllm/issues/29585
|
||||
factors.append(self.max_num_batched_tokens)
|
||||
|
||||
# PLE and other model components allocate static per-request buffers.
|
||||
# Their shapes are captured in compiled graphs.
|
||||
factors.append(self.max_num_seqs)
|
||||
|
||||
hash_str = safe_hash(str(factors).encode(), usedforsecurity=False).hexdigest()
|
||||
return hash_str
|
||||
|
||||
|
||||
@@ -47,6 +47,7 @@ MTPModelTypes = Literal[
|
||||
"exaone_moe_mtp",
|
||||
"exaone4_5_mtp",
|
||||
"qwen3_next_mtp",
|
||||
"qwen4_exp_mtp",
|
||||
"qwen3_5_mtp",
|
||||
"longcat_flash_mtp",
|
||||
"bailing_hybrid_v3_mtp",
|
||||
@@ -425,6 +426,9 @@ class SpeculativeConfig:
|
||||
"""The specific revision to use for the draft model code on Hugging Face
|
||||
Hub. It can be a branch name, a tag name, or a commit id. If unspecified,
|
||||
will use the default version."""
|
||||
index_share_for_mtp_iteration: bool | None = None
|
||||
"""Override whether MTP iterations reuse the first step's sparse indices.
|
||||
If `None`, use the value from the draft model's Hugging Face config."""
|
||||
|
||||
# Advanced control
|
||||
disable_padded_drafter_batch: bool = False
|
||||
@@ -623,6 +627,15 @@ class SpeculativeConfig:
|
||||
# Convert to tuple to make it hashable
|
||||
factors.append(tuple(layer_ids))
|
||||
|
||||
if self.method == "mtp" and self.draft_model_config is not None:
|
||||
factors.append(
|
||||
getattr(
|
||||
self.draft_model_config.hf_config,
|
||||
"index_share_for_mtp_iteration",
|
||||
False,
|
||||
)
|
||||
)
|
||||
|
||||
hash_str = safe_hash(str(factors).encode(), usedforsecurity=False).hexdigest()
|
||||
return hash_str
|
||||
|
||||
@@ -805,6 +818,27 @@ class SpeculativeConfig:
|
||||
hf_config.update(
|
||||
{"n_predict": n_predict, "architectures": ["Qwen3NextMTP"]}
|
||||
)
|
||||
if hf_config.model_type in {"qwen4_exp", "qwen4_exp_text"}:
|
||||
hf_config.model_type = "qwen4_exp_mtp"
|
||||
if hf_config.model_type == "qwen4_exp_mtp":
|
||||
text_config = get_hf_text_config(hf_config)
|
||||
n_predict = getattr(
|
||||
text_config,
|
||||
"mtp_num_hidden_layers",
|
||||
getattr(text_config, "num_nextn_predict_layers", None),
|
||||
)
|
||||
share_mtp_indices = getattr(
|
||||
text_config, "index_share_for_mtp_iteration", False
|
||||
)
|
||||
hf_config.update(
|
||||
{
|
||||
# hc_count is the HC stream multiplier for Qwen MTP feedback.
|
||||
"hc_mult": int(text_config.hc_count),
|
||||
"n_predict": n_predict,
|
||||
"architectures": ["Qwen4ExpMTP"],
|
||||
"index_share_for_mtp_iteration": share_mtp_indices,
|
||||
}
|
||||
)
|
||||
|
||||
architectures = getattr(hf_config, "architectures", []) or []
|
||||
if initial_architecture == "BailingMoeV3ForCausalLM":
|
||||
@@ -1461,6 +1495,15 @@ class SpeculativeConfig:
|
||||
)
|
||||
)
|
||||
|
||||
if self.index_share_for_mtp_iteration is not None:
|
||||
if self.method != "mtp" or self.draft_model_config is None:
|
||||
raise ValueError(
|
||||
"index_share_for_mtp_iteration is only supported with method='mtp'"
|
||||
)
|
||||
self.draft_model_config.hf_config.index_share_for_mtp_iteration = (
|
||||
self.index_share_for_mtp_iteration
|
||||
)
|
||||
|
||||
if self.method != "dspark" and self.enable_adaptive_verification:
|
||||
raise ValueError("Adaptive verification only supported with DSpark")
|
||||
|
||||
|
||||
@@ -405,6 +405,8 @@ class CustomAllreduce:
|
||||
def should_custom_ar(self, inp: torch.Tensor):
|
||||
if self.disabled or self.world_size > 8:
|
||||
return False
|
||||
if inp.dtype not in (torch.float32, torch.float16, torch.bfloat16):
|
||||
return False
|
||||
inp_size = inp.numel() * inp.element_size()
|
||||
# custom allreduce requires input byte size to be multiples of 16
|
||||
if inp_size % 16 != 0:
|
||||
|
||||
@@ -72,12 +72,15 @@ from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.network_utils import make_zmq_path
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KVCacheLayout,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowMLASpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
iter_layer_specs,
|
||||
)
|
||||
from vllm.v1.worker.block_table import BlockTable
|
||||
from vllm.v1.worker.utils import select_common_block_size
|
||||
@@ -118,7 +121,12 @@ class NixlBaseConnectorWorker:
|
||||
# NIXL regions per SSM layer = conv sub-projections + 1 SSM temporal
|
||||
# (Mamba2/GDN: 3+1=4; Mamba1: 1+1=2).
|
||||
ssm_regions_per_layer = len(self._conv_decomp.local_conv_offsets) + 1
|
||||
num_ssm_regions = len(self.block_len_per_layer) * ssm_regions_per_layer
|
||||
# Count only the regions that actually hold SSM state; must match
|
||||
# the descriptors emitted by _build_mamba_local.
|
||||
num_ssm_regions = (
|
||||
len(self._ssm_region_indices or self.block_len_per_layer)
|
||||
* ssm_regions_per_layer
|
||||
)
|
||||
|
||||
num_blocks = dst_num_blocks
|
||||
if block_size_ratio is not None:
|
||||
@@ -146,12 +154,19 @@ class NixlBaseConnectorWorker:
|
||||
all_descs: list[np.ndarray] = []
|
||||
for i, group in enumerate(block_ids):
|
||||
group_arr = np.asarray(group)
|
||||
if _is_attention_spec(self._group_spec_types[i]):
|
||||
fa_region_ids = np.arange(self.num_regions)[:, None]
|
||||
spec_type = self._group_spec_types[i]
|
||||
if _is_attention_spec(spec_type):
|
||||
# A scratch cache lives only in its own regions; every other
|
||||
# attention group spans all of them.
|
||||
fa_region_ids = (
|
||||
np.asarray(self._scratch_region_indices, dtype=np.int64)
|
||||
if spec_type is CircularBufferSpec
|
||||
else np.arange(self.num_regions)
|
||||
)[:, None]
|
||||
all_descs.append(
|
||||
(fa_region_ids * num_blocks + group_arr[None, :]).flatten()
|
||||
)
|
||||
elif _is_ssm_spec(self._group_spec_types[i]):
|
||||
elif _is_ssm_spec(spec_type):
|
||||
# NOTE (NickLucche) SSM and Attention block regions can
|
||||
# be exchanged arbitrarily by manager. Therefore, descs
|
||||
# are laid out as:
|
||||
@@ -159,7 +174,11 @@ class NixlBaseConnectorWorker:
|
||||
# num_fa_descs offset must be computed per-engine since
|
||||
# P and D can have different num_blocks (and thus
|
||||
# different FA desc counts).
|
||||
ssm_region_ids = np.arange(num_ssm_regions)[:, None]
|
||||
ssm_region_ids = (
|
||||
np.arange(num_ssm_regions, num_ssm_regions + 1)
|
||||
if i == self._ple_group_index
|
||||
else np.arange(num_ssm_regions)
|
||||
)[:, None]
|
||||
all_descs.append(
|
||||
(
|
||||
ssm_region_ids * logical_blocks
|
||||
@@ -206,6 +225,10 @@ class NixlBaseConnectorWorker:
|
||||
|
||||
# Per-FA-descriptor replicate flag, in _build_fa_local emission order.
|
||||
fa_desc_replicated = self._fa_desc_replicated(num_fa_descs)
|
||||
sharded_desc_end = len(src_blocks_data) - (
|
||||
self._logical_num_blocks if self._ple_region_index is not None else 0
|
||||
)
|
||||
assert num_fa_descs <= sharded_desc_end
|
||||
|
||||
assert block_size_ratio == 1 or fa_num_splits == 1 or all(fa_desc_replicated), (
|
||||
"Head-sharded attention reads with P_TP > D_TP and heterogeneous "
|
||||
@@ -226,9 +249,11 @@ class NixlBaseConnectorWorker:
|
||||
# SPLIT (full-attn): this rank's head slice.
|
||||
chunk = local_len // fa_num_splits
|
||||
handle.append((addr + fa_slot * chunk, chunk, dev))
|
||||
else:
|
||||
elif j < sharded_desc_end:
|
||||
chunk = local_len // ssm_num_splits
|
||||
handle.append((addr + p_idx * chunk, chunk, dev))
|
||||
else:
|
||||
handle.append((addr, local_len, dev))
|
||||
yield handle
|
||||
|
||||
def _needs_split_local_xfer_handles(self, tp_ratio: int, plan: TPMapping) -> bool:
|
||||
@@ -307,19 +332,22 @@ class NixlBaseConnectorWorker:
|
||||
)
|
||||
)
|
||||
|
||||
self.kv_cache_config = kv_cache_config
|
||||
# Per-layer specs, unwrapping UniformTypeKVCacheSpecs group wrappers.
|
||||
self._layer_specs: dict[str, KVCacheSpec] = {}
|
||||
for group in kv_cache_config.transfer_groups:
|
||||
group_spec = group.kv_cache_spec
|
||||
if isinstance(group_spec, UniformTypeKVCacheSpecs):
|
||||
self._layer_specs.update(group_spec.kv_cache_specs)
|
||||
else:
|
||||
self._layer_specs.update(dict.fromkeys(group.layer_names, group_spec))
|
||||
self._is_hma_required = (
|
||||
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
|
||||
and any(
|
||||
not isinstance(g.kv_cache_spec, FullAttentionSpec)
|
||||
for g in kv_cache_config.transfer_groups
|
||||
not isinstance(spec, FullAttentionSpec)
|
||||
for spec in self._layer_specs.values()
|
||||
)
|
||||
)
|
||||
self.kv_cache_config = kv_cache_config
|
||||
self._layer_specs = {
|
||||
layer: group.kv_cache_spec
|
||||
for group in kv_cache_config.transfer_groups
|
||||
for layer in group.layer_names
|
||||
}
|
||||
|
||||
# ---- Model state (derived from model config) ----
|
||||
mamba_ssm_size = (0, 0)
|
||||
@@ -328,9 +356,29 @@ class NixlBaseConnectorWorker:
|
||||
# conv sub-projections are contiguous in memory.
|
||||
self._conv_decomp: MambaConvSplitInfo | None = None
|
||||
self._has_mamba = any(
|
||||
isinstance(g.kv_cache_spec, MambaSpec)
|
||||
for g in kv_cache_config.transfer_groups
|
||||
isinstance(spec, MambaSpec) for spec in self._layer_specs.values()
|
||||
)
|
||||
self._is_csa_linear = any(
|
||||
get_representative_spec_type(group.kv_cache_spec) is CircularBufferSpec
|
||||
for group in kv_cache_config.kv_cache_groups
|
||||
)
|
||||
self._ple_group_index: int | None = None
|
||||
if self._is_csa_linear:
|
||||
ple_groups = [
|
||||
(index, group)
|
||||
for index, group in enumerate(kv_cache_config.kv_cache_groups)
|
||||
if group.layer_names
|
||||
and all(
|
||||
isinstance(spec, MambaSpec) and spec.tp_replicated
|
||||
for spec in iter_layer_specs(group.kv_cache_spec)
|
||||
)
|
||||
]
|
||||
if len(ple_groups) != 1 or len(ple_groups[0][1].layer_names) != 1:
|
||||
raise ValueError(
|
||||
"CSA-linear NIXL requires exactly one PLE cache owner."
|
||||
)
|
||||
self._ple_group_index = ple_groups[0][0]
|
||||
|
||||
if self._has_mamba:
|
||||
assert self._is_hma_required
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
@@ -345,6 +393,7 @@ class NixlBaseConnectorWorker:
|
||||
spec
|
||||
for spec in self._layer_specs.values()
|
||||
if isinstance(spec, MambaSpec)
|
||||
and (not self._is_csa_linear or not spec.tp_replicated)
|
||||
)
|
||||
self._conv_decomp = derive_mamba_conv_split(
|
||||
mamba_spec,
|
||||
@@ -579,6 +628,12 @@ class NixlBaseConnectorWorker:
|
||||
# (MLA), False -> SPLIT (head-sharded full-attn). Mixed only for models
|
||||
# combining both (e.g. GQA main + MLA Eagle-3 draft).
|
||||
self._region_is_mla = list[bool]()
|
||||
self._ssm_region_indices = list[int]()
|
||||
# Regions holding a scratch cache (the CSA compressor circular buffer).
|
||||
# Tracked explicitly because a scratch page shares its address with the
|
||||
# page it overlays, so its regions cannot be recovered from spec type.
|
||||
self._scratch_region_indices = list[int]()
|
||||
self._ple_region_index: int | None = None
|
||||
|
||||
# Enable different block lengths for different layers *only* when MLA is used.
|
||||
# This is not used for SSM layers, which use the counterpart `mamba_ssm_size`.
|
||||
@@ -632,6 +687,24 @@ class NixlBaseConnectorWorker:
|
||||
self.block_size = kernel_block_size
|
||||
self.num_blocks *= self._physical_blocks_per_logical_kv_block
|
||||
|
||||
def _validate_csa_linear_tp_layout(self, remote_tp_size: int) -> None:
|
||||
"""Reject P/D pairs whose main-KV pages have different head layouts.
|
||||
|
||||
A rank at TP < total KV heads holds several heads per page, while a
|
||||
rank at TP >= total KV heads holds exactly one (replicated beyond
|
||||
that). CSA-linear pages are shared tensors that transfer as whole
|
||||
block units, so endpoints on opposite sides of that boundary lay
|
||||
them out differently and cannot exchange them.
|
||||
"""
|
||||
assert self.transfer_topo is not None
|
||||
total_kv_heads = self.transfer_topo.total_num_kv_heads
|
||||
if (self.world_size < total_kv_heads) != (remote_tp_size < total_kv_heads):
|
||||
raise ValueError(
|
||||
"CSA-linear NIXL requires both endpoints on the same side of "
|
||||
f"the KV-head sharding boundary ({total_kv_heads} KV heads): "
|
||||
f"got local TP {self.world_size}, remote TP {remote_tp_size}."
|
||||
)
|
||||
|
||||
def _nixl_handshake(
|
||||
self,
|
||||
host: str,
|
||||
@@ -643,6 +716,8 @@ class NixlBaseConnectorWorker:
|
||||
notif_agents_only: bool = False,
|
||||
) -> tuple[dict[tuple[int, int], str], float]:
|
||||
"""Do a NIXL handshake with a remote instance."""
|
||||
if self._is_csa_linear:
|
||||
self._validate_csa_linear_tp_layout(remote_tp_size)
|
||||
|
||||
# the first time we connect to a remote agent.
|
||||
# be careful, the handshake happens in a background thread.
|
||||
@@ -1034,6 +1109,11 @@ class NixlBaseConnectorWorker:
|
||||
transfer_mode=self._TRANSFER_MODE,
|
||||
)
|
||||
|
||||
if self._is_csa_linear and self.use_host_buffer:
|
||||
raise NotImplementedError(
|
||||
"NIXL host staging does not preserve CSA-linear shared tensors."
|
||||
)
|
||||
|
||||
if self.use_host_buffer:
|
||||
self.initialize_host_xfer_buffer(kv_caches=kv_caches)
|
||||
assert len(self.host_xfer_buffers) == len(kv_caches), (
|
||||
@@ -1059,8 +1139,14 @@ class NixlBaseConnectorWorker:
|
||||
caches_data = []
|
||||
seen_storage_addresses: set[int] = set()
|
||||
seen_base_addresses: list[int] = []
|
||||
self._ssm_region_indices = []
|
||||
self._scratch_region_indices = []
|
||||
self._ple_region_index = None
|
||||
|
||||
packed_storage = _share_storage_and_block_stride(list(xfer_buffers.values()))
|
||||
# CSA-linear needs separate logical regions for attention and state
|
||||
# aliases even though every view shares one block-major allocation.
|
||||
packed_storage = packed_storage and not self._is_csa_linear
|
||||
|
||||
# K and V are packed into the content dim, so each attention layer is a
|
||||
# single NIXL region whose block transfers as one unit. Mamba layers instead
|
||||
@@ -1112,14 +1198,16 @@ class NixlBaseConnectorWorker:
|
||||
)
|
||||
|
||||
if isinstance(layer_spec, MambaSpec):
|
||||
physical_ratio = self._physical_blocks_per_logical_kv_block
|
||||
block_len = physical_page_size // physical_ratio
|
||||
block_stride = physical_page_size
|
||||
if self._is_csa_linear and cache.ndim != 1:
|
||||
# CSA-linear packs multiple cache owners in each block, so
|
||||
# state views advance by the packed block stride.
|
||||
block_stride = cache.stride(0) * cache.element_size()
|
||||
assert block_stride % physical_ratio == 0
|
||||
region_specs = [
|
||||
(
|
||||
cache.data_ptr(),
|
||||
physical_page_size
|
||||
// self._physical_blocks_per_logical_kv_block,
|
||||
physical_page_size
|
||||
// self._physical_blocks_per_logical_kv_block,
|
||||
)
|
||||
(cache.data_ptr(), block_len, block_stride // physical_ratio)
|
||||
]
|
||||
else:
|
||||
if cache.ndim == 1:
|
||||
@@ -1131,19 +1219,18 @@ class NixlBaseConnectorWorker:
|
||||
block_stride = cache.stride(0) * cache.element_size()
|
||||
storage_is_block_major = num_blocks * block_stride == storage.nbytes()
|
||||
# A layer whose [H, N, C] interior is dense addresses its own page
|
||||
# as one chunk, so it can own a region. When it does not, or when
|
||||
# every layer is packed into this one storage, the block's whole
|
||||
# row is the only contiguous unit.
|
||||
# as one chunk. Otherwise the block's whole row is the only
|
||||
# contiguous transfer unit.
|
||||
hnc_contiguous = (
|
||||
cache.ndim == 4
|
||||
and cache.stride(2) == cache.shape[3]
|
||||
and cache.stride(1) == cache.shape[2] * cache.shape[3]
|
||||
)
|
||||
if storage_is_block_major and (
|
||||
(packed_storage and is_mla_region) or not hnc_contiguous
|
||||
(packed_storage and is_mla_region)
|
||||
or (not hnc_contiguous and not self._is_csa_linear)
|
||||
):
|
||||
# TODO(Lucas): handle TP slicing for packed_storage; for now
|
||||
# restrict to MLA (DSv4) where kv is replicated.
|
||||
# Packed MLA layouts transfer the complete storage row.
|
||||
storage_block_len = storage.nbytes() // num_blocks
|
||||
region_specs = [
|
||||
(storage_addr, storage_block_len, storage_block_len)
|
||||
@@ -1170,17 +1257,27 @@ class NixlBaseConnectorWorker:
|
||||
|
||||
for base_addr, block_len, block_stride in region_specs:
|
||||
if base_addr in seen_base_addresses:
|
||||
if is_mla_region:
|
||||
# Dual-purpose HMA tensor: an MLA layer shares a region
|
||||
# that a non-MLA layer registered first. MLA is not
|
||||
# head-sharded, so the region must be flagged MLA.
|
||||
idx = seen_base_addresses.index(base_addr)
|
||||
self._region_is_mla[idx] = True
|
||||
continue
|
||||
seen_base_addresses.append(base_addr)
|
||||
self.block_len_per_layer.append(block_len)
|
||||
self.block_stride_per_layer.append(block_stride)
|
||||
self._region_is_mla.append(is_mla_region)
|
||||
region_index = seen_base_addresses.index(base_addr)
|
||||
self._region_is_mla[region_index] |= is_mla_region
|
||||
else:
|
||||
region_index = len(seen_base_addresses)
|
||||
seen_base_addresses.append(base_addr)
|
||||
self.block_len_per_layer.append(block_len)
|
||||
self.block_stride_per_layer.append(block_stride)
|
||||
self._region_is_mla.append(is_mla_region)
|
||||
|
||||
if isinstance(layer_spec, MambaSpec):
|
||||
if layer_spec.tp_replicated:
|
||||
assert self._is_csa_linear
|
||||
assert self._ple_region_index in (None, region_index)
|
||||
self._ple_region_index = region_index
|
||||
elif region_index not in self._ssm_region_indices:
|
||||
self._ssm_region_indices.append(region_index)
|
||||
elif (
|
||||
isinstance(layer_spec, CircularBufferSpec)
|
||||
and region_index not in self._scratch_region_indices
|
||||
):
|
||||
self._scratch_region_indices.append(region_index)
|
||||
|
||||
# When there's a mismatch between kbs<>bs, we rely on HMA to ensure
|
||||
# caches are either [NB, PS] or [NB*r, PS/r] where r is bs/kbs.
|
||||
@@ -1210,6 +1307,8 @@ class NixlBaseConnectorWorker:
|
||||
== len(self._region_is_mla)
|
||||
== len(self.block_stride_per_layer)
|
||||
)
|
||||
# Descriptor ids must be region-ordered, matching the remote side.
|
||||
self._scratch_region_indices.sort()
|
||||
|
||||
self.kv_caches_base_addr[self.engine_id][self.tp_rank] = seen_base_addresses
|
||||
self.num_regions = len(seen_base_addresses)
|
||||
@@ -1324,17 +1423,26 @@ class NixlBaseConnectorWorker:
|
||||
physical_per_logical = self._physical_blocks_per_logical_kv_block
|
||||
device_id = self.device_id
|
||||
block_arange = np.arange(num_blocks, dtype=np.uint64)
|
||||
|
||||
parts: list[np.ndarray] = []
|
||||
for i, base_addr in enumerate(base_addresses):
|
||||
# Jump one page_size, but ssm page_size may be bigger when kernel
|
||||
# locks block size to a specific value (physical_per_logical scale).
|
||||
page_stride = self.block_len_per_layer[i] * physical_per_logical
|
||||
blk_addrs = base_addr + block_arange * page_stride
|
||||
|
||||
region_indices = self._ssm_region_indices or range(len(base_addresses))
|
||||
for i in region_indices:
|
||||
base_addr = base_addresses[i]
|
||||
block_stride = self.block_stride_per_layer[i] * physical_per_logical
|
||||
blk_addrs = base_addr + block_arange * block_stride
|
||||
for off, sz in conv_offsets:
|
||||
parts.append(self._stack_descs(blk_addrs + off, sz, device_id))
|
||||
# SSM temporal state follows the conv state.
|
||||
parts.append(self._stack_descs(blk_addrs + conv_size, ssm_size, device_id))
|
||||
|
||||
if (region_index := self._ple_region_index) is not None:
|
||||
block_len = self.block_len_per_layer[region_index] * physical_per_logical
|
||||
block_stride = (
|
||||
self.block_stride_per_layer[region_index] * physical_per_logical
|
||||
)
|
||||
block_addrs = base_addresses[region_index] + block_arange * block_stride
|
||||
parts.append(self._stack_descs(block_addrs, block_len, self.device_id))
|
||||
|
||||
return np.concatenate(parts)
|
||||
|
||||
def _build_mamba_remote(
|
||||
@@ -1370,14 +1478,44 @@ class NixlBaseConnectorWorker:
|
||||
parts: list[np.ndarray] = []
|
||||
# NOTE (ZhanqiuHu): use per-layer block_lens[i], not [0], in case
|
||||
# block lengths vary across layers (e.g. MLA).
|
||||
for i, base_addr in enumerate(nixl_agent_meta.kv_caches_base_addr):
|
||||
page_stride = nixl_agent_meta.block_lens[i] * remote_physical_per_logical
|
||||
blk_addrs = base_addr + block_arange * page_stride
|
||||
region_indices = self._ssm_region_indices or range(
|
||||
len(nixl_agent_meta.kv_caches_base_addr)
|
||||
)
|
||||
for i in region_indices:
|
||||
base_addr = nixl_agent_meta.kv_caches_base_addr[i]
|
||||
block_stride = (
|
||||
nixl_agent_meta.block_strides[i] * remote_physical_per_logical
|
||||
)
|
||||
blk_addrs = base_addr + block_arange * block_stride
|
||||
for off, sz in conv_offsets:
|
||||
parts.append(self._stack_descs(blk_addrs + off, sz, device_id))
|
||||
# SSM temporal state is also TP-sharded on the heads dimension.
|
||||
ssm_addrs = blk_addrs + conv_size_remote + local_offset * ssm_read_size
|
||||
parts.append(self._stack_descs(ssm_addrs, ssm_read_size, device_id))
|
||||
|
||||
if (region_index := self._ple_region_index) is not None:
|
||||
local_block_len = (
|
||||
self.block_len_per_layer[region_index]
|
||||
* self._physical_blocks_per_logical_kv_block
|
||||
)
|
||||
remote_block_len = (
|
||||
nixl_agent_meta.block_lens[region_index] * remote_physical_per_logical
|
||||
)
|
||||
if local_block_len != remote_block_len:
|
||||
raise ValueError(
|
||||
"PLE pages require identical P/D geometry: "
|
||||
f"local={local_block_len}, remote={remote_block_len}."
|
||||
)
|
||||
remote_block_stride = (
|
||||
nixl_agent_meta.block_strides[region_index]
|
||||
* remote_physical_per_logical
|
||||
)
|
||||
block_addrs = (
|
||||
nixl_agent_meta.kv_caches_base_addr[region_index]
|
||||
+ block_arange * remote_block_stride
|
||||
)
|
||||
parts.append(self._stack_descs(block_addrs, remote_block_len, device_id))
|
||||
|
||||
return np.concatenate(parts)
|
||||
|
||||
@staticmethod
|
||||
@@ -2410,8 +2548,7 @@ class NixlBaseConnectorWorker:
|
||||
group_specs = self.kv_cache_config.transfer_groups
|
||||
physical_block_ids = []
|
||||
for i, group in enumerate(block_ids):
|
||||
spec = group_specs[i].kv_cache_spec
|
||||
if isinstance(spec, MambaSpec):
|
||||
if _is_ssm_spec(get_representative_spec_type(group_specs[i].kv_cache_spec)):
|
||||
physical_block_ids.append(group)
|
||||
else:
|
||||
physical_block_ids.append(
|
||||
|
||||
@@ -536,7 +536,8 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
def group_ids(block_ids: BlockIds, rank: int) -> BlockIds:
|
||||
return [
|
||||
list(block_ids[g])
|
||||
if (replicate_attn and _is_attention_spec(self._group_spec_types[g]))
|
||||
if (self._is_csa_linear and tp_ratio < 0)
|
||||
or (replicate_attn and _is_attention_spec(self._group_spec_types[g]))
|
||||
or rank in plan.source_ranks_per_group[g]
|
||||
else []
|
||||
for g in range(num_groups)
|
||||
|
||||
@@ -39,6 +39,7 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
is_deep_gemm_e8m0_used,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
FP8_DTYPE,
|
||||
QuantKey,
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8DynamicTensorSym,
|
||||
@@ -153,6 +154,19 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
|
||||
(kFp8StaticTensorSym, kFp8StaticTensorSym),
|
||||
(kFp8StaticTensorSym, kFp8DynamicTensorSym),
|
||||
]
|
||||
# Block-quantized FP8 with arbitrary block shape: the Triton
|
||||
# kernels take the block shape as a runtime argument.
|
||||
if (
|
||||
weight_key is not None
|
||||
and activation_key == kFp8Dynamic128Sym
|
||||
and weight_key.dtype == FP8_DTYPE
|
||||
and weight_key.symmetric
|
||||
and weight_key.scale.static
|
||||
and weight_key.scale.dtype == torch.float32
|
||||
and weight_key.scale.group_shape.row > 1
|
||||
and weight_key.scale.group_shape.col > 1
|
||||
):
|
||||
return True
|
||||
return (weight_key, activation_key) in supported
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import math
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
|
||||
@@ -268,6 +269,39 @@ def map_fp8_backend(runner_backend: MoEBackend) -> Fp8MoeBackend:
|
||||
)
|
||||
|
||||
|
||||
def refine_fp8_moe_block_shape(
|
||||
config: FusedMoEConfig,
|
||||
weight_block_size: list[int],
|
||||
) -> list[int] | None:
|
||||
"""
|
||||
Compute a refined block shape for block-quantized FP8 MoE weights whose
|
||||
checkpoint blocks cannot be sharded exactly across TP ranks.
|
||||
|
||||
TP shards the intermediate dim of the expert weights, so a per-shard size
|
||||
that is not a multiple of the checkpoint's block size makes the
|
||||
checkpoint's block scales impossible to shard exactly. When a finer block
|
||||
size (>= 32) divides both the checkpoint blocks and all involved dims,
|
||||
the weight scales can be refined to that granularity at load time (a
|
||||
lossless upsampling, since the refined block divides the checkpoint
|
||||
block). Only Triton-based kernels can consume the refined block shape:
|
||||
they take it as a runtime argument, while the other backends require the
|
||||
native 128x128 blocks. The refined shape is encoded in the QuantKey used
|
||||
for backend selection, so backends that only support 128x128 blocks are
|
||||
rejected by the oracle automatically.
|
||||
|
||||
Returns the refined [block_n, block_k] shape, or None if no refinement
|
||||
is needed or possible.
|
||||
"""
|
||||
block_n, block_k = weight_block_size
|
||||
ispp = config.intermediate_size_per_partition
|
||||
if ispp % block_n == 0 and (config.tp_size == 1 or ispp % block_k == 0):
|
||||
return None
|
||||
refine = math.gcd(block_n, block_k, ispp, config.hidden_dim)
|
||||
if refine < 32:
|
||||
return None
|
||||
return [refine, refine]
|
||||
|
||||
|
||||
def select_fp8_moe_backend(
|
||||
config: FusedMoEConfig,
|
||||
weight_key: QuantKey | None,
|
||||
|
||||
@@ -474,6 +474,39 @@ def _get_requested_backends(
|
||||
return _filter_by_activation(backends, requested_activation_key)
|
||||
|
||||
|
||||
def _requires_qwen38_tep8_emulation(
|
||||
config: FusedMoEConfig,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
"""Avoid the inaccurate native gfx950 kernel for Qwen3.8 TEP8.
|
||||
|
||||
Qwen3.8 Flash Next's routed experts use the distinctive
|
||||
``E=512, H=2560, N=640`` W4A4 shape. With eight-way expert parallelism,
|
||||
AITER operates on 64 local experts and pads ``N`` to 768. That native path
|
||||
is not numerically reliable on gfx950, while OCP MX emulation preserves
|
||||
model accuracy. Keep this guard exact so other MXFP4 shapes and smaller EP
|
||||
configurations retain the native backend.
|
||||
"""
|
||||
parallel = config.moe_parallel_config
|
||||
if not (
|
||||
activation_key == kMxfp4Dynamic
|
||||
and parallel.use_ep
|
||||
and parallel.ep_size == 8
|
||||
and config.num_experts == 512
|
||||
and config.num_local_experts == 64
|
||||
and config.hidden_dim == 2560
|
||||
and config.intermediate_size == 640
|
||||
):
|
||||
return False
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
return on_gfx950()
|
||||
|
||||
|
||||
def select_mxfp4_moe_backend(
|
||||
config: FusedMoEConfig,
|
||||
activation_key: QuantKey | None = None,
|
||||
@@ -532,6 +565,21 @@ def select_mxfp4_moe_backend(
|
||||
assert last_error is not None
|
||||
raise last_error
|
||||
|
||||
if _requires_qwen38_tep8_emulation(config, requested_activation_key):
|
||||
backend = Mxfp4MoeBackend.EMULATION
|
||||
logger.warning_once(
|
||||
"Using OCP MX emulation for the Qwen3.8 Flash Next TEP8 routed "
|
||||
"experts on gfx950 because the native AITER W4A4 kernel is not "
|
||||
"numerically reliable for this shape. Performance will be lower."
|
||||
)
|
||||
return _return_or_raise(
|
||||
backend,
|
||||
config,
|
||||
kMxfp4Static,
|
||||
requested_activation_key,
|
||||
activation_format,
|
||||
)
|
||||
|
||||
# Select kernels in order of backend.
|
||||
AVAILABLE_BACKENDS = _filter_by_activation(
|
||||
_get_priority_backends_for_gpt_oss(), requested_activation_key
|
||||
|
||||
@@ -345,14 +345,13 @@ def convert_to_unquantized_kernel_format(
|
||||
|
||||
elif unquantized_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM:
|
||||
is_act_and_mul = moe_config.is_act_and_mul
|
||||
if not is_act_and_mul:
|
||||
# Kernel requires intermediate_size_per_partition % 128 == 0 (BlockMajorK
|
||||
# weight layout uses block_k=128). Pad along the intermediate dim when
|
||||
# the model + TP split don't satisfy the constraint.
|
||||
w13_weight, w2_weight, padded_intermediate = align_moe_weights_for_fi(
|
||||
w13_weight, w2_weight, is_act_and_mul, min_alignment=128
|
||||
)
|
||||
moe_config.intermediate_size_per_partition = padded_intermediate
|
||||
# Kernel requires intermediate_size_per_partition % 128 == 0 (BlockMajorK
|
||||
# weight layout uses block_k=128). Pad along the intermediate dim when
|
||||
# the model + TP split don't satisfy the constraint.
|
||||
w13_weight, w2_weight, padded_intermediate = align_moe_weights_for_fi(
|
||||
w13_weight, w2_weight, is_act_and_mul, min_alignment=128
|
||||
)
|
||||
moe_config.intermediate_size_per_partition = padded_intermediate
|
||||
|
||||
_cache_permute_indices: dict[torch.Size, torch.Tensor] = {}
|
||||
w13_weight, w2_weight = convert_moe_weights_to_flashinfer_trtllm_block_layout(
|
||||
|
||||
@@ -832,6 +832,20 @@ class RoutedExperts(PluggableLayer):
|
||||
FusedMoeWeightScaleSupported.GROUP.value,
|
||||
FusedMoeWeightScaleSupported.BLOCK.value,
|
||||
]:
|
||||
scale_refine = getattr(self.quant_method, "weight_scale_refine", None)
|
||||
if (
|
||||
quant_method == FusedMoeWeightScaleSupported.BLOCK.value
|
||||
and scale_refine is not None
|
||||
):
|
||||
# FP8 block scales are stored per (block_n, block_k) tile
|
||||
# of the unsharded weight, while the TP-sharded parameters
|
||||
# use a refined block grid (see Fp8MoEMethod). Upsample the
|
||||
# scales to the refined grid (lossless: the refined block
|
||||
# divides the checkpoint block) so per-rank slicing stays
|
||||
# exact. Dim -2 is the weight's N dim, dim -1 is K.
|
||||
loaded_weight = loaded_weight.repeat_interleave(
|
||||
scale_refine[0], dim=-2
|
||||
).repeat_interleave(scale_refine[1], dim=-1)
|
||||
self._load_model_weight_or_group_weight_scale(
|
||||
shard_id=shard_id,
|
||||
shard_dim=shard_dim,
|
||||
|
||||
@@ -56,6 +56,10 @@ class MambaBase(AttentionLayerBase):
|
||||
def mamba_type(self) -> MambaAttentionBackendEnum:
|
||||
pass
|
||||
|
||||
@property
|
||||
def is_kv_cache_tp_replicated(self) -> bool:
|
||||
return False
|
||||
|
||||
@abstractmethod
|
||||
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
|
||||
pass
|
||||
@@ -70,6 +74,7 @@ class MambaBase(AttentionLayerBase):
|
||||
block_size=mamba_block_size,
|
||||
page_size_padded=page_size_padded,
|
||||
mamba_type=self.mamba_type,
|
||||
tp_replicated=self.is_kv_cache_tp_replicated,
|
||||
mamba_cache_mode=vllm_config.cache_config.mamba_cache_mode,
|
||||
# RecoverSSM verifies the whole window off one checkpoint, so it
|
||||
# never writes the baseline's per-draft-token state slots.
|
||||
|
||||
@@ -519,7 +519,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
self.gqa_interleaved_layout
|
||||
or self.head_k_dim != 128
|
||||
or self.head_v_dim != 128
|
||||
or self.norm.activation != "silu"
|
||||
or self.norm.activation not in ("silu", "sigmoid")
|
||||
or vllm_config.model_config.dtype != torch.bfloat16
|
||||
or conv_state_dtype != torch.bfloat16
|
||||
or recurrent_state_dtype not in FUSED_GDN_STATE_DTYPES
|
||||
@@ -527,9 +527,9 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
):
|
||||
return (
|
||||
"the fused CUDA kernel requires a BF16 GDN model with "
|
||||
"K=V=128, SiLU gating, non-interleaved GQA layout, BF16 "
|
||||
"convolution cache, BF16 or FP32 recurrent state, and a "
|
||||
"GPU with compute capability 8.0+"
|
||||
"K=V=128, SiLU or sigmoid gating, non-interleaved GQA "
|
||||
"layout, BF16 convolution cache, BF16 or FP32 recurrent "
|
||||
"state, and a GPU with compute capability 8.0+"
|
||||
)
|
||||
if not hasattr(torch.ops._C, "fused_gdn_decode_post_conv_mtp"):
|
||||
return "torch.ops._C.fused_gdn_decode_post_conv_mtp is not built"
|
||||
@@ -1766,6 +1766,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
out=core_attn_out,
|
||||
scale=self.head_k_dim**-0.5,
|
||||
norm_eps=self.layer_norm_epsilon,
|
||||
output_gate_activation=self.norm.activation,
|
||||
)
|
||||
|
||||
def _forward_core_fused_norm_packed(
|
||||
|
||||
@@ -17,6 +17,7 @@ from vllm.utils.torch_utils import (
|
||||
STR_DTYPE_TO_TORCH_DTYPE,
|
||||
get_kv_cache_torch_dtype,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -352,6 +353,10 @@ Parameters:
|
||||
num_accepted_tokens: int - number of accepted tokens used to compute the copy offset.
|
||||
Range: 1 .. 1 + num_speculative_tokens (inclusive).
|
||||
"""
|
||||
MambaStateCopyFuncs: TypeAlias = tuple[MambaStateCopyFunc, ...]
|
||||
MambaStateCopyFuncsByType: TypeAlias = dict[
|
||||
MambaAttentionBackendEnum, MambaStateCopyFuncs
|
||||
]
|
||||
|
||||
|
||||
def get_conv_copy_spec(
|
||||
|
||||
@@ -31,6 +31,7 @@ from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
convert_to_fp8_moe_kernel_format,
|
||||
make_fp8_moe_kernel,
|
||||
make_fp8_moe_quant_config,
|
||||
refine_fp8_moe_block_shape,
|
||||
select_fp8_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
@@ -63,7 +64,6 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8Static128BlockSym,
|
||||
kFp8StaticTensorSym,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
|
||||
@@ -481,9 +481,41 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
"weight_scale_inv" if self.block_quant else "weight_scale"
|
||||
)
|
||||
|
||||
self.weight_scale_refine: tuple[int, int] | None = None
|
||||
self.moe_block_shape = self.weight_block_size
|
||||
|
||||
# Set weight key and activation key for kernel compatibility
|
||||
if self.block_quant:
|
||||
weight_key = kFp8Static128BlockSym
|
||||
assert self.weight_block_size is not None
|
||||
# TP shards the intermediate dim of the expert weights, so a
|
||||
# per-shard size that is not a multiple of the checkpoint's block
|
||||
# size makes the checkpoint's block scales impossible to shard
|
||||
# exactly. When a finer block size (>= 32) divides both the
|
||||
# checkpoint blocks and all involved dims, the weight scales are
|
||||
# refined to that granularity at load time (a lossless upsampling,
|
||||
# since the refined block divides the checkpoint block). The
|
||||
# refined block shape is encoded in the weight key, so the oracle
|
||||
# only selects kernels that support it (e.g. Triton, which takes
|
||||
# the block shape as a runtime argument).
|
||||
refined_shape = refine_fp8_moe_block_shape(self.moe, self.weight_block_size)
|
||||
if refined_shape is not None:
|
||||
block_n, block_k = self.weight_block_size
|
||||
self.weight_scale_refine = (
|
||||
block_n // refined_shape[0],
|
||||
block_k // refined_shape[1],
|
||||
)
|
||||
self.moe_block_shape = refined_shape
|
||||
logger.info_once(
|
||||
"FP8 MoE block scales refined from %s to %s to fit "
|
||||
"the TP-sharded intermediate size %d.",
|
||||
str(self.weight_block_size),
|
||||
str(refined_shape),
|
||||
self.moe.intermediate_size_per_partition,
|
||||
)
|
||||
assert self.moe_block_shape is not None
|
||||
weight_key = create_fp8_quant_key(
|
||||
static=True, group_shape=GroupShape(*self.moe_block_shape)
|
||||
)
|
||||
activation_key = kFp8Dynamic128Sym
|
||||
else:
|
||||
weight_key = kFp8StaticTensorSym
|
||||
@@ -519,6 +551,8 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
|
||||
if self.block_quant:
|
||||
assert self.weight_block_size is not None
|
||||
assert self.moe_block_shape is not None
|
||||
moe_block_shape = self.moe_block_shape
|
||||
layer.weight_block_size = self.weight_block_size
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
block_n, block_k = (
|
||||
@@ -530,18 +564,24 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
# layers must be divisible by block_n.
|
||||
# Required by column parallel or enabling merged weights
|
||||
if intermediate_size_per_partition % block_n != 0:
|
||||
raise ValueError(
|
||||
f"The output_size of gate's and up's weight = "
|
||||
f"{intermediate_size_per_partition} is not divisible by "
|
||||
f"weight quantization block_n = {block_n}."
|
||||
)
|
||||
if self.weight_scale_refine is None:
|
||||
raise ValueError(
|
||||
f"The output_size of gate's and up's weight = "
|
||||
f"{intermediate_size_per_partition} is not divisible by "
|
||||
f"weight quantization block_n = {block_n}."
|
||||
)
|
||||
# Use the refined block grid for the scale parameters; the
|
||||
# loader upsamples the checkpoint scales accordingly.
|
||||
block_n, block_k = moe_block_shape
|
||||
if tp_size > 1 and intermediate_size_per_partition % block_k != 0:
|
||||
# Required by row parallel
|
||||
raise ValueError(
|
||||
f"The input_size of down's weight = "
|
||||
f"{intermediate_size_per_partition} is not divisible by "
|
||||
f"weight quantization block_k = {block_k}."
|
||||
)
|
||||
if self.weight_scale_refine is None:
|
||||
raise ValueError(
|
||||
f"The input_size of down's weight = "
|
||||
f"{intermediate_size_per_partition} is not divisible by "
|
||||
f"weight quantization block_k = {block_k}."
|
||||
)
|
||||
block_n, block_k = moe_block_shape
|
||||
|
||||
# WEIGHTS
|
||||
w13_weight = torch.nn.Parameter(
|
||||
@@ -747,7 +787,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
w2_scale=w2_scale,
|
||||
a1_scale=a1_scale,
|
||||
a2_scale=a2_scale,
|
||||
block_shape=self.weight_block_size,
|
||||
block_shape=self.moe_block_shape,
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
|
||||
@@ -161,6 +161,16 @@ class QuarkOCP_MX(QuarkScheme):
|
||||
weight_loader: Callable,
|
||||
**kwargs,
|
||||
):
|
||||
if input_size_per_partition % OCP_MX_BLOCK_SIZE != 0:
|
||||
layer_name = getattr(layer, "prefix", "") or type(layer).__name__
|
||||
raise ValueError(
|
||||
f"OCP MX linear layer {layer_name!r} has an input size per "
|
||||
f"partition of {input_size_per_partition}, which must be "
|
||||
f"divisible by the OCP MX group size {OCP_MX_BLOCK_SIZE}. "
|
||||
"Choose a compatible tensor-parallel size or avoid "
|
||||
"tensor-parallel sharding for this layer."
|
||||
)
|
||||
|
||||
if self.dynamic_mxfp4_quant:
|
||||
weight = ModelWeightParameter(
|
||||
data=torch.empty(
|
||||
|
||||
@@ -9,6 +9,41 @@ if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
|
||||
|
||||
def get_quark_ocp_mx_group_size(
|
||||
quant_config: "QuantizationConfig | None",
|
||||
layer_name: str,
|
||||
) -> int | None:
|
||||
"""Return the OCP MX group size for a quantized Quark linear layer."""
|
||||
if quant_config is None:
|
||||
return None
|
||||
|
||||
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
|
||||
from vllm.model_executor.layers.quantization.quark.utils import should_ignore_layer
|
||||
|
||||
if not isinstance(quant_config, QuarkConfig):
|
||||
return None
|
||||
|
||||
if should_ignore_layer(
|
||||
layer_name,
|
||||
ignore=quant_config.quant_config.get("exclude") or [],
|
||||
fused_mapping=quant_config.packed_modules_mapping,
|
||||
):
|
||||
return None
|
||||
|
||||
layer_quant_config = (
|
||||
quant_config.get_layer_quant_config_from_name(layer_name)
|
||||
or quant_config.quant_config.get("global_quant_config")
|
||||
or {}
|
||||
)
|
||||
weight_quant = layer_quant_config.get("weight")
|
||||
input_quant = layer_quant_config.get("input_tensors")
|
||||
if not quant_config._is_w_ocp_mx_a_x(weight_quant, input_quant):
|
||||
return None
|
||||
|
||||
assert weight_quant is not None
|
||||
return int(weight_quant["group_size"])
|
||||
|
||||
|
||||
def is_shared_expert_quant_fse_compatible(
|
||||
quant_config: "QuantizationConfig | None",
|
||||
expert_prefix: str,
|
||||
|
||||
@@ -232,7 +232,16 @@ def align_fp4_moe_weights_for_fi(
|
||||
|
||||
# Pad w13 and w2 along its intermediate dimension.
|
||||
padded_w13 = w13.new_zeros((num_experts, padded_gate_up_dim, hidden_size // 2))
|
||||
padded_w13[:, : w13.shape[1], :] = w13
|
||||
if is_act_and_mul:
|
||||
# Keep the two logical projections independently aligned. Copying the
|
||||
# fused [gate, up] tensor contiguously would move the up projection into
|
||||
# the padded tail of gate when intermediate is rounded up.
|
||||
padded_w13[:, :intermediate, :] = w13[:, :intermediate, :]
|
||||
padded_w13[:, padded_intermediate : padded_intermediate + intermediate, :] = (
|
||||
w13[:, intermediate:, :]
|
||||
)
|
||||
else:
|
||||
padded_w13[:, : w13.shape[1], :] = w13
|
||||
|
||||
padded_w2 = w2.new_zeros((num_experts, hidden_size, padded_intermediate // 2))
|
||||
padded_w2[:, :, : w2.shape[2]] = w2
|
||||
@@ -240,7 +249,13 @@ def align_fp4_moe_weights_for_fi(
|
||||
padded_w13_scale = w13_scale.new_zeros(
|
||||
(num_experts, padded_gate_up_dim, hidden_size // 16)
|
||||
)
|
||||
padded_w13_scale[:, : w13_scale.shape[1], :] = w13_scale
|
||||
if is_act_and_mul:
|
||||
padded_w13_scale[:, :intermediate, :] = w13_scale[:, :intermediate, :]
|
||||
padded_w13_scale[
|
||||
:, padded_intermediate : padded_intermediate + intermediate, :
|
||||
] = w13_scale[:, intermediate:, :]
|
||||
else:
|
||||
padded_w13_scale[:, : w13_scale.shape[1], :] = w13_scale
|
||||
|
||||
padded_w2_scale = w2_scale.new_zeros(
|
||||
(num_experts, hidden_size, padded_intermediate // 16)
|
||||
@@ -322,7 +337,13 @@ def align_moe_weights_for_fi(
|
||||
|
||||
# Pad w13 and w2 along its intermediate dimension.
|
||||
padded_w13 = w13.new_zeros((num_experts, padded_gate_up_dim, hidden_size))
|
||||
padded_w13[:, : w13.shape[1], :] = w13
|
||||
if is_act_and_mul:
|
||||
padded_w13[:, :intermediate, :] = w13[:, :intermediate, :]
|
||||
padded_w13[:, padded_intermediate : padded_intermediate + intermediate, :] = (
|
||||
w13[:, intermediate:, :]
|
||||
)
|
||||
else:
|
||||
padded_w13[:, :intermediate, :] = w13
|
||||
|
||||
padded_w2 = w2.new_zeros((num_experts, hidden_size, padded_intermediate))
|
||||
padded_w2[:, :, :intermediate] = w2
|
||||
|
||||
@@ -233,6 +233,7 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
quant_config: quant config for the layer
|
||||
prefix: full name of the layer in the state dict
|
||||
disable_tp: If true, tensor parallelism will be disabled for this layer.
|
||||
quant_method: Preselected quantization method for model-specific layers.
|
||||
""" # noqa: E501
|
||||
|
||||
# --8<-- [end:vocab_parallel_embedding]
|
||||
@@ -248,6 +249,7 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
prefix: str = "",
|
||||
*,
|
||||
disable_tp: bool = False,
|
||||
quant_method: QuantizeMethodBase | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
@@ -281,8 +283,9 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
)
|
||||
self.embedding_dim = embedding_dim
|
||||
|
||||
quant_method = None
|
||||
if quant_config is not None:
|
||||
# Avoid overriding a preselected model-specific method with generic
|
||||
# config-based dispatch.
|
||||
if quant_method is None and quant_config is not None:
|
||||
quant_method = quant_config.get_quant_method(self, prefix=prefix)
|
||||
if quant_method is None:
|
||||
quant_method = UnquantizedEmbeddingMethod()
|
||||
@@ -501,8 +504,16 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
output_parallel = self.quant_method.embedding(self, masked_input.long())
|
||||
# Mask the output embedding.
|
||||
if self.tp_size > 1:
|
||||
if output_parallel.dtype in (
|
||||
torch.float8_e4m3fn,
|
||||
torch.float8_e5m2,
|
||||
):
|
||||
# Each vocab token has one owner, so FP8 bytes can use int8 SUM.
|
||||
comm_output = output_parallel.view(torch.int8)
|
||||
comm_output.masked_fill_(input_mask.unsqueeze(-1), 0)
|
||||
output = tensor_model_parallel_all_reduce(comm_output)
|
||||
return output.view(output_parallel.dtype)
|
||||
output_parallel.masked_fill_(input_mask.unsqueeze(-1), 0)
|
||||
# Reduce across all the model parallel GPUs.
|
||||
return tensor_model_parallel_all_reduce(output_parallel)
|
||||
return output_parallel
|
||||
|
||||
|
||||
@@ -289,12 +289,17 @@ def get_quant_config(
|
||||
# if hf_quant_config is None, we will try to get config from
|
||||
# hf_overrides
|
||||
hf_overrides = model_config.hf_overrides
|
||||
if callable(hf_overrides):
|
||||
# A callable hf_overrides is a config-to-config transform (e.g. the
|
||||
# one SpeculativeConfig installs on draft model configs); it cannot
|
||||
# carry quantization config entries, so treat it as no overrides.
|
||||
hf_overrides = {}
|
||||
if not isinstance(hf_overrides, dict):
|
||||
raise ValueError(
|
||||
"hf_overrides must be a dict for get_quant_config "
|
||||
"to get the quantization config from it."
|
||||
)
|
||||
quantization_config_file = hf_overrides.get("quantization_config_file", None)
|
||||
quantization_config_file = hf_overrides.get("quantization_config_file")
|
||||
if quantization_config_file is not None:
|
||||
if hasattr(quant_cls, "from_config_file"):
|
||||
return quant_cls.from_config_file(quantization_config_file)
|
||||
@@ -304,7 +309,7 @@ def get_quant_config(
|
||||
"but quant_cls.from_config_file is not implemented in "
|
||||
f"{quant_cls}"
|
||||
)
|
||||
quantization_config_json = hf_overrides.get("quantization_config_dict_json", None)
|
||||
quantization_config_json = hf_overrides.get("quantization_config_dict_json")
|
||||
if quantization_config_json is not None:
|
||||
if hasattr(quant_cls, "from_config_dict_json"):
|
||||
return quant_cls.from_config_dict_json(quantization_config_json)
|
||||
|
||||
@@ -839,6 +839,83 @@ class Qwen3_5ForCausalLMConfig(Qwen3_5ForConditionalGenerationConfig):
|
||||
rope_parameters.pop("mrope_interleaved", None)
|
||||
|
||||
|
||||
def _strip_qwen4_exp_mrope(model_config: "ModelConfig") -> None:
|
||||
configs = {
|
||||
id(config): config
|
||||
for config in (
|
||||
getattr(model_config, "hf_config", None),
|
||||
model_config.hf_text_config,
|
||||
)
|
||||
if config is not None
|
||||
}
|
||||
for config in configs.values():
|
||||
rope_parameters = getattr(config, "rope_parameters", None)
|
||||
if rope_parameters is not None:
|
||||
rope_parameters.pop("mrope_section", None)
|
||||
rope_parameters.pop("mrope_interleaved", None)
|
||||
|
||||
|
||||
class Qwen4ExpForConditionalGenerationConfig(Qwen3_5ForConditionalGenerationConfig):
|
||||
"""Apply the Qwen3.5 hybrid-cache contract to Qwen4Exp."""
|
||||
|
||||
@staticmethod
|
||||
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
|
||||
Qwen3_5ForConditionalGenerationConfig.verify_and_update_config(vllm_config)
|
||||
text_config = vllm_config.model_config.hf_text_config
|
||||
if text_config.hc_count <= 1:
|
||||
raise ValueError("Qwen4Exp requires hc_count > 1")
|
||||
parallel_config = vllm_config.parallel_config
|
||||
uses_ple_or_qsa = bool(text_config.ple_layer_ids) or (
|
||||
getattr(text_config, "indexer_n_heads", None) is not None
|
||||
)
|
||||
if uses_ple_or_qsa and (
|
||||
parallel_config.enable_dbo or parallel_config.ubatch_size > 1
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp PLE/QSA does not support dual-batch overlap or microbatching"
|
||||
)
|
||||
# Checked again in Qwen4ExpModelState; rejecting it here keeps the
|
||||
# engine from loading weights first.
|
||||
if text_config.ple_layer_ids and parallel_config.pipeline_parallel_size > 1:
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp N-gram PLE embedding requires pipeline_parallel_size=1 "
|
||||
"because non-first pipeline ranks do not receive the raw input_ids "
|
||||
"it needs. Please run with PP=1."
|
||||
)
|
||||
multimodal_config = vllm_config.model_config.multimodal_config
|
||||
if multimodal_config is not None and multimodal_config.language_model_only:
|
||||
_strip_qwen4_exp_mrope(vllm_config.model_config)
|
||||
spec_config = vllm_config.speculative_config
|
||||
if spec_config is not None and spec_config.method not in {
|
||||
"mtp",
|
||||
"ngram",
|
||||
"ngram_gpu",
|
||||
}:
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp speculative decoding supports only its native MTP "
|
||||
"checkpoint and linear n-gram proposers"
|
||||
)
|
||||
|
||||
|
||||
class Qwen4ExpForCausalLMConfig(Qwen4ExpForConditionalGenerationConfig):
|
||||
@staticmethod
|
||||
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
|
||||
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(vllm_config)
|
||||
|
||||
_strip_qwen4_exp_mrope(vllm_config.model_config)
|
||||
|
||||
|
||||
class Qwen4ExpMTPConfig(Qwen4ExpForConditionalGenerationConfig):
|
||||
"""Preserve MRoPE for a VL target and use 1D RoPE for a text target."""
|
||||
|
||||
@staticmethod
|
||||
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
|
||||
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(vllm_config)
|
||||
if hasattr(vllm_config.model_config.hf_config, "vision_config"):
|
||||
return
|
||||
_strip_qwen4_exp_mrope(vllm_config.model_config)
|
||||
|
||||
|
||||
class ColQwen3_5Config(Qwen3_5ForConditionalGenerationConfig):
|
||||
"""Apply the attention contract declared by a ColQwen3.5 checkpoint."""
|
||||
|
||||
@@ -963,6 +1040,9 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
|
||||
"Qwen3_5ForConditionalGeneration": Qwen3_5ForConditionalGenerationConfig,
|
||||
"Qwen3_5MoeForCausalLM": Qwen3_5ForCausalLMConfig,
|
||||
"Qwen3_5MoeForConditionalGeneration": Qwen3_5ForConditionalGenerationConfig,
|
||||
"Qwen4ExpForCausalLM": Qwen4ExpForCausalLMConfig,
|
||||
"Qwen4ExpForConditionalGeneration": (Qwen4ExpForConditionalGenerationConfig),
|
||||
"Qwen4ExpMTP": Qwen4ExpMTPConfig,
|
||||
"UnlimitedOCRForCausalLM": UnlimitedOCRForCausalLMConfig,
|
||||
"VoyageQwen3BidirectionalEmbedModel": VoyageQwen3BidirectionalEmbedModelConfig,
|
||||
"XLMRobertaModel": JinaRobertaModelConfig,
|
||||
|
||||
@@ -49,13 +49,17 @@ if TYPE_CHECKING:
|
||||
from vllm.lora.model_manager import LoRAModelManager
|
||||
from vllm.model_executor.layers.fused_moe import MoERunner
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateCopyFunc,
|
||||
MambaStateCopyFuncsByType,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalKwargsItem
|
||||
from vllm.multimodal.registry import _ProcessorFactories
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tasks import ScoreType
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import (
|
||||
EncoderCudaGraphCaptureInputs,
|
||||
EncoderCudaGraphConfig,
|
||||
@@ -970,6 +974,14 @@ class IsHybrid(Protocol):
|
||||
"""
|
||||
...
|
||||
|
||||
@classmethod
|
||||
def get_mamba_state_copy_funcs(
|
||||
cls,
|
||||
mamba_types: set["MambaAttentionBackendEnum"],
|
||||
) -> "MambaStateCopyFuncsByType":
|
||||
copy_funcs = cls.get_mamba_state_copy_func()
|
||||
return {mamba_type: copy_funcs for mamba_type in mamba_types}
|
||||
|
||||
|
||||
@overload
|
||||
def is_hybrid(model: type[object]) -> TypeIs[type[IsHybrid]]: ...
|
||||
@@ -1131,6 +1143,20 @@ class SupportsMambaPrefixCaching(Protocol):
|
||||
|
||||
supports_mamba_prefix_caching: ClassVar[Literal[True]] = True
|
||||
|
||||
@classmethod
|
||||
def get_mamba_state_copy_func(cls) -> tuple["MambaStateCopyFunc", ...]:
|
||||
"""Return copy functions for the model's Mamba states."""
|
||||
...
|
||||
|
||||
@classmethod
|
||||
def get_mamba_state_copy_funcs(
|
||||
cls,
|
||||
mamba_types: set["MambaAttentionBackendEnum"],
|
||||
) -> "MambaStateCopyFuncsByType":
|
||||
"""Map legacy copy functions to each requested Mamba backend."""
|
||||
copy_funcs = cls.get_mamba_state_copy_func()
|
||||
return {mamba_type: copy_funcs for mamba_type in mamba_types}
|
||||
|
||||
|
||||
@overload
|
||||
def supports_mamba_prefix_caching(
|
||||
|
||||
@@ -80,15 +80,17 @@ class Qwen2MoeMLP(nn.Module):
|
||||
reduce_results: bool = True,
|
||||
expert_gate: torch.nn.Linear | None = None,
|
||||
is_sequence_parallel: bool = False,
|
||||
disable_tp: bool = False,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
disable_tp = disable_tp or is_sequence_parallel
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
hidden_size,
|
||||
[intermediate_size] * 2,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
disable_tp=is_sequence_parallel,
|
||||
disable_tp=disable_tp,
|
||||
prefix=f"{prefix}.gate_up_proj",
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
@@ -97,7 +99,7 @@ class Qwen2MoeMLP(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
disable_tp=is_sequence_parallel,
|
||||
disable_tp=disable_tp,
|
||||
prefix=f"{prefix}.down_proj",
|
||||
)
|
||||
if hidden_act != "silu":
|
||||
|
||||
@@ -43,6 +43,9 @@ from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateShapeCalculator,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.utils.config_utils import (
|
||||
get_quark_ocp_mx_group_size,
|
||||
)
|
||||
from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding, get_rope
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
@@ -90,6 +93,38 @@ def _should_use_sequence_parallel(vllm_config: VllmConfig) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _should_replicate_misaligned_shared_expert(
|
||||
intermediate_size: int,
|
||||
tp_size: int,
|
||||
group_size: int | None,
|
||||
enable_expert_parallel: bool,
|
||||
is_sequence_parallel: bool,
|
||||
) -> bool:
|
||||
if intermediate_size <= 0 or group_size is None:
|
||||
return False
|
||||
|
||||
partition_size, remainder = divmod(intermediate_size, tp_size)
|
||||
if remainder == 0 and partition_size % group_size == 0:
|
||||
return False
|
||||
|
||||
if enable_expert_parallel or is_sequence_parallel:
|
||||
return True
|
||||
|
||||
if remainder != 0:
|
||||
raise ValueError(
|
||||
f"Shared-expert intermediate size {intermediate_size} must be "
|
||||
f"divisible by tensor-parallel size {tp_size}."
|
||||
)
|
||||
|
||||
raise ValueError(
|
||||
"The Quark OCP MX shared expert cannot be tensor-parallelized: "
|
||||
f"intermediate size {intermediate_size} with TP size {tp_size} "
|
||||
f"produces a partition of {partition_size}, which is not divisible by "
|
||||
f"the OCP MX group size {group_size}. Choose a compatible "
|
||||
"tensor-parallel size or enable expert parallelism."
|
||||
)
|
||||
|
||||
|
||||
class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
def __init__(self, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
@@ -106,6 +141,17 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
|
||||
self.is_sequence_parallel = parallel_config.use_sequence_parallel_moe
|
||||
|
||||
shared_expert_group_size = get_quark_ocp_mx_group_size(
|
||||
quant_config,
|
||||
f"{prefix}.shared_expert.down_proj",
|
||||
)
|
||||
self.replicate_shared_expert = _should_replicate_misaligned_shared_expert(
|
||||
config.shared_expert_intermediate_size,
|
||||
self.tp_size,
|
||||
shared_expert_group_size,
|
||||
parallel_config.enable_expert_parallel,
|
||||
self.is_sequence_parallel,
|
||||
)
|
||||
if self.tp_size > config.num_experts:
|
||||
raise ValueError(
|
||||
f"Tensor parallel size {self.tp_size} is greater than "
|
||||
@@ -138,7 +184,10 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
)
|
||||
|
||||
self.is_fused_shared_expert_enabled = False
|
||||
if config.shared_expert_intermediate_size > 0:
|
||||
if (
|
||||
config.shared_expert_intermediate_size > 0
|
||||
and not self.replicate_shared_expert
|
||||
):
|
||||
self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert(
|
||||
quant_config,
|
||||
prefix,
|
||||
@@ -159,11 +208,14 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
reduce_results=False,
|
||||
expert_gate=self.shared_expert_gate,
|
||||
is_sequence_parallel=self.is_sequence_parallel,
|
||||
disable_tp=self.replicate_shared_expert,
|
||||
prefix=f"{prefix}.shared_expert",
|
||||
)
|
||||
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=self.shared_expert,
|
||||
shared_experts=(
|
||||
None if self.replicate_shared_expert else self.shared_expert
|
||||
),
|
||||
gate=self.gate,
|
||||
num_experts=self.n_routed_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
@@ -195,9 +247,16 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
if self.is_sequence_parallel and not already_sequence_parallel:
|
||||
hidden_states = sequence_parallel_chunk(hidden_states)
|
||||
|
||||
replicated_shared_output = (
|
||||
self.shared_expert(hidden_states)
|
||||
if self.replicate_shared_expert and self.shared_expert is not None
|
||||
else None
|
||||
)
|
||||
final_hidden_states = self.experts(
|
||||
hidden_states=hidden_states, router_logits=hidden_states
|
||||
)
|
||||
if replicated_shared_output is not None:
|
||||
final_hidden_states += replicated_shared_output
|
||||
|
||||
if self.is_sequence_parallel and not already_sequence_parallel:
|
||||
final_hidden_states = tensor_model_parallel_all_gather(
|
||||
@@ -488,7 +547,7 @@ class Qwen3NextDecoderLayer(nn.Module):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
positions: torch.Tensor = None,
|
||||
positions: torch.Tensor,
|
||||
**kwargs: object,
|
||||
):
|
||||
full_num_tokens = positions.shape[-1]
|
||||
|
||||
@@ -111,6 +111,10 @@ _TEXT_GENERATION_MODELS = {
|
||||
"Gemma3nForCausalLM": ("gemma3n", "Gemma3nForCausalLM"),
|
||||
"Gemma4ForCausalLM": ("gemma4", "Gemma4ForCausalLM"),
|
||||
"Qwen3NextForCausalLM": ("qwen3_next", "Qwen3NextForCausalLM"),
|
||||
"Qwen4ExpForCausalLM": (
|
||||
"vllm.models.qwen4_exp",
|
||||
"Qwen4ExpForCausalLM",
|
||||
),
|
||||
"GlmForCausalLM": ("glm", "GlmForCausalLM"),
|
||||
"Glm4ForCausalLM": ("glm4", "Glm4ForCausalLM"),
|
||||
"Glm4MoeForCausalLM": ("glm4_moe", "Glm4MoeForCausalLM"),
|
||||
@@ -573,6 +577,10 @@ _MULTIMODAL_MODELS = {
|
||||
"qwen3_5",
|
||||
"Qwen3_5MoeForConditionalGeneration",
|
||||
),
|
||||
"Qwen4ExpForConditionalGeneration": (
|
||||
"vllm.models.qwen4_exp",
|
||||
"Qwen4ExpForConditionalGeneration",
|
||||
),
|
||||
"RForConditionalGeneration": ("rvl", "RForConditionalGeneration"),
|
||||
"SkyworkR1VChatModel": ("skyworkr1v", "SkyworkR1VChatModel"),
|
||||
"SmolVLMForConditionalGeneration": ("smolvlm", "SmolVLMForConditionalGeneration"),
|
||||
@@ -659,6 +667,7 @@ _SPECULATIVE_DECODING_MODELS = {
|
||||
"MedusaModel": ("medusa", "Medusa"),
|
||||
"OpenPanguMTPModel": ("openpangu_mtp", "OpenPanguMTP"),
|
||||
"Qwen3NextMTP": ("qwen3_next_mtp", "Qwen3NextMTP"),
|
||||
"Qwen4ExpMTP": ("vllm.models.qwen4_exp", "Qwen4ExpMTP"),
|
||||
"Step3p5MTP": ("step3p5_mtp", "Step3p5MTP"),
|
||||
"Qwen3_5MTP": ("qwen3_5_mtp", "Qwen3_5MTP"),
|
||||
"Qwen3_5MoeMTP": ("qwen3_5_mtp", "Qwen3_5MoeMTP"),
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Qwen4Exp model package."""
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from .common.hyperconnection import (
|
||||
GatedResidual,
|
||||
GroupedGemmaRMSNorm,
|
||||
HyperConnectionBase,
|
||||
HyperConnectionConfig,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .nvidia.model import (
|
||||
Qwen4ExpForCausalLM,
|
||||
Qwen4ExpForConditionalGeneration,
|
||||
)
|
||||
from .nvidia.mtp import Qwen4ExpMTP
|
||||
|
||||
|
||||
def __getattr__(name: str) -> Any:
|
||||
if name in {
|
||||
"Qwen4ExpForCausalLM",
|
||||
"Qwen4ExpForConditionalGeneration",
|
||||
"Qwen4ExpMTP",
|
||||
}:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_xpu() or current_platform.is_tpu():
|
||||
raise NotImplementedError("Qwen4Exp currently supports CUDA and ROCm only")
|
||||
if current_platform.is_rocm():
|
||||
from .amd.model import (
|
||||
Qwen4ExpForCausalLM,
|
||||
Qwen4ExpForConditionalGeneration,
|
||||
)
|
||||
from .amd.mtp import Qwen4ExpMTP
|
||||
else:
|
||||
from .nvidia.model import (
|
||||
Qwen4ExpForCausalLM,
|
||||
Qwen4ExpForConditionalGeneration,
|
||||
)
|
||||
from .nvidia.mtp import Qwen4ExpMTP
|
||||
|
||||
return {
|
||||
"Qwen4ExpForCausalLM": Qwen4ExpForCausalLM,
|
||||
"Qwen4ExpForConditionalGeneration": (Qwen4ExpForConditionalGeneration),
|
||||
"Qwen4ExpMTP": Qwen4ExpMTP,
|
||||
}[name]
|
||||
raise AttributeError(name)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"GatedResidual",
|
||||
"GroupedGemmaRMSNorm",
|
||||
"HyperConnectionBase",
|
||||
"HyperConnectionConfig",
|
||||
"Qwen4ExpForCausalLM",
|
||||
"Qwen4ExpForConditionalGeneration",
|
||||
"Qwen4ExpMTP",
|
||||
]
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,205 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""HyperConnection (Gated Residual) utilities for the AMD model variant.
|
||||
|
||||
Implements the HyperConnection residual scheme proposed in
|
||||
"HyperConnections" (https://arxiv.org/abs/2409.19606). This AMD variant
|
||||
delays each HC combine to the following HC mix boundary. HC glue kernels,
|
||||
including fused combine+RMSNorm, live in ``ops/hc.py``; projections remain
|
||||
standard vLLM Linear modules.
|
||||
|
||||
Hidden states between layers have shape ``[..., HC*HS]`` with HS inner
|
||||
(HC outer, HS inner — checkpoint-native layout).
|
||||
|
||||
Typical usage inside a transformer decoder layer::
|
||||
|
||||
self.attn_hc = GatedResidual(hc_config)
|
||||
|
||||
hidden_states, block_input, injection = self.attn_hc.mix(hidden_states)
|
||||
attention_output = attention(block_input)
|
||||
hidden_states, block_input, injection = self.mlp_hc.combine_and_mix(
|
||||
hidden_states, attention_output, injection
|
||||
)
|
||||
"""
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
)
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
|
||||
from ..common.hyperconnection import (
|
||||
GroupedGemmaRMSNorm,
|
||||
HyperConnectionConfig,
|
||||
)
|
||||
from .ops.hc import (
|
||||
grouped_gemma_rmsnorm,
|
||||
hc_combine,
|
||||
hc_combine_norm,
|
||||
hc_gate_mix,
|
||||
hc_silu,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Gated-residual variant
|
||||
# ---------------------------------------------------------------------------
|
||||
class GatedResidual(nn.Module):
|
||||
"""Gated HyperConnection with learnable low-rank mixing and injection.
|
||||
|
||||
``combine_and_mix()`` runs the pre pipeline (grouped GemmaRMSNorm -> merged
|
||||
low-rank down+inject GEMM -> silu -> up GEMM -> sigmoid -> gated mean
|
||||
over the HC streams). When passed a pending block output and an injection,
|
||||
it fuses their residual combine with the RMSNorm. Final mixers use
|
||||
``use_combine=False`` and do not produce a new injection.
|
||||
|
||||
Weights: the norm owns the grouped GemmaRMSNorm affine; the projections
|
||||
are vLLM Linear modules (merged replicated linear for down+inject), so
|
||||
GEMM dispatch (e.g. the low-latency skinny GEMM) applies through the
|
||||
standard quant_method mechanism.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: HyperConnectionConfig,
|
||||
use_combine: bool = True,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.lora_rank = config.hc_lowrank
|
||||
self.hc_count = config.hc_count
|
||||
self.hidden_size = config.hidden_size
|
||||
self.use_combine = use_combine
|
||||
|
||||
norm_size = (
|
||||
self.hyper_hidden_size if config.hc_per_branch_norm else config.hidden_size
|
||||
)
|
||||
group_size = config.hidden_size if config.hc_per_branch_norm else None
|
||||
# Normalize each H-sized HC stream independently while retaining a
|
||||
# separate affine weight for every element of the HC*H layout.
|
||||
self.hc_norm = GroupedGemmaRMSNorm(
|
||||
norm_size,
|
||||
eps=config.rms_norm_eps,
|
||||
group_size=group_size,
|
||||
dtype=config.params_dtype,
|
||||
)
|
||||
|
||||
# -- vLLM Linear weights --------------------------------------------
|
||||
# The merged skinny-GEMM shape is physically padded to 16 rows for
|
||||
# alignment and efficient backend dispatch.
|
||||
self.pad_size = (-(self.lora_rank + self.hc_count)) % 16 if use_combine else 0
|
||||
if use_combine:
|
||||
self.input_mix_weight_down_block_inject = MergedColumnParallelLinear(
|
||||
self.hyper_hidden_size,
|
||||
[self.lora_rank, self.hc_count]
|
||||
+ ([self.pad_size] if self.pad_size else []),
|
||||
bias=False,
|
||||
params_dtype=config.params_dtype,
|
||||
quant_config=None,
|
||||
prefix=maybe_prefix(prefix, "input_mix_weight_down_block_inject"),
|
||||
return_bias=False,
|
||||
disable_tp=True,
|
||||
)
|
||||
else:
|
||||
self.input_mix_weight_down = ReplicatedLinear(
|
||||
self.hyper_hidden_size,
|
||||
self.lora_rank,
|
||||
bias=False,
|
||||
params_dtype=config.params_dtype,
|
||||
quant_config=None,
|
||||
prefix=maybe_prefix(prefix, "input_mix_weight_down"),
|
||||
return_bias=False,
|
||||
)
|
||||
self.input_mix_weight_up = ReplicatedLinear(
|
||||
self.lora_rank,
|
||||
self.hyper_hidden_size,
|
||||
bias=False,
|
||||
params_dtype=config.params_dtype,
|
||||
quant_config=None,
|
||||
prefix=maybe_prefix(prefix, "input_mix_weight_up"),
|
||||
return_bias=False,
|
||||
)
|
||||
|
||||
def mix(
|
||||
self, hidden_states: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
|
||||
xn = grouped_gemma_rmsnorm(
|
||||
hidden_states,
|
||||
self.hc_norm.weight,
|
||||
self.config.rms_norm_eps,
|
||||
self.hc_count,
|
||||
)
|
||||
|
||||
if self.use_combine:
|
||||
# produce injection logits for combine
|
||||
split_sizes = [self.lora_rank, self.hc_count, self.pad_size]
|
||||
down_and_injection = self.input_mix_weight_down_block_inject(xn)
|
||||
lora, injection, _ = down_and_injection.split(split_sizes, dim=-1)
|
||||
else:
|
||||
lora = self.input_mix_weight_down(xn)
|
||||
injection = None
|
||||
|
||||
lora = hc_silu(lora, self.hc_count)
|
||||
gate = self.input_mix_weight_up(lora) # [M, D]
|
||||
block_input = hc_gate_mix(xn, gate, self.hc_count)
|
||||
|
||||
return hidden_states, block_input, injection
|
||||
|
||||
def combine_and_mix(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
prev_block_output: torch.Tensor,
|
||||
prev_injection: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
|
||||
"""Consume a pending combine, then prepare the next block input.
|
||||
|
||||
``hidden_states`` is the multi-stream state from before the pending
|
||||
block's mix. Its combine with ``block_output`` is fused with this
|
||||
module's input RMSNorm.
|
||||
"""
|
||||
hidden_states, xn = hc_combine_norm(
|
||||
hidden_states,
|
||||
prev_block_output,
|
||||
prev_injection,
|
||||
self.hc_norm.weight,
|
||||
self.config.rms_norm_eps,
|
||||
self.hc_count,
|
||||
)
|
||||
|
||||
if self.use_combine:
|
||||
# produce injection logits for combine
|
||||
split_sizes = [self.lora_rank, self.hc_count, self.pad_size]
|
||||
down_and_injection = self.input_mix_weight_down_block_inject(xn)
|
||||
lora, injection, _ = down_and_injection.split(split_sizes, dim=-1)
|
||||
else:
|
||||
lora = self.input_mix_weight_down(xn)
|
||||
injection = None
|
||||
|
||||
lora = hc_silu(lora, self.hc_count)
|
||||
gate = self.input_mix_weight_up(lora) # [M, D]
|
||||
block_input = hc_gate_mix(xn, gate, self.hc_count)
|
||||
|
||||
return hidden_states, block_input, injection
|
||||
|
||||
def combine(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
block_output: torch.Tensor,
|
||||
injection: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return hc_combine(hidden_states, block_output, injection, self.hc_count)
|
||||
|
||||
@property
|
||||
def hyper_hidden_size(self) -> int:
|
||||
return self.hc_count * self.hidden_size
|
||||
|
||||
|
||||
__all__ = [
|
||||
"GatedResidual",
|
||||
"GroupedGemmaRMSNorm",
|
||||
"HyperConnectionConfig",
|
||||
]
|
||||
@@ -0,0 +1,321 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Qwen4Exp weight-free QSA indexer."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import cast
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding.mrope import triton_mrope
|
||||
from vllm.transformers_utils.configs.qwen4_exp import (
|
||||
Qwen4ExpTextConfig,
|
||||
)
|
||||
|
||||
from ..common.qsa_cache import (
|
||||
QSACompressedKeyCache,
|
||||
QSAForwardMetadata,
|
||||
QSAKeyStateCache,
|
||||
canonical_qsa_rope_positions,
|
||||
)
|
||||
|
||||
|
||||
def apply_qsa_rope(
|
||||
rotary_emb: nn.Module,
|
||||
positions: torch.Tensor,
|
||||
tensor: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Apply the main attention's exact 1D/MRoPE composition to QSA heads."""
|
||||
|
||||
num_tokens, _, head_dim = tensor.shape
|
||||
rotary_dim = rotary_emb.rotary_dim
|
||||
cache = rotary_emb._match_cos_sin_cache_dtype(tensor) # noqa: SLF001
|
||||
cos_sin = cache[positions]
|
||||
cos, sin = cos_sin.chunk(2, dim=-1)
|
||||
if positions.ndim == 2:
|
||||
shape = tensor.shape
|
||||
tensor, _ = triton_mrope(
|
||||
tensor.reshape(num_tokens, -1),
|
||||
tensor.new_empty((num_tokens, head_dim)),
|
||||
cos,
|
||||
sin,
|
||||
rotary_emb.mrope_section,
|
||||
head_dim,
|
||||
rotary_dim,
|
||||
rotary_emb.mrope_interleaved,
|
||||
rotary_emb.is_neox_style,
|
||||
)
|
||||
return tensor.reshape(shape)
|
||||
|
||||
rotated = rotary_emb.apply_rotary_emb(
|
||||
tensor[..., :rotary_dim],
|
||||
cos,
|
||||
sin,
|
||||
)
|
||||
return torch.cat((rotated, tensor[..., rotary_dim:]), dim=-1)
|
||||
|
||||
|
||||
def apply_qsa_rmsnorm(
|
||||
norm: GemmaRMSNorm,
|
||||
tensor: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Use vLLM's portable RMSNorm implementation on ROCm."""
|
||||
|
||||
return cast(torch.Tensor, norm(tensor))
|
||||
|
||||
|
||||
class QSAIndexer(nn.Module):
|
||||
"""Replicated Q/K projection plus paged, weight-free QSA selection.
|
||||
|
||||
``prefix`` must be the checkpoint's indexer prefix, normally
|
||||
``model.layers.N.self_attn.indexer``. Consequently the trainable names are
|
||||
``index_qk_proj``, ``q_layernorm`` and ``k_layernorm`` under that prefix.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
vllm_config: VllmConfig,
|
||||
config: Qwen4ExpTextConfig,
|
||||
layer_id: int,
|
||||
rotary_emb: nn.Module,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if vllm_config.cache_config is None:
|
||||
raise ValueError("QSA requires a paged KV cache")
|
||||
if vllm_config.model_config.dtype != torch.bfloat16:
|
||||
raise NotImplementedError("Qwen4Exp QSA currently requires BF16")
|
||||
|
||||
self.layer_id = int(layer_id)
|
||||
self.index_n_heads = int(config.indexer_n_heads)
|
||||
self.index_kv_heads = int(config.indexer_kv_heads)
|
||||
self.index_head_dim = int(config.indexer_head_dim)
|
||||
self.token_topk = int(config.indexer_budget)
|
||||
self.compress_ratio = int(config.indexer_compress_ratio)
|
||||
self.rotary_emb = rotary_emb
|
||||
self.prefix = prefix
|
||||
# MTP step 0 selects the target-aligned rows; later steps reuse them
|
||||
# while continuing to update the QSA side cache.
|
||||
self.skip_topk = False
|
||||
|
||||
self.index_qk_proj = ReplicatedLinear(
|
||||
int(config.hidden_size),
|
||||
(self.index_n_heads + self.index_kv_heads) * self.index_head_dim,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.index_qk_proj" if prefix else "index_qk_proj",
|
||||
)
|
||||
self.q_layernorm = GemmaRMSNorm(
|
||||
self.index_head_dim,
|
||||
eps=float(getattr(config, "rms_norm_eps", 1e-6)),
|
||||
)
|
||||
self.k_layernorm = GemmaRMSNorm(
|
||||
self.index_head_dim,
|
||||
eps=float(getattr(config, "rms_norm_eps", 1e-6)),
|
||||
)
|
||||
|
||||
cache_config = vllm_config.cache_config
|
||||
cache_prefix = f"{prefix}." if prefix else ""
|
||||
self.raw_key_cache = QSAKeyStateCache(
|
||||
head_size=self.index_head_dim,
|
||||
dtype=torch.bfloat16,
|
||||
cache_rope_positions=vllm_config.model_config.uses_mrope,
|
||||
prefix=f"{cache_prefix}raw_key_cache",
|
||||
cache_config=cache_config,
|
||||
compress_ratio=self.compress_ratio,
|
||||
vllm_config=vllm_config,
|
||||
)
|
||||
self.compressed_key_cache = QSACompressedKeyCache(
|
||||
head_size=self.index_head_dim,
|
||||
dtype=torch.bfloat16,
|
||||
compress_ratio=self.compress_ratio,
|
||||
prefix=f"{cache_prefix}compressed_key_cache",
|
||||
cache_config=cache_config,
|
||||
vllm_config=vllm_config,
|
||||
)
|
||||
|
||||
@property
|
||||
def output_width(self) -> int:
|
||||
return self.token_topk + self.compress_ratio - 1
|
||||
|
||||
def project_qk(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Project replicated Q/K, normalize+rotate Q, and preserve raw K."""
|
||||
|
||||
qk, _ = self.index_qk_proj(hidden_states)
|
||||
q_raw, token_k = qk.split(
|
||||
(
|
||||
self.index_n_heads * self.index_head_dim,
|
||||
self.index_kv_heads * self.index_head_dim,
|
||||
),
|
||||
dim=-1,
|
||||
)
|
||||
q = q_raw.reshape(-1, self.index_n_heads, self.index_head_dim)
|
||||
q = apply_qsa_rmsnorm(
|
||||
self.q_layernorm,
|
||||
q.reshape(-1, self.index_head_dim),
|
||||
).reshape_as(q)
|
||||
q = apply_qsa_rope(self.rotary_emb, positions, q)
|
||||
return q, token_k.reshape(-1, 1, self.index_head_dim)
|
||||
|
||||
def normalize_compressed_keys(
|
||||
self,
|
||||
compressed_keys: torch.Tensor,
|
||||
first_rope_positions: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Normalize pooled K and apply the first token's exact group position."""
|
||||
|
||||
keys = compressed_keys.reshape(-1, self.index_head_dim)
|
||||
keys = apply_qsa_rmsnorm(self.k_layernorm, keys).reshape(
|
||||
-1, 1, self.index_head_dim
|
||||
)
|
||||
if getattr(self.rotary_emb, "mrope_section", None):
|
||||
positions = first_rope_positions.transpose(0, 1)
|
||||
else:
|
||||
positions = first_rope_positions[:, 0]
|
||||
return apply_qsa_rope(self.rotary_emb, positions, keys)
|
||||
|
||||
def _metadata(
|
||||
self,
|
||||
) -> tuple[QSAForwardMetadata, QSAForwardMetadata] | None:
|
||||
metadata = get_forward_context().attn_metadata
|
||||
if isinstance(metadata, list):
|
||||
metadata = metadata[0]
|
||||
if not isinstance(metadata, dict):
|
||||
return None
|
||||
raw = cast(QSAForwardMetadata, metadata[self.raw_key_cache.prefix])
|
||||
compressed = cast(
|
||||
QSAForwardMetadata, metadata[self.compressed_key_cache.prefix]
|
||||
)
|
||||
if raw.num_actual_tokens != compressed.num_actual_tokens:
|
||||
raise RuntimeError("QSA side-cache metadata token counts disagree")
|
||||
if not raw.logical_positions.is_cuda and (
|
||||
not torch.equal(raw.logical_positions, compressed.logical_positions)
|
||||
):
|
||||
raise RuntimeError("QSA side-cache metadata positions disagree")
|
||||
return raw, compressed
|
||||
|
||||
def _update_and_compress(
|
||||
self,
|
||||
token_k: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
raw_metadata: QSAForwardMetadata,
|
||||
compressed_metadata: QSAForwardMetadata,
|
||||
) -> None:
|
||||
num_tokens = raw_metadata.num_actual_tokens
|
||||
raw_key_cache = self.raw_key_cache.key_cache
|
||||
rope_position_cache = self.raw_key_cache.rope_position_cache
|
||||
from .ops.qsa import qsa_compress_groups_with_ratio, qsa_store_cache_rows
|
||||
|
||||
if rope_position_cache is None:
|
||||
position_rows = raw_metadata.logical_positions.view(-1, 1, 1).expand(
|
||||
-1, 1, 3
|
||||
)
|
||||
else:
|
||||
position_rows = canonical_qsa_rope_positions(positions)[:num_tokens].to(
|
||||
device=raw_key_cache.device
|
||||
)
|
||||
pooled, first_positions = qsa_compress_groups_with_ratio(
|
||||
token_k[:num_tokens],
|
||||
position_rows,
|
||||
raw_key_cache,
|
||||
raw_metadata.block_table,
|
||||
raw_metadata.token_to_req,
|
||||
raw_metadata.query_start_loc,
|
||||
raw_metadata.logical_positions,
|
||||
compressed_metadata.slot_mapping,
|
||||
self.compress_ratio,
|
||||
rope_position_cache,
|
||||
)
|
||||
normalized = self.normalize_compressed_keys(pooled, first_positions)
|
||||
qsa_store_cache_rows(
|
||||
self.compressed_key_cache.kv_cache,
|
||||
compressed_metadata.slot_mapping,
|
||||
normalized,
|
||||
)
|
||||
qsa_store_cache_rows(
|
||||
raw_key_cache,
|
||||
raw_metadata.slot_mapping,
|
||||
token_k[:num_tokens],
|
||||
)
|
||||
if rope_position_cache is not None:
|
||||
qsa_store_cache_rows(
|
||||
rope_position_cache,
|
||||
raw_metadata.slot_mapping,
|
||||
position_rows,
|
||||
)
|
||||
|
||||
def _select(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
metadata: QSAForwardMetadata,
|
||||
out: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
from .ops.qsa import qsa_select_paged_tokens
|
||||
|
||||
return qsa_select_paged_tokens(
|
||||
q,
|
||||
self.compressed_key_cache.kv_cache,
|
||||
metadata.block_table,
|
||||
metadata.token_to_req,
|
||||
metadata.logical_positions,
|
||||
metadata.seq_lens,
|
||||
self.token_topk,
|
||||
self.compress_ratio,
|
||||
out,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Return fixed-width request-relative token indices padded with ``-1``."""
|
||||
|
||||
metadata = self._metadata()
|
||||
if metadata is None:
|
||||
# Preserve step-0 indices when later MTP steps reuse the buffer.
|
||||
if self.skip_topk and out is not None:
|
||||
return out
|
||||
result = torch.full(
|
||||
(hidden_states.shape[0], self.output_width),
|
||||
-1,
|
||||
dtype=torch.int32,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
if out is not None:
|
||||
out.copy_(result)
|
||||
return out
|
||||
return result
|
||||
raw_metadata, compressed_metadata = metadata
|
||||
num_tokens = raw_metadata.num_actual_tokens
|
||||
q, token_k = self.project_qk(
|
||||
hidden_states[:num_tokens], positions[..., :num_tokens]
|
||||
)
|
||||
self._update_and_compress(
|
||||
token_k,
|
||||
positions[..., :num_tokens],
|
||||
raw_metadata,
|
||||
compressed_metadata,
|
||||
)
|
||||
if self.skip_topk:
|
||||
if out is None:
|
||||
raise RuntimeError("QSA top-k reuse requires an output buffer")
|
||||
return out
|
||||
return self._select(q, compressed_metadata, out)
|
||||
|
||||
|
||||
__all__ = ["QSAIndexer", "apply_qsa_rope"]
|
||||
@@ -0,0 +1,15 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Qwen4Exp low-latency GEMM hook for AMD ROCm."""
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
|
||||
def enable_qwen4_exp_low_latency_gemm(
|
||||
module: nn.Module,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
"""Keep the standard vLLM linear methods on AMD ROCm."""
|
||||
|
||||
del module, dtype
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,143 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Model-runner state for Qwen4Exp PLE inputs."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
|
||||
|
||||
class Qwen4ExpModelState(MambaHybridModelState):
|
||||
"""Add rollback-safe PLE n-gram context to the model inputs."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
model: nn.Module,
|
||||
encoder_cache: EncoderCache | None,
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
super().__init__(vllm_config, model, encoder_cache, device)
|
||||
config = self.model_config.hf_text_config
|
||||
self.uses_ngram_embedding = bool(config.ple_layer_ids)
|
||||
if not self.uses_ngram_embedding:
|
||||
self.ngram_context_len = 0
|
||||
self.ngram_eos_token_id = 0
|
||||
return
|
||||
|
||||
if vllm_config.parallel_config.pipeline_parallel_size > 1:
|
||||
raise RuntimeError(
|
||||
"N-gram PLE embedding currently requires "
|
||||
"pipeline_parallel_size=1 because non-first pipeline ranks do "
|
||||
"not receive the raw input_ids required by PLE. Please run "
|
||||
"with PP=1."
|
||||
)
|
||||
|
||||
self.ngram_context_len = int(config.ngram_size) - 1
|
||||
if self.ngram_context_len <= 0:
|
||||
raise ValueError("N-gram embedding requires context length >= 1.")
|
||||
self.ngram_eos_token_id = int(config.eos_token_id)
|
||||
self.ngram_context = torch.full(
|
||||
(self.max_num_reqs, self.ngram_context_len),
|
||||
self.ngram_eos_token_id,
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
self.ngram_context_offsets = torch.arange(
|
||||
-self.ngram_context_len,
|
||||
0,
|
||||
dtype=torch.int64,
|
||||
device=self.device,
|
||||
)
|
||||
self.ple_query_start_loc = torch.zeros(
|
||||
self.max_num_reqs + 1,
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
def _prepare_ngram_context(
|
||||
self,
|
||||
input_batch: InputBatch,
|
||||
req_states: RequestState,
|
||||
) -> torch.Tensor:
|
||||
num_reqs = input_batch.num_reqs
|
||||
num_reqs_padded = input_batch.num_reqs_after_padding
|
||||
context = self.ngram_context[:num_reqs_padded]
|
||||
context.fill_(self.ngram_eos_token_id)
|
||||
if num_reqs == 0:
|
||||
return context
|
||||
|
||||
request_indices = input_batch.idx_mapping[:num_reqs].long()
|
||||
context_end = req_states.num_computed_tokens.gpu[request_indices].long()
|
||||
token_indices = context_end.unsqueeze(1) + self.ngram_context_offsets
|
||||
valid_tokens = token_indices >= 0
|
||||
token_indices.clamp_min_(0)
|
||||
context_tokens = req_states.all_token_ids.gpu[
|
||||
request_indices.unsqueeze(1), token_indices
|
||||
]
|
||||
context[:num_reqs].copy_(
|
||||
torch.where(
|
||||
valid_tokens,
|
||||
context_tokens,
|
||||
context_tokens.new_full((), self.ngram_eos_token_id),
|
||||
)
|
||||
)
|
||||
return context
|
||||
|
||||
def prepare_inputs(
|
||||
self,
|
||||
input_batch: InputBatch,
|
||||
req_states: RequestState,
|
||||
) -> dict[str, Any]:
|
||||
model_inputs = super().prepare_inputs(input_batch, req_states)
|
||||
if not self.uses_ngram_embedding:
|
||||
return model_inputs
|
||||
|
||||
num_reqs_padded = input_batch.num_reqs_after_padding
|
||||
query_start_loc = self.ple_query_start_loc[: num_reqs_padded + 1]
|
||||
query_start_loc.copy_(input_batch.query_start_loc[: num_reqs_padded + 1])
|
||||
model_inputs.update(
|
||||
query_start_loc=query_start_loc,
|
||||
ngram_context=self._prepare_ngram_context(input_batch, req_states),
|
||||
)
|
||||
return model_inputs
|
||||
|
||||
def prepare_dummy_inputs(
|
||||
self,
|
||||
num_reqs: int,
|
||||
num_tokens: int,
|
||||
) -> dict[str, Any]:
|
||||
model_inputs = super().prepare_dummy_inputs(num_reqs, num_tokens)
|
||||
if not self.uses_ngram_embedding:
|
||||
return model_inputs
|
||||
|
||||
query_start_loc = self.ple_query_start_loc[: num_reqs + 1]
|
||||
query_start_loc[0] = 0
|
||||
tokens_per_req, num_extra_tokens = divmod(num_tokens, num_reqs)
|
||||
query_lens = torch.full(
|
||||
(num_reqs,),
|
||||
tokens_per_req,
|
||||
dtype=query_start_loc.dtype,
|
||||
device=query_start_loc.device,
|
||||
)
|
||||
if num_extra_tokens > 0:
|
||||
query_lens[-num_extra_tokens:] += 1
|
||||
torch.cumsum(query_lens, dim=0, out=query_start_loc[1:])
|
||||
|
||||
ngram_context = self.ngram_context[:num_reqs]
|
||||
ngram_context.fill_(self.ngram_eos_token_id)
|
||||
model_inputs.update(
|
||||
query_start_loc=query_start_loc,
|
||||
ngram_context=ngram_context,
|
||||
)
|
||||
return model_inputs
|
||||
|
||||
|
||||
__all__ = ["Qwen4ExpModelState"]
|
||||
@@ -0,0 +1,462 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inference-only Qwen4Exp MTP (Multi-Token Predictor) model.
|
||||
|
||||
The MTP draft model reuses the Qwen4Exp backbone (PLE/HC/MoE) but:
|
||||
- drops all multi-modal handling (text-only),
|
||||
- forces PLE off while keeping the main model's HC stream count,
|
||||
- fuses the backbone hidden and the new-token embedding via
|
||||
``residual_linear_shared`` (fc_embedding + shared fc_hidden) instead of
|
||||
the ``Linear(2H, H)`` + repeat used by other MTP variants,
|
||||
- emits TWO hidden streams per step (scheme A): a single stream [T, H]
|
||||
(final-mixer collapsed, fed to the LM head) and a pre-final-mixer
|
||||
multi stream [T, hc_count*H] (fed to the next draft step).
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig, replace, set_current_vllm_config
|
||||
from vllm.distributed import get_pp_group
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.utils import configure_quant_config
|
||||
from vllm.model_executor.models.interfaces import SupportsPP
|
||||
from vllm.model_executor.models.qwen3_5 import Qwen3_5Model
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
WeightsMapper,
|
||||
get_draft_quant_config,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
maybe_fuse_shared_experts,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.qwen4_exp import (
|
||||
Qwen4ExpTextConfig,
|
||||
)
|
||||
|
||||
from .hyperconnection import GatedResidual, HyperConnectionConfig
|
||||
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
|
||||
from .model import (
|
||||
_HC_WEIGHTS_MAPPER,
|
||||
_QWEN4_EXP_IGNORED_MISSING_SUFFIXES,
|
||||
Qwen4ExpDecoderLayer,
|
||||
Qwen4ExpMixtureOfExperts,
|
||||
Qwen4ExpSparseMoeBlock,
|
||||
)
|
||||
|
||||
|
||||
def _remap_ignored_layers(
|
||||
ignored_layers: list[str],
|
||||
mtp_start_layer_idx: int,
|
||||
) -> list[str]:
|
||||
remapped: list[str] = []
|
||||
for name in ignored_layers:
|
||||
if name.startswith("mtp."):
|
||||
new_name = re.sub(
|
||||
r"(?<=\.layers\.)\d+",
|
||||
lambda m: str(mtp_start_layer_idx + int(m.group(0))),
|
||||
name,
|
||||
)
|
||||
remapped.append(new_name)
|
||||
else:
|
||||
remapped.append(name)
|
||||
return remapped
|
||||
|
||||
|
||||
def _remap_mtp_weight_name(name: str) -> str | None:
|
||||
"""Map Qwen4Exp checkpoint paths into the standalone draft model."""
|
||||
|
||||
for checkpoint_prefix in (
|
||||
"model.language_model.",
|
||||
"language_model.",
|
||||
):
|
||||
if name.startswith(checkpoint_prefix):
|
||||
name = name.removeprefix(checkpoint_prefix)
|
||||
break
|
||||
|
||||
if name.startswith("embed_tokens."):
|
||||
name = f"model.{name}"
|
||||
if name.startswith("model.mtp."):
|
||||
name = name.removeprefix("model.")
|
||||
if name.startswith("mtp.shared_head.head."):
|
||||
return name.replace("mtp.shared_head.head.", "lm_head.", 1)
|
||||
if name.startswith("model.shared_head.head."):
|
||||
return name.replace("model.shared_head.head.", "lm_head.", 1)
|
||||
if name.startswith("shared_head.head."):
|
||||
return name.replace("shared_head.head.", "lm_head.", 1)
|
||||
if name.startswith("model.lm_head."):
|
||||
return name.removeprefix("model.")
|
||||
if name.startswith("mtp."):
|
||||
return name.replace("mtp.", "model.", 1)
|
||||
if name.startswith("model.embed_tokens.") or name.startswith("lm_head."):
|
||||
return name
|
||||
return None
|
||||
|
||||
|
||||
def _make_draft_vllm_config(
|
||||
vllm_config: VllmConfig,
|
||||
mtp_start_layer_idx: int,
|
||||
) -> VllmConfig:
|
||||
"""Ensure that the draft model config is set in the vLLM config."""
|
||||
speculative_config = vllm_config.speculative_config
|
||||
if speculative_config is None or speculative_config.draft_model_config is None:
|
||||
raise ValueError("speculative_config.draft_model_config must be set")
|
||||
|
||||
draft_quant_config = get_draft_quant_config(vllm_config)
|
||||
|
||||
# inject packed and ignored modules to the quantization config of draft model
|
||||
if draft_quant_config is not None:
|
||||
configure_quant_config(draft_quant_config, Qwen4ExpMTP)
|
||||
ignored_layers = getattr(draft_quant_config, "ignored_layers", None)
|
||||
if ignored_layers:
|
||||
setattr( # noqa: B010
|
||||
draft_quant_config,
|
||||
"ignored_layers",
|
||||
_remap_ignored_layers(ignored_layers, mtp_start_layer_idx),
|
||||
)
|
||||
exclude_modules = getattr(draft_quant_config, "exclude_modules", None)
|
||||
if exclude_modules:
|
||||
setattr( # noqa: B010
|
||||
draft_quant_config,
|
||||
"exclude_modules",
|
||||
_remap_ignored_layers(exclude_modules, mtp_start_layer_idx),
|
||||
)
|
||||
|
||||
draft_vllm_config = replace(
|
||||
vllm_config,
|
||||
model_config=speculative_config.draft_model_config,
|
||||
)
|
||||
# VllmConfig post-init derives the target quant config, so restore the
|
||||
# independently resolved draft quant config after replacement.
|
||||
draft_vllm_config.quant_config = draft_quant_config
|
||||
return draft_vllm_config
|
||||
|
||||
|
||||
@support_torch_compile(
|
||||
dynamic_arg_dims={
|
||||
"input_ids": 0,
|
||||
"positions": -1,
|
||||
"intermediate_tensors": 0,
|
||||
"inputs_embeds": 0,
|
||||
"hidden_states": 0,
|
||||
}
|
||||
)
|
||||
class Qwen4ExpMultiTokenPredictor(nn.Module):
|
||||
hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _HC_WEIGHTS_MAPPER
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
|
||||
model_config = vllm_config.model_config
|
||||
config: Qwen4ExpTextConfig = model_config.hf_text_config
|
||||
|
||||
self.config = config
|
||||
self.vocab_size = config.vocab_size
|
||||
|
||||
self.mtp_start_layer_idx = config.num_hidden_layers
|
||||
self.num_mtp_layers = getattr(config, "mtp_num_hidden_layers", 1)
|
||||
|
||||
self.hidden_size = config.hidden_size
|
||||
self.hc_count = config.hc_count
|
||||
|
||||
self.embed_tokens = VocabParallelEmbedding(self.vocab_size, self.hidden_size)
|
||||
draft_vllm_config = _make_draft_vllm_config(
|
||||
vllm_config,
|
||||
self.mtp_start_layer_idx,
|
||||
)
|
||||
with set_current_vllm_config(draft_vllm_config, prefix=prefix):
|
||||
# residual_linear_shared fusion: fc_embedding projects the token
|
||||
# embedding, fc_hidden (shared across HC branches) projects the
|
||||
# backbone hidden; the embedding is added as a residual to every
|
||||
# branch (see mtp_residual_linear_shared.md).
|
||||
self.fc_embedding = ColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
self.hidden_size,
|
||||
gather_output=True,
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
quant_config=draft_vllm_config.quant_config,
|
||||
prefix=f"{prefix}.fc_embedding",
|
||||
)
|
||||
self.fc_hidden = ColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
self.hidden_size,
|
||||
gather_output=True,
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
quant_config=draft_vllm_config.quant_config,
|
||||
prefix=f"{prefix}.fc_hidden",
|
||||
)
|
||||
self.layers = nn.ModuleList(
|
||||
Qwen4ExpDecoderLayer(
|
||||
draft_vllm_config,
|
||||
layer_type="full_attention",
|
||||
prefix=f"{prefix}.layers.{self.mtp_start_layer_idx + idx}",
|
||||
)
|
||||
for idx in range(self.num_mtp_layers)
|
||||
)
|
||||
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
|
||||
self.layers,
|
||||
Qwen4ExpSparseMoeBlock,
|
||||
"mlp",
|
||||
)
|
||||
|
||||
self.pre_fc_norm_embedding = GemmaRMSNorm(
|
||||
self.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.pre_fc_norm_hidden = GemmaRMSNorm(
|
||||
self.hidden_size * self.hc_count, eps=config.rms_norm_eps
|
||||
)
|
||||
# HC final mixer collapses the multi stream into [T, H] for the LM head.
|
||||
hc_config = HyperConnectionConfig(
|
||||
hc_count=config.hc_count,
|
||||
hidden_size=config.hidden_size,
|
||||
params_dtype=torch.bfloat16,
|
||||
hc_lowrank=config.hc_lowrank,
|
||||
rms_norm_eps=config.rms_norm_eps,
|
||||
hc_per_branch_norm=True,
|
||||
)
|
||||
self.hyper_connection_mixer = GatedResidual(
|
||||
hc_config,
|
||||
use_combine=False,
|
||||
prefix=maybe_prefix(prefix, "hyper_connection_mixer"),
|
||||
)
|
||||
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
|
||||
["hidden_states"], self.hidden_size * self.hc_count
|
||||
)
|
||||
|
||||
def _iter_qsa_attentions(self):
|
||||
"""Yield MTP attention modules that own a QSA indexer."""
|
||||
|
||||
for layer in self.layers:
|
||||
attention = getattr(layer, "self_attn", None)
|
||||
if (
|
||||
attention is not None
|
||||
and getattr(attention, "indexer", None) is not None
|
||||
):
|
||||
yield attention
|
||||
|
||||
def set_skip_topk(self, skip: bool) -> None:
|
||||
"""Select on MTP step 0 and reuse its QSA indices on later steps."""
|
||||
|
||||
for attention in self._iter_qsa_attentions():
|
||||
attention.indexer.skip_topk = skip
|
||||
|
||||
def compact_topk_indices(self, row_indices: torch.Tensor) -> None:
|
||||
"""Keep each request's target-aligned step-0 sparse-index row."""
|
||||
|
||||
num_rows = row_indices.numel()
|
||||
for attention in self._iter_qsa_attentions():
|
||||
buffer = attention.topk_indices_buffer
|
||||
selected = buffer.index_select(0, row_indices)
|
||||
buffer[:num_rows].copy_(selected)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor | None = None,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor] | IntermediateTensors:
|
||||
hc_count = self.hc_count
|
||||
hidden_size = self.hidden_size
|
||||
|
||||
if get_pp_group().is_first_rank:
|
||||
assert hidden_states is not None
|
||||
if inputs_embeds is None:
|
||||
assert input_ids is not None
|
||||
inputs_embeds = self.embed_input_ids(input_ids)
|
||||
# Embedding branch: pre-norm -> fc_embedding -> [T, H].
|
||||
inputs_embeds = self.pre_fc_norm_embedding(inputs_embeds)
|
||||
inputs_embeds = self.fc_embedding(inputs_embeds)
|
||||
|
||||
# Backbone hidden is multi-stream [T, hc_count*H] (scheme A:
|
||||
# the main model truly emits the pre-final-mixer multi stream
|
||||
# on the first step; subsequent steps reuse the prior draft
|
||||
# step's multi stream).
|
||||
num_tokens = hidden_states.shape[0]
|
||||
hidden_states = hidden_states.view(num_tokens, hc_count, hidden_size)
|
||||
hidden_states = self.pre_fc_norm_hidden(hidden_states.flatten(-2)).view(
|
||||
num_tokens, hc_count, hidden_size
|
||||
)
|
||||
hidden_states = self.fc_hidden(hidden_states)
|
||||
# Add the embedding residual to every branch, then fold back
|
||||
# to [T, hc_count*H] (HC outer, HS inner) for the HC decoder.
|
||||
hidden_states = inputs_embeds.unsqueeze(-2) + hidden_states
|
||||
hidden_states = hidden_states.flatten(-2)
|
||||
else:
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
layer = self.layers[current_step_idx]
|
||||
hidden_states, block_output, injection = layer(
|
||||
hidden_states=hidden_states,
|
||||
prev_block_output=None,
|
||||
prev_injection=None,
|
||||
positions=positions,
|
||||
input_ids=None,
|
||||
query_start_loc=None,
|
||||
ngram_context=None,
|
||||
)
|
||||
if not get_pp_group().is_last_rank:
|
||||
# As in the target model, PP carries a materialized tensor rather
|
||||
# than the delayed hidden/output/injection tuple.
|
||||
hidden_states = layer.mlp_hyper_connection.combine(
|
||||
hidden_states, block_output, injection
|
||||
)
|
||||
return IntermediateTensors({"hidden_states": hidden_states})
|
||||
|
||||
# Last PP rank finalize. Keep both:
|
||||
# (A) sample_hidden_states [T, H] -> single stream for the LM head
|
||||
# (B) multi_hidden [T, hc_count*H] -> pre-final-mixer multi stream
|
||||
# for the next draft step (zero extra compute, just kept).
|
||||
multi_hidden, sample_hidden_states, _ = (
|
||||
self.hyper_connection_mixer.combine_and_mix(
|
||||
hidden_states, block_output, injection
|
||||
)
|
||||
)
|
||||
return sample_hidden_states, multi_hidden
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
weights = maybe_fuse_shared_experts(
|
||||
weights,
|
||||
enabled=self.is_fused_shared_expert_enabled,
|
||||
n_routed_experts=getattr(self.config, "num_experts", 0) or 0,
|
||||
n_shared_experts=1,
|
||||
ckpt_prefix="mlp.shared_expert",
|
||||
)
|
||||
mapper = self.hf_to_vllm_mapper | WeightsMapper(
|
||||
orig_to_new_substr={"hyper_connection_mixer.block_inject_weight": None}
|
||||
)
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(),
|
||||
)
|
||||
return loader.load_weights(weights, mapper=mapper)
|
||||
|
||||
|
||||
@support_torch_compile(
|
||||
dynamic_arg_dims={
|
||||
"input_ids": 0,
|
||||
"positions": -1,
|
||||
"intermediate_tensors": 0,
|
||||
"inputs_embeds": 0,
|
||||
"hidden_states": 0,
|
||||
}
|
||||
)
|
||||
class Qwen4ExpMTP(nn.Module, SupportsPP, Qwen4ExpMixtureOfExperts):
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||
"in_proj_qkvz": ["in_proj_qkv", "in_proj_z"],
|
||||
"in_proj_ba": ["in_proj_b", "in_proj_a"],
|
||||
"input_mix_weight_down_block_inject": [
|
||||
"input_mix_weight_down",
|
||||
"block_inject_weight",
|
||||
"_input_mix_padding",
|
||||
],
|
||||
}
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config
|
||||
self.vllm_config = vllm_config
|
||||
cache_config = vllm_config.cache_config
|
||||
if cache_config.mamba_cache_mode == "all":
|
||||
raise NotImplementedError(
|
||||
"Qwen4ExpMTP currently does not support 'all' prefix caching, "
|
||||
"please use '--mamba-cache-mode=align' instead"
|
||||
)
|
||||
|
||||
self.quant_config = vllm_config.quant_config
|
||||
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.model = Qwen4ExpMultiTokenPredictor(
|
||||
vllm_config=vllm_config,
|
||||
prefix=maybe_prefix(prefix, "mtp"),
|
||||
)
|
||||
|
||||
if get_pp_group().is_last_rank:
|
||||
if config.tie_word_embeddings:
|
||||
self.lm_head = self.model.embed_tokens
|
||||
else:
|
||||
self.lm_head = ParallelLMHead(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
else:
|
||||
self.lm_head = PPMissingLayer()
|
||||
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size)
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.model.make_empty_intermediate_tensors
|
||||
)
|
||||
self.set_moe_parameters(self.model.layers)
|
||||
enable_qwen4_exp_low_latency_gemm(self, vllm_config.model_config.dtype)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor | None = None,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor] | IntermediateTensors:
|
||||
return self.model(
|
||||
input_ids,
|
||||
positions,
|
||||
hidden_states,
|
||||
intermediate_tensors,
|
||||
inputs_embeds,
|
||||
spec_step_idx=spec_step_idx,
|
||||
)
|
||||
|
||||
def compute_logits(
|
||||
self, hidden_states: torch.Tensor, spec_step_idx: int = 0
|
||||
) -> torch.Tensor | None:
|
||||
return self.logits_processor(self.lm_head, hidden_states)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
def remap_weight_names():
|
||||
for name, weight in weights:
|
||||
remapped_name = _remap_mtp_weight_name(name)
|
||||
if remapped_name is not None:
|
||||
yield remapped_name, weight
|
||||
|
||||
mapper = WeightsMapper(
|
||||
orig_to_new_substr={"hyper_connection_mixer.block_inject_weight": None}
|
||||
)
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(),
|
||||
)
|
||||
return loader.load_weights(remap_weight_names(), mapper=mapper)
|
||||
|
||||
|
||||
__all__ = ["Qwen4ExpMTP", "Qwen4ExpMultiTokenPredictor"]
|
||||
@@ -0,0 +1,3 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""AMD ROCm Qwen4Exp kernels; import leaf modules directly."""
|
||||
@@ -0,0 +1,487 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""AMD ROCm HyperConnection kernels for Qwen4Exp."""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _grouped_gemma_rmsnorm_kernel(
|
||||
x_ptr,
|
||||
w_ptr,
|
||||
y_ptr,
|
||||
stride_x,
|
||||
stride_y,
|
||||
DIM: tl.constexpr,
|
||||
NUM_GROUPS: tl.constexpr,
|
||||
W_SHARED: tl.constexpr,
|
||||
EPS: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
) -> None:
|
||||
GROUP_DIM: tl.constexpr = DIM // NUM_GROUPS
|
||||
BLOCK_SIZE: tl.constexpr = triton.next_power_of_2(GROUP_DIM)
|
||||
|
||||
pid = tl.program_id(0)
|
||||
group_id = pid % NUM_GROUPS
|
||||
row = pid // NUM_GROUPS
|
||||
|
||||
offs_g = tl.arange(0, BLOCK_SIZE)
|
||||
offsets = group_id * GROUP_DIM + offs_g
|
||||
mask = offs_g < GROUP_DIM
|
||||
# A [GROUP_DIM] affine is shared; a [DIM] affine follows the grouped
|
||||
# checkpoint layout.
|
||||
w_offs = offs_g if W_SHARED else offsets
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
|
||||
x = tl.load(x_ptr + row * stride_x + offsets, mask, other=0.0).to(tl.float32)
|
||||
w = tl.load(w_ptr + w_offs, mask, other=0.0)
|
||||
|
||||
rrms = tl.rsqrt(tl.sum(x * x) / GROUP_DIM + EPS)
|
||||
# Gemma's (1 + w) affine is written this way to lower to an FMA.
|
||||
y = x * rrms
|
||||
y += y * w.to(tl.float32)
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
tl.store(y_ptr + row * stride_y + offsets, y, mask)
|
||||
|
||||
|
||||
def _grouped_gemma_rmsnorm(
|
||||
x: torch.Tensor, weight: torch.Tensor, eps: float, num_groups: int
|
||||
) -> torch.Tensor:
|
||||
N, DIM = x.shape
|
||||
assert x.stride(1) == 1, "grouped Gemma RMSNorm requires unit inner stride"
|
||||
assert weight.is_contiguous(), "grouped Gemma RMSNorm weight must be contiguous"
|
||||
assert DIM % num_groups == 0
|
||||
group_dim = DIM // num_groups
|
||||
assert weight.numel() in (group_dim, DIM)
|
||||
|
||||
y = x.new_empty(x.shape)
|
||||
_grouped_gemma_rmsnorm_kernel[(N * num_groups,)](
|
||||
x,
|
||||
weight,
|
||||
y,
|
||||
x.stride(0),
|
||||
y.stride(0),
|
||||
DIM,
|
||||
num_groups,
|
||||
W_SHARED=weight.numel() == group_dim,
|
||||
EPS=eps,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
)
|
||||
return y
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _hc_silu_kernel(
|
||||
x_ptr,
|
||||
y_ptr,
|
||||
stride_x,
|
||||
stride_y,
|
||||
DIM: tl.constexpr,
|
||||
HC: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
) -> None:
|
||||
BLOCK_SIZE: tl.constexpr = triton.next_power_of_2(DIM)
|
||||
|
||||
row = tl.program_id(0)
|
||||
offs = tl.arange(0, BLOCK_SIZE)
|
||||
mask = offs < DIM
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
|
||||
x = tl.load(x_ptr + row * stride_x + offs, mask).to(tl.float32) / HC
|
||||
y = x * tl.sigmoid(x)
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
tl.store(y_ptr + row * stride_y + offs, y, mask)
|
||||
|
||||
|
||||
def _hc_silu(x: torch.Tensor, hc_count: int) -> torch.Tensor:
|
||||
num_tokens, DIM = x.shape
|
||||
assert x.stride(1) == 1
|
||||
|
||||
output = x.new_empty(x.shape)
|
||||
_hc_silu_kernel[(num_tokens,)](
|
||||
x,
|
||||
output,
|
||||
x.stride(0),
|
||||
output.stride(0),
|
||||
DIM=DIM,
|
||||
HC=hc_count,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _hc_gate_mix_kernel(
|
||||
x_ptr,
|
||||
g_ptr,
|
||||
y_ptr,
|
||||
stride_x,
|
||||
stride_g,
|
||||
stride_y,
|
||||
DIM: tl.constexpr,
|
||||
HC: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
) -> None:
|
||||
HC_DIM: tl.constexpr = DIM // HC
|
||||
|
||||
row = tl.program_id(0)
|
||||
tile_id = tl.program_id(1)
|
||||
offs_inner = tile_id * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offs_inner < HC_DIM
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
|
||||
# The constexpr loop is unrolled and keeps one stream live at a time.
|
||||
# Materializing [HC, BLOCK_SIZE] more than doubles latency at large M.
|
||||
acc = tl.zeros([BLOCK_SIZE], dtype=tl.float32)
|
||||
for stream in tl.static_range(HC):
|
||||
offsets = stream * HC_DIM + offs_inner
|
||||
g = tl.load(g_ptr + row * stride_g + offsets, mask, other=0.0)
|
||||
x = tl.load(x_ptr + row * stride_x + offsets, mask, other=0.0)
|
||||
acc += tl.sigmoid(g.to(tl.float32)) * x.to(tl.float32)
|
||||
acc /= HC
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
tl.store(y_ptr + row * stride_y + offs_inner, acc, mask)
|
||||
|
||||
|
||||
def _hc_gate_mix(x: torch.Tensor, gate: torch.Tensor, hc_count: int) -> torch.Tensor:
|
||||
N, DIM = gate.shape
|
||||
assert x.shape == gate.shape
|
||||
assert DIM % hc_count == 0
|
||||
assert x.stride(1) == 1
|
||||
assert gate.stride(1) == 1
|
||||
|
||||
HC_DIM = DIM // hc_count
|
||||
out = x.new_empty(N, HC_DIM)
|
||||
BLOCK_SIZE = 512
|
||||
_hc_gate_mix_kernel[(N, triton.cdiv(HC_DIM, BLOCK_SIZE))](
|
||||
x,
|
||||
gate,
|
||||
out,
|
||||
x.stride(0),
|
||||
gate.stride(0),
|
||||
out.stride(0),
|
||||
DIM,
|
||||
hc_count,
|
||||
BLOCK_SIZE,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _hc_combine_kernel(
|
||||
block_ptr,
|
||||
res_ptr,
|
||||
inj_ptr,
|
||||
out_ptr,
|
||||
stride_block,
|
||||
stride_res,
|
||||
stride_inj,
|
||||
stride_out,
|
||||
HC_DIM: tl.constexpr,
|
||||
HC: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
) -> None:
|
||||
HC_PAD: tl.constexpr = triton.next_power_of_2(HC)
|
||||
|
||||
row = tl.program_id(0)
|
||||
tile_id = tl.program_id(1)
|
||||
|
||||
offs_inner = tile_id * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask_inner = offs_inner < HC_DIM
|
||||
offs_hc = tl.arange(0, HC_PAD)
|
||||
mask_hc = offs_hc < HC
|
||||
offs = offs_hc[:, None] * HC_DIM + offs_inner[None, :]
|
||||
mask = mask_hc[:, None] & mask_inner[None, :]
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
|
||||
inj = tl.load(inj_ptr + row * stride_inj + offs_hc, mask_hc, other=0.0)
|
||||
block = tl.load(block_ptr + row * stride_block + offs_inner, mask_inner, other=0.0)
|
||||
res = tl.load(res_ptr + row * stride_res + offs, mask, other=0.0)
|
||||
|
||||
# Keeping HC as a broadcast dimension is faster here than four separate
|
||||
# residual load/store sequences.
|
||||
inj = 2.0 * tl.sigmoid(inj.to(tl.float32) / HC)
|
||||
out = res.to(tl.float32) + block.to(tl.float32)[None, :] * inj[:, None]
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
tl.store(out_ptr + row * stride_out + offs, out, mask=mask)
|
||||
|
||||
|
||||
def _hc_combine(
|
||||
residual: torch.Tensor,
|
||||
block_output: torch.Tensor,
|
||||
injection_logits: torch.Tensor,
|
||||
hc_count: int,
|
||||
) -> torch.Tensor:
|
||||
N, DIM = residual.shape
|
||||
assert DIM % hc_count == 0
|
||||
hc_dim = DIM // hc_count
|
||||
assert block_output.shape == (N, hc_dim)
|
||||
assert injection_logits.shape == (N, hc_count)
|
||||
assert residual.stride(1) == 1
|
||||
assert block_output.stride(1) == 1
|
||||
assert injection_logits.stride(1) == 1
|
||||
|
||||
out = residual.new_empty(residual.shape)
|
||||
BLOCK_SIZE = 512
|
||||
_hc_combine_kernel[(N, triton.cdiv(hc_dim, BLOCK_SIZE))](
|
||||
block_output,
|
||||
residual,
|
||||
injection_logits,
|
||||
out,
|
||||
block_output.stride(0),
|
||||
residual.stride(0),
|
||||
injection_logits.stride(0),
|
||||
out.stride(0),
|
||||
hc_dim,
|
||||
hc_count,
|
||||
BLOCK_SIZE,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _hc_combine_norm_kernel(
|
||||
block_ptr,
|
||||
res_ptr,
|
||||
inj_ptr,
|
||||
w_ptr,
|
||||
out_ptr,
|
||||
y_ptr,
|
||||
stride_block,
|
||||
stride_res,
|
||||
stride_inj,
|
||||
stride_out,
|
||||
stride_y,
|
||||
HC_DIM: tl.constexpr,
|
||||
HC: tl.constexpr,
|
||||
W_SHARED: tl.constexpr,
|
||||
EPS: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
) -> None:
|
||||
HC_PAD: tl.constexpr = triton.next_power_of_2(HC)
|
||||
NUM_TILES: tl.constexpr = triton.cdiv(HC_DIM, BLOCK_SIZE)
|
||||
NUM_TILES_PAD: tl.constexpr = triton.next_power_of_2(NUM_TILES)
|
||||
|
||||
row = tl.program_id(0)
|
||||
stream = tl.program_id(1)
|
||||
offs_hc = tl.arange(0, HC_PAD)
|
||||
mask_hc = offs_hc < HC
|
||||
tile_ids = tl.arange(0, NUM_TILES_PAD)
|
||||
offs_inner = tile_ids[:, None] * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)[None, :]
|
||||
mask_inner = offs_inner < HC_DIM
|
||||
offs = stream * HC_DIM + offs_inner
|
||||
# Shared norm weights repeat across streams; per-branch weights use the
|
||||
# same flattened HC layout as the residual.
|
||||
w_offs = offs_inner if W_SHARED else offs
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
|
||||
# Start the uncached residual load first, then issue the other combine
|
||||
# loads before consuming any of them.
|
||||
res = tl.load(res_ptr + row * stride_res + offs, mask_inner, other=0.0)
|
||||
inj = tl.load(inj_ptr + row * stride_inj + offs_hc, mask_hc, other=0.0)
|
||||
block = tl.load(block_ptr + row * stride_block + offs_inner, mask_inner, other=0.0)
|
||||
inj = 2.0 * tl.sigmoid(inj.to(tl.float32) / HC)
|
||||
inj = tl.sum(tl.where(offs_hc == stream, inj, 0.0))
|
||||
# Round the materialized combine result before normalization. This matches
|
||||
# the unfused combine -> RMSNorm boundary.
|
||||
out = (res.to(tl.float32) + block.to(tl.float32) * inj).to(out_ptr.dtype.element_ty)
|
||||
tl.store(out_ptr + row * stride_out + offs, out, mask=mask_inner)
|
||||
|
||||
out = out.to(tl.float32)
|
||||
# Keep the two-axis reduction: flattening the padded tile is ~40% slower
|
||||
# at decode sizes.
|
||||
sum_sq = tl.sum(tl.sum(out * out, axis=1), axis=0)
|
||||
rrms = tl.rsqrt(sum_sq / HC_DIM + EPS)
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
# Loading the weight earlier helps decode but keeps the tile live across
|
||||
# the reduction and regresses larger batches, so defer it to the norm.
|
||||
w = tl.load(w_ptr + w_offs, mask_inner, other=0.0)
|
||||
y = out * rrms
|
||||
y += y * w.to(tl.float32)
|
||||
tl.store(y_ptr + row * stride_y + offs, y, mask_inner)
|
||||
|
||||
|
||||
def _hc_combine_norm(
|
||||
residual: torch.Tensor,
|
||||
block_output: torch.Tensor,
|
||||
injection_logits: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
eps: float,
|
||||
hc_count: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
N, DIM = residual.shape
|
||||
assert DIM % hc_count == 0
|
||||
hc_dim = DIM // hc_count
|
||||
assert block_output.shape == (N, hc_dim)
|
||||
assert injection_logits.shape == (N, hc_count)
|
||||
assert residual.stride(1) == 1
|
||||
assert block_output.stride(1) == 1
|
||||
assert injection_logits.stride(1) == 1
|
||||
assert norm_weight.is_contiguous()
|
||||
assert norm_weight.numel() in (hc_dim, DIM)
|
||||
|
||||
out = residual.new_empty(residual.shape)
|
||||
y = residual.new_empty(residual.shape)
|
||||
BLOCK_SIZE = 512
|
||||
_hc_combine_norm_kernel[(N, hc_count)](
|
||||
block_output,
|
||||
residual,
|
||||
injection_logits,
|
||||
norm_weight,
|
||||
out,
|
||||
y,
|
||||
block_output.stride(0),
|
||||
residual.stride(0),
|
||||
injection_logits.stride(0),
|
||||
out.stride(0),
|
||||
y.stride(0),
|
||||
hc_dim,
|
||||
hc_count,
|
||||
W_SHARED=norm_weight.numel() == hc_dim,
|
||||
EPS=eps,
|
||||
BLOCK_SIZE=BLOCK_SIZE,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
)
|
||||
return out, y
|
||||
|
||||
|
||||
def _same_shape_fake(x: torch.Tensor, *args) -> torch.Tensor:
|
||||
return x.new_empty(x.shape)
|
||||
|
||||
|
||||
def _hc_gate_mix_fake(
|
||||
x: torch.Tensor, gate: torch.Tensor, hc_count: int
|
||||
) -> torch.Tensor:
|
||||
del gate
|
||||
return x.new_empty((x.shape[0], x.shape[1] // hc_count))
|
||||
|
||||
|
||||
def _hc_combine_fake(
|
||||
residual: torch.Tensor,
|
||||
block_output: torch.Tensor,
|
||||
injection_logits: torch.Tensor,
|
||||
hc_count: int,
|
||||
) -> torch.Tensor:
|
||||
del block_output, injection_logits, hc_count
|
||||
return residual.new_empty(residual.shape)
|
||||
|
||||
|
||||
def _hc_combine_norm_fake(
|
||||
residual: torch.Tensor,
|
||||
block_output: torch.Tensor,
|
||||
injection_logits: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
eps: float,
|
||||
hc_count: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
del block_output, injection_logits, norm_weight, eps, hc_count
|
||||
return residual.new_empty(residual.shape), residual.new_empty(residual.shape)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_grouped_gemma_rmsnorm",
|
||||
op_func=_grouped_gemma_rmsnorm,
|
||||
fake_impl=_same_shape_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_hc_silu",
|
||||
op_func=_hc_silu,
|
||||
fake_impl=_same_shape_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_hc_gate_mix",
|
||||
op_func=_hc_gate_mix,
|
||||
fake_impl=_hc_gate_mix_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_hc_combine",
|
||||
op_func=_hc_combine,
|
||||
fake_impl=_hc_combine_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_hc_combine_norm",
|
||||
op_func=_hc_combine_norm,
|
||||
fake_impl=_hc_combine_norm_fake,
|
||||
)
|
||||
|
||||
|
||||
def grouped_gemma_rmsnorm(
|
||||
x: torch.Tensor, weight: torch.Tensor, eps: float, num_groups: int
|
||||
) -> torch.Tensor:
|
||||
return torch.ops.vllm.qwen4_exp_grouped_gemma_rmsnorm(x, weight, eps, num_groups)
|
||||
|
||||
|
||||
def hc_silu(x: torch.Tensor, hc_count: int) -> torch.Tensor:
|
||||
return torch.ops.vllm.qwen4_exp_hc_silu(x, hc_count)
|
||||
|
||||
|
||||
def hc_gate_mix(x: torch.Tensor, gate: torch.Tensor, hc_count: int) -> torch.Tensor:
|
||||
return torch.ops.vllm.qwen4_exp_hc_gate_mix(x, gate, hc_count)
|
||||
|
||||
|
||||
def hc_combine(
|
||||
residual: torch.Tensor,
|
||||
block_output: torch.Tensor,
|
||||
injection_logits: torch.Tensor,
|
||||
hc_count: int,
|
||||
) -> torch.Tensor:
|
||||
return torch.ops.vllm.qwen4_exp_hc_combine(
|
||||
residual, block_output, injection_logits, hc_count
|
||||
)
|
||||
|
||||
|
||||
def hc_combine_norm(
|
||||
residual: torch.Tensor,
|
||||
block_output: torch.Tensor,
|
||||
injection_logits: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
eps: float,
|
||||
hc_count: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return torch.ops.vllm.qwen4_exp_hc_combine_norm(
|
||||
residual,
|
||||
block_output,
|
||||
injection_logits,
|
||||
norm_weight,
|
||||
eps,
|
||||
hc_count,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"grouped_gemma_rmsnorm",
|
||||
"hc_combine",
|
||||
"hc_combine_norm",
|
||||
"hc_gate_mix",
|
||||
"hc_silu",
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,474 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""AMD ROCm QSA owner with Triton kernels."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import ClassVar, cast
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.attention.attention import (
|
||||
set_default_quant_scales,
|
||||
)
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
|
||||
from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.models.qwen3_next import Qwen3NextAttention
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.configs.qwen4_exp import (
|
||||
Qwen4ExpTextConfig,
|
||||
)
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
_resolve_layer_name,
|
||||
canonicalize_singleton_dim_strides,
|
||||
direct_register_custom_op,
|
||||
kv_cache_dtype_str_to_dtype,
|
||||
)
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionCGSupport,
|
||||
AttentionType,
|
||||
MultipleOf,
|
||||
)
|
||||
from vllm.v1.attention.backends.fa_utils import is_flash_attn_varlen_func_available
|
||||
from vllm.v1.attention.backends.flash_attn import (
|
||||
FlashAttentionBackend,
|
||||
FlashAttentionImpl,
|
||||
FlashAttentionMetadata,
|
||||
FlashAttentionMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheSpec,
|
||||
get_kv_quant_mode,
|
||||
)
|
||||
|
||||
from ..common.qsa_cache import QSAForwardMetadata
|
||||
from . import model
|
||||
from .indexer_qsa import QSAIndexer
|
||||
|
||||
|
||||
class Qwen4ExpQSAMetadataBuilder(FlashAttentionMetadataBuilder):
|
||||
"""Flash metadata supporting uniform decode and target-verify graphs."""
|
||||
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
||||
|
||||
|
||||
class Qwen4ExpQSAFlashAttentionBackend(FlashAttentionBackend):
|
||||
"""FullAttentionSpec backend used by the merged QSA owner."""
|
||||
|
||||
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
|
||||
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = ["auto", "bfloat16"]
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "QWEN4_EXP_QSA_TRITON"
|
||||
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes() -> list[int | MultipleOf]:
|
||||
# QSA consumes manager pages directly and does not use FA4 paged attention.
|
||||
return [MultipleOf(16)]
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls() -> type[Qwen4ExpQSAFlashAttentionImpl]:
|
||||
return Qwen4ExpQSAFlashAttentionImpl
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type[Qwen4ExpQSAMetadataBuilder]:
|
||||
return Qwen4ExpQSAMetadataBuilder
|
||||
|
||||
@classmethod
|
||||
def is_sparse(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def supports_kv_connector(cls) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
class Qwen4ExpQSAFlashAttentionImpl(FlashAttentionImpl):
|
||||
"""Run paged sparse GQA with the QSA Triton kernel."""
|
||||
|
||||
supports_dcp: bool = False
|
||||
supports_pcp: bool = False
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
if not is_flash_attn_varlen_func_available():
|
||||
raise NotImplementedError("Qwen4Exp QSA requires FlashAttention")
|
||||
if self.dcp_world_size != 1:
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp QSA does not support decode context parallelism"
|
||||
)
|
||||
if self.kv_cache_dtype not in ("auto", "bfloat16"):
|
||||
raise NotImplementedError("Qwen4Exp QSA requires a BF16 main KV cache")
|
||||
self.supports_quant_query_input = False
|
||||
|
||||
def forward_qsa(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
attn_metadata: FlashAttentionMetadata,
|
||||
output: torch.Tensor,
|
||||
token_to_req: torch.Tensor,
|
||||
output_scale: torch.Tensor | None = None,
|
||||
output_block_scale: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
del key, value
|
||||
if output_scale is not None or output_block_scale is not None:
|
||||
raise NotImplementedError("QSA does not support fused output quantization")
|
||||
if self.alibi_slopes is not None or self.sinks is not None:
|
||||
raise NotImplementedError("QSA does not support ALiBi or attention sinks")
|
||||
if self.sliding_window != (-1, -1):
|
||||
raise NotImplementedError("QSA does not support sliding-window attention")
|
||||
|
||||
num_tokens = attn_metadata.num_actual_tokens
|
||||
output.zero_()
|
||||
if num_tokens == 0:
|
||||
return output
|
||||
|
||||
topk_buffer = getattr(layer, "topk_indices_buffer", None)
|
||||
if topk_buffer is None:
|
||||
raise RuntimeError("QSA owner did not provide its top-k buffer")
|
||||
logical_indices = topk_buffer[:num_tokens]
|
||||
token_to_req = token_to_req[:num_tokens]
|
||||
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
|
||||
key_cache = canonicalize_singleton_dim_strides(key_cache)
|
||||
value_cache = canonicalize_singleton_dim_strides(value_cache)
|
||||
if key_cache.dtype != torch.bfloat16 or query.dtype != torch.bfloat16:
|
||||
raise NotImplementedError("Qwen4Exp QSA requires BF16 Q/K/V")
|
||||
|
||||
from .ops.qsa import qsa_sparse_paged_attention
|
||||
|
||||
qsa_sparse_paged_attention(
|
||||
query[:num_tokens],
|
||||
key_cache,
|
||||
value_cache,
|
||||
logical_indices,
|
||||
attn_metadata.block_table,
|
||||
token_to_req,
|
||||
output[:num_tokens],
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
|
||||
"""Merged Qwen full-attention owner with a QSA index side branch."""
|
||||
|
||||
supports_dcp = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
vllm_config: VllmConfig,
|
||||
config: Qwen4ExpTextConfig,
|
||||
layer_id: int,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
reduce_results: bool = True,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
nn.Module.__init__(self)
|
||||
cache_config = vllm_config.cache_config
|
||||
model_config = vllm_config.model_config
|
||||
if cache_config is None:
|
||||
raise ValueError("Qwen4Exp QSA requires a paged KV cache")
|
||||
if model_config.dtype != torch.bfloat16:
|
||||
raise NotImplementedError("Qwen4Exp QSA currently requires BF16")
|
||||
if cache_config.cache_dtype not in ("auto", "bfloat16"):
|
||||
raise NotImplementedError("Qwen4Exp QSA requires a BF16 main KV cache")
|
||||
if getattr(quant_config, "kv_cache_scheme", None) is not None:
|
||||
raise NotImplementedError("Qwen4Exp QSA does not support KV quantization")
|
||||
parallel_config = vllm_config.parallel_config
|
||||
if (
|
||||
parallel_config.prefill_context_parallel_size > 1
|
||||
or parallel_config.decode_context_parallel_size > 1
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp QSA does not support context parallelism"
|
||||
)
|
||||
if not getattr(config, "is_causal", True):
|
||||
raise NotImplementedError("Qwen4Exp QSA requires causal decoder attention")
|
||||
|
||||
self.config = config
|
||||
self.hidden_size = int(config.hidden_size)
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
self.total_num_heads = int(config.num_attention_heads)
|
||||
if self.total_num_heads % tp_size:
|
||||
raise ValueError("QSA attention heads must be divisible by TP size")
|
||||
self.num_heads = self.total_num_heads // tp_size
|
||||
self.total_num_kv_heads = int(config.num_key_value_heads)
|
||||
if self.total_num_kv_heads >= tp_size:
|
||||
if self.total_num_kv_heads % tp_size:
|
||||
raise ValueError("QSA KV heads must be divisible by TP size")
|
||||
elif tp_size % self.total_num_kv_heads:
|
||||
raise ValueError("TP size must be divisible by replicated QSA KV heads")
|
||||
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
|
||||
self.head_dim = int(config.head_dim or self.hidden_size // self.num_heads)
|
||||
self.q_size = self.num_heads * self.head_dim
|
||||
self.kv_size = self.num_kv_heads * self.head_dim
|
||||
self.scaling = self.head_dim**-0.5
|
||||
self.dual_chunk_attention_config = getattr(
|
||||
config, "dual_chunk_attention_config", None
|
||||
)
|
||||
if self.dual_chunk_attention_config is not None:
|
||||
raise NotImplementedError("Qwen4Exp QSA does not support dual-chunk RoPE")
|
||||
# Qwen4Exp full-attention checkpoints always pack a sigmoid output
|
||||
# gate next to Q, even when an inherited config default says otherwise.
|
||||
self.attn_output_gate = True
|
||||
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
self.hidden_size,
|
||||
self.head_dim,
|
||||
self.total_num_heads * (1 + self.attn_output_gate),
|
||||
self.total_num_kv_heads,
|
||||
bias=False,
|
||||
quant_config=model.without_modelopt_fp4(quant_config),
|
||||
prefix=f"{prefix}.qkv_proj",
|
||||
)
|
||||
self.o_proj = RowParallelLinear(
|
||||
self.total_num_heads * self.head_dim,
|
||||
self.hidden_size,
|
||||
bias=False,
|
||||
reduce_results=reduce_results,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
self.rotary_emb = get_rope(
|
||||
head_size=self.head_dim,
|
||||
max_position=config.max_position_embeddings,
|
||||
rope_parameters=config.rope_parameters,
|
||||
)
|
||||
self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
|
||||
mm_config = model_config.multimodal_config
|
||||
text_only = mm_config is None or mm_config.language_model_only
|
||||
self.use_fused_qk_norm_rope_gate = (
|
||||
self.attn_output_gate
|
||||
and getattr(self.rotary_emb, "is_neox_style", False)
|
||||
and current_platform.is_cuda()
|
||||
and text_only
|
||||
)
|
||||
|
||||
self.layer_name = f"{prefix}.attn"
|
||||
self.attn_type = AttentionType.DECODER
|
||||
self.kv_cache_dtype = cache_config.cache_dtype
|
||||
self.kv_cache_torch_dtype = kv_cache_dtype_str_to_dtype(
|
||||
self.kv_cache_dtype, model_config
|
||||
)
|
||||
if self.kv_cache_torch_dtype != torch.bfloat16:
|
||||
raise NotImplementedError("Qwen4Exp QSA requires BF16 cache storage")
|
||||
self.kv_sharing_target_layer_name = None
|
||||
self.kv_cache = torch.tensor([])
|
||||
set_default_quant_scales(self, register_buffer=True)
|
||||
|
||||
self.attn_backend = Qwen4ExpQSAFlashAttentionBackend
|
||||
self.impl = Qwen4ExpQSAFlashAttentionImpl(
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
self.scaling,
|
||||
self.num_kv_heads,
|
||||
None,
|
||||
None,
|
||||
self.kv_cache_dtype,
|
||||
None,
|
||||
AttentionType.DECODER,
|
||||
None,
|
||||
)
|
||||
self.indexer = QSAIndexer(
|
||||
vllm_config=vllm_config,
|
||||
config=config,
|
||||
layer_id=layer_id,
|
||||
rotary_emb=self.rotary_emb,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.indexer",
|
||||
)
|
||||
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
self.register_buffer(
|
||||
"topk_indices_buffer",
|
||||
torch.empty(
|
||||
max_tokens,
|
||||
self.indexer.output_width,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
persistent=False,
|
||||
)
|
||||
|
||||
static_context = vllm_config.compilation_config.static_forward_context
|
||||
if self.layer_name in static_context:
|
||||
raise ValueError(f"Duplicate layer name: {self.layer_name}")
|
||||
static_context[self.layer_name] = self
|
||||
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
return self.attn_backend
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
|
||||
return FullAttentionSpec(
|
||||
block_size=vllm_config.cache_config.block_size,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_size=self.head_dim,
|
||||
head_size_v=self.head_dim,
|
||||
dtype=self.kv_cache_torch_dtype,
|
||||
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
|
||||
)
|
||||
|
||||
def _run_qsa(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
metadata = get_forward_context().attn_metadata
|
||||
if isinstance(metadata, list):
|
||||
metadata = metadata[0]
|
||||
if not isinstance(metadata, dict):
|
||||
output.zero_()
|
||||
return
|
||||
main_metadata = cast(FlashAttentionMetadata, metadata[self.layer_name])
|
||||
if self.kv_cache.numel() == 0:
|
||||
raise RuntimeError("QSA main K/V cache is not bound")
|
||||
|
||||
num_tokens = main_metadata.num_actual_tokens
|
||||
side_metadata = cast(
|
||||
QSAForwardMetadata,
|
||||
metadata[self.indexer.raw_key_cache.prefix],
|
||||
)
|
||||
if side_metadata.num_actual_tokens != num_tokens:
|
||||
raise RuntimeError("QSA main and side metadata token counts disagree")
|
||||
selected = self.indexer(
|
||||
hidden_states,
|
||||
positions,
|
||||
self.topk_indices_buffer[:num_tokens],
|
||||
)
|
||||
if selected.shape != (
|
||||
num_tokens,
|
||||
self.indexer.output_width,
|
||||
):
|
||||
raise RuntimeError("QSA indexer returned an invalid selection shape")
|
||||
impl = cast(Qwen4ExpQSAFlashAttentionImpl, self.impl)
|
||||
impl.do_kv_cache_update(
|
||||
self,
|
||||
key,
|
||||
value,
|
||||
self.kv_cache,
|
||||
main_metadata.slot_mapping,
|
||||
)
|
||||
impl.forward_qsa(
|
||||
self,
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
self.kv_cache,
|
||||
main_metadata,
|
||||
output,
|
||||
token_to_req=side_metadata.token_to_req,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
q, k, v, gate = self._project_qkv_gate(qkv, positions)
|
||||
num_tokens = hidden_states.shape[0]
|
||||
query = q.view(num_tokens, self.num_heads, self.head_dim)
|
||||
key = k.view(num_tokens, self.num_kv_heads, self.head_dim)
|
||||
value = v.view(num_tokens, self.num_kv_heads, self.head_dim)
|
||||
attn_output = torch.empty_like(query)
|
||||
encoded_layer_name = _encode_layer_name(self.layer_name)
|
||||
if current_platform.opaque_attention_op():
|
||||
torch.ops.vllm.qwen4_exp_qsa_with_output(
|
||||
hidden_states,
|
||||
positions,
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
attn_output,
|
||||
encoded_layer_name,
|
||||
)
|
||||
else:
|
||||
qwen4_exp_qsa_with_output(
|
||||
hidden_states,
|
||||
positions,
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
attn_output,
|
||||
encoded_layer_name,
|
||||
)
|
||||
flat_output = attn_output.view(num_tokens, -1)
|
||||
if gate is not None:
|
||||
flat_output = flat_output * torch.sigmoid(gate)
|
||||
output, _ = self.o_proj(flat_output)
|
||||
return output
|
||||
|
||||
|
||||
def qwen4_exp_qsa_with_output(
|
||||
hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
layer_name: LayerNameType,
|
||||
) -> None:
|
||||
"""Run the complete QSA state/update/attend transaction."""
|
||||
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
layer = get_forward_context().no_compile_layers[layer_name]
|
||||
if not isinstance(layer, Qwen4ExpQSAAttention):
|
||||
raise TypeError(f"{layer_name} is not a Qwen4Exp QSA owner")
|
||||
layer._run_qsa(
|
||||
hidden_states,
|
||||
positions,
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
output,
|
||||
)
|
||||
|
||||
|
||||
def qwen4_exp_qsa_with_output_fake(
|
||||
hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
layer_name: LayerNameType,
|
||||
) -> None:
|
||||
del hidden_states, positions, query, key, value, output, layer_name
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_qsa_with_output",
|
||||
op_func=qwen4_exp_qsa_with_output,
|
||||
mutates_args=["output"],
|
||||
fake_impl=qwen4_exp_qsa_with_output_fake,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"QSAIndexer",
|
||||
"Qwen4ExpQSAAttention",
|
||||
"Qwen4ExpQSAFlashAttentionBackend",
|
||||
"Qwen4ExpQSAFlashAttentionImpl",
|
||||
"qwen4_exp_qsa_with_output",
|
||||
]
|
||||
@@ -0,0 +1,17 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Common Qwen4Exp model components."""
|
||||
|
||||
from .hyperconnection import (
|
||||
GatedResidual,
|
||||
GroupedGemmaRMSNorm,
|
||||
HyperConnectionBase,
|
||||
HyperConnectionConfig,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"GatedResidual",
|
||||
"GroupedGemmaRMSNorm",
|
||||
"HyperConnectionBase",
|
||||
"HyperConnectionConfig",
|
||||
]
|
||||
@@ -0,0 +1,250 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""HyperConnection (Gated Residual) utilities.
|
||||
|
||||
Implements the HyperConnection residual scheme proposed in
|
||||
"HyperConnections" (https://arxiv.org/abs/2409.19606).
|
||||
|
||||
The two concrete variants are:
|
||||
- ``HyperConnectionBase`` - simple average pooling across hc_count parallel
|
||||
streams (equivalent to hyperconnection_average).
|
||||
- ``GatedResidual`` - learnable low-rank gated mixing and injection
|
||||
(gated_residual).
|
||||
|
||||
Hidden states between layers have shape ``[..., HC*HS]`` with HS inner
|
||||
(HC outer, HS inner — checkpoint-native layout). The local torch
|
||||
implementation consumes the hyper input viewed as ``[..., HC, HS]``.
|
||||
|
||||
Typical usage inside a transformer decoder layer::
|
||||
|
||||
self.attn_hc = GatedResidual(hc_config, role="attn")
|
||||
self.mlp_hc = GatedResidual(hc_config, role="mlp")
|
||||
|
||||
hidden_states, residual = self.attn_hc.mix(hidden_states)
|
||||
hidden_states = attention(hidden_states)
|
||||
hidden_states = self.attn_hc.combine(hidden_states, residual)
|
||||
|
||||
hidden_states, residual = self.mlp_hc.mix(hidden_states)
|
||||
hidden_states = mlp(hidden_states)
|
||||
hidden_states = self.mlp_hc.combine(hidden_states, residual)
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config
|
||||
# ---------------------------------------------------------------------------
|
||||
@dataclass
|
||||
class HyperConnectionConfig:
|
||||
"""Configuration shared by all HyperConnection variants."""
|
||||
|
||||
hc_count: int = 4
|
||||
hidden_size: int = 64
|
||||
params_dtype: torch.dtype = torch.bfloat16
|
||||
mtp_hc: bool = False
|
||||
hc_lowrank: int = 16
|
||||
rms_norm_eps: float = 1e-6
|
||||
hc_per_branch_norm: bool = False
|
||||
|
||||
|
||||
class GroupedGemmaRMSNorm(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
eps: float,
|
||||
group_size: int | None,
|
||||
dtype: torch.dtype | None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if group_size is not None and hidden_size % group_size:
|
||||
raise ValueError(
|
||||
f"hidden_size ({hidden_size}) must be divisible by "
|
||||
f"group_size ({group_size})"
|
||||
)
|
||||
self.variance_epsilon = eps
|
||||
self.group_size = group_size
|
||||
self.weight = nn.Parameter(torch.zeros(hidden_size, dtype=dtype))
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
input_dtype = hidden_states.dtype
|
||||
hidden_states = hidden_states.float()
|
||||
if self.group_size is None:
|
||||
variance = hidden_states.square().mean(dim=-1, keepdim=True)
|
||||
normalized = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
|
||||
else:
|
||||
grouped = hidden_states.unflatten(
|
||||
-1, (hidden_states.shape[-1] // self.group_size, self.group_size)
|
||||
)
|
||||
variance = grouped.square().mean(dim=-1, keepdim=True)
|
||||
normalized = (
|
||||
grouped * torch.rsqrt(variance + self.variance_epsilon)
|
||||
).flatten(-2)
|
||||
return (normalized * (1.0 + self.weight.float())).to(input_dtype)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Average-pooling variant
|
||||
# ---------------------------------------------------------------------------
|
||||
class HyperConnectionBase(nn.Module):
|
||||
"""Average-pooling HyperConnection (``hyperconnection_average``).
|
||||
|
||||
Splits the incoming ``[..., HC*HS]`` tensor (HC outer, HS inner) into
|
||||
``HC`` parallel streams, averages them for the block input, and
|
||||
broadcasts the block output back to every stream.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: HyperConnectionConfig,
|
||||
layer_idx: int | None = None,
|
||||
role: str | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.hc_count = config.hc_count
|
||||
self.hidden_size = config.hidden_size
|
||||
self.layer_idx = layer_idx
|
||||
self.role = role
|
||||
|
||||
@property
|
||||
def hyper_hidden_size(self) -> int:
|
||||
return self.hc_count * self.hidden_size
|
||||
|
||||
def mix(self, hyper_input: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Average the HC streams into a single block input."""
|
||||
assert hyper_input.shape[-1] == self.hc_count * self.hidden_size
|
||||
# [*, HC, HS] — mean over HC (dim=-2).
|
||||
unflat = hyper_input.unflatten(-1, (self.hc_count, self.hidden_size))
|
||||
mixed_input = unflat.mean(dim=-2)
|
||||
return mixed_input, hyper_input
|
||||
|
||||
def combine(
|
||||
self, block_output: torch.Tensor, residual: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Broadcast the block output back to every stream."""
|
||||
assert residual.shape[-1] == self.hc_count * self.hidden_size
|
||||
assert block_output.shape[-1] == self.hidden_size
|
||||
residual_reshaped = residual.unflatten(-1, (self.hc_count, self.hidden_size))
|
||||
combined = residual_reshaped + block_output.unsqueeze(-2)
|
||||
return combined.flatten(-2)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Gated-residual variant
|
||||
# ---------------------------------------------------------------------------
|
||||
class GatedResidual(HyperConnectionBase):
|
||||
"""Gated HyperConnection with learnable low-rank mixing and injection.
|
||||
|
||||
``mix()`` applies GemmaRMSNorm per HC stream and projects through a
|
||||
low-rank sigmoid gate to produce a single block input. ``combine()``
|
||||
injects the block output back into each stream through a learned
|
||||
per-stream injection weight.
|
||||
|
||||
This implementation uses only PyTorch operators. Tensor-parallel
|
||||
collectives are supplied by its caller.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: HyperConnectionConfig,
|
||||
layer_idx: int | None = None,
|
||||
role: str | None = None,
|
||||
use_mix: bool = True,
|
||||
use_combine: bool = True,
|
||||
) -> None:
|
||||
super().__init__(config, layer_idx, role)
|
||||
norm_size = (
|
||||
self.hyper_hidden_size if config.hc_per_branch_norm else config.hidden_size
|
||||
)
|
||||
group_size = config.hidden_size if config.hc_per_branch_norm else None
|
||||
# Normalize each H-sized HC stream independently while retaining a
|
||||
# separate affine weight for every element of the HC*H layout.
|
||||
self.hc_norm = GroupedGemmaRMSNorm(
|
||||
norm_size,
|
||||
eps=config.rms_norm_eps,
|
||||
group_size=group_size,
|
||||
dtype=config.params_dtype,
|
||||
)
|
||||
|
||||
# -- raw Linear weights (checkpoint-compatible) ----------------------
|
||||
if use_mix:
|
||||
self.input_mix_weight_down = nn.Linear(
|
||||
self.hyper_hidden_size,
|
||||
config.hc_lowrank,
|
||||
bias=False,
|
||||
dtype=config.params_dtype,
|
||||
)
|
||||
self.input_mix_weight_up = nn.Linear(
|
||||
config.hc_lowrank,
|
||||
self.hyper_hidden_size,
|
||||
bias=False,
|
||||
dtype=config.params_dtype,
|
||||
)
|
||||
if use_combine:
|
||||
self.block_inject_weight = nn.Linear(
|
||||
self.hyper_hidden_size,
|
||||
self.hc_count,
|
||||
bias=False,
|
||||
dtype=config.params_dtype,
|
||||
)
|
||||
|
||||
def _normalize(self, hyper_input: torch.Tensor) -> torch.Tensor:
|
||||
if self.config.hc_per_branch_norm:
|
||||
return self.hc_norm(hyper_input)
|
||||
return self.hc_norm(
|
||||
hyper_input.unflatten(-1, (self.hc_count, self.hidden_size))
|
||||
).flatten(-2)
|
||||
|
||||
def mix(
|
||||
self, hyper_input: torch.Tensor
|
||||
) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]:
|
||||
"""Mix: RMSNorm -> low-rank gate -> gated mean."""
|
||||
assert hyper_input.shape[-1] == self.hc_count * self.hidden_size
|
||||
if not hasattr(self, "input_mix_weight_down"):
|
||||
raise RuntimeError("mix was disabled for this hyper-connection")
|
||||
hyper_input_normed = self._normalize(hyper_input)
|
||||
# Gate — original mix order: linear+silu then linear+sigmoid.
|
||||
gate = F.silu(
|
||||
F.linear(hyper_input_normed, self.input_mix_weight_down.weight)
|
||||
/ self.hc_count
|
||||
)
|
||||
gate = torch.sigmoid(F.linear(gate, self.input_mix_weight_up.weight)).unflatten(
|
||||
-1, (self.hc_count, self.hidden_size)
|
||||
)
|
||||
mixed_input = (
|
||||
gate * hyper_input_normed.unflatten(-1, (self.hc_count, self.hidden_size))
|
||||
).mean(dim=-2)
|
||||
return mixed_input.to(hyper_input.dtype), (hyper_input, hyper_input_normed)
|
||||
|
||||
def combine(
|
||||
self,
|
||||
block_output: torch.Tensor,
|
||||
residuals: tuple[torch.Tensor, torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
if not hasattr(self, "block_inject_weight"):
|
||||
raise RuntimeError("combine was disabled for this hyper-connection")
|
||||
hyper_input, hyper_input_normed = residuals
|
||||
assert hyper_input.shape[-1] == self.hc_count * self.hidden_size
|
||||
assert block_output.shape[-1] == self.hidden_size
|
||||
residual = hyper_input.unflatten(-1, (self.hc_count, self.hidden_size))
|
||||
# The paired mix keeps its normalized hyper input so combine uses the
|
||||
# same HC module's injection weight.
|
||||
injection_weight = 2.0 * torch.sigmoid(
|
||||
F.linear(hyper_input_normed, self.block_inject_weight.weight)
|
||||
/ self.hc_count
|
||||
)
|
||||
output = residual + block_output.unsqueeze(-2) * injection_weight.unsqueeze(-1)
|
||||
return output.flatten(-2).to(hyper_input.dtype)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"GatedResidual",
|
||||
"GroupedGemmaRMSNorm",
|
||||
"HyperConnectionBase",
|
||||
"HyperConnectionConfig",
|
||||
]
|
||||
@@ -0,0 +1,100 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Common Qwen4Exp PLE helpers."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PLEShardOverlap:
|
||||
"""Source and destination slices for one checkpoint embedding shard."""
|
||||
|
||||
source_start: int
|
||||
destination_start: int
|
||||
row_count: int
|
||||
|
||||
|
||||
def compute_ple_shard_overlap(
|
||||
*,
|
||||
checkpoint_start: int,
|
||||
checkpoint_rows: int,
|
||||
tp_start: int,
|
||||
tp_end: int,
|
||||
) -> PLEShardOverlap | None:
|
||||
"""Compute the overlap of a checkpoint shard and one TP vocabulary range."""
|
||||
|
||||
if checkpoint_start < 0 or checkpoint_rows < 0:
|
||||
raise ValueError("checkpoint shard bounds must be non-negative")
|
||||
if tp_start < 0 or tp_end < tp_start:
|
||||
raise ValueError("invalid TP vocabulary range")
|
||||
checkpoint_end = checkpoint_start + checkpoint_rows
|
||||
overlap_start = max(checkpoint_start, tp_start)
|
||||
overlap_end = min(checkpoint_end, tp_end)
|
||||
if overlap_start >= overlap_end:
|
||||
return None
|
||||
return PLEShardOverlap(
|
||||
source_start=overlap_start - checkpoint_start,
|
||||
destination_start=overlap_start - tp_start,
|
||||
row_count=overlap_end - overlap_start,
|
||||
)
|
||||
|
||||
|
||||
def copy_ple_embedding_shard_(
|
||||
destination: torch.Tensor,
|
||||
loaded_weight: torch.Tensor,
|
||||
*,
|
||||
checkpoint_start: int,
|
||||
tp_start: int,
|
||||
tp_end: int,
|
||||
) -> int:
|
||||
"""Copy the overlapping rows of a PLE checkpoint shard into a TP table."""
|
||||
|
||||
if destination.ndim == 0 or loaded_weight.ndim != destination.ndim:
|
||||
raise ValueError("destination and loaded weight must have matching ranks")
|
||||
if destination.shape[1:] != loaded_weight.shape[1:]:
|
||||
raise ValueError(
|
||||
"embedding shard dimensions do not match: "
|
||||
f"{tuple(destination.shape[1:])} != {tuple(loaded_weight.shape[1:])}"
|
||||
)
|
||||
if destination.shape[0] < tp_end - tp_start:
|
||||
raise ValueError("destination does not cover the requested TP range")
|
||||
overlap = compute_ple_shard_overlap(
|
||||
checkpoint_start=checkpoint_start,
|
||||
checkpoint_rows=loaded_weight.shape[0],
|
||||
tp_start=tp_start,
|
||||
tp_end=tp_end,
|
||||
)
|
||||
if overlap is None:
|
||||
return 0
|
||||
source = loaded_weight.narrow(0, overlap.source_start, overlap.row_count)
|
||||
target = destination.narrow(0, overlap.destination_start, overlap.row_count)
|
||||
with torch.no_grad():
|
||||
target.copy_(source.to(device=target.device, dtype=target.dtype))
|
||||
return overlap.row_count
|
||||
|
||||
|
||||
class PLEVocabParallelEmbedding(VocabParallelEmbedding):
|
||||
"""Vocab-parallel embedding that accepts checkpoint row shards."""
|
||||
|
||||
def weight_loader(
|
||||
self,
|
||||
param: torch.Tensor,
|
||||
loaded_weight: torch.Tensor,
|
||||
checkpoint_start: int | None = None,
|
||||
) -> None:
|
||||
if checkpoint_start is None:
|
||||
super().weight_loader(param, loaded_weight)
|
||||
return
|
||||
copy_ple_embedding_shard_(
|
||||
param,
|
||||
loaded_weight,
|
||||
checkpoint_start=checkpoint_start,
|
||||
tp_start=self.shard_indices.org_vocab_start_index,
|
||||
tp_end=self.shard_indices.org_vocab_end_index,
|
||||
)
|
||||
@@ -0,0 +1,807 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Paged side-cache ownership and metadata for Qwen4Exp QSA.
|
||||
|
||||
Each QSA layer keeps a fixed circular buffer of raw index keys (the
|
||||
compressor state) and one compressed key. MRoPE models pack exact three-axis
|
||||
positions beside the raw keys; text models derive group positions from
|
||||
logical positions. The compressor state uses one block per request, while
|
||||
the compressed owner uses ``MLAAttentionSpec.tokens_per_state`` so its block
|
||||
table follows the main KV-cache lifecycle. Their physical tensor storage is
|
||||
shared by the generic cache-layout planner.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from functools import cache
|
||||
from typing import ClassVar
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON, tl, triton
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionCGSupport,
|
||||
AttentionMetadata,
|
||||
AttentionMetadataBuilder,
|
||||
CommonAttentionMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
CircularBufferSpec,
|
||||
KVCacheLayout,
|
||||
KVCacheSpec,
|
||||
MLAAttentionSpec,
|
||||
)
|
||||
|
||||
|
||||
def canonical_qsa_rope_positions(positions: torch.Tensor) -> torch.Tensor:
|
||||
"""Return exact per-token positions as ``[tokens, 1, 3]`` int64 rows."""
|
||||
|
||||
if positions.ndim == 1:
|
||||
positions = positions.unsqueeze(0).expand(3, -1)
|
||||
elif positions.ndim != 2 or positions.shape[0] not in (1, 3):
|
||||
raise ValueError("QSA RoPE positions must be [tokens] or [1|3, tokens]")
|
||||
if positions.shape[0] == 1:
|
||||
positions = positions.expand(3, -1)
|
||||
return positions.transpose(0, 1).unsqueeze(1).to(torch.int64)
|
||||
|
||||
|
||||
def _logical_positions(
|
||||
query_start_loc: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
token_to_req: torch.Tensor,
|
||||
num_tokens: int,
|
||||
) -> torch.Tensor:
|
||||
if num_tokens == 0:
|
||||
return seq_lens.new_empty((0,), dtype=torch.int64)
|
||||
arange = torch.arange(num_tokens, device=query_start_loc.device)
|
||||
requests = token_to_req[:num_tokens].long()
|
||||
query_lens = torch.diff(query_start_loc)
|
||||
within_query = arange - query_start_loc.index_select(0, requests)
|
||||
return (
|
||||
seq_lens.index_select(0, requests).long()
|
||||
- query_lens.index_select(0, requests).long()
|
||||
+ within_query.long()
|
||||
)
|
||||
|
||||
|
||||
def _logical_to_physical_qsa_slots(
|
||||
block_table: torch.Tensor,
|
||||
request_indices: torch.Tensor,
|
||||
logical_positions: torch.Tensor,
|
||||
block_size: int,
|
||||
) -> torch.Tensor:
|
||||
if block_size <= 0:
|
||||
raise ValueError("QSA cache block size must be positive")
|
||||
if block_table.ndim != 2:
|
||||
raise ValueError("QSA block table must be two-dimensional")
|
||||
if request_indices.shape != logical_positions.shape:
|
||||
request_indices = torch.broadcast_to(request_indices, logical_positions.shape)
|
||||
|
||||
requests = request_indices.to(device=block_table.device, dtype=torch.long)
|
||||
positions = logical_positions.to(device=block_table.device, dtype=torch.long)
|
||||
valid = (requests >= 0) & (requests < block_table.shape[0]) & (positions >= 0)
|
||||
logical_blocks = torch.div(
|
||||
positions.clamp_min(0), block_size, rounding_mode="floor"
|
||||
)
|
||||
valid &= logical_blocks < block_table.shape[1]
|
||||
safe_requests = requests.clamp(0, max(block_table.shape[0] - 1, 0))
|
||||
safe_blocks = logical_blocks.clamp(0, max(block_table.shape[1] - 1, 0))
|
||||
if not all(block_table.shape):
|
||||
return torch.full_like(positions, PAD_SLOT_ID)
|
||||
physical_blocks = block_table[safe_requests, safe_blocks].long()
|
||||
valid &= physical_blocks >= 0
|
||||
slots = physical_blocks * block_size + positions.remainder(block_size)
|
||||
return torch.where(valid, slots, PAD_SLOT_ID)
|
||||
|
||||
|
||||
def circular_qsa_slot_mapping(
|
||||
block_table: torch.Tensor,
|
||||
token_to_req: torch.Tensor,
|
||||
logical_positions: torch.Tensor,
|
||||
compressor_state_size: int,
|
||||
query_start_loc: torch.Tensor | None = None,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Map each request to its fixed physical block as a circular token ring."""
|
||||
|
||||
if compressor_state_size <= 0:
|
||||
raise ValueError("QSA circular buffer size must be positive")
|
||||
if block_table.ndim != 2:
|
||||
raise ValueError("QSA block table must be two-dimensional")
|
||||
|
||||
requests = token_to_req.to(device=block_table.device, dtype=torch.long)
|
||||
positions = logical_positions.to(device=block_table.device, dtype=torch.long)
|
||||
if not all(block_table.shape):
|
||||
slots = torch.full_like(positions, PAD_SLOT_ID)
|
||||
else:
|
||||
valid = (requests >= 0) & (requests < block_table.shape[0]) & (positions >= 0)
|
||||
safe_requests = requests.clamp(0, block_table.shape[0] - 1)
|
||||
physical_blocks = block_table[safe_requests, 0].long()
|
||||
valid &= physical_blocks >= 0
|
||||
slots = physical_blocks * compressor_state_size + positions.remainder(
|
||||
compressor_state_size
|
||||
)
|
||||
slots = torch.where(valid, slots, PAD_SLOT_ID)
|
||||
|
||||
if query_start_loc is not None:
|
||||
if query_start_loc.ndim != 1 or query_start_loc.shape[0] < 2:
|
||||
raise ValueError("QSA query starts must contain a terminal offset")
|
||||
query_start_loc = query_start_loc.to(block_table.device)
|
||||
num_requests = query_start_loc.shape[0] - 1
|
||||
safe_requests = requests.clamp(0, num_requests - 1)
|
||||
request_ends = query_start_loc.index_select(0, safe_requests + 1)
|
||||
rows = torch.arange(slots.numel(), device=slots.device)
|
||||
keep = (
|
||||
(requests >= 0)
|
||||
& (requests < num_requests)
|
||||
& (rows + compressor_state_size >= request_ends)
|
||||
)
|
||||
slots = torch.where(keep, slots, PAD_SLOT_ID)
|
||||
|
||||
slots = slots.to(torch.int64)
|
||||
if out is not None:
|
||||
out.fill_(PAD_SLOT_ID)
|
||||
out[: slots.numel()].copy_(slots)
|
||||
return out[: slots.numel()]
|
||||
return slots
|
||||
|
||||
|
||||
def compressed_qsa_slot_mapping(
|
||||
block_table: torch.Tensor,
|
||||
token_to_req: torch.Tensor,
|
||||
logical_positions: torch.Tensor,
|
||||
storage_block_size: int,
|
||||
compress_ratio: int,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Build boundary-only slots for an ``MLAAttentionSpec`` QSA cache."""
|
||||
|
||||
if storage_block_size <= 0 or compress_ratio <= 0:
|
||||
raise ValueError("QSA block size and compression ratio must be positive")
|
||||
compressed_positions = torch.div(
|
||||
logical_positions.clamp_min(0), compress_ratio, rounding_mode="floor"
|
||||
)
|
||||
slots = _logical_to_physical_qsa_slots(
|
||||
block_table,
|
||||
token_to_req,
|
||||
compressed_positions,
|
||||
storage_block_size,
|
||||
)
|
||||
valid = (logical_positions >= 0) & (
|
||||
(logical_positions + 1).remainder(compress_ratio) == 0
|
||||
)
|
||||
slots = torch.where(valid, slots, PAD_SLOT_ID).to(torch.int64)
|
||||
if out is not None:
|
||||
out.fill_(PAD_SLOT_ID)
|
||||
out[: slots.numel()].copy_(slots)
|
||||
return out[: slots.numel()]
|
||||
return slots
|
||||
|
||||
|
||||
@cache
|
||||
def _metadata_launch_pdl() -> bool:
|
||||
return current_platform.is_arch_support_pdl()
|
||||
|
||||
|
||||
@triton.jit(
|
||||
do_not_specialize=[
|
||||
"num_reqs",
|
||||
"num_mapped_tokens",
|
||||
"num_tokens",
|
||||
"max_num_work",
|
||||
"num_search_steps",
|
||||
"work_search_steps",
|
||||
]
|
||||
)
|
||||
def _build_qsa_metadata_kernel(
|
||||
query_start_loc_ptr,
|
||||
seq_lens_ptr,
|
||||
common_slot_mapping_ptr,
|
||||
block_table_ptr,
|
||||
token_to_req_ptr,
|
||||
logical_positions_ptr,
|
||||
slot_mapping_ptr,
|
||||
k_work_metadata_ptr,
|
||||
block_table_stride_0: tl.constexpr,
|
||||
block_table_stride_1: tl.constexpr,
|
||||
num_reqs,
|
||||
num_mapped_tokens,
|
||||
num_tokens,
|
||||
max_num_work,
|
||||
num_search_steps,
|
||||
work_search_steps,
|
||||
storage_block_size: tl.constexpr,
|
||||
compress_ratio: tl.constexpr,
|
||||
circular_buffer_size: tl.constexpr,
|
||||
num_block_table_columns: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
TOKEN_BLOCK_SIZE: tl.constexpr,
|
||||
REQUEST_SCAN_SIZE: tl.constexpr,
|
||||
WORK_BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
|
||||
pid = tl.program_id(0)
|
||||
token_idx = pid * TOKEN_BLOCK_SIZE + tl.arange(0, TOKEN_BLOCK_SIZE)
|
||||
store_mask = token_idx < num_tokens
|
||||
mapped = token_idx < num_mapped_tokens
|
||||
search_token_idx = tl.minimum(token_idx, num_mapped_tokens - 1)
|
||||
request_idx = tl.zeros((TOKEN_BLOCK_SIZE,), tl.int32)
|
||||
# Find the last query start at or before each token. The dynamic loop avoids
|
||||
# compiling a kernel variant for every ceil(log2(num_reqs)).
|
||||
for step in tl.range(0, num_search_steps):
|
||||
candidate = request_idx + (1 << (num_search_steps - step - 1))
|
||||
valid_candidate = candidate < num_reqs
|
||||
candidate_start = tl.load(
|
||||
query_start_loc_ptr + candidate,
|
||||
mask=valid_candidate,
|
||||
other=num_mapped_tokens + 1,
|
||||
)
|
||||
advance = valid_candidate & (candidate_start <= search_token_idx)
|
||||
request_idx = tl.where(advance, candidate, request_idx)
|
||||
query_start = tl.load(query_start_loc_ptr + request_idx, mask=mapped, other=0)
|
||||
query_end = tl.load(query_start_loc_ptr + request_idx + 1, mask=mapped, other=0)
|
||||
seq_len = tl.load(seq_lens_ptr + request_idx, mask=mapped, other=0)
|
||||
logical_position = seq_len - (query_end - query_start) + token_idx - query_start
|
||||
logical_position = tl.where(mapped, logical_position, -1)
|
||||
tl.store(
|
||||
token_to_req_ptr + token_idx,
|
||||
tl.where(mapped, request_idx, 0),
|
||||
mask=store_mask,
|
||||
)
|
||||
tl.store(
|
||||
logical_positions_ptr + token_idx,
|
||||
logical_position,
|
||||
mask=store_mask,
|
||||
)
|
||||
|
||||
# circular_buffer_size is constexpr, so each builder instance compiles out
|
||||
# the other QSA cache owner's slot-mapping rule.
|
||||
if circular_buffer_size > 0:
|
||||
valid = (
|
||||
mapped
|
||||
& (logical_position >= 0)
|
||||
& (token_idx + circular_buffer_size >= query_end)
|
||||
& (num_block_table_columns > 0)
|
||||
)
|
||||
physical_block = tl.load(
|
||||
block_table_ptr + request_idx * block_table_stride_0,
|
||||
mask=valid,
|
||||
other=-1,
|
||||
)
|
||||
valid &= physical_block >= 0
|
||||
slot = physical_block * circular_buffer_size + (
|
||||
logical_position % circular_buffer_size
|
||||
)
|
||||
elif compress_ratio != 1:
|
||||
compressed_position = tl.maximum(logical_position, 0) // compress_ratio
|
||||
logical_block = compressed_position // storage_block_size
|
||||
valid = (
|
||||
mapped
|
||||
& (logical_position >= 0)
|
||||
& ((logical_position + 1) % compress_ratio == 0)
|
||||
& (logical_block < num_block_table_columns)
|
||||
)
|
||||
physical_block = tl.load(
|
||||
block_table_ptr
|
||||
+ request_idx * block_table_stride_0
|
||||
+ logical_block * block_table_stride_1,
|
||||
mask=valid,
|
||||
other=-1,
|
||||
)
|
||||
valid &= physical_block >= 0
|
||||
valid &= (
|
||||
tl.load(common_slot_mapping_ptr + token_idx, mask=mapped, other=-1) >= 0
|
||||
)
|
||||
slot = physical_block * storage_block_size + (
|
||||
compressed_position % storage_block_size
|
||||
)
|
||||
if (circular_buffer_size > 0) or (compress_ratio != 1):
|
||||
tl.store(
|
||||
slot_mapping_ptr + token_idx,
|
||||
tl.where(valid, slot, -1),
|
||||
mask=store_mask,
|
||||
)
|
||||
work_tile_start = pid * WORK_BLOCK_SIZE
|
||||
has_work_tile = work_tile_start < max_num_work
|
||||
if k_work_metadata_ptr is not None and has_work_tile:
|
||||
# Every work CTA builds the request prefix in registers. Recomputing this
|
||||
# small vector lets CTAs write disjoint work tiles without a grid barrier.
|
||||
requests = tl.arange(0, REQUEST_SCAN_SIZE)
|
||||
valid_request = requests < num_reqs
|
||||
request_query_start = tl.load(
|
||||
query_start_loc_ptr + requests, mask=valid_request, other=0
|
||||
)
|
||||
request_query_end = tl.load(
|
||||
query_start_loc_ptr + requests + 1, mask=valid_request, other=0
|
||||
)
|
||||
request_seq_len = tl.load(seq_lens_ptr + requests, mask=valid_request, other=0)
|
||||
request_query_len = request_query_end - request_query_start
|
||||
chunk_start = request_seq_len - request_query_len
|
||||
num_groups = request_seq_len // compress_ratio - chunk_start // compress_ratio
|
||||
# Nonempty requests need one item even without a completed compression
|
||||
# group because work item zero also commits the current raw-K suffix.
|
||||
work_counts = tl.where(request_query_len > 0, tl.maximum(num_groups, 1), 0)
|
||||
work_ends = tl.cumsum(tl.where(valid_request, work_counts, 0), axis=0)
|
||||
total_work = tl.sum(tl.where(valid_request, work_counts, 0), axis=0)
|
||||
work_offsets = tl.arange(0, WORK_BLOCK_SIZE)
|
||||
work = work_tile_start + work_offsets
|
||||
in_bounds = work < max_num_work
|
||||
active = in_bounds & (work < total_work)
|
||||
request = tl.zeros((WORK_BLOCK_SIZE,), dtype=tl.int32)
|
||||
# Request zero starts at zero for every active work item. Descending
|
||||
# steps find the last request starting at or before this item.
|
||||
for step_idx in tl.range(0, work_search_steps):
|
||||
step = 1 << (work_search_steps - step_idx - 1)
|
||||
candidate = request + step
|
||||
valid_candidate = candidate < num_reqs
|
||||
candidate_start = tl.gather(work_ends, candidate - 1, 0)
|
||||
advance = active & valid_candidate & (candidate_start <= work)
|
||||
request = tl.where(advance, candidate, request)
|
||||
|
||||
if launch_pdl:
|
||||
# Let the dependent grid start launch setup while these CTAs finish
|
||||
# stores; its gdc_wait still orders access to the completed metadata.
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
owner_work_start = tl.gather(work_ends, tl.maximum(request - 1, 0), 0)
|
||||
owner_work_start = tl.where(request == 0, 0, owner_work_start)
|
||||
work_in_request = work - owner_work_start
|
||||
tl.store(
|
||||
k_work_metadata_ptr + work * 2,
|
||||
tl.where(active, request, -1),
|
||||
mask=in_bounds,
|
||||
)
|
||||
tl.store(
|
||||
k_work_metadata_ptr + work * 2 + 1,
|
||||
tl.where(active, work_in_request, -1),
|
||||
mask=in_bounds,
|
||||
)
|
||||
|
||||
else:
|
||||
if launch_pdl:
|
||||
# A dependent grid launches only after every CTA has signaled.
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
|
||||
def build_qsa_metadata_triton(
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
token_to_req_buffer: torch.Tensor,
|
||||
logical_positions_buffer: torch.Tensor,
|
||||
slot_mapping_buffer: torch.Tensor,
|
||||
*,
|
||||
storage_block_size: int,
|
||||
compress_ratio: int,
|
||||
circular_buffer_size: int = 0,
|
||||
k_work_metadata_buffer: torch.Tensor | None = None,
|
||||
request_capacity: int | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Build QSA side-cache and optional pre-indexer work metadata."""
|
||||
num_tokens = common_attn_metadata.num_actual_tokens
|
||||
num_mapped_tokens = int(common_attn_metadata.query_start_loc_cpu[-1])
|
||||
token_to_req = token_to_req_buffer[:num_tokens]
|
||||
logical_positions = logical_positions_buffer[:num_tokens]
|
||||
slot_mapping = slot_mapping_buffer[:num_tokens]
|
||||
num_reqs = common_attn_metadata.query_start_loc.shape[0] - 1
|
||||
assert num_reqs > 0
|
||||
|
||||
if k_work_metadata_buffer is not None:
|
||||
if request_capacity is None:
|
||||
request_capacity = num_reqs
|
||||
assert request_capacity >= num_reqs
|
||||
# Pad for tl.arange while keeping the scan width stable across live batches.
|
||||
request_scan_size = 1 << int(math.ceil(math.log2(request_capacity)))
|
||||
max_num_work = k_work_metadata_buffer.shape[0]
|
||||
else:
|
||||
request_scan_size = 1
|
||||
max_num_work = 0
|
||||
|
||||
if num_tokens == 0 and k_work_metadata_buffer is None:
|
||||
return token_to_req, logical_positions, slot_mapping
|
||||
|
||||
block_table = common_attn_metadata.block_table_tensor
|
||||
num_search_steps = int(math.ceil(math.log2(num_reqs)))
|
||||
work_search_steps = int(math.ceil(math.log2(num_reqs)))
|
||||
# The same grid covers token tiles and, for the compressed cache, work tiles.
|
||||
num_token_blocks = cdiv(num_tokens, 128)
|
||||
num_work_blocks = (
|
||||
cdiv(max_num_work, 256) if k_work_metadata_buffer is not None else 0
|
||||
)
|
||||
_build_qsa_metadata_kernel[(max(num_token_blocks, num_work_blocks, 1),)](
|
||||
common_attn_metadata.query_start_loc,
|
||||
common_attn_metadata.seq_lens,
|
||||
common_attn_metadata.slot_mapping,
|
||||
block_table,
|
||||
token_to_req,
|
||||
logical_positions,
|
||||
slot_mapping,
|
||||
k_work_metadata_buffer,
|
||||
block_table.stride(0),
|
||||
block_table.stride(1),
|
||||
num_reqs,
|
||||
num_mapped_tokens,
|
||||
num_tokens,
|
||||
max_num_work,
|
||||
num_search_steps,
|
||||
work_search_steps,
|
||||
storage_block_size,
|
||||
compress_ratio,
|
||||
circular_buffer_size,
|
||||
block_table.shape[1],
|
||||
launch_pdl=_metadata_launch_pdl(),
|
||||
TOKEN_BLOCK_SIZE=128,
|
||||
REQUEST_SCAN_SIZE=request_scan_size,
|
||||
WORK_BLOCK_SIZE=256,
|
||||
num_warps=4,
|
||||
)
|
||||
if circular_buffer_size == 0 and compress_ratio == 1:
|
||||
slot_mapping = common_attn_metadata.slot_mapping[:num_tokens]
|
||||
return token_to_req, logical_positions, slot_mapping
|
||||
|
||||
|
||||
def _build_qsa_metadata_torch(
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
token_to_req_buffer: torch.Tensor,
|
||||
logical_positions_buffer: torch.Tensor,
|
||||
slot_mapping_buffer: torch.Tensor,
|
||||
*,
|
||||
storage_block_size: int,
|
||||
compress_ratio: int,
|
||||
circular_buffer_size: int = 0,
|
||||
k_work_metadata_buffer: torch.Tensor | None = None,
|
||||
request_capacity: int | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
del request_capacity
|
||||
num_tokens = common_attn_metadata.num_actual_tokens
|
||||
num_mapped_tokens = int(common_attn_metadata.query_start_loc_cpu[-1])
|
||||
logical_positions = logical_positions_buffer[:num_tokens]
|
||||
|
||||
token_to_req = common_attn_metadata.token_to_req_indices(token_to_req_buffer)[
|
||||
:num_tokens
|
||||
]
|
||||
logical_positions[:num_mapped_tokens].copy_(
|
||||
_logical_positions(
|
||||
common_attn_metadata.query_start_loc,
|
||||
common_attn_metadata.seq_lens,
|
||||
token_to_req[:num_mapped_tokens],
|
||||
num_mapped_tokens,
|
||||
)
|
||||
)
|
||||
if num_mapped_tokens < num_tokens:
|
||||
logical_positions[num_mapped_tokens:].fill_(-1)
|
||||
if circular_buffer_size > 0:
|
||||
slot_mapping = circular_qsa_slot_mapping(
|
||||
common_attn_metadata.block_table_tensor,
|
||||
token_to_req,
|
||||
logical_positions,
|
||||
circular_buffer_size,
|
||||
query_start_loc=common_attn_metadata.query_start_loc,
|
||||
out=slot_mapping_buffer,
|
||||
)
|
||||
elif compress_ratio == 1:
|
||||
slot_mapping = common_attn_metadata.slot_mapping[:num_tokens]
|
||||
else:
|
||||
slot_mapping = compressed_qsa_slot_mapping(
|
||||
common_attn_metadata.block_table_tensor,
|
||||
token_to_req,
|
||||
logical_positions,
|
||||
storage_block_size,
|
||||
compress_ratio,
|
||||
slot_mapping_buffer,
|
||||
)
|
||||
slot_mapping.masked_fill_(
|
||||
common_attn_metadata.slot_mapping[:num_tokens] < 0, -1
|
||||
)
|
||||
if k_work_metadata_buffer is not None:
|
||||
query_lens = (
|
||||
common_attn_metadata.query_start_loc[1:]
|
||||
- common_attn_metadata.query_start_loc[:-1]
|
||||
)
|
||||
chunk_starts = common_attn_metadata.seq_lens - query_lens
|
||||
num_work_per_request = (
|
||||
common_attn_metadata.seq_lens // compress_ratio
|
||||
- chunk_starts // compress_ratio
|
||||
)
|
||||
num_work_per_request = torch.where(
|
||||
query_lens > 0, num_work_per_request.clamp_min(1), 0
|
||||
)
|
||||
k_start_loc = torch.empty(
|
||||
query_lens.shape[0] + 1,
|
||||
dtype=torch.int32,
|
||||
device=query_lens.device,
|
||||
)
|
||||
k_start_loc[0] = 0
|
||||
torch.cumsum(num_work_per_request, 0, out=k_start_loc[1:])
|
||||
work = torch.arange(
|
||||
k_work_metadata_buffer.shape[0],
|
||||
device=k_work_metadata_buffer.device,
|
||||
)
|
||||
requests = torch.searchsorted(k_start_loc[1:], work, right=True)
|
||||
active = work < k_start_loc[-1]
|
||||
work_in_request = (
|
||||
work - k_start_loc[requests.clamp_max(query_lens.shape[0] - 1)]
|
||||
)
|
||||
k_work_metadata_buffer[:, 0].copy_(
|
||||
torch.where(active, requests, -1).to(torch.int32)
|
||||
)
|
||||
k_work_metadata_buffer[:, 1].copy_(
|
||||
torch.where(active, work_in_request, -1).to(torch.int32)
|
||||
)
|
||||
return token_to_req, logical_positions, slot_mapping
|
||||
|
||||
|
||||
# Resolve the fallback outside the per-step metadata hot path.
|
||||
build_qsa_metadata = (
|
||||
build_qsa_metadata_triton if HAS_TRITON else _build_qsa_metadata_torch
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class QSAForwardMetadata(AttentionMetadata):
|
||||
"""Common per-forward metadata for one QSA side cache."""
|
||||
|
||||
block_table: torch.Tensor
|
||||
slot_mapping: torch.Tensor
|
||||
seq_lens: torch.Tensor
|
||||
query_start_loc: torch.Tensor
|
||||
token_to_req: torch.Tensor
|
||||
logical_positions: torch.Tensor
|
||||
k_work_metadata: torch.Tensor
|
||||
num_actual_tokens: int
|
||||
storage_block_size: int
|
||||
compress_ratio: int
|
||||
|
||||
|
||||
class QSAMetadataBuilder(AttentionMetadataBuilder[QSAForwardMetadata]):
|
||||
"""Build QSA metadata from vLLM's cache-group-specific common metadata."""
|
||||
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_spec: AttentionSpec,
|
||||
layer_names: list[str],
|
||||
vllm_config: VllmConfig,
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||
self.is_circular_buffer = isinstance(kv_cache_spec, CircularBufferSpec)
|
||||
if isinstance(kv_cache_spec, MLAAttentionSpec):
|
||||
compress_ratio = kv_cache_spec.tokens_per_state
|
||||
assert isinstance(compress_ratio, int), (
|
||||
"QSA compression requires an integer tokens_per_state"
|
||||
)
|
||||
self.compress_ratio = compress_ratio
|
||||
else:
|
||||
self.compress_ratio = 1
|
||||
self.storage_block_size = kv_cache_spec.num_states
|
||||
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
self.token_to_req_buffer = torch.empty(
|
||||
max_tokens, dtype=torch.int32, device=device
|
||||
)
|
||||
self.slot_mapping_buffer = torch.empty(
|
||||
max_tokens, dtype=torch.int64, device=device
|
||||
)
|
||||
self.logical_positions_buffer = torch.empty(
|
||||
max_tokens, dtype=torch.int64, device=device
|
||||
)
|
||||
max_requests = vllm_config.scheduler_config.max_num_seqs
|
||||
self.request_capacity = max_requests
|
||||
if not self.is_circular_buffer and self.compress_ratio != 1:
|
||||
max_k_work = (
|
||||
max_tokens + (self.compress_ratio - 1) * max_requests
|
||||
) // self.compress_ratio
|
||||
self.k_work_metadata_buffer = torch.empty(
|
||||
max_k_work, 2, dtype=torch.int32, device=device
|
||||
)
|
||||
else:
|
||||
self.k_work_metadata_buffer = torch.empty(
|
||||
0, 2, dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
def build(
|
||||
self,
|
||||
common_prefix_len: int,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
fast_build: bool = False,
|
||||
) -> QSAForwardMetadata:
|
||||
del common_prefix_len, fast_build
|
||||
num_tokens = common_attn_metadata.num_actual_tokens
|
||||
build_k_work = not self.is_circular_buffer and self.compress_ratio != 1
|
||||
k_work_metadata = self.k_work_metadata_buffer
|
||||
request_capacity = None
|
||||
if build_k_work:
|
||||
num_requests = common_attn_metadata.query_start_loc.shape[0] - 1
|
||||
request_capacity = self.request_capacity
|
||||
max_num_work = (
|
||||
num_tokens + (self.compress_ratio - 1) * num_requests
|
||||
) // self.compress_ratio
|
||||
k_work_metadata = self.k_work_metadata_buffer[:max_num_work]
|
||||
token_to_req, logical_positions, slot_mapping = build_qsa_metadata(
|
||||
common_attn_metadata,
|
||||
self.token_to_req_buffer,
|
||||
self.logical_positions_buffer,
|
||||
self.slot_mapping_buffer,
|
||||
storage_block_size=self.storage_block_size,
|
||||
compress_ratio=self.compress_ratio,
|
||||
circular_buffer_size=(
|
||||
self.kv_cache_spec.block_size if self.is_circular_buffer else 0
|
||||
),
|
||||
k_work_metadata_buffer=k_work_metadata if build_k_work else None,
|
||||
request_capacity=request_capacity,
|
||||
)
|
||||
return QSAForwardMetadata(
|
||||
block_table=common_attn_metadata.block_table_tensor,
|
||||
slot_mapping=slot_mapping,
|
||||
seq_lens=common_attn_metadata.seq_lens,
|
||||
query_start_loc=common_attn_metadata.query_start_loc,
|
||||
token_to_req=token_to_req,
|
||||
logical_positions=logical_positions,
|
||||
k_work_metadata=k_work_metadata,
|
||||
num_actual_tokens=num_tokens,
|
||||
storage_block_size=self.storage_block_size,
|
||||
compress_ratio=self.compress_ratio,
|
||||
)
|
||||
|
||||
|
||||
class QSAStateBackend(AttentionBackend):
|
||||
"""Key-only dummy backend for out-of-band BF16 QSA side-cache operations."""
|
||||
|
||||
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
|
||||
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = ["auto", "bfloat16"]
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "QWEN4_EXP_EXP_QSA_STATE"
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls():
|
||||
raise NotImplementedError(
|
||||
"QSA state caches run out-of-band and have no attention impl"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type[QSAMetadataBuilder]:
|
||||
return QSAMetadataBuilder
|
||||
|
||||
@classmethod
|
||||
def supported_kv_cache_layouts(cls) -> tuple[KVCacheLayout, ...]:
|
||||
# QSA pages are packed beside the main KV pages within each block.
|
||||
return (KVCacheLayout.BLNHC, KVCacheLayout.BLHNC)
|
||||
|
||||
|
||||
class _QSAStateCache(nn.Module, AttentionLayerBase):
|
||||
supports_dcp = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
head_size: int,
|
||||
dtype: torch.dtype,
|
||||
cache_config: CacheConfig,
|
||||
prefix: str,
|
||||
vllm_config: VllmConfig,
|
||||
compress_ratio: int = 1,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if head_size <= 0:
|
||||
raise ValueError("QSA cache head size must be positive")
|
||||
if compress_ratio <= 0:
|
||||
raise ValueError("QSA compression ratio must be positive")
|
||||
if cache_config.block_size % compress_ratio:
|
||||
raise ValueError(
|
||||
"QSA cache block size must be divisible by the compression ratio"
|
||||
)
|
||||
self.head_size = head_size
|
||||
self.dtype = dtype
|
||||
self.cache_config = cache_config
|
||||
self.prefix = prefix
|
||||
self.compress_ratio = compress_ratio
|
||||
self.kv_cache = torch.tensor([])
|
||||
|
||||
static_context = vllm_config.compilation_config.static_forward_context
|
||||
if prefix in static_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
static_context[prefix] = self
|
||||
|
||||
def forward(self) -> None: ...
|
||||
|
||||
def bind_kv_cache(self, kv_cache: torch.Tensor) -> None:
|
||||
"""Adapt the unified [B, H, N, C] view to QSA's [B, N, H, C]."""
|
||||
if kv_cache.ndim != 4 or kv_cache.shape[1] != 1:
|
||||
raise ValueError("QSA state cache must be [blocks, 1, states, width]")
|
||||
if kv_cache.dtype != torch.bfloat16 or kv_cache.shape[3] != self.head_size:
|
||||
raise ValueError("QSA state cache does not match its packed BF16 spec")
|
||||
super().bind_kv_cache(kv_cache.transpose(1, 2))
|
||||
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
return QSAStateBackend
|
||||
|
||||
|
||||
class QSAKeyStateCache(_QSAStateCache):
|
||||
"""Raw BF16 key, optionally followed by exact int64 MRoPE positions."""
|
||||
|
||||
_BF16_PER_INT64 = 4
|
||||
_NUM_ROPE_AXES = 3
|
||||
|
||||
def __init__(self, *, cache_rope_positions: bool = False, **kwargs) -> None:
|
||||
key_head_size = int(kwargs.pop("head_size"))
|
||||
self.key_head_size = key_head_size
|
||||
self.cache_rope_positions = bool(cache_rope_positions)
|
||||
self.rope_position_offset = (
|
||||
(key_head_size + self._BF16_PER_INT64 - 1) // self._BF16_PER_INT64
|
||||
) * self._BF16_PER_INT64
|
||||
storage_head_size = key_head_size
|
||||
if self.cache_rope_positions:
|
||||
storage_head_size = self.rope_position_offset + (
|
||||
self._NUM_ROPE_AXES * self._BF16_PER_INT64
|
||||
)
|
||||
super().__init__(head_size=storage_head_size, **kwargs)
|
||||
|
||||
def bind_kv_cache(self, kv_cache: torch.Tensor) -> None:
|
||||
super().bind_kv_cache(kv_cache)
|
||||
qsa_cache = self.kv_cache
|
||||
self.key_cache = qsa_cache[..., : self.key_head_size]
|
||||
if self.cache_rope_positions:
|
||||
position_tail = qsa_cache[..., self.rope_position_offset :]
|
||||
self.rope_position_cache = position_tail.view(torch.int64)
|
||||
else:
|
||||
self.rope_position_cache = None
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
|
||||
# Hold the open group's committed keys plus every row a speculative
|
||||
# step stores before acceptance is known, rounded up to whole groups so
|
||||
# the ring divides the attention block size (it joins the LCM that sets
|
||||
# the scheduler block size). Anything narrower lets a rejected draft row
|
||||
# overwrite a committed key the next step needs to close the group.
|
||||
span = self.compress_ratio + vllm_config.num_speculative_tokens
|
||||
capacity = self.compress_ratio * cdiv(span, self.compress_ratio)
|
||||
assert self.cache_config.block_size % capacity == 0, (
|
||||
f"QSA ring capacity {capacity} must divide the attention block "
|
||||
f"size {self.cache_config.block_size}"
|
||||
)
|
||||
return CircularBufferSpec(
|
||||
block_size=capacity,
|
||||
num_kv_heads=1,
|
||||
head_size=self.head_size,
|
||||
head_size_v=0,
|
||||
dtype=self.dtype,
|
||||
)
|
||||
|
||||
|
||||
class QSACompressedKeyCache(_QSAStateCache):
|
||||
"""Normalized, group-first-RoPE BF16 key at one row per complete group."""
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
|
||||
del vllm_config
|
||||
return MLAAttentionSpec(
|
||||
block_size=self.cache_config.block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=self.head_size,
|
||||
dtype=self.dtype,
|
||||
tokens_per_state=self.compress_ratio,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"QSACompressedKeyCache",
|
||||
"QSAForwardMetadata",
|
||||
"QSAKeyStateCache",
|
||||
"QSAMetadataBuilder",
|
||||
"QSAStateBackend",
|
||||
"canonical_qsa_rope_positions",
|
||||
"circular_qsa_slot_mapping",
|
||||
"compressed_qsa_slot_mapping",
|
||||
]
|
||||
@@ -0,0 +1,252 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Qwen4Exp model configuration."""
|
||||
|
||||
from typing import Any, ClassVar, cast
|
||||
|
||||
from transformers import PretrainedConfig
|
||||
from transformers.models.qwen3_vl.configuration_qwen3_vl import (
|
||||
Qwen3VLVisionConfig,
|
||||
)
|
||||
|
||||
from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig
|
||||
|
||||
_QSA_CONFIG_FIELDS = (
|
||||
"indexer_n_heads",
|
||||
"indexer_kv_heads",
|
||||
"indexer_head_dim",
|
||||
"indexer_budget",
|
||||
"indexer_compress_ratio",
|
||||
)
|
||||
|
||||
|
||||
class Qwen4ExpVisionConfig(Qwen3VLVisionConfig):
|
||||
model_type = "qwen4_exp"
|
||||
base_config_key = "vision_config"
|
||||
|
||||
|
||||
class Qwen4ExpTextConfig(Qwen3NextConfig):
|
||||
model_type = "qwen4_exp_text"
|
||||
base_config_key = "text_config"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hc_count: int = 4,
|
||||
hc_lowrank: int = 320,
|
||||
ple_layer_ids: list[int] | None = None,
|
||||
ple_embed_dim: int | None = None,
|
||||
ple_conv_kernel_size: int = 4,
|
||||
ngram_size: int = 3,
|
||||
heads_per_ngram: int = 8,
|
||||
ngram_vocab_size_base: int = 20_000_000,
|
||||
make_ngram_vocab_size_divisible_by: int = 128,
|
||||
output_gate_type: str = "sigmoid",
|
||||
rope_parameters: dict[str, Any] | None = None,
|
||||
layer_types: list[str] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
if hc_count <= 1:
|
||||
raise ValueError(f"Qwen4Exp requires hc_count > 1, got {hc_count}.")
|
||||
|
||||
if rope_parameters is not None:
|
||||
if kwargs.get("rope_scaling") is None:
|
||||
kwargs["rope_scaling"] = rope_parameters
|
||||
if kwargs.get("rope_theta") is None and "rope_theta" in rope_parameters:
|
||||
kwargs["rope_theta"] = rope_parameters["rope_theta"]
|
||||
if (
|
||||
kwargs.get("partial_rotary_factor") is None
|
||||
and "partial_rotary_factor" in rope_parameters
|
||||
):
|
||||
kwargs["partial_rotary_factor"] = rope_parameters[
|
||||
"partial_rotary_factor"
|
||||
]
|
||||
|
||||
rope_scaling = kwargs.get("rope_scaling")
|
||||
rope_theta = kwargs.get("rope_theta", 10_000.0)
|
||||
super().__init__(layer_types=layer_types, **kwargs)
|
||||
|
||||
normalized_rope_parameters = self.rope_parameters
|
||||
self.rope_scaling = (
|
||||
rope_scaling or rope_parameters or normalized_rope_parameters
|
||||
)
|
||||
self.rope_parameters = rope_parameters or normalized_rope_parameters
|
||||
self.rope_theta = rope_theta
|
||||
|
||||
self.hc_count = hc_count
|
||||
self.hc_lowrank = hc_lowrank
|
||||
self.ple_layer_ids = ple_layer_ids or []
|
||||
self.ple_embed_dim = (
|
||||
self.hidden_size if ple_embed_dim is None else ple_embed_dim
|
||||
)
|
||||
self.ple_conv_kernel_size = ple_conv_kernel_size
|
||||
self.ngram_size = ngram_size
|
||||
self.heads_per_ngram = heads_per_ngram
|
||||
self.ngram_vocab_size_base = ngram_vocab_size_base
|
||||
self.make_ngram_vocab_size_divisible_by = make_ngram_vocab_size_divisible_by
|
||||
self.output_gate_type = output_gate_type
|
||||
|
||||
self._validate_ple_config()
|
||||
self._validate_ple_layer_ids()
|
||||
self._validate_qsa_config()
|
||||
|
||||
def _validate_ple_config(self) -> None:
|
||||
if self.hc_lowrank <= 0:
|
||||
raise ValueError(f"hc_lowrank must be positive, got {self.hc_lowrank}")
|
||||
if self.ngram_size < 2:
|
||||
raise ValueError(f"ngram_size must be >= 2, got {self.ngram_size}")
|
||||
if self.heads_per_ngram <= 0:
|
||||
raise ValueError(
|
||||
f"heads_per_ngram must be positive, got {self.heads_per_ngram}"
|
||||
)
|
||||
if self.ple_embed_dim <= 0:
|
||||
raise ValueError(
|
||||
f"ple_embed_dim must be positive, got {self.ple_embed_dim}"
|
||||
)
|
||||
ngram_heads = (self.ngram_size - 1) * self.heads_per_ngram
|
||||
if self.ple_embed_dim % ngram_heads:
|
||||
raise ValueError(
|
||||
"ple_embed_dim must be divisible by total ngram heads: "
|
||||
f"{self.ple_embed_dim} % {ngram_heads} != 0"
|
||||
)
|
||||
if self.ple_conv_kernel_size <= 0:
|
||||
raise ValueError(
|
||||
"ple_conv_kernel_size must be positive, got "
|
||||
f"{self.ple_conv_kernel_size}"
|
||||
)
|
||||
if self.ngram_vocab_size_base <= 0:
|
||||
raise ValueError("ngram_vocab_size_base must be positive")
|
||||
if self.make_ngram_vocab_size_divisible_by <= 0:
|
||||
raise ValueError("make_ngram_vocab_size_divisible_by must be positive")
|
||||
|
||||
def _validate_ple_layer_ids(self) -> None:
|
||||
invalid = [
|
||||
layer_id
|
||||
for layer_id in self.ple_layer_ids
|
||||
if not 1 <= int(layer_id) <= self.num_hidden_layers
|
||||
]
|
||||
if invalid:
|
||||
raise ValueError(
|
||||
"ple_layer_ids are 1-based and must refer to an existing layer; "
|
||||
f"got {invalid} for {self.num_hidden_layers} layers"
|
||||
)
|
||||
|
||||
def _validate_qsa_config(self) -> None:
|
||||
configured = {name: getattr(self, name, None) for name in _QSA_CONFIG_FIELDS}
|
||||
if all(value is None for value in configured.values()):
|
||||
return
|
||||
|
||||
missing = [name for name, value in configured.items() if value is None]
|
||||
if missing:
|
||||
raise ValueError(f"QSA config is missing required fields: {missing}")
|
||||
|
||||
values = {name: int(cast(int, value)) for name, value in configured.items()}
|
||||
if any(value <= 0 for value in values.values()):
|
||||
raise ValueError(f"QSA config values must be positive: {values}")
|
||||
if values["indexer_kv_heads"] != 1:
|
||||
raise ValueError("the QSA MQA operators require indexer_kv_heads=1")
|
||||
if values["indexer_budget"] % values["indexer_compress_ratio"] != 0:
|
||||
raise ValueError(
|
||||
"indexer_budget must be divisible by indexer_compress_ratio"
|
||||
)
|
||||
block_topk = values["indexer_budget"] // values["indexer_compress_ratio"]
|
||||
if block_topk not in (512, 2048):
|
||||
raise ValueError(
|
||||
"QSA requires indexer_budget / indexer_compress_ratio "
|
||||
f"to be 512 or 2048, got {block_topk}"
|
||||
)
|
||||
rotary_dim = int(self.head_dim * self.partial_rotary_factor)
|
||||
if rotary_dim > values["indexer_head_dim"]:
|
||||
raise ValueError(
|
||||
"QSA indexer_head_dim must cover the attention rotary "
|
||||
f"dimension, got {values['indexer_head_dim']} < {rotary_dim}"
|
||||
)
|
||||
|
||||
@property
|
||||
def layers_block_type(self) -> list[str]:
|
||||
return [
|
||||
"attention" if layer_type == "full_attention" else layer_type
|
||||
for layer_type in self.layer_types
|
||||
]
|
||||
|
||||
@property
|
||||
def short_conv_layer_ids(self) -> list[int]:
|
||||
if not self.ple_layer_ids:
|
||||
return []
|
||||
return sorted({int(layer_id) - 1 for layer_id in self.ple_layer_ids})
|
||||
|
||||
@property
|
||||
def short_conv_state_shape(self) -> tuple[int, int] | None:
|
||||
if not self.short_conv_layer_ids:
|
||||
return None
|
||||
ple_state_len = (self.ple_conv_kernel_size - 1) * self.ngram_size
|
||||
ple_channels = self.hidden_size * self.hc_count
|
||||
return ple_channels, ple_state_len
|
||||
|
||||
@property
|
||||
def ngram_context_len(self) -> int:
|
||||
if not self.ple_layer_ids:
|
||||
return 0
|
||||
return max(int(self.ngram_size) - 1, 0)
|
||||
|
||||
|
||||
class Qwen4ExpConfig(PretrainedConfig):
|
||||
model_type = "qwen4_exp"
|
||||
sub_configs: ClassVar[dict[str, type[PretrainedConfig]]] = {
|
||||
"vision_config": Qwen4ExpVisionConfig,
|
||||
"text_config": Qwen4ExpTextConfig,
|
||||
}
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
text_config: Qwen4ExpTextConfig | dict[str, Any] | None = None,
|
||||
vision_config: Qwen4ExpVisionConfig | dict[str, Any] | None = None,
|
||||
image_token_id: int = 248056,
|
||||
video_token_id: int = 248057,
|
||||
vision_start_token_id: int = 248053,
|
||||
vision_end_token_id: int = 248054,
|
||||
tie_word_embeddings: bool = False,
|
||||
rope_parameters: dict[str, Any] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
if text_config is not None:
|
||||
kwargs.pop("split_ngram_parts", None)
|
||||
|
||||
text_kwargs = (
|
||||
dict(kwargs)
|
||||
if text_config is None
|
||||
and "hidden_size" in kwargs
|
||||
and "num_hidden_layers" in kwargs
|
||||
else {}
|
||||
)
|
||||
|
||||
if isinstance(vision_config, dict):
|
||||
self.vision_config = self.sub_configs["vision_config"](**vision_config)
|
||||
elif vision_config is None:
|
||||
self.vision_config = self.sub_configs["vision_config"]()
|
||||
else:
|
||||
self.vision_config = vision_config
|
||||
|
||||
if isinstance(text_config, dict):
|
||||
self.text_config = self.sub_configs["text_config"](**text_config)
|
||||
elif text_config is None:
|
||||
self.text_config = self.sub_configs["text_config"](**text_kwargs)
|
||||
else:
|
||||
self.text_config = text_config
|
||||
|
||||
self.image_token_id = image_token_id
|
||||
self.video_token_id = video_token_id
|
||||
self.vision_start_token_id = vision_start_token_id
|
||||
self.vision_end_token_id = vision_end_token_id
|
||||
self.rope_parameters = rope_parameters or getattr(
|
||||
self.text_config, "rope_parameters", {}
|
||||
)
|
||||
super().__init__(**kwargs, tie_word_embeddings=tie_word_embeddings)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"Qwen4ExpConfig",
|
||||
"Qwen4ExpTextConfig",
|
||||
"Qwen4ExpVisionConfig",
|
||||
]
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,205 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""HyperConnection (Gated Residual) utilities — NVIDIA model variant.
|
||||
|
||||
Implements the HyperConnection residual scheme proposed in
|
||||
"HyperConnections" (https://arxiv.org/abs/2409.19606). This NVIDIA variant
|
||||
delays each HC combine to the following HC mix boundary. HC glue kernels,
|
||||
including fused combine+RMSNorm, live in ``ops/hc.py``; projections remain
|
||||
standard vLLM Linear modules.
|
||||
|
||||
Hidden states between layers have shape ``[..., HC*HS]`` with HS inner
|
||||
(HC outer, HS inner — checkpoint-native layout).
|
||||
|
||||
Typical usage inside a transformer decoder layer::
|
||||
|
||||
self.attn_hc = GatedResidual(hc_config)
|
||||
|
||||
hidden_states, block_input, injection = self.attn_hc.mix(hidden_states)
|
||||
attention_output = attention(block_input)
|
||||
hidden_states, block_input, injection = self.mlp_hc.combine_and_mix(
|
||||
hidden_states, attention_output, injection
|
||||
)
|
||||
"""
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
)
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
|
||||
from ..common.hyperconnection import (
|
||||
GroupedGemmaRMSNorm,
|
||||
HyperConnectionConfig,
|
||||
)
|
||||
from .ops.hc import (
|
||||
grouped_gemma_rmsnorm,
|
||||
hc_combine,
|
||||
hc_combine_norm,
|
||||
hc_gate_mix,
|
||||
hc_silu,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Gated-residual variant
|
||||
# ---------------------------------------------------------------------------
|
||||
class GatedResidual(nn.Module):
|
||||
"""Gated HyperConnection with learnable low-rank mixing and injection.
|
||||
|
||||
``combine_and_mix()`` runs the pre pipeline (grouped GemmaRMSNorm -> merged
|
||||
low-rank down+inject GEMM -> silu -> up GEMM -> sigmoid -> gated mean
|
||||
over the HC streams). When passed a pending block output and an injection,
|
||||
it fuses their residual combine with the RMSNorm. Final mixers use
|
||||
``use_combine=False`` and do not produce a new injection.
|
||||
|
||||
Weights: the norm owns the grouped GemmaRMSNorm affine; the projections
|
||||
are vLLM Linear modules (merged replicated linear for down+inject), so
|
||||
GEMM dispatch (e.g. the low-latency skinny GEMM) applies through the
|
||||
standard quant_method mechanism.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: HyperConnectionConfig,
|
||||
use_combine: bool = True,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.lora_rank = config.hc_lowrank
|
||||
self.hc_count = config.hc_count
|
||||
self.hidden_size = config.hidden_size
|
||||
self.use_combine = use_combine
|
||||
|
||||
norm_size = (
|
||||
self.hyper_hidden_size if config.hc_per_branch_norm else config.hidden_size
|
||||
)
|
||||
group_size = config.hidden_size if config.hc_per_branch_norm else None
|
||||
# Normalize each H-sized HC stream independently while retaining a
|
||||
# separate affine weight for every element of the HC*H layout.
|
||||
self.hc_norm = GroupedGemmaRMSNorm(
|
||||
norm_size,
|
||||
eps=config.rms_norm_eps,
|
||||
group_size=group_size,
|
||||
dtype=config.params_dtype,
|
||||
)
|
||||
|
||||
# -- vLLM Linear weights --------------------------------------------
|
||||
# The merged skinny-GEMM shape is physically padded to 16 rows to ensure
|
||||
# good alignment and performant implementation chosen by CuBLAS heuristics.
|
||||
self.pad_size = (-(self.lora_rank + self.hc_count)) % 16 if use_combine else 0
|
||||
if use_combine:
|
||||
self.input_mix_weight_down_block_inject = MergedColumnParallelLinear(
|
||||
self.hyper_hidden_size,
|
||||
[self.lora_rank, self.hc_count]
|
||||
+ ([self.pad_size] if self.pad_size else []),
|
||||
bias=False,
|
||||
params_dtype=config.params_dtype,
|
||||
quant_config=None,
|
||||
prefix=maybe_prefix(prefix, "input_mix_weight_down_block_inject"),
|
||||
return_bias=False,
|
||||
disable_tp=True,
|
||||
)
|
||||
else:
|
||||
self.input_mix_weight_down = ReplicatedLinear(
|
||||
self.hyper_hidden_size,
|
||||
self.lora_rank,
|
||||
bias=False,
|
||||
params_dtype=config.params_dtype,
|
||||
quant_config=None,
|
||||
prefix=maybe_prefix(prefix, "input_mix_weight_down"),
|
||||
return_bias=False,
|
||||
)
|
||||
self.input_mix_weight_up = ReplicatedLinear(
|
||||
self.lora_rank,
|
||||
self.hyper_hidden_size,
|
||||
bias=False,
|
||||
params_dtype=config.params_dtype,
|
||||
quant_config=None,
|
||||
prefix=maybe_prefix(prefix, "input_mix_weight_up"),
|
||||
return_bias=False,
|
||||
)
|
||||
|
||||
def mix(
|
||||
self, hidden_states: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
|
||||
xn = grouped_gemma_rmsnorm(
|
||||
hidden_states,
|
||||
self.hc_norm.weight,
|
||||
self.config.rms_norm_eps,
|
||||
self.hc_count,
|
||||
)
|
||||
|
||||
if self.use_combine:
|
||||
# produce injection logits for combine
|
||||
split_sizes = [self.lora_rank, self.hc_count, self.pad_size]
|
||||
down_and_injection = self.input_mix_weight_down_block_inject(xn)
|
||||
lora, injection, _ = down_and_injection.split(split_sizes, dim=-1)
|
||||
else:
|
||||
lora = self.input_mix_weight_down(xn)
|
||||
injection = None
|
||||
|
||||
lora = hc_silu(lora, self.hc_count)
|
||||
gate = self.input_mix_weight_up(lora) # [M, D]
|
||||
block_input = hc_gate_mix(xn, gate, self.hc_count)
|
||||
|
||||
return hidden_states, block_input, injection
|
||||
|
||||
def combine_and_mix(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
prev_block_output: torch.Tensor,
|
||||
prev_injection: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
|
||||
"""Consume a pending combine, then prepare the next block input.
|
||||
|
||||
``hidden_states`` is the multi-stream state from before the pending
|
||||
block's mix. Its combine with ``block_output`` is fused with this
|
||||
module's input RMSNorm.
|
||||
"""
|
||||
hidden_states, xn = hc_combine_norm(
|
||||
hidden_states,
|
||||
prev_block_output,
|
||||
prev_injection,
|
||||
self.hc_norm.weight,
|
||||
self.config.rms_norm_eps,
|
||||
self.hc_count,
|
||||
)
|
||||
|
||||
if self.use_combine:
|
||||
# produce injection logits for combine
|
||||
split_sizes = [self.lora_rank, self.hc_count, self.pad_size]
|
||||
down_and_injection = self.input_mix_weight_down_block_inject(xn)
|
||||
lora, injection, _ = down_and_injection.split(split_sizes, dim=-1)
|
||||
else:
|
||||
lora = self.input_mix_weight_down(xn)
|
||||
injection = None
|
||||
|
||||
lora = hc_silu(lora, self.hc_count)
|
||||
gate = self.input_mix_weight_up(lora) # [M, D]
|
||||
block_input = hc_gate_mix(xn, gate, self.hc_count)
|
||||
|
||||
return hidden_states, block_input, injection
|
||||
|
||||
def combine(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
block_output: torch.Tensor,
|
||||
injection: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return hc_combine(hidden_states, block_output, injection, self.hc_count)
|
||||
|
||||
@property
|
||||
def hyper_hidden_size(self) -> int:
|
||||
return self.hc_count * self.hidden_size
|
||||
|
||||
|
||||
__all__ = [
|
||||
"GatedResidual",
|
||||
"GroupedGemmaRMSNorm",
|
||||
"HyperConnectionConfig",
|
||||
]
|
||||
@@ -0,0 +1,356 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Qwen4Exp weight-free QSA indexer."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import cast
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding.mrope import triton_mrope
|
||||
from vllm.transformers_utils.configs.qwen4_exp import (
|
||||
Qwen4ExpTextConfig,
|
||||
)
|
||||
|
||||
from ..common.qsa_cache import (
|
||||
QSACompressedKeyCache,
|
||||
QSAForwardMetadata,
|
||||
QSAKeyStateCache,
|
||||
canonical_qsa_rope_positions,
|
||||
)
|
||||
from .ops.qsa_pre_indexer import qsa_pre_indexer
|
||||
|
||||
|
||||
def apply_qsa_rope(
|
||||
rotary_emb: nn.Module,
|
||||
positions: torch.Tensor,
|
||||
tensor: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Apply the main attention's exact 1D/MRoPE composition to QSA heads."""
|
||||
|
||||
num_tokens, _, head_dim = tensor.shape
|
||||
rotary_dim = rotary_emb.rotary_dim
|
||||
cache = rotary_emb._match_cos_sin_cache_dtype(tensor) # noqa: SLF001
|
||||
cos_sin = cache[positions]
|
||||
cos, sin = cos_sin.chunk(2, dim=-1)
|
||||
if positions.ndim == 2:
|
||||
shape = tensor.shape
|
||||
tensor, _ = triton_mrope(
|
||||
tensor.reshape(num_tokens, -1),
|
||||
tensor.new_empty((num_tokens, head_dim)),
|
||||
cos,
|
||||
sin,
|
||||
rotary_emb.mrope_section,
|
||||
head_dim,
|
||||
rotary_dim,
|
||||
rotary_emb.mrope_interleaved,
|
||||
rotary_emb.is_neox_style,
|
||||
)
|
||||
return tensor.reshape(shape)
|
||||
|
||||
rotated = rotary_emb.apply_rotary_emb.forward_cuda(
|
||||
tensor[..., :rotary_dim],
|
||||
cos,
|
||||
sin,
|
||||
)
|
||||
return torch.cat((rotated, tensor[..., rotary_dim:]), dim=-1)
|
||||
|
||||
|
||||
def _supports_fused_pre_indexer(
|
||||
rotary_emb: nn.Module,
|
||||
head_dim: int,
|
||||
num_kv_heads: int,
|
||||
compress_ratio: int,
|
||||
) -> bool:
|
||||
rotary_dim = int(rotary_emb.rotary_dim)
|
||||
mrope_section = getattr(rotary_emb, "mrope_section", None)
|
||||
return (
|
||||
bool(getattr(rotary_emb, "is_neox_style", False))
|
||||
and (
|
||||
not mrope_section
|
||||
or (
|
||||
len(mrope_section) == 3
|
||||
and sum(mrope_section) == rotary_dim // 2
|
||||
and bool(getattr(rotary_emb, "mrope_interleaved", False))
|
||||
)
|
||||
)
|
||||
and head_dim == 128
|
||||
and rotary_dim == 64
|
||||
and num_kv_heads == 1
|
||||
and compress_ratio > 1
|
||||
and compress_ratio & (compress_ratio - 1) == 0
|
||||
)
|
||||
|
||||
|
||||
class QSAIndexer(nn.Module):
|
||||
"""Replicated Q/K projection plus paged, weight-free QSA selection.
|
||||
|
||||
``prefix`` must be the checkpoint's indexer prefix, normally
|
||||
``model.layers.N.self_attn.indexer``. Consequently the trainable names are
|
||||
``index_qk_proj``, ``q_layernorm`` and ``k_layernorm`` under that prefix.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
vllm_config: VllmConfig,
|
||||
config: Qwen4ExpTextConfig,
|
||||
layer_id: int,
|
||||
rotary_emb: nn.Module,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if vllm_config.cache_config is None:
|
||||
raise ValueError("QSA requires a paged KV cache")
|
||||
if vllm_config.model_config.dtype != torch.bfloat16:
|
||||
raise NotImplementedError("Qwen4Exp QSA currently requires BF16")
|
||||
|
||||
self.layer_id = int(layer_id)
|
||||
self.index_n_heads = int(config.indexer_n_heads)
|
||||
self.index_kv_heads = int(config.indexer_kv_heads)
|
||||
self.index_head_dim = int(config.indexer_head_dim)
|
||||
self.token_topk = int(config.indexer_budget)
|
||||
self.compress_ratio = int(config.indexer_compress_ratio)
|
||||
self.rotary_emb = rotary_emb
|
||||
self.use_fused_pre_indexer = _supports_fused_pre_indexer(
|
||||
rotary_emb,
|
||||
self.index_head_dim,
|
||||
self.index_kv_heads,
|
||||
self.compress_ratio,
|
||||
)
|
||||
self.prefix = prefix
|
||||
# MTP step 0 selects the target-aligned rows; later steps reuse them
|
||||
# while continuing to update the QSA side cache.
|
||||
self.skip_topk = False
|
||||
|
||||
self.index_qk_proj = ReplicatedLinear(
|
||||
int(config.hidden_size),
|
||||
(self.index_n_heads + self.index_kv_heads) * self.index_head_dim,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.index_qk_proj" if prefix else "index_qk_proj",
|
||||
)
|
||||
self.q_layernorm = GemmaRMSNorm(
|
||||
self.index_head_dim,
|
||||
eps=float(getattr(config, "rms_norm_eps", 1e-6)),
|
||||
)
|
||||
self.k_layernorm = GemmaRMSNorm(
|
||||
self.index_head_dim,
|
||||
eps=float(getattr(config, "rms_norm_eps", 1e-6)),
|
||||
)
|
||||
|
||||
cache_config = vllm_config.cache_config
|
||||
cache_prefix = f"{prefix}." if prefix else ""
|
||||
self.raw_key_cache = QSAKeyStateCache(
|
||||
head_size=self.index_head_dim,
|
||||
dtype=torch.bfloat16,
|
||||
cache_rope_positions=vllm_config.model_config.uses_mrope,
|
||||
prefix=f"{cache_prefix}raw_key_cache",
|
||||
cache_config=cache_config,
|
||||
compress_ratio=self.compress_ratio,
|
||||
vllm_config=vllm_config,
|
||||
)
|
||||
self.compressed_key_cache = QSACompressedKeyCache(
|
||||
head_size=self.index_head_dim,
|
||||
dtype=torch.bfloat16,
|
||||
compress_ratio=self.compress_ratio,
|
||||
prefix=f"{cache_prefix}compressed_key_cache",
|
||||
cache_config=cache_config,
|
||||
vllm_config=vllm_config,
|
||||
)
|
||||
|
||||
@property
|
||||
def output_width(self) -> int:
|
||||
return self.token_topk + self.compress_ratio - 1
|
||||
|
||||
def _metadata(
|
||||
self,
|
||||
) -> tuple[QSAForwardMetadata, QSAForwardMetadata] | None:
|
||||
metadata = get_forward_context().attn_metadata
|
||||
if isinstance(metadata, list):
|
||||
metadata = metadata[0]
|
||||
if not isinstance(metadata, dict):
|
||||
return None
|
||||
raw = cast(QSAForwardMetadata, metadata[self.raw_key_cache.prefix])
|
||||
compressed = cast(
|
||||
QSAForwardMetadata, metadata[self.compressed_key_cache.prefix]
|
||||
)
|
||||
if raw.num_actual_tokens != compressed.num_actual_tokens:
|
||||
raise RuntimeError("QSA side-cache metadata token counts disagree")
|
||||
if not raw.logical_positions.is_cuda and (
|
||||
not torch.equal(raw.logical_positions, compressed.logical_positions)
|
||||
):
|
||||
raise RuntimeError("QSA side-cache metadata positions disagree")
|
||||
return raw, compressed
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Return fixed-width request-relative token indices padded with ``-1``."""
|
||||
|
||||
metadata = self._metadata()
|
||||
if metadata is None:
|
||||
# Preserve step-0 indices when later MTP steps reuse the buffer.
|
||||
if self.skip_topk and out is not None:
|
||||
return out
|
||||
result = torch.full(
|
||||
(hidden_states.shape[0], self.output_width),
|
||||
-1,
|
||||
dtype=torch.int32,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
if out is not None:
|
||||
out.copy_(result)
|
||||
return out
|
||||
return result
|
||||
|
||||
from .ops.qsa import (
|
||||
qsa_compress_groups_with_ratio,
|
||||
qsa_select_paged_tokens,
|
||||
qsa_store_cache_rows,
|
||||
)
|
||||
|
||||
raw_metadata, compressed_metadata = metadata
|
||||
num_tokens = raw_metadata.num_actual_tokens
|
||||
hidden_states = hidden_states[:num_tokens]
|
||||
positions = positions[..., :num_tokens]
|
||||
|
||||
# Q/K projection
|
||||
projected_qk, _ = self.index_qk_proj(hidden_states)
|
||||
projected_q, raw_keys = projected_qk.split(
|
||||
(
|
||||
self.index_n_heads * self.index_head_dim,
|
||||
self.index_kv_heads * self.index_head_dim,
|
||||
),
|
||||
dim=-1,
|
||||
)
|
||||
raw_key_state_cache = self.raw_key_cache
|
||||
compressed_key_cache = self.compressed_key_cache.kv_cache
|
||||
|
||||
if self.use_fused_pre_indexer:
|
||||
q = projected_q.new_empty(
|
||||
num_tokens,
|
||||
self.index_n_heads,
|
||||
self.index_head_dim,
|
||||
)
|
||||
qsa_pre_indexer(
|
||||
projected_q,
|
||||
raw_keys,
|
||||
positions,
|
||||
self.rotary_emb.cos_sin_cache,
|
||||
self.q_layernorm.weight,
|
||||
self.k_layernorm.weight,
|
||||
self.q_layernorm.variance_epsilon,
|
||||
q,
|
||||
raw_key_state_cache.kv_cache,
|
||||
raw_metadata.slot_mapping,
|
||||
raw_metadata.block_table,
|
||||
raw_metadata.query_start_loc,
|
||||
raw_metadata.logical_positions,
|
||||
compressed_key_cache,
|
||||
compressed_metadata.slot_mapping,
|
||||
compressed_metadata.k_work_metadata,
|
||||
compress_ratio=self.compress_ratio,
|
||||
mrope_section=getattr(self.rotary_emb, "mrope_section", None),
|
||||
rope_pos_offset=(
|
||||
raw_key_state_cache.rope_position_offset
|
||||
if raw_key_state_cache.rope_position_cache is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
else:
|
||||
# Unfused reference path
|
||||
from flashinfer.norm import gemma_rmsnorm
|
||||
|
||||
q = projected_q.reshape(-1, self.index_n_heads, self.index_head_dim)
|
||||
q = gemma_rmsnorm(
|
||||
q.reshape(-1, self.index_head_dim),
|
||||
self.q_layernorm.weight,
|
||||
self.q_layernorm.variance_epsilon,
|
||||
).reshape_as(q)
|
||||
q = apply_qsa_rope(self.rotary_emb, positions, q)
|
||||
|
||||
raw_key_cache = raw_key_state_cache.key_cache
|
||||
rope_position_cache = raw_key_state_cache.rope_position_cache
|
||||
if rope_position_cache is None:
|
||||
position_rows = raw_metadata.logical_positions.view(-1, 1, 1).expand(
|
||||
-1, 1, 3
|
||||
)
|
||||
else:
|
||||
position_rows = canonical_qsa_rope_positions(positions).to(
|
||||
device=raw_key_cache.device
|
||||
)
|
||||
pooled, first_positions = qsa_compress_groups_with_ratio(
|
||||
raw_keys.reshape(-1, 1, self.index_head_dim),
|
||||
position_rows,
|
||||
raw_key_cache,
|
||||
raw_metadata.block_table,
|
||||
raw_metadata.token_to_req,
|
||||
raw_metadata.query_start_loc,
|
||||
raw_metadata.logical_positions,
|
||||
compressed_metadata.slot_mapping,
|
||||
self.compress_ratio,
|
||||
rope_position_cache,
|
||||
)
|
||||
compressed_keys = gemma_rmsnorm(
|
||||
pooled.reshape(-1, self.index_head_dim),
|
||||
self.k_layernorm.weight,
|
||||
self.k_layernorm.variance_epsilon,
|
||||
).reshape(-1, 1, self.index_head_dim)
|
||||
if getattr(self.rotary_emb, "mrope_section", None):
|
||||
first_positions = first_positions.transpose(0, 1)
|
||||
else:
|
||||
first_positions = first_positions[:, 0]
|
||||
compressed_keys = apply_qsa_rope(
|
||||
self.rotary_emb,
|
||||
first_positions,
|
||||
compressed_keys,
|
||||
)
|
||||
qsa_store_cache_rows(
|
||||
compressed_key_cache,
|
||||
compressed_metadata.slot_mapping,
|
||||
compressed_keys,
|
||||
)
|
||||
qsa_store_cache_rows(
|
||||
raw_key_cache,
|
||||
raw_metadata.slot_mapping,
|
||||
raw_keys,
|
||||
)
|
||||
if rope_position_cache is not None:
|
||||
qsa_store_cache_rows(
|
||||
rope_position_cache,
|
||||
raw_metadata.slot_mapping,
|
||||
position_rows,
|
||||
)
|
||||
|
||||
if self.skip_topk:
|
||||
if out is None:
|
||||
raise RuntimeError("QSA top-k reuse requires an output buffer")
|
||||
return out
|
||||
|
||||
# Score compressed keys, select blocks, then expand them to token indices.
|
||||
return qsa_select_paged_tokens(
|
||||
q,
|
||||
compressed_key_cache,
|
||||
compressed_metadata.block_table,
|
||||
compressed_metadata.token_to_req,
|
||||
compressed_metadata.logical_positions,
|
||||
compressed_metadata.seq_lens,
|
||||
self.token_topk,
|
||||
self.compress_ratio,
|
||||
out,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["QSAIndexer", "apply_qsa_rope"]
|
||||
@@ -0,0 +1,185 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Qwen4Exp decode GEMM selection on Blackwell.
|
||||
|
||||
Dispatch follows Kimi-K3 and uses the local ``(N, K)`` shape and token count.
|
||||
Plans contain measured CUDA graph capture sizes; other token counts use the
|
||||
standard linear implementation.
|
||||
"""
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.skinny_gemm import (
|
||||
SkinnyGemmConfig,
|
||||
shape_dynamic_skinny_gemm,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
UnquantizedEmbeddingMethod,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
QWEN4_EXP_GEMM_PLANS: dict[tuple[int, int], dict[int, SkinnyGemmConfig]] = {
|
||||
# GDN fused QKVZ projection, TP=4.
|
||||
(4096, 2560): {
|
||||
1: SkinnyGemmConfig(1, 64, 4, k_unroll=4),
|
||||
2: SkinnyGemmConfig(2, 64, 4, k_unroll=4),
|
||||
},
|
||||
# GDN and QSA output projections, TP=4.
|
||||
(2560, 1536): {
|
||||
1: SkinnyGemmConfig(1, 128, 2, k_unroll=2, vector_width=4),
|
||||
2: SkinnyGemmConfig(2, 128, 2, k_unroll=2, vector_width=4),
|
||||
4: SkinnyGemmConfig(4, 64, 2, k_unroll=2),
|
||||
},
|
||||
# GDN fused B/A projection, TP=4.
|
||||
(24, 2560): {
|
||||
1: SkinnyGemmConfig(1, 128, 2, k_unroll=4, vector_width=4),
|
||||
2: SkinnyGemmConfig(2, 128, 2, k_unroll=4, vector_width=4),
|
||||
4: SkinnyGemmConfig(4, 128, 2, k_unroll=4, vector_width=4),
|
||||
8: SkinnyGemmConfig(8, 128, 1, k_unroll=4, vector_width=4),
|
||||
16: SkinnyGemmConfig(16, 128, 1, k_unroll=4, vector_width=4),
|
||||
},
|
||||
# QSA fused QKV/gate projection, TP=4.
|
||||
(3584, 2560): {
|
||||
1: SkinnyGemmConfig(1, 128, 4, k_unroll=4, vector_width=4),
|
||||
2: SkinnyGemmConfig(2, 64, 2, k_unroll=2),
|
||||
4: SkinnyGemmConfig(4, 64, 2, k_unroll=2),
|
||||
},
|
||||
# QSA indexer Q/K projection, replicated in a TP=4 deployment.
|
||||
(640, 2560): {
|
||||
1: SkinnyGemmConfig(1, 128, 1, k_unroll=4, vector_width=4),
|
||||
2: SkinnyGemmConfig(2, 128, 1, k_unroll=4, vector_width=4),
|
||||
4: SkinnyGemmConfig(4, 128, 1, k_unroll=4, vector_width=4),
|
||||
8: SkinnyGemmConfig(8, 128, 1, k_unroll=4, vector_width=4),
|
||||
},
|
||||
# Shared-expert fused gate/up projection, TP=4.
|
||||
(320, 2560): {
|
||||
1: SkinnyGemmConfig(1, 128, 2, k_unroll=4, vector_width=4),
|
||||
2: SkinnyGemmConfig(2, 128, 2, k_unroll=4, vector_width=4),
|
||||
4: SkinnyGemmConfig(4, 128, 2, k_unroll=4, vector_width=4),
|
||||
8: SkinnyGemmConfig(8, 64, 1, k_unroll=4),
|
||||
16: SkinnyGemmConfig(16, 128, 2, k_unroll=4, vector_width=4),
|
||||
},
|
||||
# LM head, TP=4.
|
||||
(62080, 2560): {
|
||||
1: SkinnyGemmConfig(1, 64, 4, k_unroll=2),
|
||||
2: SkinnyGemmConfig(2, 32, 4, k_unroll=2),
|
||||
},
|
||||
# HC merged down/injection projection, replicated in a TP=4 deployment.
|
||||
(336, 10240): {
|
||||
1: SkinnyGemmConfig(1, 128, 1, static_k=10240),
|
||||
2: SkinnyGemmConfig(2, 128, 1, static_k=10240),
|
||||
4: SkinnyGemmConfig(4, 128, 2, static_k=10240),
|
||||
8: SkinnyGemmConfig(8, 128, 1, k_unroll=4),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _is_sm103() -> bool:
|
||||
return current_platform.is_device_capability((10, 3))
|
||||
|
||||
|
||||
def _is_packed_row_major(tensor: torch.Tensor) -> bool:
|
||||
return tensor.dim() == 2 and tensor.stride() == (tensor.shape[1], 1)
|
||||
|
||||
|
||||
def _runtime_ok(x: torch.Tensor, weight: torch.Tensor) -> bool:
|
||||
return (
|
||||
not envs.VLLM_BATCH_INVARIANT
|
||||
and _is_packed_row_major(x)
|
||||
and _is_packed_row_major(weight)
|
||||
and x.dtype == torch.bfloat16
|
||||
and weight.dtype == torch.bfloat16
|
||||
and x.is_cuda
|
||||
and weight.is_cuda
|
||||
and x.device == weight.device
|
||||
and x.shape[1] == weight.shape[1]
|
||||
)
|
||||
|
||||
|
||||
class _Qwen4ExpLowLatencyApply:
|
||||
def apply(
|
||||
self,
|
||||
layer: nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if bias is None and not envs.VLLM_BATCH_INVARIANT:
|
||||
return torch.ops.vllm.qwen4_exp_low_latency_gemm(x, layer.weight)
|
||||
return super().apply(layer, x, bias) # type: ignore[misc]
|
||||
|
||||
|
||||
class Qwen4ExpLowLatencyLinearMethod(_Qwen4ExpLowLatencyApply, UnquantizedLinearMethod):
|
||||
pass
|
||||
|
||||
|
||||
class Qwen4ExpLowLatencyEmbeddingMethod(
|
||||
_Qwen4ExpLowLatencyApply, UnquantizedEmbeddingMethod
|
||||
):
|
||||
pass
|
||||
|
||||
|
||||
def _qwen4_exp_low_latency_gemm(x: torch.Tensor, weight: torch.Tensor) -> torch.Tensor:
|
||||
plan = QWEN4_EXP_GEMM_PLANS.get((weight.shape[0], weight.shape[1]))
|
||||
config = None if plan is None else plan.get(x.shape[0])
|
||||
if (
|
||||
config is not None
|
||||
and _runtime_ok(x, weight)
|
||||
and shape_dynamic_skinny_gemm.is_available()
|
||||
):
|
||||
return shape_dynamic_skinny_gemm(x, weight, config)
|
||||
return torch.nn.functional.linear(x, weight)
|
||||
|
||||
|
||||
def _qwen4_exp_low_latency_gemm_fake(
|
||||
x: torch.Tensor, weight: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
return x.new_empty((*x.shape[:-1], weight.shape[0]))
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_low_latency_gemm",
|
||||
op_func=_qwen4_exp_low_latency_gemm,
|
||||
fake_impl=_qwen4_exp_low_latency_gemm_fake,
|
||||
)
|
||||
|
||||
|
||||
def enable_qwen4_exp_low_latency_gemm(
|
||||
module: nn.Module,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
if dtype != torch.bfloat16 or not _is_sm103():
|
||||
return
|
||||
if not shape_dynamic_skinny_gemm.is_available():
|
||||
return
|
||||
|
||||
warmup_configs: set[SkinnyGemmConfig] = set()
|
||||
for child in module.modules():
|
||||
is_linear = (
|
||||
isinstance(child, LinearBase)
|
||||
and type(child.quant_method) is UnquantizedLinearMethod
|
||||
)
|
||||
is_head = (
|
||||
isinstance(child, ParallelLMHead)
|
||||
and type(child.quant_method) is UnquantizedEmbeddingMethod
|
||||
)
|
||||
if not (is_linear or is_head):
|
||||
continue
|
||||
weight = getattr(child, "weight", None)
|
||||
if weight is None or weight.dim() != 2:
|
||||
continue
|
||||
plan = QWEN4_EXP_GEMM_PLANS.get((weight.shape[0], weight.shape[1]))
|
||||
if plan is None:
|
||||
continue
|
||||
if is_linear:
|
||||
child.quant_method = Qwen4ExpLowLatencyLinearMethod()
|
||||
else:
|
||||
child.quant_method = Qwen4ExpLowLatencyEmbeddingMethod()
|
||||
warmup_configs.update(plan.values())
|
||||
|
||||
if warmup_configs:
|
||||
shape_dynamic_skinny_gemm.request_warmup_configs(dtype, warmup_configs)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,143 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Model-runner state for Qwen4Exp PLE inputs."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
|
||||
|
||||
class Qwen4ExpModelState(MambaHybridModelState):
|
||||
"""Add rollback-safe PLE n-gram context to the model inputs."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
model: nn.Module,
|
||||
encoder_cache: EncoderCache | None,
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
super().__init__(vllm_config, model, encoder_cache, device)
|
||||
config = self.model_config.hf_text_config
|
||||
self.uses_ngram_embedding = bool(config.ple_layer_ids)
|
||||
if not self.uses_ngram_embedding:
|
||||
self.ngram_context_len = 0
|
||||
self.ngram_eos_token_id = 0
|
||||
return
|
||||
|
||||
if vllm_config.parallel_config.pipeline_parallel_size > 1:
|
||||
raise RuntimeError(
|
||||
"N-gram PLE embedding currently requires "
|
||||
"pipeline_parallel_size=1 because non-first pipeline ranks do "
|
||||
"not receive the raw input_ids required by PLE. Please run "
|
||||
"with PP=1."
|
||||
)
|
||||
|
||||
self.ngram_context_len = int(config.ngram_size) - 1
|
||||
if self.ngram_context_len <= 0:
|
||||
raise ValueError("N-gram embedding requires context length >= 1.")
|
||||
self.ngram_eos_token_id = int(config.eos_token_id)
|
||||
self.ngram_context = torch.full(
|
||||
(self.max_num_reqs, self.ngram_context_len),
|
||||
self.ngram_eos_token_id,
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
self.ngram_context_offsets = torch.arange(
|
||||
-self.ngram_context_len,
|
||||
0,
|
||||
dtype=torch.int64,
|
||||
device=self.device,
|
||||
)
|
||||
self.ple_query_start_loc = torch.zeros(
|
||||
self.max_num_reqs + 1,
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
def _prepare_ngram_context(
|
||||
self,
|
||||
input_batch: InputBatch,
|
||||
req_states: RequestState,
|
||||
) -> torch.Tensor:
|
||||
num_reqs = input_batch.num_reqs
|
||||
num_reqs_padded = input_batch.num_reqs_after_padding
|
||||
context = self.ngram_context[:num_reqs_padded]
|
||||
context.fill_(self.ngram_eos_token_id)
|
||||
if num_reqs == 0:
|
||||
return context
|
||||
|
||||
request_indices = input_batch.idx_mapping[:num_reqs].long()
|
||||
context_end = req_states.num_computed_tokens.gpu[request_indices].long()
|
||||
token_indices = context_end.unsqueeze(1) + self.ngram_context_offsets
|
||||
valid_tokens = token_indices >= 0
|
||||
token_indices.clamp_min_(0)
|
||||
context_tokens = req_states.all_token_ids.gpu[
|
||||
request_indices.unsqueeze(1), token_indices
|
||||
]
|
||||
context[:num_reqs].copy_(
|
||||
torch.where(
|
||||
valid_tokens,
|
||||
context_tokens,
|
||||
context_tokens.new_full((), self.ngram_eos_token_id),
|
||||
)
|
||||
)
|
||||
return context
|
||||
|
||||
def prepare_inputs(
|
||||
self,
|
||||
input_batch: InputBatch,
|
||||
req_states: RequestState,
|
||||
) -> dict[str, Any]:
|
||||
model_inputs = super().prepare_inputs(input_batch, req_states)
|
||||
if not self.uses_ngram_embedding:
|
||||
return model_inputs
|
||||
|
||||
num_reqs_padded = input_batch.num_reqs_after_padding
|
||||
query_start_loc = self.ple_query_start_loc[: num_reqs_padded + 1]
|
||||
query_start_loc.copy_(input_batch.query_start_loc[: num_reqs_padded + 1])
|
||||
model_inputs.update(
|
||||
query_start_loc=query_start_loc,
|
||||
ngram_context=self._prepare_ngram_context(input_batch, req_states),
|
||||
)
|
||||
return model_inputs
|
||||
|
||||
def prepare_dummy_inputs(
|
||||
self,
|
||||
num_reqs: int,
|
||||
num_tokens: int,
|
||||
) -> dict[str, Any]:
|
||||
model_inputs = super().prepare_dummy_inputs(num_reqs, num_tokens)
|
||||
if not self.uses_ngram_embedding:
|
||||
return model_inputs
|
||||
|
||||
query_start_loc = self.ple_query_start_loc[: num_reqs + 1]
|
||||
query_start_loc[0] = 0
|
||||
tokens_per_req, num_extra_tokens = divmod(num_tokens, num_reqs)
|
||||
query_lens = torch.full(
|
||||
(num_reqs,),
|
||||
tokens_per_req,
|
||||
dtype=query_start_loc.dtype,
|
||||
device=query_start_loc.device,
|
||||
)
|
||||
if num_extra_tokens > 0:
|
||||
query_lens[-num_extra_tokens:] += 1
|
||||
torch.cumsum(query_lens, dim=0, out=query_start_loc[1:])
|
||||
|
||||
ngram_context = self.ngram_context[:num_reqs]
|
||||
ngram_context.fill_(self.ngram_eos_token_id)
|
||||
model_inputs.update(
|
||||
query_start_loc=query_start_loc,
|
||||
ngram_context=ngram_context,
|
||||
)
|
||||
return model_inputs
|
||||
|
||||
|
||||
__all__ = ["Qwen4ExpModelState"]
|
||||
@@ -0,0 +1,462 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inference-only Qwen4Exp MTP (Multi-Token Predictor) model.
|
||||
|
||||
The MTP draft model reuses the Qwen4Exp backbone (PLE/HC/MoE) but:
|
||||
- drops all multi-modal handling (text-only),
|
||||
- forces PLE off while keeping the main model's HC stream count,
|
||||
- fuses the backbone hidden and the new-token embedding via
|
||||
``residual_linear_shared`` (fc_embedding + shared fc_hidden) instead of
|
||||
the ``Linear(2H, H)`` + repeat used by other MTP variants,
|
||||
- emits TWO hidden streams per step (scheme A): a single stream [T, H]
|
||||
(final-mixer collapsed, fed to the LM head) and a pre-final-mixer
|
||||
multi stream [T, hc_count*H] (fed to the next draft step).
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig, replace, set_current_vllm_config
|
||||
from vllm.distributed import get_pp_group
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.utils import configure_quant_config
|
||||
from vllm.model_executor.models.interfaces import SupportsPP
|
||||
from vllm.model_executor.models.qwen3_5 import Qwen3_5Model
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
WeightsMapper,
|
||||
get_draft_quant_config,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
maybe_fuse_shared_experts,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.qwen4_exp import (
|
||||
Qwen4ExpTextConfig,
|
||||
)
|
||||
|
||||
from .hyperconnection import GatedResidual, HyperConnectionConfig
|
||||
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
|
||||
from .model import (
|
||||
_HC_WEIGHTS_MAPPER,
|
||||
_QWEN4_EXP_IGNORED_MISSING_SUFFIXES,
|
||||
Qwen4ExpDecoderLayer,
|
||||
Qwen4ExpMixtureOfExperts,
|
||||
Qwen4ExpSparseMoeBlock,
|
||||
)
|
||||
|
||||
|
||||
def _remap_ignored_layers(
|
||||
ignored_layers: list[str],
|
||||
mtp_start_layer_idx: int,
|
||||
) -> list[str]:
|
||||
remapped: list[str] = []
|
||||
for name in ignored_layers:
|
||||
if name.startswith("mtp."):
|
||||
new_name = re.sub(
|
||||
r"(?<=\.layers\.)\d+",
|
||||
lambda m: str(mtp_start_layer_idx + int(m.group(0))),
|
||||
name,
|
||||
)
|
||||
remapped.append(new_name)
|
||||
else:
|
||||
remapped.append(name)
|
||||
return remapped
|
||||
|
||||
|
||||
def _remap_mtp_weight_name(name: str) -> str | None:
|
||||
"""Map Qwen4Exp checkpoint paths into the standalone draft model."""
|
||||
|
||||
for checkpoint_prefix in (
|
||||
"model.language_model.",
|
||||
"language_model.",
|
||||
):
|
||||
if name.startswith(checkpoint_prefix):
|
||||
name = name.removeprefix(checkpoint_prefix)
|
||||
break
|
||||
|
||||
if name.startswith("embed_tokens."):
|
||||
name = f"model.{name}"
|
||||
if name.startswith("model.mtp."):
|
||||
name = name.removeprefix("model.")
|
||||
if name.startswith("mtp.shared_head.head."):
|
||||
return name.replace("mtp.shared_head.head.", "lm_head.", 1)
|
||||
if name.startswith("model.shared_head.head."):
|
||||
return name.replace("model.shared_head.head.", "lm_head.", 1)
|
||||
if name.startswith("shared_head.head."):
|
||||
return name.replace("shared_head.head.", "lm_head.", 1)
|
||||
if name.startswith("model.lm_head."):
|
||||
return name.removeprefix("model.")
|
||||
if name.startswith("mtp."):
|
||||
return name.replace("mtp.", "model.", 1)
|
||||
if name.startswith("model.embed_tokens.") or name.startswith("lm_head."):
|
||||
return name
|
||||
return None
|
||||
|
||||
|
||||
def _make_draft_vllm_config(
|
||||
vllm_config: VllmConfig,
|
||||
mtp_start_layer_idx: int,
|
||||
) -> VllmConfig:
|
||||
"""Ensure that the draft model config is set in the vLLM config."""
|
||||
speculative_config = vllm_config.speculative_config
|
||||
if speculative_config is None or speculative_config.draft_model_config is None:
|
||||
raise ValueError("speculative_config.draft_model_config must be set")
|
||||
|
||||
draft_quant_config = get_draft_quant_config(vllm_config)
|
||||
|
||||
# inject packed and ignored modules to the quantization config of draft model
|
||||
if draft_quant_config is not None:
|
||||
configure_quant_config(draft_quant_config, Qwen4ExpMTP)
|
||||
ignored_layers = getattr(draft_quant_config, "ignored_layers", None)
|
||||
if ignored_layers:
|
||||
setattr( # noqa: B010
|
||||
draft_quant_config,
|
||||
"ignored_layers",
|
||||
_remap_ignored_layers(ignored_layers, mtp_start_layer_idx),
|
||||
)
|
||||
exclude_modules = getattr(draft_quant_config, "exclude_modules", None)
|
||||
if exclude_modules:
|
||||
setattr( # noqa: B010
|
||||
draft_quant_config,
|
||||
"exclude_modules",
|
||||
_remap_ignored_layers(exclude_modules, mtp_start_layer_idx),
|
||||
)
|
||||
|
||||
draft_vllm_config = replace(
|
||||
vllm_config,
|
||||
model_config=speculative_config.draft_model_config,
|
||||
)
|
||||
# VllmConfig post-init derives the target quant config, so restore the
|
||||
# independently resolved draft quant config after replacement.
|
||||
draft_vllm_config.quant_config = draft_quant_config
|
||||
return draft_vllm_config
|
||||
|
||||
|
||||
@support_torch_compile(
|
||||
dynamic_arg_dims={
|
||||
"input_ids": 0,
|
||||
"positions": -1,
|
||||
"intermediate_tensors": 0,
|
||||
"inputs_embeds": 0,
|
||||
"hidden_states": 0,
|
||||
}
|
||||
)
|
||||
class Qwen4ExpMultiTokenPredictor(nn.Module):
|
||||
hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _HC_WEIGHTS_MAPPER
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
|
||||
model_config = vllm_config.model_config
|
||||
config: Qwen4ExpTextConfig = model_config.hf_text_config
|
||||
|
||||
self.config = config
|
||||
self.vocab_size = config.vocab_size
|
||||
|
||||
self.mtp_start_layer_idx = config.num_hidden_layers
|
||||
self.num_mtp_layers = getattr(config, "mtp_num_hidden_layers", 1)
|
||||
|
||||
self.hidden_size = config.hidden_size
|
||||
self.hc_count = config.hc_count
|
||||
|
||||
self.embed_tokens = VocabParallelEmbedding(self.vocab_size, self.hidden_size)
|
||||
draft_vllm_config = _make_draft_vllm_config(
|
||||
vllm_config,
|
||||
self.mtp_start_layer_idx,
|
||||
)
|
||||
with set_current_vllm_config(draft_vllm_config, prefix=prefix):
|
||||
# residual_linear_shared fusion: fc_embedding projects the token
|
||||
# embedding, fc_hidden (shared across HC branches) projects the
|
||||
# backbone hidden; the embedding is added as a residual to every
|
||||
# branch (see mtp_residual_linear_shared.md).
|
||||
self.fc_embedding = ColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
self.hidden_size,
|
||||
gather_output=True,
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
quant_config=draft_vllm_config.quant_config,
|
||||
prefix=f"{prefix}.fc_embedding",
|
||||
)
|
||||
self.fc_hidden = ColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
self.hidden_size,
|
||||
gather_output=True,
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
quant_config=draft_vllm_config.quant_config,
|
||||
prefix=f"{prefix}.fc_hidden",
|
||||
)
|
||||
self.layers = nn.ModuleList(
|
||||
Qwen4ExpDecoderLayer(
|
||||
draft_vllm_config,
|
||||
layer_type="full_attention",
|
||||
prefix=f"{prefix}.layers.{self.mtp_start_layer_idx + idx}",
|
||||
)
|
||||
for idx in range(self.num_mtp_layers)
|
||||
)
|
||||
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
|
||||
self.layers,
|
||||
Qwen4ExpSparseMoeBlock,
|
||||
"mlp",
|
||||
)
|
||||
|
||||
self.pre_fc_norm_embedding = GemmaRMSNorm(
|
||||
self.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.pre_fc_norm_hidden = GemmaRMSNorm(
|
||||
self.hidden_size * self.hc_count, eps=config.rms_norm_eps
|
||||
)
|
||||
# HC final mixer collapses the multi stream into [T, H] for the LM head.
|
||||
hc_config = HyperConnectionConfig(
|
||||
hc_count=config.hc_count,
|
||||
hidden_size=config.hidden_size,
|
||||
params_dtype=torch.bfloat16,
|
||||
hc_lowrank=config.hc_lowrank,
|
||||
rms_norm_eps=config.rms_norm_eps,
|
||||
hc_per_branch_norm=True,
|
||||
)
|
||||
self.hyper_connection_mixer = GatedResidual(
|
||||
hc_config,
|
||||
use_combine=False,
|
||||
prefix=maybe_prefix(prefix, "hyper_connection_mixer"),
|
||||
)
|
||||
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
|
||||
["hidden_states"], self.hidden_size * self.hc_count
|
||||
)
|
||||
|
||||
def _iter_qsa_attentions(self):
|
||||
"""Yield MTP attention modules that own a QSA indexer."""
|
||||
|
||||
for layer in self.layers:
|
||||
attention = getattr(layer, "self_attn", None)
|
||||
if (
|
||||
attention is not None
|
||||
and getattr(attention, "indexer", None) is not None
|
||||
):
|
||||
yield attention
|
||||
|
||||
def set_skip_topk(self, skip: bool) -> None:
|
||||
"""Select on MTP step 0 and reuse its QSA indices on later steps."""
|
||||
|
||||
for attention in self._iter_qsa_attentions():
|
||||
attention.indexer.skip_topk = skip
|
||||
|
||||
def compact_topk_indices(self, row_indices: torch.Tensor) -> None:
|
||||
"""Keep each request's target-aligned step-0 sparse-index row."""
|
||||
|
||||
num_rows = row_indices.numel()
|
||||
for attention in self._iter_qsa_attentions():
|
||||
buffer = attention.topk_indices_buffer
|
||||
selected = buffer.index_select(0, row_indices)
|
||||
buffer[:num_rows].copy_(selected)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor | None = None,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor] | IntermediateTensors:
|
||||
hc_count = self.hc_count
|
||||
hidden_size = self.hidden_size
|
||||
|
||||
if get_pp_group().is_first_rank:
|
||||
assert hidden_states is not None
|
||||
if inputs_embeds is None:
|
||||
assert input_ids is not None
|
||||
inputs_embeds = self.embed_input_ids(input_ids)
|
||||
# Embedding branch: pre-norm -> fc_embedding -> [T, H].
|
||||
inputs_embeds = self.pre_fc_norm_embedding(inputs_embeds)
|
||||
inputs_embeds = self.fc_embedding(inputs_embeds)
|
||||
|
||||
# Backbone hidden is multi-stream [T, hc_count*H] (scheme A:
|
||||
# the main model truly emits the pre-final-mixer multi stream
|
||||
# on the first step; subsequent steps reuse the prior draft
|
||||
# step's multi stream).
|
||||
num_tokens = hidden_states.shape[0]
|
||||
hidden_states = hidden_states.view(num_tokens, hc_count, hidden_size)
|
||||
hidden_states = self.pre_fc_norm_hidden(hidden_states.flatten(-2)).view(
|
||||
num_tokens, hc_count, hidden_size
|
||||
)
|
||||
hidden_states = self.fc_hidden(hidden_states)
|
||||
# Add the embedding residual to every branch, then fold back
|
||||
# to [T, hc_count*H] (HC outer, HS inner) for the HC decoder.
|
||||
hidden_states = inputs_embeds.unsqueeze(-2) + hidden_states
|
||||
hidden_states = hidden_states.flatten(-2)
|
||||
else:
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
layer = self.layers[current_step_idx]
|
||||
hidden_states, block_output, injection = layer(
|
||||
hidden_states=hidden_states,
|
||||
prev_block_output=None,
|
||||
prev_injection=None,
|
||||
positions=positions,
|
||||
input_ids=None,
|
||||
query_start_loc=None,
|
||||
ngram_context=None,
|
||||
)
|
||||
if not get_pp_group().is_last_rank:
|
||||
# As in the target model, PP carries a materialized tensor rather
|
||||
# than the delayed hidden/output/injection tuple.
|
||||
hidden_states = layer.mlp_hyper_connection.combine(
|
||||
hidden_states, block_output, injection
|
||||
)
|
||||
return IntermediateTensors({"hidden_states": hidden_states})
|
||||
|
||||
# Last PP rank finalize. Keep both:
|
||||
# (A) sample_hidden_states [T, H] -> single stream for the LM head
|
||||
# (B) multi_hidden [T, hc_count*H] -> pre-final-mixer multi stream
|
||||
# for the next draft step (zero extra compute, just kept).
|
||||
multi_hidden, sample_hidden_states, _ = (
|
||||
self.hyper_connection_mixer.combine_and_mix(
|
||||
hidden_states, block_output, injection
|
||||
)
|
||||
)
|
||||
return sample_hidden_states, multi_hidden
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
weights = maybe_fuse_shared_experts(
|
||||
weights,
|
||||
enabled=self.is_fused_shared_expert_enabled,
|
||||
n_routed_experts=getattr(self.config, "num_experts", 0) or 0,
|
||||
n_shared_experts=1,
|
||||
ckpt_prefix="mlp.shared_expert",
|
||||
)
|
||||
mapper = self.hf_to_vllm_mapper | WeightsMapper(
|
||||
orig_to_new_substr={"hyper_connection_mixer.block_inject_weight": None}
|
||||
)
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(),
|
||||
)
|
||||
return loader.load_weights(weights, mapper=mapper)
|
||||
|
||||
|
||||
@support_torch_compile(
|
||||
dynamic_arg_dims={
|
||||
"input_ids": 0,
|
||||
"positions": -1,
|
||||
"intermediate_tensors": 0,
|
||||
"inputs_embeds": 0,
|
||||
"hidden_states": 0,
|
||||
}
|
||||
)
|
||||
class Qwen4ExpMTP(nn.Module, SupportsPP, Qwen4ExpMixtureOfExperts):
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||
"in_proj_qkvz": ["in_proj_qkv", "in_proj_z"],
|
||||
"in_proj_ba": ["in_proj_b", "in_proj_a"],
|
||||
"input_mix_weight_down_block_inject": [
|
||||
"input_mix_weight_down",
|
||||
"block_inject_weight",
|
||||
"_input_mix_padding",
|
||||
],
|
||||
}
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config
|
||||
self.vllm_config = vllm_config
|
||||
cache_config = vllm_config.cache_config
|
||||
if cache_config.mamba_cache_mode == "all":
|
||||
raise NotImplementedError(
|
||||
"Qwen4ExpMTP currently does not support 'all' prefix caching, "
|
||||
"please use '--mamba-cache-mode=align' instead"
|
||||
)
|
||||
|
||||
self.quant_config = vllm_config.quant_config
|
||||
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.model = Qwen4ExpMultiTokenPredictor(
|
||||
vllm_config=vllm_config,
|
||||
prefix=maybe_prefix(prefix, "mtp"),
|
||||
)
|
||||
|
||||
if get_pp_group().is_last_rank:
|
||||
if config.tie_word_embeddings:
|
||||
self.lm_head = self.model.embed_tokens
|
||||
else:
|
||||
self.lm_head = ParallelLMHead(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
else:
|
||||
self.lm_head = PPMissingLayer()
|
||||
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size)
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.model.make_empty_intermediate_tensors
|
||||
)
|
||||
self.set_moe_parameters(self.model.layers)
|
||||
enable_qwen4_exp_low_latency_gemm(self, vllm_config.model_config.dtype)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor | None = None,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor] | IntermediateTensors:
|
||||
return self.model(
|
||||
input_ids,
|
||||
positions,
|
||||
hidden_states,
|
||||
intermediate_tensors,
|
||||
inputs_embeds,
|
||||
spec_step_idx=spec_step_idx,
|
||||
)
|
||||
|
||||
def compute_logits(
|
||||
self, hidden_states: torch.Tensor, spec_step_idx: int = 0
|
||||
) -> torch.Tensor | None:
|
||||
return self.logits_processor(self.lm_head, hidden_states)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
def remap_weight_names():
|
||||
for name, weight in weights:
|
||||
remapped_name = _remap_mtp_weight_name(name)
|
||||
if remapped_name is not None:
|
||||
yield remapped_name, weight
|
||||
|
||||
mapper = WeightsMapper(
|
||||
orig_to_new_substr={"hyper_connection_mixer.block_inject_weight": None}
|
||||
)
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(),
|
||||
)
|
||||
return loader.load_weights(remap_weight_names(), mapper=mapper)
|
||||
|
||||
|
||||
__all__ = ["Qwen4ExpMTP", "Qwen4ExpMultiTokenPredictor"]
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user