[Model] Support Qwen3.8-Flash-Next (#53896)

Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com>
Signed-off-by: andyluo7 <andy.luo@amd.com>
Signed-off-by: lishunyang12 <lishunyang03@gmail.com>
Signed-off-by: Harry Huang <huanghaoyan.hhy@alibaba-inc.com>
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: KungYork <firelikest@163.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: andyluo7 <43718156+andyluo7@users.noreply.github.com>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: andyluo7 <andy.luo@amd.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: lishunyang12 <lishunyang03@gmail.com>
Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Harry Huang
2026-08-31 05:57:56 +00:00
committed by GitHub
co-authored by zjy0516 KungYork Thien Tran Yifan Qiao andyluo7 Isotr0py andyluo7 OpenAI Codex lishunyang12 Vadim Gimpelson Isotr0py Lucas Wilkinson Claude Opus 5 OpenAI Codex
parent 687db59744
commit e126687a9a
124 changed files with 19424 additions and 450 deletions
+36
View File
@@ -331,6 +331,42 @@ steps:
- uv pip install --system 'gpt-oss[eval]==0.0.5'
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-b200.txt
- label: ":nvidia: (B200) Qwen3.8-Flash-Next-FP8 Accuracy Eval"
key: accuracy-eval-qwen3-8-flash-next-fp8-b200
timeout_in_minutes: 60
device: b200-k8s
optional: true
num_devices: 4
source_file_dependencies:
- csrc/libtorch_stable/gdn/
- tests/evals/qwen4_exp/
- vllm/model_executor/layers/mamba/
- vllm/models/qwen4_exp/
- vllm/transformers_utils/configs/qwen4_exp.py
- vllm/v1/attention/backends/short_conv_attn.py
- vllm/v1/spec_decode/qwen4_exp.py
commands:
- uv pip install --system 'evalscope==1.10.0'
- pytest -s -v evals/qwen4_exp/test_accuracy.py --config-list-file=configs/models-b200.txt
- label: ":nvidia: (H200) Qwen3.8-Flash-Next-FP8 Accuracy Eval"
key: accuracy-eval-qwen3-8-flash-next-fp8-h200
timeout_in_minutes: 60
device: h200
optional: true
num_devices: 4
source_file_dependencies:
- csrc/libtorch_stable/gdn/
- tests/evals/qwen4_exp/
- vllm/model_executor/layers/mamba/
- vllm/models/qwen4_exp/
- vllm/transformers_utils/configs/qwen4_exp.py
- vllm/v1/attention/backends/short_conv_attn.py
- vllm/v1/spec_decode/qwen4_exp.py
commands:
- uv pip install --system 'evalscope==1.10.0'
- pytest -s -v evals/qwen4_exp/test_accuracy.py --config-list-file=configs/models-h200.txt
- label: ":nvidia: (DGX) Spark GPQA Eval (GPT-OSS)"
key: gpqa-eval-gpt-oss-spark
timeout_in_minutes: 35
+37
View File
@@ -86,6 +86,43 @@ steps:
# The native NVIDIA Kimi K3 kernels require the SM100 family.
- pytest -v -s models/kimi_k3 kernels/attention/test_kimi_k3_mla_fused_epilogue.py kernels/test_bf16_skinny_gemm.py
- label: ":nvidia: (H200) Qwen4 Exp"
key: qwen4-exp-unit-tests
timeout_in_minutes: 40
device: h200_35gb
source_file_dependencies:
- vllm/models/qwen4_exp/
- vllm/transformers_utils/configs/qwen4_exp.py
- vllm/config/compilation.py
- vllm/v1/attention/backends/short_conv_attn.py
- tests/models/qwen4_exp/
commands:
- pytest -v -s models/qwen4_exp/test_hc_ops.py models/qwen4_exp/test_qsa_pre_indexer.py models/qwen4_exp/test_qsa_reference.py
mirror:
amd:
label: ":amd: (MI300) Qwen4 Exp"
dind: false
device: mi300_1
depends_on:
- image-build-amd
commands:
- pytest -v -s models/qwen4_exp/test_qsa_amd.py
- label: ":computer: (CPU) Qwen4 Exp"
key: qwen4-exp-unit-tests-cpu
depends_on:
- image-build-cpu
timeout_in_minutes: 20
device: cpu-small
source_file_dependencies:
- vllm/models/qwen4_exp/
- vllm/transformers_utils/configs/qwen4_exp.py
- vllm/config/compilation.py
- vllm/v1/attention/backends/short_conv_attn.py
- tests/models/qwen4_exp/
commands:
- pytest -v -s models/qwen4_exp/test_config.py models/qwen4_exp/test_ple.py
- label: ":computer: (CPU) Basic Models Other"
key: basic-models-test-other-cpu
depends_on:
+1
View File
@@ -11,6 +11,7 @@ steps:
- vllm/distributed/
- vllm/inputs/
- vllm/model_executor/
- vllm/models/qwen4_exp/
- vllm/platforms/
- vllm/sampling_params.py
- vllm/transformers_utils/
@@ -4,6 +4,7 @@
*/
#include <cstdint>
#include <string>
#include <cuda_bf16.h>
#include <cuda_fp16.h>
#include <cuda_runtime.h>
@@ -146,7 +147,7 @@ __device__ __forceinline__ Sum2 warp_reduce_sum_pair(float x, float y) {
return {x, y};
}
template <typename StateT, int ValueHeadsPerKeyHead>
template <typename StateT, int ValueHeadsPerKeyHead, bool SigmoidGate>
__global__ __launch_bounds__(kThreads, 2) void gdn_decode_post_conv_mtp_kernel(
const __nv_bfloat16* __restrict__ mixed_qkv,
const __nv_bfloat16* __restrict__ a, const __nv_bfloat16* __restrict__ b,
@@ -354,9 +355,11 @@ __global__ __launch_bounds__(kThreads, 2) void gdn_decode_post_conv_mtp_kernel(
#pragma unroll
for (int i = 0; i < 4; ++i) {
const int value = lane + i * 32;
const float gate = silu_fast(__bfloat162float(
const float gate_input = __bfloat162float(
output_gate[static_cast<int64_t>(token) * strides.gate_row +
value_head * kDimV + value]));
value_head * kDimV + value]);
const float gate =
SigmoidGate ? sigmoid_fast(gate_input) : silu_fast(gate_input);
const float weight =
norm_weight_is_bf16
? __bfloat162float(
@@ -370,7 +373,7 @@ __global__ __launch_bounds__(kThreads, 2) void gdn_decode_post_conv_mtp_kernel(
}
}
template <typename StateT, int ValueHeadsPerKeyHead>
template <typename StateT, int ValueHeadsPerKeyHead, bool SigmoidGate>
void launch_gdn_decode_post_conv_mtp(
torch::stable::Tensor const& mixed_qkv, torch::stable::Tensor const& a_log,
torch::stable::Tensor const& dt_bias,
@@ -395,7 +398,7 @@ void launch_gdn_decode_post_conv_mtp(
get_current_cuda_stream(mixed_qkv.get_device_index());
const int num_requests = static_cast<int>(state_indices.size(0));
const dim3 grid(num_requests, num_value_heads);
gdn_decode_post_conv_mtp_kernel<StateT, ValueHeadsPerKeyHead>
gdn_decode_post_conv_mtp_kernel<StateT, ValueHeadsPerKeyHead, SigmoidGate>
<<<grid, kThreads, 0, stream>>>(
static_cast<const __nv_bfloat16*>(mixed_qkv.data_ptr()), a, b,
static_cast<const float*>(a_log.data_ptr()), dt_bias.data_ptr(),
@@ -425,7 +428,7 @@ void fused_gdn_decode_post_conv_mtp(
torch::stable::Tensor const& num_accepted_tokens,
torch::stable::Tensor& state, torch::stable::Tensor const& output_gate,
torch::stable::Tensor const& norm_weight, torch::stable::Tensor& out,
double scale, double norm_eps) {
double scale, double norm_eps, const std::string& output_gate_activation) {
using torch::headeronly::ScalarType;
STD_TORCH_CHECK(
@@ -466,6 +469,9 @@ void fused_gdn_decode_post_conv_mtp(
"norm_weight must be a CUDA float32 or bfloat16 tensor");
STD_TORCH_CHECK(out.is_cuda() && out.scalar_type() == ScalarType::BFloat16,
"out must be a CUDA bfloat16 tensor");
STD_TORCH_CHECK(
output_gate_activation == "silu" || output_gate_activation == "sigmoid",
"output_gate_activation must be 'silu' or 'sigmoid'");
STD_TORCH_CHECK(mixed_qkv.dim() == 2,
"mixed_qkv must have shape [L, 2 * H * 128 + HV * 128]");
@@ -547,8 +553,9 @@ void fused_gdn_decode_post_conv_mtp(
const auto* b_ptr = static_cast<const __nv_bfloat16*>(b.data_ptr());
const auto* output_gate_ptr =
static_cast<const __nv_bfloat16*>(output_gate.data_ptr());
const auto launch = [&]<typename StateT, int ValueHeadsPerKeyHead>() {
launch_gdn_decode_post_conv_mtp<StateT, ValueHeadsPerKeyHead>(
const auto launch = [&]<typename StateT, int ValueHeadsPerKeyHead,
bool SigmoidGate>() {
launch_gdn_decode_post_conv_mtp<StateT, ValueHeadsPerKeyHead, SigmoidGate>(
mixed_qkv, a_log, dt_bias, state_indices, cu_seqlens,
num_accepted_tokens, state, norm_weight, out, a_ptr, b_ptr,
output_gate_ptr, num_key_heads, num_value_heads, scale, norm_eps,
@@ -556,9 +563,18 @@ void fused_gdn_decode_post_conv_mtp(
};
const auto dispatch_state_type = [&]<int ValueHeadsPerKeyHead>() {
if (state_scalar_type == ScalarType::Float) {
launch.template operator()<float, ValueHeadsPerKeyHead>();
if (output_gate_activation == "sigmoid") {
launch.template operator()<float, ValueHeadsPerKeyHead, true>();
} else {
launch.template operator()<float, ValueHeadsPerKeyHead, false>();
}
} else {
launch.template operator()<__nv_bfloat16, ValueHeadsPerKeyHead>();
if (output_gate_activation == "sigmoid") {
launch.template operator()<__nv_bfloat16, ValueHeadsPerKeyHead, true>();
} else {
launch
.template operator()<__nv_bfloat16, ValueHeadsPerKeyHead, false>();
}
}
};
switch (value_heads_per_key_head) {
+1 -1
View File
@@ -405,7 +405,7 @@ void fused_gdn_decode_post_conv_mtp(
torch::stable::Tensor const& num_accepted_tokens,
torch::stable::Tensor& state, torch::stable::Tensor const& output_gate,
torch::stable::Tensor const& norm_weight, torch::stable::Tensor& out,
double scale, double norm_eps);
double scale, double norm_eps, const std::string& output_gate_activation);
#endif
+2 -1
View File
@@ -535,7 +535,8 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
"Tensor mixed_qkv, Tensor a, Tensor b, Tensor A_log, Tensor dt_bias, "
"Tensor state_indices, Tensor cu_seqlens, Tensor num_accepted_tokens, "
"Tensor! state, Tensor output_gate, Tensor norm_weight, Tensor! out, "
"float scale, float norm_eps=1e-5) -> ()");
"float scale, float norm_eps=1e-5, "
"str output_gate_activation='silu') -> ()");
#endif
#ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES
+19
View File
@@ -7,6 +7,7 @@ from enum import Enum
import pytest
from vllm.config.cache import CacheConfig
from vllm.config.scheduler import SchedulerConfig
from vllm.config.utils import get_hash_factors, hash_factors, normalize_value
# Helpers
@@ -216,6 +217,24 @@ def test_cache_config_hash_ignores_kv_cache_sizing_knobs():
assert CacheConfig(gpu_memory_utilization=0.5).compute_hash() == base_hash
def test_scheduler_config_hash_includes_max_num_seqs():
"""Per-request workspace sizes must invalidate compiled graphs."""
base_hash = SchedulerConfig(
max_model_len=8192,
is_encoder_decoder=False,
max_num_batched_tokens=8192,
max_num_seqs=128,
).compute_hash()
larger_batch_hash = SchedulerConfig(
max_model_len=8192,
is_encoder_decoder=False,
max_num_batched_tokens=8192,
max_num_seqs=1024,
).compute_hash()
assert larger_batch_hash != base_hash
def test_cache_config_hash_ignores_prefix_cache_retention_interval():
base_hash = CacheConfig().compute_hash()
assert CacheConfig(prefix_cache_retention_interval=64).compute_hash() == base_hash
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for SpeculativeConfig.compose_draft_hf_overrides.
"""Tests for draft config overrides used by SpeculativeConfig.
Callable ``hf_overrides`` on the target model config (e.g. the
``dummy_hf_overrides`` shrink used by ``tests/models/test_initialization.py``)
@@ -12,10 +12,12 @@ when the target itself is shrunk — which is what kept spec-decode archs like
"""
import functools
from unittest.mock import MagicMock, patch
import pytest
from transformers import PretrainedConfig
from vllm.config.parallel import ParallelConfig
from vllm.config.speculative import SpeculativeConfig
@@ -134,3 +136,52 @@ def test_composed_override_is_picklable():
out = composed(_make_hf_config())
assert out.num_hidden_layers == 1
def _make_mtp_speculative_config(
override: bool | None,
checkpoint_value: bool,
) -> SpeculativeConfig:
draft_hf_config = _make_hf_config(
architectures=["Qwen4ExpMTP"],
model_type="qwen4_exp_mtp",
n_predict=1,
index_share_for_mtp_iteration=checkpoint_value,
)
draft_model_config = MagicMock(
model="draft",
hf_config=draft_hf_config,
architectures=draft_hf_config.architectures,
max_model_len=128,
)
target_model_config = MagicMock(
model="target",
max_model_len=128,
quantization=None,
hf_overrides={},
)
with patch("vllm.config.speculative.ModelConfig", return_value=draft_model_config):
return SpeculativeConfig(
model="draft",
method="mtp",
num_speculative_tokens=1,
index_share_for_mtp_iteration=override,
target_model_config=target_model_config,
target_parallel_config=ParallelConfig(),
)
@pytest.mark.cpu_test
@pytest.mark.parametrize(
("override", "checkpoint_value", "expected"),
[(None, True, True), (False, True, False), (True, False, True)],
)
def test_mtp_index_share_override(
override: bool | None, checkpoint_value: bool, expected: bool
):
speculative_config = _make_mtp_speculative_config(override, checkpoint_value)
assert (
speculative_config.draft_model_config.hf_config.index_share_for_mtp_iteration
is expected
)
@@ -24,6 +24,29 @@ for i, v in enumerate(test_sizes):
test_sizes[i] -= v % 8
@pytest.mark.parametrize(
("dtype", "expected"),
[
(torch.float32, True),
(torch.float16, True),
(torch.bfloat16, True),
(torch.int8, False),
(torch.float8_e4m3fn, False),
],
)
def test_custom_allreduce_filters_dtype(
dtype: torch.dtype,
expected: bool,
) -> None:
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
communicator.disabled = False
communicator.world_size = 2
communicator.max_size = 1024
communicator._ptr = 0
assert communicator.should_custom_ar(torch.empty(16, dtype=dtype)) is expected
@pytest.mark.parametrize(
("major", "local_multicast", "expected"),
[
+14
View File
@@ -0,0 +1,14 @@
# Qwen4Exp accuracy evaluation
This suite starts a Qwen3.8-Flash-Next-FP8 OpenAI-compatible server once and
uses EvalScope to evaluate GSM8K and AIME25.
```bash
# B200
pytest -s -v tests/evals/qwen4_exp/test_accuracy.py \
--config-list-file=configs/models-b200.txt
# H200
pytest -s -v tests/evals/qwen4_exp/test_accuracy.py \
--config-list-file=configs/models-h200.txt
```
+3
View File
@@ -0,0 +1,3 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,26 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
model_name: "Qwen/Qwen3.8-Flash-Next-FP8"
datasets:
gsm8k:
metric_threshold: 0.98
tolerance: 0.02
aime25:
metric_threshold: 0.90
tolerance: 0.05
eval_batch_size: 32
startup_max_wait_seconds: 1800
generation_config:
seed: 1236
do_sample: true
temperature: 0.7
top_p: 0.8
top_k: 20
repetition_penalty: 1.0
presence_penalty: 1.5
max_tokens: 32768
stream: true
extra_body:
chat_template_kwargs:
enable_thinking: true
server_args: "--tensor-parallel-size 4"
@@ -0,0 +1,2 @@
# B200 configuration for Qwen3.8-Flash-Next-FP8 accuracy evaluation
Qwen3.8-Flash-Next-FP8.yaml
@@ -0,0 +1,2 @@
# H200 configuration for Qwen3.8-Flash-Next-FP8 accuracy evaluation
Qwen3.8-Flash-Next-FP8.yaml
+42
View File
@@ -0,0 +1,42 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from pathlib import Path
def pytest_addoption(parser):
parser.addoption(
"--config-list-file",
required=True,
help="File containing Qwen4Exp evaluation config files",
)
def pytest_generate_tests(metafunc):
if "config_filename" not in metafunc.fixturenames:
return
config_list_path = Path(metafunc.config.getoption("--config-list-file"))
if not config_list_path.is_absolute():
test_relative_path = Path(__file__).parent / config_list_path
config_list_path = (
test_relative_path
if test_relative_path.exists()
else Path.cwd() / config_list_path
)
config_dir = config_list_path.parent
config_files = [
config_dir / line.strip()
for line in config_list_path.read_text(encoding="utf-8").splitlines()
if line.strip() and not line.lstrip().startswith("#")
]
missing = [path for path in config_files if not path.exists()]
if missing:
raise FileNotFoundError(f"Missing evaluation configs: {missing}")
metafunc.parametrize(
"config_filename",
config_files,
ids=[config_file.stem for config_file in config_files],
)
+68
View File
@@ -0,0 +1,68 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""GSM8K and AIME25 accuracy evaluation for Qwen3.8-Flash-Next-FP8."""
import shlex
from pathlib import Path
from typing import Any
import yaml
from tests.utils import RemoteOpenAIServer
def run_evalscope(
eval_config: dict[str, Any], base_url: str, work_dir: Path
) -> dict[str, float]:
from evalscope.run import run_task
reports = run_task(
{
"model": eval_config["model_name"],
"api_url": base_url,
"api_key": "EMPTY_TOKEN",
"datasets": list(eval_config["datasets"]),
"eval_batch_size": eval_config.get("eval_batch_size", 32),
"generation_config": eval_config["generation_config"],
"work_dir": str(work_dir),
"no_timestamp": True,
}
)
if not isinstance(reports, dict):
raise TypeError(f"Expected EvalScope reports to be a dict, got {type(reports)}")
return {dataset: float(report.score) for dataset, report in reports.items()}
def test_qwen4_exp_accuracy(config_filename: Path, tmp_path: Path):
eval_config = yaml.safe_load(config_filename.read_text(encoding="utf-8"))
server_args = shlex.split(eval_config.get("server_args", ""))
server_args.extend(["--trust-remote-code", "--disable-uvicorn-access-log"])
model_name = eval_config["model_name"]
print(f"Starting Qwen4Exp evaluation for model: {model_name}")
print(f"Datasets: {', '.join(eval_config['datasets'])}")
print(f"Server args: {' '.join(server_args)}")
with RemoteOpenAIServer(
model_name,
server_args,
env_dict=eval_config.get("env"),
max_wait_seconds=eval_config.get("startup_max_wait_seconds", 1800),
) as remote_server:
scores = run_evalscope(eval_config, remote_server.url_for("v1"), tmp_path)
for dataset, metric_config in eval_config["datasets"].items():
score = scores[dataset]
threshold = metric_config["metric_threshold"]
tolerance = metric_config.get("tolerance", 0.0)
minimum_score = threshold - tolerance
print(
f"{dataset}: measured={score:.4f}, expected={threshold:.4f}, "
f"tolerance={tolerance:.4f}"
)
assert score >= minimum_score, (
f"{dataset} score too low: {score:.4f} < {threshold:.4f} - "
f"{tolerance:.4f} = {minimum_score:.4f}"
)
+8 -3
View File
@@ -57,13 +57,13 @@ EPS = 1e-6
class _TestGatedNorm:
def __init__(self, weight: torch.Tensor) -> None:
def __init__(self, weight: torch.Tensor, activation: str) -> None:
self.weight = weight
self.bias = None
self.eps = EPS
self.group_size = None
self.norm_before_gate = True
self.activation = "silu"
self.activation = activation
def __call__(self, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor:
return rmsnorm_fn(
@@ -99,6 +99,7 @@ def _build_layer(
dt_bias: torch.Tensor,
conv_weight: torch.Tensor,
norm_weight: torch.Tensor,
output_gate_activation: str,
):
layer = types.SimpleNamespace(
prefix=PREFIX,
@@ -116,7 +117,7 @@ def _build_layer(
dt_bias=dt_bias,
conv1d=types.SimpleNamespace(weight=conv_weight, bias=None),
kv_cache=(conv_state, ssm_state),
norm=_TestGatedNorm(norm_weight),
norm=_TestGatedNorm(norm_weight, output_gate_activation),
layer_norm_epsilon=EPS,
gdn_decode_kernel="cuda",
)
@@ -250,12 +251,14 @@ def test_fused_forward_uses_packed_entrypoint() -> None:
pytest.param([128], [1], [-1], 0, id="pure-decode"),
],
)
@pytest.mark.parametrize("output_gate_activation", ["silu", "sigmoid"])
@torch.inference_mode()
def test_fused_model_path_matches_reference(
seq_lens: list[int],
query_lens: list[int],
draft_tokens: list[int],
expected_fused_calls: int,
output_gate_activation: str,
) -> None:
"""Fused MTP and its mixed/prefill/decode fallbacks match the reference."""
torch.manual_seed(1)
@@ -334,6 +337,7 @@ def test_fused_model_path_matches_reference(
dt_bias,
conv_weight,
norm_weight,
output_gate_activation,
)
reference_out = torch.zeros_like(output_gate)
with patch.object(
@@ -355,6 +359,7 @@ def test_fused_model_path_matches_reference(
dt_bias,
conv_weight,
norm_weight,
output_gate_activation,
)
context.no_compile_layers = {PREFIX: fused_layer}
fused_out = torch.zeros_like(output_gate)
@@ -366,7 +366,7 @@ def test_preprocess_fused_align_matches_scalar_bookkeeping(monkeypatch, token_bi
input_batch=scalar_case[1],
requests=scalar_case[2],
forward_context={},
mamba_state_copy_funcs=(),
mamba_state_copy_funcs={},
copy_bufs=scalar_copy_bufs,
)
@@ -384,7 +384,7 @@ def test_preprocess_fused_align_matches_scalar_bookkeeping(monkeypatch, token_bi
input_batch=fused_case[1],
requests=fused_case[2],
forward_context={},
mamba_state_copy_funcs=(),
mamba_state_copy_funcs={},
copy_bufs=fused_copy_bufs,
align_ctx=ctx,
)
+104
View File
@@ -351,3 +351,107 @@ def test_w8a8_block_fp8_deep_gemm_fused_moe(M, N, K, E, topk, seed, monkeypatch)
torch.accelerator.synchronize()
torch.testing.assert_close(out, ref_out, atol=0.035, rtol=0.035)
def _dequant_block_fp8(
w_fp8: torch.Tensor, w_s: torch.Tensor, block_shape: list[int]
) -> torch.Tensor:
"""Dequantize a [N, K] fp8 weight with [Nb, Kb] block scales to fp32."""
n, k = w_fp8.shape
bn, bk = block_shape
s = w_s.repeat_interleave(bn, dim=0).repeat_interleave(bk, dim=1)
return w_fp8.to(torch.float32) * s[:n, :k]
@pytest.mark.parametrize("tp_rank", [0, 1, 2, 3])
@torch.inference_mode()
def test_w8a8_block_fp8_fused_moe_refined_block_scales(tp_rank, workspace_init):
"""TP-misaligned blockwise FP8 MoE (e.g. Qwen4Exp, intermediate
640 at TP=4 -> 160 per rank, not divisible by the checkpoint's 128 block).
The per-rank scale grid is refined from 128 to 32 (lossless: each 32-block
lies inside one global 128-block) so the Triton kernel can consume exact
per-shard scales. Simulates one TP rank's shard and checks:
1. the refined+sharded scales dequantize exactly like the global scales;
2. the Triton fused MoE kernel with block_shape=[32, 32] matches the
native-torch blockwise reference on the same shard.
"""
torch.manual_seed(0)
dtype = torch.bfloat16
M, E, topk = 8, 4, 2
hidden = 256
inter_full = 640
tp_size = 4
n_shard = inter_full // tp_size # 160
ckpt_block = [128, 128]
refined_block = [32, 32]
factor = ckpt_block[0] // refined_block[0] # 4
a = torch.randn((M, hidden), dtype=dtype) / 10
score = torch.randn((M, E), dtype=dtype)
(_, w1_full, w1_s_full, _), (_, w2_full, w2_s_full, _) = make_test_weights(
E,
inter_full,
hidden,
dtype,
torch.float8_e4m3fn,
per_out_ch_quant=False,
block_shape=ckpt_block,
)
assert w1_s_full is not None
assert w2_s_full is not None
# TP shard the weights (exact slicing, no scale involvement).
lo, hi = tp_rank * n_shard, (tp_rank + 1) * n_shard
w1 = torch.cat(
[w1_full[:, lo:hi], w1_full[:, inter_full + lo : inter_full + hi]], dim=1
)
w2 = w2_full[:, :, lo:hi]
# Refine the global 128-block scales to 32 blocks, then take the shard's
# slice -- mirrors the Fp8MoEMethod/RoutedExperts loading path.
w1_s32 = w1_s_full.repeat_interleave(factor, dim=-2).repeat_interleave(
factor, dim=-1
)
nb = n_shard // refined_block[0] # 5 local 32-blocks per projection
gate_hi = w1_s32.shape[1] // 2
w1_s = torch.cat(
[
w1_s32[:, tp_rank * nb : (tp_rank + 1) * nb],
w1_s32[:, gate_hi + tp_rank * nb : gate_hi + (tp_rank + 1) * nb],
],
dim=1,
)
w2_s32 = w2_s_full.repeat_interleave(factor, dim=-2).repeat_interleave(
factor, dim=-1
)
w2_s = w2_s32[:, :, tp_rank * nb : (tp_rank + 1) * nb]
# The refined per-shard scales must reproduce the global-scale dequant
# exactly (dequant full-width, then slice to the shard).
for e in range(E):
d32 = _dequant_block_fp8(w1[e, :n_shard], w1_s[e, :nb], refined_block)
d128 = _dequant_block_fp8(w1_full[e], w1_s_full[e], ckpt_block)[lo:hi]
assert torch.equal(d32, d128)
d32 = _dequant_block_fp8(w2[e], w2_s[e], refined_block)
d128 = _dequant_block_fp8(w2_full[e], w2_s_full[e], ckpt_block)[:, lo:hi]
assert torch.equal(d32, d128)
quant_config = fp8_w8a8_moe_quant_config(
w1_scale=w1_s,
w2_scale=w2_s,
block_shape=refined_block,
)
topk_weights, topk_ids, _ = fused_topk(a, score.float(), topk, False)
with set_current_vllm_config(vllm_config):
ref_out = torch_w8a8_block_fp8_moe(
a, w1, w2, w1_s, w2_s, topk_weights, topk_ids, refined_block
)
out = fused_experts(
a, w1, w2, topk_weights, topk_ids, quant_config=quant_config
)
torch.testing.assert_close(out, ref_out, atol=0.035, rtol=0.035)
+10 -6
View File
@@ -28,8 +28,9 @@ from vllm.model_executor.layers.fused_moe.config import (
from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import (
TrtLlmBf16ExpertsModular,
)
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
convert_moe_weights_to_flashinfer_trtllm_block_layout,
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
UnquantizedMoeBackend,
convert_to_unquantized_kernel_format,
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
@@ -46,9 +47,8 @@ if pytest and (
)
# (m, n, k) = (tokens, intermediate_size_per_partition, hidden_dim).
# Covers larger NvFP4-like shapes while keeping BF16's FlashInfer TRTLLM
# intermediate-size multiple-of-128 requirement.
MNK_FACTORS = [
(2, 160, 2560),
(2, 1024, 1024),
(64, 2048, 1536),
(64, 1024, 4096),
@@ -97,11 +97,15 @@ def test_trtllm_bf16_moe_modular_no_graph(
max_num_tokens=next_power_of_2(m),
)
trtllm_w1, trtllm_w2 = convert_moe_weights_to_flashinfer_trtllm_block_layout(
{},
trtllm_w1, trtllm_w2 = convert_to_unquantized_kernel_format(
UnquantizedMoeBackend.FLASHINFER_TRTLLM,
moe_config,
w1,
w2,
)
expected_n = (n + 127) // 128 * 128
assert moe_config.intermediate_size_per_partition == expected_n
assert trtllm_w2.numel() == e * k * expected_n
trtllm_experts = mk.FusedMoEKernel(
maybe_make_prepare_finalize(
+4 -1
View File
@@ -267,6 +267,7 @@ def test_fused_post_conv_l0():
),
],
)
@pytest.mark.parametrize("output_gate_activation", ["silu", "sigmoid"])
@torch.inference_mode()
def test_fused_gdn_decode_post_conv_mtp_head_ratios(
head_ratio: int,
@@ -274,6 +275,7 @@ def test_fused_gdn_decode_post_conv_mtp_head_ratios(
query_lengths: tuple[int, ...],
state_dtype: torch.dtype,
norm_dtype: torch.dtype,
output_gate_activation: str,
) -> None:
if torch.cuda.get_device_capability() < (8, 0):
pytest.skip("fused GDN decode MTP requires compute capability 8.0+")
@@ -357,7 +359,7 @@ def test_fused_gdn_decode_post_conv_mtp_head_ratios(
z=output_gate,
eps=eps,
norm_before_gate=True,
activation="silu",
activation=output_gate_activation,
)
actual = ops.fused_gdn_decode_post_conv_mtp(
mixed_qkv=mixed_qkv,
@@ -374,6 +376,7 @@ def test_fused_gdn_decode_post_conv_mtp_head_ratios(
out=torch.empty_like(output_gate),
scale=scale,
norm_eps=eps,
output_gate_activation=output_gate_activation,
)
output_error = (actual.float() - expected.float()).norm()
@@ -18,8 +18,12 @@ from vllm.model_executor.layers.fused_moe.layer import determine_expert_counts
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
from vllm.model_executor.layers.quantization.utils.config_utils import (
get_quark_ocp_mx_group_size,
is_shared_expert_quant_fse_compatible,
)
from vllm.model_executor.models.qwen3_next import (
_should_replicate_misaligned_shared_expert,
)
from vllm.model_executor.models.utils import PPMissingLayer
from vllm.models.deepseek_v4 import quant_config as deepseek_v4_quant_config
from vllm.models.minimax_m3.amd import model as minimax_m3_model
@@ -605,6 +609,82 @@ def test_quark_shared_expert_fse_compatibility(
)
def test_quark_ocp_mx_group_size_for_layer() -> None:
quant_config = QuarkConfig({**_QUARK_FSE_CONFIG, "exclude": []})
assert (
get_quark_ocp_mx_group_size(
quant_config,
"model.layers.0.mlp.shared_expert.down_proj",
)
== 32
)
quant_config.quant_config["exclude"] = ["model.layers.0.mlp.shared_expert_gate"]
assert (
get_quark_ocp_mx_group_size(
quant_config,
"model.layers.0.mlp.shared_expert.down_proj",
)
== 32
)
quant_config.quant_config["exclude"] = [
"model.layers.0.mlp.shared_expert.down_proj"
]
assert (
get_quark_ocp_mx_group_size(
quant_config,
"model.layers.0.mlp.shared_expert.down_proj",
)
is None
)
@pytest.mark.parametrize("tp_size", [1, 2, 4])
def test_quark_ocp_mx_shared_expert_uses_supported_tp_shards(tp_size: int) -> None:
assert not _should_replicate_misaligned_shared_expert(
intermediate_size=640,
tp_size=tp_size,
group_size=32,
enable_expert_parallel=False,
is_sequence_parallel=False,
)
def test_quark_ocp_mx_shared_expert_rejects_plain_tp8() -> None:
with pytest.raises(
ValueError,
match="TP size 8 produces a partition of 80",
):
_should_replicate_misaligned_shared_expert(
intermediate_size=640,
tp_size=8,
group_size=32,
enable_expert_parallel=False,
is_sequence_parallel=False,
)
def test_quark_ocp_mx_shared_expert_is_replicated_for_tep8() -> None:
assert _should_replicate_misaligned_shared_expert(
intermediate_size=640,
tp_size=8,
group_size=32,
enable_expert_parallel=True,
is_sequence_parallel=False,
)
def test_quark_ocp_mx_shared_expert_is_replicated_when_not_tp_divisible() -> None:
assert _should_replicate_misaligned_shared_expert(
intermediate_size=672,
tp_size=8,
group_size=32,
enable_expert_parallel=True,
is_sequence_parallel=False,
)
def test_quark_shared_expert_fse_requires_matching_layer_quant_configs() -> None:
global_quant_config = {"weight": {"dtype": "fp4"}}
quant_config = QuarkConfig(
+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+200
View File
@@ -0,0 +1,200 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from unittest.mock import patch
import pytest
import torch
from vllm.config.speculative import SpeculativeConfig
from vllm.model_executor.models.config import (
Qwen3_5ForConditionalGenerationConfig,
Qwen4ExpForConditionalGenerationConfig,
)
from vllm.models.qwen4_exp.config import (
Qwen4ExpConfig,
Qwen4ExpTextConfig,
)
from vllm.models.qwen4_exp.nvidia.model_state import Qwen4ExpModelState
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
def _text_config(**kwargs) -> Qwen4ExpTextConfig:
values = {
"vocab_size": 64,
"hidden_size": 16,
"intermediate_size": 32,
"num_hidden_layers": 2,
"num_attention_heads": 2,
"num_key_value_heads": 1,
"head_dim": 8,
"layer_types": ["linear_attention", "full_attention"],
"linear_num_key_heads": 2,
"linear_num_value_heads": 2,
"linear_key_head_dim": 8,
"linear_value_head_dim": 8,
"num_experts": 0,
"hc_count": 2,
"hc_lowrank": 4,
"ple_layer_ids": [1],
"mtp_num_hidden_layers": 1,
"mtp": {"hybrid": True},
}
values.update(kwargs)
return Qwen4ExpTextConfig(**values)
def test_qwen4_exp_mtp_returns_sample_and_multi_streams() -> None:
from vllm.models.qwen4_exp.nvidia.mtp import (
Qwen4ExpMultiTokenPredictor,
)
model = object.__new__(Qwen4ExpMultiTokenPredictor)
torch.nn.Module.__init__(model)
model.hc_count = 2
model.hidden_size = 4
model.num_mtp_layers = 1
model.layers = [
lambda **kwargs: (
kwargs["hidden_states"],
kwargs["hidden_states"],
torch.zeros(kwargs["hidden_states"].shape[0], 2),
),
]
model.hyper_connection_mixer = SimpleNamespace(
combine_and_mix=lambda hidden_states, block_output, injection: (
hidden_states,
hidden_states.unflatten(-1, (2, 4)).mean(-2),
None,
),
)
multi_hidden = torch.arange(16, dtype=torch.float32).reshape(2, 8)
pp_group = SimpleNamespace(is_first_rank=False, is_last_rank=True)
with patch(
"vllm.models.qwen4_exp.nvidia.mtp.get_pp_group",
return_value=pp_group,
):
sample_hidden, returned_multi_hidden = model.forward(
input_ids=None,
positions=torch.arange(2),
intermediate_tensors={"hidden_states": multi_hidden},
)
torch.testing.assert_close(
sample_hidden,
multi_hidden.unflatten(-1, (2, 4)).mean(dim=-2),
)
assert returned_multi_hidden is multi_hidden
@pytest.mark.parametrize("wrapped_config", [False, True])
def test_qwen4_exp_mtp_override_sets_draft_config(
wrapped_config: bool,
) -> None:
text_config = _text_config(
architectures=["Qwen4ExpForCausalLM"],
index_share_for_mtp_iteration=True,
)
config = (
Qwen4ExpConfig(
architectures=["Qwen4ExpForConditionalGeneration"],
text_config=text_config,
)
if wrapped_config
else text_config
)
draft_config = SpeculativeConfig.hf_config_override(config)
assert draft_config.index_share_for_mtp_iteration is True
assert draft_config.model_type == "qwen4_exp_mtp"
assert draft_config.architectures == ["Qwen4ExpMTP"]
assert draft_config.hc_mult == 2
assert draft_config.n_predict == 1
@pytest.mark.parametrize("ple_layer_ids", [[1], []])
def test_qwen4_exp_rejects_pipeline_parallel_only_with_ple(ple_layer_ids) -> None:
"""PLE needs raw input_ids, which non-first pipeline ranks never see. The
rest of the architecture is PP-capable, so the refusal must be conditional
-- and must land before the engine spends time loading weights."""
vllm_config = SimpleNamespace(
model_config=SimpleNamespace(
hf_text_config=_text_config(ple_layer_ids=ple_layer_ids),
multimodal_config=None,
),
parallel_config=SimpleNamespace(
pipeline_parallel_size=2, enable_dbo=False, ubatch_size=1
),
speculative_config=None,
)
with patch.object(
Qwen3_5ForConditionalGenerationConfig, "verify_and_update_config"
):
if ple_layer_ids:
with pytest.raises(NotImplementedError, match="pipeline_parallel_size=1"):
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(
vllm_config
)
else:
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(vllm_config)
def test_qwen4_exp_model_state_prepares_ngram_context() -> None:
model_state = object.__new__(Qwen4ExpModelState)
model_state.uses_ngram_embedding = True
model_state.ngram_context_len = 3
model_state.ngram_eos_token_id = 99
model_state.ngram_context = torch.empty((4, 3), dtype=torch.int32)
model_state.ngram_context_offsets = torch.arange(-3, 0, dtype=torch.int64)
model_state.ple_query_start_loc = torch.empty(5, dtype=torch.int32)
input_batch = SimpleNamespace(
num_reqs=2,
num_reqs_after_padding=3,
idx_mapping=torch.tensor([1, 0]),
query_start_loc=torch.tensor([0, 2, 3, 3], dtype=torch.int32),
)
req_states = SimpleNamespace(
num_computed_tokens=SimpleNamespace(gpu=torch.tensor([3, 1])),
all_token_ids=SimpleNamespace(
gpu=torch.tensor([[1, 2, 3, 4], [20, 21, 22, 23]], dtype=torch.int32)
),
)
with patch.object(MambaHybridModelState, "prepare_inputs", return_value={}):
model_inputs = model_state.prepare_inputs(input_batch, req_states)
torch.testing.assert_close(
model_inputs["query_start_loc"],
torch.tensor([0, 2, 3, 3], dtype=torch.int32),
)
torch.testing.assert_close(
model_inputs["ngram_context"],
torch.tensor([[99, 99, 20], [1, 2, 3], [99, 99, 99]], dtype=torch.int32),
)
def test_qwen4_exp_model_state_prepares_stable_dummy_ngram_inputs() -> None:
model_state = object.__new__(Qwen4ExpModelState)
model_state.uses_ngram_embedding = True
model_state.ngram_eos_token_id = 99
model_state.ngram_context = torch.empty((4, 3), dtype=torch.int32)
model_state.ple_query_start_loc = torch.empty(5, dtype=torch.int32)
with patch.object(MambaHybridModelState, "prepare_dummy_inputs", return_value={}):
first = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=8)
query_start_loc_ptr = first["query_start_loc"].data_ptr()
ngram_context_ptr = first["ngram_context"].data_ptr()
second = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=8)
torch.testing.assert_close(
second["query_start_loc"], torch.tensor([0, 2, 5, 8], dtype=torch.int32)
)
torch.testing.assert_close(
second["ngram_context"], torch.full((3, 3), 99, dtype=torch.int32)
)
assert second["query_start_loc"].data_ptr() == query_start_loc_ptr
assert second["ngram_context"].data_ptr() == ngram_context_ptr
+94
View File
@@ -0,0 +1,94 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.models.qwen4_exp.nvidia.ops.hc import (
grouped_gemma_rmsnorm,
hc_combine,
hc_combine_norm,
hc_gate_mix,
)
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda() or not HAS_TRITON,
reason="HC kernels require CUDA and Triton",
)
HC = 4
HIDDEN_SIZE = 2560
HYPER_HIDDEN_SIZE = HC * HIDDEN_SIZE
EPS = 1e-6
def test_grouped_gemma_rmsnorm() -> None:
torch.manual_seed(0)
x = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
actual = grouped_gemma_rmsnorm(x, weight, EPS, HC)
grouped = x.float().unflatten(-1, (HC, HIDDEN_SIZE))
variance = grouped.square().mean(-1, keepdim=True)
expected = grouped * torch.rsqrt(variance + EPS)
expected = expected.flatten(-2) * (1.0 + weight.float())
torch.testing.assert_close(actual, expected.to(torch.bfloat16))
def test_hc_gate_mix() -> None:
torch.manual_seed(0)
x = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
gate = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
actual = hc_gate_mix(x, gate, HC)
expected = (
torch.sigmoid(gate.float().unflatten(-1, (HC, HIDDEN_SIZE)))
* x.float().unflatten(-1, (HC, HIDDEN_SIZE))
).mean(-2)
torch.testing.assert_close(actual, expected.to(torch.bfloat16))
def test_hc_combine() -> None:
torch.manual_seed(0)
block_output = torch.randn(2, HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
residual = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
injection = torch.randn(2, HC, dtype=torch.bfloat16, device="cuda")
actual = hc_combine(residual, block_output, injection, HC)
injection_weight = 2.0 * torch.sigmoid(injection.float() / HC)
expected = residual.float().unflatten(-1, (HC, HIDDEN_SIZE))
expected = expected + block_output.float().unsqueeze(
-2
) * injection_weight.unsqueeze(-1)
torch.testing.assert_close(actual, expected.flatten(-2).to(torch.bfloat16))
def test_hc_combine_norm() -> None:
torch.manual_seed(0)
block_output = torch.randn(2, HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
residual = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
injection = torch.randn(2, HC, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
actual, actual_norm = hc_combine_norm(
residual, block_output, injection, weight, EPS, HC
)
injection_weight = 2.0 * torch.sigmoid(injection.float() / HC)
expected = residual.float().unflatten(-1, (HC, HIDDEN_SIZE))
expected = expected + block_output.float().unsqueeze(
-2
) * injection_weight.unsqueeze(-1)
expected = expected.flatten(-2).to(residual.dtype)
grouped = expected.float().unflatten(-1, (HC, HIDDEN_SIZE))
variance = grouped.square().mean(-1, keepdim=True)
expected_norm = grouped * torch.rsqrt(variance + EPS)
expected_norm = expected_norm.flatten(-2) * (1.0 + weight.float())
torch.testing.assert_close(actual, expected)
torch.testing.assert_close(actual_norm, expected_norm.to(torch.bfloat16))
+401
View File
@@ -0,0 +1,401 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from functools import partial
from types import SimpleNamespace
import pytest
import torch
from torch import nn
from torch.nn import functional as F
import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module
import vllm.model_executor.parameter as parameter_module
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.models.qwen4_exp.common.ple import (
PLEShardOverlap,
compute_ple_shard_overlap,
copy_ple_embedding_shard_,
)
from vllm.models.qwen4_exp.nvidia import ple_layer as ple_layer_module
from vllm.models.qwen4_exp.nvidia.ple_layer import (
Qwen4ExpNGramEmbedding,
Qwen4ExpPLEFp8EmbeddingMethod,
Qwen4ExpPLELayer,
_get_ple_embedding_quant_method,
)
def _make_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding:
module = Qwen4ExpNGramEmbedding.__new__(Qwen4ExpNGramEmbedding)
nn.Module.__init__(module)
module.split_ngram_parts = 2
module.register_buffer("layer_multipliers", torch.zeros(1, dtype=torch.long))
module.register_buffer("ngram_heads_offsets", torch.zeros(1, dtype=torch.long))
module.register_buffer("ngram_heads_vocab_sizes", torch.zeros(1, dtype=torch.long))
module.ngram_embedding = SimpleNamespace(
org_vocab_size=8,
embedding_dim=2,
weight=nn.Parameter(torch.full((4, 2), -1.0)),
shard_indices=SimpleNamespace(
org_vocab_start_index=2,
org_vocab_end_index=6,
),
)
_set_test_embedding_weight_loader(module.ngram_embedding)
return module
def _set_test_embedding_weight_loader(embedding) -> None:
embedding.weight.weight_loader = partial(
copy_ple_embedding_shard_,
tp_start=embedding.shard_indices.org_vocab_start_index,
tp_end=embedding.shard_indices.org_vocab_end_index,
)
def _make_fp8_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding:
module = _make_ngram_embedding_for_load_test()
embedding = nn.Module()
embedding.org_vocab_size = 8
embedding.embedding_dim = 2
embedding.shard_indices = SimpleNamespace(
org_vocab_start_index=2,
org_vocab_end_index=6,
)
embedding.register_parameter(
"weight",
nn.Parameter(
torch.full((4, 2), -1.0).to(torch.float8_e4m3fn),
requires_grad=False,
),
)
embedding.register_parameter(
"weight_scale",
nn.Parameter(torch.zeros(1, dtype=torch.bfloat16), requires_grad=False),
)
_set_test_embedding_weight_loader(embedding)
module.ngram_embedding = embedding
return module
def test_ple_shard_overlap_and_copy() -> None:
overlap = compute_ple_shard_overlap(
checkpoint_start=2, checkpoint_rows=5, tp_start=4, tp_end=8
)
assert overlap == PLEShardOverlap(source_start=2, destination_start=0, row_count=3)
destination = torch.full((4, 2), -1.0)
loaded = torch.arange(10, dtype=torch.float64).reshape(5, 2)
copied = copy_ple_embedding_shard_(
destination,
loaded,
checkpoint_start=2,
tp_start=4,
tp_end=8,
)
assert copied == 3
torch.testing.assert_close(destination[:3], loaded[2:5].float())
torch.testing.assert_close(destination[3], torch.tensor([-1.0, -1.0]))
def test_ple_shard_copy_is_a_noop_without_overlap() -> None:
destination = torch.ones(4, 2)
copied = copy_ple_embedding_shard_(
destination,
torch.zeros(2, 2),
checkpoint_start=10,
tp_start=4,
tp_end=8,
)
assert copied == 0
assert torch.equal(destination, torch.ones_like(destination))
def test_ngram_embedding_loads_shards_and_ignores_legacy_token_lookup() -> None:
module = _make_ngram_embedding_for_load_test()
shard_0 = torch.arange(8, dtype=torch.float32).reshape(4, 2)
shard_1 = torch.arange(8, 16, dtype=torch.float32).reshape(4, 2)
loaded = module.load_weights(
[
("ngram_embedding.shard_0.weight", shard_0),
("ngram_embedding.shard_1.weight", shard_1),
("token_lookup", torch.tensor([2, 1, 0])),
]
)
assert loaded == {"ngram_embedding.weight"}
torch.testing.assert_close(
module.ngram_embedding.weight,
torch.cat((shard_0[2:4], shard_1[0:2])),
)
def test_ngram_embedding_loads_fp8_shards_and_global_scale() -> None:
module = _make_fp8_ngram_embedding_for_load_test()
shard_0 = torch.arange(8, dtype=torch.float32).reshape(4, 2).to(torch.float8_e4m3fn)
shard_1 = (
torch.arange(8, 16, dtype=torch.float32).reshape(4, 2).to(torch.float8_e4m3fn)
)
weight_scale = torch.tensor([0.25], dtype=torch.bfloat16)
loaded = module.load_weights(
[
("ngram_embedding.shard_0.weight", shard_0),
("ngram_embedding.shard_1.weight", shard_1),
("ngram_embedding.weight_scale", weight_scale),
]
)
assert loaded == {"ngram_embedding.weight", "ngram_embedding.weight_scale"}
assert module.ngram_embedding.weight.dtype == torch.float8_e4m3fn
assert torch.equal(
module.ngram_embedding.weight.float(),
torch.cat((shard_0[2:4], shard_1[0:2])).float(),
)
assert torch.equal(module.ngram_embedding.weight_scale, weight_scale)
def _make_fp8_embedding_layer(
monkeypatch: pytest.MonkeyPatch,
) -> embedding_module.VocabParallelEmbedding:
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
embedding_module, "get_tensor_model_parallel_world_size", lambda: 1
)
monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
parameter_module, "get_tensor_model_parallel_world_size", lambda: 1
)
method = Qwen4ExpPLEFp8EmbeddingMethod()
layer = embedding_module.VocabParallelEmbedding(
3,
2,
params_dtype=torch.bfloat16,
padding_size=1,
quant_method=method,
)
weight = torch.tensor([[1.0, 2.0], [4.0, 8.0], [16.0, 32.0]])
layer.weight.data.copy_(weight.to(torch.float8_e4m3fn))
layer.weight_scale.data.copy_(torch.tensor([0.25], dtype=torch.bfloat16))
return layer
def test_ple_fp8_embedding_dequantizes_in_ple_layer(monkeypatch) -> None:
layer = _make_fp8_embedding_layer(monkeypatch)
quantized_output = layer(torch.tensor([2, 0]))
ple_layer = Qwen4ExpPLELayer.__new__(Qwen4ExpPLELayer)
nn.Module.__init__(ple_layer)
ple_layer.ple_embedding = nn.Module()
ple_layer.ple_embedding.ngram_embedding = layer
output = ple_layer._dequantize_embeddings(
quantized_output,
torch.bfloat16,
)
assert layer.weight.dtype == torch.float8_e4m3fn
assert layer.weight_scale.dtype == torch.bfloat16
assert quantized_output.dtype == torch.float8_e4m3fn
assert output.dtype == torch.bfloat16
weight = torch.tensor([[1.0, 2.0], [4.0, 8.0], [16.0, 32.0]])
torch.testing.assert_close(output, (weight[[2, 0]] * 0.25).bfloat16())
def test_ple_fp8_embedding_uses_int8_for_tp_reduce(monkeypatch) -> None:
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
embedding_module, "get_tensor_model_parallel_world_size", lambda: 2
)
monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
parameter_module, "get_tensor_model_parallel_world_size", lambda: 2
)
monkeypatch.setattr(
embedding_module,
"get_masked_input_and_mask",
lambda *args: (
torch.tensor([0, 0]),
torch.tensor([False, True]),
),
)
reduced_dtypes = []
def all_reduce(tensor: torch.Tensor) -> torch.Tensor:
reduced_dtypes.append(tensor.dtype)
return tensor.clone()
monkeypatch.setattr(
embedding_module,
"tensor_model_parallel_all_reduce",
all_reduce,
)
layer = embedding_module.VocabParallelEmbedding(
4,
2,
params_dtype=torch.bfloat16,
padding_size=1,
quant_method=Qwen4ExpPLEFp8EmbeddingMethod(),
)
layer.weight.data.copy_(
torch.tensor([[1.0, 2.0], [4.0, 8.0]]).to(torch.float8_e4m3fn)
)
output = layer(torch.tensor([0, 2]))
assert reduced_dtypes == [torch.int8]
assert output.dtype == torch.float8_e4m3fn
torch.testing.assert_close(output[0].float(), layer.weight[0].float())
assert torch.count_nonzero(output[1].float()) == 0
def test_ple_fp8_embedding_respects_checkpoint_shard_exclusions() -> None:
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
quant_config = Fp8Config(
is_checkpoint_fp8_serialized=True,
ignored_layers=[],
weight_block_size=[128, 128],
)
assert isinstance(
_get_ple_embedding_quant_method(quant_config, prefix),
Qwen4ExpPLEFp8EmbeddingMethod,
)
quant_config.ignored_layers = [f"{prefix}.shard_0"]
assert _get_ple_embedding_quant_method(quant_config, prefix) is None
def test_ple_ngram_ids_custom_op_uses_current_request_layout(monkeypatch) -> None:
class RuntimeNGramEmbedding(nn.Module):
def compute_ngram_ids(
self,
input_ids: torch.Tensor,
query_start_loc: torch.Tensor,
ngram_context: torch.Tensor,
) -> torch.Tensor:
del input_ids, ngram_context
num_reqs = query_start_loc.numel() - 1
return torch.full((4, 2), num_reqs, dtype=torch.long)
layer = Qwen4ExpPLELayer.__new__(Qwen4ExpPLELayer)
nn.Module.__init__(layer)
layer.ple_embedding = RuntimeNGramEmbedding()
monkeypatch.setattr(
ple_layer_module,
"get_forward_context",
lambda: SimpleNamespace(no_compile_layers={"ple": layer}),
)
input_ids = torch.arange(4)
ngram_context = torch.zeros(2, 2, dtype=torch.long)
output = torch.empty(4, 2, dtype=torch.long)
ple_layer_module.qwen4_exp_compute_ple_ngram_ids(
input_ids,
torch.tensor([0, 4]),
ngram_context,
output,
"ple",
)
assert torch.equal(output, torch.ones_like(output))
ple_layer_module.qwen4_exp_compute_ple_ngram_ids(
input_ids,
torch.tensor([0, 2, 4]),
ngram_context,
output,
"ple",
)
assert torch.equal(output, torch.full_like(output, 2))
def test_dilated_ple_spec_state_rolls_back_before_next_forward() -> None:
module = Qwen4ExpPLELayer.__new__(Qwen4ExpPLELayer)
nn.Module.__init__(module)
module.conv_state_len = 6
module.short_conv_dilation = 2
conv_weights = torch.tensor([[0.25, -0.5, 0.75, 1.0]])
conv_state = torch.zeros(2, 1, 9)
conv_state[1] = torch.arange(1, 10, dtype=torch.float32).reshape(1, 9)
first_inputs = torch.tensor([[10.0], [20.0], [30.0], [40.0]])
initial_state = conv_state[1:].clone()
first_history = torch.cat(
(initial_state[..., : module.conv_state_len], first_inputs.T.unsqueeze(0)),
dim=-1,
)
graph_padded_inputs = F.pad(first_inputs, (0, 0, 0, 4))
first_output = module._short_conv_dilated_spec_batched(
graph_padded_inputs,
conv_state,
conv_weights,
torch.tensor([1, 0]),
torch.tensor([0, 4, 4]),
torch.tensor([1, 0]),
spec_query_len=4,
)
expected_first_output = F.silu(
F.conv1d(
first_history,
conv_weights.unsqueeze(1),
groups=1,
dilation=module.short_conv_dilation,
)
).transpose(1, 2)[0]
expected_first_state = first_history[..., 1:10]
torch.testing.assert_close(first_output[:4], expected_first_output)
assert torch.count_nonzero(first_output[4:]) == 0
assert torch.count_nonzero(conv_state[0]) == 0
torch.testing.assert_close(conv_state[1:], expected_first_state)
second_inputs = torch.tensor([[50.0], [60.0]])
rollback_state = expected_first_state[..., 1:7]
padded_second_inputs = F.pad(second_inputs.T.unsqueeze(0), (0, 2))
second_history = torch.cat((rollback_state, padded_second_inputs), dim=-1)
expected_second_state = expected_first_state.clone()
expected_second_state[..., :7] = second_history[..., 1:8]
second_output = module._short_conv_dilated_spec_batched(
second_inputs,
conv_state,
conv_weights,
torch.tensor([1]),
torch.tensor([0, 2]),
torch.tensor([2]),
spec_query_len=4,
)
expected_second_output = F.silu(
F.conv1d(
second_history,
conv_weights.unsqueeze(1),
groups=1,
dilation=module.short_conv_dilation,
)
).transpose(1, 2)[0, :2]
torch.testing.assert_close(second_output, expected_second_output)
torch.testing.assert_close(conv_state[1:], expected_second_state)
def test_ple_short_conv_uses_fallback_when_profile_metadata_is_omitted(
monkeypatch: pytest.MonkeyPatch,
) -> None:
module = Qwen4ExpPLELayer.__new__(Qwen4ExpPLELayer)
nn.Module.__init__(module)
module.prefix = "model.layers.1.ple"
inputs = torch.arange(6, dtype=torch.float32).reshape(3, 2)
expected = inputs + 1
monkeypatch.setattr(module, "_short_conv_fallback", lambda _: expected)
monkeypatch.setattr(
ple_layer_module,
"get_forward_context",
lambda: SimpleNamespace(attn_metadata={"model.layers.0.self_attn": object()}),
)
output = module._short_conv(inputs)
assert output is expected
+297
View File
@@ -0,0 +1,297 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from types import SimpleNamespace
from typing import Any
import pytest
import torch
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
from vllm.models.qwen4_exp.amd import (
model as _qwen4_exp_model, # noqa: F401
)
from vllm.models.qwen4_exp.amd import ple_layer as ple_layer_module
from vllm.models.qwen4_exp.amd.indexer_qsa import (
apply_qsa_rmsnorm,
apply_qsa_rope,
)
from vllm.models.qwen4_exp.amd.ops import qsa as qsa_ops
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(),
reason="AMD QSA requires ROCm",
)
requires_qsa_kernels = pytest.mark.skipif(
not HAS_TRITON,
reason="AMD QSA kernels require Triton",
)
def test_ple_ngram_embedding_custom_op_uses_resident_weight(
monkeypatch: pytest.MonkeyPatch,
) -> None:
layer_name = "model.layers.0.ple"
layer = ple_layer_module.Qwen4ExpPLELayer.__new__(ple_layer_module.Qwen4ExpPLELayer)
torch.nn.Module.__init__(layer)
layer.ple_embedding = torch.nn.Module()
layer.ple_embedding.ngram_embedding = torch.nn.Embedding(8, 3)
context = SimpleNamespace(no_compile_layers={layer_name: layer})
monkeypatch.setattr(ple_layer_module, "get_forward_context", lambda: context)
ngram_ids = torch.tensor([[0, 1], [2, 3]])
output = torch.empty(2, 6)
ple_layer_module.qwen4_exp_amd_ple_ngram_embedding(
ngram_ids,
output,
layer_name,
)
expected = layer.ple_embedding.ngram_embedding(ngram_ids).flatten(-2)
torch.testing.assert_close(output, expected)
def _qsa_sparse_paged_attention_reference(
q: torch.Tensor,
k_cache: torch.Tensor,
v_cache: torch.Tensor,
logical_indices: torch.Tensor,
block_table: torch.Tensor,
token_to_req: torch.Tensor,
softmax_scale: float,
) -> torch.Tensor:
output = torch.zeros_like(q)
repeats = q.shape[1] // k_cache.shape[2]
page_size = k_cache.shape[1]
for row in range(q.shape[0]):
logical = logical_indices[row]
logical = logical[logical >= 0].long()
if not logical.numel():
continue
request = token_to_req[row].long()
pages = block_table[request, logical // page_size].long()
offsets = logical % page_size
keys = k_cache[pages, offsets].repeat_interleave(repeats, dim=1)
values = v_cache[pages, offsets].repeat_interleave(repeats, dim=1)
scores = torch.einsum("hd,khd->hk", q[row].float(), keys.float())
probabilities = torch.softmax(scores * softmax_scale, dim=-1)
output[row] = torch.einsum("hk,khd->hd", probabilities, values.float()).to(
q.dtype
)
return output
def test_qsa_rope_uses_platform_dispatch() -> None:
tensor = torch.arange(16, dtype=torch.float32).reshape(2, 2, 4)
positions = torch.tensor([0, 1])
calls = []
def apply_rotary_emb(
rotary_input: torch.Tensor,
cos: torch.Tensor,
sin: torch.Tensor,
) -> torch.Tensor:
calls.append((rotary_input, cos, sin))
return rotary_input + 1
rotary_emb = SimpleNamespace(
rotary_dim=2,
apply_rotary_emb=apply_rotary_emb,
_match_cos_sin_cache_dtype=lambda _: torch.zeros(2, 4),
)
output = apply_qsa_rope(rotary_emb, positions, tensor)
assert len(calls) == 1
torch.testing.assert_close(output[..., :2], tensor[..., :2] + 1)
torch.testing.assert_close(output[..., 2:], tensor[..., 2:])
def test_qsa_rmsnorm_uses_portable_implementation(default_vllm_config) -> None:
norm = GemmaRMSNorm(4, eps=1e-6)
norm.weight.data.copy_(torch.tensor([0.1, -0.2, 0.3, -0.4]))
tensor = torch.arange(8, dtype=torch.float32).reshape(2, 4)
output = apply_qsa_rmsnorm(norm, tensor)
torch.testing.assert_close(output, norm.forward_native(tensor))
def test_qsa_selection_uses_portable_topk_on_rocm(
monkeypatch: pytest.MonkeyPatch,
) -> None:
rows = 2
token_topk = 8
compress_ratio = 4
block_topk = token_topk // compress_ratio
blocks = torch.empty((rows, block_topk), dtype=torch.int32)
visible_blocks = torch.tensor([4, 6], dtype=torch.int32)
logits = torch.empty((rows, 8), dtype=torch.float32)
selection_output = torch.empty(
(rows, token_topk + compress_ratio - 1), dtype=torch.int32
)
topk_call: dict[str, Any] = {}
monkeypatch.setattr(
qsa_ops,
"qsa_mqa_paged",
lambda *args, **kwargs: (logits, visible_blocks),
)
monkeypatch.setattr(qsa_ops.current_platform, "is_cuda", lambda: False)
def top_k_per_row_decode(
topk_logits,
next_n,
seq_lens,
raw_topk_indices,
num_rows,
stride0,
stride1,
topk_tokens,
) -> None:
topk_call.update(
logits=topk_logits,
next_n=next_n,
seq_lens=seq_lens,
raw_topk_indices=raw_topk_indices,
num_rows=num_rows,
strides=(stride0, stride1),
topk_tokens=topk_tokens,
)
raw_topk_indices.zero_()
def expand_qsa_block_indices(*args) -> None:
args[-1].fill_(-1)
monkeypatch.setattr(qsa_ops.ops, "top_k_per_row_decode", top_k_per_row_decode)
monkeypatch.setattr(
qsa_ops,
"expand_qsa_block_indices_cuda",
expand_qsa_block_indices,
)
output = qsa_ops.qsa_select_paged_tokens(
torch.empty((rows, 1, 1)),
torch.empty((1, 4, 1, 1)),
torch.empty((1, 2), dtype=torch.int32),
torch.zeros(rows, dtype=torch.int32),
torch.arange(rows, dtype=torch.int32),
torch.tensor([8], dtype=torch.int32),
token_topk,
compress_ratio,
selection_output,
)
assert output is selection_output
assert torch.all(output == -1)
assert topk_call["logits"] is logits
assert topk_call["next_n"] == 1
assert topk_call["seq_lens"] is visible_blocks
assert topk_call["raw_topk_indices"].shape == blocks.shape
assert topk_call["num_rows"] == rows
assert topk_call["strides"] == (logits.stride(0), logits.stride(1))
assert topk_call["topk_tokens"] == block_topk
@requires_qsa_kernels
@pytest.mark.parametrize(
("num_rows", "num_query_heads", "num_kv_heads", "page_size"),
[
pytest.param(1, 24, 2, 1792, id="tp1_split64"),
pytest.param(16, 12, 1, 1792, id="tp2_split32"),
pytest.param(32, 6, 1, 1024, id="tp4_split8"),
pytest.param(257, 6, 1, 1024, id="tp4_split4"),
pytest.param(513, 6, 1, 1024, id="tp4_split1"),
],
)
def test_qsa_sparse_paged_attention_matches_reference(
num_rows: int,
num_query_heads: int,
num_kv_heads: int,
page_size: int,
) -> None:
torch.manual_seed(2)
head_dim = 256
num_requests = 2
num_selected_pages = 64
num_pages_per_request = num_selected_pages + 1
num_cache_blocks = num_requests * num_pages_per_request
indexer_budget = 2048
indexer_compress_ratio = 4
selection_width = indexer_budget + indexer_compress_ratio - 1
q = torch.randn(
num_rows, num_query_heads, head_dim, device="cuda", dtype=torch.bfloat16
)
kv_cache = torch.randn(
num_cache_blocks,
page_size,
num_kv_heads,
2 * head_dim,
device="cuda",
dtype=torch.bfloat16,
)
k_cache, v_cache = kv_cache.split(head_dim, dim=-1)
block_table = (
torch.randperm(num_cache_blocks, device="cuda")
.reshape(num_requests, num_pages_per_request)
.to(torch.int32)
)
rows_per_request = math.ceil(num_rows / num_requests)
row_indices = torch.arange(num_rows, device="cuda", dtype=torch.int32)
token_to_req = row_indices // rows_per_request
request_row_counts = torch.tensor(
[rows_per_request, num_rows - rows_per_request],
device="cuda",
dtype=torch.int32,
)
context_length = num_pages_per_request * page_size - 1
block_topk = indexer_budget // indexer_compress_ratio
compressed_blocks_per_page = page_size // indexer_compress_ratio
selection = torch.arange(block_topk, device="cuda")
selected_pages = selection % num_selected_pages
selected_offsets = selection // num_selected_pages
row_shifts = 2 * row_indices.unsqueeze(1)
selected_offsets = (selected_offsets + row_shifts) % compressed_blocks_per_page
block_indices = (selected_pages * compressed_blocks_per_page + selected_offsets).to(
torch.int32
)
rows_within_request = row_indices % rows_per_request
query_positions = (
context_length - request_row_counts[token_to_req.long()] + rows_within_request
).to(torch.int64)
sequence_lengths = torch.full(
(num_requests,), context_length, device="cuda", dtype=torch.int32
)
logical_indices = qsa_ops.expand_qsa_block_indices_cuda(
block_indices,
query_positions,
sequence_lengths,
token_to_req,
indexer_compress_ratio,
indexer_budget,
)
assert logical_indices.shape == (num_rows, selection_width)
actual = qsa_ops.qsa_sparse_paged_attention(
q,
k_cache,
v_cache,
logical_indices,
block_table,
token_to_req,
)
expected = _qsa_sparse_paged_attention_reference(
q,
k_cache,
v_cache,
logical_indices,
block_table,
token_to_req,
q.shape[-1] ** -0.5,
)
torch.testing.assert_close(actual, expected, rtol=2e-2, atol=2e-2)
@@ -0,0 +1,297 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness tests for the fused QSA pre-indexer."""
import pytest
import torch
from vllm.models.qwen4_exp.common.qsa_cache import (
canonical_qsa_rope_positions,
circular_qsa_slot_mapping,
compressed_qsa_slot_mapping,
)
from vllm.models.qwen4_exp.nvidia.indexer_qsa import apply_qsa_rope
from vllm.models.qwen4_exp.nvidia.ops.qsa import (
qsa_compress_groups_with_ratio,
qsa_store_cache_rows,
)
from vllm.models.qwen4_exp.nvidia.ops.qsa_pre_indexer import (
qsa_pre_indexer,
)
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
requires_qsa_kernels = pytest.mark.skipif(
not current_platform.is_cuda() or not HAS_TRITON,
reason="QSA kernels require CUDA and Triton",
)
HQ, D = 4, 128
CR = 4
MROPE_SECTION = (11, 11, 10)
EPS = 1e-6
BLOCK_SIZE = 16
COMP_PAGE = BLOCK_SIZE // CR
ROPE_POS_OFFSET = D
RTOL = 1.6e-2
ATOL = 1e-2
MIXED_BATCH = ([260, 259, 138], [1, 1, 37], [8, 8, 8])
def _make_block_table(block_counts):
num_blocks = sum(block_counts)
table = torch.full((len(block_counts), max(block_counts)), -1, dtype=torch.int32)
physical_blocks = torch.randperm(num_blocks)
offset = 0
for request, count in enumerate(block_counts):
table[request, :count] = physical_blocks[offset : offset + count]
offset += count
return table, num_blocks
@requires_qsa_kernels
@pytest.mark.usefixtures("default_vllm_config")
@pytest.mark.parametrize(
"mrope,is_2d_positions,cache_rope_positions,state_size,seq_lens,query_lens,history_lens",
[
pytest.param(True, True, True, 4, *MIXED_BATCH, id="mrope"),
pytest.param(False, False, False, 4, *MIXED_BATCH, id="text"),
pytest.param(True, False, True, 8, *MIXED_BATCH, id="mrope-model-1d"),
pytest.param(True, True, False, 4, *MIXED_BATCH, id="mrope-no-position-cache"),
pytest.param(
True,
False,
False,
8,
*MIXED_BATCH,
id="mrope-model-1d-no-position-cache",
),
pytest.param(
False, False, True, 4, *MIXED_BATCH, id="text-with-position-cache"
),
pytest.param(True, True, True, 4, [37], [37], [0], id="fresh"),
pytest.param(True, True, True, 4, [4097], [4097], [0], id="tiled"),
],
)
def test_qsa_fused_pre_indexer_matches_unfused(
mrope,
is_2d_positions,
cache_rope_positions,
state_size,
seq_lens,
query_lens,
history_lens,
) -> None:
from flashinfer.norm import gemma_rmsnorm
from vllm.model_executor.layers.rotary_embedding import get_rope
device = "cuda"
rope_params = {
"partial_rotary_factor": 0.25,
"rope_theta": 10000000,
"rope_type": "default",
}
if mrope:
rope_params["mrope_interleaved"] = True
rope_params["mrope_section"] = list(MROPE_SECTION)
with torch.device(device):
rope = get_rope(
head_size=256,
max_position=32768,
rope_parameters=rope_params,
dtype=torch.bfloat16,
)
token_to_req = torch.cat(
[
torch.full((length,), request, dtype=torch.int32)
for request, length in enumerate(query_lens)
]
).to(device)
logical_positions = torch.cat(
[
torch.arange(seq_len - query_len, seq_len, dtype=torch.int64)
for seq_len, query_len in zip(seq_lens, query_lens)
]
).to(device)
num_tokens = logical_positions.numel()
query_start_loc = torch.tensor(
[0, *torch.tensor(query_lens).cumsum(0).tolist()], dtype=torch.int32
).to(device)
positions = (
torch.stack(
[
logical_positions,
logical_positions // 7 + 3,
logical_positions // 13 + 11,
]
)
if is_2d_positions
else logical_positions
)
position_rows = (
canonical_qsa_rope_positions(positions)
if cache_rope_positions
else logical_positions.view(-1, 1, 1).expand(-1, 1, 3)
)
compressed_block_counts = [
(seq_len // CR + COMP_PAGE - 1) // COMP_PAGE for seq_len in seq_lens
]
raw_block_table, num_raw_blocks = _make_block_table([1] * len(seq_lens))
compressed_block_table, num_compressed_blocks = _make_block_table(
compressed_block_counts
)
raw_block_table = raw_block_table.to(device)
raw_slots = circular_qsa_slot_mapping(
raw_block_table,
token_to_req,
logical_positions,
state_size,
query_start_loc,
)
compressed_slots = compressed_qsa_slot_mapping(
compressed_block_table.to(device),
token_to_req,
logical_positions,
COMP_PAGE,
CR,
)
group_counts = torch.tensor(
[
seq_len // CR - (seq_len - query_len) // CR
for seq_len, query_len in zip(seq_lens, query_lens)
],
dtype=torch.int32,
device=device,
)
k_work_counts = torch.maximum(group_counts, torch.ones_like(group_counts))
k_start_loc = torch.cat([k_work_counts.new_zeros(1), k_work_counts.cumsum(0)])
work_requests = torch.repeat_interleave(
torch.arange(len(query_lens), dtype=torch.int32, device=device),
k_work_counts,
)
local_work = torch.arange(
int(k_start_loc[-1]), dtype=torch.int32, device=device
) - torch.repeat_interleave(k_start_loc[:-1], k_work_counts)
max_k_work = (num_tokens + (CR - 1) * len(query_lens)) // CR
k_work_metadata = torch.full((max_k_work, 2), -1, dtype=torch.int32, device=device)
k_work_metadata[: work_requests.numel()] = torch.stack(
(work_requests, local_work), dim=1
)
raw_width = D + 12 if cache_rope_positions else D
# Match vLLM's padded-page cache layout: rows are contiguous, while physical
# blocks have a larger stride than their logical contents.
raw_page_elements = state_size * raw_width
fused_raw_storage = torch.zeros(
num_raw_blocks,
raw_page_elements + 16,
dtype=torch.bfloat16,
device=device,
)
fused_raw = torch.as_strided(
fused_raw_storage,
(num_raw_blocks, state_size, 1, raw_width),
(raw_page_elements + 16, raw_width, raw_width, 1),
)
for request, history_len in enumerate(history_lens):
history_end = seq_lens[request] - query_lens[request]
for position in range(history_end - history_len, history_end):
block = int(raw_block_table[request, 0])
row = fused_raw[block, position % state_size, 0]
row[:D] = torch.randn(D, dtype=torch.bfloat16, device=device)
if cache_rope_positions:
row[ROPE_POS_OFFSET:].view(torch.int64).copy_(
torch.tensor(
[position, position // 7 + 3, position // 13 + 11],
dtype=torch.int64,
device=device,
)
)
unfused_raw = fused_raw.clone()
compressed_page_elements = COMP_PAGE * D
fused_compressed_storage = torch.zeros(
num_compressed_blocks,
compressed_page_elements + 16,
dtype=torch.bfloat16,
device=device,
)
fused_compressed = torch.as_strided(
fused_compressed_storage,
(num_compressed_blocks, COMP_PAGE, 1, D),
(compressed_page_elements + 16, D, D, 1),
)
unfused_compressed = fused_compressed.clone()
projected_qk = torch.randn(
num_tokens, (HQ + 1) * D, dtype=torch.bfloat16, device=device
)
q_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
k_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
fused_query = torch.empty(num_tokens, HQ, D, dtype=torch.bfloat16, device=device)
qsa_pre_indexer(
projected_qk[:, : HQ * D],
projected_qk[:, HQ * D :],
positions,
rope.cos_sin_cache,
q_weight,
k_weight,
EPS,
fused_query,
fused_raw,
raw_slots,
raw_block_table,
query_start_loc,
logical_positions,
fused_compressed,
compressed_slots,
k_work_metadata,
compress_ratio=CR,
mrope_section=MROPE_SECTION if mrope else None,
rope_pos_offset=ROPE_POS_OFFSET if cache_rope_positions else None,
)
unfused_query = projected_qk[:, : HQ * D].reshape(num_tokens, HQ, D)
unfused_query = gemma_rmsnorm(
unfused_query.reshape(-1, D), q_weight, EPS
).reshape_as(unfused_query)
unfused_query = apply_qsa_rope(rope, positions, unfused_query)
raw_keys = unfused_raw[..., :D]
rope_positions = (
unfused_raw[..., ROPE_POS_OFFSET:].view(torch.int64)
if cache_rope_positions
else None
)
pooled, first_positions = qsa_compress_groups_with_ratio(
projected_qk[:, HQ * D :].reshape(-1, 1, D),
position_rows,
raw_keys,
raw_block_table,
token_to_req,
query_start_loc,
logical_positions,
compressed_slots,
CR,
rope_positions,
)
compressed_rows = gemma_rmsnorm(pooled.reshape(-1, D), k_weight, EPS).reshape(
-1, 1, D
)
group_positions = (
first_positions.transpose(0, 1) if mrope else first_positions[:, 0]
)
compressed_rows = apply_qsa_rope(rope, group_positions, compressed_rows)
qsa_store_cache_rows(unfused_compressed, compressed_slots, compressed_rows)
qsa_store_cache_rows(raw_keys, raw_slots, projected_qk[:, HQ * D :])
if rope_positions is not None:
qsa_store_cache_rows(rope_positions, raw_slots, position_rows)
torch.testing.assert_close(fused_query, unfused_query, rtol=RTOL, atol=ATOL)
assert torch.equal(fused_raw.view(torch.int16), unfused_raw.view(torch.int16))
torch.testing.assert_close(
fused_compressed, unfused_compressed, rtol=RTOL, atol=ATOL
)
@@ -0,0 +1,973 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from types import SimpleNamespace
import pytest
import torch
from vllm.models.qwen4_exp.common import qsa_cache
from vllm.models.qwen4_exp.common.qsa_cache import QSAMetadataBuilder
from vllm.models.qwen4_exp.nvidia import indexer_qsa
from vllm.models.qwen4_exp.nvidia import (
model as _qwen4_exp_model, # noqa: F401
)
from vllm.models.qwen4_exp.nvidia.ops import qsa as qsa_ops
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
requires_qsa_kernels = pytest.mark.skipif(
not current_platform.is_cuda() or not HAS_TRITON,
reason="QSA kernels require CUDA and Triton",
)
def test_qsa_mtp_index_share_updates_cache_but_skips_selection(
monkeypatch: pytest.MonkeyPatch,
) -> None:
rows = torch.tensor([[3, 1, -1], [5, 2, 0]], dtype=torch.int32)
raw_metadata = SimpleNamespace(
num_actual_tokens=2,
slot_mapping=torch.arange(2),
block_table=torch.empty(0),
query_start_loc=torch.arange(3),
logical_positions=torch.arange(2),
)
compressed_metadata = SimpleNamespace(
num_actual_tokens=2,
slot_mapping=torch.arange(2),
k_work_metadata=torch.empty(0),
)
updates = []
selections = []
indexer = SimpleNamespace(
skip_topk=True,
_metadata=lambda: (raw_metadata, compressed_metadata),
index_qk_proj=lambda hidden: (torch.zeros(2, 2), None),
index_n_heads=1,
index_kv_heads=1,
index_head_dim=1,
raw_key_cache=SimpleNamespace(
kv_cache=torch.empty(0),
rope_position_cache=None,
rope_position_offset=0,
),
compressed_key_cache=SimpleNamespace(kv_cache=torch.empty(0)),
use_fused_pre_indexer=True,
rotary_emb=SimpleNamespace(cos_sin_cache=torch.empty(0)),
q_layernorm=SimpleNamespace(weight=torch.ones(1), variance_epsilon=1e-6),
k_layernorm=SimpleNamespace(weight=torch.ones(1)),
compress_ratio=2,
)
monkeypatch.setattr(
indexer_qsa,
"qsa_pre_indexer",
lambda *args, **kwargs: updates.append((args, kwargs)),
)
monkeypatch.setattr(
qsa_ops,
"qsa_select_paged_tokens",
lambda *args, **kwargs: selections.append((args, kwargs)),
)
actual = indexer_qsa.QSAIndexer.forward(
indexer,
torch.zeros(2, 4),
torch.tensor([7, 8]),
rows,
)
assert actual is rows
assert len(updates) == 1
assert not selections
def _qsa_mqa_paged_reference(
q: torch.Tensor,
k_cache: torch.Tensor,
page_table: torch.Tensor,
token_to_req: torch.Tensor,
visible_lengths: torch.Tensor,
) -> torch.Tensor:
pages = page_table.index_select(0, token_to_req.long()).long()
keys = k_cache[pages, :, 0, :].flatten(1, 2)
scores = torch.einsum("rhd,rnd->rnh", q.float(), keys.float())
logits = torch.relu(scores).sum(dim=-1) / math.sqrt(q.shape[-1])
positions = torch.arange(keys.shape[1], device=q.device).unsqueeze(0)
return logits.masked_fill(positions >= visible_lengths.unsqueeze(1), -torch.inf)
def _qsa_relative_topk_reference(
logits: torch.Tensor,
row_starts: torch.Tensor,
row_ends: torch.Tensor,
topk: int,
) -> torch.Tensor:
output = torch.full(
(logits.shape[0], topk), -1, dtype=torch.int32, device=logits.device
)
for row in range(logits.shape[0]):
start = int(row_starts[row].item())
length = int((row_ends[row] - row_starts[row]).item())
width = min(length, topk)
if width:
output[row, :width] = torch.topk(
logits[row, start : start + length], width
).indices.to(torch.int32)
return output
def _expand_qsa_indices_reference(
block_indices: torch.Tensor,
query_positions: torch.Tensor,
sequence_lengths: torch.Tensor,
compress_ratio: int,
token_topk: int,
) -> torch.Tensor:
rows = block_indices.shape[0]
block_topk = token_topk // compress_ratio
output_width = token_topk + compress_ratio - 1
offsets = torch.arange(compress_ratio, device=block_indices.device)
blocks = block_indices.long()
expanded = blocks.unsqueeze(-1) * compress_ratio + offsets
expanded = torch.where(
blocks.unsqueeze(-1) >= 0, expanded, torch.full_like(expanded, -1)
).reshape(rows, block_topk * compress_ratio)
expanded = expanded[:, :token_topk]
expanded = torch.where(
(expanded >= 0) & (expanded < sequence_lengths.unsqueeze(1)),
expanded,
torch.full_like(expanded, -1),
)
tail_offsets = torch.arange(compress_ratio - 1, device=block_indices.device)
visible_tokens = query_positions + 1
tail_start = visible_tokens // compress_ratio * compress_ratio
tail = tail_start.unsqueeze(1) + tail_offsets.unsqueeze(0)
tail_count = (visible_tokens - tail_start).unsqueeze(1)
tail_valid = (tail_offsets.unsqueeze(0) < tail_count) & (
tail < sequence_lengths.unsqueeze(1)
)
tail = torch.where(tail_valid, tail, torch.full_like(tail, -1))
result = torch.cat((expanded, tail), dim=1)
order = torch.arange(output_width, device=result.device).expand(rows, -1)
sort_key = torch.where(result >= 0, order, order + output_width)
return result.gather(1, torch.argsort(sort_key, dim=1, stable=True)).to(torch.int32)
def _qsa_select_paged_tokens_reference(
q: torch.Tensor,
k_cache: torch.Tensor,
page_table: torch.Tensor,
token_to_req: torch.Tensor,
query_positions: torch.Tensor,
sequence_lengths: torch.Tensor,
token_topk: int,
compress_ratio: int,
) -> torch.Tensor:
row_sequence_lengths = sequence_lengths.index_select(0, token_to_req.long())
visible_blocks = torch.minimum(
(query_positions + 1) // compress_ratio,
row_sequence_lengths // compress_ratio,
).to(torch.int32)
logits = _qsa_mqa_paged_reference(
q,
k_cache,
page_table,
token_to_req,
visible_blocks,
)
starts = torch.zeros_like(visible_blocks)
blocks = _qsa_relative_topk_reference(
logits,
starts,
visible_blocks,
token_topk // compress_ratio,
)
return _expand_qsa_indices_reference(
blocks,
query_positions,
row_sequence_lengths,
compress_ratio,
token_topk,
)
def _qsa_sparse_paged_attention_reference(
q: torch.Tensor,
k_cache: torch.Tensor,
v_cache: torch.Tensor,
logical_indices: torch.Tensor,
block_table: torch.Tensor,
token_to_req: torch.Tensor,
softmax_scale: float,
) -> torch.Tensor:
output = torch.zeros_like(q)
repeats = q.shape[1] // k_cache.shape[2]
page_size = k_cache.shape[1]
for row in range(q.shape[0]):
logical = logical_indices[row]
logical = logical[logical >= 0].long()
if not logical.numel():
continue
request = token_to_req[row].long()
pages = block_table[request, logical // page_size].long()
offsets = logical % page_size
keys = k_cache[pages, offsets].repeat_interleave(repeats, dim=1)
values = v_cache[pages, offsets].repeat_interleave(repeats, dim=1)
scores = torch.einsum("hd,khd->hk", q[row].float(), keys.float())
probabilities = torch.softmax(scores * softmax_scale, dim=-1)
output[row] = torch.einsum("hk,khd->hd", probabilities, values.float()).to(
q.dtype
)
return output
@requires_qsa_kernels
def test_qsa_side_metadata_marks_cudagraph_padding_inert() -> None:
device = torch.device("cuda")
builder = QSAMetadataBuilder.__new__(QSAMetadataBuilder)
builder.compress_ratio = 1
builder.is_circular_buffer = False
builder.storage_block_size = 64
builder.token_to_req_buffer = torch.empty(16, dtype=torch.int32, device=device)
builder.slot_mapping_buffer = torch.empty(16, dtype=torch.int64, device=device)
builder.logical_positions_buffer = torch.empty(16, dtype=torch.int64, device=device)
builder.k_work_metadata_buffer = torch.empty(0, 2, dtype=torch.int32, device=device)
query_start_loc = torch.tensor([0, 4, 8, 12, 12], dtype=torch.int32, device=device)
token_to_req = torch.tensor([0] * 4 + [1] * 4 + [2] * 4 + [0] * 4, device=device)
common = SimpleNamespace(
num_actual_tokens=16,
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc.cpu(),
seq_lens=torch.tensor([68, 68, 68, 0], dtype=torch.int32, device=device),
slot_mapping=torch.tensor(list(range(12)) + [-1] * 4, device=device),
block_table_tensor=torch.empty((4, 0), dtype=torch.int32, device=device),
token_to_req_indices=lambda buffer: buffer.copy_(token_to_req),
)
metadata = builder.build(0, common)
assert metadata.logical_positions.tolist() == [
64,
65,
66,
67,
64,
65,
66,
67,
64,
65,
66,
67,
-1,
-1,
-1,
-1,
]
assert metadata.slot_mapping.tolist() == list(range(12)) + [-1] * 4
@requires_qsa_kernels
def test_qsa_circular_buffer_metadata_keeps_only_each_requests_suffix() -> None:
device = torch.device("cuda")
builder = QSAMetadataBuilder.__new__(QSAMetadataBuilder)
builder.compress_ratio = 4
builder.is_circular_buffer = True
builder.kv_cache_spec = SimpleNamespace(block_size=4)
builder.storage_block_size = 4
builder.token_to_req_buffer = torch.empty(16, dtype=torch.int32, device=device)
builder.slot_mapping_buffer = torch.empty(16, dtype=torch.int64, device=device)
builder.logical_positions_buffer = torch.empty(16, dtype=torch.int64, device=device)
builder.k_work_metadata_buffer = torch.empty(0, 2, dtype=torch.int32, device=device)
query_start_loc = torch.tensor([0, 7, 13, 13], dtype=torch.int32, device=device)
token_to_req = torch.tensor([0] * 7 + [1] * 6 + [0] * 3, device=device)
block_table = torch.tensor([[1], [0], [2]], dtype=torch.int32, device=device)
common = SimpleNamespace(
num_actual_tokens=16,
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc.cpu(),
seq_lens=torch.tensor([9, 11, 0], dtype=torch.int32, device=device),
slot_mapping=torch.full((16,), -1, dtype=torch.int64, device=device),
block_table_tensor=block_table,
token_to_req_indices=lambda buffer: buffer.copy_(token_to_req),
)
metadata = builder.build(0, common)
expected = [
-1,
-1,
-1,
5,
6,
7,
4,
-1,
-1,
3,
0,
1,
2,
-1,
-1,
-1,
]
assert metadata.slot_mapping.tolist() == expected
@pytest.mark.parametrize("chunk_start", list(range(8)))
def test_qsa_circular_buffer_survives_one_speculative_step(chunk_start: int) -> None:
"""A speculative step must not overwrite the open group's committed keys.
The step stores every row it computes, drafts included, before acceptance
is known, while the next step still reads the earlier members of the group
being compressed from the ring. A ring sized at the compression ratio makes
those rows alias, so a rejected draft silently replaces a committed key.
"""
compress_ratio = 4
num_spec = 3
capacity = compress_ratio * -(-(compress_ratio + num_spec) // compress_ratio)
query_len = num_spec + 1
slots = qsa_cache.circular_qsa_slot_mapping(
torch.tensor([[0]], dtype=torch.int32),
torch.zeros(query_len, dtype=torch.int32),
torch.arange(chunk_start, chunk_start + query_len),
capacity,
query_start_loc=torch.tensor([0, query_len], dtype=torch.int32),
)
committed = torch.arange(chunk_start - chunk_start % compress_ratio, chunk_start)
assert set(slots.tolist()).isdisjoint((committed % capacity).tolist())
def _qsa_key_cache(block_size: int, compress_ratio: int) -> qsa_cache.QSAKeyStateCache:
return qsa_cache.QSAKeyStateCache(
head_size=64,
dtype=torch.bfloat16,
cache_config=SimpleNamespace(block_size=block_size),
prefix=f"raw.{block_size}.{compress_ratio}",
vllm_config=SimpleNamespace(
compilation_config=SimpleNamespace(static_forward_context={})
),
compress_ratio=compress_ratio,
)
def test_qsa_state_caches_adapt_the_unified_logical_layout() -> None:
raw_cache = _qsa_key_cache(block_size=32, compress_ratio=4)
compressed_cache = qsa_cache.QSACompressedKeyCache(
head_size=64,
dtype=torch.bfloat16,
cache_config=SimpleNamespace(block_size=32),
prefix="compressed.bind",
vllm_config=SimpleNamespace(
compilation_config=SimpleNamespace(static_forward_context={})
),
compress_ratio=4,
)
raw_view = torch.empty(2, 1, 8, 64, dtype=torch.bfloat16)
compressed_view = torch.empty(2, 1, 8, 64, dtype=torch.bfloat16)
raw_cache.bind_kv_cache(raw_view)
compressed_cache.bind_kv_cache(compressed_view)
assert raw_cache.kv_cache.shape == (2, 8, 1, 64)
assert compressed_cache.kv_cache.shape == (2, 8, 1, 64)
assert raw_cache.kv_cache.data_ptr() == raw_view.data_ptr()
assert compressed_cache.kv_cache.data_ptr() == compressed_view.data_ptr()
@pytest.mark.parametrize(
("compress_ratio", "num_spec", "expected"),
[(4, 0, 4), (4, 1, 8), (4, 3, 8), (4, 4, 8), (4, 5, 12), (2, 3, 6)],
)
def test_qsa_ring_capacity_covers_one_speculative_step(
compress_ratio: int, num_spec: int, expected: int
) -> None:
"""Capacity spans the open group plus one speculative step, in whole groups."""
spec = _qsa_key_cache(
block_size=48, compress_ratio=compress_ratio
).get_kv_cache_spec(SimpleNamespace(num_speculative_tokens=num_spec))
assert spec.block_size == expected
@requires_qsa_kernels
def test_qsa_compressed_metadata_keeps_dummy_slots_inert() -> None:
device = torch.device("cuda")
builder = QSAMetadataBuilder.__new__(QSAMetadataBuilder)
builder.compress_ratio = 4
builder.is_circular_buffer = False
builder.storage_block_size = 16
builder.token_to_req_buffer = torch.empty(8, dtype=torch.int32, device=device)
builder.slot_mapping_buffer = torch.empty(8, dtype=torch.int64, device=device)
builder.logical_positions_buffer = torch.empty(8, dtype=torch.int64, device=device)
# Simulate max_num_seqs exceeding the three live requests below.
builder.request_capacity = 8
builder.k_work_metadata_buffer = torch.empty(4, 2, dtype=torch.int32, device=device)
query_start_loc = torch.tensor([0, 3, 3, 8], dtype=torch.int32, device=device)
token_to_req = torch.tensor(
[0, 0, 0, 2, 2, 2, 2, 2], dtype=torch.int32, device=device
)
common = SimpleNamespace(
num_actual_tokens=8,
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc.cpu(),
seq_lens=torch.tensor([7, 0, 12], dtype=torch.int32, device=device),
slot_mapping=torch.full((8,), -1, dtype=torch.int64, device=device),
block_table_tensor=torch.zeros((3, 1), dtype=torch.int32, device=device),
token_to_req_indices=lambda buffer: buffer.copy_(token_to_req),
)
metadata = builder.build(0, common)
assert metadata.slot_mapping.tolist() == [-1] * 8
assert metadata.k_work_metadata.tolist() == [[0, 0], [2, 0], [2, 1], [-1, -1]]
@requires_qsa_kernels
@pytest.mark.parametrize("compress_ratio", [1, 4])
@pytest.mark.parametrize("num_reqs", [2, 3, 4, 7, 8, 9])
def test_qsa_triton_metadata_matches_pytorch(
compress_ratio: int, num_reqs: int
) -> None:
device = torch.device("cuda")
num_tokens = 8
query_start_loc = torch.tensor(
[0, 3, *([3] * (num_reqs - 2)), 8], dtype=torch.int32, device=device
)
token_to_req = torch.tensor(
[0, 0, 0, *([num_reqs - 1] * 5)],
dtype=torch.int32,
device=device,
)
block_table_rows = torch.tensor(
[
[4, -1, 8, -1, 12, -1],
[1, -1, 2, -1, 3, -1],
[7, -1, 9, -1, 11, -1],
],
dtype=torch.int32,
device=device,
)
block_table_storage = block_table_rows[
torch.arange(num_reqs, device=device) % block_table_rows.shape[0]
]
seq_lens = torch.zeros(num_reqs, dtype=torch.int32, device=device)
seq_lens[0] = 10
seq_lens[-1] = 20
common = SimpleNamespace(
num_actual_tokens=num_tokens,
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc.cpu(),
seq_lens=seq_lens,
slot_mapping=torch.tensor(
[0, 1, -1, 3, 4, -1, -1, -1], dtype=torch.int64, device=device
),
block_table_tensor=block_table_storage[:, ::2],
token_to_req_indices=lambda buffer: buffer.copy_(token_to_req),
)
def make_buffers() -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
return (
torch.empty(num_tokens, dtype=torch.int32, device=device),
torch.empty(num_tokens, dtype=torch.int64, device=device),
torch.empty(num_tokens, dtype=torch.int64, device=device),
)
max_num_work = (
(num_tokens + (compress_ratio - 1) * num_reqs) // compress_ratio
if compress_ratio != 1
else 0
)
actual_k_work = (
torch.empty(max_num_work, 2, dtype=torch.int32, device=device)
if max_num_work
else None
)
actual_buffers = make_buffers()
actual = qsa_cache.build_qsa_metadata_triton(
common,
*actual_buffers,
storage_block_size=2,
compress_ratio=compress_ratio,
k_work_metadata_buffer=actual_k_work,
request_capacity=num_reqs,
)
expected_k_work = (
torch.empty_like(actual_k_work) if actual_k_work is not None else None
)
expected_buffers = make_buffers()
expected = qsa_cache._build_qsa_metadata_torch(
common,
*expected_buffers,
storage_block_size=2,
compress_ratio=compress_ratio,
k_work_metadata_buffer=expected_k_work,
request_capacity=num_reqs,
)
for actual_tensor, expected_tensor in zip(actual, expected):
torch.testing.assert_close(actual_tensor, expected_tensor)
if actual_k_work is not None:
torch.testing.assert_close(actual_k_work, expected_k_work)
@requires_qsa_kernels
def test_qsa_fused_metadata_matches_pytorch_for_large_padded_prefill() -> None:
device = torch.device("cuda")
num_mapped_tokens = 4096
num_tokens = 4224
query_start_loc = torch.tensor(
[0, num_mapped_tokens], dtype=torch.int32, device=device
)
common = SimpleNamespace(
num_actual_tokens=num_tokens,
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc.cpu(),
seq_lens=torch.tensor(
[num_mapped_tokens + 32], dtype=torch.int32, device=device
),
block_table_tensor=torch.arange(256, dtype=torch.int32, device=device)[None],
slot_mapping=torch.tensor(
[0] * num_mapped_tokens + [-1] * (num_tokens - num_mapped_tokens),
dtype=torch.int64,
device=device,
),
token_to_req_indices=lambda buffer: buffer.zero_(),
)
def make_buffers() -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
return (
torch.empty(num_tokens, dtype=torch.int32, device=device),
torch.empty(num_tokens, dtype=torch.int64, device=device),
torch.empty(num_tokens, dtype=torch.int64, device=device),
)
max_num_work = (num_tokens + 3) // 4
actual_k_work = torch.empty(max_num_work, 2, dtype=torch.int32, device=device)
expected_k_work = torch.empty_like(actual_k_work)
actual = qsa_cache.build_qsa_metadata_triton(
common,
*make_buffers(),
storage_block_size=8,
compress_ratio=4,
k_work_metadata_buffer=actual_k_work,
)
expected = qsa_cache._build_qsa_metadata_torch(
common,
*make_buffers(),
storage_block_size=8,
compress_ratio=4,
k_work_metadata_buffer=expected_k_work,
)
for actual_tensor, expected_tensor in zip(actual, expected):
torch.testing.assert_close(actual_tensor, expected_tensor)
torch.testing.assert_close(actual_k_work, expected_k_work)
@requires_qsa_kernels
@pytest.mark.parametrize(
"num_rows",
[
pytest.param(3, id="one_tile_per_program"),
pytest.param(33, id="looped_tiles"),
],
)
def test_qsa_mqa_paged_matches_test_reference(num_rows: int) -> None:
torch.manual_seed(1)
head_dim = 128
q = torch.randn(num_rows, 4, head_dim, device="cuda", dtype=torch.bfloat16)
cache = torch.randn(40, 4, 1, head_dim, device="cuda", dtype=torch.bfloat16)
page_table = torch.randperm(40, device="cuda", dtype=torch.int32).reshape(2, 20)
request_rows = [(num_rows + 1) // 2, num_rows // 2]
token_to_req = torch.repeat_interleave(
torch.arange(2, device="cuda", dtype=torch.int32),
torch.tensor(request_rows, device="cuda"),
)
sequence_length_values = [320, 264]
sequence_lengths = torch.tensor(
sequence_length_values, device="cuda", dtype=torch.int32
)
query_positions = torch.cat(
[
torch.arange(length - rows, length, device="cuda", dtype=torch.int32)
for rows, length in zip(request_rows, sequence_length_values, strict=True)
]
)
compress_ratio = 4
visible_lengths = (query_positions + 1) // compress_ratio
actual, actual_visible_blocks = qsa_ops.qsa_mqa_paged(
q,
cache,
page_table,
token_to_req,
query_positions,
sequence_lengths,
compress_ratio=compress_ratio,
)
expected = _qsa_mqa_paged_reference(
q, cache, page_table, token_to_req, visible_lengths
)
torch.testing.assert_close(actual_visible_blocks, visible_lengths)
# Top-k is bounded by visible_blocks; later columns are intentionally unwritten.
columns = torch.arange(actual.shape[1], device=actual.device)
visible = columns[None, :] < actual_visible_blocks[:, None]
torch.testing.assert_close(actual[visible], expected[visible], rtol=1e-3, atol=1e-3)
@requires_qsa_kernels
def test_qsa_block_expansion_matches_test_reference() -> None:
blocks = torch.tensor([[0, -1], [1, 0]], device="cuda", dtype=torch.int32)
query_positions = torch.tensor([5, 10], device="cuda")
sequence_lengths = torch.tensor([6, 11], device="cuda")
token_to_req = torch.tensor([0, 1], device="cuda", dtype=torch.int32)
actual = qsa_ops.expand_qsa_block_indices_cuda(
blocks,
query_positions,
sequence_lengths,
token_to_req,
compress_ratio=4,
token_topk=8,
)
expected = _expand_qsa_indices_reference(
blocks,
query_positions,
sequence_lengths,
compress_ratio=4,
token_topk=8,
)
torch.testing.assert_close(actual, expected)
@requires_qsa_kernels
@pytest.mark.parametrize(
("num_rows", "num_query_heads", "num_kv_heads", "page_size"),
[
# Kernel-visible pages with --block-size 256 and hybrid-cache alignment.
pytest.param(1, 24, 2, 1792, id="tp1_split64"),
pytest.param(16, 12, 1, 1792, id="tp2_split32"),
pytest.param(32, 6, 1, 1024, id="tp4_split8"),
pytest.param(257, 6, 1, 1024, id="tp4_split4"),
pytest.param(513, 6, 1, 1024, id="tp4_split1"),
],
)
def test_qsa_sparse_paged_attention_matches_test_reference(
num_rows: int,
num_query_heads: int,
num_kv_heads: int,
page_size: int,
) -> None:
torch.manual_seed(2)
head_dim = 256
num_requests = 2
num_selected_pages = 64
# Keep the newest page outside the synthetic top-k as causal headroom.
num_pages_per_request = num_selected_pages + 1
num_cache_blocks = num_requests * num_pages_per_request
indexer_budget = 2048
indexer_compress_ratio = 4
selection_width = indexer_budget + indexer_compress_ratio - 1
q = torch.randn(
num_rows, num_query_heads, head_dim, device="cuda", dtype=torch.bfloat16
)
kv_cache = torch.randn(
num_cache_blocks,
page_size,
num_kv_heads,
2 * head_dim,
device="cuda",
dtype=torch.bfloat16,
)
k_cache, v_cache = kv_cache.split(head_dim, dim=-1)
block_table = (
torch.randperm(num_cache_blocks, device="cuda")
.reshape(num_requests, num_pages_per_request)
.to(torch.int32)
)
rows_per_request = math.ceil(num_rows / num_requests)
row_indices = torch.arange(num_rows, device="cuda", dtype=torch.int32)
token_to_req = row_indices // rows_per_request
request_row_counts = torch.tensor(
[rows_per_request, num_rows - rows_per_request],
device="cuda",
dtype=torch.int32,
)
context_length = num_pages_per_request * page_size - 1
block_topk = indexer_budget // indexer_compress_ratio
compressed_blocks_per_page = page_size // indexer_compress_ratio
selection = torch.arange(block_topk, device="cuda")
selected_pages = selection % num_selected_pages
selected_offsets = selection // num_selected_pages
row_shifts = 2 * row_indices.unsqueeze(1)
# Eight blocks per page; adjacent rows overlap by six of those eight.
selected_offsets = (selected_offsets + row_shifts) % compressed_blocks_per_page
block_indices = (selected_pages * compressed_blocks_per_page + selected_offsets).to(
torch.int32
)
rows_within_request = row_indices % rows_per_request
query_positions = (
context_length - request_row_counts[token_to_req.long()] + rows_within_request
).to(torch.int64)
sequence_lengths = torch.full(
(num_requests,), context_length, device="cuda", dtype=torch.int32
)
logical_indices = qsa_ops.expand_qsa_block_indices_cuda(
block_indices,
query_positions,
sequence_lengths,
token_to_req,
indexer_compress_ratio,
indexer_budget,
)
assert logical_indices.shape == (num_rows, selection_width)
scale = q.shape[-1] ** -0.5
actual = qsa_ops.qsa_sparse_paged_attention(
q,
k_cache,
v_cache,
logical_indices,
block_table,
token_to_req,
)
expected = _qsa_sparse_paged_attention_reference(
q,
k_cache,
v_cache,
logical_indices,
block_table,
token_to_req,
scale,
)
torch.testing.assert_close(actual, expected, rtol=2e-2, atol=2e-2)
@requires_qsa_kernels
def test_qsa_selection_chunks_workspace_and_matches_test_reference(
monkeypatch: pytest.MonkeyPatch,
workspace_init,
) -> None:
rows, keys, heads, head_dim = 65, 640, 4, 16
token_topk, compress_ratio = 2048, 4
torch.manual_seed(3)
q = torch.randn(rows, heads, head_dim, device="cuda", dtype=torch.bfloat16)
cache = torch.randn(40, 16, 1, head_dim, device="cuda", dtype=torch.bfloat16)
page_table = torch.randperm(40, device="cuda", dtype=torch.int32).unsqueeze(0)
token_to_req = torch.zeros(rows, device="cuda", dtype=torch.int32)
query_positions = torch.full((rows,), 2559, device="cuda", dtype=torch.int32)
sequence_lengths = torch.tensor([2560], device="cuda", dtype=torch.int32)
monkeypatch.setattr(qsa_ops, "_LOGITS_WORKSPACE_BYTES", 32 * keys * 4)
original_score = qsa_ops.qsa_mqa_paged
scored_row_counts = []
def record_score(query: torch.Tensor, *args, **kwargs):
scored_row_counts.append(query.shape[0])
return original_score(query, *args, **kwargs)
monkeypatch.setattr(qsa_ops, "qsa_mqa_paged", record_score)
actual = qsa_ops.qsa_select_paged_tokens(
q,
cache,
page_table,
token_to_req,
query_positions,
sequence_lengths,
token_topk,
compress_ratio,
)
expected = _qsa_select_paged_tokens_reference(
q,
cache,
page_table,
token_to_req,
query_positions,
sequence_lengths,
token_topk,
compress_ratio,
)
torch.testing.assert_close(actual.sort().values, expected.sort().values)
assert scored_row_counts == [32, 32, 1]
@requires_qsa_kernels
def test_qsa_selection_handles_no_complete_compressed_blocks(workspace_init) -> None:
q = torch.zeros(2, 4, 8, device="cuda", dtype=torch.bfloat16)
cache = torch.zeros(1, 16, 1, 8, device="cuda", dtype=torch.bfloat16)
page_table = torch.zeros(1, 1, device="cuda", dtype=torch.int32)
token_to_req = torch.zeros(2, device="cuda", dtype=torch.int32)
query_positions = torch.tensor([1, 2], device="cuda", dtype=torch.int32)
sequence_lengths = torch.tensor([3], device="cuda", dtype=torch.int32)
selected = qsa_ops.qsa_select_paged_tokens(
q,
cache,
page_table,
token_to_req,
query_positions,
sequence_lengths,
token_topk=2048,
compress_ratio=4,
)
assert selected[0, :2].tolist() == [0, 1]
assert selected[1, :3].tolist() == [0, 1, 2]
assert torch.all(selected[0, 2:] == -1)
assert torch.all(selected[1, 3:] == -1)
@requires_qsa_kernels
def test_qsa_streaming_compression_and_compressor_state_store_match_reference() -> None:
head_dim = 8
current_pairs = [
*((0, position) for position in range(2, 9)),
*((1, position) for position in range(5, 11)),
]
def key_row(request: int, position: int) -> torch.Tensor:
return (
torch.arange(head_dim, dtype=torch.float32) + request * 1000 + position * 10
)
def position_row(request: int, position: int) -> torch.Tensor:
return torch.tensor(
[
request * 1000 + position,
request * 1000 + position + 100,
request * 1000 + position + 200,
],
dtype=torch.int64,
)
raw_keys = (
torch.stack([key_row(request, position) for request, position in current_pairs])
.unsqueeze(1)
.to(device="cuda", dtype=torch.bfloat16)
)
raw_positions = (
torch.stack(
[position_row(request, position) for request, position in current_pairs]
)
.unsqueeze(1)
.to(device="cuda")
)
token_to_req = torch.tensor(
[request for request, _ in current_pairs],
dtype=torch.int32,
device="cuda",
)
logical_positions = torch.tensor(
[position for _, position in current_pairs],
dtype=torch.int64,
device="cuda",
)
query_start_loc = torch.tensor([0, 7, 13], dtype=torch.int32, device="cuda")
compressor_state_block_table = torch.tensor(
[[1], [0]], dtype=torch.int32, device="cuda"
)
compressor_state_cache = torch.zeros(
2, 4, 1, head_dim, dtype=torch.bfloat16, device="cuda"
)
rope_cache = torch.zeros(2, 4, 1, 3, dtype=torch.int64, device="cuda")
for request, position, block in ((0, 0, 1), (0, 1, 1), (1, 4, 0)):
compressor_state_cache[block, position % 4, 0] = key_row(request, position).to(
device="cuda", dtype=torch.bfloat16
)
rope_cache[block, position % 4, 0] = position_row(request, position).to("cuda")
compressed_slots = torch.full(
(len(current_pairs),), -1, dtype=torch.int64, device="cuda"
)
valid_rows = torch.tensor([1, 5, 9], dtype=torch.int64, device="cuda")
compressed_slots[valid_rows] = torch.arange(3, device="cuda")
pooled, first_positions = qsa_ops.qsa_compress_groups_with_ratio(
raw_keys,
raw_positions,
compressor_state_cache,
compressor_state_block_table,
token_to_req,
query_start_loc,
logical_positions,
compressed_slots,
compress_ratio=4,
rope_cache=rope_cache,
)
pooled_without_rope, scalar_first_positions = (
qsa_ops.qsa_compress_groups_with_ratio(
raw_keys,
raw_positions,
compressor_state_cache,
compressor_state_block_table,
token_to_req,
query_start_loc,
logical_positions,
compressed_slots,
compress_ratio=4,
)
)
groups = [
[(0, position) for position in range(0, 4)],
[(0, position) for position in range(4, 8)],
[(1, position) for position in range(4, 8)],
]
expected_pooled = (
torch.stack(
[
torch.stack([key_row(*pair) for pair in group]).mean(dim=0)
for group in groups
]
)
.unsqueeze(1)
.to(device="cuda", dtype=torch.bfloat16)
)
expected_positions = torch.stack(
[position_row(0, 0), position_row(0, 4), position_row(1, 4)]
).to("cuda")
expected_scalar_positions = torch.tensor(
[[0, 0, 0], [4, 4, 4], [4, 4, 4]],
dtype=torch.int64,
device="cuda",
)
torch.testing.assert_close(pooled[valid_rows], expected_pooled)
torch.testing.assert_close(pooled_without_rope[valid_rows], expected_pooled)
torch.testing.assert_close(first_positions[valid_rows], expected_positions)
torch.testing.assert_close(
scalar_first_positions[valid_rows], expected_scalar_positions
)
compressor_state_slots = torch.tensor(
[-1, -1, -1, 5, 6, 7, 4, -1, -1, 3, 0, 1, 2],
dtype=torch.int64,
device="cuda",
)
qsa_ops.qsa_store_cache_rows(
compressor_state_cache, compressor_state_slots, raw_keys
)
qsa_ops.qsa_store_cache_rows(rope_cache, compressor_state_slots, raw_positions)
for request, positions, block in ((0, range(5, 9), 1), (1, range(7, 11), 0)):
for position in positions:
torch.testing.assert_close(
compressor_state_cache[block, position % 4, 0],
key_row(request, position).to(device="cuda", dtype=torch.bfloat16),
)
torch.testing.assert_close(
rope_cache[block, position % 4, 0],
position_row(request, position).to("cuda"),
)
+6
View File
@@ -508,6 +508,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
extras={"native-prefix": "imdatta0/small_qwen3_5_20b"},
max_model_len=4096,
),
"Qwen4ExpForCausalLM": _HfExamplesInfo("", is_available_online=False),
"MellumForCausalLM": _HfExamplesInfo("JetBrains/Mellum2-12B-A2.5B-Base"),
"Qwen3NextForCausalLM": _HfExamplesInfo(
"Qwen/Qwen3-Next-80B-A3B-Instruct",
@@ -1348,6 +1349,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"Qwen/Qwen3.5-35B-A3B",
max_model_len=4096,
),
"Qwen4ExpForConditionalGeneration": _HfExamplesInfo(
"",
is_available_online=False,
),
"Qwen3OmniMoeForConditionalGeneration": _HfExamplesInfo(
"Qwen/Qwen3-Omni-30B-A3B-Instruct",
max_model_len=4096,
@@ -1797,6 +1802,7 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
"Qwen/Qwen3.5-35B-A3B",
speculative_model="Qwen/Qwen3.5-35B-A3B",
),
"Qwen4ExpMTP": _HfExamplesInfo("", is_available_online=False),
"Step3p5MTP": _HfExamplesInfo(
"stepfun-ai/Step-3.5-Flash",
speculative_model="stepfun-ai/Step-3.5-Flash",
+44
View File
@@ -6,6 +6,7 @@ Run `pytest tests/quantization/test_fp8.py --forked`.
"""
import logging
from types import SimpleNamespace
import pytest
import regex as re
@@ -31,6 +32,10 @@ from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
from vllm.model_executor.layers.quantization.online.fp8 import (
Fp8PerTensorOnlineLinearMethod,
)
from vllm.model_executor.layers.quantization.utils import flashinfer_utils
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
prepare_fp8_moe_layer_for_fi,
)
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
process_fp8_input_tensor_strategy_moe,
)
@@ -50,6 +55,45 @@ MODELS = [
]
def test_prepare_gated_trtllm_fp8_moe_weights_pads_each_projection(monkeypatch):
monkeypatch.setattr(
flashinfer_utils,
"rotate_weights_for_fi_trtllm_fp8_per_tensor_moe",
lambda *args: None,
)
intermediate = 17
padded_intermediate = 32
hidden_size = 4
gate = torch.ones((1, intermediate, hidden_size), dtype=torch.float8_e4m3fn)
up = torch.full_like(gate, 2)
w13 = torch.cat((gate, up), dim=1)
w2 = torch.ones((1, hidden_size, intermediate), dtype=torch.float8_e4m3fn)
layer = SimpleNamespace(
activation=SimpleNamespace(is_gated=True),
moe_config=SimpleNamespace(
is_act_and_mul=True,
intermediate_size_per_partition=intermediate,
),
)
padded_w31, _, _, _ = prepare_fp8_moe_layer_for_fi(
layer,
w13,
w2,
w13_scale=torch.ones(1),
w13_input_scale=torch.ones(1),
w2_scale=torch.ones(1),
w2_input_scale=torch.ones(1),
is_trtllm=True,
)
expected = w13.new_zeros((1, 2 * padded_intermediate, hidden_size))
expected[:, :intermediate] = up
expected[:, padded_intermediate : padded_intermediate + intermediate] = gate
assert layer.moe_config.intermediate_size_per_partition == padded_intermediate
assert torch.equal(padded_w31, expected)
def test_static_fp8_moe_input_scales_remain_scalar() -> None:
a1_scale, a2_scale = process_fp8_input_tensor_strategy_moe(
torch.tensor([0.25, 0.5]),
+86
View File
@@ -16,6 +16,7 @@ from vllm.model_executor.layers.fused_moe.config import (
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
Mxfp4MoeBackend,
_requires_qwen38_tep8_emulation,
select_mxfp4_moe_backend,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
@@ -72,6 +73,91 @@ def _make_w4a4_moe_config(moe_backend: str = "auto") -> FusedMoEConfig:
)
def _make_qwen38_tep8_moe_config() -> FusedMoEConfig:
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
return FusedMoEConfig(
num_experts=512,
experts_per_token=9,
hidden_dim=2560,
intermediate_size=640,
num_local_experts=64,
num_logical_experts=512,
moe_parallel_config=FusedMoEParallelConfig(
tp_size=1,
tp_rank=0,
pcp_size=1,
pcp_rank=0,
dp_size=1,
dp_rank=0,
ep_size=8,
ep_rank=0,
sp_size=1,
use_ep=True,
all2all_backend="allgather_reducescatter",
enable_eplb=False,
),
activation=MoEActivation.SILU,
in_dtype=torch.bfloat16,
device="cuda",
routing_method=RoutingMethodType.Renormalize,
)
def test_qwen38_tep8_requires_emulation_only_on_gfx950(monkeypatch):
import vllm.model_executor.layers.fused_moe.oracle.mxfp4 as mxfp4_oracle
config = _make_qwen38_tep8_moe_config()
monkeypatch.setattr(current_platform, "is_rocm", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: True)
assert _requires_qwen38_tep8_emulation(config, kMxfp4Dynamic)
config.moe_parallel_config.ep_size = 4
config.num_local_experts = 128
assert not _requires_qwen38_tep8_emulation(config, kMxfp4Dynamic)
config = _make_qwen38_tep8_moe_config()
monkeypatch.setattr(mxfp4_oracle.current_platform, "is_rocm", lambda: False)
assert not _requires_qwen38_tep8_emulation(config, kMxfp4Dynamic)
@pytest.mark.parametrize(
"requested_backend,expected_backend",
[
("auto", Mxfp4MoeBackend.EMULATION),
("aiter", Mxfp4MoeBackend.AITER_MXFP4_MXFP4),
],
)
def test_qwen38_tep8_auto_fallback_respects_explicit_backend(
requested_backend,
expected_backend,
monkeypatch,
):
import vllm.model_executor.layers.fused_moe.oracle.mxfp4 as mxfp4_oracle
class SupportedExperts:
@staticmethod
def is_supported_config(*args, **kwargs):
return True, None
config = _make_qwen38_tep8_moe_config()
config.moe_backend = requested_backend
monkeypatch.setattr(current_platform, "is_rocm", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: True)
monkeypatch.setattr(mxfp4_oracle, "_user_moe_activation_override", lambda: None)
monkeypatch.setattr(
mxfp4_oracle, "backend_to_kernel_cls", lambda backend: [SupportedExperts]
)
backend, experts_cls = select_mxfp4_moe_backend(
config, activation_key=kMxfp4Dynamic
)
assert backend == expected_backend
assert experts_cls is SupportedExperts
@pytest.fixture
def mxfp4_oracle_config():
"""Stub the config the oracle reads (``model_config.quantization_config``)
+21
View File
@@ -604,3 +604,24 @@ def test_online_mxfp4_dense_matches_quark(
keys=("weight", "weight_scale"),
packed_weight_keys=("weight",),
)
def test_quark_ocp_mx_dense_rejects_unaligned_input_partition():
scheme = object.__new__(QuarkOCP_MX)
layer = torch.nn.Module()
layer.prefix = "model.layers.0.mlp.shared_expert.down_proj"
with pytest.raises(
ValueError,
match=(
"input size per partition of 80, which must be divisible by "
"the OCP MX group size 32"
),
):
scheme.create_weights(
layer=layer,
output_partition_sizes=[2560],
input_size_per_partition=80,
params_dtype=torch.bfloat16,
weight_loader=lambda *_: None,
)
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
prepare_nvfp4_moe_layer_for_fi_or_cutlass,
)
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
align_fp4_moe_weights_for_fi,
align_trtllm_fp4_moe_hidden_dim_for_fi,
)
@@ -107,3 +108,53 @@ def test_align_trtllm_fp4_moe_hidden_dim_pads_to_256_multiple():
assert torch.count_nonzero(out_w13_scale[:, :, hidden_dim // 16 :]) == 0
assert torch.count_nonzero(out_w2[:, hidden_dim:, :]) == 0
assert torch.count_nonzero(out_w2_scale[:, hidden_dim:, :]) == 0
def test_align_trtllm_fp4_moe_intermediate_pads_gate_and_up_separately():
# Qwen3.8-Flash-Next has intermediate_size=640. At TP4 each rank owns
# 160 values, which TRTLLM pads to the next 64-value boundary (192).
intermediate = 160
padded_intermediate = 192
hidden_dim = 32
gate = torch.ones((1, intermediate, hidden_dim // 2), dtype=torch.uint8)
up = torch.full_like(gate, 2)
w13 = torch.cat((gate, up), dim=1)
gate_scale = torch.full((1, intermediate, hidden_dim // 16), 3, dtype=torch.uint8)
up_scale = torch.full_like(gate_scale, 4)
w13_scale = torch.cat((gate_scale, up_scale), dim=1)
w2 = torch.full((1, hidden_dim, intermediate // 2), 5, dtype=torch.uint8)
w2_scale = torch.full((1, hidden_dim, intermediate // 16), 6, dtype=torch.uint8)
out_w13, out_w13_scale, out_w2, out_w2_scale, out_intermediate = (
align_fp4_moe_weights_for_fi(
w13,
w13_scale,
w2,
w2_scale,
is_act_and_mul=True,
min_alignment=64,
)
)
assert out_intermediate == padded_intermediate
torch.testing.assert_close(out_w13[:, :intermediate], gate)
torch.testing.assert_close(
out_w13[:, padded_intermediate : padded_intermediate + intermediate], up
)
torch.testing.assert_close(out_w13_scale[:, :intermediate], gate_scale)
torch.testing.assert_close(
out_w13_scale[:, padded_intermediate : padded_intermediate + intermediate],
up_scale,
)
assert torch.count_nonzero(out_w13[:, intermediate:padded_intermediate]) == 0
assert torch.count_nonzero(out_w13[:, padded_intermediate + intermediate :]) == 0
assert torch.count_nonzero(out_w13_scale[:, intermediate:padded_intermediate]) == 0
assert (
torch.count_nonzero(out_w13_scale[:, padded_intermediate + intermediate :]) == 0
)
torch.testing.assert_close(out_w2[:, :, : intermediate // 2], w2)
torch.testing.assert_close(out_w2_scale[:, :, : intermediate // 16], w2_scale)
assert torch.count_nonzero(out_w2[:, :, intermediate // 2 :]) == 0
assert torch.count_nonzero(out_w2_scale[:, :, intermediate // 16 :]) == 0
+312
View File
@@ -9,23 +9,34 @@ block dim (a contiguous region per layer) or inside it (all layers' pages within
block); the allocation is the same either way.
"""
from dataclasses import replace
from unittest.mock import MagicMock
import pytest
import torch
from vllm.config import CacheConfig
from vllm.v1.core.kv_cache_manager import KVCacheManager
from vllm.v1.core.kv_cache_utils import (
_get_kv_cache_bytes_per_block,
_get_packed_kv_cache_groups,
_pool_bytes_per_block,
generate_scheduler_kv_cache_config,
get_kv_cache_config_from_groups,
get_kv_cache_groups,
resolve_kv_cache_block_sizes,
)
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
FullAttentionSpec,
KVCacheGroupSpec,
KVCacheLayout,
KVCacheSpec,
MambaSpec,
MLAAttentionSpec,
SlidingWindowMLASpec,
UniformTypeKVCacheSpecs,
iter_layer_specs,
)
from vllm.v1.worker.utils import allocate_kv_cache
@@ -86,6 +97,307 @@ def _bind(config, layout: str):
return allocate_kv_cache(config, torch.device("cpu"), KVCacheLayout[layout], None)
MAIN_KV_PAGE_BYTES = 2_048
COMPRESSED_PAGE_BYTES = 128
NUM_CACHE_TUPLES = 3
BYTES_PER_BLOCK = NUM_CACHE_TUPLES * (MAIN_KV_PAGE_BYTES + COMPRESSED_PAGE_BYTES)
def _main_kv_name(layer_index: int) -> str:
return f"model.layers.{layer_index}.self_attn"
def _compressed_name(layer_index: int) -> str:
return f"model.layers.{layer_index}.self_attn.indexer.compressed_key_cache"
def _compressor_state_name(layer_index: int) -> str:
return f"model.layers.{layer_index}.self_attn.indexer.raw_key_cache"
def _make_csa_linear_specs(
num_mamba: int = 7,
num_tuples: int = NUM_CACHE_TUPLES,
) -> dict[str, KVCacheSpec]:
specs: dict[str, KVCacheSpec] = {}
for layer_index in range(num_mamba):
specs[f"model.layers.{layer_index}.linear_attn"] = MambaSpec(
block_size=16,
shapes=((32,),),
dtypes=(torch.bfloat16,),
)
specs["model.layers.0.ple"] = MambaSpec(
block_size=16,
shapes=((24,),),
dtypes=(torch.bfloat16,),
tp_replicated=True,
)
for layer_index in range(num_tuples):
specs[_main_kv_name(layer_index)] = FullAttentionSpec(
block_size=16,
num_kv_heads=2,
head_size=16,
head_size_v=16,
dtype=torch.bfloat16,
)
specs[_compressed_name(layer_index)] = MLAAttentionSpec(
block_size=16,
num_kv_heads=1,
head_size=16,
dtype=torch.bfloat16,
tokens_per_state=4,
)
specs[_compressor_state_name(layer_index)] = CircularBufferSpec(
block_size=4,
num_kv_heads=1,
head_size=8,
head_size_v=0,
dtype=torch.bfloat16,
)
return specs
def _shared_layout_config():
config = _mock_vllm_config("BLNHC")
config.scheduler_config.disable_hybrid_kv_cache_manager = False
config.speculative_config = None
config.model_config.max_model_len = 16
config.model_config.get_total_num_hidden_layers.return_value = 64
config.model_config.get_total_num_kv_heads.return_value = 2
config.model_config.get_num_kv_heads.return_value = 2
config.parallel_config.pipeline_parallel_size = 1
config.parallel_config.decode_context_parallel_size = 1
config.cache_config.block_size = 16
config.cache_config.enable_prefix_caching = False
config.cache_config.prefix_match_unit = None
config.cache_config.mamba_cache_mode = "none"
return config
class TestCSALinearGrouping:
"""A CSA + linear-attention model (sparse attention with a compressor ring,
plus sharded GDN and one TP-replicated PLE state) goes through the generic
packed-group path; no model-specific branch is involved."""
@staticmethod
def _mamba_groups(groups):
"""(group, per-layer mamba spec) for every group holding mamba state."""
out = []
for group in groups:
spec = next(iter(iter_layer_specs(group.kv_cache_spec)))
if isinstance(spec, MambaSpec):
out.append((group, spec))
return out
def test_replicated_state_gets_its_own_group(self):
"""The PLE state is TP-replicated and a different size, so it must not
share a manager group with the sharded GDN states (the NIXL worker
requires exactly one single-layer replicated group)."""
groups = get_kv_cache_groups(_shared_layout_config(), _make_csa_linear_specs())
mamba_groups = self._mamba_groups(groups)
replicated = [g for g, spec in mamba_groups if spec.tp_replicated]
assert len(replicated) == 1
assert replicated[0].layer_names == ["model.layers.0.ple"]
sharded = [g for g, spec in mamba_groups if not spec.tp_replicated]
assert sharded, "GDN states must keep their own groups"
assert sorted(n for g in sharded for n in g.layer_names) == sorted(
f"model.layers.{i}.linear_attn" for i in range(7)
)
def test_roles_land_in_separate_groups(self):
groups = get_kv_cache_groups(_shared_layout_config(), _make_csa_linear_specs())
owner = next(g for g in groups if _main_kv_name(0) in g.layer_names)
assert sorted(owner.layer_names) == sorted(
[
*(_main_kv_name(i) for i in range(NUM_CACHE_TUPLES)),
*(_compressed_name(i) for i in range(NUM_CACHE_TUPLES)),
]
)
assert owner.kv_cache_spec.prefix_cacheable
scratch = next(g for g in groups if _compressor_state_name(0) in g.layer_names)
assert scratch.layer_names == [
_compressor_state_name(i) for i in range(NUM_CACHE_TUPLES)
]
assert not scratch.kv_cache_spec.prefix_cacheable
def test_every_group_fits_one_packed_block(self):
config = _shared_layout_config()
groups = get_kv_cache_groups(config, _make_csa_linear_specs())
bytes_per_block = _get_kv_cache_bytes_per_block(groups)
pages = _pages(groups)
for group in groups:
assert sum(pages[n] for n in group.layer_names) <= bytes_per_block
kv_cache_config = get_kv_cache_config_from_groups(
config, groups, available_memory=bytes_per_block * 32
)
assert kv_cache_config.num_blocks == 32
# Groups overlay from byte 0 of each block, so a layer never addresses
# past the block it belongs to.
for tensor in kv_cache_config.kv_cache_tensors:
assert tensor.block_stride == bytes_per_block
assert tensor.offset < bytes_per_block
def test_scratch_group_survives_computed_block_truncation(self):
"""The scratch group contributes no computed blocks, so truncating a
lookup result must skip it: its block size is the ring capacity, which
neither divides the hit length nor bounds the (empty) block list."""
config = _shared_layout_config()
config.cache_config.enable_prefix_caching = True
groups = get_kv_cache_groups(config, _make_csa_linear_specs())
kv_cache_config = get_kv_cache_config_from_groups(
config, groups, available_memory=BYTES_PER_BLOCK * 64
)
scheduler_config = generate_scheduler_kv_cache_config([kv_cache_config])
manager = KVCacheManager(
scheduler_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=16,
scheduler_block_size=16,
)
blocks = manager.create_kv_cache_blocks(
tuple(
manager.block_pool.get_new_blocks(3)
if group.kv_cache_spec.prefix_cacheable
else []
for group in scheduler_config.kv_cache_groups
)
)
truncated = manager.truncate_computed_blocks(blocks, 48)
scratch_index = next(
i
for i, group in enumerate(scheduler_config.kv_cache_groups)
if not group.kv_cache_spec.prefix_cacheable
)
assert truncated.blocks[scratch_index] == []
def test_prefix_hits_respect_compression_alignment(self):
config = _shared_layout_config()
config.cache_config.enable_prefix_caching = True
config.cache_config.prefix_match_unit = 16
config.cache_config.mamba_cache_mode = "align"
specs = {
name: replace(spec, mamba_cache_mode="align")
if isinstance(spec, MambaSpec)
else spec
for name, spec in _make_csa_linear_specs(num_tuples=1).items()
}
groups = get_kv_cache_groups(config, specs)
kv_cache_config = get_kv_cache_config_from_groups(
config, groups, available_memory=8 * MAIN_KV_PAGE_BYTES
)
assert resolve_kv_cache_block_sizes(kv_cache_config, config) == (16, 16)
# 2 tokens is not a multiple of the compression ratio 4: a prefix hit
# could land inside a partially filled compressed state.
config.cache_config.prefix_match_unit = 2
with pytest.raises(ValueError, match="per-state compression"):
resolve_kv_cache_block_sizes(kv_cache_config, config)
def test_scratch_ring_does_not_drag_hash_granularity(self):
config = _shared_layout_config()
config.cache_config.enable_prefix_caching = True
config.cache_config.mamba_cache_mode = "align"
specs = {
name: replace(spec, mamba_cache_mode="align")
if isinstance(spec, MambaSpec)
else spec
for name, spec in _make_csa_linear_specs(num_tuples=1).items()
}
groups = get_kv_cache_groups(config, specs)
kv_cache_config = get_kv_cache_config_from_groups(
config, groups, available_memory=8 * MAIN_KV_PAGE_BYTES
)
# The hash granularity is the GCD over prefix-cacheable groups only;
# the 4-token scratch ring is excluded (it would drag it to 4).
assert resolve_kv_cache_block_sizes(kv_cache_config, config) == (16, 16)
def test_compressed_attention_hashes_can_be_finer_than_cache_hits(self):
config = _shared_layout_config()
config.cache_config.enable_prefix_caching = True
specs = {
"compressed.4": MLAAttentionSpec(
block_size=256,
num_kv_heads=1,
head_size=16,
dtype=torch.bfloat16,
tokens_per_state=4,
),
"compressed.128": MLAAttentionSpec(
block_size=256,
num_kv_heads=1,
head_size=16,
dtype=torch.bfloat16,
tokens_per_state=128,
),
"compressor_state.4": SlidingWindowMLASpec(
block_size=4,
num_kv_heads=1,
head_size=8,
head_size_v=0,
dtype=torch.bfloat16,
sliding_window=8,
),
}
groups = get_kv_cache_groups(config, specs)
kv_cache_config = get_kv_cache_config_from_groups(
config, groups, available_memory=8 * MAIN_KV_PAGE_BYTES
)
# Hashes are computed every 4 tokens, but without an align-mode Mamba
# group cache hits remain on the 256-token scheduler boundary.
assert resolve_kv_cache_block_sizes(kv_cache_config, config) == (256, 4)
@pytest.mark.parametrize("wide", ["unbalanced_attention", "unsplittable_state"])
def test_mamba_split_measures_the_block_the_other_groups_already_force(self, wide):
"""Whatever fixes the block stride -- a bucket with unequal layer counts
per page size, which is emitted whole, or a single-layer state that
cannot be split at all -- the mamba layers must be sized against it.
Sizing them against a narrower bucket splits them past what the block
already fits, spending a pool block per extra group for no saving."""
config = _mock_vllm_config("BLNHC")
config.speculative_config = None
specs = {}
if wide == "unbalanced_attention":
for i in range(3):
specs[f"wide.{i}"] = _mla(1024)
specs["wide.odd"] = _mla(800)
else:
specs["wide.state"] = MambaSpec(
block_size=16,
shapes=((51_200,),),
dtypes=(torch.bfloat16,),
tp_replicated=True,
)
# Mixed and balanced, but far narrower than the bucket above.
for i in range(10):
specs[f"narrow.a.{i}"] = FullAttentionSpec(
block_size=16, num_kv_heads=1, head_size=8, dtype=torch.uint8
)
specs[f"narrow.b.{i}"] = FullAttentionSpec(
block_size=16, num_kv_heads=1, head_size=16, dtype=torch.uint8
)
for i in range(100):
specs[f"gdn.{i}"] = MambaSpec(
block_size=16, shapes=((512,),), dtypes=(torch.bfloat16,)
)
groups = _get_packed_kv_cache_groups(config, specs)
gdn = [g for g in groups if g.layer_names[0].startswith("gdn.")]
assert _get_kv_cache_bytes_per_block(groups) == sum(
specs[name].page_size_bytes for name in specs if name.startswith("wide.")
)
# That block holds every GDN state at once, so the repeat pattern alone
# decides the split; the cap must not add groups on top of it.
assert len(gdn) == 10
class TestDensePacking:
def test_bytes_per_block_is_largest_group(self):
groups, g1, g2 = _mixed_page_groups()
+39 -48
View File
@@ -12,7 +12,7 @@ import pytest
import torch
import vllm.v1.core.kv_cache_utils as kv_cache_utils
from vllm.config import ModelConfig, SchedulerConfig, VllmConfig
from vllm.config import CacheConfig, ModelConfig, SchedulerConfig, VllmConfig
from vllm.config.kv_events import KVEventsConfig
from vllm.lora.request import LoRARequest
from vllm.multimodal.inputs import (
@@ -37,7 +37,6 @@ from vllm.v1.core.kv_cache_utils import (
get_kv_cache_groups,
get_max_concurrency_for_kv_cache_config,
get_request_block_hasher,
group_and_unify_kv_cache_specs,
hash_block_tokens,
init_none_hash,
is_kv_cache_spec_uniform,
@@ -2226,38 +2225,6 @@ def new_swa_mla_spec(head_size=576, sliding_window=128, model_version=None):
)
def test_group_and_unify_kv_cache_specs_no_swa_mla_returns_none():
# Without any SlidingWindowMLASpec the function does not apply.
specs = {"mla.0": new_mla_spec(), "mla.1": new_mla_spec()}
assert group_and_unify_kv_cache_specs(specs) is None
def test_group_and_unify_kv_cache_specs_uniform_page_size_returns_none():
# A non-DeepseekV4 model that mixes full MLA and sliding-window MLA layers
# with a uniform page size must not fall into the DeepseekV4 tuple-packing
# path; it should defer to the generic uniform-page-size grouping instead.
mla_spec = new_mla_spec()
swa_spec = new_swa_mla_spec()
assert mla_spec.page_size_bytes == swa_spec.page_size_bytes
specs = {"mla.0": mla_spec, "mla.1": new_mla_spec(), "swa.0": swa_spec}
assert group_and_unify_kv_cache_specs(specs) is None
def test_group_and_unify_kv_cache_specs_mixed_page_size_groups():
# DeepseekV4-style: differing page sizes across MLA and sliding-window MLA
# layers do require tuple packing, so grouping must still be produced.
mla_spec = new_mla_spec()
swa_spec = new_swa_mla_spec(head_size=1024)
assert mla_spec.page_size_bytes != swa_spec.page_size_bytes
specs = {"mla.0": mla_spec, "mla.1": new_mla_spec(), "swa.0": swa_spec}
grouped = group_and_unify_kv_cache_specs(specs)
assert grouped is not None
# One MLA group plus one sliding-window MLA group.
assert len(grouped) == 2
layer_names = {name for g in grouped for name in g.kv_cache_specs}
assert layer_names == {"mla.0", "mla.1", "swa.0"}
def new_indexer_mla_spec(block_size=16):
# Sparse-attention indexer k_cache: an MLAAttentionSpec with a much smaller
# page size than the main MLA attention (uint8, small head), so their pages
@@ -2270,10 +2237,30 @@ def new_indexer_mla_spec(block_size=16):
)
def test_mixed_page_size_groups_use_spec_compatibility():
specs = {}
for i in range(3):
specs[f"mla.{i}"] = new_mla_spec()
specs[f"indexer.{i}"] = new_indexer_mla_spec()
specs.update({f"swa.{i}": new_swa_mla_spec(head_size=1024) for i in range(5)})
config = _grouping_config()
config.cache_config = CacheConfig()
config.cache_config.kv_cache_layout = "BLNHC"
groups = get_kv_cache_groups(config, specs)
assert len(groups) == 3
assert {name for group in groups for name in group.layer_names} == set(specs)
assert sorted(len(group.layer_names) for group in groups) == [2, 3, 6]
def _grouping_config():
cache_config = CacheConfig()
cache_config.kv_cache_layout = "LBNHC"
return SimpleNamespace(
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
speculative_config=None,
cache_config=cache_config,
)
@@ -3281,10 +3268,16 @@ def test_iter_layer_specs_returns_group_members():
assert list(iter_layer_specs(wrapped)) == [full, mla]
def _spec_decode_grouping_config(method="dspark"):
def _spec_decode_grouping_config(method="dspark", model_type=None):
"""Grouping config with an EAGLE-family speculative method enabled."""
return SimpleNamespace(
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
cache_config=SimpleNamespace(
get_resolved_kv_cache_layout=lambda: SimpleNamespace(
is_block_outermost=True
)
),
model_config=SimpleNamespace(hf_config=SimpleNamespace(model_type=model_type)),
speculative_config=SimpleNamespace(
method=method,
use_eagle=lambda: True,
@@ -3322,13 +3315,12 @@ def test_draft_group_annotated_on_hybrid_general_path():
flagged = [g for g in groups if g.is_eagle_group]
assert len(flagged) == 1
assert flagged[0].layer_names == ["draft.attn.0"]
assert "draft.attn.0" in flagged[0].layer_names
def test_mamba_groups_never_flagged_even_when_draft_shares_a_group():
# MLAAttentionSpec.merge ORs non_causal_multi_token_decode rather than
# requiring equality, so a draft layer can share a group with target
# layers; that combined group still holds volatile draft KV and must be
# Packed uniform-type groups can contain distinct target and draft layer
# specs; the combined group still holds volatile draft KV and must be
# flagged. What must never happen is a Mamba group being flagged: that
# widens its lookup window to two consecutive chunks, which align-mode
# checkpointing never produces, zeroing every lookup.
@@ -3347,10 +3339,8 @@ def test_mamba_groups_never_flagged_even_when_draft_shares_a_group():
def test_draft_group_not_annotated_without_spec_decode():
# The marker alone must not flag anything; the eagle semantics only apply
# when a speculative method is actually enabled.
config = SimpleNamespace(
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
speculative_config=None,
)
config = _spec_decode_grouping_config()
config.speculative_config = None
groups = get_kv_cache_groups(config, _hybrid_specs_with_draft(draft=True))
assert not any(g.is_eagle_group for g in groups)
@@ -3396,12 +3386,13 @@ def _deepseek_v4_specs(model_version="deepseek_v4"):
}
def test_deepseek_v4_draft_group_annotated_on_group_and_unify_path():
def test_deepseek_v4_draft_group_annotated_on_packed_path():
# DeepseekV4's MTP block reuses the target's decoder layer, so its spec
# carries no draft marker and only the positional rule can find it. This
# pins the pre-existing behaviour that the unified annotator must preserve.
groups = get_kv_cache_groups(
_spec_decode_grouping_config(method="mtp"), _deepseek_v4_specs()
_spec_decode_grouping_config(method="mtp", model_type="deepseek_v4"),
_deepseek_v4_specs(model_version=None),
)
flagged = [g for g in groups if g.is_eagle_group]
@@ -3409,13 +3400,13 @@ def test_deepseek_v4_draft_group_annotated_on_group_and_unify_path():
assert "model.layers.3.self_attn.attn" in flagged[0].layer_names
def test_deepseek_v4_annotation_requires_model_version():
def test_deepseek_v4_annotation_requires_model_type():
# The positional rule is only sound for DeepseekV4, where the draft layer
# is known to be registered last. Without that model gate nothing may be
# flagged, however the grouping happens to fall out.
groups = get_kv_cache_groups(
_spec_decode_grouping_config(method="mtp"),
_deepseek_v4_specs(model_version=None),
_spec_decode_grouping_config(method="mtp", model_type="other"),
_deepseek_v4_specs(),
)
assert not any(g.is_eagle_group for g in groups)
+46
View File
@@ -1761,6 +1761,52 @@ def test_spec_decode_padding_skipped_for_diffusion():
assert r2.request_id not in out.scheduled_spec_decode_tokens
@pytest.mark.parametrize("aligned_tokens", [1, 2, 3])
def test_spec_decode_padding_dropped_when_recurrent_alignment_clips(
aligned_tokens: int,
):
"""A clipped padded request must drop its padding, not shorten it.
The sampler derives a request's row window from its draft count, so a
padded request that keeps fewer than 1 + num_spec query rows would select
the preceding request's hidden states.
"""
num_spec = 3
scheduler = create_scheduler(
num_speculative_tokens=num_spec,
enable_prefix_caching=True,
block_size=16,
)
r1, r2 = create_requests(
num_requests=2, num_tokens=33, same_prompt=True, max_tokens=16
)
scheduler.add_request(r1)
out = scheduler.schedule()
assert out.num_scheduled_tokens[r1.request_id] == 33
_model_output(scheduler, out, [[100]])
scheduler.update_draft_token_ids(DraftTokenIds([r1.request_id], [[1, 2, 3]]))
# Model the recurrent align split that clips a padded 1+3 prompt tail at
# the next cache boundary. The real QSA block size produces all three cases
# depending on prompt length modulo four.
scheduler.need_mamba_block_aligned_split = True
original_split = scheduler._mamba_block_aligned_split
def align_prompt_tail(request, num_new_tokens, *args):
if request is r2:
return aligned_tokens
return original_split(request, num_new_tokens, *args)
scheduler._mamba_block_aligned_split = align_prompt_tail
scheduler.add_request(r2)
out = scheduler.schedule()
assert out.scheduled_spec_decode_tokens[r1.request_id] == [1, 2, 3]
assert out.num_scheduled_tokens[r2.request_id] == 1
assert r2.request_id not in out.scheduled_spec_decode_tokens
def test_spec_decode_padding_skipped_with_prefill_in_batch():
"""Padding is skipped when the batch contains a prefill chunk: the batch is
already mixed/non-uniform, so padding a new decode request buys nothing.
@@ -14,6 +14,7 @@ from vllm.v1.core.kv_cache_utils import (
)
from vllm.v1.core.single_type_kv_cache_manager import (
ChunkedLocalAttentionManager,
CircularBufferManager,
FullAttentionManager,
MambaManager,
RSWAManager,
@@ -21,6 +22,7 @@ from vllm.v1.core.single_type_kv_cache_manager import (
)
from vllm.v1.kv_cache_interface import (
ChunkedLocalAttentionSpec,
CircularBufferSpec,
FullAttentionSpec,
MambaSpec,
RSWASpec,
@@ -136,6 +138,67 @@ def get_chunked_local_attention_manager(
)
def test_circular_buffer_allocates_one_block_for_the_request_lifetime():
block_size = 4
spec = CircularBufferSpec(
block_size=block_size,
num_kv_heads=1,
head_size=8,
dtype=torch.bfloat16,
)
block_pool = BlockPool(
num_gpu_blocks=10, enable_caching=True, hash_block_size=block_size
)
manager = CircularBufferManager(
spec,
block_pool=block_pool,
enable_caching=True,
kv_cache_group_id=0,
scheduler_block_size=block_size,
needs_kv_cache_zeroing=True,
max_admission_blocks_per_request=1,
)
request_id = "request"
assert (
manager.get_num_blocks_to_allocate(
request_id,
num_tokens=128,
new_computed_blocks=[block_pool.blocks[1]],
total_computed_tokens=0,
num_local_computed_tokens=0,
num_tokens_main_model=128,
)
== 1
)
blocks = manager.allocate_new_blocks(
request_id, num_tokens=128, num_tokens_main_model=128
)
assert len(blocks) == 1
assert not manager.records_new_block_ids
assert manager.take_new_block_ids() == []
for num_tokens in (256, 1024):
assert (
manager.get_num_blocks_to_allocate(
request_id,
num_tokens,
[],
total_computed_tokens=num_tokens - 1,
num_local_computed_tokens=0,
num_tokens_main_model=num_tokens,
)
== 0
)
assert manager.allocate_new_blocks(request_id, num_tokens, num_tokens) == []
manager.cache_blocks(request_id, 1024)
manager.remove_skipped_blocks(request_id, 1024)
assert manager.get_num_common_prefix_blocks(request_id) == 0
assert manager.get_num_skipped_tokens(1024) == 0
assert manager.req_to_blocks[request_id] == blocks
def test_sliding_window_records_new_blocks_for_zeroing():
block_size = 2
spec = SlidingWindowSpec(
@@ -234,7 +234,8 @@ def get_fake_execute_model_fn(original_execute_model_fn: Callable):
iter(scheduler_output.num_scheduled_tokens.values())
)
assert num_scheduled_tokens == cur_step_action.num_scheduled_tokens
mamba_group_ids, mamba_spec = get_mamba_groups(self.kv_cache_config)
mamba_groups = get_mamba_groups(self.kv_cache_config)
mamba_spec, mamba_group_ids = next(iter(mamba_groups.items()))
mamba_group_id = mamba_group_ids[0]
mamba_layer_name = self.kv_cache_config.kv_cache_groups[
mamba_group_id
@@ -321,7 +322,8 @@ def get_fake_process_mamba_fn(
assert len(copy_info[0]) == len(copy_info[1]) == len(copy_info[2]) == 0
else:
assert len(copy_info[0]) == len(copy_info[1]) == len(copy_info[2]) == 2
mamba_group_ids, mamba_spec = get_mamba_groups(kv_cache_config)
mamba_groups = get_mamba_groups(kv_cache_config)
mamba_spec, mamba_group_ids = next(iter(mamba_groups.items()))
mamba_group_id = mamba_group_ids[0]
mamba_layer_name = kv_cache_config.kv_cache_groups[
mamba_group_id
@@ -936,12 +938,12 @@ def _run_mamba_prefix_cache_mrv1(
cleanup_dist_env_and_memory()
@create_new_process_for_each_test()
@create_new_process_for_each_test("spawn")
def test_mamba_prefix_cache_mrv1(monkeypatch: pytest.MonkeyPatch):
_run_mamba_prefix_cache_mrv1(monkeypatch, async_scheduling=False)
@create_new_process_for_each_test()
@create_new_process_for_each_test("spawn")
def test_mamba_prefix_cache_mrv1_async(monkeypatch: pytest.MonkeyPatch):
_run_mamba_prefix_cache_mrv1(monkeypatch, async_scheduling=True)
@@ -975,11 +977,12 @@ def _run_mamba_prefix_cache_mrv2(
forward_context = (
model_state.vllm_config.compilation_config.static_forward_context
)
group_ids, _ = get_mamba_groups(kv_cache_config)
for group_id in group_ids:
block_table = block_tables[group_id]
for layer_name in kv_cache_config.kv_cache_groups[group_id].layer_names:
yield forward_context[layer_name].kv_cache[-1], block_table
mamba_groups = get_mamba_groups(kv_cache_config)
for group_ids in mamba_groups.values():
for group_id in group_ids:
block_table = block_tables[group_id]
for layer_name in kv_cache_config.kv_cache_groups[group_id].layer_names:
yield forward_context[layer_name].kv_cache[-1], block_table
def temporal_block(temporal_state, block_table, col):
# Resolving the block id for assertions is a deliberate D2H.
@@ -15,6 +15,8 @@ from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
HiddenStateCacheSpec,
KVCacheGroupSpec,
KVCacheLayout,
MambaSpec,
MLAAttentionSpec,
SlidingWindowMLASpec,
)
@@ -99,12 +101,12 @@ def test_get_block_size_falls_back_to_cache_config_when_no_kv_cache_config():
assert block_size == 16
# ---- MLA-verifier absorption ------------------------------------------------
# ---- Packed MLA grouping ----------------------------------------------------
def test_find_group_id_errors_clearly_when_absorbed_by_mla_swa_verifier():
# HiddenStateCacheSpec subclasses MLAAttentionSpec, so an MLA + sliding-
# window MLA verifier absorbs it into the MLA group instead of isolating it.
def test_hidden_state_group_isolated_from_packed_mla_groups():
# HiddenStateCacheSpec subclasses MLAAttentionSpec, but grouping must pull
# it out before packing compatible MLA cache specs.
dt = torch.bfloat16
spec = {
"layers.0.mla": MLAAttentionSpec(
@@ -116,11 +118,54 @@ def test_find_group_id_errors_clearly_when_absorbed_by_mla_swa_verifier():
"cache_only_layers.61": _hidden(64),
}
vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(
get_resolved_kv_cache_layout=lambda: KVCacheLayout.BLHNC
),
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
speculative_config=None,
)
groups = get_kv_cache_groups(vllm_config, spec)
assert not any(isinstance(g.kv_cache_spec, HiddenStateCacheSpec) for g in groups)
hidden_group_ids = [
i
for i, group in enumerate(groups)
if isinstance(group.kv_cache_spec, HiddenStateCacheSpec)
]
assert len(hidden_group_ids) == 1
cfg = SimpleNamespace(kv_cache_groups=groups)
with pytest.raises(ValueError, match="MLA verifiers are unsupported"):
ExampleHiddenStatesConnector._find_cache_kv_group_id(cfg)
assert (
ExampleHiddenStatesConnector._find_cache_kv_group_id(cfg) == hidden_group_ids[0]
)
def test_hidden_state_group_isolated_from_packed_mixed_page_groups():
# Packed grouping keeps groups with unequal page sizes (blocks are strided
# by the widest group), so the hidden group is appended without padding.
hidden = _hidden(16)
spec = {
"layers.0.attn": _full(16),
"layers.1.mamba": MambaSpec(
block_size=16, shapes=((1024,),), dtypes=(torch.float32,)
),
"cache_only_layers.61": hidden,
}
vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(
get_resolved_kv_cache_layout=lambda: KVCacheLayout.BLHNC
),
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
speculative_config=None,
)
groups = get_kv_cache_groups(vllm_config, spec)
page_sizes = {g.kv_cache_spec.page_size_bytes for g in groups}
assert len(page_sizes) > 1, "spec must exercise the packed grouping path"
hidden_group_ids = [
i
for i, group in enumerate(groups)
if isinstance(group.kv_cache_spec, HiddenStateCacheSpec)
]
assert len(hidden_group_ids) == 1
assert groups[hidden_group_ids[0]].kv_cache_spec == hidden
cfg = SimpleNamespace(kv_cache_groups=groups)
assert (
ExampleHiddenStatesConnector._find_cache_kv_group_id(cfg) == hidden_group_ids[0]
)
@@ -865,6 +865,10 @@ def _make_mock_worker_for_desc_ids(
worker._group_spec_types = group_spec_types
worker.block_len_per_layer = block_len_per_layer or [100]
worker._conv_decomp = None
worker._is_csa_linear = False
worker._ssm_region_indices = [0] if has_mamba else []
worker._ple_group_index = None
worker._ple_region_index = None
if has_mamba:
from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import ( # noqa: E501
MambaConvSplitInfo,
@@ -1823,6 +1827,7 @@ def test_push_write_hybrid_mla_replicates_attention():
worker.shutdown = lambda: None # skeleton worker: silence __del__
worker.use_mla = True
worker._has_mamba = True
worker._is_csa_linear = False
worker._group_spec_types = (MLAAttentionSpec, MambaSpec)
worker.transfer_topo = MagicMock()
worker.transfer_topo.tp_ratio.return_value = -2
@@ -645,3 +645,302 @@ def test_mismatched_mla_kernel_page_rejected_for_mla_hybrid():
meta_r.block_lens = [x // 2 for x in worker.block_len_per_layer]
with pytest.raises((AssertionError, RuntimeError)):
worker.add_remote_agent(meta_r, remote_tp_rank=0, remote_tp_size=2)
def _make_csa_linear_ple_worker(scratch_aliases: str = "compressed"):
"""``scratch_aliases`` selects which pages the compressor ring overlays."""
from unittest.mock import MagicMock
from vllm.config import set_current_vllm_config
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker as bw
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
NixlConnectorWorker,
)
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheTensor,
MambaSpec,
MLAAttentionSpec,
UniformTypeKVCacheSpecs,
)
main_kv_specs = {
f"main_kv.{index}": FullAttentionSpec(
block_size=4,
num_kv_heads=1,
head_size=16,
dtype=torch.float16,
)
for index in range(2)
}
compressed_specs = {
f"compressed.{index}": MLAAttentionSpec(
block_size=4,
num_kv_heads=1,
head_size=64,
dtype=torch.float16,
tokens_per_state=2,
)
for index in range(2)
}
compressor_state_specs = {
f"compressor_state.{index}": CircularBufferSpec(
block_size=2,
num_kv_heads=1,
head_size=32,
dtype=torch.float16,
)
for index in range(2)
}
sharded_mamba_spec = MambaSpec(
block_size=1,
shapes=((8, 3), (1, 4, 4)),
dtypes=(torch.float16, torch.float32),
page_size_padded=256,
mamba_type=MambaAttentionBackendEnum.GDN_ATTN,
)
ple_spec = MambaSpec(
block_size=1,
shapes=((6, 3),),
dtypes=(torch.float16,),
page_size_padded=256,
mamba_type=MambaAttentionBackendEnum.SHORT_CONV,
tp_replicated=True,
)
compressed_sparse_specs = {
name: spec
for index in range(2)
for name, spec in (
(f"main_kv.{index}", main_kv_specs[f"main_kv.{index}"]),
(f"compressed.{index}", compressed_specs[f"compressed.{index}"]),
)
}
def group(specs):
uniform = UniformTypeKVCacheSpecs.from_specs(specs)
assert uniform is not None
return KVCacheGroupSpec(list(specs), uniform)
tensor_regions: tuple[tuple[str, ...], ...]
if scratch_aliases == "compressed":
tensor_regions = (
("main_kv.0", "mamba.sharded", "mamba.ple"),
("main_kv.1",),
("compressed.0", "compressor_state.0"),
("compressed.1", "compressor_state.1"),
)
else:
assert scratch_aliases == "main_kv"
tensor_regions = (
("main_kv.0", "mamba.sharded", "mamba.ple", "compressor_state.0"),
("main_kv.1", "compressor_state.1"),
("compressed.0",),
("compressed.1",),
)
region_size = 512
page_size = 256
kv_cache_config = KVCacheConfig(
num_blocks=2,
kv_cache_tensors=[
KVCacheTensor(
size=len(tensor_regions) * region_size,
layers=[layer_name],
layer_stride=region_size,
block_stride=page_size,
offset=region_index * region_size,
)
for region_index, layer_names in enumerate(tensor_regions)
for layer_name in layer_names
],
kv_cache_groups=[
group(compressed_sparse_specs),
group(compressor_state_specs),
KVCacheGroupSpec(["mamba.sharded"], sharded_mamba_spec),
KVCacheGroupSpec(["mamba.ple"], ple_spec),
],
)
vllm_config = create_vllm_config(block_size=4)
vllm_config.cache_config.enable_prefix_caching = False
vllm_config.kv_transfer_config.kv_buffer_device = "cuda"
fake_backend = MagicMock()
fake_backend.get_supported_kernel_block_sizes.return_value = [4]
fake_backend.get_name.return_value = "TEST_ATTN"
fake_backend.full_cls_name.return_value = "test.AttentionBackend"
fake_platform = MagicMock()
fake_platform.device_type = "cuda"
fake_platform.get_nixl_memory_type.return_value = "VRAM"
with (
patch.object(bw, "NixlWrapper", _RecordingNixl),
patch.object(bw, "get_tensor_model_parallel_rank", return_value=0),
patch.object(bw, "get_tensor_model_parallel_world_size", return_value=1),
patch.object(bw, "get_current_attn_backends", return_value=[fake_backend]),
patch.object(bw, "current_platform", fake_platform),
patch(
"vllm.model_executor.layers.mamba.mamba_utils.get_conv_state_layout",
return_value="DS",
),
set_current_vllm_config(vllm_config),
):
worker = NixlConnectorWorker(vllm_config, "local-engine", kv_cache_config)
tensors = [torch.zeros((2, 256), dtype=torch.uint8) for _ in range(4)]
worker.register_kv_caches(
{
layer_name: tensors[region_index]
for region_index, layer_names in enumerate(tensor_regions)
for layer_name in layer_names
}
)
return worker
@pytest.mark.cpu_test
def test_csa_linear_registration_discovers_shared_regions():
worker = _make_csa_linear_ple_worker()
assert worker._ssm_region_indices == [0]
assert worker._ple_group_index == 3
assert worker._ple_region_index == 0
assert worker._region_is_mla == [False, False, True, True]
assert worker._scratch_region_indices == [2, 3]
@pytest.mark.cpu_test
def test_csa_linear_scratch_descs_follow_the_pages_they_overlay():
"""The scratch ring is addressed by the regions it actually registered in,
not by inferring that it must sit on the MLA pages: overlaying the main KV
pages instead moves its descriptors with it."""
worker = _make_csa_linear_ple_worker(scratch_aliases="main_kv")
# Same MLA regions as before, but the ring now lives in the non-MLA ones.
assert worker._region_is_mla == [False, False, True, True]
assert worker._scratch_region_indices == [0, 1]
desc_ids = worker._compute_desc_ids(
block_ids=([1], [0], [1], [1]),
dst_num_blocks=2,
block_size_ratio=None,
physical_blocks_per_logical=1,
)
# Scratch group (block 0) now maps to regions 0 and 1 -> descs 0 and 2,
# where the compressed-page placement gave 4 and 6.
assert desc_ids.tolist()[4:6] == [0, 2]
@pytest.mark.cpu_test
def test_csa_linear_selects_compressed_sparse_compressor_state_and_mamba_regions():
worker = _make_csa_linear_ple_worker()
desc_ids = worker._compute_desc_ids(
block_ids=([1], [0], [1], [1]),
dst_num_blocks=2,
block_size_ratio=None,
physical_blocks_per_logical=1,
)
assert desc_ids.tolist() == [
1,
3,
5,
7,
4,
6,
9,
11,
13,
15,
17,
]
mamba_only = worker._compute_desc_ids(
block_ids=([], [], [1], [1]),
dst_num_blocks=2,
block_size_ratio=None,
physical_blocks_per_logical=1,
)
assert mamba_only.tolist() == [9, 11, 13, 15, 17]
@pytest.mark.cpu_test
def test_csa_linear_ple_descriptor_is_not_split():
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
TPMapping,
)
worker = _make_csa_linear_ple_worker()
bases = [0x10000, 0x20000, 0x30000, 0x40000]
attention = worker._build_fa_local(bases, block_size_ratio=1)
mamba = worker._build_mamba_local(bases)
assert attention.shape == (8, 3)
assert mamba.shape == (10, 3)
assert mamba[-2:, 0].tolist() == [0x10000, 0x10100]
assert mamba[-2:, 1].tolist() == [256, 256]
mapping = TPMapping(
source_ranks_per_group=((0,), (0,), (0, 1), (0, 1)),
all_source_ranks=(0, 1),
rank_to_attention_slot={0: 0, 1: 0},
rank_offset_factor=0,
)
splits = list(
worker._build_local_splits_from_plan(
mapping,
np.concatenate([attention, mamba]),
num_fa_descs=len(attention),
)
)
assert len(splits) == 2
assert [row[1] for row in splits[0][-2:]] == [256, 256]
assert [row[1] for row in splits[1][-2:]] == [256, 256]
assert splits[0][-3][1] * 2 == mamba[-3, 1]
assert splits[1][-3][1] * 2 == mamba[-3, 1]
@pytest.mark.cpu_test
def test_csa_linear_remote_ple_is_copied_whole():
from types import SimpleNamespace
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
NixlAgentMetadata,
)
worker = _make_csa_linear_ple_worker()
metadata = NixlAgentMetadata(
engine_id="remote",
agent_metadata=b"agent",
kv_caches_base_addr=[
0x10000,
0x20000,
0x30000,
0x40000,
],
device_id=0,
num_blocks=2,
block_lens=[256] * 4,
block_strides=[256] * 4,
kv_cache_layout="HND",
block_size=4,
ssm_sizes=(24, 32),
attn_backend_name="test",
physical_blocks_per_logical_kv_block=1,
)
descriptors = worker._build_mamba_remote(
metadata,
tp_ratio=-2,
transfer_info=SimpleNamespace(remote_physical_blocks_per_logical=1),
)
assert descriptors.shape == (10, 3)
assert descriptors[-2:, 0].tolist() == [0x10000, 0x10100]
assert descriptors[-2:, 1].tolist() == [256, 256]
metadata.block_lens[0] = 128
with pytest.raises(ValueError, match="PLE pages require identical"):
worker._build_mamba_remote(
metadata,
tp_ratio=-2,
transfer_info=SimpleNamespace(remote_physical_blocks_per_logical=1),
)
@@ -359,6 +359,7 @@ class _StubWriterWorker(NixlPushConnectorWorker):
w._physical_blocks_per_logical_kv_block = 1
# Single non-hybrid attention group, matching the stub block id lists.
w._has_mamba = False
w._is_csa_linear = False
w._group_spec_types = (FullAttentionSpec,)
w._engine_ttl = 0.0
w._engine_last_active = {}
@@ -1198,6 +1199,75 @@ class TestPushWriterMlaReplication:
# engine thread never sees a partial set and double-frees the request.
assert sorted(w._sending_transfers["p-req"]) == [1000, 1001]
def test_csa_linear_writes_every_group_to_each_target(self):
from types import SimpleNamespace
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
RemoteMeta,
ReqMeta,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
TPMapping,
)
engine_id = "decode-engine"
w = _StubWriterWorker.fresh()
w.world_size = 2
w.use_mla = False
w._has_mamba = True
w._is_csa_linear = True
w.nixl_wrapper = MagicMock()
w.transfer_topo = MagicMock()
w.transfer_topo.get_engine_info.return_value = SimpleNamespace(
remote_tp_size=4,
remote_block_size=16,
remote_physical_blocks_per_logical=1,
)
w.transfer_topo.tp_ratio.return_value = -2
w.transfer_topo.handshake_target_ranks.return_value = (0, 1)
w.tp_mappings = {
engine_id: TPMapping(
source_ranks_per_group=((0,), (0,), (0, 1), (0, 1)),
all_source_ranks=(0, 1),
rank_to_attention_slot={0: 0, 1: 0},
rank_offset_factor=0,
)
}
w._logical_to_kernel_block_ids = lambda block_ids, ratio: block_ids
w.dst_xfer_side_handles = {engine_id: {0: 1000, 1: 1001}}
w.src_xfer_handles_by_tp_ratio = {(-2, 16): [2000, 2001]}
writes = []
def _fake_xfer(**kwargs):
writes.append(kwargs)
return 3000 + kwargs["read_spec"].remote_rank
w._xfer_blocks = _fake_xfer
block_ids = ([10], [20], [30], [40])
meta = ReqMeta(
local_block_ids=block_ids,
local_physical_block_ids=block_ids,
tp_size=2,
remote=RemoteMeta(
block_ids=([50], [60], [70], [80]),
host="",
port=0,
engine_id=engine_id,
request_id="d-req",
),
)
w._xfer_blocks_for_req(req_id="p-req", meta=meta)
assert [write["read_spec"].remote_rank for write in writes] == [0, 1]
assert [write["read_spec"].local_block_ids for write in writes] == [
list(block_ids),
list(block_ids),
]
assert [write["local_xfer_side_handle"] for write in writes] == [2000, 2001]
assert sorted(w._sending_transfers["p-req"]) == [3000, 3001]
class TestPushPrefixCaching:
"""Partial prefix-cache hit on D: D preallocates only its *uncomputed*
@@ -154,6 +154,9 @@ def _make_mock_worker_for_splits(group_spec_types):
worker.block_len_per_layer = []
worker.num_regions = 0
worker._region_is_mla = []
worker._conv_decomp = SimpleNamespace(local_conv_offsets=())
worker._ssm_region_indices = [0] if MambaSpec in group_spec_types else []
worker._ple_region_index = None
return worker
@@ -282,3 +285,31 @@ class TestMambaPlanSplitHandles:
with pytest.raises(AssertionError, match="Head-sharded"):
list(worker._build_local_splits_from_plan(plan, src_blocks_data, 2, 2))
@pytest.mark.parametrize(
("total_kv_heads", "local_tp", "remote_tp", "compatible"),
[
# 2 heads: TP1 packs both heads into a page, TP >= 2 packs one.
(2, 1, 2, False),
(2, 2, 1, False),
(2, 1, 1, True),
(2, 2, 4, True),
(2, 4, 2, True),
# 4 heads: the boundary moves with the head count.
(4, 2, 4, False),
(4, 2, 2, True),
(4, 4, 8, True),
],
)
def test_csa_linear_tp_layout_boundary(total_kv_heads, local_tp, remote_tp, compatible):
worker = object.__new__(NixlConnectorWorker)
worker.world_size = local_tp
worker._is_csa_linear = True
worker.transfer_topo = SimpleNamespace(total_num_kv_heads=total_kv_heads)
if compatible:
worker._validate_csa_linear_tp_layout(remote_tp)
else:
with pytest.raises(ValueError, match="KV-head sharding boundary"):
worker._validate_csa_linear_tp_layout(remote_tp)
@@ -104,6 +104,47 @@ def _make_speculator(
return speculator
@pytest.mark.parametrize(("hc_mult", "expected"), [(None, 64), (4, 256)])
def test_speculator_uses_draft_model_hidden_size(monkeypatch, hc_mult, expected):
# Qwen4Exp targets expose multi-stream HC residuals to the drafter.
monkeypatch.setattr(base_spec_module, "_target_feeds_hc_residual", lambda _: True)
hf_config = SimpleNamespace()
if hc_mult is not None:
hf_config.hc_mult = hc_mult
draft_model_config = SimpleNamespace(
hf_config=hf_config,
get_hidden_size=lambda: 64,
get_vocab_size=lambda: 32,
)
speculative_config = SimpleNamespace(
method="mtp",
num_speculative_tokens=3,
draft_model_config=draft_model_config,
use_local_argmax_reduction=False,
draft_sample_method="greedy",
)
vllm_config = SimpleNamespace(
speculative_config=speculative_config,
scheduler_config=SimpleNamespace(
max_num_seqs=2,
max_num_batched_tokens=8,
),
model_config=SimpleNamespace(
max_model_len=32,
dtype=torch.float32,
use_fp64_gumbel=False,
),
parallel_config=SimpleNamespace(
data_parallel_size=1,
data_parallel_rank=0,
),
)
speculator = _TestSpeculator(vllm_config, torch.device("cpu"))
assert speculator.hidden_size == expected
def test_mm_support_configured_after_model_load(monkeypatch):
target_model_config = object()
draft_model_config = object()
@@ -197,9 +238,11 @@ def test_load_model_disables_mm_support_for_text_only_drafter(monkeypatch):
assert not speculator.supports_mm_inputs
assert warning_messages == [
"Draft model _TextOnlyDraftModel does not support external multimodal "
"embeddings. Embeddings from the target model will not be passed to the "
"drafter; using text-only draft inputs instead."
(
"Draft model _TextOnlyDraftModel does not support external multimodal "
"embeddings. Embeddings from the target model will not be passed to the "
"drafter; using text-only draft inputs instead."
)
]
+36
View File
@@ -132,6 +132,42 @@ def test_block_tables_apply_staged_writes_single_group():
)
def test_block_tables_skip_custom_slot_mapping_groups():
device = torch.device("cuda")
block_tables = BlockTables(
block_sizes=[8, 262144],
max_num_reqs=1,
max_num_batched_tokens=4,
max_num_blocks_per_group=[1, 1],
device=device,
kernel_block_sizes=[8, 262144],
slot_mapping_enabled=[False, True],
)
block_tables.append_block_ids(
req_index=0,
new_block_ids=([7], [12]),
overwrite=True,
)
block_tables.apply_staged_writes()
idx_mapping = torch.tensor([0], dtype=torch.int32, device=device)
query_start_loc = torch.tensor([0, 2], dtype=torch.int32, device=device)
positions = torch.tensor([153797, 165757], dtype=torch.int64, device=device)
slot_mappings = block_tables.compute_slot_mappings(
idx_mapping,
query_start_loc,
positions,
num_tokens_padded=2,
)
torch.accelerator.synchronize()
assert slot_mappings[0].tolist() == [-1, -1]
assert slot_mappings[1].tolist() == [
12 * 262144 + 153797,
12 * 262144 + 165757,
]
@pytest.mark.parametrize("cp_rank", range(4))
def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
"""DCP interleave is expressed in logical-block token coordinates."""
@@ -8,15 +8,107 @@ import torch
import vllm.v1.worker.gpu.model_runner as model_runner_module
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
MambaSpec,
UniformTypeKVCacheSpecs,
)
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
def test_qsa_circular_group_uses_custom_slot_mapping(monkeypatch):
runner = GPUModelRunner.__new__(GPUModelRunner)
runner.max_model_len = 262144
runner.is_encoder_decoder = False
runner.dcp_size = 1
runner.dcp_rank = 0
runner.cp_interleave = 1
runner.cache_config = SimpleNamespace(enable_prefix_caching=True)
parallel_config = SimpleNamespace(
decode_context_parallel_size=1,
cp_kv_cache_interleave_size=1,
)
runner.parallel_config = parallel_config
runner.vllm_config = SimpleNamespace(
parallel_config=parallel_config,
cache_config=SimpleNamespace(mamba_cache_mode="none"),
)
runner.model_state = SimpleNamespace(
get_additional_cg_support=lambda: (),
num_new_sampled_tokens_per_step=1,
)
runner.speculator = None
runner.req_states = []
runner.input_buffers = SimpleNamespace(query_start_loc=None)
runner.vocab_size = 1
runner.max_num_reqs = 1
runner.max_num_tokens = 2
runner.device = torch.device("cuda")
raw_spec = CircularBufferSpec(
block_size=8,
num_kv_heads=1,
head_size=128,
dtype=torch.bfloat16,
)
compressed_spec = FullAttentionSpec(
block_size=262144,
num_kv_heads=1,
head_size=128,
dtype=torch.bfloat16,
)
kv_cache_config = KVCacheConfig(
num_blocks=1,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
layer_names=["raw"],
kv_cache_spec=UniformTypeKVCacheSpecs(
block_size=8,
kv_cache_specs={"raw": raw_spec},
),
),
KVCacheGroupSpec(layer_names=["compressed"], kv_cache_spec=compressed_spec),
],
)
class FakeAttnCGSupport:
def narrow(self, *args):
return self
attn_cg_support = FakeAttnCGSupport()
monkeypatch.setattr(
model_runner_module,
"init_attn_backend",
lambda *args: ([], attn_cg_support, [8, 262144]),
)
monkeypatch.setattr(
model_runner_module,
"maybe_create_adaptive_verification_manager",
lambda **kwargs: None,
)
captured = {}
class BlockTablesCaptured(Exception):
pass
def capture_block_tables(**kwargs):
captured.update(kwargs)
raise BlockTablesCaptured
monkeypatch.setattr(model_runner_module, "BlockTables", capture_block_tables)
with pytest.raises(BlockTablesCaptured):
runner.initialize_kv_cache(kv_cache_config)
assert captured["max_num_blocks_per_group"] == [1, 1]
assert captured["slot_mapping_enabled"] == [False, True]
@pytest.mark.parametrize(
("mamba_cache_mode", "num_speculative_blocks", "expected"),
[
+15 -2
View File
@@ -20,6 +20,7 @@ from vllm.config.vllm import VllmConfig
from vllm.platforms import current_platform
from vllm.utils.math_utils import cdiv
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
@@ -53,6 +54,18 @@ def _attention_group() -> KVCacheGroupSpec:
)
def _circular_group() -> KVCacheGroupSpec:
return KVCacheGroupSpec(
["circular"],
CircularBufferSpec(
block_size=BLOCK_SIZE,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
)
def _mamba_group(mamba_cache_mode: str) -> KVCacheGroupSpec:
# Name carries the mode so several groups can coexist in one config.
return KVCacheGroupSpec(
@@ -212,8 +225,7 @@ def test_warmup_reserves_mamba_speculative_blocks(mamba_cache_mode):
def _hybrid_kv_cache_config(num_blocks: int) -> KVCacheConfig:
"""Full attention plus one Mamba group per cache mode, so a single manager
exercises every branch `_reserved_block_count` has.
"""Attention, circular, and Mamba groups exercise every reservation branch.
"none" and "all" reach the same branch of both `_reserved_block_count` and
`MambaManager`, which tests only for "align". They are still both listed:
@@ -226,6 +238,7 @@ def _hybrid_kv_cache_config(num_blocks: int) -> KVCacheConfig:
kv_cache_tensors=[],
kv_cache_groups=[
_attention_group(),
_circular_group(),
_mamba_group("none"),
_mamba_group("all"),
_mamba_group("align"),
+188 -12
View File
@@ -1,6 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from dataclasses import dataclass
from typing import Any
from unittest.mock import MagicMock, patch
@@ -10,11 +9,20 @@ import pytest
import torch
from vllm.model_executor.layers.mamba.mamba_utils import (
MambaStateCopyFunc,
MambaStateCopyFuncsByType,
get_conv_copy_spec,
get_temporal_copy_spec,
)
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput
from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheGroupSpec, MambaSpec
from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
MambaSpec,
UniformTypeKVCacheSpecs,
)
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
from vllm.v1.worker.mamba_utils import (
MambaCopyBuffers,
MambaSpecDecodeGPUContext,
@@ -22,17 +30,21 @@ from vllm.v1.worker.mamba_utils import (
batch_memcpy,
collect_mamba_copy_meta,
do_mamba_copy_block,
get_mamba_groups,
preprocess_mamba,
stage_postprocess_inputs_to_gpu,
)
MambaStateCopyFunc = Callable[..., Any]
# Conv + temporal copy specs, in the order the tests' MambaSpec shapes expect.
_COPY_FUNCS: tuple[MambaStateCopyFunc, ...] = (
_DEFAULT_COPY_FUNCS: tuple[MambaStateCopyFunc, ...] = (
get_conv_copy_spec,
get_temporal_copy_spec,
)
_COPY_FUNCS: MambaStateCopyFuncsByType = {
MambaAttentionBackendEnum.MAMBA2: _DEFAULT_COPY_FUNCS,
MambaAttentionBackendEnum.GDN_ATTN: _DEFAULT_COPY_FUNCS,
MambaAttentionBackendEnum.SHORT_CONV: (get_conv_copy_spec,),
}
def postprocess_mamba(
@@ -41,7 +53,7 @@ def postprocess_mamba(
input_batch: Any,
requests: dict[str, Any],
forward_context: dict[str, Any],
mamba_state_copy_funcs: tuple[MambaStateCopyFunc, ...],
mamba_state_copy_funcs: MambaStateCopyFuncsByType,
copy_bufs: "MambaCopyBuffers",
):
"""CPU reference for the align-mode postprocess.
@@ -147,7 +159,7 @@ def test_resumed_req_ids_cleared_from_mamba_state_idx():
input_batch,
{}, # requests
{}, # forward_context
(), # mamba_state_copy_funcs
{}, # mamba_state_copy_funcs
copy_bufs,
)
@@ -430,7 +442,10 @@ def _make_requests(
def _make_copy_bufs(
cfg: _TestConfig, kv_cache_config: KVCacheConfig, device: torch.device
cfg: _TestConfig,
kv_cache_config: KVCacheConfig,
device: torch.device,
copy_funcs: MambaStateCopyFuncsByType = _COPY_FUNCS,
) -> MambaCopyBuffers:
"""Create MambaCopyBuffers for the Python path."""
@@ -440,13 +455,16 @@ def _make_copy_bufs(
return MambaCopyBuffers.create(
max_num_reqs=cfg.max_num_reqs,
kv_cache_config=kv_cache_config,
copy_funcs=(get_conv_copy_spec, get_temporal_copy_spec),
copy_funcs=copy_funcs,
make_buffer=make_buffer,
)
def _make_gpu_ctx(
cfg: _TestConfig, kv_cache_config: KVCacheConfig, device: torch.device
cfg: _TestConfig,
kv_cache_config: KVCacheConfig,
device: torch.device,
copy_funcs: MambaStateCopyFuncsByType = _COPY_FUNCS,
) -> MambaSpecDecodeGPUContext:
"""Create MambaSpecDecodeGPUContext for the GPU path."""
@@ -456,12 +474,170 @@ def _make_gpu_ctx(
return MambaSpecDecodeGPUContext.create(
max_num_reqs=cfg.max_num_reqs,
kv_cache_config=kv_cache_config,
num_state_types=2,
copy_funcs=copy_funcs,
device=device,
make_buffer=make_buffer,
)
def test_mamba_groups_support_different_state_specs():
gdn_spec = MambaSpec(
block_size=16,
shapes=((4, 4), (2, 4, 4)),
dtypes=(torch.float16, torch.float32),
mamba_type=MambaAttentionBackendEnum.GDN_ATTN,
mamba_cache_mode="align",
num_speculative_blocks=3,
)
short_conv_spec = MambaSpec(
block_size=16,
shapes=((12, 8),),
dtypes=(torch.float16,),
mamba_type=MambaAttentionBackendEnum.SHORT_CONV,
mamba_cache_mode="align",
num_speculative_blocks=3,
)
kv_cache_config = KVCacheConfig(
num_blocks=8,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(["gdn.0", "gdn.1"], gdn_spec),
KVCacheGroupSpec(["ple.0"], short_conv_spec),
],
)
mamba_groups = get_mamba_groups(kv_cache_config)
assert mamba_groups == {gdn_spec: [0], short_conv_spec: [1]}
model_state = object.__new__(MambaHybridModelState)
model_state._mamba_group_ids = []
model_state._mamba_spec = None
group_ids, representative_spec = model_state._get_mamba_group_info(kv_cache_config)
assert group_ids == [0, 1]
assert representative_spec is gdn_spec
cfg = _TestConfig()
copy_bufs = _make_copy_bufs(cfg, kv_cache_config, torch.device("cpu"))
assert copy_bufs.src_ptrs.cpu.numel() == cfg.max_num_reqs * 5
assert copy_bufs.dst_ptrs.cpu.numel() == cfg.max_num_reqs * 5
assert copy_bufs.sizes.cpu.numel() == cfg.max_num_reqs * 5
ctx = _make_gpu_ctx(cfg, kv_cache_config, torch.device("cpu"))
assert ctx.num_states == 5
forward_context = {}
gdn_spec_layer_names = ["gdn.0", "gdn.1"]
for layer_name in gdn_spec_layer_names:
attention = MagicMock()
attention.kv_cache = (
torch.empty(8, 4, 4, dtype=torch.float16),
torch.empty(8, 2, 4, 4, dtype=torch.float32),
)
forward_context[layer_name] = attention
ple_attention = MagicMock()
ple_attention.kv_cache = (torch.empty(8, 12, 8, dtype=torch.float16),)
forward_context["ple.0"] = ple_attention
assert gdn_spec_layer_names == kv_cache_config.kv_cache_groups[0].layer_names
block_tables = [
torch.zeros(4, 8, dtype=torch.int32),
torch.zeros(4, 8, dtype=torch.int32),
]
ctx.initialize_from_forward_context(
kv_cache_config,
forward_context,
_COPY_FUNCS,
block_tables,
)
assert ctx.is_initialized
assert ctx.state_group_indices.tolist() == [0, 0, 0, 0, 1]
assert ctx.state_conv_widths.tolist() == [4, 0, 4, 0, 12]
def test_mamba_groups_support_mixed_specs_in_uniform_group():
gdn_spec = MambaSpec(
block_size=16,
shapes=((4, 4), (2, 4, 4)),
dtypes=(torch.float16, torch.float32),
mamba_type=MambaAttentionBackendEnum.GDN_ATTN,
mamba_cache_mode="align",
num_speculative_blocks=3,
)
short_conv_spec = MambaSpec(
block_size=16,
shapes=((12, 8),),
dtypes=(torch.float16,),
mamba_type=MambaAttentionBackendEnum.SHORT_CONV,
mamba_cache_mode="align",
num_speculative_blocks=3,
)
layer_specs = {
"gdn.0": gdn_spec,
"gdn.1": gdn_spec,
"ple.0": short_conv_spec,
}
kv_cache_config = KVCacheConfig(
num_blocks=8,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
list(layer_specs),
UniformTypeKVCacheSpecs(
block_size=16,
kv_cache_specs=layer_specs,
),
)
],
)
assert kv_cache_config.has_mamba_layers
assert get_mamba_groups(kv_cache_config) == {
gdn_spec: [0],
short_conv_spec: [0],
}
# The group is listed once per spec, so its id must not be repeated.
model_state = object.__new__(MambaHybridModelState)
model_state._mamba_group_ids = []
model_state._mamba_spec = None
group_ids, representative_spec = model_state._get_mamba_group_info(kv_cache_config)
assert group_ids == [0]
assert representative_spec is gdn_spec
cfg = _TestConfig()
copy_bufs = _make_copy_bufs(cfg, kv_cache_config, torch.device("cpu"))
assert copy_bufs.mamba_group_ids == [0]
assert copy_bufs.src_ptrs.cpu.numel() == cfg.max_num_reqs * 5
ctx = _make_gpu_ctx(cfg, kv_cache_config, torch.device("cpu"))
assert ctx.mamba_group_ids == [0]
assert ctx.num_states == 5
forward_context = {
"gdn.0": MagicMock(
kv_cache=(
torch.empty(8, 4, 4, dtype=torch.float16),
torch.empty(8, 2, 4, 4, dtype=torch.float32),
)
),
"gdn.1": MagicMock(
kv_cache=(
torch.empty(8, 4, 4, dtype=torch.float16),
torch.empty(8, 2, 4, 4, dtype=torch.float32),
)
),
"ple.0": MagicMock(kv_cache=(torch.empty(8, 12, 8, dtype=torch.float16),)),
}
ctx.initialize_from_forward_context(
kv_cache_config,
forward_context,
_COPY_FUNCS,
[torch.zeros(4, 8, dtype=torch.int32)],
)
assert ctx.state_group_indices.tolist() == [0, 0, 0, 0, 0]
assert ctx.state_conv_widths.tolist() == [4, 0, 4, 0, 12]
# -----------------------------------------------------------------------------
# stage_postprocess_inputs_to_gpu: single-pass staging into pinned views
# -----------------------------------------------------------------------------
@@ -620,7 +796,7 @@ def _run_gpu_postprocess(
*,
kv_cache_config: KVCacheConfig,
forward_context: dict[str, Any],
copy_funcs: tuple,
copy_funcs: MambaStateCopyFuncsByType,
block_table: torch.Tensor,
req_ids: list[str],
num_accepted_tokens: list[int],
+2
View File
@@ -2817,6 +2817,7 @@ def fused_gdn_decode_post_conv_mtp(
out: torch.Tensor | None = None,
scale: float = 128**-0.5,
norm_eps: float = 1e-5,
output_gate_activation: str = "silu",
) -> torch.Tensor:
if out is None:
out = torch.empty_like(output_gate)
@@ -2835,6 +2836,7 @@ def fused_gdn_decode_post_conv_mtp(
out,
scale,
norm_eps,
output_gate_activation,
)
return out
+3
View File
@@ -768,6 +768,9 @@ class CompilationConfig:
"vllm::mamba_mixer2",
"vllm::mamba_mixer",
"vllm::short_conv",
"vllm::qwen4_exp_compute_ple_ngram_ids",
"vllm::qwen4_exp_ple_short_conv",
"vllm::qwen4_exp_qsa_with_output",
"vllm::linear_attention",
"vllm::qwen_gdn_attention_core",
"vllm::qwen_gdn_attention_core_fused_norm_packed",
+4
View File
@@ -215,6 +215,10 @@ class SchedulerConfig:
# https://github.com/vllm-project/vllm/issues/29585
factors.append(self.max_num_batched_tokens)
# PLE and other model components allocate static per-request buffers.
# Their shapes are captured in compiled graphs.
factors.append(self.max_num_seqs)
hash_str = safe_hash(str(factors).encode(), usedforsecurity=False).hexdigest()
return hash_str
+43
View File
@@ -47,6 +47,7 @@ MTPModelTypes = Literal[
"exaone_moe_mtp",
"exaone4_5_mtp",
"qwen3_next_mtp",
"qwen4_exp_mtp",
"qwen3_5_mtp",
"longcat_flash_mtp",
"bailing_hybrid_v3_mtp",
@@ -425,6 +426,9 @@ class SpeculativeConfig:
"""The specific revision to use for the draft model code on Hugging Face
Hub. It can be a branch name, a tag name, or a commit id. If unspecified,
will use the default version."""
index_share_for_mtp_iteration: bool | None = None
"""Override whether MTP iterations reuse the first step's sparse indices.
If `None`, use the value from the draft model's Hugging Face config."""
# Advanced control
disable_padded_drafter_batch: bool = False
@@ -623,6 +627,15 @@ class SpeculativeConfig:
# Convert to tuple to make it hashable
factors.append(tuple(layer_ids))
if self.method == "mtp" and self.draft_model_config is not None:
factors.append(
getattr(
self.draft_model_config.hf_config,
"index_share_for_mtp_iteration",
False,
)
)
hash_str = safe_hash(str(factors).encode(), usedforsecurity=False).hexdigest()
return hash_str
@@ -805,6 +818,27 @@ class SpeculativeConfig:
hf_config.update(
{"n_predict": n_predict, "architectures": ["Qwen3NextMTP"]}
)
if hf_config.model_type in {"qwen4_exp", "qwen4_exp_text"}:
hf_config.model_type = "qwen4_exp_mtp"
if hf_config.model_type == "qwen4_exp_mtp":
text_config = get_hf_text_config(hf_config)
n_predict = getattr(
text_config,
"mtp_num_hidden_layers",
getattr(text_config, "num_nextn_predict_layers", None),
)
share_mtp_indices = getattr(
text_config, "index_share_for_mtp_iteration", False
)
hf_config.update(
{
# hc_count is the HC stream multiplier for Qwen MTP feedback.
"hc_mult": int(text_config.hc_count),
"n_predict": n_predict,
"architectures": ["Qwen4ExpMTP"],
"index_share_for_mtp_iteration": share_mtp_indices,
}
)
architectures = getattr(hf_config, "architectures", []) or []
if initial_architecture == "BailingMoeV3ForCausalLM":
@@ -1461,6 +1495,15 @@ class SpeculativeConfig:
)
)
if self.index_share_for_mtp_iteration is not None:
if self.method != "mtp" or self.draft_model_config is None:
raise ValueError(
"index_share_for_mtp_iteration is only supported with method='mtp'"
)
self.draft_model_config.hf_config.index_share_for_mtp_iteration = (
self.index_share_for_mtp_iteration
)
if self.method != "dspark" and self.enable_adaptive_verification:
raise ValueError("Adaptive verification only supported with DSpark")
@@ -405,6 +405,8 @@ class CustomAllreduce:
def should_custom_ar(self, inp: torch.Tensor):
if self.disabled or self.world_size > 8:
return False
if inp.dtype not in (torch.float32, torch.float16, torch.bfloat16):
return False
inp_size = inp.numel() * inp.element_size()
# custom allreduce requires input byte size to be multiples of 16
if inp_size % 16 != 0:
@@ -72,12 +72,15 @@ from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.network_utils import make_zmq_path
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
FullAttentionSpec,
KVCacheLayout,
KVCacheSpec,
MambaSpec,
MLAAttentionSpec,
SlidingWindowMLASpec,
UniformTypeKVCacheSpecs,
iter_layer_specs,
)
from vllm.v1.worker.block_table import BlockTable
from vllm.v1.worker.utils import select_common_block_size
@@ -118,7 +121,12 @@ class NixlBaseConnectorWorker:
# NIXL regions per SSM layer = conv sub-projections + 1 SSM temporal
# (Mamba2/GDN: 3+1=4; Mamba1: 1+1=2).
ssm_regions_per_layer = len(self._conv_decomp.local_conv_offsets) + 1
num_ssm_regions = len(self.block_len_per_layer) * ssm_regions_per_layer
# Count only the regions that actually hold SSM state; must match
# the descriptors emitted by _build_mamba_local.
num_ssm_regions = (
len(self._ssm_region_indices or self.block_len_per_layer)
* ssm_regions_per_layer
)
num_blocks = dst_num_blocks
if block_size_ratio is not None:
@@ -146,12 +154,19 @@ class NixlBaseConnectorWorker:
all_descs: list[np.ndarray] = []
for i, group in enumerate(block_ids):
group_arr = np.asarray(group)
if _is_attention_spec(self._group_spec_types[i]):
fa_region_ids = np.arange(self.num_regions)[:, None]
spec_type = self._group_spec_types[i]
if _is_attention_spec(spec_type):
# A scratch cache lives only in its own regions; every other
# attention group spans all of them.
fa_region_ids = (
np.asarray(self._scratch_region_indices, dtype=np.int64)
if spec_type is CircularBufferSpec
else np.arange(self.num_regions)
)[:, None]
all_descs.append(
(fa_region_ids * num_blocks + group_arr[None, :]).flatten()
)
elif _is_ssm_spec(self._group_spec_types[i]):
elif _is_ssm_spec(spec_type):
# NOTE (NickLucche) SSM and Attention block regions can
# be exchanged arbitrarily by manager. Therefore, descs
# are laid out as:
@@ -159,7 +174,11 @@ class NixlBaseConnectorWorker:
# num_fa_descs offset must be computed per-engine since
# P and D can have different num_blocks (and thus
# different FA desc counts).
ssm_region_ids = np.arange(num_ssm_regions)[:, None]
ssm_region_ids = (
np.arange(num_ssm_regions, num_ssm_regions + 1)
if i == self._ple_group_index
else np.arange(num_ssm_regions)
)[:, None]
all_descs.append(
(
ssm_region_ids * logical_blocks
@@ -206,6 +225,10 @@ class NixlBaseConnectorWorker:
# Per-FA-descriptor replicate flag, in _build_fa_local emission order.
fa_desc_replicated = self._fa_desc_replicated(num_fa_descs)
sharded_desc_end = len(src_blocks_data) - (
self._logical_num_blocks if self._ple_region_index is not None else 0
)
assert num_fa_descs <= sharded_desc_end
assert block_size_ratio == 1 or fa_num_splits == 1 or all(fa_desc_replicated), (
"Head-sharded attention reads with P_TP > D_TP and heterogeneous "
@@ -226,9 +249,11 @@ class NixlBaseConnectorWorker:
# SPLIT (full-attn): this rank's head slice.
chunk = local_len // fa_num_splits
handle.append((addr + fa_slot * chunk, chunk, dev))
else:
elif j < sharded_desc_end:
chunk = local_len // ssm_num_splits
handle.append((addr + p_idx * chunk, chunk, dev))
else:
handle.append((addr, local_len, dev))
yield handle
def _needs_split_local_xfer_handles(self, tp_ratio: int, plan: TPMapping) -> bool:
@@ -307,19 +332,22 @@ class NixlBaseConnectorWorker:
)
)
self.kv_cache_config = kv_cache_config
# Per-layer specs, unwrapping UniformTypeKVCacheSpecs group wrappers.
self._layer_specs: dict[str, KVCacheSpec] = {}
for group in kv_cache_config.transfer_groups:
group_spec = group.kv_cache_spec
if isinstance(group_spec, UniformTypeKVCacheSpecs):
self._layer_specs.update(group_spec.kv_cache_specs)
else:
self._layer_specs.update(dict.fromkeys(group.layer_names, group_spec))
self._is_hma_required = (
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
and any(
not isinstance(g.kv_cache_spec, FullAttentionSpec)
for g in kv_cache_config.transfer_groups
not isinstance(spec, FullAttentionSpec)
for spec in self._layer_specs.values()
)
)
self.kv_cache_config = kv_cache_config
self._layer_specs = {
layer: group.kv_cache_spec
for group in kv_cache_config.transfer_groups
for layer in group.layer_names
}
# ---- Model state (derived from model config) ----
mamba_ssm_size = (0, 0)
@@ -328,9 +356,29 @@ class NixlBaseConnectorWorker:
# conv sub-projections are contiguous in memory.
self._conv_decomp: MambaConvSplitInfo | None = None
self._has_mamba = any(
isinstance(g.kv_cache_spec, MambaSpec)
for g in kv_cache_config.transfer_groups
isinstance(spec, MambaSpec) for spec in self._layer_specs.values()
)
self._is_csa_linear = any(
get_representative_spec_type(group.kv_cache_spec) is CircularBufferSpec
for group in kv_cache_config.kv_cache_groups
)
self._ple_group_index: int | None = None
if self._is_csa_linear:
ple_groups = [
(index, group)
for index, group in enumerate(kv_cache_config.kv_cache_groups)
if group.layer_names
and all(
isinstance(spec, MambaSpec) and spec.tp_replicated
for spec in iter_layer_specs(group.kv_cache_spec)
)
]
if len(ple_groups) != 1 or len(ple_groups[0][1].layer_names) != 1:
raise ValueError(
"CSA-linear NIXL requires exactly one PLE cache owner."
)
self._ple_group_index = ple_groups[0][0]
if self._has_mamba:
assert self._is_hma_required
from vllm.model_executor.layers.mamba.mamba_utils import (
@@ -345,6 +393,7 @@ class NixlBaseConnectorWorker:
spec
for spec in self._layer_specs.values()
if isinstance(spec, MambaSpec)
and (not self._is_csa_linear or not spec.tp_replicated)
)
self._conv_decomp = derive_mamba_conv_split(
mamba_spec,
@@ -579,6 +628,12 @@ class NixlBaseConnectorWorker:
# (MLA), False -> SPLIT (head-sharded full-attn). Mixed only for models
# combining both (e.g. GQA main + MLA Eagle-3 draft).
self._region_is_mla = list[bool]()
self._ssm_region_indices = list[int]()
# Regions holding a scratch cache (the CSA compressor circular buffer).
# Tracked explicitly because a scratch page shares its address with the
# page it overlays, so its regions cannot be recovered from spec type.
self._scratch_region_indices = list[int]()
self._ple_region_index: int | None = None
# Enable different block lengths for different layers *only* when MLA is used.
# This is not used for SSM layers, which use the counterpart `mamba_ssm_size`.
@@ -632,6 +687,24 @@ class NixlBaseConnectorWorker:
self.block_size = kernel_block_size
self.num_blocks *= self._physical_blocks_per_logical_kv_block
def _validate_csa_linear_tp_layout(self, remote_tp_size: int) -> None:
"""Reject P/D pairs whose main-KV pages have different head layouts.
A rank at TP < total KV heads holds several heads per page, while a
rank at TP >= total KV heads holds exactly one (replicated beyond
that). CSA-linear pages are shared tensors that transfer as whole
block units, so endpoints on opposite sides of that boundary lay
them out differently and cannot exchange them.
"""
assert self.transfer_topo is not None
total_kv_heads = self.transfer_topo.total_num_kv_heads
if (self.world_size < total_kv_heads) != (remote_tp_size < total_kv_heads):
raise ValueError(
"CSA-linear NIXL requires both endpoints on the same side of "
f"the KV-head sharding boundary ({total_kv_heads} KV heads): "
f"got local TP {self.world_size}, remote TP {remote_tp_size}."
)
def _nixl_handshake(
self,
host: str,
@@ -643,6 +716,8 @@ class NixlBaseConnectorWorker:
notif_agents_only: bool = False,
) -> tuple[dict[tuple[int, int], str], float]:
"""Do a NIXL handshake with a remote instance."""
if self._is_csa_linear:
self._validate_csa_linear_tp_layout(remote_tp_size)
# the first time we connect to a remote agent.
# be careful, the handshake happens in a background thread.
@@ -1034,6 +1109,11 @@ class NixlBaseConnectorWorker:
transfer_mode=self._TRANSFER_MODE,
)
if self._is_csa_linear and self.use_host_buffer:
raise NotImplementedError(
"NIXL host staging does not preserve CSA-linear shared tensors."
)
if self.use_host_buffer:
self.initialize_host_xfer_buffer(kv_caches=kv_caches)
assert len(self.host_xfer_buffers) == len(kv_caches), (
@@ -1059,8 +1139,14 @@ class NixlBaseConnectorWorker:
caches_data = []
seen_storage_addresses: set[int] = set()
seen_base_addresses: list[int] = []
self._ssm_region_indices = []
self._scratch_region_indices = []
self._ple_region_index = None
packed_storage = _share_storage_and_block_stride(list(xfer_buffers.values()))
# CSA-linear needs separate logical regions for attention and state
# aliases even though every view shares one block-major allocation.
packed_storage = packed_storage and not self._is_csa_linear
# K and V are packed into the content dim, so each attention layer is a
# single NIXL region whose block transfers as one unit. Mamba layers instead
@@ -1112,14 +1198,16 @@ class NixlBaseConnectorWorker:
)
if isinstance(layer_spec, MambaSpec):
physical_ratio = self._physical_blocks_per_logical_kv_block
block_len = physical_page_size // physical_ratio
block_stride = physical_page_size
if self._is_csa_linear and cache.ndim != 1:
# CSA-linear packs multiple cache owners in each block, so
# state views advance by the packed block stride.
block_stride = cache.stride(0) * cache.element_size()
assert block_stride % physical_ratio == 0
region_specs = [
(
cache.data_ptr(),
physical_page_size
// self._physical_blocks_per_logical_kv_block,
physical_page_size
// self._physical_blocks_per_logical_kv_block,
)
(cache.data_ptr(), block_len, block_stride // physical_ratio)
]
else:
if cache.ndim == 1:
@@ -1131,19 +1219,18 @@ class NixlBaseConnectorWorker:
block_stride = cache.stride(0) * cache.element_size()
storage_is_block_major = num_blocks * block_stride == storage.nbytes()
# A layer whose [H, N, C] interior is dense addresses its own page
# as one chunk, so it can own a region. When it does not, or when
# every layer is packed into this one storage, the block's whole
# row is the only contiguous unit.
# as one chunk. Otherwise the block's whole row is the only
# contiguous transfer unit.
hnc_contiguous = (
cache.ndim == 4
and cache.stride(2) == cache.shape[3]
and cache.stride(1) == cache.shape[2] * cache.shape[3]
)
if storage_is_block_major and (
(packed_storage and is_mla_region) or not hnc_contiguous
(packed_storage and is_mla_region)
or (not hnc_contiguous and not self._is_csa_linear)
):
# TODO(Lucas): handle TP slicing for packed_storage; for now
# restrict to MLA (DSv4) where kv is replicated.
# Packed MLA layouts transfer the complete storage row.
storage_block_len = storage.nbytes() // num_blocks
region_specs = [
(storage_addr, storage_block_len, storage_block_len)
@@ -1170,17 +1257,27 @@ class NixlBaseConnectorWorker:
for base_addr, block_len, block_stride in region_specs:
if base_addr in seen_base_addresses:
if is_mla_region:
# Dual-purpose HMA tensor: an MLA layer shares a region
# that a non-MLA layer registered first. MLA is not
# head-sharded, so the region must be flagged MLA.
idx = seen_base_addresses.index(base_addr)
self._region_is_mla[idx] = True
continue
seen_base_addresses.append(base_addr)
self.block_len_per_layer.append(block_len)
self.block_stride_per_layer.append(block_stride)
self._region_is_mla.append(is_mla_region)
region_index = seen_base_addresses.index(base_addr)
self._region_is_mla[region_index] |= is_mla_region
else:
region_index = len(seen_base_addresses)
seen_base_addresses.append(base_addr)
self.block_len_per_layer.append(block_len)
self.block_stride_per_layer.append(block_stride)
self._region_is_mla.append(is_mla_region)
if isinstance(layer_spec, MambaSpec):
if layer_spec.tp_replicated:
assert self._is_csa_linear
assert self._ple_region_index in (None, region_index)
self._ple_region_index = region_index
elif region_index not in self._ssm_region_indices:
self._ssm_region_indices.append(region_index)
elif (
isinstance(layer_spec, CircularBufferSpec)
and region_index not in self._scratch_region_indices
):
self._scratch_region_indices.append(region_index)
# When there's a mismatch between kbs<>bs, we rely on HMA to ensure
# caches are either [NB, PS] or [NB*r, PS/r] where r is bs/kbs.
@@ -1210,6 +1307,8 @@ class NixlBaseConnectorWorker:
== len(self._region_is_mla)
== len(self.block_stride_per_layer)
)
# Descriptor ids must be region-ordered, matching the remote side.
self._scratch_region_indices.sort()
self.kv_caches_base_addr[self.engine_id][self.tp_rank] = seen_base_addresses
self.num_regions = len(seen_base_addresses)
@@ -1324,17 +1423,26 @@ class NixlBaseConnectorWorker:
physical_per_logical = self._physical_blocks_per_logical_kv_block
device_id = self.device_id
block_arange = np.arange(num_blocks, dtype=np.uint64)
parts: list[np.ndarray] = []
for i, base_addr in enumerate(base_addresses):
# Jump one page_size, but ssm page_size may be bigger when kernel
# locks block size to a specific value (physical_per_logical scale).
page_stride = self.block_len_per_layer[i] * physical_per_logical
blk_addrs = base_addr + block_arange * page_stride
region_indices = self._ssm_region_indices or range(len(base_addresses))
for i in region_indices:
base_addr = base_addresses[i]
block_stride = self.block_stride_per_layer[i] * physical_per_logical
blk_addrs = base_addr + block_arange * block_stride
for off, sz in conv_offsets:
parts.append(self._stack_descs(blk_addrs + off, sz, device_id))
# SSM temporal state follows the conv state.
parts.append(self._stack_descs(blk_addrs + conv_size, ssm_size, device_id))
if (region_index := self._ple_region_index) is not None:
block_len = self.block_len_per_layer[region_index] * physical_per_logical
block_stride = (
self.block_stride_per_layer[region_index] * physical_per_logical
)
block_addrs = base_addresses[region_index] + block_arange * block_stride
parts.append(self._stack_descs(block_addrs, block_len, self.device_id))
return np.concatenate(parts)
def _build_mamba_remote(
@@ -1370,14 +1478,44 @@ class NixlBaseConnectorWorker:
parts: list[np.ndarray] = []
# NOTE (ZhanqiuHu): use per-layer block_lens[i], not [0], in case
# block lengths vary across layers (e.g. MLA).
for i, base_addr in enumerate(nixl_agent_meta.kv_caches_base_addr):
page_stride = nixl_agent_meta.block_lens[i] * remote_physical_per_logical
blk_addrs = base_addr + block_arange * page_stride
region_indices = self._ssm_region_indices or range(
len(nixl_agent_meta.kv_caches_base_addr)
)
for i in region_indices:
base_addr = nixl_agent_meta.kv_caches_base_addr[i]
block_stride = (
nixl_agent_meta.block_strides[i] * remote_physical_per_logical
)
blk_addrs = base_addr + block_arange * block_stride
for off, sz in conv_offsets:
parts.append(self._stack_descs(blk_addrs + off, sz, device_id))
# SSM temporal state is also TP-sharded on the heads dimension.
ssm_addrs = blk_addrs + conv_size_remote + local_offset * ssm_read_size
parts.append(self._stack_descs(ssm_addrs, ssm_read_size, device_id))
if (region_index := self._ple_region_index) is not None:
local_block_len = (
self.block_len_per_layer[region_index]
* self._physical_blocks_per_logical_kv_block
)
remote_block_len = (
nixl_agent_meta.block_lens[region_index] * remote_physical_per_logical
)
if local_block_len != remote_block_len:
raise ValueError(
"PLE pages require identical P/D geometry: "
f"local={local_block_len}, remote={remote_block_len}."
)
remote_block_stride = (
nixl_agent_meta.block_strides[region_index]
* remote_physical_per_logical
)
block_addrs = (
nixl_agent_meta.kv_caches_base_addr[region_index]
+ block_arange * remote_block_stride
)
parts.append(self._stack_descs(block_addrs, remote_block_len, device_id))
return np.concatenate(parts)
@staticmethod
@@ -2410,8 +2548,7 @@ class NixlBaseConnectorWorker:
group_specs = self.kv_cache_config.transfer_groups
physical_block_ids = []
for i, group in enumerate(block_ids):
spec = group_specs[i].kv_cache_spec
if isinstance(spec, MambaSpec):
if _is_ssm_spec(get_representative_spec_type(group_specs[i].kv_cache_spec)):
physical_block_ids.append(group)
else:
physical_block_ids.append(
@@ -536,7 +536,8 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
def group_ids(block_ids: BlockIds, rank: int) -> BlockIds:
return [
list(block_ids[g])
if (replicate_attn and _is_attention_spec(self._group_spec_types[g]))
if (self._is_csa_linear and tp_ratio < 0)
or (replicate_attn and _is_attention_spec(self._group_spec_types[g]))
or rank in plan.source_ranks_per_group[g]
else []
for g in range(num_groups)
@@ -39,6 +39,7 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
is_deep_gemm_e8m0_used,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
FP8_DTYPE,
QuantKey,
kFp8Dynamic128Sym,
kFp8DynamicTensorSym,
@@ -153,6 +154,19 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
(kFp8StaticTensorSym, kFp8StaticTensorSym),
(kFp8StaticTensorSym, kFp8DynamicTensorSym),
]
# Block-quantized FP8 with arbitrary block shape: the Triton
# kernels take the block shape as a runtime argument.
if (
weight_key is not None
and activation_key == kFp8Dynamic128Sym
and weight_key.dtype == FP8_DTYPE
and weight_key.symmetric
and weight_key.scale.static
and weight_key.scale.dtype == torch.float32
and weight_key.scale.group_shape.row > 1
and weight_key.scale.group_shape.col > 1
):
return True
return (weight_key, activation_key) in supported
@staticmethod
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from enum import Enum
from typing import Any
@@ -268,6 +269,39 @@ def map_fp8_backend(runner_backend: MoEBackend) -> Fp8MoeBackend:
)
def refine_fp8_moe_block_shape(
config: FusedMoEConfig,
weight_block_size: list[int],
) -> list[int] | None:
"""
Compute a refined block shape for block-quantized FP8 MoE weights whose
checkpoint blocks cannot be sharded exactly across TP ranks.
TP shards the intermediate dim of the expert weights, so a per-shard size
that is not a multiple of the checkpoint's block size makes the
checkpoint's block scales impossible to shard exactly. When a finer block
size (>= 32) divides both the checkpoint blocks and all involved dims,
the weight scales can be refined to that granularity at load time (a
lossless upsampling, since the refined block divides the checkpoint
block). Only Triton-based kernels can consume the refined block shape:
they take it as a runtime argument, while the other backends require the
native 128x128 blocks. The refined shape is encoded in the QuantKey used
for backend selection, so backends that only support 128x128 blocks are
rejected by the oracle automatically.
Returns the refined [block_n, block_k] shape, or None if no refinement
is needed or possible.
"""
block_n, block_k = weight_block_size
ispp = config.intermediate_size_per_partition
if ispp % block_n == 0 and (config.tp_size == 1 or ispp % block_k == 0):
return None
refine = math.gcd(block_n, block_k, ispp, config.hidden_dim)
if refine < 32:
return None
return [refine, refine]
def select_fp8_moe_backend(
config: FusedMoEConfig,
weight_key: QuantKey | None,
@@ -474,6 +474,39 @@ def _get_requested_backends(
return _filter_by_activation(backends, requested_activation_key)
def _requires_qwen38_tep8_emulation(
config: FusedMoEConfig,
activation_key: QuantKey | None,
) -> bool:
"""Avoid the inaccurate native gfx950 kernel for Qwen3.8 TEP8.
Qwen3.8 Flash Next's routed experts use the distinctive
``E=512, H=2560, N=640`` W4A4 shape. With eight-way expert parallelism,
AITER operates on 64 local experts and pads ``N`` to 768. That native path
is not numerically reliable on gfx950, while OCP MX emulation preserves
model accuracy. Keep this guard exact so other MXFP4 shapes and smaller EP
configurations retain the native backend.
"""
parallel = config.moe_parallel_config
if not (
activation_key == kMxfp4Dynamic
and parallel.use_ep
and parallel.ep_size == 8
and config.num_experts == 512
and config.num_local_experts == 64
and config.hidden_dim == 2560
and config.intermediate_size == 640
):
return False
if not current_platform.is_rocm():
return False
from vllm.platforms.rocm import on_gfx950
return on_gfx950()
def select_mxfp4_moe_backend(
config: FusedMoEConfig,
activation_key: QuantKey | None = None,
@@ -532,6 +565,21 @@ def select_mxfp4_moe_backend(
assert last_error is not None
raise last_error
if _requires_qwen38_tep8_emulation(config, requested_activation_key):
backend = Mxfp4MoeBackend.EMULATION
logger.warning_once(
"Using OCP MX emulation for the Qwen3.8 Flash Next TEP8 routed "
"experts on gfx950 because the native AITER W4A4 kernel is not "
"numerically reliable for this shape. Performance will be lower."
)
return _return_or_raise(
backend,
config,
kMxfp4Static,
requested_activation_key,
activation_format,
)
# Select kernels in order of backend.
AVAILABLE_BACKENDS = _filter_by_activation(
_get_priority_backends_for_gpt_oss(), requested_activation_key
@@ -345,14 +345,13 @@ def convert_to_unquantized_kernel_format(
elif unquantized_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM:
is_act_and_mul = moe_config.is_act_and_mul
if not is_act_and_mul:
# Kernel requires intermediate_size_per_partition % 128 == 0 (BlockMajorK
# weight layout uses block_k=128). Pad along the intermediate dim when
# the model + TP split don't satisfy the constraint.
w13_weight, w2_weight, padded_intermediate = align_moe_weights_for_fi(
w13_weight, w2_weight, is_act_and_mul, min_alignment=128
)
moe_config.intermediate_size_per_partition = padded_intermediate
# Kernel requires intermediate_size_per_partition % 128 == 0 (BlockMajorK
# weight layout uses block_k=128). Pad along the intermediate dim when
# the model + TP split don't satisfy the constraint.
w13_weight, w2_weight, padded_intermediate = align_moe_weights_for_fi(
w13_weight, w2_weight, is_act_and_mul, min_alignment=128
)
moe_config.intermediate_size_per_partition = padded_intermediate
_cache_permute_indices: dict[torch.Size, torch.Tensor] = {}
w13_weight, w2_weight = convert_moe_weights_to_flashinfer_trtllm_block_layout(
@@ -832,6 +832,20 @@ class RoutedExperts(PluggableLayer):
FusedMoeWeightScaleSupported.GROUP.value,
FusedMoeWeightScaleSupported.BLOCK.value,
]:
scale_refine = getattr(self.quant_method, "weight_scale_refine", None)
if (
quant_method == FusedMoeWeightScaleSupported.BLOCK.value
and scale_refine is not None
):
# FP8 block scales are stored per (block_n, block_k) tile
# of the unsharded weight, while the TP-sharded parameters
# use a refined block grid (see Fp8MoEMethod). Upsample the
# scales to the refined grid (lossless: the refined block
# divides the checkpoint block) so per-rank slicing stays
# exact. Dim -2 is the weight's N dim, dim -1 is K.
loaded_weight = loaded_weight.repeat_interleave(
scale_refine[0], dim=-2
).repeat_interleave(scale_refine[1], dim=-1)
self._load_model_weight_or_group_weight_scale(
shard_id=shard_id,
shard_dim=shard_dim,
@@ -56,6 +56,10 @@ class MambaBase(AttentionLayerBase):
def mamba_type(self) -> MambaAttentionBackendEnum:
pass
@property
def is_kv_cache_tp_replicated(self) -> bool:
return False
@abstractmethod
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
pass
@@ -70,6 +74,7 @@ class MambaBase(AttentionLayerBase):
block_size=mamba_block_size,
page_size_padded=page_size_padded,
mamba_type=self.mamba_type,
tp_replicated=self.is_kv_cache_tp_replicated,
mamba_cache_mode=vllm_config.cache_config.mamba_cache_mode,
# RecoverSSM verifies the whole window off one checkpoint, so it
# never writes the baseline's per-draft-token state slots.
@@ -519,7 +519,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
self.gqa_interleaved_layout
or self.head_k_dim != 128
or self.head_v_dim != 128
or self.norm.activation != "silu"
or self.norm.activation not in ("silu", "sigmoid")
or vllm_config.model_config.dtype != torch.bfloat16
or conv_state_dtype != torch.bfloat16
or recurrent_state_dtype not in FUSED_GDN_STATE_DTYPES
@@ -527,9 +527,9 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
):
return (
"the fused CUDA kernel requires a BF16 GDN model with "
"K=V=128, SiLU gating, non-interleaved GQA layout, BF16 "
"convolution cache, BF16 or FP32 recurrent state, and a "
"GPU with compute capability 8.0+"
"K=V=128, SiLU or sigmoid gating, non-interleaved GQA "
"layout, BF16 convolution cache, BF16 or FP32 recurrent "
"state, and a GPU with compute capability 8.0+"
)
if not hasattr(torch.ops._C, "fused_gdn_decode_post_conv_mtp"):
return "torch.ops._C.fused_gdn_decode_post_conv_mtp is not built"
@@ -1766,6 +1766,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
out=core_attn_out,
scale=self.head_k_dim**-0.5,
norm_eps=self.layer_norm_epsilon,
output_gate_activation=self.norm.activation,
)
def _forward_core_fused_norm_packed(
@@ -17,6 +17,7 @@ from vllm.utils.torch_utils import (
STR_DTYPE_TO_TORCH_DTYPE,
get_kv_cache_torch_dtype,
)
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
logger = init_logger(__name__)
@@ -352,6 +353,10 @@ Parameters:
num_accepted_tokens: int - number of accepted tokens used to compute the copy offset.
Range: 1 .. 1 + num_speculative_tokens (inclusive).
"""
MambaStateCopyFuncs: TypeAlias = tuple[MambaStateCopyFunc, ...]
MambaStateCopyFuncsByType: TypeAlias = dict[
MambaAttentionBackendEnum, MambaStateCopyFuncs
]
def get_conv_copy_spec(
+53 -13
View File
@@ -31,6 +31,7 @@ from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
convert_to_fp8_moe_kernel_format,
make_fp8_moe_kernel,
make_fp8_moe_quant_config,
refine_fp8_moe_block_shape,
select_fp8_moe_backend,
)
from vllm.model_executor.layers.linear import (
@@ -63,7 +64,6 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Dynamic128Sym,
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8Static128BlockSym,
kFp8StaticTensorSym,
)
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
@@ -481,9 +481,41 @@ class Fp8MoEMethod(FusedMoEMethodBase):
"weight_scale_inv" if self.block_quant else "weight_scale"
)
self.weight_scale_refine: tuple[int, int] | None = None
self.moe_block_shape = self.weight_block_size
# Set weight key and activation key for kernel compatibility
if self.block_quant:
weight_key = kFp8Static128BlockSym
assert self.weight_block_size is not None
# TP shards the intermediate dim of the expert weights, so a
# per-shard size that is not a multiple of the checkpoint's block
# size makes the checkpoint's block scales impossible to shard
# exactly. When a finer block size (>= 32) divides both the
# checkpoint blocks and all involved dims, the weight scales are
# refined to that granularity at load time (a lossless upsampling,
# since the refined block divides the checkpoint block). The
# refined block shape is encoded in the weight key, so the oracle
# only selects kernels that support it (e.g. Triton, which takes
# the block shape as a runtime argument).
refined_shape = refine_fp8_moe_block_shape(self.moe, self.weight_block_size)
if refined_shape is not None:
block_n, block_k = self.weight_block_size
self.weight_scale_refine = (
block_n // refined_shape[0],
block_k // refined_shape[1],
)
self.moe_block_shape = refined_shape
logger.info_once(
"FP8 MoE block scales refined from %s to %s to fit "
"the TP-sharded intermediate size %d.",
str(self.weight_block_size),
str(refined_shape),
self.moe.intermediate_size_per_partition,
)
assert self.moe_block_shape is not None
weight_key = create_fp8_quant_key(
static=True, group_shape=GroupShape(*self.moe_block_shape)
)
activation_key = kFp8Dynamic128Sym
else:
weight_key = kFp8StaticTensorSym
@@ -519,6 +551,8 @@ class Fp8MoEMethod(FusedMoEMethodBase):
if self.block_quant:
assert self.weight_block_size is not None
assert self.moe_block_shape is not None
moe_block_shape = self.moe_block_shape
layer.weight_block_size = self.weight_block_size
tp_size = get_tensor_model_parallel_world_size()
block_n, block_k = (
@@ -530,18 +564,24 @@ class Fp8MoEMethod(FusedMoEMethodBase):
# layers must be divisible by block_n.
# Required by column parallel or enabling merged weights
if intermediate_size_per_partition % block_n != 0:
raise ValueError(
f"The output_size of gate's and up's weight = "
f"{intermediate_size_per_partition} is not divisible by "
f"weight quantization block_n = {block_n}."
)
if self.weight_scale_refine is None:
raise ValueError(
f"The output_size of gate's and up's weight = "
f"{intermediate_size_per_partition} is not divisible by "
f"weight quantization block_n = {block_n}."
)
# Use the refined block grid for the scale parameters; the
# loader upsamples the checkpoint scales accordingly.
block_n, block_k = moe_block_shape
if tp_size > 1 and intermediate_size_per_partition % block_k != 0:
# Required by row parallel
raise ValueError(
f"The input_size of down's weight = "
f"{intermediate_size_per_partition} is not divisible by "
f"weight quantization block_k = {block_k}."
)
if self.weight_scale_refine is None:
raise ValueError(
f"The input_size of down's weight = "
f"{intermediate_size_per_partition} is not divisible by "
f"weight quantization block_k = {block_k}."
)
block_n, block_k = moe_block_shape
# WEIGHTS
w13_weight = torch.nn.Parameter(
@@ -747,7 +787,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
w2_scale=w2_scale,
a1_scale=a1_scale,
a2_scale=a2_scale,
block_shape=self.weight_block_size,
block_shape=self.moe_block_shape,
swiglu_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
@@ -161,6 +161,16 @@ class QuarkOCP_MX(QuarkScheme):
weight_loader: Callable,
**kwargs,
):
if input_size_per_partition % OCP_MX_BLOCK_SIZE != 0:
layer_name = getattr(layer, "prefix", "") or type(layer).__name__
raise ValueError(
f"OCP MX linear layer {layer_name!r} has an input size per "
f"partition of {input_size_per_partition}, which must be "
f"divisible by the OCP MX group size {OCP_MX_BLOCK_SIZE}. "
"Choose a compatible tensor-parallel size or avoid "
"tensor-parallel sharding for this layer."
)
if self.dynamic_mxfp4_quant:
weight = ModelWeightParameter(
data=torch.empty(
@@ -9,6 +9,41 @@ if TYPE_CHECKING:
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
def get_quark_ocp_mx_group_size(
quant_config: "QuantizationConfig | None",
layer_name: str,
) -> int | None:
"""Return the OCP MX group size for a quantized Quark linear layer."""
if quant_config is None:
return None
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
from vllm.model_executor.layers.quantization.quark.utils import should_ignore_layer
if not isinstance(quant_config, QuarkConfig):
return None
if should_ignore_layer(
layer_name,
ignore=quant_config.quant_config.get("exclude") or [],
fused_mapping=quant_config.packed_modules_mapping,
):
return None
layer_quant_config = (
quant_config.get_layer_quant_config_from_name(layer_name)
or quant_config.quant_config.get("global_quant_config")
or {}
)
weight_quant = layer_quant_config.get("weight")
input_quant = layer_quant_config.get("input_tensors")
if not quant_config._is_w_ocp_mx_a_x(weight_quant, input_quant):
return None
assert weight_quant is not None
return int(weight_quant["group_size"])
def is_shared_expert_quant_fse_compatible(
quant_config: "QuantizationConfig | None",
expert_prefix: str,
@@ -232,7 +232,16 @@ def align_fp4_moe_weights_for_fi(
# Pad w13 and w2 along its intermediate dimension.
padded_w13 = w13.new_zeros((num_experts, padded_gate_up_dim, hidden_size // 2))
padded_w13[:, : w13.shape[1], :] = w13
if is_act_and_mul:
# Keep the two logical projections independently aligned. Copying the
# fused [gate, up] tensor contiguously would move the up projection into
# the padded tail of gate when intermediate is rounded up.
padded_w13[:, :intermediate, :] = w13[:, :intermediate, :]
padded_w13[:, padded_intermediate : padded_intermediate + intermediate, :] = (
w13[:, intermediate:, :]
)
else:
padded_w13[:, : w13.shape[1], :] = w13
padded_w2 = w2.new_zeros((num_experts, hidden_size, padded_intermediate // 2))
padded_w2[:, :, : w2.shape[2]] = w2
@@ -240,7 +249,13 @@ def align_fp4_moe_weights_for_fi(
padded_w13_scale = w13_scale.new_zeros(
(num_experts, padded_gate_up_dim, hidden_size // 16)
)
padded_w13_scale[:, : w13_scale.shape[1], :] = w13_scale
if is_act_and_mul:
padded_w13_scale[:, :intermediate, :] = w13_scale[:, :intermediate, :]
padded_w13_scale[
:, padded_intermediate : padded_intermediate + intermediate, :
] = w13_scale[:, intermediate:, :]
else:
padded_w13_scale[:, : w13_scale.shape[1], :] = w13_scale
padded_w2_scale = w2_scale.new_zeros(
(num_experts, hidden_size, padded_intermediate // 16)
@@ -322,7 +337,13 @@ def align_moe_weights_for_fi(
# Pad w13 and w2 along its intermediate dimension.
padded_w13 = w13.new_zeros((num_experts, padded_gate_up_dim, hidden_size))
padded_w13[:, : w13.shape[1], :] = w13
if is_act_and_mul:
padded_w13[:, :intermediate, :] = w13[:, :intermediate, :]
padded_w13[:, padded_intermediate : padded_intermediate + intermediate, :] = (
w13[:, intermediate:, :]
)
else:
padded_w13[:, :intermediate, :] = w13
padded_w2 = w2.new_zeros((num_experts, hidden_size, padded_intermediate))
padded_w2[:, :, :intermediate] = w2
@@ -233,6 +233,7 @@ class VocabParallelEmbedding(PluggableLayer):
quant_config: quant config for the layer
prefix: full name of the layer in the state dict
disable_tp: If true, tensor parallelism will be disabled for this layer.
quant_method: Preselected quantization method for model-specific layers.
""" # noqa: E501
# --8<-- [end:vocab_parallel_embedding]
@@ -248,6 +249,7 @@ class VocabParallelEmbedding(PluggableLayer):
prefix: str = "",
*,
disable_tp: bool = False,
quant_method: QuantizeMethodBase | None = None,
):
super().__init__()
@@ -281,8 +283,9 @@ class VocabParallelEmbedding(PluggableLayer):
)
self.embedding_dim = embedding_dim
quant_method = None
if quant_config is not None:
# Avoid overriding a preselected model-specific method with generic
# config-based dispatch.
if quant_method is None and quant_config is not None:
quant_method = quant_config.get_quant_method(self, prefix=prefix)
if quant_method is None:
quant_method = UnquantizedEmbeddingMethod()
@@ -501,8 +504,16 @@ class VocabParallelEmbedding(PluggableLayer):
output_parallel = self.quant_method.embedding(self, masked_input.long())
# Mask the output embedding.
if self.tp_size > 1:
if output_parallel.dtype in (
torch.float8_e4m3fn,
torch.float8_e5m2,
):
# Each vocab token has one owner, so FP8 bytes can use int8 SUM.
comm_output = output_parallel.view(torch.int8)
comm_output.masked_fill_(input_mask.unsqueeze(-1), 0)
output = tensor_model_parallel_all_reduce(comm_output)
return output.view(output_parallel.dtype)
output_parallel.masked_fill_(input_mask.unsqueeze(-1), 0)
# Reduce across all the model parallel GPUs.
return tensor_model_parallel_all_reduce(output_parallel)
return output_parallel
@@ -289,12 +289,17 @@ def get_quant_config(
# if hf_quant_config is None, we will try to get config from
# hf_overrides
hf_overrides = model_config.hf_overrides
if callable(hf_overrides):
# A callable hf_overrides is a config-to-config transform (e.g. the
# one SpeculativeConfig installs on draft model configs); it cannot
# carry quantization config entries, so treat it as no overrides.
hf_overrides = {}
if not isinstance(hf_overrides, dict):
raise ValueError(
"hf_overrides must be a dict for get_quant_config "
"to get the quantization config from it."
)
quantization_config_file = hf_overrides.get("quantization_config_file", None)
quantization_config_file = hf_overrides.get("quantization_config_file")
if quantization_config_file is not None:
if hasattr(quant_cls, "from_config_file"):
return quant_cls.from_config_file(quantization_config_file)
@@ -304,7 +309,7 @@ def get_quant_config(
"but quant_cls.from_config_file is not implemented in "
f"{quant_cls}"
)
quantization_config_json = hf_overrides.get("quantization_config_dict_json", None)
quantization_config_json = hf_overrides.get("quantization_config_dict_json")
if quantization_config_json is not None:
if hasattr(quant_cls, "from_config_dict_json"):
return quant_cls.from_config_dict_json(quantization_config_json)
+80
View File
@@ -839,6 +839,83 @@ class Qwen3_5ForCausalLMConfig(Qwen3_5ForConditionalGenerationConfig):
rope_parameters.pop("mrope_interleaved", None)
def _strip_qwen4_exp_mrope(model_config: "ModelConfig") -> None:
configs = {
id(config): config
for config in (
getattr(model_config, "hf_config", None),
model_config.hf_text_config,
)
if config is not None
}
for config in configs.values():
rope_parameters = getattr(config, "rope_parameters", None)
if rope_parameters is not None:
rope_parameters.pop("mrope_section", None)
rope_parameters.pop("mrope_interleaved", None)
class Qwen4ExpForConditionalGenerationConfig(Qwen3_5ForConditionalGenerationConfig):
"""Apply the Qwen3.5 hybrid-cache contract to Qwen4Exp."""
@staticmethod
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
Qwen3_5ForConditionalGenerationConfig.verify_and_update_config(vllm_config)
text_config = vllm_config.model_config.hf_text_config
if text_config.hc_count <= 1:
raise ValueError("Qwen4Exp requires hc_count > 1")
parallel_config = vllm_config.parallel_config
uses_ple_or_qsa = bool(text_config.ple_layer_ids) or (
getattr(text_config, "indexer_n_heads", None) is not None
)
if uses_ple_or_qsa and (
parallel_config.enable_dbo or parallel_config.ubatch_size > 1
):
raise NotImplementedError(
"Qwen4Exp PLE/QSA does not support dual-batch overlap or microbatching"
)
# Checked again in Qwen4ExpModelState; rejecting it here keeps the
# engine from loading weights first.
if text_config.ple_layer_ids and parallel_config.pipeline_parallel_size > 1:
raise NotImplementedError(
"Qwen4Exp N-gram PLE embedding requires pipeline_parallel_size=1 "
"because non-first pipeline ranks do not receive the raw input_ids "
"it needs. Please run with PP=1."
)
multimodal_config = vllm_config.model_config.multimodal_config
if multimodal_config is not None and multimodal_config.language_model_only:
_strip_qwen4_exp_mrope(vllm_config.model_config)
spec_config = vllm_config.speculative_config
if spec_config is not None and spec_config.method not in {
"mtp",
"ngram",
"ngram_gpu",
}:
raise NotImplementedError(
"Qwen4Exp speculative decoding supports only its native MTP "
"checkpoint and linear n-gram proposers"
)
class Qwen4ExpForCausalLMConfig(Qwen4ExpForConditionalGenerationConfig):
@staticmethod
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(vllm_config)
_strip_qwen4_exp_mrope(vllm_config.model_config)
class Qwen4ExpMTPConfig(Qwen4ExpForConditionalGenerationConfig):
"""Preserve MRoPE for a VL target and use 1D RoPE for a text target."""
@staticmethod
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(vllm_config)
if hasattr(vllm_config.model_config.hf_config, "vision_config"):
return
_strip_qwen4_exp_mrope(vllm_config.model_config)
class ColQwen3_5Config(Qwen3_5ForConditionalGenerationConfig):
"""Apply the attention contract declared by a ColQwen3.5 checkpoint."""
@@ -963,6 +1040,9 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
"Qwen3_5ForConditionalGeneration": Qwen3_5ForConditionalGenerationConfig,
"Qwen3_5MoeForCausalLM": Qwen3_5ForCausalLMConfig,
"Qwen3_5MoeForConditionalGeneration": Qwen3_5ForConditionalGenerationConfig,
"Qwen4ExpForCausalLM": Qwen4ExpForCausalLMConfig,
"Qwen4ExpForConditionalGeneration": (Qwen4ExpForConditionalGenerationConfig),
"Qwen4ExpMTP": Qwen4ExpMTPConfig,
"UnlimitedOCRForCausalLM": UnlimitedOCRForCausalLMConfig,
"VoyageQwen3BidirectionalEmbedModel": VoyageQwen3BidirectionalEmbedModelConfig,
"XLMRobertaModel": JinaRobertaModelConfig,
+27 -1
View File
@@ -49,13 +49,17 @@ if TYPE_CHECKING:
from vllm.lora.model_manager import LoRAModelManager
from vllm.model_executor.layers.fused_moe import MoERunner
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc
from vllm.model_executor.layers.mamba.mamba_utils import (
MambaStateCopyFunc,
MambaStateCopyFuncsByType,
)
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.models.utils import WeightsMapper
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalKwargsItem
from vllm.multimodal.registry import _ProcessorFactories
from vllm.sequence import IntermediateTensors
from vllm.tasks import ScoreType
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.worker.encoder_cudagraph_defs import (
EncoderCudaGraphCaptureInputs,
EncoderCudaGraphConfig,
@@ -970,6 +974,14 @@ class IsHybrid(Protocol):
"""
...
@classmethod
def get_mamba_state_copy_funcs(
cls,
mamba_types: set["MambaAttentionBackendEnum"],
) -> "MambaStateCopyFuncsByType":
copy_funcs = cls.get_mamba_state_copy_func()
return {mamba_type: copy_funcs for mamba_type in mamba_types}
@overload
def is_hybrid(model: type[object]) -> TypeIs[type[IsHybrid]]: ...
@@ -1131,6 +1143,20 @@ class SupportsMambaPrefixCaching(Protocol):
supports_mamba_prefix_caching: ClassVar[Literal[True]] = True
@classmethod
def get_mamba_state_copy_func(cls) -> tuple["MambaStateCopyFunc", ...]:
"""Return copy functions for the model's Mamba states."""
...
@classmethod
def get_mamba_state_copy_funcs(
cls,
mamba_types: set["MambaAttentionBackendEnum"],
) -> "MambaStateCopyFuncsByType":
"""Map legacy copy functions to each requested Mamba backend."""
copy_funcs = cls.get_mamba_state_copy_func()
return {mamba_type: copy_funcs for mamba_type in mamba_types}
@overload
def supports_mamba_prefix_caching(
+4 -2
View File
@@ -80,15 +80,17 @@ class Qwen2MoeMLP(nn.Module):
reduce_results: bool = True,
expert_gate: torch.nn.Linear | None = None,
is_sequence_parallel: bool = False,
disable_tp: bool = False,
prefix: str = "",
) -> None:
super().__init__()
disable_tp = disable_tp or is_sequence_parallel
self.gate_up_proj = MergedColumnParallelLinear(
hidden_size,
[intermediate_size] * 2,
bias=False,
quant_config=quant_config,
disable_tp=is_sequence_parallel,
disable_tp=disable_tp,
prefix=f"{prefix}.gate_up_proj",
)
self.down_proj = RowParallelLinear(
@@ -97,7 +99,7 @@ class Qwen2MoeMLP(nn.Module):
bias=False,
quant_config=quant_config,
reduce_results=reduce_results,
disable_tp=is_sequence_parallel,
disable_tp=disable_tp,
prefix=f"{prefix}.down_proj",
)
if hidden_act != "silu":
+62 -3
View File
@@ -43,6 +43,9 @@ from vllm.model_executor.layers.mamba.mamba_utils import (
MambaStateShapeCalculator,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.utils.config_utils import (
get_quark_ocp_mx_group_size,
)
from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding, get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
@@ -90,6 +93,38 @@ def _should_use_sequence_parallel(vllm_config: VllmConfig) -> bool:
)
def _should_replicate_misaligned_shared_expert(
intermediate_size: int,
tp_size: int,
group_size: int | None,
enable_expert_parallel: bool,
is_sequence_parallel: bool,
) -> bool:
if intermediate_size <= 0 or group_size is None:
return False
partition_size, remainder = divmod(intermediate_size, tp_size)
if remainder == 0 and partition_size % group_size == 0:
return False
if enable_expert_parallel or is_sequence_parallel:
return True
if remainder != 0:
raise ValueError(
f"Shared-expert intermediate size {intermediate_size} must be "
f"divisible by tensor-parallel size {tp_size}."
)
raise ValueError(
"The Quark OCP MX shared expert cannot be tensor-parallelized: "
f"intermediate size {intermediate_size} with TP size {tp_size} "
f"produces a partition of {partition_size}, which is not divisible by "
f"the OCP MX group size {group_size}. Choose a compatible "
"tensor-parallel size or enable expert parallelism."
)
class Qwen3NextSparseMoeBlock(nn.Module):
def __init__(self, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -106,6 +141,17 @@ class Qwen3NextSparseMoeBlock(nn.Module):
self.is_sequence_parallel = parallel_config.use_sequence_parallel_moe
shared_expert_group_size = get_quark_ocp_mx_group_size(
quant_config,
f"{prefix}.shared_expert.down_proj",
)
self.replicate_shared_expert = _should_replicate_misaligned_shared_expert(
config.shared_expert_intermediate_size,
self.tp_size,
shared_expert_group_size,
parallel_config.enable_expert_parallel,
self.is_sequence_parallel,
)
if self.tp_size > config.num_experts:
raise ValueError(
f"Tensor parallel size {self.tp_size} is greater than "
@@ -138,7 +184,10 @@ class Qwen3NextSparseMoeBlock(nn.Module):
)
self.is_fused_shared_expert_enabled = False
if config.shared_expert_intermediate_size > 0:
if (
config.shared_expert_intermediate_size > 0
and not self.replicate_shared_expert
):
self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert(
quant_config,
prefix,
@@ -159,11 +208,14 @@ class Qwen3NextSparseMoeBlock(nn.Module):
reduce_results=False,
expert_gate=self.shared_expert_gate,
is_sequence_parallel=self.is_sequence_parallel,
disable_tp=self.replicate_shared_expert,
prefix=f"{prefix}.shared_expert",
)
self.experts = FusedMoEFactory(
shared_experts=self.shared_expert,
shared_experts=(
None if self.replicate_shared_expert else self.shared_expert
),
gate=self.gate,
num_experts=self.n_routed_experts,
top_k=config.num_experts_per_tok,
@@ -195,9 +247,16 @@ class Qwen3NextSparseMoeBlock(nn.Module):
if self.is_sequence_parallel and not already_sequence_parallel:
hidden_states = sequence_parallel_chunk(hidden_states)
replicated_shared_output = (
self.shared_expert(hidden_states)
if self.replicate_shared_expert and self.shared_expert is not None
else None
)
final_hidden_states = self.experts(
hidden_states=hidden_states, router_logits=hidden_states
)
if replicated_shared_output is not None:
final_hidden_states += replicated_shared_output
if self.is_sequence_parallel and not already_sequence_parallel:
final_hidden_states = tensor_model_parallel_all_gather(
@@ -488,7 +547,7 @@ class Qwen3NextDecoderLayer(nn.Module):
self,
hidden_states: torch.Tensor,
residual: torch.Tensor | None,
positions: torch.Tensor = None,
positions: torch.Tensor,
**kwargs: object,
):
full_num_tokens = positions.shape[-1]
+9
View File
@@ -111,6 +111,10 @@ _TEXT_GENERATION_MODELS = {
"Gemma3nForCausalLM": ("gemma3n", "Gemma3nForCausalLM"),
"Gemma4ForCausalLM": ("gemma4", "Gemma4ForCausalLM"),
"Qwen3NextForCausalLM": ("qwen3_next", "Qwen3NextForCausalLM"),
"Qwen4ExpForCausalLM": (
"vllm.models.qwen4_exp",
"Qwen4ExpForCausalLM",
),
"GlmForCausalLM": ("glm", "GlmForCausalLM"),
"Glm4ForCausalLM": ("glm4", "Glm4ForCausalLM"),
"Glm4MoeForCausalLM": ("glm4_moe", "Glm4MoeForCausalLM"),
@@ -573,6 +577,10 @@ _MULTIMODAL_MODELS = {
"qwen3_5",
"Qwen3_5MoeForConditionalGeneration",
),
"Qwen4ExpForConditionalGeneration": (
"vllm.models.qwen4_exp",
"Qwen4ExpForConditionalGeneration",
),
"RForConditionalGeneration": ("rvl", "RForConditionalGeneration"),
"SkyworkR1VChatModel": ("skyworkr1v", "SkyworkR1VChatModel"),
"SmolVLMForConditionalGeneration": ("smolvlm", "SmolVLMForConditionalGeneration"),
@@ -659,6 +667,7 @@ _SPECULATIVE_DECODING_MODELS = {
"MedusaModel": ("medusa", "Medusa"),
"OpenPanguMTPModel": ("openpangu_mtp", "OpenPanguMTP"),
"Qwen3NextMTP": ("qwen3_next_mtp", "Qwen3NextMTP"),
"Qwen4ExpMTP": ("vllm.models.qwen4_exp", "Qwen4ExpMTP"),
"Step3p5MTP": ("step3p5_mtp", "Step3p5MTP"),
"Qwen3_5MTP": ("qwen3_5_mtp", "Qwen3_5MTP"),
"Qwen3_5MoeMTP": ("qwen3_5_mtp", "Qwen3_5MoeMTP"),
+61
View File
@@ -0,0 +1,61 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Qwen4Exp model package."""
from typing import TYPE_CHECKING, Any
from .common.hyperconnection import (
GatedResidual,
GroupedGemmaRMSNorm,
HyperConnectionBase,
HyperConnectionConfig,
)
if TYPE_CHECKING:
from .nvidia.model import (
Qwen4ExpForCausalLM,
Qwen4ExpForConditionalGeneration,
)
from .nvidia.mtp import Qwen4ExpMTP
def __getattr__(name: str) -> Any:
if name in {
"Qwen4ExpForCausalLM",
"Qwen4ExpForConditionalGeneration",
"Qwen4ExpMTP",
}:
from vllm.platforms import current_platform
if current_platform.is_xpu() or current_platform.is_tpu():
raise NotImplementedError("Qwen4Exp currently supports CUDA and ROCm only")
if current_platform.is_rocm():
from .amd.model import (
Qwen4ExpForCausalLM,
Qwen4ExpForConditionalGeneration,
)
from .amd.mtp import Qwen4ExpMTP
else:
from .nvidia.model import (
Qwen4ExpForCausalLM,
Qwen4ExpForConditionalGeneration,
)
from .nvidia.mtp import Qwen4ExpMTP
return {
"Qwen4ExpForCausalLM": Qwen4ExpForCausalLM,
"Qwen4ExpForConditionalGeneration": (Qwen4ExpForConditionalGeneration),
"Qwen4ExpMTP": Qwen4ExpMTP,
}[name]
raise AttributeError(name)
__all__ = [
"GatedResidual",
"GroupedGemmaRMSNorm",
"HyperConnectionBase",
"HyperConnectionConfig",
"Qwen4ExpForCausalLM",
"Qwen4ExpForConditionalGeneration",
"Qwen4ExpMTP",
]
+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,205 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""HyperConnection (Gated Residual) utilities for the AMD model variant.
Implements the HyperConnection residual scheme proposed in
"HyperConnections" (https://arxiv.org/abs/2409.19606). This AMD variant
delays each HC combine to the following HC mix boundary. HC glue kernels,
including fused combine+RMSNorm, live in ``ops/hc.py``; projections remain
standard vLLM Linear modules.
Hidden states between layers have shape ``[..., HC*HS]`` with HS inner
(HC outer, HS inner — checkpoint-native layout).
Typical usage inside a transformer decoder layer::
self.attn_hc = GatedResidual(hc_config)
hidden_states, block_input, injection = self.attn_hc.mix(hidden_states)
attention_output = attention(block_input)
hidden_states, block_input, injection = self.mlp_hc.combine_and_mix(
hidden_states, attention_output, injection
)
"""
import torch
from torch import nn
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
ReplicatedLinear,
)
from vllm.model_executor.models.utils import maybe_prefix
from ..common.hyperconnection import (
GroupedGemmaRMSNorm,
HyperConnectionConfig,
)
from .ops.hc import (
grouped_gemma_rmsnorm,
hc_combine,
hc_combine_norm,
hc_gate_mix,
hc_silu,
)
# ---------------------------------------------------------------------------
# Gated-residual variant
# ---------------------------------------------------------------------------
class GatedResidual(nn.Module):
"""Gated HyperConnection with learnable low-rank mixing and injection.
``combine_and_mix()`` runs the pre pipeline (grouped GemmaRMSNorm -> merged
low-rank down+inject GEMM -> silu -> up GEMM -> sigmoid -> gated mean
over the HC streams). When passed a pending block output and an injection,
it fuses their residual combine with the RMSNorm. Final mixers use
``use_combine=False`` and do not produce a new injection.
Weights: the norm owns the grouped GemmaRMSNorm affine; the projections
are vLLM Linear modules (merged replicated linear for down+inject), so
GEMM dispatch (e.g. the low-latency skinny GEMM) applies through the
standard quant_method mechanism.
"""
def __init__(
self,
config: HyperConnectionConfig,
use_combine: bool = True,
prefix: str = "",
) -> None:
super().__init__()
self.config = config
self.lora_rank = config.hc_lowrank
self.hc_count = config.hc_count
self.hidden_size = config.hidden_size
self.use_combine = use_combine
norm_size = (
self.hyper_hidden_size if config.hc_per_branch_norm else config.hidden_size
)
group_size = config.hidden_size if config.hc_per_branch_norm else None
# Normalize each H-sized HC stream independently while retaining a
# separate affine weight for every element of the HC*H layout.
self.hc_norm = GroupedGemmaRMSNorm(
norm_size,
eps=config.rms_norm_eps,
group_size=group_size,
dtype=config.params_dtype,
)
# -- vLLM Linear weights --------------------------------------------
# The merged skinny-GEMM shape is physically padded to 16 rows for
# alignment and efficient backend dispatch.
self.pad_size = (-(self.lora_rank + self.hc_count)) % 16 if use_combine else 0
if use_combine:
self.input_mix_weight_down_block_inject = MergedColumnParallelLinear(
self.hyper_hidden_size,
[self.lora_rank, self.hc_count]
+ ([self.pad_size] if self.pad_size else []),
bias=False,
params_dtype=config.params_dtype,
quant_config=None,
prefix=maybe_prefix(prefix, "input_mix_weight_down_block_inject"),
return_bias=False,
disable_tp=True,
)
else:
self.input_mix_weight_down = ReplicatedLinear(
self.hyper_hidden_size,
self.lora_rank,
bias=False,
params_dtype=config.params_dtype,
quant_config=None,
prefix=maybe_prefix(prefix, "input_mix_weight_down"),
return_bias=False,
)
self.input_mix_weight_up = ReplicatedLinear(
self.lora_rank,
self.hyper_hidden_size,
bias=False,
params_dtype=config.params_dtype,
quant_config=None,
prefix=maybe_prefix(prefix, "input_mix_weight_up"),
return_bias=False,
)
def mix(
self, hidden_states: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
xn = grouped_gemma_rmsnorm(
hidden_states,
self.hc_norm.weight,
self.config.rms_norm_eps,
self.hc_count,
)
if self.use_combine:
# produce injection logits for combine
split_sizes = [self.lora_rank, self.hc_count, self.pad_size]
down_and_injection = self.input_mix_weight_down_block_inject(xn)
lora, injection, _ = down_and_injection.split(split_sizes, dim=-1)
else:
lora = self.input_mix_weight_down(xn)
injection = None
lora = hc_silu(lora, self.hc_count)
gate = self.input_mix_weight_up(lora) # [M, D]
block_input = hc_gate_mix(xn, gate, self.hc_count)
return hidden_states, block_input, injection
def combine_and_mix(
self,
hidden_states: torch.Tensor,
prev_block_output: torch.Tensor,
prev_injection: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
"""Consume a pending combine, then prepare the next block input.
``hidden_states`` is the multi-stream state from before the pending
block's mix. Its combine with ``block_output`` is fused with this
module's input RMSNorm.
"""
hidden_states, xn = hc_combine_norm(
hidden_states,
prev_block_output,
prev_injection,
self.hc_norm.weight,
self.config.rms_norm_eps,
self.hc_count,
)
if self.use_combine:
# produce injection logits for combine
split_sizes = [self.lora_rank, self.hc_count, self.pad_size]
down_and_injection = self.input_mix_weight_down_block_inject(xn)
lora, injection, _ = down_and_injection.split(split_sizes, dim=-1)
else:
lora = self.input_mix_weight_down(xn)
injection = None
lora = hc_silu(lora, self.hc_count)
gate = self.input_mix_weight_up(lora) # [M, D]
block_input = hc_gate_mix(xn, gate, self.hc_count)
return hidden_states, block_input, injection
def combine(
self,
hidden_states: torch.Tensor,
block_output: torch.Tensor,
injection: torch.Tensor,
) -> torch.Tensor:
return hc_combine(hidden_states, block_output, injection, self.hc_count)
@property
def hyper_hidden_size(self) -> int:
return self.hc_count * self.hidden_size
__all__ = [
"GatedResidual",
"GroupedGemmaRMSNorm",
"HyperConnectionConfig",
]
+321
View File
@@ -0,0 +1,321 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Qwen4Exp weight-free QSA indexer."""
from __future__ import annotations
from typing import cast
import torch
from torch import nn
from vllm.config import VllmConfig
from vllm.forward_context import get_forward_context
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding.mrope import triton_mrope
from vllm.transformers_utils.configs.qwen4_exp import (
Qwen4ExpTextConfig,
)
from ..common.qsa_cache import (
QSACompressedKeyCache,
QSAForwardMetadata,
QSAKeyStateCache,
canonical_qsa_rope_positions,
)
def apply_qsa_rope(
rotary_emb: nn.Module,
positions: torch.Tensor,
tensor: torch.Tensor,
) -> torch.Tensor:
"""Apply the main attention's exact 1D/MRoPE composition to QSA heads."""
num_tokens, _, head_dim = tensor.shape
rotary_dim = rotary_emb.rotary_dim
cache = rotary_emb._match_cos_sin_cache_dtype(tensor) # noqa: SLF001
cos_sin = cache[positions]
cos, sin = cos_sin.chunk(2, dim=-1)
if positions.ndim == 2:
shape = tensor.shape
tensor, _ = triton_mrope(
tensor.reshape(num_tokens, -1),
tensor.new_empty((num_tokens, head_dim)),
cos,
sin,
rotary_emb.mrope_section,
head_dim,
rotary_dim,
rotary_emb.mrope_interleaved,
rotary_emb.is_neox_style,
)
return tensor.reshape(shape)
rotated = rotary_emb.apply_rotary_emb(
tensor[..., :rotary_dim],
cos,
sin,
)
return torch.cat((rotated, tensor[..., rotary_dim:]), dim=-1)
def apply_qsa_rmsnorm(
norm: GemmaRMSNorm,
tensor: torch.Tensor,
) -> torch.Tensor:
"""Use vLLM's portable RMSNorm implementation on ROCm."""
return cast(torch.Tensor, norm(tensor))
class QSAIndexer(nn.Module):
"""Replicated Q/K projection plus paged, weight-free QSA selection.
``prefix`` must be the checkpoint's indexer prefix, normally
``model.layers.N.self_attn.indexer``. Consequently the trainable names are
``index_qk_proj``, ``q_layernorm`` and ``k_layernorm`` under that prefix.
"""
def __init__(
self,
*,
vllm_config: VllmConfig,
config: Qwen4ExpTextConfig,
layer_id: int,
rotary_emb: nn.Module,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
if vllm_config.cache_config is None:
raise ValueError("QSA requires a paged KV cache")
if vllm_config.model_config.dtype != torch.bfloat16:
raise NotImplementedError("Qwen4Exp QSA currently requires BF16")
self.layer_id = int(layer_id)
self.index_n_heads = int(config.indexer_n_heads)
self.index_kv_heads = int(config.indexer_kv_heads)
self.index_head_dim = int(config.indexer_head_dim)
self.token_topk = int(config.indexer_budget)
self.compress_ratio = int(config.indexer_compress_ratio)
self.rotary_emb = rotary_emb
self.prefix = prefix
# MTP step 0 selects the target-aligned rows; later steps reuse them
# while continuing to update the QSA side cache.
self.skip_topk = False
self.index_qk_proj = ReplicatedLinear(
int(config.hidden_size),
(self.index_n_heads + self.index_kv_heads) * self.index_head_dim,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.index_qk_proj" if prefix else "index_qk_proj",
)
self.q_layernorm = GemmaRMSNorm(
self.index_head_dim,
eps=float(getattr(config, "rms_norm_eps", 1e-6)),
)
self.k_layernorm = GemmaRMSNorm(
self.index_head_dim,
eps=float(getattr(config, "rms_norm_eps", 1e-6)),
)
cache_config = vllm_config.cache_config
cache_prefix = f"{prefix}." if prefix else ""
self.raw_key_cache = QSAKeyStateCache(
head_size=self.index_head_dim,
dtype=torch.bfloat16,
cache_rope_positions=vllm_config.model_config.uses_mrope,
prefix=f"{cache_prefix}raw_key_cache",
cache_config=cache_config,
compress_ratio=self.compress_ratio,
vllm_config=vllm_config,
)
self.compressed_key_cache = QSACompressedKeyCache(
head_size=self.index_head_dim,
dtype=torch.bfloat16,
compress_ratio=self.compress_ratio,
prefix=f"{cache_prefix}compressed_key_cache",
cache_config=cache_config,
vllm_config=vllm_config,
)
@property
def output_width(self) -> int:
return self.token_topk + self.compress_ratio - 1
def project_qk(
self,
hidden_states: torch.Tensor,
positions: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Project replicated Q/K, normalize+rotate Q, and preserve raw K."""
qk, _ = self.index_qk_proj(hidden_states)
q_raw, token_k = qk.split(
(
self.index_n_heads * self.index_head_dim,
self.index_kv_heads * self.index_head_dim,
),
dim=-1,
)
q = q_raw.reshape(-1, self.index_n_heads, self.index_head_dim)
q = apply_qsa_rmsnorm(
self.q_layernorm,
q.reshape(-1, self.index_head_dim),
).reshape_as(q)
q = apply_qsa_rope(self.rotary_emb, positions, q)
return q, token_k.reshape(-1, 1, self.index_head_dim)
def normalize_compressed_keys(
self,
compressed_keys: torch.Tensor,
first_rope_positions: torch.Tensor,
) -> torch.Tensor:
"""Normalize pooled K and apply the first token's exact group position."""
keys = compressed_keys.reshape(-1, self.index_head_dim)
keys = apply_qsa_rmsnorm(self.k_layernorm, keys).reshape(
-1, 1, self.index_head_dim
)
if getattr(self.rotary_emb, "mrope_section", None):
positions = first_rope_positions.transpose(0, 1)
else:
positions = first_rope_positions[:, 0]
return apply_qsa_rope(self.rotary_emb, positions, keys)
def _metadata(
self,
) -> tuple[QSAForwardMetadata, QSAForwardMetadata] | None:
metadata = get_forward_context().attn_metadata
if isinstance(metadata, list):
metadata = metadata[0]
if not isinstance(metadata, dict):
return None
raw = cast(QSAForwardMetadata, metadata[self.raw_key_cache.prefix])
compressed = cast(
QSAForwardMetadata, metadata[self.compressed_key_cache.prefix]
)
if raw.num_actual_tokens != compressed.num_actual_tokens:
raise RuntimeError("QSA side-cache metadata token counts disagree")
if not raw.logical_positions.is_cuda and (
not torch.equal(raw.logical_positions, compressed.logical_positions)
):
raise RuntimeError("QSA side-cache metadata positions disagree")
return raw, compressed
def _update_and_compress(
self,
token_k: torch.Tensor,
positions: torch.Tensor,
raw_metadata: QSAForwardMetadata,
compressed_metadata: QSAForwardMetadata,
) -> None:
num_tokens = raw_metadata.num_actual_tokens
raw_key_cache = self.raw_key_cache.key_cache
rope_position_cache = self.raw_key_cache.rope_position_cache
from .ops.qsa import qsa_compress_groups_with_ratio, qsa_store_cache_rows
if rope_position_cache is None:
position_rows = raw_metadata.logical_positions.view(-1, 1, 1).expand(
-1, 1, 3
)
else:
position_rows = canonical_qsa_rope_positions(positions)[:num_tokens].to(
device=raw_key_cache.device
)
pooled, first_positions = qsa_compress_groups_with_ratio(
token_k[:num_tokens],
position_rows,
raw_key_cache,
raw_metadata.block_table,
raw_metadata.token_to_req,
raw_metadata.query_start_loc,
raw_metadata.logical_positions,
compressed_metadata.slot_mapping,
self.compress_ratio,
rope_position_cache,
)
normalized = self.normalize_compressed_keys(pooled, first_positions)
qsa_store_cache_rows(
self.compressed_key_cache.kv_cache,
compressed_metadata.slot_mapping,
normalized,
)
qsa_store_cache_rows(
raw_key_cache,
raw_metadata.slot_mapping,
token_k[:num_tokens],
)
if rope_position_cache is not None:
qsa_store_cache_rows(
rope_position_cache,
raw_metadata.slot_mapping,
position_rows,
)
def _select(
self,
q: torch.Tensor,
metadata: QSAForwardMetadata,
out: torch.Tensor | None,
) -> torch.Tensor:
from .ops.qsa import qsa_select_paged_tokens
return qsa_select_paged_tokens(
q,
self.compressed_key_cache.kv_cache,
metadata.block_table,
metadata.token_to_req,
metadata.logical_positions,
metadata.seq_lens,
self.token_topk,
self.compress_ratio,
out,
)
def forward(
self,
hidden_states: torch.Tensor,
positions: torch.Tensor,
out: torch.Tensor | None = None,
) -> torch.Tensor:
"""Return fixed-width request-relative token indices padded with ``-1``."""
metadata = self._metadata()
if metadata is None:
# Preserve step-0 indices when later MTP steps reuse the buffer.
if self.skip_topk and out is not None:
return out
result = torch.full(
(hidden_states.shape[0], self.output_width),
-1,
dtype=torch.int32,
device=hidden_states.device,
)
if out is not None:
out.copy_(result)
return out
return result
raw_metadata, compressed_metadata = metadata
num_tokens = raw_metadata.num_actual_tokens
q, token_k = self.project_qk(
hidden_states[:num_tokens], positions[..., :num_tokens]
)
self._update_and_compress(
token_k,
positions[..., :num_tokens],
raw_metadata,
compressed_metadata,
)
if self.skip_topk:
if out is None:
raise RuntimeError("QSA top-k reuse requires an output buffer")
return out
return self._select(q, compressed_metadata, out)
__all__ = ["QSAIndexer", "apply_qsa_rope"]
@@ -0,0 +1,15 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Qwen4Exp low-latency GEMM hook for AMD ROCm."""
import torch
from torch import nn
def enable_qwen4_exp_low_latency_gemm(
module: nn.Module,
dtype: torch.dtype,
) -> None:
"""Keep the standard vLLM linear methods on AMD ROCm."""
del module, dtype
File diff suppressed because it is too large Load Diff
+143
View File
@@ -0,0 +1,143 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Model-runner state for Qwen4Exp PLE inputs."""
from typing import Any
import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
from vllm.v1.worker.gpu.states import RequestState
class Qwen4ExpModelState(MambaHybridModelState):
"""Add rollback-safe PLE n-gram context to the model inputs."""
def __init__(
self,
vllm_config: VllmConfig,
model: nn.Module,
encoder_cache: EncoderCache | None,
device: torch.device,
) -> None:
super().__init__(vllm_config, model, encoder_cache, device)
config = self.model_config.hf_text_config
self.uses_ngram_embedding = bool(config.ple_layer_ids)
if not self.uses_ngram_embedding:
self.ngram_context_len = 0
self.ngram_eos_token_id = 0
return
if vllm_config.parallel_config.pipeline_parallel_size > 1:
raise RuntimeError(
"N-gram PLE embedding currently requires "
"pipeline_parallel_size=1 because non-first pipeline ranks do "
"not receive the raw input_ids required by PLE. Please run "
"with PP=1."
)
self.ngram_context_len = int(config.ngram_size) - 1
if self.ngram_context_len <= 0:
raise ValueError("N-gram embedding requires context length >= 1.")
self.ngram_eos_token_id = int(config.eos_token_id)
self.ngram_context = torch.full(
(self.max_num_reqs, self.ngram_context_len),
self.ngram_eos_token_id,
dtype=torch.int32,
device=self.device,
)
self.ngram_context_offsets = torch.arange(
-self.ngram_context_len,
0,
dtype=torch.int64,
device=self.device,
)
self.ple_query_start_loc = torch.zeros(
self.max_num_reqs + 1,
dtype=torch.int32,
device=self.device,
)
def _prepare_ngram_context(
self,
input_batch: InputBatch,
req_states: RequestState,
) -> torch.Tensor:
num_reqs = input_batch.num_reqs
num_reqs_padded = input_batch.num_reqs_after_padding
context = self.ngram_context[:num_reqs_padded]
context.fill_(self.ngram_eos_token_id)
if num_reqs == 0:
return context
request_indices = input_batch.idx_mapping[:num_reqs].long()
context_end = req_states.num_computed_tokens.gpu[request_indices].long()
token_indices = context_end.unsqueeze(1) + self.ngram_context_offsets
valid_tokens = token_indices >= 0
token_indices.clamp_min_(0)
context_tokens = req_states.all_token_ids.gpu[
request_indices.unsqueeze(1), token_indices
]
context[:num_reqs].copy_(
torch.where(
valid_tokens,
context_tokens,
context_tokens.new_full((), self.ngram_eos_token_id),
)
)
return context
def prepare_inputs(
self,
input_batch: InputBatch,
req_states: RequestState,
) -> dict[str, Any]:
model_inputs = super().prepare_inputs(input_batch, req_states)
if not self.uses_ngram_embedding:
return model_inputs
num_reqs_padded = input_batch.num_reqs_after_padding
query_start_loc = self.ple_query_start_loc[: num_reqs_padded + 1]
query_start_loc.copy_(input_batch.query_start_loc[: num_reqs_padded + 1])
model_inputs.update(
query_start_loc=query_start_loc,
ngram_context=self._prepare_ngram_context(input_batch, req_states),
)
return model_inputs
def prepare_dummy_inputs(
self,
num_reqs: int,
num_tokens: int,
) -> dict[str, Any]:
model_inputs = super().prepare_dummy_inputs(num_reqs, num_tokens)
if not self.uses_ngram_embedding:
return model_inputs
query_start_loc = self.ple_query_start_loc[: num_reqs + 1]
query_start_loc[0] = 0
tokens_per_req, num_extra_tokens = divmod(num_tokens, num_reqs)
query_lens = torch.full(
(num_reqs,),
tokens_per_req,
dtype=query_start_loc.dtype,
device=query_start_loc.device,
)
if num_extra_tokens > 0:
query_lens[-num_extra_tokens:] += 1
torch.cumsum(query_lens, dim=0, out=query_start_loc[1:])
ngram_context = self.ngram_context[:num_reqs]
ngram_context.fill_(self.ngram_eos_token_id)
model_inputs.update(
query_start_loc=query_start_loc,
ngram_context=ngram_context,
)
return model_inputs
__all__ = ["Qwen4ExpModelState"]
+462
View File
@@ -0,0 +1,462 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inference-only Qwen4Exp MTP (Multi-Token Predictor) model.
The MTP draft model reuses the Qwen4Exp backbone (PLE/HC/MoE) but:
- drops all multi-modal handling (text-only),
- forces PLE off while keeping the main model's HC stream count,
- fuses the backbone hidden and the new-token embedding via
``residual_linear_shared`` (fc_embedding + shared fc_hidden) instead of
the ``Linear(2H, H)`` + repeat used by other MTP variants,
- emits TWO hidden streams per step (scheme A): a single stream [T, H]
(final-mixer collapsed, fed to the LM head) and a pre-final-mixer
multi stream [T, hc_count*H] (fed to the next draft step).
"""
from collections.abc import Iterable
import regex as re
import torch
from torch import nn
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig, replace, set_current_vllm_config
from vllm.distributed import get_pp_group
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.utils import configure_quant_config
from vllm.model_executor.models.interfaces import SupportsPP
from vllm.model_executor.models.qwen3_5 import Qwen3_5Model
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
PPMissingLayer,
WeightsMapper,
get_draft_quant_config,
make_empty_intermediate_tensors_factory,
maybe_fuse_shared_experts,
maybe_prefix,
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.qwen4_exp import (
Qwen4ExpTextConfig,
)
from .hyperconnection import GatedResidual, HyperConnectionConfig
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
from .model import (
_HC_WEIGHTS_MAPPER,
_QWEN4_EXP_IGNORED_MISSING_SUFFIXES,
Qwen4ExpDecoderLayer,
Qwen4ExpMixtureOfExperts,
Qwen4ExpSparseMoeBlock,
)
def _remap_ignored_layers(
ignored_layers: list[str],
mtp_start_layer_idx: int,
) -> list[str]:
remapped: list[str] = []
for name in ignored_layers:
if name.startswith("mtp."):
new_name = re.sub(
r"(?<=\.layers\.)\d+",
lambda m: str(mtp_start_layer_idx + int(m.group(0))),
name,
)
remapped.append(new_name)
else:
remapped.append(name)
return remapped
def _remap_mtp_weight_name(name: str) -> str | None:
"""Map Qwen4Exp checkpoint paths into the standalone draft model."""
for checkpoint_prefix in (
"model.language_model.",
"language_model.",
):
if name.startswith(checkpoint_prefix):
name = name.removeprefix(checkpoint_prefix)
break
if name.startswith("embed_tokens."):
name = f"model.{name}"
if name.startswith("model.mtp."):
name = name.removeprefix("model.")
if name.startswith("mtp.shared_head.head."):
return name.replace("mtp.shared_head.head.", "lm_head.", 1)
if name.startswith("model.shared_head.head."):
return name.replace("model.shared_head.head.", "lm_head.", 1)
if name.startswith("shared_head.head."):
return name.replace("shared_head.head.", "lm_head.", 1)
if name.startswith("model.lm_head."):
return name.removeprefix("model.")
if name.startswith("mtp."):
return name.replace("mtp.", "model.", 1)
if name.startswith("model.embed_tokens.") or name.startswith("lm_head."):
return name
return None
def _make_draft_vllm_config(
vllm_config: VllmConfig,
mtp_start_layer_idx: int,
) -> VllmConfig:
"""Ensure that the draft model config is set in the vLLM config."""
speculative_config = vllm_config.speculative_config
if speculative_config is None or speculative_config.draft_model_config is None:
raise ValueError("speculative_config.draft_model_config must be set")
draft_quant_config = get_draft_quant_config(vllm_config)
# inject packed and ignored modules to the quantization config of draft model
if draft_quant_config is not None:
configure_quant_config(draft_quant_config, Qwen4ExpMTP)
ignored_layers = getattr(draft_quant_config, "ignored_layers", None)
if ignored_layers:
setattr( # noqa: B010
draft_quant_config,
"ignored_layers",
_remap_ignored_layers(ignored_layers, mtp_start_layer_idx),
)
exclude_modules = getattr(draft_quant_config, "exclude_modules", None)
if exclude_modules:
setattr( # noqa: B010
draft_quant_config,
"exclude_modules",
_remap_ignored_layers(exclude_modules, mtp_start_layer_idx),
)
draft_vllm_config = replace(
vllm_config,
model_config=speculative_config.draft_model_config,
)
# VllmConfig post-init derives the target quant config, so restore the
# independently resolved draft quant config after replacement.
draft_vllm_config.quant_config = draft_quant_config
return draft_vllm_config
@support_torch_compile(
dynamic_arg_dims={
"input_ids": 0,
"positions": -1,
"intermediate_tensors": 0,
"inputs_embeds": 0,
"hidden_states": 0,
}
)
class Qwen4ExpMultiTokenPredictor(nn.Module):
hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _HC_WEIGHTS_MAPPER
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
model_config = vllm_config.model_config
config: Qwen4ExpTextConfig = model_config.hf_text_config
self.config = config
self.vocab_size = config.vocab_size
self.mtp_start_layer_idx = config.num_hidden_layers
self.num_mtp_layers = getattr(config, "mtp_num_hidden_layers", 1)
self.hidden_size = config.hidden_size
self.hc_count = config.hc_count
self.embed_tokens = VocabParallelEmbedding(self.vocab_size, self.hidden_size)
draft_vllm_config = _make_draft_vllm_config(
vllm_config,
self.mtp_start_layer_idx,
)
with set_current_vllm_config(draft_vllm_config, prefix=prefix):
# residual_linear_shared fusion: fc_embedding projects the token
# embedding, fc_hidden (shared across HC branches) projects the
# backbone hidden; the embedding is added as a residual to every
# branch (see mtp_residual_linear_shared.md).
self.fc_embedding = ColumnParallelLinear(
self.hidden_size,
self.hidden_size,
gather_output=True,
bias=False,
return_bias=False,
quant_config=draft_vllm_config.quant_config,
prefix=f"{prefix}.fc_embedding",
)
self.fc_hidden = ColumnParallelLinear(
self.hidden_size,
self.hidden_size,
gather_output=True,
bias=False,
return_bias=False,
quant_config=draft_vllm_config.quant_config,
prefix=f"{prefix}.fc_hidden",
)
self.layers = nn.ModuleList(
Qwen4ExpDecoderLayer(
draft_vllm_config,
layer_type="full_attention",
prefix=f"{prefix}.layers.{self.mtp_start_layer_idx + idx}",
)
for idx in range(self.num_mtp_layers)
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
Qwen4ExpSparseMoeBlock,
"mlp",
)
self.pre_fc_norm_embedding = GemmaRMSNorm(
self.hidden_size, eps=config.rms_norm_eps
)
self.pre_fc_norm_hidden = GemmaRMSNorm(
self.hidden_size * self.hc_count, eps=config.rms_norm_eps
)
# HC final mixer collapses the multi stream into [T, H] for the LM head.
hc_config = HyperConnectionConfig(
hc_count=config.hc_count,
hidden_size=config.hidden_size,
params_dtype=torch.bfloat16,
hc_lowrank=config.hc_lowrank,
rms_norm_eps=config.rms_norm_eps,
hc_per_branch_norm=True,
)
self.hyper_connection_mixer = GatedResidual(
hc_config,
use_combine=False,
prefix=maybe_prefix(prefix, "hyper_connection_mixer"),
)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states"], self.hidden_size * self.hc_count
)
def _iter_qsa_attentions(self):
"""Yield MTP attention modules that own a QSA indexer."""
for layer in self.layers:
attention = getattr(layer, "self_attn", None)
if (
attention is not None
and getattr(attention, "indexer", None) is not None
):
yield attention
def set_skip_topk(self, skip: bool) -> None:
"""Select on MTP step 0 and reuse its QSA indices on later steps."""
for attention in self._iter_qsa_attentions():
attention.indexer.skip_topk = skip
def compact_topk_indices(self, row_indices: torch.Tensor) -> None:
"""Keep each request's target-aligned step-0 sparse-index row."""
num_rows = row_indices.numel()
for attention in self._iter_qsa_attentions():
buffer = attention.topk_indices_buffer
selected = buffer.index_select(0, row_indices)
buffer[:num_rows].copy_(selected)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
hidden_states: torch.Tensor | None = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor] | IntermediateTensors:
hc_count = self.hc_count
hidden_size = self.hidden_size
if get_pp_group().is_first_rank:
assert hidden_states is not None
if inputs_embeds is None:
assert input_ids is not None
inputs_embeds = self.embed_input_ids(input_ids)
# Embedding branch: pre-norm -> fc_embedding -> [T, H].
inputs_embeds = self.pre_fc_norm_embedding(inputs_embeds)
inputs_embeds = self.fc_embedding(inputs_embeds)
# Backbone hidden is multi-stream [T, hc_count*H] (scheme A:
# the main model truly emits the pre-final-mixer multi stream
# on the first step; subsequent steps reuse the prior draft
# step's multi stream).
num_tokens = hidden_states.shape[0]
hidden_states = hidden_states.view(num_tokens, hc_count, hidden_size)
hidden_states = self.pre_fc_norm_hidden(hidden_states.flatten(-2)).view(
num_tokens, hc_count, hidden_size
)
hidden_states = self.fc_hidden(hidden_states)
# Add the embedding residual to every branch, then fold back
# to [T, hc_count*H] (HC outer, HS inner) for the HC decoder.
hidden_states = inputs_embeds.unsqueeze(-2) + hidden_states
hidden_states = hidden_states.flatten(-2)
else:
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"]
current_step_idx = spec_step_idx % self.num_mtp_layers
layer = self.layers[current_step_idx]
hidden_states, block_output, injection = layer(
hidden_states=hidden_states,
prev_block_output=None,
prev_injection=None,
positions=positions,
input_ids=None,
query_start_loc=None,
ngram_context=None,
)
if not get_pp_group().is_last_rank:
# As in the target model, PP carries a materialized tensor rather
# than the delayed hidden/output/injection tuple.
hidden_states = layer.mlp_hyper_connection.combine(
hidden_states, block_output, injection
)
return IntermediateTensors({"hidden_states": hidden_states})
# Last PP rank finalize. Keep both:
# (A) sample_hidden_states [T, H] -> single stream for the LM head
# (B) multi_hidden [T, hc_count*H] -> pre-final-mixer multi stream
# for the next draft step (zero extra compute, just kept).
multi_hidden, sample_hidden_states, _ = (
self.hyper_connection_mixer.combine_and_mix(
hidden_states, block_output, injection
)
)
return sample_hidden_states, multi_hidden
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
weights = maybe_fuse_shared_experts(
weights,
enabled=self.is_fused_shared_expert_enabled,
n_routed_experts=getattr(self.config, "num_experts", 0) or 0,
n_shared_experts=1,
ckpt_prefix="mlp.shared_expert",
)
mapper = self.hf_to_vllm_mapper | WeightsMapper(
orig_to_new_substr={"hyper_connection_mixer.block_inject_weight": None}
)
loader = AutoWeightsLoader(
self,
ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(),
)
return loader.load_weights(weights, mapper=mapper)
@support_torch_compile(
dynamic_arg_dims={
"input_ids": 0,
"positions": -1,
"intermediate_tensors": 0,
"inputs_embeds": 0,
"hidden_states": 0,
}
)
class Qwen4ExpMTP(nn.Module, SupportsPP, Qwen4ExpMixtureOfExperts):
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
"gate_up_proj": ["gate_proj", "up_proj"],
"in_proj_qkvz": ["in_proj_qkv", "in_proj_z"],
"in_proj_ba": ["in_proj_b", "in_proj_a"],
"input_mix_weight_down_block_inject": [
"input_mix_weight_down",
"block_inject_weight",
"_input_mix_padding",
],
}
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config
self.vllm_config = vllm_config
cache_config = vllm_config.cache_config
if cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Qwen4ExpMTP currently does not support 'all' prefix caching, "
"please use '--mamba-cache-mode=align' instead"
)
self.quant_config = vllm_config.quant_config
super().__init__()
self.config = config
self.model = Qwen4ExpMultiTokenPredictor(
vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "mtp"),
)
if get_pp_group().is_last_rank:
if config.tie_word_embeddings:
self.lm_head = self.model.embed_tokens
else:
self.lm_head = ParallelLMHead(
config.vocab_size,
config.hidden_size,
prefix=maybe_prefix(prefix, "lm_head"),
)
else:
self.lm_head = PPMissingLayer()
self.logits_processor = LogitsProcessor(config.vocab_size)
self.make_empty_intermediate_tensors = (
self.model.make_empty_intermediate_tensors
)
self.set_moe_parameters(self.model.layers)
enable_qwen4_exp_low_latency_gemm(self, vllm_config.model_config.dtype)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
hidden_states: torch.Tensor | None = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor] | IntermediateTensors:
return self.model(
input_ids,
positions,
hidden_states,
intermediate_tensors,
inputs_embeds,
spec_step_idx=spec_step_idx,
)
def compute_logits(
self, hidden_states: torch.Tensor, spec_step_idx: int = 0
) -> torch.Tensor | None:
return self.logits_processor(self.lm_head, hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
def remap_weight_names():
for name, weight in weights:
remapped_name = _remap_mtp_weight_name(name)
if remapped_name is not None:
yield remapped_name, weight
mapper = WeightsMapper(
orig_to_new_substr={"hyper_connection_mixer.block_inject_weight": None}
)
loader = AutoWeightsLoader(
self,
ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(),
)
return loader.load_weights(remap_weight_names(), mapper=mapper)
__all__ = ["Qwen4ExpMTP", "Qwen4ExpMultiTokenPredictor"]
@@ -0,0 +1,3 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""AMD ROCm Qwen4Exp kernels; import leaf modules directly."""
+487
View File
@@ -0,0 +1,487 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""AMD ROCm HyperConnection kernels for Qwen4Exp."""
import torch
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import direct_register_custom_op
@triton.jit
def _grouped_gemma_rmsnorm_kernel(
x_ptr,
w_ptr,
y_ptr,
stride_x,
stride_y,
DIM: tl.constexpr,
NUM_GROUPS: tl.constexpr,
W_SHARED: tl.constexpr,
EPS: tl.constexpr,
launch_pdl: tl.constexpr,
) -> None:
GROUP_DIM: tl.constexpr = DIM // NUM_GROUPS
BLOCK_SIZE: tl.constexpr = triton.next_power_of_2(GROUP_DIM)
pid = tl.program_id(0)
group_id = pid % NUM_GROUPS
row = pid // NUM_GROUPS
offs_g = tl.arange(0, BLOCK_SIZE)
offsets = group_id * GROUP_DIM + offs_g
mask = offs_g < GROUP_DIM
# A [GROUP_DIM] affine is shared; a [DIM] affine follows the grouped
# checkpoint layout.
w_offs = offs_g if W_SHARED else offsets
if launch_pdl:
tl.extra.cuda.gdc_wait()
x = tl.load(x_ptr + row * stride_x + offsets, mask, other=0.0).to(tl.float32)
w = tl.load(w_ptr + w_offs, mask, other=0.0)
rrms = tl.rsqrt(tl.sum(x * x) / GROUP_DIM + EPS)
# Gemma's (1 + w) affine is written this way to lower to an FMA.
y = x * rrms
y += y * w.to(tl.float32)
if launch_pdl:
tl.extra.cuda.gdc_launch_dependents()
tl.store(y_ptr + row * stride_y + offsets, y, mask)
def _grouped_gemma_rmsnorm(
x: torch.Tensor, weight: torch.Tensor, eps: float, num_groups: int
) -> torch.Tensor:
N, DIM = x.shape
assert x.stride(1) == 1, "grouped Gemma RMSNorm requires unit inner stride"
assert weight.is_contiguous(), "grouped Gemma RMSNorm weight must be contiguous"
assert DIM % num_groups == 0
group_dim = DIM // num_groups
assert weight.numel() in (group_dim, DIM)
y = x.new_empty(x.shape)
_grouped_gemma_rmsnorm_kernel[(N * num_groups,)](
x,
weight,
y,
x.stride(0),
y.stride(0),
DIM,
num_groups,
W_SHARED=weight.numel() == group_dim,
EPS=eps,
launch_pdl=current_platform.is_arch_support_pdl(),
)
return y
@triton.jit
def _hc_silu_kernel(
x_ptr,
y_ptr,
stride_x,
stride_y,
DIM: tl.constexpr,
HC: tl.constexpr,
launch_pdl: tl.constexpr,
) -> None:
BLOCK_SIZE: tl.constexpr = triton.next_power_of_2(DIM)
row = tl.program_id(0)
offs = tl.arange(0, BLOCK_SIZE)
mask = offs < DIM
if launch_pdl:
tl.extra.cuda.gdc_wait()
x = tl.load(x_ptr + row * stride_x + offs, mask).to(tl.float32) / HC
y = x * tl.sigmoid(x)
if launch_pdl:
tl.extra.cuda.gdc_launch_dependents()
tl.store(y_ptr + row * stride_y + offs, y, mask)
def _hc_silu(x: torch.Tensor, hc_count: int) -> torch.Tensor:
num_tokens, DIM = x.shape
assert x.stride(1) == 1
output = x.new_empty(x.shape)
_hc_silu_kernel[(num_tokens,)](
x,
output,
x.stride(0),
output.stride(0),
DIM=DIM,
HC=hc_count,
launch_pdl=current_platform.is_arch_support_pdl(),
)
return output
@triton.jit
def _hc_gate_mix_kernel(
x_ptr,
g_ptr,
y_ptr,
stride_x,
stride_g,
stride_y,
DIM: tl.constexpr,
HC: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
launch_pdl: tl.constexpr,
) -> None:
HC_DIM: tl.constexpr = DIM // HC
row = tl.program_id(0)
tile_id = tl.program_id(1)
offs_inner = tile_id * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = offs_inner < HC_DIM
if launch_pdl:
tl.extra.cuda.gdc_wait()
# The constexpr loop is unrolled and keeps one stream live at a time.
# Materializing [HC, BLOCK_SIZE] more than doubles latency at large M.
acc = tl.zeros([BLOCK_SIZE], dtype=tl.float32)
for stream in tl.static_range(HC):
offsets = stream * HC_DIM + offs_inner
g = tl.load(g_ptr + row * stride_g + offsets, mask, other=0.0)
x = tl.load(x_ptr + row * stride_x + offsets, mask, other=0.0)
acc += tl.sigmoid(g.to(tl.float32)) * x.to(tl.float32)
acc /= HC
if launch_pdl:
tl.extra.cuda.gdc_launch_dependents()
tl.store(y_ptr + row * stride_y + offs_inner, acc, mask)
def _hc_gate_mix(x: torch.Tensor, gate: torch.Tensor, hc_count: int) -> torch.Tensor:
N, DIM = gate.shape
assert x.shape == gate.shape
assert DIM % hc_count == 0
assert x.stride(1) == 1
assert gate.stride(1) == 1
HC_DIM = DIM // hc_count
out = x.new_empty(N, HC_DIM)
BLOCK_SIZE = 512
_hc_gate_mix_kernel[(N, triton.cdiv(HC_DIM, BLOCK_SIZE))](
x,
gate,
out,
x.stride(0),
gate.stride(0),
out.stride(0),
DIM,
hc_count,
BLOCK_SIZE,
launch_pdl=current_platform.is_arch_support_pdl(),
)
return out
@triton.jit
def _hc_combine_kernel(
block_ptr,
res_ptr,
inj_ptr,
out_ptr,
stride_block,
stride_res,
stride_inj,
stride_out,
HC_DIM: tl.constexpr,
HC: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
launch_pdl: tl.constexpr,
) -> None:
HC_PAD: tl.constexpr = triton.next_power_of_2(HC)
row = tl.program_id(0)
tile_id = tl.program_id(1)
offs_inner = tile_id * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask_inner = offs_inner < HC_DIM
offs_hc = tl.arange(0, HC_PAD)
mask_hc = offs_hc < HC
offs = offs_hc[:, None] * HC_DIM + offs_inner[None, :]
mask = mask_hc[:, None] & mask_inner[None, :]
if launch_pdl:
tl.extra.cuda.gdc_wait()
inj = tl.load(inj_ptr + row * stride_inj + offs_hc, mask_hc, other=0.0)
block = tl.load(block_ptr + row * stride_block + offs_inner, mask_inner, other=0.0)
res = tl.load(res_ptr + row * stride_res + offs, mask, other=0.0)
# Keeping HC as a broadcast dimension is faster here than four separate
# residual load/store sequences.
inj = 2.0 * tl.sigmoid(inj.to(tl.float32) / HC)
out = res.to(tl.float32) + block.to(tl.float32)[None, :] * inj[:, None]
if launch_pdl:
tl.extra.cuda.gdc_launch_dependents()
tl.store(out_ptr + row * stride_out + offs, out, mask=mask)
def _hc_combine(
residual: torch.Tensor,
block_output: torch.Tensor,
injection_logits: torch.Tensor,
hc_count: int,
) -> torch.Tensor:
N, DIM = residual.shape
assert DIM % hc_count == 0
hc_dim = DIM // hc_count
assert block_output.shape == (N, hc_dim)
assert injection_logits.shape == (N, hc_count)
assert residual.stride(1) == 1
assert block_output.stride(1) == 1
assert injection_logits.stride(1) == 1
out = residual.new_empty(residual.shape)
BLOCK_SIZE = 512
_hc_combine_kernel[(N, triton.cdiv(hc_dim, BLOCK_SIZE))](
block_output,
residual,
injection_logits,
out,
block_output.stride(0),
residual.stride(0),
injection_logits.stride(0),
out.stride(0),
hc_dim,
hc_count,
BLOCK_SIZE,
launch_pdl=current_platform.is_arch_support_pdl(),
)
return out
@triton.jit
def _hc_combine_norm_kernel(
block_ptr,
res_ptr,
inj_ptr,
w_ptr,
out_ptr,
y_ptr,
stride_block,
stride_res,
stride_inj,
stride_out,
stride_y,
HC_DIM: tl.constexpr,
HC: tl.constexpr,
W_SHARED: tl.constexpr,
EPS: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
launch_pdl: tl.constexpr,
) -> None:
HC_PAD: tl.constexpr = triton.next_power_of_2(HC)
NUM_TILES: tl.constexpr = triton.cdiv(HC_DIM, BLOCK_SIZE)
NUM_TILES_PAD: tl.constexpr = triton.next_power_of_2(NUM_TILES)
row = tl.program_id(0)
stream = tl.program_id(1)
offs_hc = tl.arange(0, HC_PAD)
mask_hc = offs_hc < HC
tile_ids = tl.arange(0, NUM_TILES_PAD)
offs_inner = tile_ids[:, None] * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)[None, :]
mask_inner = offs_inner < HC_DIM
offs = stream * HC_DIM + offs_inner
# Shared norm weights repeat across streams; per-branch weights use the
# same flattened HC layout as the residual.
w_offs = offs_inner if W_SHARED else offs
if launch_pdl:
tl.extra.cuda.gdc_wait()
# Start the uncached residual load first, then issue the other combine
# loads before consuming any of them.
res = tl.load(res_ptr + row * stride_res + offs, mask_inner, other=0.0)
inj = tl.load(inj_ptr + row * stride_inj + offs_hc, mask_hc, other=0.0)
block = tl.load(block_ptr + row * stride_block + offs_inner, mask_inner, other=0.0)
inj = 2.0 * tl.sigmoid(inj.to(tl.float32) / HC)
inj = tl.sum(tl.where(offs_hc == stream, inj, 0.0))
# Round the materialized combine result before normalization. This matches
# the unfused combine -> RMSNorm boundary.
out = (res.to(tl.float32) + block.to(tl.float32) * inj).to(out_ptr.dtype.element_ty)
tl.store(out_ptr + row * stride_out + offs, out, mask=mask_inner)
out = out.to(tl.float32)
# Keep the two-axis reduction: flattening the padded tile is ~40% slower
# at decode sizes.
sum_sq = tl.sum(tl.sum(out * out, axis=1), axis=0)
rrms = tl.rsqrt(sum_sq / HC_DIM + EPS)
if launch_pdl:
tl.extra.cuda.gdc_launch_dependents()
# Loading the weight earlier helps decode but keeps the tile live across
# the reduction and regresses larger batches, so defer it to the norm.
w = tl.load(w_ptr + w_offs, mask_inner, other=0.0)
y = out * rrms
y += y * w.to(tl.float32)
tl.store(y_ptr + row * stride_y + offs, y, mask_inner)
def _hc_combine_norm(
residual: torch.Tensor,
block_output: torch.Tensor,
injection_logits: torch.Tensor,
norm_weight: torch.Tensor,
eps: float,
hc_count: int,
) -> tuple[torch.Tensor, torch.Tensor]:
N, DIM = residual.shape
assert DIM % hc_count == 0
hc_dim = DIM // hc_count
assert block_output.shape == (N, hc_dim)
assert injection_logits.shape == (N, hc_count)
assert residual.stride(1) == 1
assert block_output.stride(1) == 1
assert injection_logits.stride(1) == 1
assert norm_weight.is_contiguous()
assert norm_weight.numel() in (hc_dim, DIM)
out = residual.new_empty(residual.shape)
y = residual.new_empty(residual.shape)
BLOCK_SIZE = 512
_hc_combine_norm_kernel[(N, hc_count)](
block_output,
residual,
injection_logits,
norm_weight,
out,
y,
block_output.stride(0),
residual.stride(0),
injection_logits.stride(0),
out.stride(0),
y.stride(0),
hc_dim,
hc_count,
W_SHARED=norm_weight.numel() == hc_dim,
EPS=eps,
BLOCK_SIZE=BLOCK_SIZE,
launch_pdl=current_platform.is_arch_support_pdl(),
)
return out, y
def _same_shape_fake(x: torch.Tensor, *args) -> torch.Tensor:
return x.new_empty(x.shape)
def _hc_gate_mix_fake(
x: torch.Tensor, gate: torch.Tensor, hc_count: int
) -> torch.Tensor:
del gate
return x.new_empty((x.shape[0], x.shape[1] // hc_count))
def _hc_combine_fake(
residual: torch.Tensor,
block_output: torch.Tensor,
injection_logits: torch.Tensor,
hc_count: int,
) -> torch.Tensor:
del block_output, injection_logits, hc_count
return residual.new_empty(residual.shape)
def _hc_combine_norm_fake(
residual: torch.Tensor,
block_output: torch.Tensor,
injection_logits: torch.Tensor,
norm_weight: torch.Tensor,
eps: float,
hc_count: int,
) -> tuple[torch.Tensor, torch.Tensor]:
del block_output, injection_logits, norm_weight, eps, hc_count
return residual.new_empty(residual.shape), residual.new_empty(residual.shape)
direct_register_custom_op(
op_name="qwen4_exp_grouped_gemma_rmsnorm",
op_func=_grouped_gemma_rmsnorm,
fake_impl=_same_shape_fake,
)
direct_register_custom_op(
op_name="qwen4_exp_hc_silu",
op_func=_hc_silu,
fake_impl=_same_shape_fake,
)
direct_register_custom_op(
op_name="qwen4_exp_hc_gate_mix",
op_func=_hc_gate_mix,
fake_impl=_hc_gate_mix_fake,
)
direct_register_custom_op(
op_name="qwen4_exp_hc_combine",
op_func=_hc_combine,
fake_impl=_hc_combine_fake,
)
direct_register_custom_op(
op_name="qwen4_exp_hc_combine_norm",
op_func=_hc_combine_norm,
fake_impl=_hc_combine_norm_fake,
)
def grouped_gemma_rmsnorm(
x: torch.Tensor, weight: torch.Tensor, eps: float, num_groups: int
) -> torch.Tensor:
return torch.ops.vllm.qwen4_exp_grouped_gemma_rmsnorm(x, weight, eps, num_groups)
def hc_silu(x: torch.Tensor, hc_count: int) -> torch.Tensor:
return torch.ops.vllm.qwen4_exp_hc_silu(x, hc_count)
def hc_gate_mix(x: torch.Tensor, gate: torch.Tensor, hc_count: int) -> torch.Tensor:
return torch.ops.vllm.qwen4_exp_hc_gate_mix(x, gate, hc_count)
def hc_combine(
residual: torch.Tensor,
block_output: torch.Tensor,
injection_logits: torch.Tensor,
hc_count: int,
) -> torch.Tensor:
return torch.ops.vllm.qwen4_exp_hc_combine(
residual, block_output, injection_logits, hc_count
)
def hc_combine_norm(
residual: torch.Tensor,
block_output: torch.Tensor,
injection_logits: torch.Tensor,
norm_weight: torch.Tensor,
eps: float,
hc_count: int,
) -> tuple[torch.Tensor, torch.Tensor]:
return torch.ops.vllm.qwen4_exp_hc_combine_norm(
residual,
block_output,
injection_logits,
norm_weight,
eps,
hc_count,
)
__all__ = [
"grouped_gemma_rmsnorm",
"hc_combine",
"hc_combine_norm",
"hc_gate_mix",
"hc_silu",
]
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+474
View File
@@ -0,0 +1,474 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""AMD ROCm QSA owner with Triton kernels."""
from __future__ import annotations
from typing import ClassVar, cast
import torch
from torch import nn
from vllm.config import VllmConfig
from vllm.config.cache import CacheDType
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.forward_context import get_forward_context
from vllm.model_executor.layers.attention.attention import (
set_default_quant_scales,
)
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.models.qwen3_next import Qwen3NextAttention
from vllm.platforms import current_platform
from vllm.transformers_utils.configs.qwen4_exp import (
Qwen4ExpTextConfig,
)
from vllm.utils.torch_utils import (
LayerNameType,
_encode_layer_name,
_resolve_layer_name,
canonicalize_singleton_dim_strides,
direct_register_custom_op,
kv_cache_dtype_str_to_dtype,
)
from vllm.v1.attention.backend import (
AttentionBackend,
AttentionCGSupport,
AttentionType,
MultipleOf,
)
from vllm.v1.attention.backends.fa_utils import is_flash_attn_varlen_func_available
from vllm.v1.attention.backends.flash_attn import (
FlashAttentionBackend,
FlashAttentionImpl,
FlashAttentionMetadata,
FlashAttentionMetadataBuilder,
)
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheSpec,
get_kv_quant_mode,
)
from ..common.qsa_cache import QSAForwardMetadata
from . import model
from .indexer_qsa import QSAIndexer
class Qwen4ExpQSAMetadataBuilder(FlashAttentionMetadataBuilder):
"""Flash metadata supporting uniform decode and target-verify graphs."""
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
class Qwen4ExpQSAFlashAttentionBackend(FlashAttentionBackend):
"""FullAttentionSpec backend used by the merged QSA owner."""
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = ["auto", "bfloat16"]
@staticmethod
def get_name() -> str:
return "QWEN4_EXP_QSA_TRITON"
@staticmethod
def get_supported_kernel_block_sizes() -> list[int | MultipleOf]:
# QSA consumes manager pages directly and does not use FA4 paged attention.
return [MultipleOf(16)]
@staticmethod
def get_impl_cls() -> type[Qwen4ExpQSAFlashAttentionImpl]:
return Qwen4ExpQSAFlashAttentionImpl
@staticmethod
def get_builder_cls() -> type[Qwen4ExpQSAMetadataBuilder]:
return Qwen4ExpQSAMetadataBuilder
@classmethod
def is_sparse(cls) -> bool:
return True
@classmethod
def supports_kv_connector(cls) -> bool:
return False
class Qwen4ExpQSAFlashAttentionImpl(FlashAttentionImpl):
"""Run paged sparse GQA with the QSA Triton kernel."""
supports_dcp: bool = False
supports_pcp: bool = False
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
if not is_flash_attn_varlen_func_available():
raise NotImplementedError("Qwen4Exp QSA requires FlashAttention")
if self.dcp_world_size != 1:
raise NotImplementedError(
"Qwen4Exp QSA does not support decode context parallelism"
)
if self.kv_cache_dtype not in ("auto", "bfloat16"):
raise NotImplementedError("Qwen4Exp QSA requires a BF16 main KV cache")
self.supports_quant_query_input = False
def forward_qsa(
self,
layer: torch.nn.Module,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
kv_cache: torch.Tensor,
attn_metadata: FlashAttentionMetadata,
output: torch.Tensor,
token_to_req: torch.Tensor,
output_scale: torch.Tensor | None = None,
output_block_scale: torch.Tensor | None = None,
) -> torch.Tensor:
del key, value
if output_scale is not None or output_block_scale is not None:
raise NotImplementedError("QSA does not support fused output quantization")
if self.alibi_slopes is not None or self.sinks is not None:
raise NotImplementedError("QSA does not support ALiBi or attention sinks")
if self.sliding_window != (-1, -1):
raise NotImplementedError("QSA does not support sliding-window attention")
num_tokens = attn_metadata.num_actual_tokens
output.zero_()
if num_tokens == 0:
return output
topk_buffer = getattr(layer, "topk_indices_buffer", None)
if topk_buffer is None:
raise RuntimeError("QSA owner did not provide its top-k buffer")
logical_indices = topk_buffer[:num_tokens]
token_to_req = token_to_req[:num_tokens]
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
key_cache = canonicalize_singleton_dim_strides(key_cache)
value_cache = canonicalize_singleton_dim_strides(value_cache)
if key_cache.dtype != torch.bfloat16 or query.dtype != torch.bfloat16:
raise NotImplementedError("Qwen4Exp QSA requires BF16 Q/K/V")
from .ops.qsa import qsa_sparse_paged_attention
qsa_sparse_paged_attention(
query[:num_tokens],
key_cache,
value_cache,
logical_indices,
attn_metadata.block_table,
token_to_req,
output[:num_tokens],
)
return output
class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
"""Merged Qwen full-attention owner with a QSA index side branch."""
supports_dcp = False
def __init__(
self,
*,
vllm_config: VllmConfig,
config: Qwen4ExpTextConfig,
layer_id: int,
quant_config: QuantizationConfig | None = None,
reduce_results: bool = True,
prefix: str = "",
) -> None:
nn.Module.__init__(self)
cache_config = vllm_config.cache_config
model_config = vllm_config.model_config
if cache_config is None:
raise ValueError("Qwen4Exp QSA requires a paged KV cache")
if model_config.dtype != torch.bfloat16:
raise NotImplementedError("Qwen4Exp QSA currently requires BF16")
if cache_config.cache_dtype not in ("auto", "bfloat16"):
raise NotImplementedError("Qwen4Exp QSA requires a BF16 main KV cache")
if getattr(quant_config, "kv_cache_scheme", None) is not None:
raise NotImplementedError("Qwen4Exp QSA does not support KV quantization")
parallel_config = vllm_config.parallel_config
if (
parallel_config.prefill_context_parallel_size > 1
or parallel_config.decode_context_parallel_size > 1
):
raise NotImplementedError(
"Qwen4Exp QSA does not support context parallelism"
)
if not getattr(config, "is_causal", True):
raise NotImplementedError("Qwen4Exp QSA requires causal decoder attention")
self.config = config
self.hidden_size = int(config.hidden_size)
tp_size = get_tensor_model_parallel_world_size()
self.total_num_heads = int(config.num_attention_heads)
if self.total_num_heads % tp_size:
raise ValueError("QSA attention heads must be divisible by TP size")
self.num_heads = self.total_num_heads // tp_size
self.total_num_kv_heads = int(config.num_key_value_heads)
if self.total_num_kv_heads >= tp_size:
if self.total_num_kv_heads % tp_size:
raise ValueError("QSA KV heads must be divisible by TP size")
elif tp_size % self.total_num_kv_heads:
raise ValueError("TP size must be divisible by replicated QSA KV heads")
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
self.head_dim = int(config.head_dim or self.hidden_size // self.num_heads)
self.q_size = self.num_heads * self.head_dim
self.kv_size = self.num_kv_heads * self.head_dim
self.scaling = self.head_dim**-0.5
self.dual_chunk_attention_config = getattr(
config, "dual_chunk_attention_config", None
)
if self.dual_chunk_attention_config is not None:
raise NotImplementedError("Qwen4Exp QSA does not support dual-chunk RoPE")
# Qwen4Exp full-attention checkpoints always pack a sigmoid output
# gate next to Q, even when an inherited config default says otherwise.
self.attn_output_gate = True
self.qkv_proj = QKVParallelLinear(
self.hidden_size,
self.head_dim,
self.total_num_heads * (1 + self.attn_output_gate),
self.total_num_kv_heads,
bias=False,
quant_config=model.without_modelopt_fp4(quant_config),
prefix=f"{prefix}.qkv_proj",
)
self.o_proj = RowParallelLinear(
self.total_num_heads * self.head_dim,
self.hidden_size,
bias=False,
reduce_results=reduce_results,
quant_config=quant_config,
prefix=f"{prefix}.o_proj",
)
self.rotary_emb = get_rope(
head_size=self.head_dim,
max_position=config.max_position_embeddings,
rope_parameters=config.rope_parameters,
)
self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
mm_config = model_config.multimodal_config
text_only = mm_config is None or mm_config.language_model_only
self.use_fused_qk_norm_rope_gate = (
self.attn_output_gate
and getattr(self.rotary_emb, "is_neox_style", False)
and current_platform.is_cuda()
and text_only
)
self.layer_name = f"{prefix}.attn"
self.attn_type = AttentionType.DECODER
self.kv_cache_dtype = cache_config.cache_dtype
self.kv_cache_torch_dtype = kv_cache_dtype_str_to_dtype(
self.kv_cache_dtype, model_config
)
if self.kv_cache_torch_dtype != torch.bfloat16:
raise NotImplementedError("Qwen4Exp QSA requires BF16 cache storage")
self.kv_sharing_target_layer_name = None
self.kv_cache = torch.tensor([])
set_default_quant_scales(self, register_buffer=True)
self.attn_backend = Qwen4ExpQSAFlashAttentionBackend
self.impl = Qwen4ExpQSAFlashAttentionImpl(
self.num_heads,
self.head_dim,
self.scaling,
self.num_kv_heads,
None,
None,
self.kv_cache_dtype,
None,
AttentionType.DECODER,
None,
)
self.indexer = QSAIndexer(
vllm_config=vllm_config,
config=config,
layer_id=layer_id,
rotary_emb=self.rotary_emb,
quant_config=quant_config,
prefix=f"{prefix}.indexer",
)
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
self.register_buffer(
"topk_indices_buffer",
torch.empty(
max_tokens,
self.indexer.output_width,
dtype=torch.int32,
),
persistent=False,
)
static_context = vllm_config.compilation_config.static_forward_context
if self.layer_name in static_context:
raise ValueError(f"Duplicate layer name: {self.layer_name}")
static_context[self.layer_name] = self
def get_attn_backend(self) -> type[AttentionBackend]:
return self.attn_backend
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
return FullAttentionSpec(
block_size=vllm_config.cache_config.block_size,
num_kv_heads=self.num_kv_heads,
head_size=self.head_dim,
head_size_v=self.head_dim,
dtype=self.kv_cache_torch_dtype,
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
)
def _run_qsa(
self,
hidden_states: torch.Tensor,
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
output: torch.Tensor,
) -> None:
metadata = get_forward_context().attn_metadata
if isinstance(metadata, list):
metadata = metadata[0]
if not isinstance(metadata, dict):
output.zero_()
return
main_metadata = cast(FlashAttentionMetadata, metadata[self.layer_name])
if self.kv_cache.numel() == 0:
raise RuntimeError("QSA main K/V cache is not bound")
num_tokens = main_metadata.num_actual_tokens
side_metadata = cast(
QSAForwardMetadata,
metadata[self.indexer.raw_key_cache.prefix],
)
if side_metadata.num_actual_tokens != num_tokens:
raise RuntimeError("QSA main and side metadata token counts disagree")
selected = self.indexer(
hidden_states,
positions,
self.topk_indices_buffer[:num_tokens],
)
if selected.shape != (
num_tokens,
self.indexer.output_width,
):
raise RuntimeError("QSA indexer returned an invalid selection shape")
impl = cast(Qwen4ExpQSAFlashAttentionImpl, self.impl)
impl.do_kv_cache_update(
self,
key,
value,
self.kv_cache,
main_metadata.slot_mapping,
)
impl.forward_qsa(
self,
query,
key,
value,
self.kv_cache,
main_metadata,
output,
token_to_req=side_metadata.token_to_req,
)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
) -> torch.Tensor:
qkv, _ = self.qkv_proj(hidden_states)
q, k, v, gate = self._project_qkv_gate(qkv, positions)
num_tokens = hidden_states.shape[0]
query = q.view(num_tokens, self.num_heads, self.head_dim)
key = k.view(num_tokens, self.num_kv_heads, self.head_dim)
value = v.view(num_tokens, self.num_kv_heads, self.head_dim)
attn_output = torch.empty_like(query)
encoded_layer_name = _encode_layer_name(self.layer_name)
if current_platform.opaque_attention_op():
torch.ops.vllm.qwen4_exp_qsa_with_output(
hidden_states,
positions,
query,
key,
value,
attn_output,
encoded_layer_name,
)
else:
qwen4_exp_qsa_with_output(
hidden_states,
positions,
query,
key,
value,
attn_output,
encoded_layer_name,
)
flat_output = attn_output.view(num_tokens, -1)
if gate is not None:
flat_output = flat_output * torch.sigmoid(gate)
output, _ = self.o_proj(flat_output)
return output
def qwen4_exp_qsa_with_output(
hidden_states: torch.Tensor,
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
output: torch.Tensor,
layer_name: LayerNameType,
) -> None:
"""Run the complete QSA state/update/attend transaction."""
layer_name = _resolve_layer_name(layer_name)
layer = get_forward_context().no_compile_layers[layer_name]
if not isinstance(layer, Qwen4ExpQSAAttention):
raise TypeError(f"{layer_name} is not a Qwen4Exp QSA owner")
layer._run_qsa(
hidden_states,
positions,
query,
key,
value,
output,
)
def qwen4_exp_qsa_with_output_fake(
hidden_states: torch.Tensor,
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
output: torch.Tensor,
layer_name: LayerNameType,
) -> None:
del hidden_states, positions, query, key, value, output, layer_name
direct_register_custom_op(
op_name="qwen4_exp_qsa_with_output",
op_func=qwen4_exp_qsa_with_output,
mutates_args=["output"],
fake_impl=qwen4_exp_qsa_with_output_fake,
)
__all__ = [
"QSAIndexer",
"Qwen4ExpQSAAttention",
"Qwen4ExpQSAFlashAttentionBackend",
"Qwen4ExpQSAFlashAttentionImpl",
"qwen4_exp_qsa_with_output",
]
+17
View File
@@ -0,0 +1,17 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Common Qwen4Exp model components."""
from .hyperconnection import (
GatedResidual,
GroupedGemmaRMSNorm,
HyperConnectionBase,
HyperConnectionConfig,
)
__all__ = [
"GatedResidual",
"GroupedGemmaRMSNorm",
"HyperConnectionBase",
"HyperConnectionConfig",
]
@@ -0,0 +1,250 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""HyperConnection (Gated Residual) utilities.
Implements the HyperConnection residual scheme proposed in
"HyperConnections" (https://arxiv.org/abs/2409.19606).
The two concrete variants are:
- ``HyperConnectionBase`` - simple average pooling across hc_count parallel
streams (equivalent to hyperconnection_average).
- ``GatedResidual`` - learnable low-rank gated mixing and injection
(gated_residual).
Hidden states between layers have shape ``[..., HC*HS]`` with HS inner
(HC outer, HS inner — checkpoint-native layout). The local torch
implementation consumes the hyper input viewed as ``[..., HC, HS]``.
Typical usage inside a transformer decoder layer::
self.attn_hc = GatedResidual(hc_config, role="attn")
self.mlp_hc = GatedResidual(hc_config, role="mlp")
hidden_states, residual = self.attn_hc.mix(hidden_states)
hidden_states = attention(hidden_states)
hidden_states = self.attn_hc.combine(hidden_states, residual)
hidden_states, residual = self.mlp_hc.mix(hidden_states)
hidden_states = mlp(hidden_states)
hidden_states = self.mlp_hc.combine(hidden_states, residual)
"""
from dataclasses import dataclass
import torch
import torch.nn.functional as F
from torch import nn
# ---------------------------------------------------------------------------
# Config
# ---------------------------------------------------------------------------
@dataclass
class HyperConnectionConfig:
"""Configuration shared by all HyperConnection variants."""
hc_count: int = 4
hidden_size: int = 64
params_dtype: torch.dtype = torch.bfloat16
mtp_hc: bool = False
hc_lowrank: int = 16
rms_norm_eps: float = 1e-6
hc_per_branch_norm: bool = False
class GroupedGemmaRMSNorm(nn.Module):
def __init__(
self,
hidden_size: int,
eps: float,
group_size: int | None,
dtype: torch.dtype | None,
) -> None:
super().__init__()
if group_size is not None and hidden_size % group_size:
raise ValueError(
f"hidden_size ({hidden_size}) must be divisible by "
f"group_size ({group_size})"
)
self.variance_epsilon = eps
self.group_size = group_size
self.weight = nn.Parameter(torch.zeros(hidden_size, dtype=dtype))
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
input_dtype = hidden_states.dtype
hidden_states = hidden_states.float()
if self.group_size is None:
variance = hidden_states.square().mean(dim=-1, keepdim=True)
normalized = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
else:
grouped = hidden_states.unflatten(
-1, (hidden_states.shape[-1] // self.group_size, self.group_size)
)
variance = grouped.square().mean(dim=-1, keepdim=True)
normalized = (
grouped * torch.rsqrt(variance + self.variance_epsilon)
).flatten(-2)
return (normalized * (1.0 + self.weight.float())).to(input_dtype)
# ---------------------------------------------------------------------------
# Average-pooling variant
# ---------------------------------------------------------------------------
class HyperConnectionBase(nn.Module):
"""Average-pooling HyperConnection (``hyperconnection_average``).
Splits the incoming ``[..., HC*HS]`` tensor (HC outer, HS inner) into
``HC`` parallel streams, averages them for the block input, and
broadcasts the block output back to every stream.
"""
def __init__(
self,
config: HyperConnectionConfig,
layer_idx: int | None = None,
role: str | None = None,
) -> None:
super().__init__()
self.config = config
self.hc_count = config.hc_count
self.hidden_size = config.hidden_size
self.layer_idx = layer_idx
self.role = role
@property
def hyper_hidden_size(self) -> int:
return self.hc_count * self.hidden_size
def mix(self, hyper_input: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
"""Average the HC streams into a single block input."""
assert hyper_input.shape[-1] == self.hc_count * self.hidden_size
# [*, HC, HS] — mean over HC (dim=-2).
unflat = hyper_input.unflatten(-1, (self.hc_count, self.hidden_size))
mixed_input = unflat.mean(dim=-2)
return mixed_input, hyper_input
def combine(
self, block_output: torch.Tensor, residual: torch.Tensor
) -> torch.Tensor:
"""Broadcast the block output back to every stream."""
assert residual.shape[-1] == self.hc_count * self.hidden_size
assert block_output.shape[-1] == self.hidden_size
residual_reshaped = residual.unflatten(-1, (self.hc_count, self.hidden_size))
combined = residual_reshaped + block_output.unsqueeze(-2)
return combined.flatten(-2)
# ---------------------------------------------------------------------------
# Gated-residual variant
# ---------------------------------------------------------------------------
class GatedResidual(HyperConnectionBase):
"""Gated HyperConnection with learnable low-rank mixing and injection.
``mix()`` applies GemmaRMSNorm per HC stream and projects through a
low-rank sigmoid gate to produce a single block input. ``combine()``
injects the block output back into each stream through a learned
per-stream injection weight.
This implementation uses only PyTorch operators. Tensor-parallel
collectives are supplied by its caller.
"""
def __init__(
self,
config: HyperConnectionConfig,
layer_idx: int | None = None,
role: str | None = None,
use_mix: bool = True,
use_combine: bool = True,
) -> None:
super().__init__(config, layer_idx, role)
norm_size = (
self.hyper_hidden_size if config.hc_per_branch_norm else config.hidden_size
)
group_size = config.hidden_size if config.hc_per_branch_norm else None
# Normalize each H-sized HC stream independently while retaining a
# separate affine weight for every element of the HC*H layout.
self.hc_norm = GroupedGemmaRMSNorm(
norm_size,
eps=config.rms_norm_eps,
group_size=group_size,
dtype=config.params_dtype,
)
# -- raw Linear weights (checkpoint-compatible) ----------------------
if use_mix:
self.input_mix_weight_down = nn.Linear(
self.hyper_hidden_size,
config.hc_lowrank,
bias=False,
dtype=config.params_dtype,
)
self.input_mix_weight_up = nn.Linear(
config.hc_lowrank,
self.hyper_hidden_size,
bias=False,
dtype=config.params_dtype,
)
if use_combine:
self.block_inject_weight = nn.Linear(
self.hyper_hidden_size,
self.hc_count,
bias=False,
dtype=config.params_dtype,
)
def _normalize(self, hyper_input: torch.Tensor) -> torch.Tensor:
if self.config.hc_per_branch_norm:
return self.hc_norm(hyper_input)
return self.hc_norm(
hyper_input.unflatten(-1, (self.hc_count, self.hidden_size))
).flatten(-2)
def mix(
self, hyper_input: torch.Tensor
) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]:
"""Mix: RMSNorm -> low-rank gate -> gated mean."""
assert hyper_input.shape[-1] == self.hc_count * self.hidden_size
if not hasattr(self, "input_mix_weight_down"):
raise RuntimeError("mix was disabled for this hyper-connection")
hyper_input_normed = self._normalize(hyper_input)
# Gate — original mix order: linear+silu then linear+sigmoid.
gate = F.silu(
F.linear(hyper_input_normed, self.input_mix_weight_down.weight)
/ self.hc_count
)
gate = torch.sigmoid(F.linear(gate, self.input_mix_weight_up.weight)).unflatten(
-1, (self.hc_count, self.hidden_size)
)
mixed_input = (
gate * hyper_input_normed.unflatten(-1, (self.hc_count, self.hidden_size))
).mean(dim=-2)
return mixed_input.to(hyper_input.dtype), (hyper_input, hyper_input_normed)
def combine(
self,
block_output: torch.Tensor,
residuals: tuple[torch.Tensor, torch.Tensor],
) -> torch.Tensor:
if not hasattr(self, "block_inject_weight"):
raise RuntimeError("combine was disabled for this hyper-connection")
hyper_input, hyper_input_normed = residuals
assert hyper_input.shape[-1] == self.hc_count * self.hidden_size
assert block_output.shape[-1] == self.hidden_size
residual = hyper_input.unflatten(-1, (self.hc_count, self.hidden_size))
# The paired mix keeps its normalized hyper input so combine uses the
# same HC module's injection weight.
injection_weight = 2.0 * torch.sigmoid(
F.linear(hyper_input_normed, self.block_inject_weight.weight)
/ self.hc_count
)
output = residual + block_output.unsqueeze(-2) * injection_weight.unsqueeze(-1)
return output.flatten(-2).to(hyper_input.dtype)
__all__ = [
"GatedResidual",
"GroupedGemmaRMSNorm",
"HyperConnectionBase",
"HyperConnectionConfig",
]
+100
View File
@@ -0,0 +1,100 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Common Qwen4Exp PLE helpers."""
from dataclasses import dataclass
import torch
from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
@dataclass(frozen=True)
class PLEShardOverlap:
"""Source and destination slices for one checkpoint embedding shard."""
source_start: int
destination_start: int
row_count: int
def compute_ple_shard_overlap(
*,
checkpoint_start: int,
checkpoint_rows: int,
tp_start: int,
tp_end: int,
) -> PLEShardOverlap | None:
"""Compute the overlap of a checkpoint shard and one TP vocabulary range."""
if checkpoint_start < 0 or checkpoint_rows < 0:
raise ValueError("checkpoint shard bounds must be non-negative")
if tp_start < 0 or tp_end < tp_start:
raise ValueError("invalid TP vocabulary range")
checkpoint_end = checkpoint_start + checkpoint_rows
overlap_start = max(checkpoint_start, tp_start)
overlap_end = min(checkpoint_end, tp_end)
if overlap_start >= overlap_end:
return None
return PLEShardOverlap(
source_start=overlap_start - checkpoint_start,
destination_start=overlap_start - tp_start,
row_count=overlap_end - overlap_start,
)
def copy_ple_embedding_shard_(
destination: torch.Tensor,
loaded_weight: torch.Tensor,
*,
checkpoint_start: int,
tp_start: int,
tp_end: int,
) -> int:
"""Copy the overlapping rows of a PLE checkpoint shard into a TP table."""
if destination.ndim == 0 or loaded_weight.ndim != destination.ndim:
raise ValueError("destination and loaded weight must have matching ranks")
if destination.shape[1:] != loaded_weight.shape[1:]:
raise ValueError(
"embedding shard dimensions do not match: "
f"{tuple(destination.shape[1:])} != {tuple(loaded_weight.shape[1:])}"
)
if destination.shape[0] < tp_end - tp_start:
raise ValueError("destination does not cover the requested TP range")
overlap = compute_ple_shard_overlap(
checkpoint_start=checkpoint_start,
checkpoint_rows=loaded_weight.shape[0],
tp_start=tp_start,
tp_end=tp_end,
)
if overlap is None:
return 0
source = loaded_weight.narrow(0, overlap.source_start, overlap.row_count)
target = destination.narrow(0, overlap.destination_start, overlap.row_count)
with torch.no_grad():
target.copy_(source.to(device=target.device, dtype=target.dtype))
return overlap.row_count
class PLEVocabParallelEmbedding(VocabParallelEmbedding):
"""Vocab-parallel embedding that accepts checkpoint row shards."""
def weight_loader(
self,
param: torch.Tensor,
loaded_weight: torch.Tensor,
checkpoint_start: int | None = None,
) -> None:
if checkpoint_start is None:
super().weight_loader(param, loaded_weight)
return
copy_ple_embedding_shard_(
param,
loaded_weight,
checkpoint_start=checkpoint_start,
tp_start=self.shard_indices.org_vocab_start_index,
tp_end=self.shard_indices.org_vocab_end_index,
)
+807
View File
@@ -0,0 +1,807 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Paged side-cache ownership and metadata for Qwen4Exp QSA.
Each QSA layer keeps a fixed circular buffer of raw index keys (the
compressor state) and one compressed key. MRoPE models pack exact three-axis
positions beside the raw keys; text models derive group positions from
logical positions. The compressor state uses one block per request, while
the compressed owner uses ``MLAAttentionSpec.tokens_per_state`` so its block
table follows the main KV-cache lifecycle. Their physical tensor storage is
shared by the generic cache-layout planner.
"""
from __future__ import annotations
import math
from dataclasses import dataclass
from functools import cache
from typing import ClassVar
import torch
from torch import nn
from vllm.config import CacheConfig, VllmConfig
from vllm.config.cache import CacheDType
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON, tl, triton
from vllm.utils.math_utils import cdiv
from vllm.v1.attention.backend import (
AttentionBackend,
AttentionCGSupport,
AttentionMetadata,
AttentionMetadataBuilder,
CommonAttentionMetadata,
)
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
from vllm.v1.kv_cache_interface import (
AttentionSpec,
CircularBufferSpec,
KVCacheLayout,
KVCacheSpec,
MLAAttentionSpec,
)
def canonical_qsa_rope_positions(positions: torch.Tensor) -> torch.Tensor:
"""Return exact per-token positions as ``[tokens, 1, 3]`` int64 rows."""
if positions.ndim == 1:
positions = positions.unsqueeze(0).expand(3, -1)
elif positions.ndim != 2 or positions.shape[0] not in (1, 3):
raise ValueError("QSA RoPE positions must be [tokens] or [1|3, tokens]")
if positions.shape[0] == 1:
positions = positions.expand(3, -1)
return positions.transpose(0, 1).unsqueeze(1).to(torch.int64)
def _logical_positions(
query_start_loc: torch.Tensor,
seq_lens: torch.Tensor,
token_to_req: torch.Tensor,
num_tokens: int,
) -> torch.Tensor:
if num_tokens == 0:
return seq_lens.new_empty((0,), dtype=torch.int64)
arange = torch.arange(num_tokens, device=query_start_loc.device)
requests = token_to_req[:num_tokens].long()
query_lens = torch.diff(query_start_loc)
within_query = arange - query_start_loc.index_select(0, requests)
return (
seq_lens.index_select(0, requests).long()
- query_lens.index_select(0, requests).long()
+ within_query.long()
)
def _logical_to_physical_qsa_slots(
block_table: torch.Tensor,
request_indices: torch.Tensor,
logical_positions: torch.Tensor,
block_size: int,
) -> torch.Tensor:
if block_size <= 0:
raise ValueError("QSA cache block size must be positive")
if block_table.ndim != 2:
raise ValueError("QSA block table must be two-dimensional")
if request_indices.shape != logical_positions.shape:
request_indices = torch.broadcast_to(request_indices, logical_positions.shape)
requests = request_indices.to(device=block_table.device, dtype=torch.long)
positions = logical_positions.to(device=block_table.device, dtype=torch.long)
valid = (requests >= 0) & (requests < block_table.shape[0]) & (positions >= 0)
logical_blocks = torch.div(
positions.clamp_min(0), block_size, rounding_mode="floor"
)
valid &= logical_blocks < block_table.shape[1]
safe_requests = requests.clamp(0, max(block_table.shape[0] - 1, 0))
safe_blocks = logical_blocks.clamp(0, max(block_table.shape[1] - 1, 0))
if not all(block_table.shape):
return torch.full_like(positions, PAD_SLOT_ID)
physical_blocks = block_table[safe_requests, safe_blocks].long()
valid &= physical_blocks >= 0
slots = physical_blocks * block_size + positions.remainder(block_size)
return torch.where(valid, slots, PAD_SLOT_ID)
def circular_qsa_slot_mapping(
block_table: torch.Tensor,
token_to_req: torch.Tensor,
logical_positions: torch.Tensor,
compressor_state_size: int,
query_start_loc: torch.Tensor | None = None,
out: torch.Tensor | None = None,
) -> torch.Tensor:
"""Map each request to its fixed physical block as a circular token ring."""
if compressor_state_size <= 0:
raise ValueError("QSA circular buffer size must be positive")
if block_table.ndim != 2:
raise ValueError("QSA block table must be two-dimensional")
requests = token_to_req.to(device=block_table.device, dtype=torch.long)
positions = logical_positions.to(device=block_table.device, dtype=torch.long)
if not all(block_table.shape):
slots = torch.full_like(positions, PAD_SLOT_ID)
else:
valid = (requests >= 0) & (requests < block_table.shape[0]) & (positions >= 0)
safe_requests = requests.clamp(0, block_table.shape[0] - 1)
physical_blocks = block_table[safe_requests, 0].long()
valid &= physical_blocks >= 0
slots = physical_blocks * compressor_state_size + positions.remainder(
compressor_state_size
)
slots = torch.where(valid, slots, PAD_SLOT_ID)
if query_start_loc is not None:
if query_start_loc.ndim != 1 or query_start_loc.shape[0] < 2:
raise ValueError("QSA query starts must contain a terminal offset")
query_start_loc = query_start_loc.to(block_table.device)
num_requests = query_start_loc.shape[0] - 1
safe_requests = requests.clamp(0, num_requests - 1)
request_ends = query_start_loc.index_select(0, safe_requests + 1)
rows = torch.arange(slots.numel(), device=slots.device)
keep = (
(requests >= 0)
& (requests < num_requests)
& (rows + compressor_state_size >= request_ends)
)
slots = torch.where(keep, slots, PAD_SLOT_ID)
slots = slots.to(torch.int64)
if out is not None:
out.fill_(PAD_SLOT_ID)
out[: slots.numel()].copy_(slots)
return out[: slots.numel()]
return slots
def compressed_qsa_slot_mapping(
block_table: torch.Tensor,
token_to_req: torch.Tensor,
logical_positions: torch.Tensor,
storage_block_size: int,
compress_ratio: int,
out: torch.Tensor | None = None,
) -> torch.Tensor:
"""Build boundary-only slots for an ``MLAAttentionSpec`` QSA cache."""
if storage_block_size <= 0 or compress_ratio <= 0:
raise ValueError("QSA block size and compression ratio must be positive")
compressed_positions = torch.div(
logical_positions.clamp_min(0), compress_ratio, rounding_mode="floor"
)
slots = _logical_to_physical_qsa_slots(
block_table,
token_to_req,
compressed_positions,
storage_block_size,
)
valid = (logical_positions >= 0) & (
(logical_positions + 1).remainder(compress_ratio) == 0
)
slots = torch.where(valid, slots, PAD_SLOT_ID).to(torch.int64)
if out is not None:
out.fill_(PAD_SLOT_ID)
out[: slots.numel()].copy_(slots)
return out[: slots.numel()]
return slots
@cache
def _metadata_launch_pdl() -> bool:
return current_platform.is_arch_support_pdl()
@triton.jit(
do_not_specialize=[
"num_reqs",
"num_mapped_tokens",
"num_tokens",
"max_num_work",
"num_search_steps",
"work_search_steps",
]
)
def _build_qsa_metadata_kernel(
query_start_loc_ptr,
seq_lens_ptr,
common_slot_mapping_ptr,
block_table_ptr,
token_to_req_ptr,
logical_positions_ptr,
slot_mapping_ptr,
k_work_metadata_ptr,
block_table_stride_0: tl.constexpr,
block_table_stride_1: tl.constexpr,
num_reqs,
num_mapped_tokens,
num_tokens,
max_num_work,
num_search_steps,
work_search_steps,
storage_block_size: tl.constexpr,
compress_ratio: tl.constexpr,
circular_buffer_size: tl.constexpr,
num_block_table_columns: tl.constexpr,
launch_pdl: tl.constexpr,
TOKEN_BLOCK_SIZE: tl.constexpr,
REQUEST_SCAN_SIZE: tl.constexpr,
WORK_BLOCK_SIZE: tl.constexpr,
):
if launch_pdl:
tl.extra.cuda.gdc_wait()
pid = tl.program_id(0)
token_idx = pid * TOKEN_BLOCK_SIZE + tl.arange(0, TOKEN_BLOCK_SIZE)
store_mask = token_idx < num_tokens
mapped = token_idx < num_mapped_tokens
search_token_idx = tl.minimum(token_idx, num_mapped_tokens - 1)
request_idx = tl.zeros((TOKEN_BLOCK_SIZE,), tl.int32)
# Find the last query start at or before each token. The dynamic loop avoids
# compiling a kernel variant for every ceil(log2(num_reqs)).
for step in tl.range(0, num_search_steps):
candidate = request_idx + (1 << (num_search_steps - step - 1))
valid_candidate = candidate < num_reqs
candidate_start = tl.load(
query_start_loc_ptr + candidate,
mask=valid_candidate,
other=num_mapped_tokens + 1,
)
advance = valid_candidate & (candidate_start <= search_token_idx)
request_idx = tl.where(advance, candidate, request_idx)
query_start = tl.load(query_start_loc_ptr + request_idx, mask=mapped, other=0)
query_end = tl.load(query_start_loc_ptr + request_idx + 1, mask=mapped, other=0)
seq_len = tl.load(seq_lens_ptr + request_idx, mask=mapped, other=0)
logical_position = seq_len - (query_end - query_start) + token_idx - query_start
logical_position = tl.where(mapped, logical_position, -1)
tl.store(
token_to_req_ptr + token_idx,
tl.where(mapped, request_idx, 0),
mask=store_mask,
)
tl.store(
logical_positions_ptr + token_idx,
logical_position,
mask=store_mask,
)
# circular_buffer_size is constexpr, so each builder instance compiles out
# the other QSA cache owner's slot-mapping rule.
if circular_buffer_size > 0:
valid = (
mapped
& (logical_position >= 0)
& (token_idx + circular_buffer_size >= query_end)
& (num_block_table_columns > 0)
)
physical_block = tl.load(
block_table_ptr + request_idx * block_table_stride_0,
mask=valid,
other=-1,
)
valid &= physical_block >= 0
slot = physical_block * circular_buffer_size + (
logical_position % circular_buffer_size
)
elif compress_ratio != 1:
compressed_position = tl.maximum(logical_position, 0) // compress_ratio
logical_block = compressed_position // storage_block_size
valid = (
mapped
& (logical_position >= 0)
& ((logical_position + 1) % compress_ratio == 0)
& (logical_block < num_block_table_columns)
)
physical_block = tl.load(
block_table_ptr
+ request_idx * block_table_stride_0
+ logical_block * block_table_stride_1,
mask=valid,
other=-1,
)
valid &= physical_block >= 0
valid &= (
tl.load(common_slot_mapping_ptr + token_idx, mask=mapped, other=-1) >= 0
)
slot = physical_block * storage_block_size + (
compressed_position % storage_block_size
)
if (circular_buffer_size > 0) or (compress_ratio != 1):
tl.store(
slot_mapping_ptr + token_idx,
tl.where(valid, slot, -1),
mask=store_mask,
)
work_tile_start = pid * WORK_BLOCK_SIZE
has_work_tile = work_tile_start < max_num_work
if k_work_metadata_ptr is not None and has_work_tile:
# Every work CTA builds the request prefix in registers. Recomputing this
# small vector lets CTAs write disjoint work tiles without a grid barrier.
requests = tl.arange(0, REQUEST_SCAN_SIZE)
valid_request = requests < num_reqs
request_query_start = tl.load(
query_start_loc_ptr + requests, mask=valid_request, other=0
)
request_query_end = tl.load(
query_start_loc_ptr + requests + 1, mask=valid_request, other=0
)
request_seq_len = tl.load(seq_lens_ptr + requests, mask=valid_request, other=0)
request_query_len = request_query_end - request_query_start
chunk_start = request_seq_len - request_query_len
num_groups = request_seq_len // compress_ratio - chunk_start // compress_ratio
# Nonempty requests need one item even without a completed compression
# group because work item zero also commits the current raw-K suffix.
work_counts = tl.where(request_query_len > 0, tl.maximum(num_groups, 1), 0)
work_ends = tl.cumsum(tl.where(valid_request, work_counts, 0), axis=0)
total_work = tl.sum(tl.where(valid_request, work_counts, 0), axis=0)
work_offsets = tl.arange(0, WORK_BLOCK_SIZE)
work = work_tile_start + work_offsets
in_bounds = work < max_num_work
active = in_bounds & (work < total_work)
request = tl.zeros((WORK_BLOCK_SIZE,), dtype=tl.int32)
# Request zero starts at zero for every active work item. Descending
# steps find the last request starting at or before this item.
for step_idx in tl.range(0, work_search_steps):
step = 1 << (work_search_steps - step_idx - 1)
candidate = request + step
valid_candidate = candidate < num_reqs
candidate_start = tl.gather(work_ends, candidate - 1, 0)
advance = active & valid_candidate & (candidate_start <= work)
request = tl.where(advance, candidate, request)
if launch_pdl:
# Let the dependent grid start launch setup while these CTAs finish
# stores; its gdc_wait still orders access to the completed metadata.
tl.extra.cuda.gdc_launch_dependents()
owner_work_start = tl.gather(work_ends, tl.maximum(request - 1, 0), 0)
owner_work_start = tl.where(request == 0, 0, owner_work_start)
work_in_request = work - owner_work_start
tl.store(
k_work_metadata_ptr + work * 2,
tl.where(active, request, -1),
mask=in_bounds,
)
tl.store(
k_work_metadata_ptr + work * 2 + 1,
tl.where(active, work_in_request, -1),
mask=in_bounds,
)
else:
if launch_pdl:
# A dependent grid launches only after every CTA has signaled.
tl.extra.cuda.gdc_launch_dependents()
def build_qsa_metadata_triton(
common_attn_metadata: CommonAttentionMetadata,
token_to_req_buffer: torch.Tensor,
logical_positions_buffer: torch.Tensor,
slot_mapping_buffer: torch.Tensor,
*,
storage_block_size: int,
compress_ratio: int,
circular_buffer_size: int = 0,
k_work_metadata_buffer: torch.Tensor | None = None,
request_capacity: int | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Build QSA side-cache and optional pre-indexer work metadata."""
num_tokens = common_attn_metadata.num_actual_tokens
num_mapped_tokens = int(common_attn_metadata.query_start_loc_cpu[-1])
token_to_req = token_to_req_buffer[:num_tokens]
logical_positions = logical_positions_buffer[:num_tokens]
slot_mapping = slot_mapping_buffer[:num_tokens]
num_reqs = common_attn_metadata.query_start_loc.shape[0] - 1
assert num_reqs > 0
if k_work_metadata_buffer is not None:
if request_capacity is None:
request_capacity = num_reqs
assert request_capacity >= num_reqs
# Pad for tl.arange while keeping the scan width stable across live batches.
request_scan_size = 1 << int(math.ceil(math.log2(request_capacity)))
max_num_work = k_work_metadata_buffer.shape[0]
else:
request_scan_size = 1
max_num_work = 0
if num_tokens == 0 and k_work_metadata_buffer is None:
return token_to_req, logical_positions, slot_mapping
block_table = common_attn_metadata.block_table_tensor
num_search_steps = int(math.ceil(math.log2(num_reqs)))
work_search_steps = int(math.ceil(math.log2(num_reqs)))
# The same grid covers token tiles and, for the compressed cache, work tiles.
num_token_blocks = cdiv(num_tokens, 128)
num_work_blocks = (
cdiv(max_num_work, 256) if k_work_metadata_buffer is not None else 0
)
_build_qsa_metadata_kernel[(max(num_token_blocks, num_work_blocks, 1),)](
common_attn_metadata.query_start_loc,
common_attn_metadata.seq_lens,
common_attn_metadata.slot_mapping,
block_table,
token_to_req,
logical_positions,
slot_mapping,
k_work_metadata_buffer,
block_table.stride(0),
block_table.stride(1),
num_reqs,
num_mapped_tokens,
num_tokens,
max_num_work,
num_search_steps,
work_search_steps,
storage_block_size,
compress_ratio,
circular_buffer_size,
block_table.shape[1],
launch_pdl=_metadata_launch_pdl(),
TOKEN_BLOCK_SIZE=128,
REQUEST_SCAN_SIZE=request_scan_size,
WORK_BLOCK_SIZE=256,
num_warps=4,
)
if circular_buffer_size == 0 and compress_ratio == 1:
slot_mapping = common_attn_metadata.slot_mapping[:num_tokens]
return token_to_req, logical_positions, slot_mapping
def _build_qsa_metadata_torch(
common_attn_metadata: CommonAttentionMetadata,
token_to_req_buffer: torch.Tensor,
logical_positions_buffer: torch.Tensor,
slot_mapping_buffer: torch.Tensor,
*,
storage_block_size: int,
compress_ratio: int,
circular_buffer_size: int = 0,
k_work_metadata_buffer: torch.Tensor | None = None,
request_capacity: int | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
del request_capacity
num_tokens = common_attn_metadata.num_actual_tokens
num_mapped_tokens = int(common_attn_metadata.query_start_loc_cpu[-1])
logical_positions = logical_positions_buffer[:num_tokens]
token_to_req = common_attn_metadata.token_to_req_indices(token_to_req_buffer)[
:num_tokens
]
logical_positions[:num_mapped_tokens].copy_(
_logical_positions(
common_attn_metadata.query_start_loc,
common_attn_metadata.seq_lens,
token_to_req[:num_mapped_tokens],
num_mapped_tokens,
)
)
if num_mapped_tokens < num_tokens:
logical_positions[num_mapped_tokens:].fill_(-1)
if circular_buffer_size > 0:
slot_mapping = circular_qsa_slot_mapping(
common_attn_metadata.block_table_tensor,
token_to_req,
logical_positions,
circular_buffer_size,
query_start_loc=common_attn_metadata.query_start_loc,
out=slot_mapping_buffer,
)
elif compress_ratio == 1:
slot_mapping = common_attn_metadata.slot_mapping[:num_tokens]
else:
slot_mapping = compressed_qsa_slot_mapping(
common_attn_metadata.block_table_tensor,
token_to_req,
logical_positions,
storage_block_size,
compress_ratio,
slot_mapping_buffer,
)
slot_mapping.masked_fill_(
common_attn_metadata.slot_mapping[:num_tokens] < 0, -1
)
if k_work_metadata_buffer is not None:
query_lens = (
common_attn_metadata.query_start_loc[1:]
- common_attn_metadata.query_start_loc[:-1]
)
chunk_starts = common_attn_metadata.seq_lens - query_lens
num_work_per_request = (
common_attn_metadata.seq_lens // compress_ratio
- chunk_starts // compress_ratio
)
num_work_per_request = torch.where(
query_lens > 0, num_work_per_request.clamp_min(1), 0
)
k_start_loc = torch.empty(
query_lens.shape[0] + 1,
dtype=torch.int32,
device=query_lens.device,
)
k_start_loc[0] = 0
torch.cumsum(num_work_per_request, 0, out=k_start_loc[1:])
work = torch.arange(
k_work_metadata_buffer.shape[0],
device=k_work_metadata_buffer.device,
)
requests = torch.searchsorted(k_start_loc[1:], work, right=True)
active = work < k_start_loc[-1]
work_in_request = (
work - k_start_loc[requests.clamp_max(query_lens.shape[0] - 1)]
)
k_work_metadata_buffer[:, 0].copy_(
torch.where(active, requests, -1).to(torch.int32)
)
k_work_metadata_buffer[:, 1].copy_(
torch.where(active, work_in_request, -1).to(torch.int32)
)
return token_to_req, logical_positions, slot_mapping
# Resolve the fallback outside the per-step metadata hot path.
build_qsa_metadata = (
build_qsa_metadata_triton if HAS_TRITON else _build_qsa_metadata_torch
)
@dataclass
class QSAForwardMetadata(AttentionMetadata):
"""Common per-forward metadata for one QSA side cache."""
block_table: torch.Tensor
slot_mapping: torch.Tensor
seq_lens: torch.Tensor
query_start_loc: torch.Tensor
token_to_req: torch.Tensor
logical_positions: torch.Tensor
k_work_metadata: torch.Tensor
num_actual_tokens: int
storage_block_size: int
compress_ratio: int
class QSAMetadataBuilder(AttentionMetadataBuilder[QSAForwardMetadata]):
"""Build QSA metadata from vLLM's cache-group-specific common metadata."""
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
def __init__(
self,
kv_cache_spec: AttentionSpec,
layer_names: list[str],
vllm_config: VllmConfig,
device: torch.device,
) -> None:
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
self.is_circular_buffer = isinstance(kv_cache_spec, CircularBufferSpec)
if isinstance(kv_cache_spec, MLAAttentionSpec):
compress_ratio = kv_cache_spec.tokens_per_state
assert isinstance(compress_ratio, int), (
"QSA compression requires an integer tokens_per_state"
)
self.compress_ratio = compress_ratio
else:
self.compress_ratio = 1
self.storage_block_size = kv_cache_spec.num_states
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
self.token_to_req_buffer = torch.empty(
max_tokens, dtype=torch.int32, device=device
)
self.slot_mapping_buffer = torch.empty(
max_tokens, dtype=torch.int64, device=device
)
self.logical_positions_buffer = torch.empty(
max_tokens, dtype=torch.int64, device=device
)
max_requests = vllm_config.scheduler_config.max_num_seqs
self.request_capacity = max_requests
if not self.is_circular_buffer and self.compress_ratio != 1:
max_k_work = (
max_tokens + (self.compress_ratio - 1) * max_requests
) // self.compress_ratio
self.k_work_metadata_buffer = torch.empty(
max_k_work, 2, dtype=torch.int32, device=device
)
else:
self.k_work_metadata_buffer = torch.empty(
0, 2, dtype=torch.int32, device=device
)
def build(
self,
common_prefix_len: int,
common_attn_metadata: CommonAttentionMetadata,
fast_build: bool = False,
) -> QSAForwardMetadata:
del common_prefix_len, fast_build
num_tokens = common_attn_metadata.num_actual_tokens
build_k_work = not self.is_circular_buffer and self.compress_ratio != 1
k_work_metadata = self.k_work_metadata_buffer
request_capacity = None
if build_k_work:
num_requests = common_attn_metadata.query_start_loc.shape[0] - 1
request_capacity = self.request_capacity
max_num_work = (
num_tokens + (self.compress_ratio - 1) * num_requests
) // self.compress_ratio
k_work_metadata = self.k_work_metadata_buffer[:max_num_work]
token_to_req, logical_positions, slot_mapping = build_qsa_metadata(
common_attn_metadata,
self.token_to_req_buffer,
self.logical_positions_buffer,
self.slot_mapping_buffer,
storage_block_size=self.storage_block_size,
compress_ratio=self.compress_ratio,
circular_buffer_size=(
self.kv_cache_spec.block_size if self.is_circular_buffer else 0
),
k_work_metadata_buffer=k_work_metadata if build_k_work else None,
request_capacity=request_capacity,
)
return QSAForwardMetadata(
block_table=common_attn_metadata.block_table_tensor,
slot_mapping=slot_mapping,
seq_lens=common_attn_metadata.seq_lens,
query_start_loc=common_attn_metadata.query_start_loc,
token_to_req=token_to_req,
logical_positions=logical_positions,
k_work_metadata=k_work_metadata,
num_actual_tokens=num_tokens,
storage_block_size=self.storage_block_size,
compress_ratio=self.compress_ratio,
)
class QSAStateBackend(AttentionBackend):
"""Key-only dummy backend for out-of-band BF16 QSA side-cache operations."""
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = ["auto", "bfloat16"]
@staticmethod
def get_name() -> str:
return "QWEN4_EXP_EXP_QSA_STATE"
@staticmethod
def get_impl_cls():
raise NotImplementedError(
"QSA state caches run out-of-band and have no attention impl"
)
@staticmethod
def get_builder_cls() -> type[QSAMetadataBuilder]:
return QSAMetadataBuilder
@classmethod
def supported_kv_cache_layouts(cls) -> tuple[KVCacheLayout, ...]:
# QSA pages are packed beside the main KV pages within each block.
return (KVCacheLayout.BLNHC, KVCacheLayout.BLHNC)
class _QSAStateCache(nn.Module, AttentionLayerBase):
supports_dcp = False
def __init__(
self,
*,
head_size: int,
dtype: torch.dtype,
cache_config: CacheConfig,
prefix: str,
vllm_config: VllmConfig,
compress_ratio: int = 1,
) -> None:
super().__init__()
if head_size <= 0:
raise ValueError("QSA cache head size must be positive")
if compress_ratio <= 0:
raise ValueError("QSA compression ratio must be positive")
if cache_config.block_size % compress_ratio:
raise ValueError(
"QSA cache block size must be divisible by the compression ratio"
)
self.head_size = head_size
self.dtype = dtype
self.cache_config = cache_config
self.prefix = prefix
self.compress_ratio = compress_ratio
self.kv_cache = torch.tensor([])
static_context = vllm_config.compilation_config.static_forward_context
if prefix in static_context:
raise ValueError(f"Duplicate layer name: {prefix}")
static_context[prefix] = self
def forward(self) -> None: ...
def bind_kv_cache(self, kv_cache: torch.Tensor) -> None:
"""Adapt the unified [B, H, N, C] view to QSA's [B, N, H, C]."""
if kv_cache.ndim != 4 or kv_cache.shape[1] != 1:
raise ValueError("QSA state cache must be [blocks, 1, states, width]")
if kv_cache.dtype != torch.bfloat16 or kv_cache.shape[3] != self.head_size:
raise ValueError("QSA state cache does not match its packed BF16 spec")
super().bind_kv_cache(kv_cache.transpose(1, 2))
def get_attn_backend(self) -> type[AttentionBackend]:
return QSAStateBackend
class QSAKeyStateCache(_QSAStateCache):
"""Raw BF16 key, optionally followed by exact int64 MRoPE positions."""
_BF16_PER_INT64 = 4
_NUM_ROPE_AXES = 3
def __init__(self, *, cache_rope_positions: bool = False, **kwargs) -> None:
key_head_size = int(kwargs.pop("head_size"))
self.key_head_size = key_head_size
self.cache_rope_positions = bool(cache_rope_positions)
self.rope_position_offset = (
(key_head_size + self._BF16_PER_INT64 - 1) // self._BF16_PER_INT64
) * self._BF16_PER_INT64
storage_head_size = key_head_size
if self.cache_rope_positions:
storage_head_size = self.rope_position_offset + (
self._NUM_ROPE_AXES * self._BF16_PER_INT64
)
super().__init__(head_size=storage_head_size, **kwargs)
def bind_kv_cache(self, kv_cache: torch.Tensor) -> None:
super().bind_kv_cache(kv_cache)
qsa_cache = self.kv_cache
self.key_cache = qsa_cache[..., : self.key_head_size]
if self.cache_rope_positions:
position_tail = qsa_cache[..., self.rope_position_offset :]
self.rope_position_cache = position_tail.view(torch.int64)
else:
self.rope_position_cache = None
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
# Hold the open group's committed keys plus every row a speculative
# step stores before acceptance is known, rounded up to whole groups so
# the ring divides the attention block size (it joins the LCM that sets
# the scheduler block size). Anything narrower lets a rejected draft row
# overwrite a committed key the next step needs to close the group.
span = self.compress_ratio + vllm_config.num_speculative_tokens
capacity = self.compress_ratio * cdiv(span, self.compress_ratio)
assert self.cache_config.block_size % capacity == 0, (
f"QSA ring capacity {capacity} must divide the attention block "
f"size {self.cache_config.block_size}"
)
return CircularBufferSpec(
block_size=capacity,
num_kv_heads=1,
head_size=self.head_size,
head_size_v=0,
dtype=self.dtype,
)
class QSACompressedKeyCache(_QSAStateCache):
"""Normalized, group-first-RoPE BF16 key at one row per complete group."""
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
del vllm_config
return MLAAttentionSpec(
block_size=self.cache_config.block_size,
num_kv_heads=1,
head_size=self.head_size,
dtype=self.dtype,
tokens_per_state=self.compress_ratio,
)
__all__ = [
"QSACompressedKeyCache",
"QSAForwardMetadata",
"QSAKeyStateCache",
"QSAMetadataBuilder",
"QSAStateBackend",
"canonical_qsa_rope_positions",
"circular_qsa_slot_mapping",
"compressed_qsa_slot_mapping",
]
+252
View File
@@ -0,0 +1,252 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Qwen4Exp model configuration."""
from typing import Any, ClassVar, cast
from transformers import PretrainedConfig
from transformers.models.qwen3_vl.configuration_qwen3_vl import (
Qwen3VLVisionConfig,
)
from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig
_QSA_CONFIG_FIELDS = (
"indexer_n_heads",
"indexer_kv_heads",
"indexer_head_dim",
"indexer_budget",
"indexer_compress_ratio",
)
class Qwen4ExpVisionConfig(Qwen3VLVisionConfig):
model_type = "qwen4_exp"
base_config_key = "vision_config"
class Qwen4ExpTextConfig(Qwen3NextConfig):
model_type = "qwen4_exp_text"
base_config_key = "text_config"
keys_to_ignore_at_inference = ["past_key_values"]
def __init__(
self,
hc_count: int = 4,
hc_lowrank: int = 320,
ple_layer_ids: list[int] | None = None,
ple_embed_dim: int | None = None,
ple_conv_kernel_size: int = 4,
ngram_size: int = 3,
heads_per_ngram: int = 8,
ngram_vocab_size_base: int = 20_000_000,
make_ngram_vocab_size_divisible_by: int = 128,
output_gate_type: str = "sigmoid",
rope_parameters: dict[str, Any] | None = None,
layer_types: list[str] | None = None,
**kwargs: Any,
) -> None:
if hc_count <= 1:
raise ValueError(f"Qwen4Exp requires hc_count > 1, got {hc_count}.")
if rope_parameters is not None:
if kwargs.get("rope_scaling") is None:
kwargs["rope_scaling"] = rope_parameters
if kwargs.get("rope_theta") is None and "rope_theta" in rope_parameters:
kwargs["rope_theta"] = rope_parameters["rope_theta"]
if (
kwargs.get("partial_rotary_factor") is None
and "partial_rotary_factor" in rope_parameters
):
kwargs["partial_rotary_factor"] = rope_parameters[
"partial_rotary_factor"
]
rope_scaling = kwargs.get("rope_scaling")
rope_theta = kwargs.get("rope_theta", 10_000.0)
super().__init__(layer_types=layer_types, **kwargs)
normalized_rope_parameters = self.rope_parameters
self.rope_scaling = (
rope_scaling or rope_parameters or normalized_rope_parameters
)
self.rope_parameters = rope_parameters or normalized_rope_parameters
self.rope_theta = rope_theta
self.hc_count = hc_count
self.hc_lowrank = hc_lowrank
self.ple_layer_ids = ple_layer_ids or []
self.ple_embed_dim = (
self.hidden_size if ple_embed_dim is None else ple_embed_dim
)
self.ple_conv_kernel_size = ple_conv_kernel_size
self.ngram_size = ngram_size
self.heads_per_ngram = heads_per_ngram
self.ngram_vocab_size_base = ngram_vocab_size_base
self.make_ngram_vocab_size_divisible_by = make_ngram_vocab_size_divisible_by
self.output_gate_type = output_gate_type
self._validate_ple_config()
self._validate_ple_layer_ids()
self._validate_qsa_config()
def _validate_ple_config(self) -> None:
if self.hc_lowrank <= 0:
raise ValueError(f"hc_lowrank must be positive, got {self.hc_lowrank}")
if self.ngram_size < 2:
raise ValueError(f"ngram_size must be >= 2, got {self.ngram_size}")
if self.heads_per_ngram <= 0:
raise ValueError(
f"heads_per_ngram must be positive, got {self.heads_per_ngram}"
)
if self.ple_embed_dim <= 0:
raise ValueError(
f"ple_embed_dim must be positive, got {self.ple_embed_dim}"
)
ngram_heads = (self.ngram_size - 1) * self.heads_per_ngram
if self.ple_embed_dim % ngram_heads:
raise ValueError(
"ple_embed_dim must be divisible by total ngram heads: "
f"{self.ple_embed_dim} % {ngram_heads} != 0"
)
if self.ple_conv_kernel_size <= 0:
raise ValueError(
"ple_conv_kernel_size must be positive, got "
f"{self.ple_conv_kernel_size}"
)
if self.ngram_vocab_size_base <= 0:
raise ValueError("ngram_vocab_size_base must be positive")
if self.make_ngram_vocab_size_divisible_by <= 0:
raise ValueError("make_ngram_vocab_size_divisible_by must be positive")
def _validate_ple_layer_ids(self) -> None:
invalid = [
layer_id
for layer_id in self.ple_layer_ids
if not 1 <= int(layer_id) <= self.num_hidden_layers
]
if invalid:
raise ValueError(
"ple_layer_ids are 1-based and must refer to an existing layer; "
f"got {invalid} for {self.num_hidden_layers} layers"
)
def _validate_qsa_config(self) -> None:
configured = {name: getattr(self, name, None) for name in _QSA_CONFIG_FIELDS}
if all(value is None for value in configured.values()):
return
missing = [name for name, value in configured.items() if value is None]
if missing:
raise ValueError(f"QSA config is missing required fields: {missing}")
values = {name: int(cast(int, value)) for name, value in configured.items()}
if any(value <= 0 for value in values.values()):
raise ValueError(f"QSA config values must be positive: {values}")
if values["indexer_kv_heads"] != 1:
raise ValueError("the QSA MQA operators require indexer_kv_heads=1")
if values["indexer_budget"] % values["indexer_compress_ratio"] != 0:
raise ValueError(
"indexer_budget must be divisible by indexer_compress_ratio"
)
block_topk = values["indexer_budget"] // values["indexer_compress_ratio"]
if block_topk not in (512, 2048):
raise ValueError(
"QSA requires indexer_budget / indexer_compress_ratio "
f"to be 512 or 2048, got {block_topk}"
)
rotary_dim = int(self.head_dim * self.partial_rotary_factor)
if rotary_dim > values["indexer_head_dim"]:
raise ValueError(
"QSA indexer_head_dim must cover the attention rotary "
f"dimension, got {values['indexer_head_dim']} < {rotary_dim}"
)
@property
def layers_block_type(self) -> list[str]:
return [
"attention" if layer_type == "full_attention" else layer_type
for layer_type in self.layer_types
]
@property
def short_conv_layer_ids(self) -> list[int]:
if not self.ple_layer_ids:
return []
return sorted({int(layer_id) - 1 for layer_id in self.ple_layer_ids})
@property
def short_conv_state_shape(self) -> tuple[int, int] | None:
if not self.short_conv_layer_ids:
return None
ple_state_len = (self.ple_conv_kernel_size - 1) * self.ngram_size
ple_channels = self.hidden_size * self.hc_count
return ple_channels, ple_state_len
@property
def ngram_context_len(self) -> int:
if not self.ple_layer_ids:
return 0
return max(int(self.ngram_size) - 1, 0)
class Qwen4ExpConfig(PretrainedConfig):
model_type = "qwen4_exp"
sub_configs: ClassVar[dict[str, type[PretrainedConfig]]] = {
"vision_config": Qwen4ExpVisionConfig,
"text_config": Qwen4ExpTextConfig,
}
keys_to_ignore_at_inference = ["past_key_values"]
def __init__(
self,
text_config: Qwen4ExpTextConfig | dict[str, Any] | None = None,
vision_config: Qwen4ExpVisionConfig | dict[str, Any] | None = None,
image_token_id: int = 248056,
video_token_id: int = 248057,
vision_start_token_id: int = 248053,
vision_end_token_id: int = 248054,
tie_word_embeddings: bool = False,
rope_parameters: dict[str, Any] | None = None,
**kwargs: Any,
) -> None:
if text_config is not None:
kwargs.pop("split_ngram_parts", None)
text_kwargs = (
dict(kwargs)
if text_config is None
and "hidden_size" in kwargs
and "num_hidden_layers" in kwargs
else {}
)
if isinstance(vision_config, dict):
self.vision_config = self.sub_configs["vision_config"](**vision_config)
elif vision_config is None:
self.vision_config = self.sub_configs["vision_config"]()
else:
self.vision_config = vision_config
if isinstance(text_config, dict):
self.text_config = self.sub_configs["text_config"](**text_config)
elif text_config is None:
self.text_config = self.sub_configs["text_config"](**text_kwargs)
else:
self.text_config = text_config
self.image_token_id = image_token_id
self.video_token_id = video_token_id
self.vision_start_token_id = vision_start_token_id
self.vision_end_token_id = vision_end_token_id
self.rope_parameters = rope_parameters or getattr(
self.text_config, "rope_parameters", {}
)
super().__init__(**kwargs, tie_word_embeddings=tie_word_embeddings)
__all__ = [
"Qwen4ExpConfig",
"Qwen4ExpTextConfig",
"Qwen4ExpVisionConfig",
]
+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,205 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""HyperConnection (Gated Residual) utilities — NVIDIA model variant.
Implements the HyperConnection residual scheme proposed in
"HyperConnections" (https://arxiv.org/abs/2409.19606). This NVIDIA variant
delays each HC combine to the following HC mix boundary. HC glue kernels,
including fused combine+RMSNorm, live in ``ops/hc.py``; projections remain
standard vLLM Linear modules.
Hidden states between layers have shape ``[..., HC*HS]`` with HS inner
(HC outer, HS inner — checkpoint-native layout).
Typical usage inside a transformer decoder layer::
self.attn_hc = GatedResidual(hc_config)
hidden_states, block_input, injection = self.attn_hc.mix(hidden_states)
attention_output = attention(block_input)
hidden_states, block_input, injection = self.mlp_hc.combine_and_mix(
hidden_states, attention_output, injection
)
"""
import torch
from torch import nn
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
ReplicatedLinear,
)
from vllm.model_executor.models.utils import maybe_prefix
from ..common.hyperconnection import (
GroupedGemmaRMSNorm,
HyperConnectionConfig,
)
from .ops.hc import (
grouped_gemma_rmsnorm,
hc_combine,
hc_combine_norm,
hc_gate_mix,
hc_silu,
)
# ---------------------------------------------------------------------------
# Gated-residual variant
# ---------------------------------------------------------------------------
class GatedResidual(nn.Module):
"""Gated HyperConnection with learnable low-rank mixing and injection.
``combine_and_mix()`` runs the pre pipeline (grouped GemmaRMSNorm -> merged
low-rank down+inject GEMM -> silu -> up GEMM -> sigmoid -> gated mean
over the HC streams). When passed a pending block output and an injection,
it fuses their residual combine with the RMSNorm. Final mixers use
``use_combine=False`` and do not produce a new injection.
Weights: the norm owns the grouped GemmaRMSNorm affine; the projections
are vLLM Linear modules (merged replicated linear for down+inject), so
GEMM dispatch (e.g. the low-latency skinny GEMM) applies through the
standard quant_method mechanism.
"""
def __init__(
self,
config: HyperConnectionConfig,
use_combine: bool = True,
prefix: str = "",
) -> None:
super().__init__()
self.config = config
self.lora_rank = config.hc_lowrank
self.hc_count = config.hc_count
self.hidden_size = config.hidden_size
self.use_combine = use_combine
norm_size = (
self.hyper_hidden_size if config.hc_per_branch_norm else config.hidden_size
)
group_size = config.hidden_size if config.hc_per_branch_norm else None
# Normalize each H-sized HC stream independently while retaining a
# separate affine weight for every element of the HC*H layout.
self.hc_norm = GroupedGemmaRMSNorm(
norm_size,
eps=config.rms_norm_eps,
group_size=group_size,
dtype=config.params_dtype,
)
# -- vLLM Linear weights --------------------------------------------
# The merged skinny-GEMM shape is physically padded to 16 rows to ensure
# good alignment and performant implementation chosen by CuBLAS heuristics.
self.pad_size = (-(self.lora_rank + self.hc_count)) % 16 if use_combine else 0
if use_combine:
self.input_mix_weight_down_block_inject = MergedColumnParallelLinear(
self.hyper_hidden_size,
[self.lora_rank, self.hc_count]
+ ([self.pad_size] if self.pad_size else []),
bias=False,
params_dtype=config.params_dtype,
quant_config=None,
prefix=maybe_prefix(prefix, "input_mix_weight_down_block_inject"),
return_bias=False,
disable_tp=True,
)
else:
self.input_mix_weight_down = ReplicatedLinear(
self.hyper_hidden_size,
self.lora_rank,
bias=False,
params_dtype=config.params_dtype,
quant_config=None,
prefix=maybe_prefix(prefix, "input_mix_weight_down"),
return_bias=False,
)
self.input_mix_weight_up = ReplicatedLinear(
self.lora_rank,
self.hyper_hidden_size,
bias=False,
params_dtype=config.params_dtype,
quant_config=None,
prefix=maybe_prefix(prefix, "input_mix_weight_up"),
return_bias=False,
)
def mix(
self, hidden_states: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
xn = grouped_gemma_rmsnorm(
hidden_states,
self.hc_norm.weight,
self.config.rms_norm_eps,
self.hc_count,
)
if self.use_combine:
# produce injection logits for combine
split_sizes = [self.lora_rank, self.hc_count, self.pad_size]
down_and_injection = self.input_mix_weight_down_block_inject(xn)
lora, injection, _ = down_and_injection.split(split_sizes, dim=-1)
else:
lora = self.input_mix_weight_down(xn)
injection = None
lora = hc_silu(lora, self.hc_count)
gate = self.input_mix_weight_up(lora) # [M, D]
block_input = hc_gate_mix(xn, gate, self.hc_count)
return hidden_states, block_input, injection
def combine_and_mix(
self,
hidden_states: torch.Tensor,
prev_block_output: torch.Tensor,
prev_injection: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
"""Consume a pending combine, then prepare the next block input.
``hidden_states`` is the multi-stream state from before the pending
block's mix. Its combine with ``block_output`` is fused with this
module's input RMSNorm.
"""
hidden_states, xn = hc_combine_norm(
hidden_states,
prev_block_output,
prev_injection,
self.hc_norm.weight,
self.config.rms_norm_eps,
self.hc_count,
)
if self.use_combine:
# produce injection logits for combine
split_sizes = [self.lora_rank, self.hc_count, self.pad_size]
down_and_injection = self.input_mix_weight_down_block_inject(xn)
lora, injection, _ = down_and_injection.split(split_sizes, dim=-1)
else:
lora = self.input_mix_weight_down(xn)
injection = None
lora = hc_silu(lora, self.hc_count)
gate = self.input_mix_weight_up(lora) # [M, D]
block_input = hc_gate_mix(xn, gate, self.hc_count)
return hidden_states, block_input, injection
def combine(
self,
hidden_states: torch.Tensor,
block_output: torch.Tensor,
injection: torch.Tensor,
) -> torch.Tensor:
return hc_combine(hidden_states, block_output, injection, self.hc_count)
@property
def hyper_hidden_size(self) -> int:
return self.hc_count * self.hidden_size
__all__ = [
"GatedResidual",
"GroupedGemmaRMSNorm",
"HyperConnectionConfig",
]
+356
View File
@@ -0,0 +1,356 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Qwen4Exp weight-free QSA indexer."""
from __future__ import annotations
from typing import cast
import torch
from torch import nn
from vllm.config import VllmConfig
from vllm.forward_context import get_forward_context
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding.mrope import triton_mrope
from vllm.transformers_utils.configs.qwen4_exp import (
Qwen4ExpTextConfig,
)
from ..common.qsa_cache import (
QSACompressedKeyCache,
QSAForwardMetadata,
QSAKeyStateCache,
canonical_qsa_rope_positions,
)
from .ops.qsa_pre_indexer import qsa_pre_indexer
def apply_qsa_rope(
rotary_emb: nn.Module,
positions: torch.Tensor,
tensor: torch.Tensor,
) -> torch.Tensor:
"""Apply the main attention's exact 1D/MRoPE composition to QSA heads."""
num_tokens, _, head_dim = tensor.shape
rotary_dim = rotary_emb.rotary_dim
cache = rotary_emb._match_cos_sin_cache_dtype(tensor) # noqa: SLF001
cos_sin = cache[positions]
cos, sin = cos_sin.chunk(2, dim=-1)
if positions.ndim == 2:
shape = tensor.shape
tensor, _ = triton_mrope(
tensor.reshape(num_tokens, -1),
tensor.new_empty((num_tokens, head_dim)),
cos,
sin,
rotary_emb.mrope_section,
head_dim,
rotary_dim,
rotary_emb.mrope_interleaved,
rotary_emb.is_neox_style,
)
return tensor.reshape(shape)
rotated = rotary_emb.apply_rotary_emb.forward_cuda(
tensor[..., :rotary_dim],
cos,
sin,
)
return torch.cat((rotated, tensor[..., rotary_dim:]), dim=-1)
def _supports_fused_pre_indexer(
rotary_emb: nn.Module,
head_dim: int,
num_kv_heads: int,
compress_ratio: int,
) -> bool:
rotary_dim = int(rotary_emb.rotary_dim)
mrope_section = getattr(rotary_emb, "mrope_section", None)
return (
bool(getattr(rotary_emb, "is_neox_style", False))
and (
not mrope_section
or (
len(mrope_section) == 3
and sum(mrope_section) == rotary_dim // 2
and bool(getattr(rotary_emb, "mrope_interleaved", False))
)
)
and head_dim == 128
and rotary_dim == 64
and num_kv_heads == 1
and compress_ratio > 1
and compress_ratio & (compress_ratio - 1) == 0
)
class QSAIndexer(nn.Module):
"""Replicated Q/K projection plus paged, weight-free QSA selection.
``prefix`` must be the checkpoint's indexer prefix, normally
``model.layers.N.self_attn.indexer``. Consequently the trainable names are
``index_qk_proj``, ``q_layernorm`` and ``k_layernorm`` under that prefix.
"""
def __init__(
self,
*,
vllm_config: VllmConfig,
config: Qwen4ExpTextConfig,
layer_id: int,
rotary_emb: nn.Module,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
if vllm_config.cache_config is None:
raise ValueError("QSA requires a paged KV cache")
if vllm_config.model_config.dtype != torch.bfloat16:
raise NotImplementedError("Qwen4Exp QSA currently requires BF16")
self.layer_id = int(layer_id)
self.index_n_heads = int(config.indexer_n_heads)
self.index_kv_heads = int(config.indexer_kv_heads)
self.index_head_dim = int(config.indexer_head_dim)
self.token_topk = int(config.indexer_budget)
self.compress_ratio = int(config.indexer_compress_ratio)
self.rotary_emb = rotary_emb
self.use_fused_pre_indexer = _supports_fused_pre_indexer(
rotary_emb,
self.index_head_dim,
self.index_kv_heads,
self.compress_ratio,
)
self.prefix = prefix
# MTP step 0 selects the target-aligned rows; later steps reuse them
# while continuing to update the QSA side cache.
self.skip_topk = False
self.index_qk_proj = ReplicatedLinear(
int(config.hidden_size),
(self.index_n_heads + self.index_kv_heads) * self.index_head_dim,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.index_qk_proj" if prefix else "index_qk_proj",
)
self.q_layernorm = GemmaRMSNorm(
self.index_head_dim,
eps=float(getattr(config, "rms_norm_eps", 1e-6)),
)
self.k_layernorm = GemmaRMSNorm(
self.index_head_dim,
eps=float(getattr(config, "rms_norm_eps", 1e-6)),
)
cache_config = vllm_config.cache_config
cache_prefix = f"{prefix}." if prefix else ""
self.raw_key_cache = QSAKeyStateCache(
head_size=self.index_head_dim,
dtype=torch.bfloat16,
cache_rope_positions=vllm_config.model_config.uses_mrope,
prefix=f"{cache_prefix}raw_key_cache",
cache_config=cache_config,
compress_ratio=self.compress_ratio,
vllm_config=vllm_config,
)
self.compressed_key_cache = QSACompressedKeyCache(
head_size=self.index_head_dim,
dtype=torch.bfloat16,
compress_ratio=self.compress_ratio,
prefix=f"{cache_prefix}compressed_key_cache",
cache_config=cache_config,
vllm_config=vllm_config,
)
@property
def output_width(self) -> int:
return self.token_topk + self.compress_ratio - 1
def _metadata(
self,
) -> tuple[QSAForwardMetadata, QSAForwardMetadata] | None:
metadata = get_forward_context().attn_metadata
if isinstance(metadata, list):
metadata = metadata[0]
if not isinstance(metadata, dict):
return None
raw = cast(QSAForwardMetadata, metadata[self.raw_key_cache.prefix])
compressed = cast(
QSAForwardMetadata, metadata[self.compressed_key_cache.prefix]
)
if raw.num_actual_tokens != compressed.num_actual_tokens:
raise RuntimeError("QSA side-cache metadata token counts disagree")
if not raw.logical_positions.is_cuda and (
not torch.equal(raw.logical_positions, compressed.logical_positions)
):
raise RuntimeError("QSA side-cache metadata positions disagree")
return raw, compressed
def forward(
self,
hidden_states: torch.Tensor,
positions: torch.Tensor,
out: torch.Tensor | None = None,
) -> torch.Tensor:
"""Return fixed-width request-relative token indices padded with ``-1``."""
metadata = self._metadata()
if metadata is None:
# Preserve step-0 indices when later MTP steps reuse the buffer.
if self.skip_topk and out is not None:
return out
result = torch.full(
(hidden_states.shape[0], self.output_width),
-1,
dtype=torch.int32,
device=hidden_states.device,
)
if out is not None:
out.copy_(result)
return out
return result
from .ops.qsa import (
qsa_compress_groups_with_ratio,
qsa_select_paged_tokens,
qsa_store_cache_rows,
)
raw_metadata, compressed_metadata = metadata
num_tokens = raw_metadata.num_actual_tokens
hidden_states = hidden_states[:num_tokens]
positions = positions[..., :num_tokens]
# Q/K projection
projected_qk, _ = self.index_qk_proj(hidden_states)
projected_q, raw_keys = projected_qk.split(
(
self.index_n_heads * self.index_head_dim,
self.index_kv_heads * self.index_head_dim,
),
dim=-1,
)
raw_key_state_cache = self.raw_key_cache
compressed_key_cache = self.compressed_key_cache.kv_cache
if self.use_fused_pre_indexer:
q = projected_q.new_empty(
num_tokens,
self.index_n_heads,
self.index_head_dim,
)
qsa_pre_indexer(
projected_q,
raw_keys,
positions,
self.rotary_emb.cos_sin_cache,
self.q_layernorm.weight,
self.k_layernorm.weight,
self.q_layernorm.variance_epsilon,
q,
raw_key_state_cache.kv_cache,
raw_metadata.slot_mapping,
raw_metadata.block_table,
raw_metadata.query_start_loc,
raw_metadata.logical_positions,
compressed_key_cache,
compressed_metadata.slot_mapping,
compressed_metadata.k_work_metadata,
compress_ratio=self.compress_ratio,
mrope_section=getattr(self.rotary_emb, "mrope_section", None),
rope_pos_offset=(
raw_key_state_cache.rope_position_offset
if raw_key_state_cache.rope_position_cache is not None
else None
),
)
else:
# Unfused reference path
from flashinfer.norm import gemma_rmsnorm
q = projected_q.reshape(-1, self.index_n_heads, self.index_head_dim)
q = gemma_rmsnorm(
q.reshape(-1, self.index_head_dim),
self.q_layernorm.weight,
self.q_layernorm.variance_epsilon,
).reshape_as(q)
q = apply_qsa_rope(self.rotary_emb, positions, q)
raw_key_cache = raw_key_state_cache.key_cache
rope_position_cache = raw_key_state_cache.rope_position_cache
if rope_position_cache is None:
position_rows = raw_metadata.logical_positions.view(-1, 1, 1).expand(
-1, 1, 3
)
else:
position_rows = canonical_qsa_rope_positions(positions).to(
device=raw_key_cache.device
)
pooled, first_positions = qsa_compress_groups_with_ratio(
raw_keys.reshape(-1, 1, self.index_head_dim),
position_rows,
raw_key_cache,
raw_metadata.block_table,
raw_metadata.token_to_req,
raw_metadata.query_start_loc,
raw_metadata.logical_positions,
compressed_metadata.slot_mapping,
self.compress_ratio,
rope_position_cache,
)
compressed_keys = gemma_rmsnorm(
pooled.reshape(-1, self.index_head_dim),
self.k_layernorm.weight,
self.k_layernorm.variance_epsilon,
).reshape(-1, 1, self.index_head_dim)
if getattr(self.rotary_emb, "mrope_section", None):
first_positions = first_positions.transpose(0, 1)
else:
first_positions = first_positions[:, 0]
compressed_keys = apply_qsa_rope(
self.rotary_emb,
first_positions,
compressed_keys,
)
qsa_store_cache_rows(
compressed_key_cache,
compressed_metadata.slot_mapping,
compressed_keys,
)
qsa_store_cache_rows(
raw_key_cache,
raw_metadata.slot_mapping,
raw_keys,
)
if rope_position_cache is not None:
qsa_store_cache_rows(
rope_position_cache,
raw_metadata.slot_mapping,
position_rows,
)
if self.skip_topk:
if out is None:
raise RuntimeError("QSA top-k reuse requires an output buffer")
return out
# Score compressed keys, select blocks, then expand them to token indices.
return qsa_select_paged_tokens(
q,
compressed_key_cache,
compressed_metadata.block_table,
compressed_metadata.token_to_req,
compressed_metadata.logical_positions,
compressed_metadata.seq_lens,
self.token_topk,
self.compress_ratio,
out,
)
__all__ = ["QSAIndexer", "apply_qsa_rope"]
@@ -0,0 +1,185 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Qwen4Exp decode GEMM selection on Blackwell.
Dispatch follows Kimi-K3 and uses the local ``(N, K)`` shape and token count.
Plans contain measured CUDA graph capture sizes; other token counts use the
standard linear implementation.
"""
import torch
from torch import nn
import vllm.envs as envs
from vllm.model_executor.kernels.linear.cute_dsl.skinny_gemm import (
SkinnyGemmConfig,
shape_dynamic_skinny_gemm,
)
from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
UnquantizedEmbeddingMethod,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import direct_register_custom_op
QWEN4_EXP_GEMM_PLANS: dict[tuple[int, int], dict[int, SkinnyGemmConfig]] = {
# GDN fused QKVZ projection, TP=4.
(4096, 2560): {
1: SkinnyGemmConfig(1, 64, 4, k_unroll=4),
2: SkinnyGemmConfig(2, 64, 4, k_unroll=4),
},
# GDN and QSA output projections, TP=4.
(2560, 1536): {
1: SkinnyGemmConfig(1, 128, 2, k_unroll=2, vector_width=4),
2: SkinnyGemmConfig(2, 128, 2, k_unroll=2, vector_width=4),
4: SkinnyGemmConfig(4, 64, 2, k_unroll=2),
},
# GDN fused B/A projection, TP=4.
(24, 2560): {
1: SkinnyGemmConfig(1, 128, 2, k_unroll=4, vector_width=4),
2: SkinnyGemmConfig(2, 128, 2, k_unroll=4, vector_width=4),
4: SkinnyGemmConfig(4, 128, 2, k_unroll=4, vector_width=4),
8: SkinnyGemmConfig(8, 128, 1, k_unroll=4, vector_width=4),
16: SkinnyGemmConfig(16, 128, 1, k_unroll=4, vector_width=4),
},
# QSA fused QKV/gate projection, TP=4.
(3584, 2560): {
1: SkinnyGemmConfig(1, 128, 4, k_unroll=4, vector_width=4),
2: SkinnyGemmConfig(2, 64, 2, k_unroll=2),
4: SkinnyGemmConfig(4, 64, 2, k_unroll=2),
},
# QSA indexer Q/K projection, replicated in a TP=4 deployment.
(640, 2560): {
1: SkinnyGemmConfig(1, 128, 1, k_unroll=4, vector_width=4),
2: SkinnyGemmConfig(2, 128, 1, k_unroll=4, vector_width=4),
4: SkinnyGemmConfig(4, 128, 1, k_unroll=4, vector_width=4),
8: SkinnyGemmConfig(8, 128, 1, k_unroll=4, vector_width=4),
},
# Shared-expert fused gate/up projection, TP=4.
(320, 2560): {
1: SkinnyGemmConfig(1, 128, 2, k_unroll=4, vector_width=4),
2: SkinnyGemmConfig(2, 128, 2, k_unroll=4, vector_width=4),
4: SkinnyGemmConfig(4, 128, 2, k_unroll=4, vector_width=4),
8: SkinnyGemmConfig(8, 64, 1, k_unroll=4),
16: SkinnyGemmConfig(16, 128, 2, k_unroll=4, vector_width=4),
},
# LM head, TP=4.
(62080, 2560): {
1: SkinnyGemmConfig(1, 64, 4, k_unroll=2),
2: SkinnyGemmConfig(2, 32, 4, k_unroll=2),
},
# HC merged down/injection projection, replicated in a TP=4 deployment.
(336, 10240): {
1: SkinnyGemmConfig(1, 128, 1, static_k=10240),
2: SkinnyGemmConfig(2, 128, 1, static_k=10240),
4: SkinnyGemmConfig(4, 128, 2, static_k=10240),
8: SkinnyGemmConfig(8, 128, 1, k_unroll=4),
},
}
def _is_sm103() -> bool:
return current_platform.is_device_capability((10, 3))
def _is_packed_row_major(tensor: torch.Tensor) -> bool:
return tensor.dim() == 2 and tensor.stride() == (tensor.shape[1], 1)
def _runtime_ok(x: torch.Tensor, weight: torch.Tensor) -> bool:
return (
not envs.VLLM_BATCH_INVARIANT
and _is_packed_row_major(x)
and _is_packed_row_major(weight)
and x.dtype == torch.bfloat16
and weight.dtype == torch.bfloat16
and x.is_cuda
and weight.is_cuda
and x.device == weight.device
and x.shape[1] == weight.shape[1]
)
class _Qwen4ExpLowLatencyApply:
def apply(
self,
layer: nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
if bias is None and not envs.VLLM_BATCH_INVARIANT:
return torch.ops.vllm.qwen4_exp_low_latency_gemm(x, layer.weight)
return super().apply(layer, x, bias) # type: ignore[misc]
class Qwen4ExpLowLatencyLinearMethod(_Qwen4ExpLowLatencyApply, UnquantizedLinearMethod):
pass
class Qwen4ExpLowLatencyEmbeddingMethod(
_Qwen4ExpLowLatencyApply, UnquantizedEmbeddingMethod
):
pass
def _qwen4_exp_low_latency_gemm(x: torch.Tensor, weight: torch.Tensor) -> torch.Tensor:
plan = QWEN4_EXP_GEMM_PLANS.get((weight.shape[0], weight.shape[1]))
config = None if plan is None else plan.get(x.shape[0])
if (
config is not None
and _runtime_ok(x, weight)
and shape_dynamic_skinny_gemm.is_available()
):
return shape_dynamic_skinny_gemm(x, weight, config)
return torch.nn.functional.linear(x, weight)
def _qwen4_exp_low_latency_gemm_fake(
x: torch.Tensor, weight: torch.Tensor
) -> torch.Tensor:
return x.new_empty((*x.shape[:-1], weight.shape[0]))
direct_register_custom_op(
op_name="qwen4_exp_low_latency_gemm",
op_func=_qwen4_exp_low_latency_gemm,
fake_impl=_qwen4_exp_low_latency_gemm_fake,
)
def enable_qwen4_exp_low_latency_gemm(
module: nn.Module,
dtype: torch.dtype,
) -> None:
if dtype != torch.bfloat16 or not _is_sm103():
return
if not shape_dynamic_skinny_gemm.is_available():
return
warmup_configs: set[SkinnyGemmConfig] = set()
for child in module.modules():
is_linear = (
isinstance(child, LinearBase)
and type(child.quant_method) is UnquantizedLinearMethod
)
is_head = (
isinstance(child, ParallelLMHead)
and type(child.quant_method) is UnquantizedEmbeddingMethod
)
if not (is_linear or is_head):
continue
weight = getattr(child, "weight", None)
if weight is None or weight.dim() != 2:
continue
plan = QWEN4_EXP_GEMM_PLANS.get((weight.shape[0], weight.shape[1]))
if plan is None:
continue
if is_linear:
child.quant_method = Qwen4ExpLowLatencyLinearMethod()
else:
child.quant_method = Qwen4ExpLowLatencyEmbeddingMethod()
warmup_configs.update(plan.values())
if warmup_configs:
shape_dynamic_skinny_gemm.request_warmup_configs(dtype, warmup_configs)
File diff suppressed because it is too large Load Diff
+143
View File
@@ -0,0 +1,143 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Model-runner state for Qwen4Exp PLE inputs."""
from typing import Any
import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
from vllm.v1.worker.gpu.states import RequestState
class Qwen4ExpModelState(MambaHybridModelState):
"""Add rollback-safe PLE n-gram context to the model inputs."""
def __init__(
self,
vllm_config: VllmConfig,
model: nn.Module,
encoder_cache: EncoderCache | None,
device: torch.device,
) -> None:
super().__init__(vllm_config, model, encoder_cache, device)
config = self.model_config.hf_text_config
self.uses_ngram_embedding = bool(config.ple_layer_ids)
if not self.uses_ngram_embedding:
self.ngram_context_len = 0
self.ngram_eos_token_id = 0
return
if vllm_config.parallel_config.pipeline_parallel_size > 1:
raise RuntimeError(
"N-gram PLE embedding currently requires "
"pipeline_parallel_size=1 because non-first pipeline ranks do "
"not receive the raw input_ids required by PLE. Please run "
"with PP=1."
)
self.ngram_context_len = int(config.ngram_size) - 1
if self.ngram_context_len <= 0:
raise ValueError("N-gram embedding requires context length >= 1.")
self.ngram_eos_token_id = int(config.eos_token_id)
self.ngram_context = torch.full(
(self.max_num_reqs, self.ngram_context_len),
self.ngram_eos_token_id,
dtype=torch.int32,
device=self.device,
)
self.ngram_context_offsets = torch.arange(
-self.ngram_context_len,
0,
dtype=torch.int64,
device=self.device,
)
self.ple_query_start_loc = torch.zeros(
self.max_num_reqs + 1,
dtype=torch.int32,
device=self.device,
)
def _prepare_ngram_context(
self,
input_batch: InputBatch,
req_states: RequestState,
) -> torch.Tensor:
num_reqs = input_batch.num_reqs
num_reqs_padded = input_batch.num_reqs_after_padding
context = self.ngram_context[:num_reqs_padded]
context.fill_(self.ngram_eos_token_id)
if num_reqs == 0:
return context
request_indices = input_batch.idx_mapping[:num_reqs].long()
context_end = req_states.num_computed_tokens.gpu[request_indices].long()
token_indices = context_end.unsqueeze(1) + self.ngram_context_offsets
valid_tokens = token_indices >= 0
token_indices.clamp_min_(0)
context_tokens = req_states.all_token_ids.gpu[
request_indices.unsqueeze(1), token_indices
]
context[:num_reqs].copy_(
torch.where(
valid_tokens,
context_tokens,
context_tokens.new_full((), self.ngram_eos_token_id),
)
)
return context
def prepare_inputs(
self,
input_batch: InputBatch,
req_states: RequestState,
) -> dict[str, Any]:
model_inputs = super().prepare_inputs(input_batch, req_states)
if not self.uses_ngram_embedding:
return model_inputs
num_reqs_padded = input_batch.num_reqs_after_padding
query_start_loc = self.ple_query_start_loc[: num_reqs_padded + 1]
query_start_loc.copy_(input_batch.query_start_loc[: num_reqs_padded + 1])
model_inputs.update(
query_start_loc=query_start_loc,
ngram_context=self._prepare_ngram_context(input_batch, req_states),
)
return model_inputs
def prepare_dummy_inputs(
self,
num_reqs: int,
num_tokens: int,
) -> dict[str, Any]:
model_inputs = super().prepare_dummy_inputs(num_reqs, num_tokens)
if not self.uses_ngram_embedding:
return model_inputs
query_start_loc = self.ple_query_start_loc[: num_reqs + 1]
query_start_loc[0] = 0
tokens_per_req, num_extra_tokens = divmod(num_tokens, num_reqs)
query_lens = torch.full(
(num_reqs,),
tokens_per_req,
dtype=query_start_loc.dtype,
device=query_start_loc.device,
)
if num_extra_tokens > 0:
query_lens[-num_extra_tokens:] += 1
torch.cumsum(query_lens, dim=0, out=query_start_loc[1:])
ngram_context = self.ngram_context[:num_reqs]
ngram_context.fill_(self.ngram_eos_token_id)
model_inputs.update(
query_start_loc=query_start_loc,
ngram_context=ngram_context,
)
return model_inputs
__all__ = ["Qwen4ExpModelState"]
+462
View File
@@ -0,0 +1,462 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inference-only Qwen4Exp MTP (Multi-Token Predictor) model.
The MTP draft model reuses the Qwen4Exp backbone (PLE/HC/MoE) but:
- drops all multi-modal handling (text-only),
- forces PLE off while keeping the main model's HC stream count,
- fuses the backbone hidden and the new-token embedding via
``residual_linear_shared`` (fc_embedding + shared fc_hidden) instead of
the ``Linear(2H, H)`` + repeat used by other MTP variants,
- emits TWO hidden streams per step (scheme A): a single stream [T, H]
(final-mixer collapsed, fed to the LM head) and a pre-final-mixer
multi stream [T, hc_count*H] (fed to the next draft step).
"""
from collections.abc import Iterable
import regex as re
import torch
from torch import nn
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig, replace, set_current_vllm_config
from vllm.distributed import get_pp_group
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.utils import configure_quant_config
from vllm.model_executor.models.interfaces import SupportsPP
from vllm.model_executor.models.qwen3_5 import Qwen3_5Model
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
PPMissingLayer,
WeightsMapper,
get_draft_quant_config,
make_empty_intermediate_tensors_factory,
maybe_fuse_shared_experts,
maybe_prefix,
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.qwen4_exp import (
Qwen4ExpTextConfig,
)
from .hyperconnection import GatedResidual, HyperConnectionConfig
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
from .model import (
_HC_WEIGHTS_MAPPER,
_QWEN4_EXP_IGNORED_MISSING_SUFFIXES,
Qwen4ExpDecoderLayer,
Qwen4ExpMixtureOfExperts,
Qwen4ExpSparseMoeBlock,
)
def _remap_ignored_layers(
ignored_layers: list[str],
mtp_start_layer_idx: int,
) -> list[str]:
remapped: list[str] = []
for name in ignored_layers:
if name.startswith("mtp."):
new_name = re.sub(
r"(?<=\.layers\.)\d+",
lambda m: str(mtp_start_layer_idx + int(m.group(0))),
name,
)
remapped.append(new_name)
else:
remapped.append(name)
return remapped
def _remap_mtp_weight_name(name: str) -> str | None:
"""Map Qwen4Exp checkpoint paths into the standalone draft model."""
for checkpoint_prefix in (
"model.language_model.",
"language_model.",
):
if name.startswith(checkpoint_prefix):
name = name.removeprefix(checkpoint_prefix)
break
if name.startswith("embed_tokens."):
name = f"model.{name}"
if name.startswith("model.mtp."):
name = name.removeprefix("model.")
if name.startswith("mtp.shared_head.head."):
return name.replace("mtp.shared_head.head.", "lm_head.", 1)
if name.startswith("model.shared_head.head."):
return name.replace("model.shared_head.head.", "lm_head.", 1)
if name.startswith("shared_head.head."):
return name.replace("shared_head.head.", "lm_head.", 1)
if name.startswith("model.lm_head."):
return name.removeprefix("model.")
if name.startswith("mtp."):
return name.replace("mtp.", "model.", 1)
if name.startswith("model.embed_tokens.") or name.startswith("lm_head."):
return name
return None
def _make_draft_vllm_config(
vllm_config: VllmConfig,
mtp_start_layer_idx: int,
) -> VllmConfig:
"""Ensure that the draft model config is set in the vLLM config."""
speculative_config = vllm_config.speculative_config
if speculative_config is None or speculative_config.draft_model_config is None:
raise ValueError("speculative_config.draft_model_config must be set")
draft_quant_config = get_draft_quant_config(vllm_config)
# inject packed and ignored modules to the quantization config of draft model
if draft_quant_config is not None:
configure_quant_config(draft_quant_config, Qwen4ExpMTP)
ignored_layers = getattr(draft_quant_config, "ignored_layers", None)
if ignored_layers:
setattr( # noqa: B010
draft_quant_config,
"ignored_layers",
_remap_ignored_layers(ignored_layers, mtp_start_layer_idx),
)
exclude_modules = getattr(draft_quant_config, "exclude_modules", None)
if exclude_modules:
setattr( # noqa: B010
draft_quant_config,
"exclude_modules",
_remap_ignored_layers(exclude_modules, mtp_start_layer_idx),
)
draft_vllm_config = replace(
vllm_config,
model_config=speculative_config.draft_model_config,
)
# VllmConfig post-init derives the target quant config, so restore the
# independently resolved draft quant config after replacement.
draft_vllm_config.quant_config = draft_quant_config
return draft_vllm_config
@support_torch_compile(
dynamic_arg_dims={
"input_ids": 0,
"positions": -1,
"intermediate_tensors": 0,
"inputs_embeds": 0,
"hidden_states": 0,
}
)
class Qwen4ExpMultiTokenPredictor(nn.Module):
hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _HC_WEIGHTS_MAPPER
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
model_config = vllm_config.model_config
config: Qwen4ExpTextConfig = model_config.hf_text_config
self.config = config
self.vocab_size = config.vocab_size
self.mtp_start_layer_idx = config.num_hidden_layers
self.num_mtp_layers = getattr(config, "mtp_num_hidden_layers", 1)
self.hidden_size = config.hidden_size
self.hc_count = config.hc_count
self.embed_tokens = VocabParallelEmbedding(self.vocab_size, self.hidden_size)
draft_vllm_config = _make_draft_vllm_config(
vllm_config,
self.mtp_start_layer_idx,
)
with set_current_vllm_config(draft_vllm_config, prefix=prefix):
# residual_linear_shared fusion: fc_embedding projects the token
# embedding, fc_hidden (shared across HC branches) projects the
# backbone hidden; the embedding is added as a residual to every
# branch (see mtp_residual_linear_shared.md).
self.fc_embedding = ColumnParallelLinear(
self.hidden_size,
self.hidden_size,
gather_output=True,
bias=False,
return_bias=False,
quant_config=draft_vllm_config.quant_config,
prefix=f"{prefix}.fc_embedding",
)
self.fc_hidden = ColumnParallelLinear(
self.hidden_size,
self.hidden_size,
gather_output=True,
bias=False,
return_bias=False,
quant_config=draft_vllm_config.quant_config,
prefix=f"{prefix}.fc_hidden",
)
self.layers = nn.ModuleList(
Qwen4ExpDecoderLayer(
draft_vllm_config,
layer_type="full_attention",
prefix=f"{prefix}.layers.{self.mtp_start_layer_idx + idx}",
)
for idx in range(self.num_mtp_layers)
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
Qwen4ExpSparseMoeBlock,
"mlp",
)
self.pre_fc_norm_embedding = GemmaRMSNorm(
self.hidden_size, eps=config.rms_norm_eps
)
self.pre_fc_norm_hidden = GemmaRMSNorm(
self.hidden_size * self.hc_count, eps=config.rms_norm_eps
)
# HC final mixer collapses the multi stream into [T, H] for the LM head.
hc_config = HyperConnectionConfig(
hc_count=config.hc_count,
hidden_size=config.hidden_size,
params_dtype=torch.bfloat16,
hc_lowrank=config.hc_lowrank,
rms_norm_eps=config.rms_norm_eps,
hc_per_branch_norm=True,
)
self.hyper_connection_mixer = GatedResidual(
hc_config,
use_combine=False,
prefix=maybe_prefix(prefix, "hyper_connection_mixer"),
)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states"], self.hidden_size * self.hc_count
)
def _iter_qsa_attentions(self):
"""Yield MTP attention modules that own a QSA indexer."""
for layer in self.layers:
attention = getattr(layer, "self_attn", None)
if (
attention is not None
and getattr(attention, "indexer", None) is not None
):
yield attention
def set_skip_topk(self, skip: bool) -> None:
"""Select on MTP step 0 and reuse its QSA indices on later steps."""
for attention in self._iter_qsa_attentions():
attention.indexer.skip_topk = skip
def compact_topk_indices(self, row_indices: torch.Tensor) -> None:
"""Keep each request's target-aligned step-0 sparse-index row."""
num_rows = row_indices.numel()
for attention in self._iter_qsa_attentions():
buffer = attention.topk_indices_buffer
selected = buffer.index_select(0, row_indices)
buffer[:num_rows].copy_(selected)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
hidden_states: torch.Tensor | None = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor] | IntermediateTensors:
hc_count = self.hc_count
hidden_size = self.hidden_size
if get_pp_group().is_first_rank:
assert hidden_states is not None
if inputs_embeds is None:
assert input_ids is not None
inputs_embeds = self.embed_input_ids(input_ids)
# Embedding branch: pre-norm -> fc_embedding -> [T, H].
inputs_embeds = self.pre_fc_norm_embedding(inputs_embeds)
inputs_embeds = self.fc_embedding(inputs_embeds)
# Backbone hidden is multi-stream [T, hc_count*H] (scheme A:
# the main model truly emits the pre-final-mixer multi stream
# on the first step; subsequent steps reuse the prior draft
# step's multi stream).
num_tokens = hidden_states.shape[0]
hidden_states = hidden_states.view(num_tokens, hc_count, hidden_size)
hidden_states = self.pre_fc_norm_hidden(hidden_states.flatten(-2)).view(
num_tokens, hc_count, hidden_size
)
hidden_states = self.fc_hidden(hidden_states)
# Add the embedding residual to every branch, then fold back
# to [T, hc_count*H] (HC outer, HS inner) for the HC decoder.
hidden_states = inputs_embeds.unsqueeze(-2) + hidden_states
hidden_states = hidden_states.flatten(-2)
else:
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"]
current_step_idx = spec_step_idx % self.num_mtp_layers
layer = self.layers[current_step_idx]
hidden_states, block_output, injection = layer(
hidden_states=hidden_states,
prev_block_output=None,
prev_injection=None,
positions=positions,
input_ids=None,
query_start_loc=None,
ngram_context=None,
)
if not get_pp_group().is_last_rank:
# As in the target model, PP carries a materialized tensor rather
# than the delayed hidden/output/injection tuple.
hidden_states = layer.mlp_hyper_connection.combine(
hidden_states, block_output, injection
)
return IntermediateTensors({"hidden_states": hidden_states})
# Last PP rank finalize. Keep both:
# (A) sample_hidden_states [T, H] -> single stream for the LM head
# (B) multi_hidden [T, hc_count*H] -> pre-final-mixer multi stream
# for the next draft step (zero extra compute, just kept).
multi_hidden, sample_hidden_states, _ = (
self.hyper_connection_mixer.combine_and_mix(
hidden_states, block_output, injection
)
)
return sample_hidden_states, multi_hidden
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
weights = maybe_fuse_shared_experts(
weights,
enabled=self.is_fused_shared_expert_enabled,
n_routed_experts=getattr(self.config, "num_experts", 0) or 0,
n_shared_experts=1,
ckpt_prefix="mlp.shared_expert",
)
mapper = self.hf_to_vllm_mapper | WeightsMapper(
orig_to_new_substr={"hyper_connection_mixer.block_inject_weight": None}
)
loader = AutoWeightsLoader(
self,
ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(),
)
return loader.load_weights(weights, mapper=mapper)
@support_torch_compile(
dynamic_arg_dims={
"input_ids": 0,
"positions": -1,
"intermediate_tensors": 0,
"inputs_embeds": 0,
"hidden_states": 0,
}
)
class Qwen4ExpMTP(nn.Module, SupportsPP, Qwen4ExpMixtureOfExperts):
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
"gate_up_proj": ["gate_proj", "up_proj"],
"in_proj_qkvz": ["in_proj_qkv", "in_proj_z"],
"in_proj_ba": ["in_proj_b", "in_proj_a"],
"input_mix_weight_down_block_inject": [
"input_mix_weight_down",
"block_inject_weight",
"_input_mix_padding",
],
}
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config
self.vllm_config = vllm_config
cache_config = vllm_config.cache_config
if cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Qwen4ExpMTP currently does not support 'all' prefix caching, "
"please use '--mamba-cache-mode=align' instead"
)
self.quant_config = vllm_config.quant_config
super().__init__()
self.config = config
self.model = Qwen4ExpMultiTokenPredictor(
vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "mtp"),
)
if get_pp_group().is_last_rank:
if config.tie_word_embeddings:
self.lm_head = self.model.embed_tokens
else:
self.lm_head = ParallelLMHead(
config.vocab_size,
config.hidden_size,
prefix=maybe_prefix(prefix, "lm_head"),
)
else:
self.lm_head = PPMissingLayer()
self.logits_processor = LogitsProcessor(config.vocab_size)
self.make_empty_intermediate_tensors = (
self.model.make_empty_intermediate_tensors
)
self.set_moe_parameters(self.model.layers)
enable_qwen4_exp_low_latency_gemm(self, vllm_config.model_config.dtype)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
hidden_states: torch.Tensor | None = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor] | IntermediateTensors:
return self.model(
input_ids,
positions,
hidden_states,
intermediate_tensors,
inputs_embeds,
spec_step_idx=spec_step_idx,
)
def compute_logits(
self, hidden_states: torch.Tensor, spec_step_idx: int = 0
) -> torch.Tensor | None:
return self.logits_processor(self.lm_head, hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
def remap_weight_names():
for name, weight in weights:
remapped_name = _remap_mtp_weight_name(name)
if remapped_name is not None:
yield remapped_name, weight
mapper = WeightsMapper(
orig_to_new_substr={"hyper_connection_mixer.block_inject_weight": None}
)
loader = AutoWeightsLoader(
self,
ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(),
)
return loader.load_weights(remap_weight_names(), mapper=mapper)
__all__ = ["Qwen4ExpMTP", "Qwen4ExpMultiTokenPredictor"]

Some files were not shown because too many files have changed in this diff Show More