[Model] Support Qwen3.8-Flash-Next (#53896)

Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com>
Signed-off-by: andyluo7 <andy.luo@amd.com>
Signed-off-by: lishunyang12 <lishunyang03@gmail.com>
Signed-off-by: Harry Huang <huanghaoyan.hhy@alibaba-inc.com>
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: KungYork <firelikest@163.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: andyluo7 <43718156+andyluo7@users.noreply.github.com>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: andyluo7 <andy.luo@amd.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: lishunyang12 <lishunyang03@gmail.com>
Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Harry Huang
2026-08-31 05:57:56 +00:00
committed by GitHub
co-authored by zjy0516 KungYork Thien Tran Yifan Qiao andyluo7 Isotr0py andyluo7 OpenAI Codex lishunyang12 Vadim Gimpelson Isotr0py Lucas Wilkinson Claude Opus 5 OpenAI Codex
parent 687db59744
commit e126687a9a
124 changed files with 19424 additions and 450 deletions
+19
View File
@@ -7,6 +7,7 @@ from enum import Enum
import pytest
from vllm.config.cache import CacheConfig
from vllm.config.scheduler import SchedulerConfig
from vllm.config.utils import get_hash_factors, hash_factors, normalize_value
# Helpers
@@ -216,6 +217,24 @@ def test_cache_config_hash_ignores_kv_cache_sizing_knobs():
assert CacheConfig(gpu_memory_utilization=0.5).compute_hash() == base_hash
def test_scheduler_config_hash_includes_max_num_seqs():
"""Per-request workspace sizes must invalidate compiled graphs."""
base_hash = SchedulerConfig(
max_model_len=8192,
is_encoder_decoder=False,
max_num_batched_tokens=8192,
max_num_seqs=128,
).compute_hash()
larger_batch_hash = SchedulerConfig(
max_model_len=8192,
is_encoder_decoder=False,
max_num_batched_tokens=8192,
max_num_seqs=1024,
).compute_hash()
assert larger_batch_hash != base_hash
def test_cache_config_hash_ignores_prefix_cache_retention_interval():
base_hash = CacheConfig().compute_hash()
assert CacheConfig(prefix_cache_retention_interval=64).compute_hash() == base_hash
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for SpeculativeConfig.compose_draft_hf_overrides.
"""Tests for draft config overrides used by SpeculativeConfig.
Callable ``hf_overrides`` on the target model config (e.g. the
``dummy_hf_overrides`` shrink used by ``tests/models/test_initialization.py``)
@@ -12,10 +12,12 @@ when the target itself is shrunk — which is what kept spec-decode archs like
"""
import functools
from unittest.mock import MagicMock, patch
import pytest
from transformers import PretrainedConfig
from vllm.config.parallel import ParallelConfig
from vllm.config.speculative import SpeculativeConfig
@@ -134,3 +136,52 @@ def test_composed_override_is_picklable():
out = composed(_make_hf_config())
assert out.num_hidden_layers == 1
def _make_mtp_speculative_config(
override: bool | None,
checkpoint_value: bool,
) -> SpeculativeConfig:
draft_hf_config = _make_hf_config(
architectures=["Qwen4ExpMTP"],
model_type="qwen4_exp_mtp",
n_predict=1,
index_share_for_mtp_iteration=checkpoint_value,
)
draft_model_config = MagicMock(
model="draft",
hf_config=draft_hf_config,
architectures=draft_hf_config.architectures,
max_model_len=128,
)
target_model_config = MagicMock(
model="target",
max_model_len=128,
quantization=None,
hf_overrides={},
)
with patch("vllm.config.speculative.ModelConfig", return_value=draft_model_config):
return SpeculativeConfig(
model="draft",
method="mtp",
num_speculative_tokens=1,
index_share_for_mtp_iteration=override,
target_model_config=target_model_config,
target_parallel_config=ParallelConfig(),
)
@pytest.mark.cpu_test
@pytest.mark.parametrize(
("override", "checkpoint_value", "expected"),
[(None, True, True), (False, True, False), (True, False, True)],
)
def test_mtp_index_share_override(
override: bool | None, checkpoint_value: bool, expected: bool
):
speculative_config = _make_mtp_speculative_config(override, checkpoint_value)
assert (
speculative_config.draft_model_config.hf_config.index_share_for_mtp_iteration
is expected
)
@@ -24,6 +24,29 @@ for i, v in enumerate(test_sizes):
test_sizes[i] -= v % 8
@pytest.mark.parametrize(
("dtype", "expected"),
[
(torch.float32, True),
(torch.float16, True),
(torch.bfloat16, True),
(torch.int8, False),
(torch.float8_e4m3fn, False),
],
)
def test_custom_allreduce_filters_dtype(
dtype: torch.dtype,
expected: bool,
) -> None:
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
communicator.disabled = False
communicator.world_size = 2
communicator.max_size = 1024
communicator._ptr = 0
assert communicator.should_custom_ar(torch.empty(16, dtype=dtype)) is expected
@pytest.mark.parametrize(
("major", "local_multicast", "expected"),
[
+14
View File
@@ -0,0 +1,14 @@
# Qwen4Exp accuracy evaluation
This suite starts a Qwen3.8-Flash-Next-FP8 OpenAI-compatible server once and
uses EvalScope to evaluate GSM8K and AIME25.
```bash
# B200
pytest -s -v tests/evals/qwen4_exp/test_accuracy.py \
--config-list-file=configs/models-b200.txt
# H200
pytest -s -v tests/evals/qwen4_exp/test_accuracy.py \
--config-list-file=configs/models-h200.txt
```
+3
View File
@@ -0,0 +1,3 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,26 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
model_name: "Qwen/Qwen3.8-Flash-Next-FP8"
datasets:
gsm8k:
metric_threshold: 0.98
tolerance: 0.02
aime25:
metric_threshold: 0.90
tolerance: 0.05
eval_batch_size: 32
startup_max_wait_seconds: 1800
generation_config:
seed: 1236
do_sample: true
temperature: 0.7
top_p: 0.8
top_k: 20
repetition_penalty: 1.0
presence_penalty: 1.5
max_tokens: 32768
stream: true
extra_body:
chat_template_kwargs:
enable_thinking: true
server_args: "--tensor-parallel-size 4"
@@ -0,0 +1,2 @@
# B200 configuration for Qwen3.8-Flash-Next-FP8 accuracy evaluation
Qwen3.8-Flash-Next-FP8.yaml
@@ -0,0 +1,2 @@
# H200 configuration for Qwen3.8-Flash-Next-FP8 accuracy evaluation
Qwen3.8-Flash-Next-FP8.yaml
+42
View File
@@ -0,0 +1,42 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from pathlib import Path
def pytest_addoption(parser):
parser.addoption(
"--config-list-file",
required=True,
help="File containing Qwen4Exp evaluation config files",
)
def pytest_generate_tests(metafunc):
if "config_filename" not in metafunc.fixturenames:
return
config_list_path = Path(metafunc.config.getoption("--config-list-file"))
if not config_list_path.is_absolute():
test_relative_path = Path(__file__).parent / config_list_path
config_list_path = (
test_relative_path
if test_relative_path.exists()
else Path.cwd() / config_list_path
)
config_dir = config_list_path.parent
config_files = [
config_dir / line.strip()
for line in config_list_path.read_text(encoding="utf-8").splitlines()
if line.strip() and not line.lstrip().startswith("#")
]
missing = [path for path in config_files if not path.exists()]
if missing:
raise FileNotFoundError(f"Missing evaluation configs: {missing}")
metafunc.parametrize(
"config_filename",
config_files,
ids=[config_file.stem for config_file in config_files],
)
+68
View File
@@ -0,0 +1,68 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""GSM8K and AIME25 accuracy evaluation for Qwen3.8-Flash-Next-FP8."""
import shlex
from pathlib import Path
from typing import Any
import yaml
from tests.utils import RemoteOpenAIServer
def run_evalscope(
eval_config: dict[str, Any], base_url: str, work_dir: Path
) -> dict[str, float]:
from evalscope.run import run_task
reports = run_task(
{
"model": eval_config["model_name"],
"api_url": base_url,
"api_key": "EMPTY_TOKEN",
"datasets": list(eval_config["datasets"]),
"eval_batch_size": eval_config.get("eval_batch_size", 32),
"generation_config": eval_config["generation_config"],
"work_dir": str(work_dir),
"no_timestamp": True,
}
)
if not isinstance(reports, dict):
raise TypeError(f"Expected EvalScope reports to be a dict, got {type(reports)}")
return {dataset: float(report.score) for dataset, report in reports.items()}
def test_qwen4_exp_accuracy(config_filename: Path, tmp_path: Path):
eval_config = yaml.safe_load(config_filename.read_text(encoding="utf-8"))
server_args = shlex.split(eval_config.get("server_args", ""))
server_args.extend(["--trust-remote-code", "--disable-uvicorn-access-log"])
model_name = eval_config["model_name"]
print(f"Starting Qwen4Exp evaluation for model: {model_name}")
print(f"Datasets: {', '.join(eval_config['datasets'])}")
print(f"Server args: {' '.join(server_args)}")
with RemoteOpenAIServer(
model_name,
server_args,
env_dict=eval_config.get("env"),
max_wait_seconds=eval_config.get("startup_max_wait_seconds", 1800),
) as remote_server:
scores = run_evalscope(eval_config, remote_server.url_for("v1"), tmp_path)
for dataset, metric_config in eval_config["datasets"].items():
score = scores[dataset]
threshold = metric_config["metric_threshold"]
tolerance = metric_config.get("tolerance", 0.0)
minimum_score = threshold - tolerance
print(
f"{dataset}: measured={score:.4f}, expected={threshold:.4f}, "
f"tolerance={tolerance:.4f}"
)
assert score >= minimum_score, (
f"{dataset} score too low: {score:.4f} < {threshold:.4f} - "
f"{tolerance:.4f} = {minimum_score:.4f}"
)
+8 -3
View File
@@ -57,13 +57,13 @@ EPS = 1e-6
class _TestGatedNorm:
def __init__(self, weight: torch.Tensor) -> None:
def __init__(self, weight: torch.Tensor, activation: str) -> None:
self.weight = weight
self.bias = None
self.eps = EPS
self.group_size = None
self.norm_before_gate = True
self.activation = "silu"
self.activation = activation
def __call__(self, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor:
return rmsnorm_fn(
@@ -99,6 +99,7 @@ def _build_layer(
dt_bias: torch.Tensor,
conv_weight: torch.Tensor,
norm_weight: torch.Tensor,
output_gate_activation: str,
):
layer = types.SimpleNamespace(
prefix=PREFIX,
@@ -116,7 +117,7 @@ def _build_layer(
dt_bias=dt_bias,
conv1d=types.SimpleNamespace(weight=conv_weight, bias=None),
kv_cache=(conv_state, ssm_state),
norm=_TestGatedNorm(norm_weight),
norm=_TestGatedNorm(norm_weight, output_gate_activation),
layer_norm_epsilon=EPS,
gdn_decode_kernel="cuda",
)
@@ -250,12 +251,14 @@ def test_fused_forward_uses_packed_entrypoint() -> None:
pytest.param([128], [1], [-1], 0, id="pure-decode"),
],
)
@pytest.mark.parametrize("output_gate_activation", ["silu", "sigmoid"])
@torch.inference_mode()
def test_fused_model_path_matches_reference(
seq_lens: list[int],
query_lens: list[int],
draft_tokens: list[int],
expected_fused_calls: int,
output_gate_activation: str,
) -> None:
"""Fused MTP and its mixed/prefill/decode fallbacks match the reference."""
torch.manual_seed(1)
@@ -334,6 +337,7 @@ def test_fused_model_path_matches_reference(
dt_bias,
conv_weight,
norm_weight,
output_gate_activation,
)
reference_out = torch.zeros_like(output_gate)
with patch.object(
@@ -355,6 +359,7 @@ def test_fused_model_path_matches_reference(
dt_bias,
conv_weight,
norm_weight,
output_gate_activation,
)
context.no_compile_layers = {PREFIX: fused_layer}
fused_out = torch.zeros_like(output_gate)
@@ -366,7 +366,7 @@ def test_preprocess_fused_align_matches_scalar_bookkeeping(monkeypatch, token_bi
input_batch=scalar_case[1],
requests=scalar_case[2],
forward_context={},
mamba_state_copy_funcs=(),
mamba_state_copy_funcs={},
copy_bufs=scalar_copy_bufs,
)
@@ -384,7 +384,7 @@ def test_preprocess_fused_align_matches_scalar_bookkeeping(monkeypatch, token_bi
input_batch=fused_case[1],
requests=fused_case[2],
forward_context={},
mamba_state_copy_funcs=(),
mamba_state_copy_funcs={},
copy_bufs=fused_copy_bufs,
align_ctx=ctx,
)
+104
View File
@@ -351,3 +351,107 @@ def test_w8a8_block_fp8_deep_gemm_fused_moe(M, N, K, E, topk, seed, monkeypatch)
torch.accelerator.synchronize()
torch.testing.assert_close(out, ref_out, atol=0.035, rtol=0.035)
def _dequant_block_fp8(
w_fp8: torch.Tensor, w_s: torch.Tensor, block_shape: list[int]
) -> torch.Tensor:
"""Dequantize a [N, K] fp8 weight with [Nb, Kb] block scales to fp32."""
n, k = w_fp8.shape
bn, bk = block_shape
s = w_s.repeat_interleave(bn, dim=0).repeat_interleave(bk, dim=1)
return w_fp8.to(torch.float32) * s[:n, :k]
@pytest.mark.parametrize("tp_rank", [0, 1, 2, 3])
@torch.inference_mode()
def test_w8a8_block_fp8_fused_moe_refined_block_scales(tp_rank, workspace_init):
"""TP-misaligned blockwise FP8 MoE (e.g. Qwen4Exp, intermediate
640 at TP=4 -> 160 per rank, not divisible by the checkpoint's 128 block).
The per-rank scale grid is refined from 128 to 32 (lossless: each 32-block
lies inside one global 128-block) so the Triton kernel can consume exact
per-shard scales. Simulates one TP rank's shard and checks:
1. the refined+sharded scales dequantize exactly like the global scales;
2. the Triton fused MoE kernel with block_shape=[32, 32] matches the
native-torch blockwise reference on the same shard.
"""
torch.manual_seed(0)
dtype = torch.bfloat16
M, E, topk = 8, 4, 2
hidden = 256
inter_full = 640
tp_size = 4
n_shard = inter_full // tp_size # 160
ckpt_block = [128, 128]
refined_block = [32, 32]
factor = ckpt_block[0] // refined_block[0] # 4
a = torch.randn((M, hidden), dtype=dtype) / 10
score = torch.randn((M, E), dtype=dtype)
(_, w1_full, w1_s_full, _), (_, w2_full, w2_s_full, _) = make_test_weights(
E,
inter_full,
hidden,
dtype,
torch.float8_e4m3fn,
per_out_ch_quant=False,
block_shape=ckpt_block,
)
assert w1_s_full is not None
assert w2_s_full is not None
# TP shard the weights (exact slicing, no scale involvement).
lo, hi = tp_rank * n_shard, (tp_rank + 1) * n_shard
w1 = torch.cat(
[w1_full[:, lo:hi], w1_full[:, inter_full + lo : inter_full + hi]], dim=1
)
w2 = w2_full[:, :, lo:hi]
# Refine the global 128-block scales to 32 blocks, then take the shard's
# slice -- mirrors the Fp8MoEMethod/RoutedExperts loading path.
w1_s32 = w1_s_full.repeat_interleave(factor, dim=-2).repeat_interleave(
factor, dim=-1
)
nb = n_shard // refined_block[0] # 5 local 32-blocks per projection
gate_hi = w1_s32.shape[1] // 2
w1_s = torch.cat(
[
w1_s32[:, tp_rank * nb : (tp_rank + 1) * nb],
w1_s32[:, gate_hi + tp_rank * nb : gate_hi + (tp_rank + 1) * nb],
],
dim=1,
)
w2_s32 = w2_s_full.repeat_interleave(factor, dim=-2).repeat_interleave(
factor, dim=-1
)
w2_s = w2_s32[:, :, tp_rank * nb : (tp_rank + 1) * nb]
# The refined per-shard scales must reproduce the global-scale dequant
# exactly (dequant full-width, then slice to the shard).
for e in range(E):
d32 = _dequant_block_fp8(w1[e, :n_shard], w1_s[e, :nb], refined_block)
d128 = _dequant_block_fp8(w1_full[e], w1_s_full[e], ckpt_block)[lo:hi]
assert torch.equal(d32, d128)
d32 = _dequant_block_fp8(w2[e], w2_s[e], refined_block)
d128 = _dequant_block_fp8(w2_full[e], w2_s_full[e], ckpt_block)[:, lo:hi]
assert torch.equal(d32, d128)
quant_config = fp8_w8a8_moe_quant_config(
w1_scale=w1_s,
w2_scale=w2_s,
block_shape=refined_block,
)
topk_weights, topk_ids, _ = fused_topk(a, score.float(), topk, False)
with set_current_vllm_config(vllm_config):
ref_out = torch_w8a8_block_fp8_moe(
a, w1, w2, w1_s, w2_s, topk_weights, topk_ids, refined_block
)
out = fused_experts(
a, w1, w2, topk_weights, topk_ids, quant_config=quant_config
)
torch.testing.assert_close(out, ref_out, atol=0.035, rtol=0.035)
+10 -6
View File
@@ -28,8 +28,9 @@ from vllm.model_executor.layers.fused_moe.config import (
from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import (
TrtLlmBf16ExpertsModular,
)
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
convert_moe_weights_to_flashinfer_trtllm_block_layout,
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
UnquantizedMoeBackend,
convert_to_unquantized_kernel_format,
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
@@ -46,9 +47,8 @@ if pytest and (
)
# (m, n, k) = (tokens, intermediate_size_per_partition, hidden_dim).
# Covers larger NvFP4-like shapes while keeping BF16's FlashInfer TRTLLM
# intermediate-size multiple-of-128 requirement.
MNK_FACTORS = [
(2, 160, 2560),
(2, 1024, 1024),
(64, 2048, 1536),
(64, 1024, 4096),
@@ -97,11 +97,15 @@ def test_trtllm_bf16_moe_modular_no_graph(
max_num_tokens=next_power_of_2(m),
)
trtllm_w1, trtllm_w2 = convert_moe_weights_to_flashinfer_trtllm_block_layout(
{},
trtllm_w1, trtllm_w2 = convert_to_unquantized_kernel_format(
UnquantizedMoeBackend.FLASHINFER_TRTLLM,
moe_config,
w1,
w2,
)
expected_n = (n + 127) // 128 * 128
assert moe_config.intermediate_size_per_partition == expected_n
assert trtllm_w2.numel() == e * k * expected_n
trtllm_experts = mk.FusedMoEKernel(
maybe_make_prepare_finalize(
+4 -1
View File
@@ -267,6 +267,7 @@ def test_fused_post_conv_l0():
),
],
)
@pytest.mark.parametrize("output_gate_activation", ["silu", "sigmoid"])
@torch.inference_mode()
def test_fused_gdn_decode_post_conv_mtp_head_ratios(
head_ratio: int,
@@ -274,6 +275,7 @@ def test_fused_gdn_decode_post_conv_mtp_head_ratios(
query_lengths: tuple[int, ...],
state_dtype: torch.dtype,
norm_dtype: torch.dtype,
output_gate_activation: str,
) -> None:
if torch.cuda.get_device_capability() < (8, 0):
pytest.skip("fused GDN decode MTP requires compute capability 8.0+")
@@ -357,7 +359,7 @@ def test_fused_gdn_decode_post_conv_mtp_head_ratios(
z=output_gate,
eps=eps,
norm_before_gate=True,
activation="silu",
activation=output_gate_activation,
)
actual = ops.fused_gdn_decode_post_conv_mtp(
mixed_qkv=mixed_qkv,
@@ -374,6 +376,7 @@ def test_fused_gdn_decode_post_conv_mtp_head_ratios(
out=torch.empty_like(output_gate),
scale=scale,
norm_eps=eps,
output_gate_activation=output_gate_activation,
)
output_error = (actual.float() - expected.float()).norm()
@@ -18,8 +18,12 @@ from vllm.model_executor.layers.fused_moe.layer import determine_expert_counts
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
from vllm.model_executor.layers.quantization.utils.config_utils import (
get_quark_ocp_mx_group_size,
is_shared_expert_quant_fse_compatible,
)
from vllm.model_executor.models.qwen3_next import (
_should_replicate_misaligned_shared_expert,
)
from vllm.model_executor.models.utils import PPMissingLayer
from vllm.models.deepseek_v4 import quant_config as deepseek_v4_quant_config
from vllm.models.minimax_m3.amd import model as minimax_m3_model
@@ -605,6 +609,82 @@ def test_quark_shared_expert_fse_compatibility(
)
def test_quark_ocp_mx_group_size_for_layer() -> None:
quant_config = QuarkConfig({**_QUARK_FSE_CONFIG, "exclude": []})
assert (
get_quark_ocp_mx_group_size(
quant_config,
"model.layers.0.mlp.shared_expert.down_proj",
)
== 32
)
quant_config.quant_config["exclude"] = ["model.layers.0.mlp.shared_expert_gate"]
assert (
get_quark_ocp_mx_group_size(
quant_config,
"model.layers.0.mlp.shared_expert.down_proj",
)
== 32
)
quant_config.quant_config["exclude"] = [
"model.layers.0.mlp.shared_expert.down_proj"
]
assert (
get_quark_ocp_mx_group_size(
quant_config,
"model.layers.0.mlp.shared_expert.down_proj",
)
is None
)
@pytest.mark.parametrize("tp_size", [1, 2, 4])
def test_quark_ocp_mx_shared_expert_uses_supported_tp_shards(tp_size: int) -> None:
assert not _should_replicate_misaligned_shared_expert(
intermediate_size=640,
tp_size=tp_size,
group_size=32,
enable_expert_parallel=False,
is_sequence_parallel=False,
)
def test_quark_ocp_mx_shared_expert_rejects_plain_tp8() -> None:
with pytest.raises(
ValueError,
match="TP size 8 produces a partition of 80",
):
_should_replicate_misaligned_shared_expert(
intermediate_size=640,
tp_size=8,
group_size=32,
enable_expert_parallel=False,
is_sequence_parallel=False,
)
def test_quark_ocp_mx_shared_expert_is_replicated_for_tep8() -> None:
assert _should_replicate_misaligned_shared_expert(
intermediate_size=640,
tp_size=8,
group_size=32,
enable_expert_parallel=True,
is_sequence_parallel=False,
)
def test_quark_ocp_mx_shared_expert_is_replicated_when_not_tp_divisible() -> None:
assert _should_replicate_misaligned_shared_expert(
intermediate_size=672,
tp_size=8,
group_size=32,
enable_expert_parallel=True,
is_sequence_parallel=False,
)
def test_quark_shared_expert_fse_requires_matching_layer_quant_configs() -> None:
global_quant_config = {"weight": {"dtype": "fp4"}}
quant_config = QuarkConfig(
+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+200
View File
@@ -0,0 +1,200 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from unittest.mock import patch
import pytest
import torch
from vllm.config.speculative import SpeculativeConfig
from vllm.model_executor.models.config import (
Qwen3_5ForConditionalGenerationConfig,
Qwen4ExpForConditionalGenerationConfig,
)
from vllm.models.qwen4_exp.config import (
Qwen4ExpConfig,
Qwen4ExpTextConfig,
)
from vllm.models.qwen4_exp.nvidia.model_state import Qwen4ExpModelState
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
def _text_config(**kwargs) -> Qwen4ExpTextConfig:
values = {
"vocab_size": 64,
"hidden_size": 16,
"intermediate_size": 32,
"num_hidden_layers": 2,
"num_attention_heads": 2,
"num_key_value_heads": 1,
"head_dim": 8,
"layer_types": ["linear_attention", "full_attention"],
"linear_num_key_heads": 2,
"linear_num_value_heads": 2,
"linear_key_head_dim": 8,
"linear_value_head_dim": 8,
"num_experts": 0,
"hc_count": 2,
"hc_lowrank": 4,
"ple_layer_ids": [1],
"mtp_num_hidden_layers": 1,
"mtp": {"hybrid": True},
}
values.update(kwargs)
return Qwen4ExpTextConfig(**values)
def test_qwen4_exp_mtp_returns_sample_and_multi_streams() -> None:
from vllm.models.qwen4_exp.nvidia.mtp import (
Qwen4ExpMultiTokenPredictor,
)
model = object.__new__(Qwen4ExpMultiTokenPredictor)
torch.nn.Module.__init__(model)
model.hc_count = 2
model.hidden_size = 4
model.num_mtp_layers = 1
model.layers = [
lambda **kwargs: (
kwargs["hidden_states"],
kwargs["hidden_states"],
torch.zeros(kwargs["hidden_states"].shape[0], 2),
),
]
model.hyper_connection_mixer = SimpleNamespace(
combine_and_mix=lambda hidden_states, block_output, injection: (
hidden_states,
hidden_states.unflatten(-1, (2, 4)).mean(-2),
None,
),
)
multi_hidden = torch.arange(16, dtype=torch.float32).reshape(2, 8)
pp_group = SimpleNamespace(is_first_rank=False, is_last_rank=True)
with patch(
"vllm.models.qwen4_exp.nvidia.mtp.get_pp_group",
return_value=pp_group,
):
sample_hidden, returned_multi_hidden = model.forward(
input_ids=None,
positions=torch.arange(2),
intermediate_tensors={"hidden_states": multi_hidden},
)
torch.testing.assert_close(
sample_hidden,
multi_hidden.unflatten(-1, (2, 4)).mean(dim=-2),
)
assert returned_multi_hidden is multi_hidden
@pytest.mark.parametrize("wrapped_config", [False, True])
def test_qwen4_exp_mtp_override_sets_draft_config(
wrapped_config: bool,
) -> None:
text_config = _text_config(
architectures=["Qwen4ExpForCausalLM"],
index_share_for_mtp_iteration=True,
)
config = (
Qwen4ExpConfig(
architectures=["Qwen4ExpForConditionalGeneration"],
text_config=text_config,
)
if wrapped_config
else text_config
)
draft_config = SpeculativeConfig.hf_config_override(config)
assert draft_config.index_share_for_mtp_iteration is True
assert draft_config.model_type == "qwen4_exp_mtp"
assert draft_config.architectures == ["Qwen4ExpMTP"]
assert draft_config.hc_mult == 2
assert draft_config.n_predict == 1
@pytest.mark.parametrize("ple_layer_ids", [[1], []])
def test_qwen4_exp_rejects_pipeline_parallel_only_with_ple(ple_layer_ids) -> None:
"""PLE needs raw input_ids, which non-first pipeline ranks never see. The
rest of the architecture is PP-capable, so the refusal must be conditional
-- and must land before the engine spends time loading weights."""
vllm_config = SimpleNamespace(
model_config=SimpleNamespace(
hf_text_config=_text_config(ple_layer_ids=ple_layer_ids),
multimodal_config=None,
),
parallel_config=SimpleNamespace(
pipeline_parallel_size=2, enable_dbo=False, ubatch_size=1
),
speculative_config=None,
)
with patch.object(
Qwen3_5ForConditionalGenerationConfig, "verify_and_update_config"
):
if ple_layer_ids:
with pytest.raises(NotImplementedError, match="pipeline_parallel_size=1"):
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(
vllm_config
)
else:
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(vllm_config)
def test_qwen4_exp_model_state_prepares_ngram_context() -> None:
model_state = object.__new__(Qwen4ExpModelState)
model_state.uses_ngram_embedding = True
model_state.ngram_context_len = 3
model_state.ngram_eos_token_id = 99
model_state.ngram_context = torch.empty((4, 3), dtype=torch.int32)
model_state.ngram_context_offsets = torch.arange(-3, 0, dtype=torch.int64)
model_state.ple_query_start_loc = torch.empty(5, dtype=torch.int32)
input_batch = SimpleNamespace(
num_reqs=2,
num_reqs_after_padding=3,
idx_mapping=torch.tensor([1, 0]),
query_start_loc=torch.tensor([0, 2, 3, 3], dtype=torch.int32),
)
req_states = SimpleNamespace(
num_computed_tokens=SimpleNamespace(gpu=torch.tensor([3, 1])),
all_token_ids=SimpleNamespace(
gpu=torch.tensor([[1, 2, 3, 4], [20, 21, 22, 23]], dtype=torch.int32)
),
)
with patch.object(MambaHybridModelState, "prepare_inputs", return_value={}):
model_inputs = model_state.prepare_inputs(input_batch, req_states)
torch.testing.assert_close(
model_inputs["query_start_loc"],
torch.tensor([0, 2, 3, 3], dtype=torch.int32),
)
torch.testing.assert_close(
model_inputs["ngram_context"],
torch.tensor([[99, 99, 20], [1, 2, 3], [99, 99, 99]], dtype=torch.int32),
)
def test_qwen4_exp_model_state_prepares_stable_dummy_ngram_inputs() -> None:
model_state = object.__new__(Qwen4ExpModelState)
model_state.uses_ngram_embedding = True
model_state.ngram_eos_token_id = 99
model_state.ngram_context = torch.empty((4, 3), dtype=torch.int32)
model_state.ple_query_start_loc = torch.empty(5, dtype=torch.int32)
with patch.object(MambaHybridModelState, "prepare_dummy_inputs", return_value={}):
first = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=8)
query_start_loc_ptr = first["query_start_loc"].data_ptr()
ngram_context_ptr = first["ngram_context"].data_ptr()
second = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=8)
torch.testing.assert_close(
second["query_start_loc"], torch.tensor([0, 2, 5, 8], dtype=torch.int32)
)
torch.testing.assert_close(
second["ngram_context"], torch.full((3, 3), 99, dtype=torch.int32)
)
assert second["query_start_loc"].data_ptr() == query_start_loc_ptr
assert second["ngram_context"].data_ptr() == ngram_context_ptr
+94
View File
@@ -0,0 +1,94 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.models.qwen4_exp.nvidia.ops.hc import (
grouped_gemma_rmsnorm,
hc_combine,
hc_combine_norm,
hc_gate_mix,
)
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda() or not HAS_TRITON,
reason="HC kernels require CUDA and Triton",
)
HC = 4
HIDDEN_SIZE = 2560
HYPER_HIDDEN_SIZE = HC * HIDDEN_SIZE
EPS = 1e-6
def test_grouped_gemma_rmsnorm() -> None:
torch.manual_seed(0)
x = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
actual = grouped_gemma_rmsnorm(x, weight, EPS, HC)
grouped = x.float().unflatten(-1, (HC, HIDDEN_SIZE))
variance = grouped.square().mean(-1, keepdim=True)
expected = grouped * torch.rsqrt(variance + EPS)
expected = expected.flatten(-2) * (1.0 + weight.float())
torch.testing.assert_close(actual, expected.to(torch.bfloat16))
def test_hc_gate_mix() -> None:
torch.manual_seed(0)
x = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
gate = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
actual = hc_gate_mix(x, gate, HC)
expected = (
torch.sigmoid(gate.float().unflatten(-1, (HC, HIDDEN_SIZE)))
* x.float().unflatten(-1, (HC, HIDDEN_SIZE))
).mean(-2)
torch.testing.assert_close(actual, expected.to(torch.bfloat16))
def test_hc_combine() -> None:
torch.manual_seed(0)
block_output = torch.randn(2, HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
residual = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
injection = torch.randn(2, HC, dtype=torch.bfloat16, device="cuda")
actual = hc_combine(residual, block_output, injection, HC)
injection_weight = 2.0 * torch.sigmoid(injection.float() / HC)
expected = residual.float().unflatten(-1, (HC, HIDDEN_SIZE))
expected = expected + block_output.float().unsqueeze(
-2
) * injection_weight.unsqueeze(-1)
torch.testing.assert_close(actual, expected.flatten(-2).to(torch.bfloat16))
def test_hc_combine_norm() -> None:
torch.manual_seed(0)
block_output = torch.randn(2, HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
residual = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
injection = torch.randn(2, HC, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
actual, actual_norm = hc_combine_norm(
residual, block_output, injection, weight, EPS, HC
)
injection_weight = 2.0 * torch.sigmoid(injection.float() / HC)
expected = residual.float().unflatten(-1, (HC, HIDDEN_SIZE))
expected = expected + block_output.float().unsqueeze(
-2
) * injection_weight.unsqueeze(-1)
expected = expected.flatten(-2).to(residual.dtype)
grouped = expected.float().unflatten(-1, (HC, HIDDEN_SIZE))
variance = grouped.square().mean(-1, keepdim=True)
expected_norm = grouped * torch.rsqrt(variance + EPS)
expected_norm = expected_norm.flatten(-2) * (1.0 + weight.float())
torch.testing.assert_close(actual, expected)
torch.testing.assert_close(actual_norm, expected_norm.to(torch.bfloat16))
+401
View File
@@ -0,0 +1,401 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from functools import partial
from types import SimpleNamespace
import pytest
import torch
from torch import nn
from torch.nn import functional as F
import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module
import vllm.model_executor.parameter as parameter_module
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.models.qwen4_exp.common.ple import (
PLEShardOverlap,
compute_ple_shard_overlap,
copy_ple_embedding_shard_,
)
from vllm.models.qwen4_exp.nvidia import ple_layer as ple_layer_module
from vllm.models.qwen4_exp.nvidia.ple_layer import (
Qwen4ExpNGramEmbedding,
Qwen4ExpPLEFp8EmbeddingMethod,
Qwen4ExpPLELayer,
_get_ple_embedding_quant_method,
)
def _make_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding:
module = Qwen4ExpNGramEmbedding.__new__(Qwen4ExpNGramEmbedding)
nn.Module.__init__(module)
module.split_ngram_parts = 2
module.register_buffer("layer_multipliers", torch.zeros(1, dtype=torch.long))
module.register_buffer("ngram_heads_offsets", torch.zeros(1, dtype=torch.long))
module.register_buffer("ngram_heads_vocab_sizes", torch.zeros(1, dtype=torch.long))
module.ngram_embedding = SimpleNamespace(
org_vocab_size=8,
embedding_dim=2,
weight=nn.Parameter(torch.full((4, 2), -1.0)),
shard_indices=SimpleNamespace(
org_vocab_start_index=2,
org_vocab_end_index=6,
),
)
_set_test_embedding_weight_loader(module.ngram_embedding)
return module
def _set_test_embedding_weight_loader(embedding) -> None:
embedding.weight.weight_loader = partial(
copy_ple_embedding_shard_,
tp_start=embedding.shard_indices.org_vocab_start_index,
tp_end=embedding.shard_indices.org_vocab_end_index,
)
def _make_fp8_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding:
module = _make_ngram_embedding_for_load_test()
embedding = nn.Module()
embedding.org_vocab_size = 8
embedding.embedding_dim = 2
embedding.shard_indices = SimpleNamespace(
org_vocab_start_index=2,
org_vocab_end_index=6,
)
embedding.register_parameter(
"weight",
nn.Parameter(
torch.full((4, 2), -1.0).to(torch.float8_e4m3fn),
requires_grad=False,
),
)
embedding.register_parameter(
"weight_scale",
nn.Parameter(torch.zeros(1, dtype=torch.bfloat16), requires_grad=False),
)
_set_test_embedding_weight_loader(embedding)
module.ngram_embedding = embedding
return module
def test_ple_shard_overlap_and_copy() -> None:
overlap = compute_ple_shard_overlap(
checkpoint_start=2, checkpoint_rows=5, tp_start=4, tp_end=8
)
assert overlap == PLEShardOverlap(source_start=2, destination_start=0, row_count=3)
destination = torch.full((4, 2), -1.0)
loaded = torch.arange(10, dtype=torch.float64).reshape(5, 2)
copied = copy_ple_embedding_shard_(
destination,
loaded,
checkpoint_start=2,
tp_start=4,
tp_end=8,
)
assert copied == 3
torch.testing.assert_close(destination[:3], loaded[2:5].float())
torch.testing.assert_close(destination[3], torch.tensor([-1.0, -1.0]))
def test_ple_shard_copy_is_a_noop_without_overlap() -> None:
destination = torch.ones(4, 2)
copied = copy_ple_embedding_shard_(
destination,
torch.zeros(2, 2),
checkpoint_start=10,
tp_start=4,
tp_end=8,
)
assert copied == 0
assert torch.equal(destination, torch.ones_like(destination))
def test_ngram_embedding_loads_shards_and_ignores_legacy_token_lookup() -> None:
module = _make_ngram_embedding_for_load_test()
shard_0 = torch.arange(8, dtype=torch.float32).reshape(4, 2)
shard_1 = torch.arange(8, 16, dtype=torch.float32).reshape(4, 2)
loaded = module.load_weights(
[
("ngram_embedding.shard_0.weight", shard_0),
("ngram_embedding.shard_1.weight", shard_1),
("token_lookup", torch.tensor([2, 1, 0])),
]
)
assert loaded == {"ngram_embedding.weight"}
torch.testing.assert_close(
module.ngram_embedding.weight,
torch.cat((shard_0[2:4], shard_1[0:2])),
)
def test_ngram_embedding_loads_fp8_shards_and_global_scale() -> None:
module = _make_fp8_ngram_embedding_for_load_test()
shard_0 = torch.arange(8, dtype=torch.float32).reshape(4, 2).to(torch.float8_e4m3fn)
shard_1 = (
torch.arange(8, 16, dtype=torch.float32).reshape(4, 2).to(torch.float8_e4m3fn)
)
weight_scale = torch.tensor([0.25], dtype=torch.bfloat16)
loaded = module.load_weights(
[
("ngram_embedding.shard_0.weight", shard_0),
("ngram_embedding.shard_1.weight", shard_1),
("ngram_embedding.weight_scale", weight_scale),
]
)
assert loaded == {"ngram_embedding.weight", "ngram_embedding.weight_scale"}
assert module.ngram_embedding.weight.dtype == torch.float8_e4m3fn
assert torch.equal(
module.ngram_embedding.weight.float(),
torch.cat((shard_0[2:4], shard_1[0:2])).float(),
)
assert torch.equal(module.ngram_embedding.weight_scale, weight_scale)
def _make_fp8_embedding_layer(
monkeypatch: pytest.MonkeyPatch,
) -> embedding_module.VocabParallelEmbedding:
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
embedding_module, "get_tensor_model_parallel_world_size", lambda: 1
)
monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
parameter_module, "get_tensor_model_parallel_world_size", lambda: 1
)
method = Qwen4ExpPLEFp8EmbeddingMethod()
layer = embedding_module.VocabParallelEmbedding(
3,
2,
params_dtype=torch.bfloat16,
padding_size=1,
quant_method=method,
)
weight = torch.tensor([[1.0, 2.0], [4.0, 8.0], [16.0, 32.0]])
layer.weight.data.copy_(weight.to(torch.float8_e4m3fn))
layer.weight_scale.data.copy_(torch.tensor([0.25], dtype=torch.bfloat16))
return layer
def test_ple_fp8_embedding_dequantizes_in_ple_layer(monkeypatch) -> None:
layer = _make_fp8_embedding_layer(monkeypatch)
quantized_output = layer(torch.tensor([2, 0]))
ple_layer = Qwen4ExpPLELayer.__new__(Qwen4ExpPLELayer)
nn.Module.__init__(ple_layer)
ple_layer.ple_embedding = nn.Module()
ple_layer.ple_embedding.ngram_embedding = layer
output = ple_layer._dequantize_embeddings(
quantized_output,
torch.bfloat16,
)
assert layer.weight.dtype == torch.float8_e4m3fn
assert layer.weight_scale.dtype == torch.bfloat16
assert quantized_output.dtype == torch.float8_e4m3fn
assert output.dtype == torch.bfloat16
weight = torch.tensor([[1.0, 2.0], [4.0, 8.0], [16.0, 32.0]])
torch.testing.assert_close(output, (weight[[2, 0]] * 0.25).bfloat16())
def test_ple_fp8_embedding_uses_int8_for_tp_reduce(monkeypatch) -> None:
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
embedding_module, "get_tensor_model_parallel_world_size", lambda: 2
)
monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
parameter_module, "get_tensor_model_parallel_world_size", lambda: 2
)
monkeypatch.setattr(
embedding_module,
"get_masked_input_and_mask",
lambda *args: (
torch.tensor([0, 0]),
torch.tensor([False, True]),
),
)
reduced_dtypes = []
def all_reduce(tensor: torch.Tensor) -> torch.Tensor:
reduced_dtypes.append(tensor.dtype)
return tensor.clone()
monkeypatch.setattr(
embedding_module,
"tensor_model_parallel_all_reduce",
all_reduce,
)
layer = embedding_module.VocabParallelEmbedding(
4,
2,
params_dtype=torch.bfloat16,
padding_size=1,
quant_method=Qwen4ExpPLEFp8EmbeddingMethod(),
)
layer.weight.data.copy_(
torch.tensor([[1.0, 2.0], [4.0, 8.0]]).to(torch.float8_e4m3fn)
)
output = layer(torch.tensor([0, 2]))
assert reduced_dtypes == [torch.int8]
assert output.dtype == torch.float8_e4m3fn
torch.testing.assert_close(output[0].float(), layer.weight[0].float())
assert torch.count_nonzero(output[1].float()) == 0
def test_ple_fp8_embedding_respects_checkpoint_shard_exclusions() -> None:
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
quant_config = Fp8Config(
is_checkpoint_fp8_serialized=True,
ignored_layers=[],
weight_block_size=[128, 128],
)
assert isinstance(
_get_ple_embedding_quant_method(quant_config, prefix),
Qwen4ExpPLEFp8EmbeddingMethod,
)
quant_config.ignored_layers = [f"{prefix}.shard_0"]
assert _get_ple_embedding_quant_method(quant_config, prefix) is None
def test_ple_ngram_ids_custom_op_uses_current_request_layout(monkeypatch) -> None:
class RuntimeNGramEmbedding(nn.Module):
def compute_ngram_ids(
self,
input_ids: torch.Tensor,
query_start_loc: torch.Tensor,
ngram_context: torch.Tensor,
) -> torch.Tensor:
del input_ids, ngram_context
num_reqs = query_start_loc.numel() - 1
return torch.full((4, 2), num_reqs, dtype=torch.long)
layer = Qwen4ExpPLELayer.__new__(Qwen4ExpPLELayer)
nn.Module.__init__(layer)
layer.ple_embedding = RuntimeNGramEmbedding()
monkeypatch.setattr(
ple_layer_module,
"get_forward_context",
lambda: SimpleNamespace(no_compile_layers={"ple": layer}),
)
input_ids = torch.arange(4)
ngram_context = torch.zeros(2, 2, dtype=torch.long)
output = torch.empty(4, 2, dtype=torch.long)
ple_layer_module.qwen4_exp_compute_ple_ngram_ids(
input_ids,
torch.tensor([0, 4]),
ngram_context,
output,
"ple",
)
assert torch.equal(output, torch.ones_like(output))
ple_layer_module.qwen4_exp_compute_ple_ngram_ids(
input_ids,
torch.tensor([0, 2, 4]),
ngram_context,
output,
"ple",
)
assert torch.equal(output, torch.full_like(output, 2))
def test_dilated_ple_spec_state_rolls_back_before_next_forward() -> None:
module = Qwen4ExpPLELayer.__new__(Qwen4ExpPLELayer)
nn.Module.__init__(module)
module.conv_state_len = 6
module.short_conv_dilation = 2
conv_weights = torch.tensor([[0.25, -0.5, 0.75, 1.0]])
conv_state = torch.zeros(2, 1, 9)
conv_state[1] = torch.arange(1, 10, dtype=torch.float32).reshape(1, 9)
first_inputs = torch.tensor([[10.0], [20.0], [30.0], [40.0]])
initial_state = conv_state[1:].clone()
first_history = torch.cat(
(initial_state[..., : module.conv_state_len], first_inputs.T.unsqueeze(0)),
dim=-1,
)
graph_padded_inputs = F.pad(first_inputs, (0, 0, 0, 4))
first_output = module._short_conv_dilated_spec_batched(
graph_padded_inputs,
conv_state,
conv_weights,
torch.tensor([1, 0]),
torch.tensor([0, 4, 4]),
torch.tensor([1, 0]),
spec_query_len=4,
)
expected_first_output = F.silu(
F.conv1d(
first_history,
conv_weights.unsqueeze(1),
groups=1,
dilation=module.short_conv_dilation,
)
).transpose(1, 2)[0]
expected_first_state = first_history[..., 1:10]
torch.testing.assert_close(first_output[:4], expected_first_output)
assert torch.count_nonzero(first_output[4:]) == 0
assert torch.count_nonzero(conv_state[0]) == 0
torch.testing.assert_close(conv_state[1:], expected_first_state)
second_inputs = torch.tensor([[50.0], [60.0]])
rollback_state = expected_first_state[..., 1:7]
padded_second_inputs = F.pad(second_inputs.T.unsqueeze(0), (0, 2))
second_history = torch.cat((rollback_state, padded_second_inputs), dim=-1)
expected_second_state = expected_first_state.clone()
expected_second_state[..., :7] = second_history[..., 1:8]
second_output = module._short_conv_dilated_spec_batched(
second_inputs,
conv_state,
conv_weights,
torch.tensor([1]),
torch.tensor([0, 2]),
torch.tensor([2]),
spec_query_len=4,
)
expected_second_output = F.silu(
F.conv1d(
second_history,
conv_weights.unsqueeze(1),
groups=1,
dilation=module.short_conv_dilation,
)
).transpose(1, 2)[0, :2]
torch.testing.assert_close(second_output, expected_second_output)
torch.testing.assert_close(conv_state[1:], expected_second_state)
def test_ple_short_conv_uses_fallback_when_profile_metadata_is_omitted(
monkeypatch: pytest.MonkeyPatch,
) -> None:
module = Qwen4ExpPLELayer.__new__(Qwen4ExpPLELayer)
nn.Module.__init__(module)
module.prefix = "model.layers.1.ple"
inputs = torch.arange(6, dtype=torch.float32).reshape(3, 2)
expected = inputs + 1
monkeypatch.setattr(module, "_short_conv_fallback", lambda _: expected)
monkeypatch.setattr(
ple_layer_module,
"get_forward_context",
lambda: SimpleNamespace(attn_metadata={"model.layers.0.self_attn": object()}),
)
output = module._short_conv(inputs)
assert output is expected
+297
View File
@@ -0,0 +1,297 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from types import SimpleNamespace
from typing import Any
import pytest
import torch
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
from vllm.models.qwen4_exp.amd import (
model as _qwen4_exp_model, # noqa: F401
)
from vllm.models.qwen4_exp.amd import ple_layer as ple_layer_module
from vllm.models.qwen4_exp.amd.indexer_qsa import (
apply_qsa_rmsnorm,
apply_qsa_rope,
)
from vllm.models.qwen4_exp.amd.ops import qsa as qsa_ops
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(),
reason="AMD QSA requires ROCm",
)
requires_qsa_kernels = pytest.mark.skipif(
not HAS_TRITON,
reason="AMD QSA kernels require Triton",
)
def test_ple_ngram_embedding_custom_op_uses_resident_weight(
monkeypatch: pytest.MonkeyPatch,
) -> None:
layer_name = "model.layers.0.ple"
layer = ple_layer_module.Qwen4ExpPLELayer.__new__(ple_layer_module.Qwen4ExpPLELayer)
torch.nn.Module.__init__(layer)
layer.ple_embedding = torch.nn.Module()
layer.ple_embedding.ngram_embedding = torch.nn.Embedding(8, 3)
context = SimpleNamespace(no_compile_layers={layer_name: layer})
monkeypatch.setattr(ple_layer_module, "get_forward_context", lambda: context)
ngram_ids = torch.tensor([[0, 1], [2, 3]])
output = torch.empty(2, 6)
ple_layer_module.qwen4_exp_amd_ple_ngram_embedding(
ngram_ids,
output,
layer_name,
)
expected = layer.ple_embedding.ngram_embedding(ngram_ids).flatten(-2)
torch.testing.assert_close(output, expected)
def _qsa_sparse_paged_attention_reference(
q: torch.Tensor,
k_cache: torch.Tensor,
v_cache: torch.Tensor,
logical_indices: torch.Tensor,
block_table: torch.Tensor,
token_to_req: torch.Tensor,
softmax_scale: float,
) -> torch.Tensor:
output = torch.zeros_like(q)
repeats = q.shape[1] // k_cache.shape[2]
page_size = k_cache.shape[1]
for row in range(q.shape[0]):
logical = logical_indices[row]
logical = logical[logical >= 0].long()
if not logical.numel():
continue
request = token_to_req[row].long()
pages = block_table[request, logical // page_size].long()
offsets = logical % page_size
keys = k_cache[pages, offsets].repeat_interleave(repeats, dim=1)
values = v_cache[pages, offsets].repeat_interleave(repeats, dim=1)
scores = torch.einsum("hd,khd->hk", q[row].float(), keys.float())
probabilities = torch.softmax(scores * softmax_scale, dim=-1)
output[row] = torch.einsum("hk,khd->hd", probabilities, values.float()).to(
q.dtype
)
return output
def test_qsa_rope_uses_platform_dispatch() -> None:
tensor = torch.arange(16, dtype=torch.float32).reshape(2, 2, 4)
positions = torch.tensor([0, 1])
calls = []
def apply_rotary_emb(
rotary_input: torch.Tensor,
cos: torch.Tensor,
sin: torch.Tensor,
) -> torch.Tensor:
calls.append((rotary_input, cos, sin))
return rotary_input + 1
rotary_emb = SimpleNamespace(
rotary_dim=2,
apply_rotary_emb=apply_rotary_emb,
_match_cos_sin_cache_dtype=lambda _: torch.zeros(2, 4),
)
output = apply_qsa_rope(rotary_emb, positions, tensor)
assert len(calls) == 1
torch.testing.assert_close(output[..., :2], tensor[..., :2] + 1)
torch.testing.assert_close(output[..., 2:], tensor[..., 2:])
def test_qsa_rmsnorm_uses_portable_implementation(default_vllm_config) -> None:
norm = GemmaRMSNorm(4, eps=1e-6)
norm.weight.data.copy_(torch.tensor([0.1, -0.2, 0.3, -0.4]))
tensor = torch.arange(8, dtype=torch.float32).reshape(2, 4)
output = apply_qsa_rmsnorm(norm, tensor)
torch.testing.assert_close(output, norm.forward_native(tensor))
def test_qsa_selection_uses_portable_topk_on_rocm(
monkeypatch: pytest.MonkeyPatch,
) -> None:
rows = 2
token_topk = 8
compress_ratio = 4
block_topk = token_topk // compress_ratio
blocks = torch.empty((rows, block_topk), dtype=torch.int32)
visible_blocks = torch.tensor([4, 6], dtype=torch.int32)
logits = torch.empty((rows, 8), dtype=torch.float32)
selection_output = torch.empty(
(rows, token_topk + compress_ratio - 1), dtype=torch.int32
)
topk_call: dict[str, Any] = {}
monkeypatch.setattr(
qsa_ops,
"qsa_mqa_paged",
lambda *args, **kwargs: (logits, visible_blocks),
)
monkeypatch.setattr(qsa_ops.current_platform, "is_cuda", lambda: False)
def top_k_per_row_decode(
topk_logits,
next_n,
seq_lens,
raw_topk_indices,
num_rows,
stride0,
stride1,
topk_tokens,
) -> None:
topk_call.update(
logits=topk_logits,
next_n=next_n,
seq_lens=seq_lens,
raw_topk_indices=raw_topk_indices,
num_rows=num_rows,
strides=(stride0, stride1),
topk_tokens=topk_tokens,
)
raw_topk_indices.zero_()
def expand_qsa_block_indices(*args) -> None:
args[-1].fill_(-1)
monkeypatch.setattr(qsa_ops.ops, "top_k_per_row_decode", top_k_per_row_decode)
monkeypatch.setattr(
qsa_ops,
"expand_qsa_block_indices_cuda",
expand_qsa_block_indices,
)
output = qsa_ops.qsa_select_paged_tokens(
torch.empty((rows, 1, 1)),
torch.empty((1, 4, 1, 1)),
torch.empty((1, 2), dtype=torch.int32),
torch.zeros(rows, dtype=torch.int32),
torch.arange(rows, dtype=torch.int32),
torch.tensor([8], dtype=torch.int32),
token_topk,
compress_ratio,
selection_output,
)
assert output is selection_output
assert torch.all(output == -1)
assert topk_call["logits"] is logits
assert topk_call["next_n"] == 1
assert topk_call["seq_lens"] is visible_blocks
assert topk_call["raw_topk_indices"].shape == blocks.shape
assert topk_call["num_rows"] == rows
assert topk_call["strides"] == (logits.stride(0), logits.stride(1))
assert topk_call["topk_tokens"] == block_topk
@requires_qsa_kernels
@pytest.mark.parametrize(
("num_rows", "num_query_heads", "num_kv_heads", "page_size"),
[
pytest.param(1, 24, 2, 1792, id="tp1_split64"),
pytest.param(16, 12, 1, 1792, id="tp2_split32"),
pytest.param(32, 6, 1, 1024, id="tp4_split8"),
pytest.param(257, 6, 1, 1024, id="tp4_split4"),
pytest.param(513, 6, 1, 1024, id="tp4_split1"),
],
)
def test_qsa_sparse_paged_attention_matches_reference(
num_rows: int,
num_query_heads: int,
num_kv_heads: int,
page_size: int,
) -> None:
torch.manual_seed(2)
head_dim = 256
num_requests = 2
num_selected_pages = 64
num_pages_per_request = num_selected_pages + 1
num_cache_blocks = num_requests * num_pages_per_request
indexer_budget = 2048
indexer_compress_ratio = 4
selection_width = indexer_budget + indexer_compress_ratio - 1
q = torch.randn(
num_rows, num_query_heads, head_dim, device="cuda", dtype=torch.bfloat16
)
kv_cache = torch.randn(
num_cache_blocks,
page_size,
num_kv_heads,
2 * head_dim,
device="cuda",
dtype=torch.bfloat16,
)
k_cache, v_cache = kv_cache.split(head_dim, dim=-1)
block_table = (
torch.randperm(num_cache_blocks, device="cuda")
.reshape(num_requests, num_pages_per_request)
.to(torch.int32)
)
rows_per_request = math.ceil(num_rows / num_requests)
row_indices = torch.arange(num_rows, device="cuda", dtype=torch.int32)
token_to_req = row_indices // rows_per_request
request_row_counts = torch.tensor(
[rows_per_request, num_rows - rows_per_request],
device="cuda",
dtype=torch.int32,
)
context_length = num_pages_per_request * page_size - 1
block_topk = indexer_budget // indexer_compress_ratio
compressed_blocks_per_page = page_size // indexer_compress_ratio
selection = torch.arange(block_topk, device="cuda")
selected_pages = selection % num_selected_pages
selected_offsets = selection // num_selected_pages
row_shifts = 2 * row_indices.unsqueeze(1)
selected_offsets = (selected_offsets + row_shifts) % compressed_blocks_per_page
block_indices = (selected_pages * compressed_blocks_per_page + selected_offsets).to(
torch.int32
)
rows_within_request = row_indices % rows_per_request
query_positions = (
context_length - request_row_counts[token_to_req.long()] + rows_within_request
).to(torch.int64)
sequence_lengths = torch.full(
(num_requests,), context_length, device="cuda", dtype=torch.int32
)
logical_indices = qsa_ops.expand_qsa_block_indices_cuda(
block_indices,
query_positions,
sequence_lengths,
token_to_req,
indexer_compress_ratio,
indexer_budget,
)
assert logical_indices.shape == (num_rows, selection_width)
actual = qsa_ops.qsa_sparse_paged_attention(
q,
k_cache,
v_cache,
logical_indices,
block_table,
token_to_req,
)
expected = _qsa_sparse_paged_attention_reference(
q,
k_cache,
v_cache,
logical_indices,
block_table,
token_to_req,
q.shape[-1] ** -0.5,
)
torch.testing.assert_close(actual, expected, rtol=2e-2, atol=2e-2)
@@ -0,0 +1,297 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness tests for the fused QSA pre-indexer."""
import pytest
import torch
from vllm.models.qwen4_exp.common.qsa_cache import (
canonical_qsa_rope_positions,
circular_qsa_slot_mapping,
compressed_qsa_slot_mapping,
)
from vllm.models.qwen4_exp.nvidia.indexer_qsa import apply_qsa_rope
from vllm.models.qwen4_exp.nvidia.ops.qsa import (
qsa_compress_groups_with_ratio,
qsa_store_cache_rows,
)
from vllm.models.qwen4_exp.nvidia.ops.qsa_pre_indexer import (
qsa_pre_indexer,
)
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
requires_qsa_kernels = pytest.mark.skipif(
not current_platform.is_cuda() or not HAS_TRITON,
reason="QSA kernels require CUDA and Triton",
)
HQ, D = 4, 128
CR = 4
MROPE_SECTION = (11, 11, 10)
EPS = 1e-6
BLOCK_SIZE = 16
COMP_PAGE = BLOCK_SIZE // CR
ROPE_POS_OFFSET = D
RTOL = 1.6e-2
ATOL = 1e-2
MIXED_BATCH = ([260, 259, 138], [1, 1, 37], [8, 8, 8])
def _make_block_table(block_counts):
num_blocks = sum(block_counts)
table = torch.full((len(block_counts), max(block_counts)), -1, dtype=torch.int32)
physical_blocks = torch.randperm(num_blocks)
offset = 0
for request, count in enumerate(block_counts):
table[request, :count] = physical_blocks[offset : offset + count]
offset += count
return table, num_blocks
@requires_qsa_kernels
@pytest.mark.usefixtures("default_vllm_config")
@pytest.mark.parametrize(
"mrope,is_2d_positions,cache_rope_positions,state_size,seq_lens,query_lens,history_lens",
[
pytest.param(True, True, True, 4, *MIXED_BATCH, id="mrope"),
pytest.param(False, False, False, 4, *MIXED_BATCH, id="text"),
pytest.param(True, False, True, 8, *MIXED_BATCH, id="mrope-model-1d"),
pytest.param(True, True, False, 4, *MIXED_BATCH, id="mrope-no-position-cache"),
pytest.param(
True,
False,
False,
8,
*MIXED_BATCH,
id="mrope-model-1d-no-position-cache",
),
pytest.param(
False, False, True, 4, *MIXED_BATCH, id="text-with-position-cache"
),
pytest.param(True, True, True, 4, [37], [37], [0], id="fresh"),
pytest.param(True, True, True, 4, [4097], [4097], [0], id="tiled"),
],
)
def test_qsa_fused_pre_indexer_matches_unfused(
mrope,
is_2d_positions,
cache_rope_positions,
state_size,
seq_lens,
query_lens,
history_lens,
) -> None:
from flashinfer.norm import gemma_rmsnorm
from vllm.model_executor.layers.rotary_embedding import get_rope
device = "cuda"
rope_params = {
"partial_rotary_factor": 0.25,
"rope_theta": 10000000,
"rope_type": "default",
}
if mrope:
rope_params["mrope_interleaved"] = True
rope_params["mrope_section"] = list(MROPE_SECTION)
with torch.device(device):
rope = get_rope(
head_size=256,
max_position=32768,
rope_parameters=rope_params,
dtype=torch.bfloat16,
)
token_to_req = torch.cat(
[
torch.full((length,), request, dtype=torch.int32)
for request, length in enumerate(query_lens)
]
).to(device)
logical_positions = torch.cat(
[
torch.arange(seq_len - query_len, seq_len, dtype=torch.int64)
for seq_len, query_len in zip(seq_lens, query_lens)
]
).to(device)
num_tokens = logical_positions.numel()
query_start_loc = torch.tensor(
[0, *torch.tensor(query_lens).cumsum(0).tolist()], dtype=torch.int32
).to(device)
positions = (
torch.stack(
[
logical_positions,
logical_positions // 7 + 3,
logical_positions // 13 + 11,
]
)
if is_2d_positions
else logical_positions
)
position_rows = (
canonical_qsa_rope_positions(positions)
if cache_rope_positions
else logical_positions.view(-1, 1, 1).expand(-1, 1, 3)
)
compressed_block_counts = [
(seq_len // CR + COMP_PAGE - 1) // COMP_PAGE for seq_len in seq_lens
]
raw_block_table, num_raw_blocks = _make_block_table([1] * len(seq_lens))
compressed_block_table, num_compressed_blocks = _make_block_table(
compressed_block_counts
)
raw_block_table = raw_block_table.to(device)
raw_slots = circular_qsa_slot_mapping(
raw_block_table,
token_to_req,
logical_positions,
state_size,
query_start_loc,
)
compressed_slots = compressed_qsa_slot_mapping(
compressed_block_table.to(device),
token_to_req,
logical_positions,
COMP_PAGE,
CR,
)
group_counts = torch.tensor(
[
seq_len // CR - (seq_len - query_len) // CR
for seq_len, query_len in zip(seq_lens, query_lens)
],
dtype=torch.int32,
device=device,
)
k_work_counts = torch.maximum(group_counts, torch.ones_like(group_counts))
k_start_loc = torch.cat([k_work_counts.new_zeros(1), k_work_counts.cumsum(0)])
work_requests = torch.repeat_interleave(
torch.arange(len(query_lens), dtype=torch.int32, device=device),
k_work_counts,
)
local_work = torch.arange(
int(k_start_loc[-1]), dtype=torch.int32, device=device
) - torch.repeat_interleave(k_start_loc[:-1], k_work_counts)
max_k_work = (num_tokens + (CR - 1) * len(query_lens)) // CR
k_work_metadata = torch.full((max_k_work, 2), -1, dtype=torch.int32, device=device)
k_work_metadata[: work_requests.numel()] = torch.stack(
(work_requests, local_work), dim=1
)
raw_width = D + 12 if cache_rope_positions else D
# Match vLLM's padded-page cache layout: rows are contiguous, while physical
# blocks have a larger stride than their logical contents.
raw_page_elements = state_size * raw_width
fused_raw_storage = torch.zeros(
num_raw_blocks,
raw_page_elements + 16,
dtype=torch.bfloat16,
device=device,
)
fused_raw = torch.as_strided(
fused_raw_storage,
(num_raw_blocks, state_size, 1, raw_width),
(raw_page_elements + 16, raw_width, raw_width, 1),
)
for request, history_len in enumerate(history_lens):
history_end = seq_lens[request] - query_lens[request]
for position in range(history_end - history_len, history_end):
block = int(raw_block_table[request, 0])
row = fused_raw[block, position % state_size, 0]
row[:D] = torch.randn(D, dtype=torch.bfloat16, device=device)
if cache_rope_positions:
row[ROPE_POS_OFFSET:].view(torch.int64).copy_(
torch.tensor(
[position, position // 7 + 3, position // 13 + 11],
dtype=torch.int64,
device=device,
)
)
unfused_raw = fused_raw.clone()
compressed_page_elements = COMP_PAGE * D
fused_compressed_storage = torch.zeros(
num_compressed_blocks,
compressed_page_elements + 16,
dtype=torch.bfloat16,
device=device,
)
fused_compressed = torch.as_strided(
fused_compressed_storage,
(num_compressed_blocks, COMP_PAGE, 1, D),
(compressed_page_elements + 16, D, D, 1),
)
unfused_compressed = fused_compressed.clone()
projected_qk = torch.randn(
num_tokens, (HQ + 1) * D, dtype=torch.bfloat16, device=device
)
q_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
k_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
fused_query = torch.empty(num_tokens, HQ, D, dtype=torch.bfloat16, device=device)
qsa_pre_indexer(
projected_qk[:, : HQ * D],
projected_qk[:, HQ * D :],
positions,
rope.cos_sin_cache,
q_weight,
k_weight,
EPS,
fused_query,
fused_raw,
raw_slots,
raw_block_table,
query_start_loc,
logical_positions,
fused_compressed,
compressed_slots,
k_work_metadata,
compress_ratio=CR,
mrope_section=MROPE_SECTION if mrope else None,
rope_pos_offset=ROPE_POS_OFFSET if cache_rope_positions else None,
)
unfused_query = projected_qk[:, : HQ * D].reshape(num_tokens, HQ, D)
unfused_query = gemma_rmsnorm(
unfused_query.reshape(-1, D), q_weight, EPS
).reshape_as(unfused_query)
unfused_query = apply_qsa_rope(rope, positions, unfused_query)
raw_keys = unfused_raw[..., :D]
rope_positions = (
unfused_raw[..., ROPE_POS_OFFSET:].view(torch.int64)
if cache_rope_positions
else None
)
pooled, first_positions = qsa_compress_groups_with_ratio(
projected_qk[:, HQ * D :].reshape(-1, 1, D),
position_rows,
raw_keys,
raw_block_table,
token_to_req,
query_start_loc,
logical_positions,
compressed_slots,
CR,
rope_positions,
)
compressed_rows = gemma_rmsnorm(pooled.reshape(-1, D), k_weight, EPS).reshape(
-1, 1, D
)
group_positions = (
first_positions.transpose(0, 1) if mrope else first_positions[:, 0]
)
compressed_rows = apply_qsa_rope(rope, group_positions, compressed_rows)
qsa_store_cache_rows(unfused_compressed, compressed_slots, compressed_rows)
qsa_store_cache_rows(raw_keys, raw_slots, projected_qk[:, HQ * D :])
if rope_positions is not None:
qsa_store_cache_rows(rope_positions, raw_slots, position_rows)
torch.testing.assert_close(fused_query, unfused_query, rtol=RTOL, atol=ATOL)
assert torch.equal(fused_raw.view(torch.int16), unfused_raw.view(torch.int16))
torch.testing.assert_close(
fused_compressed, unfused_compressed, rtol=RTOL, atol=ATOL
)
@@ -0,0 +1,973 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from types import SimpleNamespace
import pytest
import torch
from vllm.models.qwen4_exp.common import qsa_cache
from vllm.models.qwen4_exp.common.qsa_cache import QSAMetadataBuilder
from vllm.models.qwen4_exp.nvidia import indexer_qsa
from vllm.models.qwen4_exp.nvidia import (
model as _qwen4_exp_model, # noqa: F401
)
from vllm.models.qwen4_exp.nvidia.ops import qsa as qsa_ops
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
requires_qsa_kernels = pytest.mark.skipif(
not current_platform.is_cuda() or not HAS_TRITON,
reason="QSA kernels require CUDA and Triton",
)
def test_qsa_mtp_index_share_updates_cache_but_skips_selection(
monkeypatch: pytest.MonkeyPatch,
) -> None:
rows = torch.tensor([[3, 1, -1], [5, 2, 0]], dtype=torch.int32)
raw_metadata = SimpleNamespace(
num_actual_tokens=2,
slot_mapping=torch.arange(2),
block_table=torch.empty(0),
query_start_loc=torch.arange(3),
logical_positions=torch.arange(2),
)
compressed_metadata = SimpleNamespace(
num_actual_tokens=2,
slot_mapping=torch.arange(2),
k_work_metadata=torch.empty(0),
)
updates = []
selections = []
indexer = SimpleNamespace(
skip_topk=True,
_metadata=lambda: (raw_metadata, compressed_metadata),
index_qk_proj=lambda hidden: (torch.zeros(2, 2), None),
index_n_heads=1,
index_kv_heads=1,
index_head_dim=1,
raw_key_cache=SimpleNamespace(
kv_cache=torch.empty(0),
rope_position_cache=None,
rope_position_offset=0,
),
compressed_key_cache=SimpleNamespace(kv_cache=torch.empty(0)),
use_fused_pre_indexer=True,
rotary_emb=SimpleNamespace(cos_sin_cache=torch.empty(0)),
q_layernorm=SimpleNamespace(weight=torch.ones(1), variance_epsilon=1e-6),
k_layernorm=SimpleNamespace(weight=torch.ones(1)),
compress_ratio=2,
)
monkeypatch.setattr(
indexer_qsa,
"qsa_pre_indexer",
lambda *args, **kwargs: updates.append((args, kwargs)),
)
monkeypatch.setattr(
qsa_ops,
"qsa_select_paged_tokens",
lambda *args, **kwargs: selections.append((args, kwargs)),
)
actual = indexer_qsa.QSAIndexer.forward(
indexer,
torch.zeros(2, 4),
torch.tensor([7, 8]),
rows,
)
assert actual is rows
assert len(updates) == 1
assert not selections
def _qsa_mqa_paged_reference(
q: torch.Tensor,
k_cache: torch.Tensor,
page_table: torch.Tensor,
token_to_req: torch.Tensor,
visible_lengths: torch.Tensor,
) -> torch.Tensor:
pages = page_table.index_select(0, token_to_req.long()).long()
keys = k_cache[pages, :, 0, :].flatten(1, 2)
scores = torch.einsum("rhd,rnd->rnh", q.float(), keys.float())
logits = torch.relu(scores).sum(dim=-1) / math.sqrt(q.shape[-1])
positions = torch.arange(keys.shape[1], device=q.device).unsqueeze(0)
return logits.masked_fill(positions >= visible_lengths.unsqueeze(1), -torch.inf)
def _qsa_relative_topk_reference(
logits: torch.Tensor,
row_starts: torch.Tensor,
row_ends: torch.Tensor,
topk: int,
) -> torch.Tensor:
output = torch.full(
(logits.shape[0], topk), -1, dtype=torch.int32, device=logits.device
)
for row in range(logits.shape[0]):
start = int(row_starts[row].item())
length = int((row_ends[row] - row_starts[row]).item())
width = min(length, topk)
if width:
output[row, :width] = torch.topk(
logits[row, start : start + length], width
).indices.to(torch.int32)
return output
def _expand_qsa_indices_reference(
block_indices: torch.Tensor,
query_positions: torch.Tensor,
sequence_lengths: torch.Tensor,
compress_ratio: int,
token_topk: int,
) -> torch.Tensor:
rows = block_indices.shape[0]
block_topk = token_topk // compress_ratio
output_width = token_topk + compress_ratio - 1
offsets = torch.arange(compress_ratio, device=block_indices.device)
blocks = block_indices.long()
expanded = blocks.unsqueeze(-1) * compress_ratio + offsets
expanded = torch.where(
blocks.unsqueeze(-1) >= 0, expanded, torch.full_like(expanded, -1)
).reshape(rows, block_topk * compress_ratio)
expanded = expanded[:, :token_topk]
expanded = torch.where(
(expanded >= 0) & (expanded < sequence_lengths.unsqueeze(1)),
expanded,
torch.full_like(expanded, -1),
)
tail_offsets = torch.arange(compress_ratio - 1, device=block_indices.device)
visible_tokens = query_positions + 1
tail_start = visible_tokens // compress_ratio * compress_ratio
tail = tail_start.unsqueeze(1) + tail_offsets.unsqueeze(0)
tail_count = (visible_tokens - tail_start).unsqueeze(1)
tail_valid = (tail_offsets.unsqueeze(0) < tail_count) & (
tail < sequence_lengths.unsqueeze(1)
)
tail = torch.where(tail_valid, tail, torch.full_like(tail, -1))
result = torch.cat((expanded, tail), dim=1)
order = torch.arange(output_width, device=result.device).expand(rows, -1)
sort_key = torch.where(result >= 0, order, order + output_width)
return result.gather(1, torch.argsort(sort_key, dim=1, stable=True)).to(torch.int32)
def _qsa_select_paged_tokens_reference(
q: torch.Tensor,
k_cache: torch.Tensor,
page_table: torch.Tensor,
token_to_req: torch.Tensor,
query_positions: torch.Tensor,
sequence_lengths: torch.Tensor,
token_topk: int,
compress_ratio: int,
) -> torch.Tensor:
row_sequence_lengths = sequence_lengths.index_select(0, token_to_req.long())
visible_blocks = torch.minimum(
(query_positions + 1) // compress_ratio,
row_sequence_lengths // compress_ratio,
).to(torch.int32)
logits = _qsa_mqa_paged_reference(
q,
k_cache,
page_table,
token_to_req,
visible_blocks,
)
starts = torch.zeros_like(visible_blocks)
blocks = _qsa_relative_topk_reference(
logits,
starts,
visible_blocks,
token_topk // compress_ratio,
)
return _expand_qsa_indices_reference(
blocks,
query_positions,
row_sequence_lengths,
compress_ratio,
token_topk,
)
def _qsa_sparse_paged_attention_reference(
q: torch.Tensor,
k_cache: torch.Tensor,
v_cache: torch.Tensor,
logical_indices: torch.Tensor,
block_table: torch.Tensor,
token_to_req: torch.Tensor,
softmax_scale: float,
) -> torch.Tensor:
output = torch.zeros_like(q)
repeats = q.shape[1] // k_cache.shape[2]
page_size = k_cache.shape[1]
for row in range(q.shape[0]):
logical = logical_indices[row]
logical = logical[logical >= 0].long()
if not logical.numel():
continue
request = token_to_req[row].long()
pages = block_table[request, logical // page_size].long()
offsets = logical % page_size
keys = k_cache[pages, offsets].repeat_interleave(repeats, dim=1)
values = v_cache[pages, offsets].repeat_interleave(repeats, dim=1)
scores = torch.einsum("hd,khd->hk", q[row].float(), keys.float())
probabilities = torch.softmax(scores * softmax_scale, dim=-1)
output[row] = torch.einsum("hk,khd->hd", probabilities, values.float()).to(
q.dtype
)
return output
@requires_qsa_kernels
def test_qsa_side_metadata_marks_cudagraph_padding_inert() -> None:
device = torch.device("cuda")
builder = QSAMetadataBuilder.__new__(QSAMetadataBuilder)
builder.compress_ratio = 1
builder.is_circular_buffer = False
builder.storage_block_size = 64
builder.token_to_req_buffer = torch.empty(16, dtype=torch.int32, device=device)
builder.slot_mapping_buffer = torch.empty(16, dtype=torch.int64, device=device)
builder.logical_positions_buffer = torch.empty(16, dtype=torch.int64, device=device)
builder.k_work_metadata_buffer = torch.empty(0, 2, dtype=torch.int32, device=device)
query_start_loc = torch.tensor([0, 4, 8, 12, 12], dtype=torch.int32, device=device)
token_to_req = torch.tensor([0] * 4 + [1] * 4 + [2] * 4 + [0] * 4, device=device)
common = SimpleNamespace(
num_actual_tokens=16,
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc.cpu(),
seq_lens=torch.tensor([68, 68, 68, 0], dtype=torch.int32, device=device),
slot_mapping=torch.tensor(list(range(12)) + [-1] * 4, device=device),
block_table_tensor=torch.empty((4, 0), dtype=torch.int32, device=device),
token_to_req_indices=lambda buffer: buffer.copy_(token_to_req),
)
metadata = builder.build(0, common)
assert metadata.logical_positions.tolist() == [
64,
65,
66,
67,
64,
65,
66,
67,
64,
65,
66,
67,
-1,
-1,
-1,
-1,
]
assert metadata.slot_mapping.tolist() == list(range(12)) + [-1] * 4
@requires_qsa_kernels
def test_qsa_circular_buffer_metadata_keeps_only_each_requests_suffix() -> None:
device = torch.device("cuda")
builder = QSAMetadataBuilder.__new__(QSAMetadataBuilder)
builder.compress_ratio = 4
builder.is_circular_buffer = True
builder.kv_cache_spec = SimpleNamespace(block_size=4)
builder.storage_block_size = 4
builder.token_to_req_buffer = torch.empty(16, dtype=torch.int32, device=device)
builder.slot_mapping_buffer = torch.empty(16, dtype=torch.int64, device=device)
builder.logical_positions_buffer = torch.empty(16, dtype=torch.int64, device=device)
builder.k_work_metadata_buffer = torch.empty(0, 2, dtype=torch.int32, device=device)
query_start_loc = torch.tensor([0, 7, 13, 13], dtype=torch.int32, device=device)
token_to_req = torch.tensor([0] * 7 + [1] * 6 + [0] * 3, device=device)
block_table = torch.tensor([[1], [0], [2]], dtype=torch.int32, device=device)
common = SimpleNamespace(
num_actual_tokens=16,
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc.cpu(),
seq_lens=torch.tensor([9, 11, 0], dtype=torch.int32, device=device),
slot_mapping=torch.full((16,), -1, dtype=torch.int64, device=device),
block_table_tensor=block_table,
token_to_req_indices=lambda buffer: buffer.copy_(token_to_req),
)
metadata = builder.build(0, common)
expected = [
-1,
-1,
-1,
5,
6,
7,
4,
-1,
-1,
3,
0,
1,
2,
-1,
-1,
-1,
]
assert metadata.slot_mapping.tolist() == expected
@pytest.mark.parametrize("chunk_start", list(range(8)))
def test_qsa_circular_buffer_survives_one_speculative_step(chunk_start: int) -> None:
"""A speculative step must not overwrite the open group's committed keys.
The step stores every row it computes, drafts included, before acceptance
is known, while the next step still reads the earlier members of the group
being compressed from the ring. A ring sized at the compression ratio makes
those rows alias, so a rejected draft silently replaces a committed key.
"""
compress_ratio = 4
num_spec = 3
capacity = compress_ratio * -(-(compress_ratio + num_spec) // compress_ratio)
query_len = num_spec + 1
slots = qsa_cache.circular_qsa_slot_mapping(
torch.tensor([[0]], dtype=torch.int32),
torch.zeros(query_len, dtype=torch.int32),
torch.arange(chunk_start, chunk_start + query_len),
capacity,
query_start_loc=torch.tensor([0, query_len], dtype=torch.int32),
)
committed = torch.arange(chunk_start - chunk_start % compress_ratio, chunk_start)
assert set(slots.tolist()).isdisjoint((committed % capacity).tolist())
def _qsa_key_cache(block_size: int, compress_ratio: int) -> qsa_cache.QSAKeyStateCache:
return qsa_cache.QSAKeyStateCache(
head_size=64,
dtype=torch.bfloat16,
cache_config=SimpleNamespace(block_size=block_size),
prefix=f"raw.{block_size}.{compress_ratio}",
vllm_config=SimpleNamespace(
compilation_config=SimpleNamespace(static_forward_context={})
),
compress_ratio=compress_ratio,
)
def test_qsa_state_caches_adapt_the_unified_logical_layout() -> None:
raw_cache = _qsa_key_cache(block_size=32, compress_ratio=4)
compressed_cache = qsa_cache.QSACompressedKeyCache(
head_size=64,
dtype=torch.bfloat16,
cache_config=SimpleNamespace(block_size=32),
prefix="compressed.bind",
vllm_config=SimpleNamespace(
compilation_config=SimpleNamespace(static_forward_context={})
),
compress_ratio=4,
)
raw_view = torch.empty(2, 1, 8, 64, dtype=torch.bfloat16)
compressed_view = torch.empty(2, 1, 8, 64, dtype=torch.bfloat16)
raw_cache.bind_kv_cache(raw_view)
compressed_cache.bind_kv_cache(compressed_view)
assert raw_cache.kv_cache.shape == (2, 8, 1, 64)
assert compressed_cache.kv_cache.shape == (2, 8, 1, 64)
assert raw_cache.kv_cache.data_ptr() == raw_view.data_ptr()
assert compressed_cache.kv_cache.data_ptr() == compressed_view.data_ptr()
@pytest.mark.parametrize(
("compress_ratio", "num_spec", "expected"),
[(4, 0, 4), (4, 1, 8), (4, 3, 8), (4, 4, 8), (4, 5, 12), (2, 3, 6)],
)
def test_qsa_ring_capacity_covers_one_speculative_step(
compress_ratio: int, num_spec: int, expected: int
) -> None:
"""Capacity spans the open group plus one speculative step, in whole groups."""
spec = _qsa_key_cache(
block_size=48, compress_ratio=compress_ratio
).get_kv_cache_spec(SimpleNamespace(num_speculative_tokens=num_spec))
assert spec.block_size == expected
@requires_qsa_kernels
def test_qsa_compressed_metadata_keeps_dummy_slots_inert() -> None:
device = torch.device("cuda")
builder = QSAMetadataBuilder.__new__(QSAMetadataBuilder)
builder.compress_ratio = 4
builder.is_circular_buffer = False
builder.storage_block_size = 16
builder.token_to_req_buffer = torch.empty(8, dtype=torch.int32, device=device)
builder.slot_mapping_buffer = torch.empty(8, dtype=torch.int64, device=device)
builder.logical_positions_buffer = torch.empty(8, dtype=torch.int64, device=device)
# Simulate max_num_seqs exceeding the three live requests below.
builder.request_capacity = 8
builder.k_work_metadata_buffer = torch.empty(4, 2, dtype=torch.int32, device=device)
query_start_loc = torch.tensor([0, 3, 3, 8], dtype=torch.int32, device=device)
token_to_req = torch.tensor(
[0, 0, 0, 2, 2, 2, 2, 2], dtype=torch.int32, device=device
)
common = SimpleNamespace(
num_actual_tokens=8,
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc.cpu(),
seq_lens=torch.tensor([7, 0, 12], dtype=torch.int32, device=device),
slot_mapping=torch.full((8,), -1, dtype=torch.int64, device=device),
block_table_tensor=torch.zeros((3, 1), dtype=torch.int32, device=device),
token_to_req_indices=lambda buffer: buffer.copy_(token_to_req),
)
metadata = builder.build(0, common)
assert metadata.slot_mapping.tolist() == [-1] * 8
assert metadata.k_work_metadata.tolist() == [[0, 0], [2, 0], [2, 1], [-1, -1]]
@requires_qsa_kernels
@pytest.mark.parametrize("compress_ratio", [1, 4])
@pytest.mark.parametrize("num_reqs", [2, 3, 4, 7, 8, 9])
def test_qsa_triton_metadata_matches_pytorch(
compress_ratio: int, num_reqs: int
) -> None:
device = torch.device("cuda")
num_tokens = 8
query_start_loc = torch.tensor(
[0, 3, *([3] * (num_reqs - 2)), 8], dtype=torch.int32, device=device
)
token_to_req = torch.tensor(
[0, 0, 0, *([num_reqs - 1] * 5)],
dtype=torch.int32,
device=device,
)
block_table_rows = torch.tensor(
[
[4, -1, 8, -1, 12, -1],
[1, -1, 2, -1, 3, -1],
[7, -1, 9, -1, 11, -1],
],
dtype=torch.int32,
device=device,
)
block_table_storage = block_table_rows[
torch.arange(num_reqs, device=device) % block_table_rows.shape[0]
]
seq_lens = torch.zeros(num_reqs, dtype=torch.int32, device=device)
seq_lens[0] = 10
seq_lens[-1] = 20
common = SimpleNamespace(
num_actual_tokens=num_tokens,
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc.cpu(),
seq_lens=seq_lens,
slot_mapping=torch.tensor(
[0, 1, -1, 3, 4, -1, -1, -1], dtype=torch.int64, device=device
),
block_table_tensor=block_table_storage[:, ::2],
token_to_req_indices=lambda buffer: buffer.copy_(token_to_req),
)
def make_buffers() -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
return (
torch.empty(num_tokens, dtype=torch.int32, device=device),
torch.empty(num_tokens, dtype=torch.int64, device=device),
torch.empty(num_tokens, dtype=torch.int64, device=device),
)
max_num_work = (
(num_tokens + (compress_ratio - 1) * num_reqs) // compress_ratio
if compress_ratio != 1
else 0
)
actual_k_work = (
torch.empty(max_num_work, 2, dtype=torch.int32, device=device)
if max_num_work
else None
)
actual_buffers = make_buffers()
actual = qsa_cache.build_qsa_metadata_triton(
common,
*actual_buffers,
storage_block_size=2,
compress_ratio=compress_ratio,
k_work_metadata_buffer=actual_k_work,
request_capacity=num_reqs,
)
expected_k_work = (
torch.empty_like(actual_k_work) if actual_k_work is not None else None
)
expected_buffers = make_buffers()
expected = qsa_cache._build_qsa_metadata_torch(
common,
*expected_buffers,
storage_block_size=2,
compress_ratio=compress_ratio,
k_work_metadata_buffer=expected_k_work,
request_capacity=num_reqs,
)
for actual_tensor, expected_tensor in zip(actual, expected):
torch.testing.assert_close(actual_tensor, expected_tensor)
if actual_k_work is not None:
torch.testing.assert_close(actual_k_work, expected_k_work)
@requires_qsa_kernels
def test_qsa_fused_metadata_matches_pytorch_for_large_padded_prefill() -> None:
device = torch.device("cuda")
num_mapped_tokens = 4096
num_tokens = 4224
query_start_loc = torch.tensor(
[0, num_mapped_tokens], dtype=torch.int32, device=device
)
common = SimpleNamespace(
num_actual_tokens=num_tokens,
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc.cpu(),
seq_lens=torch.tensor(
[num_mapped_tokens + 32], dtype=torch.int32, device=device
),
block_table_tensor=torch.arange(256, dtype=torch.int32, device=device)[None],
slot_mapping=torch.tensor(
[0] * num_mapped_tokens + [-1] * (num_tokens - num_mapped_tokens),
dtype=torch.int64,
device=device,
),
token_to_req_indices=lambda buffer: buffer.zero_(),
)
def make_buffers() -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
return (
torch.empty(num_tokens, dtype=torch.int32, device=device),
torch.empty(num_tokens, dtype=torch.int64, device=device),
torch.empty(num_tokens, dtype=torch.int64, device=device),
)
max_num_work = (num_tokens + 3) // 4
actual_k_work = torch.empty(max_num_work, 2, dtype=torch.int32, device=device)
expected_k_work = torch.empty_like(actual_k_work)
actual = qsa_cache.build_qsa_metadata_triton(
common,
*make_buffers(),
storage_block_size=8,
compress_ratio=4,
k_work_metadata_buffer=actual_k_work,
)
expected = qsa_cache._build_qsa_metadata_torch(
common,
*make_buffers(),
storage_block_size=8,
compress_ratio=4,
k_work_metadata_buffer=expected_k_work,
)
for actual_tensor, expected_tensor in zip(actual, expected):
torch.testing.assert_close(actual_tensor, expected_tensor)
torch.testing.assert_close(actual_k_work, expected_k_work)
@requires_qsa_kernels
@pytest.mark.parametrize(
"num_rows",
[
pytest.param(3, id="one_tile_per_program"),
pytest.param(33, id="looped_tiles"),
],
)
def test_qsa_mqa_paged_matches_test_reference(num_rows: int) -> None:
torch.manual_seed(1)
head_dim = 128
q = torch.randn(num_rows, 4, head_dim, device="cuda", dtype=torch.bfloat16)
cache = torch.randn(40, 4, 1, head_dim, device="cuda", dtype=torch.bfloat16)
page_table = torch.randperm(40, device="cuda", dtype=torch.int32).reshape(2, 20)
request_rows = [(num_rows + 1) // 2, num_rows // 2]
token_to_req = torch.repeat_interleave(
torch.arange(2, device="cuda", dtype=torch.int32),
torch.tensor(request_rows, device="cuda"),
)
sequence_length_values = [320, 264]
sequence_lengths = torch.tensor(
sequence_length_values, device="cuda", dtype=torch.int32
)
query_positions = torch.cat(
[
torch.arange(length - rows, length, device="cuda", dtype=torch.int32)
for rows, length in zip(request_rows, sequence_length_values, strict=True)
]
)
compress_ratio = 4
visible_lengths = (query_positions + 1) // compress_ratio
actual, actual_visible_blocks = qsa_ops.qsa_mqa_paged(
q,
cache,
page_table,
token_to_req,
query_positions,
sequence_lengths,
compress_ratio=compress_ratio,
)
expected = _qsa_mqa_paged_reference(
q, cache, page_table, token_to_req, visible_lengths
)
torch.testing.assert_close(actual_visible_blocks, visible_lengths)
# Top-k is bounded by visible_blocks; later columns are intentionally unwritten.
columns = torch.arange(actual.shape[1], device=actual.device)
visible = columns[None, :] < actual_visible_blocks[:, None]
torch.testing.assert_close(actual[visible], expected[visible], rtol=1e-3, atol=1e-3)
@requires_qsa_kernels
def test_qsa_block_expansion_matches_test_reference() -> None:
blocks = torch.tensor([[0, -1], [1, 0]], device="cuda", dtype=torch.int32)
query_positions = torch.tensor([5, 10], device="cuda")
sequence_lengths = torch.tensor([6, 11], device="cuda")
token_to_req = torch.tensor([0, 1], device="cuda", dtype=torch.int32)
actual = qsa_ops.expand_qsa_block_indices_cuda(
blocks,
query_positions,
sequence_lengths,
token_to_req,
compress_ratio=4,
token_topk=8,
)
expected = _expand_qsa_indices_reference(
blocks,
query_positions,
sequence_lengths,
compress_ratio=4,
token_topk=8,
)
torch.testing.assert_close(actual, expected)
@requires_qsa_kernels
@pytest.mark.parametrize(
("num_rows", "num_query_heads", "num_kv_heads", "page_size"),
[
# Kernel-visible pages with --block-size 256 and hybrid-cache alignment.
pytest.param(1, 24, 2, 1792, id="tp1_split64"),
pytest.param(16, 12, 1, 1792, id="tp2_split32"),
pytest.param(32, 6, 1, 1024, id="tp4_split8"),
pytest.param(257, 6, 1, 1024, id="tp4_split4"),
pytest.param(513, 6, 1, 1024, id="tp4_split1"),
],
)
def test_qsa_sparse_paged_attention_matches_test_reference(
num_rows: int,
num_query_heads: int,
num_kv_heads: int,
page_size: int,
) -> None:
torch.manual_seed(2)
head_dim = 256
num_requests = 2
num_selected_pages = 64
# Keep the newest page outside the synthetic top-k as causal headroom.
num_pages_per_request = num_selected_pages + 1
num_cache_blocks = num_requests * num_pages_per_request
indexer_budget = 2048
indexer_compress_ratio = 4
selection_width = indexer_budget + indexer_compress_ratio - 1
q = torch.randn(
num_rows, num_query_heads, head_dim, device="cuda", dtype=torch.bfloat16
)
kv_cache = torch.randn(
num_cache_blocks,
page_size,
num_kv_heads,
2 * head_dim,
device="cuda",
dtype=torch.bfloat16,
)
k_cache, v_cache = kv_cache.split(head_dim, dim=-1)
block_table = (
torch.randperm(num_cache_blocks, device="cuda")
.reshape(num_requests, num_pages_per_request)
.to(torch.int32)
)
rows_per_request = math.ceil(num_rows / num_requests)
row_indices = torch.arange(num_rows, device="cuda", dtype=torch.int32)
token_to_req = row_indices // rows_per_request
request_row_counts = torch.tensor(
[rows_per_request, num_rows - rows_per_request],
device="cuda",
dtype=torch.int32,
)
context_length = num_pages_per_request * page_size - 1
block_topk = indexer_budget // indexer_compress_ratio
compressed_blocks_per_page = page_size // indexer_compress_ratio
selection = torch.arange(block_topk, device="cuda")
selected_pages = selection % num_selected_pages
selected_offsets = selection // num_selected_pages
row_shifts = 2 * row_indices.unsqueeze(1)
# Eight blocks per page; adjacent rows overlap by six of those eight.
selected_offsets = (selected_offsets + row_shifts) % compressed_blocks_per_page
block_indices = (selected_pages * compressed_blocks_per_page + selected_offsets).to(
torch.int32
)
rows_within_request = row_indices % rows_per_request
query_positions = (
context_length - request_row_counts[token_to_req.long()] + rows_within_request
).to(torch.int64)
sequence_lengths = torch.full(
(num_requests,), context_length, device="cuda", dtype=torch.int32
)
logical_indices = qsa_ops.expand_qsa_block_indices_cuda(
block_indices,
query_positions,
sequence_lengths,
token_to_req,
indexer_compress_ratio,
indexer_budget,
)
assert logical_indices.shape == (num_rows, selection_width)
scale = q.shape[-1] ** -0.5
actual = qsa_ops.qsa_sparse_paged_attention(
q,
k_cache,
v_cache,
logical_indices,
block_table,
token_to_req,
)
expected = _qsa_sparse_paged_attention_reference(
q,
k_cache,
v_cache,
logical_indices,
block_table,
token_to_req,
scale,
)
torch.testing.assert_close(actual, expected, rtol=2e-2, atol=2e-2)
@requires_qsa_kernels
def test_qsa_selection_chunks_workspace_and_matches_test_reference(
monkeypatch: pytest.MonkeyPatch,
workspace_init,
) -> None:
rows, keys, heads, head_dim = 65, 640, 4, 16
token_topk, compress_ratio = 2048, 4
torch.manual_seed(3)
q = torch.randn(rows, heads, head_dim, device="cuda", dtype=torch.bfloat16)
cache = torch.randn(40, 16, 1, head_dim, device="cuda", dtype=torch.bfloat16)
page_table = torch.randperm(40, device="cuda", dtype=torch.int32).unsqueeze(0)
token_to_req = torch.zeros(rows, device="cuda", dtype=torch.int32)
query_positions = torch.full((rows,), 2559, device="cuda", dtype=torch.int32)
sequence_lengths = torch.tensor([2560], device="cuda", dtype=torch.int32)
monkeypatch.setattr(qsa_ops, "_LOGITS_WORKSPACE_BYTES", 32 * keys * 4)
original_score = qsa_ops.qsa_mqa_paged
scored_row_counts = []
def record_score(query: torch.Tensor, *args, **kwargs):
scored_row_counts.append(query.shape[0])
return original_score(query, *args, **kwargs)
monkeypatch.setattr(qsa_ops, "qsa_mqa_paged", record_score)
actual = qsa_ops.qsa_select_paged_tokens(
q,
cache,
page_table,
token_to_req,
query_positions,
sequence_lengths,
token_topk,
compress_ratio,
)
expected = _qsa_select_paged_tokens_reference(
q,
cache,
page_table,
token_to_req,
query_positions,
sequence_lengths,
token_topk,
compress_ratio,
)
torch.testing.assert_close(actual.sort().values, expected.sort().values)
assert scored_row_counts == [32, 32, 1]
@requires_qsa_kernels
def test_qsa_selection_handles_no_complete_compressed_blocks(workspace_init) -> None:
q = torch.zeros(2, 4, 8, device="cuda", dtype=torch.bfloat16)
cache = torch.zeros(1, 16, 1, 8, device="cuda", dtype=torch.bfloat16)
page_table = torch.zeros(1, 1, device="cuda", dtype=torch.int32)
token_to_req = torch.zeros(2, device="cuda", dtype=torch.int32)
query_positions = torch.tensor([1, 2], device="cuda", dtype=torch.int32)
sequence_lengths = torch.tensor([3], device="cuda", dtype=torch.int32)
selected = qsa_ops.qsa_select_paged_tokens(
q,
cache,
page_table,
token_to_req,
query_positions,
sequence_lengths,
token_topk=2048,
compress_ratio=4,
)
assert selected[0, :2].tolist() == [0, 1]
assert selected[1, :3].tolist() == [0, 1, 2]
assert torch.all(selected[0, 2:] == -1)
assert torch.all(selected[1, 3:] == -1)
@requires_qsa_kernels
def test_qsa_streaming_compression_and_compressor_state_store_match_reference() -> None:
head_dim = 8
current_pairs = [
*((0, position) for position in range(2, 9)),
*((1, position) for position in range(5, 11)),
]
def key_row(request: int, position: int) -> torch.Tensor:
return (
torch.arange(head_dim, dtype=torch.float32) + request * 1000 + position * 10
)
def position_row(request: int, position: int) -> torch.Tensor:
return torch.tensor(
[
request * 1000 + position,
request * 1000 + position + 100,
request * 1000 + position + 200,
],
dtype=torch.int64,
)
raw_keys = (
torch.stack([key_row(request, position) for request, position in current_pairs])
.unsqueeze(1)
.to(device="cuda", dtype=torch.bfloat16)
)
raw_positions = (
torch.stack(
[position_row(request, position) for request, position in current_pairs]
)
.unsqueeze(1)
.to(device="cuda")
)
token_to_req = torch.tensor(
[request for request, _ in current_pairs],
dtype=torch.int32,
device="cuda",
)
logical_positions = torch.tensor(
[position for _, position in current_pairs],
dtype=torch.int64,
device="cuda",
)
query_start_loc = torch.tensor([0, 7, 13], dtype=torch.int32, device="cuda")
compressor_state_block_table = torch.tensor(
[[1], [0]], dtype=torch.int32, device="cuda"
)
compressor_state_cache = torch.zeros(
2, 4, 1, head_dim, dtype=torch.bfloat16, device="cuda"
)
rope_cache = torch.zeros(2, 4, 1, 3, dtype=torch.int64, device="cuda")
for request, position, block in ((0, 0, 1), (0, 1, 1), (1, 4, 0)):
compressor_state_cache[block, position % 4, 0] = key_row(request, position).to(
device="cuda", dtype=torch.bfloat16
)
rope_cache[block, position % 4, 0] = position_row(request, position).to("cuda")
compressed_slots = torch.full(
(len(current_pairs),), -1, dtype=torch.int64, device="cuda"
)
valid_rows = torch.tensor([1, 5, 9], dtype=torch.int64, device="cuda")
compressed_slots[valid_rows] = torch.arange(3, device="cuda")
pooled, first_positions = qsa_ops.qsa_compress_groups_with_ratio(
raw_keys,
raw_positions,
compressor_state_cache,
compressor_state_block_table,
token_to_req,
query_start_loc,
logical_positions,
compressed_slots,
compress_ratio=4,
rope_cache=rope_cache,
)
pooled_without_rope, scalar_first_positions = (
qsa_ops.qsa_compress_groups_with_ratio(
raw_keys,
raw_positions,
compressor_state_cache,
compressor_state_block_table,
token_to_req,
query_start_loc,
logical_positions,
compressed_slots,
compress_ratio=4,
)
)
groups = [
[(0, position) for position in range(0, 4)],
[(0, position) for position in range(4, 8)],
[(1, position) for position in range(4, 8)],
]
expected_pooled = (
torch.stack(
[
torch.stack([key_row(*pair) for pair in group]).mean(dim=0)
for group in groups
]
)
.unsqueeze(1)
.to(device="cuda", dtype=torch.bfloat16)
)
expected_positions = torch.stack(
[position_row(0, 0), position_row(0, 4), position_row(1, 4)]
).to("cuda")
expected_scalar_positions = torch.tensor(
[[0, 0, 0], [4, 4, 4], [4, 4, 4]],
dtype=torch.int64,
device="cuda",
)
torch.testing.assert_close(pooled[valid_rows], expected_pooled)
torch.testing.assert_close(pooled_without_rope[valid_rows], expected_pooled)
torch.testing.assert_close(first_positions[valid_rows], expected_positions)
torch.testing.assert_close(
scalar_first_positions[valid_rows], expected_scalar_positions
)
compressor_state_slots = torch.tensor(
[-1, -1, -1, 5, 6, 7, 4, -1, -1, 3, 0, 1, 2],
dtype=torch.int64,
device="cuda",
)
qsa_ops.qsa_store_cache_rows(
compressor_state_cache, compressor_state_slots, raw_keys
)
qsa_ops.qsa_store_cache_rows(rope_cache, compressor_state_slots, raw_positions)
for request, positions, block in ((0, range(5, 9), 1), (1, range(7, 11), 0)):
for position in positions:
torch.testing.assert_close(
compressor_state_cache[block, position % 4, 0],
key_row(request, position).to(device="cuda", dtype=torch.bfloat16),
)
torch.testing.assert_close(
rope_cache[block, position % 4, 0],
position_row(request, position).to("cuda"),
)
+6
View File
@@ -508,6 +508,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
extras={"native-prefix": "imdatta0/small_qwen3_5_20b"},
max_model_len=4096,
),
"Qwen4ExpForCausalLM": _HfExamplesInfo("", is_available_online=False),
"MellumForCausalLM": _HfExamplesInfo("JetBrains/Mellum2-12B-A2.5B-Base"),
"Qwen3NextForCausalLM": _HfExamplesInfo(
"Qwen/Qwen3-Next-80B-A3B-Instruct",
@@ -1348,6 +1349,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"Qwen/Qwen3.5-35B-A3B",
max_model_len=4096,
),
"Qwen4ExpForConditionalGeneration": _HfExamplesInfo(
"",
is_available_online=False,
),
"Qwen3OmniMoeForConditionalGeneration": _HfExamplesInfo(
"Qwen/Qwen3-Omni-30B-A3B-Instruct",
max_model_len=4096,
@@ -1797,6 +1802,7 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
"Qwen/Qwen3.5-35B-A3B",
speculative_model="Qwen/Qwen3.5-35B-A3B",
),
"Qwen4ExpMTP": _HfExamplesInfo("", is_available_online=False),
"Step3p5MTP": _HfExamplesInfo(
"stepfun-ai/Step-3.5-Flash",
speculative_model="stepfun-ai/Step-3.5-Flash",
+44
View File
@@ -6,6 +6,7 @@ Run `pytest tests/quantization/test_fp8.py --forked`.
"""
import logging
from types import SimpleNamespace
import pytest
import regex as re
@@ -31,6 +32,10 @@ from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
from vllm.model_executor.layers.quantization.online.fp8 import (
Fp8PerTensorOnlineLinearMethod,
)
from vllm.model_executor.layers.quantization.utils import flashinfer_utils
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
prepare_fp8_moe_layer_for_fi,
)
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
process_fp8_input_tensor_strategy_moe,
)
@@ -50,6 +55,45 @@ MODELS = [
]
def test_prepare_gated_trtllm_fp8_moe_weights_pads_each_projection(monkeypatch):
monkeypatch.setattr(
flashinfer_utils,
"rotate_weights_for_fi_trtllm_fp8_per_tensor_moe",
lambda *args: None,
)
intermediate = 17
padded_intermediate = 32
hidden_size = 4
gate = torch.ones((1, intermediate, hidden_size), dtype=torch.float8_e4m3fn)
up = torch.full_like(gate, 2)
w13 = torch.cat((gate, up), dim=1)
w2 = torch.ones((1, hidden_size, intermediate), dtype=torch.float8_e4m3fn)
layer = SimpleNamespace(
activation=SimpleNamespace(is_gated=True),
moe_config=SimpleNamespace(
is_act_and_mul=True,
intermediate_size_per_partition=intermediate,
),
)
padded_w31, _, _, _ = prepare_fp8_moe_layer_for_fi(
layer,
w13,
w2,
w13_scale=torch.ones(1),
w13_input_scale=torch.ones(1),
w2_scale=torch.ones(1),
w2_input_scale=torch.ones(1),
is_trtllm=True,
)
expected = w13.new_zeros((1, 2 * padded_intermediate, hidden_size))
expected[:, :intermediate] = up
expected[:, padded_intermediate : padded_intermediate + intermediate] = gate
assert layer.moe_config.intermediate_size_per_partition == padded_intermediate
assert torch.equal(padded_w31, expected)
def test_static_fp8_moe_input_scales_remain_scalar() -> None:
a1_scale, a2_scale = process_fp8_input_tensor_strategy_moe(
torch.tensor([0.25, 0.5]),
+86
View File
@@ -16,6 +16,7 @@ from vllm.model_executor.layers.fused_moe.config import (
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
Mxfp4MoeBackend,
_requires_qwen38_tep8_emulation,
select_mxfp4_moe_backend,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
@@ -72,6 +73,91 @@ def _make_w4a4_moe_config(moe_backend: str = "auto") -> FusedMoEConfig:
)
def _make_qwen38_tep8_moe_config() -> FusedMoEConfig:
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
return FusedMoEConfig(
num_experts=512,
experts_per_token=9,
hidden_dim=2560,
intermediate_size=640,
num_local_experts=64,
num_logical_experts=512,
moe_parallel_config=FusedMoEParallelConfig(
tp_size=1,
tp_rank=0,
pcp_size=1,
pcp_rank=0,
dp_size=1,
dp_rank=0,
ep_size=8,
ep_rank=0,
sp_size=1,
use_ep=True,
all2all_backend="allgather_reducescatter",
enable_eplb=False,
),
activation=MoEActivation.SILU,
in_dtype=torch.bfloat16,
device="cuda",
routing_method=RoutingMethodType.Renormalize,
)
def test_qwen38_tep8_requires_emulation_only_on_gfx950(monkeypatch):
import vllm.model_executor.layers.fused_moe.oracle.mxfp4 as mxfp4_oracle
config = _make_qwen38_tep8_moe_config()
monkeypatch.setattr(current_platform, "is_rocm", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: True)
assert _requires_qwen38_tep8_emulation(config, kMxfp4Dynamic)
config.moe_parallel_config.ep_size = 4
config.num_local_experts = 128
assert not _requires_qwen38_tep8_emulation(config, kMxfp4Dynamic)
config = _make_qwen38_tep8_moe_config()
monkeypatch.setattr(mxfp4_oracle.current_platform, "is_rocm", lambda: False)
assert not _requires_qwen38_tep8_emulation(config, kMxfp4Dynamic)
@pytest.mark.parametrize(
"requested_backend,expected_backend",
[
("auto", Mxfp4MoeBackend.EMULATION),
("aiter", Mxfp4MoeBackend.AITER_MXFP4_MXFP4),
],
)
def test_qwen38_tep8_auto_fallback_respects_explicit_backend(
requested_backend,
expected_backend,
monkeypatch,
):
import vllm.model_executor.layers.fused_moe.oracle.mxfp4 as mxfp4_oracle
class SupportedExperts:
@staticmethod
def is_supported_config(*args, **kwargs):
return True, None
config = _make_qwen38_tep8_moe_config()
config.moe_backend = requested_backend
monkeypatch.setattr(current_platform, "is_rocm", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: True)
monkeypatch.setattr(mxfp4_oracle, "_user_moe_activation_override", lambda: None)
monkeypatch.setattr(
mxfp4_oracle, "backend_to_kernel_cls", lambda backend: [SupportedExperts]
)
backend, experts_cls = select_mxfp4_moe_backend(
config, activation_key=kMxfp4Dynamic
)
assert backend == expected_backend
assert experts_cls is SupportedExperts
@pytest.fixture
def mxfp4_oracle_config():
"""Stub the config the oracle reads (``model_config.quantization_config``)
+21
View File
@@ -604,3 +604,24 @@ def test_online_mxfp4_dense_matches_quark(
keys=("weight", "weight_scale"),
packed_weight_keys=("weight",),
)
def test_quark_ocp_mx_dense_rejects_unaligned_input_partition():
scheme = object.__new__(QuarkOCP_MX)
layer = torch.nn.Module()
layer.prefix = "model.layers.0.mlp.shared_expert.down_proj"
with pytest.raises(
ValueError,
match=(
"input size per partition of 80, which must be divisible by "
"the OCP MX group size 32"
),
):
scheme.create_weights(
layer=layer,
output_partition_sizes=[2560],
input_size_per_partition=80,
params_dtype=torch.bfloat16,
weight_loader=lambda *_: None,
)
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
prepare_nvfp4_moe_layer_for_fi_or_cutlass,
)
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
align_fp4_moe_weights_for_fi,
align_trtllm_fp4_moe_hidden_dim_for_fi,
)
@@ -107,3 +108,53 @@ def test_align_trtllm_fp4_moe_hidden_dim_pads_to_256_multiple():
assert torch.count_nonzero(out_w13_scale[:, :, hidden_dim // 16 :]) == 0
assert torch.count_nonzero(out_w2[:, hidden_dim:, :]) == 0
assert torch.count_nonzero(out_w2_scale[:, hidden_dim:, :]) == 0
def test_align_trtllm_fp4_moe_intermediate_pads_gate_and_up_separately():
# Qwen3.8-Flash-Next has intermediate_size=640. At TP4 each rank owns
# 160 values, which TRTLLM pads to the next 64-value boundary (192).
intermediate = 160
padded_intermediate = 192
hidden_dim = 32
gate = torch.ones((1, intermediate, hidden_dim // 2), dtype=torch.uint8)
up = torch.full_like(gate, 2)
w13 = torch.cat((gate, up), dim=1)
gate_scale = torch.full((1, intermediate, hidden_dim // 16), 3, dtype=torch.uint8)
up_scale = torch.full_like(gate_scale, 4)
w13_scale = torch.cat((gate_scale, up_scale), dim=1)
w2 = torch.full((1, hidden_dim, intermediate // 2), 5, dtype=torch.uint8)
w2_scale = torch.full((1, hidden_dim, intermediate // 16), 6, dtype=torch.uint8)
out_w13, out_w13_scale, out_w2, out_w2_scale, out_intermediate = (
align_fp4_moe_weights_for_fi(
w13,
w13_scale,
w2,
w2_scale,
is_act_and_mul=True,
min_alignment=64,
)
)
assert out_intermediate == padded_intermediate
torch.testing.assert_close(out_w13[:, :intermediate], gate)
torch.testing.assert_close(
out_w13[:, padded_intermediate : padded_intermediate + intermediate], up
)
torch.testing.assert_close(out_w13_scale[:, :intermediate], gate_scale)
torch.testing.assert_close(
out_w13_scale[:, padded_intermediate : padded_intermediate + intermediate],
up_scale,
)
assert torch.count_nonzero(out_w13[:, intermediate:padded_intermediate]) == 0
assert torch.count_nonzero(out_w13[:, padded_intermediate + intermediate :]) == 0
assert torch.count_nonzero(out_w13_scale[:, intermediate:padded_intermediate]) == 0
assert (
torch.count_nonzero(out_w13_scale[:, padded_intermediate + intermediate :]) == 0
)
torch.testing.assert_close(out_w2[:, :, : intermediate // 2], w2)
torch.testing.assert_close(out_w2_scale[:, :, : intermediate // 16], w2_scale)
assert torch.count_nonzero(out_w2[:, :, intermediate // 2 :]) == 0
assert torch.count_nonzero(out_w2_scale[:, :, intermediate // 16 :]) == 0
+312
View File
@@ -9,23 +9,34 @@ block dim (a contiguous region per layer) or inside it (all layers' pages within
block); the allocation is the same either way.
"""
from dataclasses import replace
from unittest.mock import MagicMock
import pytest
import torch
from vllm.config import CacheConfig
from vllm.v1.core.kv_cache_manager import KVCacheManager
from vllm.v1.core.kv_cache_utils import (
_get_kv_cache_bytes_per_block,
_get_packed_kv_cache_groups,
_pool_bytes_per_block,
generate_scheduler_kv_cache_config,
get_kv_cache_config_from_groups,
get_kv_cache_groups,
resolve_kv_cache_block_sizes,
)
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
FullAttentionSpec,
KVCacheGroupSpec,
KVCacheLayout,
KVCacheSpec,
MambaSpec,
MLAAttentionSpec,
SlidingWindowMLASpec,
UniformTypeKVCacheSpecs,
iter_layer_specs,
)
from vllm.v1.worker.utils import allocate_kv_cache
@@ -86,6 +97,307 @@ def _bind(config, layout: str):
return allocate_kv_cache(config, torch.device("cpu"), KVCacheLayout[layout], None)
MAIN_KV_PAGE_BYTES = 2_048
COMPRESSED_PAGE_BYTES = 128
NUM_CACHE_TUPLES = 3
BYTES_PER_BLOCK = NUM_CACHE_TUPLES * (MAIN_KV_PAGE_BYTES + COMPRESSED_PAGE_BYTES)
def _main_kv_name(layer_index: int) -> str:
return f"model.layers.{layer_index}.self_attn"
def _compressed_name(layer_index: int) -> str:
return f"model.layers.{layer_index}.self_attn.indexer.compressed_key_cache"
def _compressor_state_name(layer_index: int) -> str:
return f"model.layers.{layer_index}.self_attn.indexer.raw_key_cache"
def _make_csa_linear_specs(
num_mamba: int = 7,
num_tuples: int = NUM_CACHE_TUPLES,
) -> dict[str, KVCacheSpec]:
specs: dict[str, KVCacheSpec] = {}
for layer_index in range(num_mamba):
specs[f"model.layers.{layer_index}.linear_attn"] = MambaSpec(
block_size=16,
shapes=((32,),),
dtypes=(torch.bfloat16,),
)
specs["model.layers.0.ple"] = MambaSpec(
block_size=16,
shapes=((24,),),
dtypes=(torch.bfloat16,),
tp_replicated=True,
)
for layer_index in range(num_tuples):
specs[_main_kv_name(layer_index)] = FullAttentionSpec(
block_size=16,
num_kv_heads=2,
head_size=16,
head_size_v=16,
dtype=torch.bfloat16,
)
specs[_compressed_name(layer_index)] = MLAAttentionSpec(
block_size=16,
num_kv_heads=1,
head_size=16,
dtype=torch.bfloat16,
tokens_per_state=4,
)
specs[_compressor_state_name(layer_index)] = CircularBufferSpec(
block_size=4,
num_kv_heads=1,
head_size=8,
head_size_v=0,
dtype=torch.bfloat16,
)
return specs
def _shared_layout_config():
config = _mock_vllm_config("BLNHC")
config.scheduler_config.disable_hybrid_kv_cache_manager = False
config.speculative_config = None
config.model_config.max_model_len = 16
config.model_config.get_total_num_hidden_layers.return_value = 64
config.model_config.get_total_num_kv_heads.return_value = 2
config.model_config.get_num_kv_heads.return_value = 2
config.parallel_config.pipeline_parallel_size = 1
config.parallel_config.decode_context_parallel_size = 1
config.cache_config.block_size = 16
config.cache_config.enable_prefix_caching = False
config.cache_config.prefix_match_unit = None
config.cache_config.mamba_cache_mode = "none"
return config
class TestCSALinearGrouping:
"""A CSA + linear-attention model (sparse attention with a compressor ring,
plus sharded GDN and one TP-replicated PLE state) goes through the generic
packed-group path; no model-specific branch is involved."""
@staticmethod
def _mamba_groups(groups):
"""(group, per-layer mamba spec) for every group holding mamba state."""
out = []
for group in groups:
spec = next(iter(iter_layer_specs(group.kv_cache_spec)))
if isinstance(spec, MambaSpec):
out.append((group, spec))
return out
def test_replicated_state_gets_its_own_group(self):
"""The PLE state is TP-replicated and a different size, so it must not
share a manager group with the sharded GDN states (the NIXL worker
requires exactly one single-layer replicated group)."""
groups = get_kv_cache_groups(_shared_layout_config(), _make_csa_linear_specs())
mamba_groups = self._mamba_groups(groups)
replicated = [g for g, spec in mamba_groups if spec.tp_replicated]
assert len(replicated) == 1
assert replicated[0].layer_names == ["model.layers.0.ple"]
sharded = [g for g, spec in mamba_groups if not spec.tp_replicated]
assert sharded, "GDN states must keep their own groups"
assert sorted(n for g in sharded for n in g.layer_names) == sorted(
f"model.layers.{i}.linear_attn" for i in range(7)
)
def test_roles_land_in_separate_groups(self):
groups = get_kv_cache_groups(_shared_layout_config(), _make_csa_linear_specs())
owner = next(g for g in groups if _main_kv_name(0) in g.layer_names)
assert sorted(owner.layer_names) == sorted(
[
*(_main_kv_name(i) for i in range(NUM_CACHE_TUPLES)),
*(_compressed_name(i) for i in range(NUM_CACHE_TUPLES)),
]
)
assert owner.kv_cache_spec.prefix_cacheable
scratch = next(g for g in groups if _compressor_state_name(0) in g.layer_names)
assert scratch.layer_names == [
_compressor_state_name(i) for i in range(NUM_CACHE_TUPLES)
]
assert not scratch.kv_cache_spec.prefix_cacheable
def test_every_group_fits_one_packed_block(self):
config = _shared_layout_config()
groups = get_kv_cache_groups(config, _make_csa_linear_specs())
bytes_per_block = _get_kv_cache_bytes_per_block(groups)
pages = _pages(groups)
for group in groups:
assert sum(pages[n] for n in group.layer_names) <= bytes_per_block
kv_cache_config = get_kv_cache_config_from_groups(
config, groups, available_memory=bytes_per_block * 32
)
assert kv_cache_config.num_blocks == 32
# Groups overlay from byte 0 of each block, so a layer never addresses
# past the block it belongs to.
for tensor in kv_cache_config.kv_cache_tensors:
assert tensor.block_stride == bytes_per_block
assert tensor.offset < bytes_per_block
def test_scratch_group_survives_computed_block_truncation(self):
"""The scratch group contributes no computed blocks, so truncating a
lookup result must skip it: its block size is the ring capacity, which
neither divides the hit length nor bounds the (empty) block list."""
config = _shared_layout_config()
config.cache_config.enable_prefix_caching = True
groups = get_kv_cache_groups(config, _make_csa_linear_specs())
kv_cache_config = get_kv_cache_config_from_groups(
config, groups, available_memory=BYTES_PER_BLOCK * 64
)
scheduler_config = generate_scheduler_kv_cache_config([kv_cache_config])
manager = KVCacheManager(
scheduler_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=16,
scheduler_block_size=16,
)
blocks = manager.create_kv_cache_blocks(
tuple(
manager.block_pool.get_new_blocks(3)
if group.kv_cache_spec.prefix_cacheable
else []
for group in scheduler_config.kv_cache_groups
)
)
truncated = manager.truncate_computed_blocks(blocks, 48)
scratch_index = next(
i
for i, group in enumerate(scheduler_config.kv_cache_groups)
if not group.kv_cache_spec.prefix_cacheable
)
assert truncated.blocks[scratch_index] == []
def test_prefix_hits_respect_compression_alignment(self):
config = _shared_layout_config()
config.cache_config.enable_prefix_caching = True
config.cache_config.prefix_match_unit = 16
config.cache_config.mamba_cache_mode = "align"
specs = {
name: replace(spec, mamba_cache_mode="align")
if isinstance(spec, MambaSpec)
else spec
for name, spec in _make_csa_linear_specs(num_tuples=1).items()
}
groups = get_kv_cache_groups(config, specs)
kv_cache_config = get_kv_cache_config_from_groups(
config, groups, available_memory=8 * MAIN_KV_PAGE_BYTES
)
assert resolve_kv_cache_block_sizes(kv_cache_config, config) == (16, 16)
# 2 tokens is not a multiple of the compression ratio 4: a prefix hit
# could land inside a partially filled compressed state.
config.cache_config.prefix_match_unit = 2
with pytest.raises(ValueError, match="per-state compression"):
resolve_kv_cache_block_sizes(kv_cache_config, config)
def test_scratch_ring_does_not_drag_hash_granularity(self):
config = _shared_layout_config()
config.cache_config.enable_prefix_caching = True
config.cache_config.mamba_cache_mode = "align"
specs = {
name: replace(spec, mamba_cache_mode="align")
if isinstance(spec, MambaSpec)
else spec
for name, spec in _make_csa_linear_specs(num_tuples=1).items()
}
groups = get_kv_cache_groups(config, specs)
kv_cache_config = get_kv_cache_config_from_groups(
config, groups, available_memory=8 * MAIN_KV_PAGE_BYTES
)
# The hash granularity is the GCD over prefix-cacheable groups only;
# the 4-token scratch ring is excluded (it would drag it to 4).
assert resolve_kv_cache_block_sizes(kv_cache_config, config) == (16, 16)
def test_compressed_attention_hashes_can_be_finer_than_cache_hits(self):
config = _shared_layout_config()
config.cache_config.enable_prefix_caching = True
specs = {
"compressed.4": MLAAttentionSpec(
block_size=256,
num_kv_heads=1,
head_size=16,
dtype=torch.bfloat16,
tokens_per_state=4,
),
"compressed.128": MLAAttentionSpec(
block_size=256,
num_kv_heads=1,
head_size=16,
dtype=torch.bfloat16,
tokens_per_state=128,
),
"compressor_state.4": SlidingWindowMLASpec(
block_size=4,
num_kv_heads=1,
head_size=8,
head_size_v=0,
dtype=torch.bfloat16,
sliding_window=8,
),
}
groups = get_kv_cache_groups(config, specs)
kv_cache_config = get_kv_cache_config_from_groups(
config, groups, available_memory=8 * MAIN_KV_PAGE_BYTES
)
# Hashes are computed every 4 tokens, but without an align-mode Mamba
# group cache hits remain on the 256-token scheduler boundary.
assert resolve_kv_cache_block_sizes(kv_cache_config, config) == (256, 4)
@pytest.mark.parametrize("wide", ["unbalanced_attention", "unsplittable_state"])
def test_mamba_split_measures_the_block_the_other_groups_already_force(self, wide):
"""Whatever fixes the block stride -- a bucket with unequal layer counts
per page size, which is emitted whole, or a single-layer state that
cannot be split at all -- the mamba layers must be sized against it.
Sizing them against a narrower bucket splits them past what the block
already fits, spending a pool block per extra group for no saving."""
config = _mock_vllm_config("BLNHC")
config.speculative_config = None
specs = {}
if wide == "unbalanced_attention":
for i in range(3):
specs[f"wide.{i}"] = _mla(1024)
specs["wide.odd"] = _mla(800)
else:
specs["wide.state"] = MambaSpec(
block_size=16,
shapes=((51_200,),),
dtypes=(torch.bfloat16,),
tp_replicated=True,
)
# Mixed and balanced, but far narrower than the bucket above.
for i in range(10):
specs[f"narrow.a.{i}"] = FullAttentionSpec(
block_size=16, num_kv_heads=1, head_size=8, dtype=torch.uint8
)
specs[f"narrow.b.{i}"] = FullAttentionSpec(
block_size=16, num_kv_heads=1, head_size=16, dtype=torch.uint8
)
for i in range(100):
specs[f"gdn.{i}"] = MambaSpec(
block_size=16, shapes=((512,),), dtypes=(torch.bfloat16,)
)
groups = _get_packed_kv_cache_groups(config, specs)
gdn = [g for g in groups if g.layer_names[0].startswith("gdn.")]
assert _get_kv_cache_bytes_per_block(groups) == sum(
specs[name].page_size_bytes for name in specs if name.startswith("wide.")
)
# That block holds every GDN state at once, so the repeat pattern alone
# decides the split; the cap must not add groups on top of it.
assert len(gdn) == 10
class TestDensePacking:
def test_bytes_per_block_is_largest_group(self):
groups, g1, g2 = _mixed_page_groups()
+39 -48
View File
@@ -12,7 +12,7 @@ import pytest
import torch
import vllm.v1.core.kv_cache_utils as kv_cache_utils
from vllm.config import ModelConfig, SchedulerConfig, VllmConfig
from vllm.config import CacheConfig, ModelConfig, SchedulerConfig, VllmConfig
from vllm.config.kv_events import KVEventsConfig
from vllm.lora.request import LoRARequest
from vllm.multimodal.inputs import (
@@ -37,7 +37,6 @@ from vllm.v1.core.kv_cache_utils import (
get_kv_cache_groups,
get_max_concurrency_for_kv_cache_config,
get_request_block_hasher,
group_and_unify_kv_cache_specs,
hash_block_tokens,
init_none_hash,
is_kv_cache_spec_uniform,
@@ -2226,38 +2225,6 @@ def new_swa_mla_spec(head_size=576, sliding_window=128, model_version=None):
)
def test_group_and_unify_kv_cache_specs_no_swa_mla_returns_none():
# Without any SlidingWindowMLASpec the function does not apply.
specs = {"mla.0": new_mla_spec(), "mla.1": new_mla_spec()}
assert group_and_unify_kv_cache_specs(specs) is None
def test_group_and_unify_kv_cache_specs_uniform_page_size_returns_none():
# A non-DeepseekV4 model that mixes full MLA and sliding-window MLA layers
# with a uniform page size must not fall into the DeepseekV4 tuple-packing
# path; it should defer to the generic uniform-page-size grouping instead.
mla_spec = new_mla_spec()
swa_spec = new_swa_mla_spec()
assert mla_spec.page_size_bytes == swa_spec.page_size_bytes
specs = {"mla.0": mla_spec, "mla.1": new_mla_spec(), "swa.0": swa_spec}
assert group_and_unify_kv_cache_specs(specs) is None
def test_group_and_unify_kv_cache_specs_mixed_page_size_groups():
# DeepseekV4-style: differing page sizes across MLA and sliding-window MLA
# layers do require tuple packing, so grouping must still be produced.
mla_spec = new_mla_spec()
swa_spec = new_swa_mla_spec(head_size=1024)
assert mla_spec.page_size_bytes != swa_spec.page_size_bytes
specs = {"mla.0": mla_spec, "mla.1": new_mla_spec(), "swa.0": swa_spec}
grouped = group_and_unify_kv_cache_specs(specs)
assert grouped is not None
# One MLA group plus one sliding-window MLA group.
assert len(grouped) == 2
layer_names = {name for g in grouped for name in g.kv_cache_specs}
assert layer_names == {"mla.0", "mla.1", "swa.0"}
def new_indexer_mla_spec(block_size=16):
# Sparse-attention indexer k_cache: an MLAAttentionSpec with a much smaller
# page size than the main MLA attention (uint8, small head), so their pages
@@ -2270,10 +2237,30 @@ def new_indexer_mla_spec(block_size=16):
)
def test_mixed_page_size_groups_use_spec_compatibility():
specs = {}
for i in range(3):
specs[f"mla.{i}"] = new_mla_spec()
specs[f"indexer.{i}"] = new_indexer_mla_spec()
specs.update({f"swa.{i}": new_swa_mla_spec(head_size=1024) for i in range(5)})
config = _grouping_config()
config.cache_config = CacheConfig()
config.cache_config.kv_cache_layout = "BLNHC"
groups = get_kv_cache_groups(config, specs)
assert len(groups) == 3
assert {name for group in groups for name in group.layer_names} == set(specs)
assert sorted(len(group.layer_names) for group in groups) == [2, 3, 6]
def _grouping_config():
cache_config = CacheConfig()
cache_config.kv_cache_layout = "LBNHC"
return SimpleNamespace(
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
speculative_config=None,
cache_config=cache_config,
)
@@ -3281,10 +3268,16 @@ def test_iter_layer_specs_returns_group_members():
assert list(iter_layer_specs(wrapped)) == [full, mla]
def _spec_decode_grouping_config(method="dspark"):
def _spec_decode_grouping_config(method="dspark", model_type=None):
"""Grouping config with an EAGLE-family speculative method enabled."""
return SimpleNamespace(
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
cache_config=SimpleNamespace(
get_resolved_kv_cache_layout=lambda: SimpleNamespace(
is_block_outermost=True
)
),
model_config=SimpleNamespace(hf_config=SimpleNamespace(model_type=model_type)),
speculative_config=SimpleNamespace(
method=method,
use_eagle=lambda: True,
@@ -3322,13 +3315,12 @@ def test_draft_group_annotated_on_hybrid_general_path():
flagged = [g for g in groups if g.is_eagle_group]
assert len(flagged) == 1
assert flagged[0].layer_names == ["draft.attn.0"]
assert "draft.attn.0" in flagged[0].layer_names
def test_mamba_groups_never_flagged_even_when_draft_shares_a_group():
# MLAAttentionSpec.merge ORs non_causal_multi_token_decode rather than
# requiring equality, so a draft layer can share a group with target
# layers; that combined group still holds volatile draft KV and must be
# Packed uniform-type groups can contain distinct target and draft layer
# specs; the combined group still holds volatile draft KV and must be
# flagged. What must never happen is a Mamba group being flagged: that
# widens its lookup window to two consecutive chunks, which align-mode
# checkpointing never produces, zeroing every lookup.
@@ -3347,10 +3339,8 @@ def test_mamba_groups_never_flagged_even_when_draft_shares_a_group():
def test_draft_group_not_annotated_without_spec_decode():
# The marker alone must not flag anything; the eagle semantics only apply
# when a speculative method is actually enabled.
config = SimpleNamespace(
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
speculative_config=None,
)
config = _spec_decode_grouping_config()
config.speculative_config = None
groups = get_kv_cache_groups(config, _hybrid_specs_with_draft(draft=True))
assert not any(g.is_eagle_group for g in groups)
@@ -3396,12 +3386,13 @@ def _deepseek_v4_specs(model_version="deepseek_v4"):
}
def test_deepseek_v4_draft_group_annotated_on_group_and_unify_path():
def test_deepseek_v4_draft_group_annotated_on_packed_path():
# DeepseekV4's MTP block reuses the target's decoder layer, so its spec
# carries no draft marker and only the positional rule can find it. This
# pins the pre-existing behaviour that the unified annotator must preserve.
groups = get_kv_cache_groups(
_spec_decode_grouping_config(method="mtp"), _deepseek_v4_specs()
_spec_decode_grouping_config(method="mtp", model_type="deepseek_v4"),
_deepseek_v4_specs(model_version=None),
)
flagged = [g for g in groups if g.is_eagle_group]
@@ -3409,13 +3400,13 @@ def test_deepseek_v4_draft_group_annotated_on_group_and_unify_path():
assert "model.layers.3.self_attn.attn" in flagged[0].layer_names
def test_deepseek_v4_annotation_requires_model_version():
def test_deepseek_v4_annotation_requires_model_type():
# The positional rule is only sound for DeepseekV4, where the draft layer
# is known to be registered last. Without that model gate nothing may be
# flagged, however the grouping happens to fall out.
groups = get_kv_cache_groups(
_spec_decode_grouping_config(method="mtp"),
_deepseek_v4_specs(model_version=None),
_spec_decode_grouping_config(method="mtp", model_type="other"),
_deepseek_v4_specs(),
)
assert not any(g.is_eagle_group for g in groups)
+46
View File
@@ -1761,6 +1761,52 @@ def test_spec_decode_padding_skipped_for_diffusion():
assert r2.request_id not in out.scheduled_spec_decode_tokens
@pytest.mark.parametrize("aligned_tokens", [1, 2, 3])
def test_spec_decode_padding_dropped_when_recurrent_alignment_clips(
aligned_tokens: int,
):
"""A clipped padded request must drop its padding, not shorten it.
The sampler derives a request's row window from its draft count, so a
padded request that keeps fewer than 1 + num_spec query rows would select
the preceding request's hidden states.
"""
num_spec = 3
scheduler = create_scheduler(
num_speculative_tokens=num_spec,
enable_prefix_caching=True,
block_size=16,
)
r1, r2 = create_requests(
num_requests=2, num_tokens=33, same_prompt=True, max_tokens=16
)
scheduler.add_request(r1)
out = scheduler.schedule()
assert out.num_scheduled_tokens[r1.request_id] == 33
_model_output(scheduler, out, [[100]])
scheduler.update_draft_token_ids(DraftTokenIds([r1.request_id], [[1, 2, 3]]))
# Model the recurrent align split that clips a padded 1+3 prompt tail at
# the next cache boundary. The real QSA block size produces all three cases
# depending on prompt length modulo four.
scheduler.need_mamba_block_aligned_split = True
original_split = scheduler._mamba_block_aligned_split
def align_prompt_tail(request, num_new_tokens, *args):
if request is r2:
return aligned_tokens
return original_split(request, num_new_tokens, *args)
scheduler._mamba_block_aligned_split = align_prompt_tail
scheduler.add_request(r2)
out = scheduler.schedule()
assert out.scheduled_spec_decode_tokens[r1.request_id] == [1, 2, 3]
assert out.num_scheduled_tokens[r2.request_id] == 1
assert r2.request_id not in out.scheduled_spec_decode_tokens
def test_spec_decode_padding_skipped_with_prefill_in_batch():
"""Padding is skipped when the batch contains a prefill chunk: the batch is
already mixed/non-uniform, so padding a new decode request buys nothing.
@@ -14,6 +14,7 @@ from vllm.v1.core.kv_cache_utils import (
)
from vllm.v1.core.single_type_kv_cache_manager import (
ChunkedLocalAttentionManager,
CircularBufferManager,
FullAttentionManager,
MambaManager,
RSWAManager,
@@ -21,6 +22,7 @@ from vllm.v1.core.single_type_kv_cache_manager import (
)
from vllm.v1.kv_cache_interface import (
ChunkedLocalAttentionSpec,
CircularBufferSpec,
FullAttentionSpec,
MambaSpec,
RSWASpec,
@@ -136,6 +138,67 @@ def get_chunked_local_attention_manager(
)
def test_circular_buffer_allocates_one_block_for_the_request_lifetime():
block_size = 4
spec = CircularBufferSpec(
block_size=block_size,
num_kv_heads=1,
head_size=8,
dtype=torch.bfloat16,
)
block_pool = BlockPool(
num_gpu_blocks=10, enable_caching=True, hash_block_size=block_size
)
manager = CircularBufferManager(
spec,
block_pool=block_pool,
enable_caching=True,
kv_cache_group_id=0,
scheduler_block_size=block_size,
needs_kv_cache_zeroing=True,
max_admission_blocks_per_request=1,
)
request_id = "request"
assert (
manager.get_num_blocks_to_allocate(
request_id,
num_tokens=128,
new_computed_blocks=[block_pool.blocks[1]],
total_computed_tokens=0,
num_local_computed_tokens=0,
num_tokens_main_model=128,
)
== 1
)
blocks = manager.allocate_new_blocks(
request_id, num_tokens=128, num_tokens_main_model=128
)
assert len(blocks) == 1
assert not manager.records_new_block_ids
assert manager.take_new_block_ids() == []
for num_tokens in (256, 1024):
assert (
manager.get_num_blocks_to_allocate(
request_id,
num_tokens,
[],
total_computed_tokens=num_tokens - 1,
num_local_computed_tokens=0,
num_tokens_main_model=num_tokens,
)
== 0
)
assert manager.allocate_new_blocks(request_id, num_tokens, num_tokens) == []
manager.cache_blocks(request_id, 1024)
manager.remove_skipped_blocks(request_id, 1024)
assert manager.get_num_common_prefix_blocks(request_id) == 0
assert manager.get_num_skipped_tokens(1024) == 0
assert manager.req_to_blocks[request_id] == blocks
def test_sliding_window_records_new_blocks_for_zeroing():
block_size = 2
spec = SlidingWindowSpec(
@@ -234,7 +234,8 @@ def get_fake_execute_model_fn(original_execute_model_fn: Callable):
iter(scheduler_output.num_scheduled_tokens.values())
)
assert num_scheduled_tokens == cur_step_action.num_scheduled_tokens
mamba_group_ids, mamba_spec = get_mamba_groups(self.kv_cache_config)
mamba_groups = get_mamba_groups(self.kv_cache_config)
mamba_spec, mamba_group_ids = next(iter(mamba_groups.items()))
mamba_group_id = mamba_group_ids[0]
mamba_layer_name = self.kv_cache_config.kv_cache_groups[
mamba_group_id
@@ -321,7 +322,8 @@ def get_fake_process_mamba_fn(
assert len(copy_info[0]) == len(copy_info[1]) == len(copy_info[2]) == 0
else:
assert len(copy_info[0]) == len(copy_info[1]) == len(copy_info[2]) == 2
mamba_group_ids, mamba_spec = get_mamba_groups(kv_cache_config)
mamba_groups = get_mamba_groups(kv_cache_config)
mamba_spec, mamba_group_ids = next(iter(mamba_groups.items()))
mamba_group_id = mamba_group_ids[0]
mamba_layer_name = kv_cache_config.kv_cache_groups[
mamba_group_id
@@ -936,12 +938,12 @@ def _run_mamba_prefix_cache_mrv1(
cleanup_dist_env_and_memory()
@create_new_process_for_each_test()
@create_new_process_for_each_test("spawn")
def test_mamba_prefix_cache_mrv1(monkeypatch: pytest.MonkeyPatch):
_run_mamba_prefix_cache_mrv1(monkeypatch, async_scheduling=False)
@create_new_process_for_each_test()
@create_new_process_for_each_test("spawn")
def test_mamba_prefix_cache_mrv1_async(monkeypatch: pytest.MonkeyPatch):
_run_mamba_prefix_cache_mrv1(monkeypatch, async_scheduling=True)
@@ -975,11 +977,12 @@ def _run_mamba_prefix_cache_mrv2(
forward_context = (
model_state.vllm_config.compilation_config.static_forward_context
)
group_ids, _ = get_mamba_groups(kv_cache_config)
for group_id in group_ids:
block_table = block_tables[group_id]
for layer_name in kv_cache_config.kv_cache_groups[group_id].layer_names:
yield forward_context[layer_name].kv_cache[-1], block_table
mamba_groups = get_mamba_groups(kv_cache_config)
for group_ids in mamba_groups.values():
for group_id in group_ids:
block_table = block_tables[group_id]
for layer_name in kv_cache_config.kv_cache_groups[group_id].layer_names:
yield forward_context[layer_name].kv_cache[-1], block_table
def temporal_block(temporal_state, block_table, col):
# Resolving the block id for assertions is a deliberate D2H.
@@ -15,6 +15,8 @@ from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
HiddenStateCacheSpec,
KVCacheGroupSpec,
KVCacheLayout,
MambaSpec,
MLAAttentionSpec,
SlidingWindowMLASpec,
)
@@ -99,12 +101,12 @@ def test_get_block_size_falls_back_to_cache_config_when_no_kv_cache_config():
assert block_size == 16
# ---- MLA-verifier absorption ------------------------------------------------
# ---- Packed MLA grouping ----------------------------------------------------
def test_find_group_id_errors_clearly_when_absorbed_by_mla_swa_verifier():
# HiddenStateCacheSpec subclasses MLAAttentionSpec, so an MLA + sliding-
# window MLA verifier absorbs it into the MLA group instead of isolating it.
def test_hidden_state_group_isolated_from_packed_mla_groups():
# HiddenStateCacheSpec subclasses MLAAttentionSpec, but grouping must pull
# it out before packing compatible MLA cache specs.
dt = torch.bfloat16
spec = {
"layers.0.mla": MLAAttentionSpec(
@@ -116,11 +118,54 @@ def test_find_group_id_errors_clearly_when_absorbed_by_mla_swa_verifier():
"cache_only_layers.61": _hidden(64),
}
vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(
get_resolved_kv_cache_layout=lambda: KVCacheLayout.BLHNC
),
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
speculative_config=None,
)
groups = get_kv_cache_groups(vllm_config, spec)
assert not any(isinstance(g.kv_cache_spec, HiddenStateCacheSpec) for g in groups)
hidden_group_ids = [
i
for i, group in enumerate(groups)
if isinstance(group.kv_cache_spec, HiddenStateCacheSpec)
]
assert len(hidden_group_ids) == 1
cfg = SimpleNamespace(kv_cache_groups=groups)
with pytest.raises(ValueError, match="MLA verifiers are unsupported"):
ExampleHiddenStatesConnector._find_cache_kv_group_id(cfg)
assert (
ExampleHiddenStatesConnector._find_cache_kv_group_id(cfg) == hidden_group_ids[0]
)
def test_hidden_state_group_isolated_from_packed_mixed_page_groups():
# Packed grouping keeps groups with unequal page sizes (blocks are strided
# by the widest group), so the hidden group is appended without padding.
hidden = _hidden(16)
spec = {
"layers.0.attn": _full(16),
"layers.1.mamba": MambaSpec(
block_size=16, shapes=((1024,),), dtypes=(torch.float32,)
),
"cache_only_layers.61": hidden,
}
vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(
get_resolved_kv_cache_layout=lambda: KVCacheLayout.BLHNC
),
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
speculative_config=None,
)
groups = get_kv_cache_groups(vllm_config, spec)
page_sizes = {g.kv_cache_spec.page_size_bytes for g in groups}
assert len(page_sizes) > 1, "spec must exercise the packed grouping path"
hidden_group_ids = [
i
for i, group in enumerate(groups)
if isinstance(group.kv_cache_spec, HiddenStateCacheSpec)
]
assert len(hidden_group_ids) == 1
assert groups[hidden_group_ids[0]].kv_cache_spec == hidden
cfg = SimpleNamespace(kv_cache_groups=groups)
assert (
ExampleHiddenStatesConnector._find_cache_kv_group_id(cfg) == hidden_group_ids[0]
)
@@ -865,6 +865,10 @@ def _make_mock_worker_for_desc_ids(
worker._group_spec_types = group_spec_types
worker.block_len_per_layer = block_len_per_layer or [100]
worker._conv_decomp = None
worker._is_csa_linear = False
worker._ssm_region_indices = [0] if has_mamba else []
worker._ple_group_index = None
worker._ple_region_index = None
if has_mamba:
from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import ( # noqa: E501
MambaConvSplitInfo,
@@ -1823,6 +1827,7 @@ def test_push_write_hybrid_mla_replicates_attention():
worker.shutdown = lambda: None # skeleton worker: silence __del__
worker.use_mla = True
worker._has_mamba = True
worker._is_csa_linear = False
worker._group_spec_types = (MLAAttentionSpec, MambaSpec)
worker.transfer_topo = MagicMock()
worker.transfer_topo.tp_ratio.return_value = -2
@@ -645,3 +645,302 @@ def test_mismatched_mla_kernel_page_rejected_for_mla_hybrid():
meta_r.block_lens = [x // 2 for x in worker.block_len_per_layer]
with pytest.raises((AssertionError, RuntimeError)):
worker.add_remote_agent(meta_r, remote_tp_rank=0, remote_tp_size=2)
def _make_csa_linear_ple_worker(scratch_aliases: str = "compressed"):
"""``scratch_aliases`` selects which pages the compressor ring overlays."""
from unittest.mock import MagicMock
from vllm.config import set_current_vllm_config
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker as bw
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
NixlConnectorWorker,
)
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheTensor,
MambaSpec,
MLAAttentionSpec,
UniformTypeKVCacheSpecs,
)
main_kv_specs = {
f"main_kv.{index}": FullAttentionSpec(
block_size=4,
num_kv_heads=1,
head_size=16,
dtype=torch.float16,
)
for index in range(2)
}
compressed_specs = {
f"compressed.{index}": MLAAttentionSpec(
block_size=4,
num_kv_heads=1,
head_size=64,
dtype=torch.float16,
tokens_per_state=2,
)
for index in range(2)
}
compressor_state_specs = {
f"compressor_state.{index}": CircularBufferSpec(
block_size=2,
num_kv_heads=1,
head_size=32,
dtype=torch.float16,
)
for index in range(2)
}
sharded_mamba_spec = MambaSpec(
block_size=1,
shapes=((8, 3), (1, 4, 4)),
dtypes=(torch.float16, torch.float32),
page_size_padded=256,
mamba_type=MambaAttentionBackendEnum.GDN_ATTN,
)
ple_spec = MambaSpec(
block_size=1,
shapes=((6, 3),),
dtypes=(torch.float16,),
page_size_padded=256,
mamba_type=MambaAttentionBackendEnum.SHORT_CONV,
tp_replicated=True,
)
compressed_sparse_specs = {
name: spec
for index in range(2)
for name, spec in (
(f"main_kv.{index}", main_kv_specs[f"main_kv.{index}"]),
(f"compressed.{index}", compressed_specs[f"compressed.{index}"]),
)
}
def group(specs):
uniform = UniformTypeKVCacheSpecs.from_specs(specs)
assert uniform is not None
return KVCacheGroupSpec(list(specs), uniform)
tensor_regions: tuple[tuple[str, ...], ...]
if scratch_aliases == "compressed":
tensor_regions = (
("main_kv.0", "mamba.sharded", "mamba.ple"),
("main_kv.1",),
("compressed.0", "compressor_state.0"),
("compressed.1", "compressor_state.1"),
)
else:
assert scratch_aliases == "main_kv"
tensor_regions = (
("main_kv.0", "mamba.sharded", "mamba.ple", "compressor_state.0"),
("main_kv.1", "compressor_state.1"),
("compressed.0",),
("compressed.1",),
)
region_size = 512
page_size = 256
kv_cache_config = KVCacheConfig(
num_blocks=2,
kv_cache_tensors=[
KVCacheTensor(
size=len(tensor_regions) * region_size,
layers=[layer_name],
layer_stride=region_size,
block_stride=page_size,
offset=region_index * region_size,
)
for region_index, layer_names in enumerate(tensor_regions)
for layer_name in layer_names
],
kv_cache_groups=[
group(compressed_sparse_specs),
group(compressor_state_specs),
KVCacheGroupSpec(["mamba.sharded"], sharded_mamba_spec),
KVCacheGroupSpec(["mamba.ple"], ple_spec),
],
)
vllm_config = create_vllm_config(block_size=4)
vllm_config.cache_config.enable_prefix_caching = False
vllm_config.kv_transfer_config.kv_buffer_device = "cuda"
fake_backend = MagicMock()
fake_backend.get_supported_kernel_block_sizes.return_value = [4]
fake_backend.get_name.return_value = "TEST_ATTN"
fake_backend.full_cls_name.return_value = "test.AttentionBackend"
fake_platform = MagicMock()
fake_platform.device_type = "cuda"
fake_platform.get_nixl_memory_type.return_value = "VRAM"
with (
patch.object(bw, "NixlWrapper", _RecordingNixl),
patch.object(bw, "get_tensor_model_parallel_rank", return_value=0),
patch.object(bw, "get_tensor_model_parallel_world_size", return_value=1),
patch.object(bw, "get_current_attn_backends", return_value=[fake_backend]),
patch.object(bw, "current_platform", fake_platform),
patch(
"vllm.model_executor.layers.mamba.mamba_utils.get_conv_state_layout",
return_value="DS",
),
set_current_vllm_config(vllm_config),
):
worker = NixlConnectorWorker(vllm_config, "local-engine", kv_cache_config)
tensors = [torch.zeros((2, 256), dtype=torch.uint8) for _ in range(4)]
worker.register_kv_caches(
{
layer_name: tensors[region_index]
for region_index, layer_names in enumerate(tensor_regions)
for layer_name in layer_names
}
)
return worker
@pytest.mark.cpu_test
def test_csa_linear_registration_discovers_shared_regions():
worker = _make_csa_linear_ple_worker()
assert worker._ssm_region_indices == [0]
assert worker._ple_group_index == 3
assert worker._ple_region_index == 0
assert worker._region_is_mla == [False, False, True, True]
assert worker._scratch_region_indices == [2, 3]
@pytest.mark.cpu_test
def test_csa_linear_scratch_descs_follow_the_pages_they_overlay():
"""The scratch ring is addressed by the regions it actually registered in,
not by inferring that it must sit on the MLA pages: overlaying the main KV
pages instead moves its descriptors with it."""
worker = _make_csa_linear_ple_worker(scratch_aliases="main_kv")
# Same MLA regions as before, but the ring now lives in the non-MLA ones.
assert worker._region_is_mla == [False, False, True, True]
assert worker._scratch_region_indices == [0, 1]
desc_ids = worker._compute_desc_ids(
block_ids=([1], [0], [1], [1]),
dst_num_blocks=2,
block_size_ratio=None,
physical_blocks_per_logical=1,
)
# Scratch group (block 0) now maps to regions 0 and 1 -> descs 0 and 2,
# where the compressed-page placement gave 4 and 6.
assert desc_ids.tolist()[4:6] == [0, 2]
@pytest.mark.cpu_test
def test_csa_linear_selects_compressed_sparse_compressor_state_and_mamba_regions():
worker = _make_csa_linear_ple_worker()
desc_ids = worker._compute_desc_ids(
block_ids=([1], [0], [1], [1]),
dst_num_blocks=2,
block_size_ratio=None,
physical_blocks_per_logical=1,
)
assert desc_ids.tolist() == [
1,
3,
5,
7,
4,
6,
9,
11,
13,
15,
17,
]
mamba_only = worker._compute_desc_ids(
block_ids=([], [], [1], [1]),
dst_num_blocks=2,
block_size_ratio=None,
physical_blocks_per_logical=1,
)
assert mamba_only.tolist() == [9, 11, 13, 15, 17]
@pytest.mark.cpu_test
def test_csa_linear_ple_descriptor_is_not_split():
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
TPMapping,
)
worker = _make_csa_linear_ple_worker()
bases = [0x10000, 0x20000, 0x30000, 0x40000]
attention = worker._build_fa_local(bases, block_size_ratio=1)
mamba = worker._build_mamba_local(bases)
assert attention.shape == (8, 3)
assert mamba.shape == (10, 3)
assert mamba[-2:, 0].tolist() == [0x10000, 0x10100]
assert mamba[-2:, 1].tolist() == [256, 256]
mapping = TPMapping(
source_ranks_per_group=((0,), (0,), (0, 1), (0, 1)),
all_source_ranks=(0, 1),
rank_to_attention_slot={0: 0, 1: 0},
rank_offset_factor=0,
)
splits = list(
worker._build_local_splits_from_plan(
mapping,
np.concatenate([attention, mamba]),
num_fa_descs=len(attention),
)
)
assert len(splits) == 2
assert [row[1] for row in splits[0][-2:]] == [256, 256]
assert [row[1] for row in splits[1][-2:]] == [256, 256]
assert splits[0][-3][1] * 2 == mamba[-3, 1]
assert splits[1][-3][1] * 2 == mamba[-3, 1]
@pytest.mark.cpu_test
def test_csa_linear_remote_ple_is_copied_whole():
from types import SimpleNamespace
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
NixlAgentMetadata,
)
worker = _make_csa_linear_ple_worker()
metadata = NixlAgentMetadata(
engine_id="remote",
agent_metadata=b"agent",
kv_caches_base_addr=[
0x10000,
0x20000,
0x30000,
0x40000,
],
device_id=0,
num_blocks=2,
block_lens=[256] * 4,
block_strides=[256] * 4,
kv_cache_layout="HND",
block_size=4,
ssm_sizes=(24, 32),
attn_backend_name="test",
physical_blocks_per_logical_kv_block=1,
)
descriptors = worker._build_mamba_remote(
metadata,
tp_ratio=-2,
transfer_info=SimpleNamespace(remote_physical_blocks_per_logical=1),
)
assert descriptors.shape == (10, 3)
assert descriptors[-2:, 0].tolist() == [0x10000, 0x10100]
assert descriptors[-2:, 1].tolist() == [256, 256]
metadata.block_lens[0] = 128
with pytest.raises(ValueError, match="PLE pages require identical"):
worker._build_mamba_remote(
metadata,
tp_ratio=-2,
transfer_info=SimpleNamespace(remote_physical_blocks_per_logical=1),
)
@@ -359,6 +359,7 @@ class _StubWriterWorker(NixlPushConnectorWorker):
w._physical_blocks_per_logical_kv_block = 1
# Single non-hybrid attention group, matching the stub block id lists.
w._has_mamba = False
w._is_csa_linear = False
w._group_spec_types = (FullAttentionSpec,)
w._engine_ttl = 0.0
w._engine_last_active = {}
@@ -1198,6 +1199,75 @@ class TestPushWriterMlaReplication:
# engine thread never sees a partial set and double-frees the request.
assert sorted(w._sending_transfers["p-req"]) == [1000, 1001]
def test_csa_linear_writes_every_group_to_each_target(self):
from types import SimpleNamespace
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
RemoteMeta,
ReqMeta,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
TPMapping,
)
engine_id = "decode-engine"
w = _StubWriterWorker.fresh()
w.world_size = 2
w.use_mla = False
w._has_mamba = True
w._is_csa_linear = True
w.nixl_wrapper = MagicMock()
w.transfer_topo = MagicMock()
w.transfer_topo.get_engine_info.return_value = SimpleNamespace(
remote_tp_size=4,
remote_block_size=16,
remote_physical_blocks_per_logical=1,
)
w.transfer_topo.tp_ratio.return_value = -2
w.transfer_topo.handshake_target_ranks.return_value = (0, 1)
w.tp_mappings = {
engine_id: TPMapping(
source_ranks_per_group=((0,), (0,), (0, 1), (0, 1)),
all_source_ranks=(0, 1),
rank_to_attention_slot={0: 0, 1: 0},
rank_offset_factor=0,
)
}
w._logical_to_kernel_block_ids = lambda block_ids, ratio: block_ids
w.dst_xfer_side_handles = {engine_id: {0: 1000, 1: 1001}}
w.src_xfer_handles_by_tp_ratio = {(-2, 16): [2000, 2001]}
writes = []
def _fake_xfer(**kwargs):
writes.append(kwargs)
return 3000 + kwargs["read_spec"].remote_rank
w._xfer_blocks = _fake_xfer
block_ids = ([10], [20], [30], [40])
meta = ReqMeta(
local_block_ids=block_ids,
local_physical_block_ids=block_ids,
tp_size=2,
remote=RemoteMeta(
block_ids=([50], [60], [70], [80]),
host="",
port=0,
engine_id=engine_id,
request_id="d-req",
),
)
w._xfer_blocks_for_req(req_id="p-req", meta=meta)
assert [write["read_spec"].remote_rank for write in writes] == [0, 1]
assert [write["read_spec"].local_block_ids for write in writes] == [
list(block_ids),
list(block_ids),
]
assert [write["local_xfer_side_handle"] for write in writes] == [2000, 2001]
assert sorted(w._sending_transfers["p-req"]) == [3000, 3001]
class TestPushPrefixCaching:
"""Partial prefix-cache hit on D: D preallocates only its *uncomputed*
@@ -154,6 +154,9 @@ def _make_mock_worker_for_splits(group_spec_types):
worker.block_len_per_layer = []
worker.num_regions = 0
worker._region_is_mla = []
worker._conv_decomp = SimpleNamespace(local_conv_offsets=())
worker._ssm_region_indices = [0] if MambaSpec in group_spec_types else []
worker._ple_region_index = None
return worker
@@ -282,3 +285,31 @@ class TestMambaPlanSplitHandles:
with pytest.raises(AssertionError, match="Head-sharded"):
list(worker._build_local_splits_from_plan(plan, src_blocks_data, 2, 2))
@pytest.mark.parametrize(
("total_kv_heads", "local_tp", "remote_tp", "compatible"),
[
# 2 heads: TP1 packs both heads into a page, TP >= 2 packs one.
(2, 1, 2, False),
(2, 2, 1, False),
(2, 1, 1, True),
(2, 2, 4, True),
(2, 4, 2, True),
# 4 heads: the boundary moves with the head count.
(4, 2, 4, False),
(4, 2, 2, True),
(4, 4, 8, True),
],
)
def test_csa_linear_tp_layout_boundary(total_kv_heads, local_tp, remote_tp, compatible):
worker = object.__new__(NixlConnectorWorker)
worker.world_size = local_tp
worker._is_csa_linear = True
worker.transfer_topo = SimpleNamespace(total_num_kv_heads=total_kv_heads)
if compatible:
worker._validate_csa_linear_tp_layout(remote_tp)
else:
with pytest.raises(ValueError, match="KV-head sharding boundary"):
worker._validate_csa_linear_tp_layout(remote_tp)
@@ -104,6 +104,47 @@ def _make_speculator(
return speculator
@pytest.mark.parametrize(("hc_mult", "expected"), [(None, 64), (4, 256)])
def test_speculator_uses_draft_model_hidden_size(monkeypatch, hc_mult, expected):
# Qwen4Exp targets expose multi-stream HC residuals to the drafter.
monkeypatch.setattr(base_spec_module, "_target_feeds_hc_residual", lambda _: True)
hf_config = SimpleNamespace()
if hc_mult is not None:
hf_config.hc_mult = hc_mult
draft_model_config = SimpleNamespace(
hf_config=hf_config,
get_hidden_size=lambda: 64,
get_vocab_size=lambda: 32,
)
speculative_config = SimpleNamespace(
method="mtp",
num_speculative_tokens=3,
draft_model_config=draft_model_config,
use_local_argmax_reduction=False,
draft_sample_method="greedy",
)
vllm_config = SimpleNamespace(
speculative_config=speculative_config,
scheduler_config=SimpleNamespace(
max_num_seqs=2,
max_num_batched_tokens=8,
),
model_config=SimpleNamespace(
max_model_len=32,
dtype=torch.float32,
use_fp64_gumbel=False,
),
parallel_config=SimpleNamespace(
data_parallel_size=1,
data_parallel_rank=0,
),
)
speculator = _TestSpeculator(vllm_config, torch.device("cpu"))
assert speculator.hidden_size == expected
def test_mm_support_configured_after_model_load(monkeypatch):
target_model_config = object()
draft_model_config = object()
@@ -197,9 +238,11 @@ def test_load_model_disables_mm_support_for_text_only_drafter(monkeypatch):
assert not speculator.supports_mm_inputs
assert warning_messages == [
"Draft model _TextOnlyDraftModel does not support external multimodal "
"embeddings. Embeddings from the target model will not be passed to the "
"drafter; using text-only draft inputs instead."
(
"Draft model _TextOnlyDraftModel does not support external multimodal "
"embeddings. Embeddings from the target model will not be passed to the "
"drafter; using text-only draft inputs instead."
)
]
+36
View File
@@ -132,6 +132,42 @@ def test_block_tables_apply_staged_writes_single_group():
)
def test_block_tables_skip_custom_slot_mapping_groups():
device = torch.device("cuda")
block_tables = BlockTables(
block_sizes=[8, 262144],
max_num_reqs=1,
max_num_batched_tokens=4,
max_num_blocks_per_group=[1, 1],
device=device,
kernel_block_sizes=[8, 262144],
slot_mapping_enabled=[False, True],
)
block_tables.append_block_ids(
req_index=0,
new_block_ids=([7], [12]),
overwrite=True,
)
block_tables.apply_staged_writes()
idx_mapping = torch.tensor([0], dtype=torch.int32, device=device)
query_start_loc = torch.tensor([0, 2], dtype=torch.int32, device=device)
positions = torch.tensor([153797, 165757], dtype=torch.int64, device=device)
slot_mappings = block_tables.compute_slot_mappings(
idx_mapping,
query_start_loc,
positions,
num_tokens_padded=2,
)
torch.accelerator.synchronize()
assert slot_mappings[0].tolist() == [-1, -1]
assert slot_mappings[1].tolist() == [
12 * 262144 + 153797,
12 * 262144 + 165757,
]
@pytest.mark.parametrize("cp_rank", range(4))
def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
"""DCP interleave is expressed in logical-block token coordinates."""
@@ -8,15 +8,107 @@ import torch
import vllm.v1.worker.gpu.model_runner as model_runner_module
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
MambaSpec,
UniformTypeKVCacheSpecs,
)
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
def test_qsa_circular_group_uses_custom_slot_mapping(monkeypatch):
runner = GPUModelRunner.__new__(GPUModelRunner)
runner.max_model_len = 262144
runner.is_encoder_decoder = False
runner.dcp_size = 1
runner.dcp_rank = 0
runner.cp_interleave = 1
runner.cache_config = SimpleNamespace(enable_prefix_caching=True)
parallel_config = SimpleNamespace(
decode_context_parallel_size=1,
cp_kv_cache_interleave_size=1,
)
runner.parallel_config = parallel_config
runner.vllm_config = SimpleNamespace(
parallel_config=parallel_config,
cache_config=SimpleNamespace(mamba_cache_mode="none"),
)
runner.model_state = SimpleNamespace(
get_additional_cg_support=lambda: (),
num_new_sampled_tokens_per_step=1,
)
runner.speculator = None
runner.req_states = []
runner.input_buffers = SimpleNamespace(query_start_loc=None)
runner.vocab_size = 1
runner.max_num_reqs = 1
runner.max_num_tokens = 2
runner.device = torch.device("cuda")
raw_spec = CircularBufferSpec(
block_size=8,
num_kv_heads=1,
head_size=128,
dtype=torch.bfloat16,
)
compressed_spec = FullAttentionSpec(
block_size=262144,
num_kv_heads=1,
head_size=128,
dtype=torch.bfloat16,
)
kv_cache_config = KVCacheConfig(
num_blocks=1,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
layer_names=["raw"],
kv_cache_spec=UniformTypeKVCacheSpecs(
block_size=8,
kv_cache_specs={"raw": raw_spec},
),
),
KVCacheGroupSpec(layer_names=["compressed"], kv_cache_spec=compressed_spec),
],
)
class FakeAttnCGSupport:
def narrow(self, *args):
return self
attn_cg_support = FakeAttnCGSupport()
monkeypatch.setattr(
model_runner_module,
"init_attn_backend",
lambda *args: ([], attn_cg_support, [8, 262144]),
)
monkeypatch.setattr(
model_runner_module,
"maybe_create_adaptive_verification_manager",
lambda **kwargs: None,
)
captured = {}
class BlockTablesCaptured(Exception):
pass
def capture_block_tables(**kwargs):
captured.update(kwargs)
raise BlockTablesCaptured
monkeypatch.setattr(model_runner_module, "BlockTables", capture_block_tables)
with pytest.raises(BlockTablesCaptured):
runner.initialize_kv_cache(kv_cache_config)
assert captured["max_num_blocks_per_group"] == [1, 1]
assert captured["slot_mapping_enabled"] == [False, True]
@pytest.mark.parametrize(
("mamba_cache_mode", "num_speculative_blocks", "expected"),
[
+15 -2
View File
@@ -20,6 +20,7 @@ from vllm.config.vllm import VllmConfig
from vllm.platforms import current_platform
from vllm.utils.math_utils import cdiv
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
@@ -53,6 +54,18 @@ def _attention_group() -> KVCacheGroupSpec:
)
def _circular_group() -> KVCacheGroupSpec:
return KVCacheGroupSpec(
["circular"],
CircularBufferSpec(
block_size=BLOCK_SIZE,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
)
def _mamba_group(mamba_cache_mode: str) -> KVCacheGroupSpec:
# Name carries the mode so several groups can coexist in one config.
return KVCacheGroupSpec(
@@ -212,8 +225,7 @@ def test_warmup_reserves_mamba_speculative_blocks(mamba_cache_mode):
def _hybrid_kv_cache_config(num_blocks: int) -> KVCacheConfig:
"""Full attention plus one Mamba group per cache mode, so a single manager
exercises every branch `_reserved_block_count` has.
"""Attention, circular, and Mamba groups exercise every reservation branch.
"none" and "all" reach the same branch of both `_reserved_block_count` and
`MambaManager`, which tests only for "align". They are still both listed:
@@ -226,6 +238,7 @@ def _hybrid_kv_cache_config(num_blocks: int) -> KVCacheConfig:
kv_cache_tensors=[],
kv_cache_groups=[
_attention_group(),
_circular_group(),
_mamba_group("none"),
_mamba_group("all"),
_mamba_group("align"),
+188 -12
View File
@@ -1,6 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from dataclasses import dataclass
from typing import Any
from unittest.mock import MagicMock, patch
@@ -10,11 +9,20 @@ import pytest
import torch
from vllm.model_executor.layers.mamba.mamba_utils import (
MambaStateCopyFunc,
MambaStateCopyFuncsByType,
get_conv_copy_spec,
get_temporal_copy_spec,
)
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput
from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheGroupSpec, MambaSpec
from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
MambaSpec,
UniformTypeKVCacheSpecs,
)
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
from vllm.v1.worker.mamba_utils import (
MambaCopyBuffers,
MambaSpecDecodeGPUContext,
@@ -22,17 +30,21 @@ from vllm.v1.worker.mamba_utils import (
batch_memcpy,
collect_mamba_copy_meta,
do_mamba_copy_block,
get_mamba_groups,
preprocess_mamba,
stage_postprocess_inputs_to_gpu,
)
MambaStateCopyFunc = Callable[..., Any]
# Conv + temporal copy specs, in the order the tests' MambaSpec shapes expect.
_COPY_FUNCS: tuple[MambaStateCopyFunc, ...] = (
_DEFAULT_COPY_FUNCS: tuple[MambaStateCopyFunc, ...] = (
get_conv_copy_spec,
get_temporal_copy_spec,
)
_COPY_FUNCS: MambaStateCopyFuncsByType = {
MambaAttentionBackendEnum.MAMBA2: _DEFAULT_COPY_FUNCS,
MambaAttentionBackendEnum.GDN_ATTN: _DEFAULT_COPY_FUNCS,
MambaAttentionBackendEnum.SHORT_CONV: (get_conv_copy_spec,),
}
def postprocess_mamba(
@@ -41,7 +53,7 @@ def postprocess_mamba(
input_batch: Any,
requests: dict[str, Any],
forward_context: dict[str, Any],
mamba_state_copy_funcs: tuple[MambaStateCopyFunc, ...],
mamba_state_copy_funcs: MambaStateCopyFuncsByType,
copy_bufs: "MambaCopyBuffers",
):
"""CPU reference for the align-mode postprocess.
@@ -147,7 +159,7 @@ def test_resumed_req_ids_cleared_from_mamba_state_idx():
input_batch,
{}, # requests
{}, # forward_context
(), # mamba_state_copy_funcs
{}, # mamba_state_copy_funcs
copy_bufs,
)
@@ -430,7 +442,10 @@ def _make_requests(
def _make_copy_bufs(
cfg: _TestConfig, kv_cache_config: KVCacheConfig, device: torch.device
cfg: _TestConfig,
kv_cache_config: KVCacheConfig,
device: torch.device,
copy_funcs: MambaStateCopyFuncsByType = _COPY_FUNCS,
) -> MambaCopyBuffers:
"""Create MambaCopyBuffers for the Python path."""
@@ -440,13 +455,16 @@ def _make_copy_bufs(
return MambaCopyBuffers.create(
max_num_reqs=cfg.max_num_reqs,
kv_cache_config=kv_cache_config,
copy_funcs=(get_conv_copy_spec, get_temporal_copy_spec),
copy_funcs=copy_funcs,
make_buffer=make_buffer,
)
def _make_gpu_ctx(
cfg: _TestConfig, kv_cache_config: KVCacheConfig, device: torch.device
cfg: _TestConfig,
kv_cache_config: KVCacheConfig,
device: torch.device,
copy_funcs: MambaStateCopyFuncsByType = _COPY_FUNCS,
) -> MambaSpecDecodeGPUContext:
"""Create MambaSpecDecodeGPUContext for the GPU path."""
@@ -456,12 +474,170 @@ def _make_gpu_ctx(
return MambaSpecDecodeGPUContext.create(
max_num_reqs=cfg.max_num_reqs,
kv_cache_config=kv_cache_config,
num_state_types=2,
copy_funcs=copy_funcs,
device=device,
make_buffer=make_buffer,
)
def test_mamba_groups_support_different_state_specs():
gdn_spec = MambaSpec(
block_size=16,
shapes=((4, 4), (2, 4, 4)),
dtypes=(torch.float16, torch.float32),
mamba_type=MambaAttentionBackendEnum.GDN_ATTN,
mamba_cache_mode="align",
num_speculative_blocks=3,
)
short_conv_spec = MambaSpec(
block_size=16,
shapes=((12, 8),),
dtypes=(torch.float16,),
mamba_type=MambaAttentionBackendEnum.SHORT_CONV,
mamba_cache_mode="align",
num_speculative_blocks=3,
)
kv_cache_config = KVCacheConfig(
num_blocks=8,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(["gdn.0", "gdn.1"], gdn_spec),
KVCacheGroupSpec(["ple.0"], short_conv_spec),
],
)
mamba_groups = get_mamba_groups(kv_cache_config)
assert mamba_groups == {gdn_spec: [0], short_conv_spec: [1]}
model_state = object.__new__(MambaHybridModelState)
model_state._mamba_group_ids = []
model_state._mamba_spec = None
group_ids, representative_spec = model_state._get_mamba_group_info(kv_cache_config)
assert group_ids == [0, 1]
assert representative_spec is gdn_spec
cfg = _TestConfig()
copy_bufs = _make_copy_bufs(cfg, kv_cache_config, torch.device("cpu"))
assert copy_bufs.src_ptrs.cpu.numel() == cfg.max_num_reqs * 5
assert copy_bufs.dst_ptrs.cpu.numel() == cfg.max_num_reqs * 5
assert copy_bufs.sizes.cpu.numel() == cfg.max_num_reqs * 5
ctx = _make_gpu_ctx(cfg, kv_cache_config, torch.device("cpu"))
assert ctx.num_states == 5
forward_context = {}
gdn_spec_layer_names = ["gdn.0", "gdn.1"]
for layer_name in gdn_spec_layer_names:
attention = MagicMock()
attention.kv_cache = (
torch.empty(8, 4, 4, dtype=torch.float16),
torch.empty(8, 2, 4, 4, dtype=torch.float32),
)
forward_context[layer_name] = attention
ple_attention = MagicMock()
ple_attention.kv_cache = (torch.empty(8, 12, 8, dtype=torch.float16),)
forward_context["ple.0"] = ple_attention
assert gdn_spec_layer_names == kv_cache_config.kv_cache_groups[0].layer_names
block_tables = [
torch.zeros(4, 8, dtype=torch.int32),
torch.zeros(4, 8, dtype=torch.int32),
]
ctx.initialize_from_forward_context(
kv_cache_config,
forward_context,
_COPY_FUNCS,
block_tables,
)
assert ctx.is_initialized
assert ctx.state_group_indices.tolist() == [0, 0, 0, 0, 1]
assert ctx.state_conv_widths.tolist() == [4, 0, 4, 0, 12]
def test_mamba_groups_support_mixed_specs_in_uniform_group():
gdn_spec = MambaSpec(
block_size=16,
shapes=((4, 4), (2, 4, 4)),
dtypes=(torch.float16, torch.float32),
mamba_type=MambaAttentionBackendEnum.GDN_ATTN,
mamba_cache_mode="align",
num_speculative_blocks=3,
)
short_conv_spec = MambaSpec(
block_size=16,
shapes=((12, 8),),
dtypes=(torch.float16,),
mamba_type=MambaAttentionBackendEnum.SHORT_CONV,
mamba_cache_mode="align",
num_speculative_blocks=3,
)
layer_specs = {
"gdn.0": gdn_spec,
"gdn.1": gdn_spec,
"ple.0": short_conv_spec,
}
kv_cache_config = KVCacheConfig(
num_blocks=8,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
list(layer_specs),
UniformTypeKVCacheSpecs(
block_size=16,
kv_cache_specs=layer_specs,
),
)
],
)
assert kv_cache_config.has_mamba_layers
assert get_mamba_groups(kv_cache_config) == {
gdn_spec: [0],
short_conv_spec: [0],
}
# The group is listed once per spec, so its id must not be repeated.
model_state = object.__new__(MambaHybridModelState)
model_state._mamba_group_ids = []
model_state._mamba_spec = None
group_ids, representative_spec = model_state._get_mamba_group_info(kv_cache_config)
assert group_ids == [0]
assert representative_spec is gdn_spec
cfg = _TestConfig()
copy_bufs = _make_copy_bufs(cfg, kv_cache_config, torch.device("cpu"))
assert copy_bufs.mamba_group_ids == [0]
assert copy_bufs.src_ptrs.cpu.numel() == cfg.max_num_reqs * 5
ctx = _make_gpu_ctx(cfg, kv_cache_config, torch.device("cpu"))
assert ctx.mamba_group_ids == [0]
assert ctx.num_states == 5
forward_context = {
"gdn.0": MagicMock(
kv_cache=(
torch.empty(8, 4, 4, dtype=torch.float16),
torch.empty(8, 2, 4, 4, dtype=torch.float32),
)
),
"gdn.1": MagicMock(
kv_cache=(
torch.empty(8, 4, 4, dtype=torch.float16),
torch.empty(8, 2, 4, 4, dtype=torch.float32),
)
),
"ple.0": MagicMock(kv_cache=(torch.empty(8, 12, 8, dtype=torch.float16),)),
}
ctx.initialize_from_forward_context(
kv_cache_config,
forward_context,
_COPY_FUNCS,
[torch.zeros(4, 8, dtype=torch.int32)],
)
assert ctx.state_group_indices.tolist() == [0, 0, 0, 0, 0]
assert ctx.state_conv_widths.tolist() == [4, 0, 4, 0, 12]
# -----------------------------------------------------------------------------
# stage_postprocess_inputs_to_gpu: single-pass staging into pinned views
# -----------------------------------------------------------------------------
@@ -620,7 +796,7 @@ def _run_gpu_postprocess(
*,
kv_cache_config: KVCacheConfig,
forward_context: dict[str, Any],
copy_funcs: tuple,
copy_funcs: MambaStateCopyFuncsByType,
block_table: torch.Tensor,
req_ids: list[str],
num_accepted_tokens: list[int],