[Model] Support Qwen3.8-Flash-Next (#53896)
Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com> Signed-off-by: andyluo7 <andy.luo@amd.com> Signed-off-by: lishunyang12 <lishunyang03@gmail.com> Signed-off-by: Harry Huang <huanghaoyan.hhy@alibaba-inc.com> Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: KungYork <firelikest@163.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: andyluo7 <43718156+andyluo7@users.noreply.github.com> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: andyluo7 <andy.luo@amd.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: lishunyang12 <lishunyang03@gmail.com> Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
zjy0516
KungYork
Thien Tran
Yifan Qiao
andyluo7
Isotr0py
andyluo7
OpenAI Codex
lishunyang12
Vadim Gimpelson
Isotr0py
Lucas Wilkinson
Claude Opus 5
OpenAI Codex
parent
687db59744
commit
e126687a9a
@@ -7,6 +7,7 @@ from enum import Enum
|
||||
import pytest
|
||||
|
||||
from vllm.config.cache import CacheConfig
|
||||
from vllm.config.scheduler import SchedulerConfig
|
||||
from vllm.config.utils import get_hash_factors, hash_factors, normalize_value
|
||||
|
||||
# Helpers
|
||||
@@ -216,6 +217,24 @@ def test_cache_config_hash_ignores_kv_cache_sizing_knobs():
|
||||
assert CacheConfig(gpu_memory_utilization=0.5).compute_hash() == base_hash
|
||||
|
||||
|
||||
def test_scheduler_config_hash_includes_max_num_seqs():
|
||||
"""Per-request workspace sizes must invalidate compiled graphs."""
|
||||
base_hash = SchedulerConfig(
|
||||
max_model_len=8192,
|
||||
is_encoder_decoder=False,
|
||||
max_num_batched_tokens=8192,
|
||||
max_num_seqs=128,
|
||||
).compute_hash()
|
||||
larger_batch_hash = SchedulerConfig(
|
||||
max_model_len=8192,
|
||||
is_encoder_decoder=False,
|
||||
max_num_batched_tokens=8192,
|
||||
max_num_seqs=1024,
|
||||
).compute_hash()
|
||||
|
||||
assert larger_batch_hash != base_hash
|
||||
|
||||
|
||||
def test_cache_config_hash_ignores_prefix_cache_retention_interval():
|
||||
base_hash = CacheConfig().compute_hash()
|
||||
assert CacheConfig(prefix_cache_retention_interval=64).compute_hash() == base_hash
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for SpeculativeConfig.compose_draft_hf_overrides.
|
||||
"""Tests for draft config overrides used by SpeculativeConfig.
|
||||
|
||||
Callable ``hf_overrides`` on the target model config (e.g. the
|
||||
``dummy_hf_overrides`` shrink used by ``tests/models/test_initialization.py``)
|
||||
@@ -12,10 +12,12 @@ when the target itself is shrunk — which is what kept spec-decode archs like
|
||||
"""
|
||||
|
||||
import functools
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.config.speculative import SpeculativeConfig
|
||||
|
||||
|
||||
@@ -134,3 +136,52 @@ def test_composed_override_is_picklable():
|
||||
|
||||
out = composed(_make_hf_config())
|
||||
assert out.num_hidden_layers == 1
|
||||
|
||||
|
||||
def _make_mtp_speculative_config(
|
||||
override: bool | None,
|
||||
checkpoint_value: bool,
|
||||
) -> SpeculativeConfig:
|
||||
draft_hf_config = _make_hf_config(
|
||||
architectures=["Qwen4ExpMTP"],
|
||||
model_type="qwen4_exp_mtp",
|
||||
n_predict=1,
|
||||
index_share_for_mtp_iteration=checkpoint_value,
|
||||
)
|
||||
draft_model_config = MagicMock(
|
||||
model="draft",
|
||||
hf_config=draft_hf_config,
|
||||
architectures=draft_hf_config.architectures,
|
||||
max_model_len=128,
|
||||
)
|
||||
target_model_config = MagicMock(
|
||||
model="target",
|
||||
max_model_len=128,
|
||||
quantization=None,
|
||||
hf_overrides={},
|
||||
)
|
||||
|
||||
with patch("vllm.config.speculative.ModelConfig", return_value=draft_model_config):
|
||||
return SpeculativeConfig(
|
||||
model="draft",
|
||||
method="mtp",
|
||||
num_speculative_tokens=1,
|
||||
index_share_for_mtp_iteration=override,
|
||||
target_model_config=target_model_config,
|
||||
target_parallel_config=ParallelConfig(),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize(
|
||||
("override", "checkpoint_value", "expected"),
|
||||
[(None, True, True), (False, True, False), (True, False, True)],
|
||||
)
|
||||
def test_mtp_index_share_override(
|
||||
override: bool | None, checkpoint_value: bool, expected: bool
|
||||
):
|
||||
speculative_config = _make_mtp_speculative_config(override, checkpoint_value)
|
||||
assert (
|
||||
speculative_config.draft_model_config.hf_config.index_share_for_mtp_iteration
|
||||
is expected
|
||||
)
|
||||
|
||||
@@ -24,6 +24,29 @@ for i, v in enumerate(test_sizes):
|
||||
test_sizes[i] -= v % 8
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("dtype", "expected"),
|
||||
[
|
||||
(torch.float32, True),
|
||||
(torch.float16, True),
|
||||
(torch.bfloat16, True),
|
||||
(torch.int8, False),
|
||||
(torch.float8_e4m3fn, False),
|
||||
],
|
||||
)
|
||||
def test_custom_allreduce_filters_dtype(
|
||||
dtype: torch.dtype,
|
||||
expected: bool,
|
||||
) -> None:
|
||||
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
|
||||
communicator.disabled = False
|
||||
communicator.world_size = 2
|
||||
communicator.max_size = 1024
|
||||
communicator._ptr = 0
|
||||
|
||||
assert communicator.should_custom_ar(torch.empty(16, dtype=dtype)) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("major", "local_multicast", "expected"),
|
||||
[
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
# Qwen4Exp accuracy evaluation
|
||||
|
||||
This suite starts a Qwen3.8-Flash-Next-FP8 OpenAI-compatible server once and
|
||||
uses EvalScope to evaluate GSM8K and AIME25.
|
||||
|
||||
```bash
|
||||
# B200
|
||||
pytest -s -v tests/evals/qwen4_exp/test_accuracy.py \
|
||||
--config-list-file=configs/models-b200.txt
|
||||
|
||||
# H200
|
||||
pytest -s -v tests/evals/qwen4_exp/test_accuracy.py \
|
||||
--config-list-file=configs/models-h200.txt
|
||||
```
|
||||
@@ -0,0 +1,3 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
model_name: "Qwen/Qwen3.8-Flash-Next-FP8"
|
||||
datasets:
|
||||
gsm8k:
|
||||
metric_threshold: 0.98
|
||||
tolerance: 0.02
|
||||
aime25:
|
||||
metric_threshold: 0.90
|
||||
tolerance: 0.05
|
||||
eval_batch_size: 32
|
||||
startup_max_wait_seconds: 1800
|
||||
generation_config:
|
||||
seed: 1236
|
||||
do_sample: true
|
||||
temperature: 0.7
|
||||
top_p: 0.8
|
||||
top_k: 20
|
||||
repetition_penalty: 1.0
|
||||
presence_penalty: 1.5
|
||||
max_tokens: 32768
|
||||
stream: true
|
||||
extra_body:
|
||||
chat_template_kwargs:
|
||||
enable_thinking: true
|
||||
server_args: "--tensor-parallel-size 4"
|
||||
@@ -0,0 +1,2 @@
|
||||
# B200 configuration for Qwen3.8-Flash-Next-FP8 accuracy evaluation
|
||||
Qwen3.8-Flash-Next-FP8.yaml
|
||||
@@ -0,0 +1,2 @@
|
||||
# H200 configuration for Qwen3.8-Flash-Next-FP8 accuracy evaluation
|
||||
Qwen3.8-Flash-Next-FP8.yaml
|
||||
@@ -0,0 +1,42 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def pytest_addoption(parser):
|
||||
parser.addoption(
|
||||
"--config-list-file",
|
||||
required=True,
|
||||
help="File containing Qwen4Exp evaluation config files",
|
||||
)
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc):
|
||||
if "config_filename" not in metafunc.fixturenames:
|
||||
return
|
||||
|
||||
config_list_path = Path(metafunc.config.getoption("--config-list-file"))
|
||||
if not config_list_path.is_absolute():
|
||||
test_relative_path = Path(__file__).parent / config_list_path
|
||||
config_list_path = (
|
||||
test_relative_path
|
||||
if test_relative_path.exists()
|
||||
else Path.cwd() / config_list_path
|
||||
)
|
||||
|
||||
config_dir = config_list_path.parent
|
||||
config_files = [
|
||||
config_dir / line.strip()
|
||||
for line in config_list_path.read_text(encoding="utf-8").splitlines()
|
||||
if line.strip() and not line.lstrip().startswith("#")
|
||||
]
|
||||
missing = [path for path in config_files if not path.exists()]
|
||||
if missing:
|
||||
raise FileNotFoundError(f"Missing evaluation configs: {missing}")
|
||||
|
||||
metafunc.parametrize(
|
||||
"config_filename",
|
||||
config_files,
|
||||
ids=[config_file.stem for config_file in config_files],
|
||||
)
|
||||
@@ -0,0 +1,68 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""GSM8K and AIME25 accuracy evaluation for Qwen3.8-Flash-Next-FP8."""
|
||||
|
||||
import shlex
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
|
||||
def run_evalscope(
|
||||
eval_config: dict[str, Any], base_url: str, work_dir: Path
|
||||
) -> dict[str, float]:
|
||||
from evalscope.run import run_task
|
||||
|
||||
reports = run_task(
|
||||
{
|
||||
"model": eval_config["model_name"],
|
||||
"api_url": base_url,
|
||||
"api_key": "EMPTY_TOKEN",
|
||||
"datasets": list(eval_config["datasets"]),
|
||||
"eval_batch_size": eval_config.get("eval_batch_size", 32),
|
||||
"generation_config": eval_config["generation_config"],
|
||||
"work_dir": str(work_dir),
|
||||
"no_timestamp": True,
|
||||
}
|
||||
)
|
||||
if not isinstance(reports, dict):
|
||||
raise TypeError(f"Expected EvalScope reports to be a dict, got {type(reports)}")
|
||||
|
||||
return {dataset: float(report.score) for dataset, report in reports.items()}
|
||||
|
||||
|
||||
def test_qwen4_exp_accuracy(config_filename: Path, tmp_path: Path):
|
||||
eval_config = yaml.safe_load(config_filename.read_text(encoding="utf-8"))
|
||||
server_args = shlex.split(eval_config.get("server_args", ""))
|
||||
server_args.extend(["--trust-remote-code", "--disable-uvicorn-access-log"])
|
||||
|
||||
model_name = eval_config["model_name"]
|
||||
print(f"Starting Qwen4Exp evaluation for model: {model_name}")
|
||||
print(f"Datasets: {', '.join(eval_config['datasets'])}")
|
||||
print(f"Server args: {' '.join(server_args)}")
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
model_name,
|
||||
server_args,
|
||||
env_dict=eval_config.get("env"),
|
||||
max_wait_seconds=eval_config.get("startup_max_wait_seconds", 1800),
|
||||
) as remote_server:
|
||||
scores = run_evalscope(eval_config, remote_server.url_for("v1"), tmp_path)
|
||||
|
||||
for dataset, metric_config in eval_config["datasets"].items():
|
||||
score = scores[dataset]
|
||||
threshold = metric_config["metric_threshold"]
|
||||
tolerance = metric_config.get("tolerance", 0.0)
|
||||
minimum_score = threshold - tolerance
|
||||
|
||||
print(
|
||||
f"{dataset}: measured={score:.4f}, expected={threshold:.4f}, "
|
||||
f"tolerance={tolerance:.4f}"
|
||||
)
|
||||
assert score >= minimum_score, (
|
||||
f"{dataset} score too low: {score:.4f} < {threshold:.4f} - "
|
||||
f"{tolerance:.4f} = {minimum_score:.4f}"
|
||||
)
|
||||
@@ -57,13 +57,13 @@ EPS = 1e-6
|
||||
|
||||
|
||||
class _TestGatedNorm:
|
||||
def __init__(self, weight: torch.Tensor) -> None:
|
||||
def __init__(self, weight: torch.Tensor, activation: str) -> None:
|
||||
self.weight = weight
|
||||
self.bias = None
|
||||
self.eps = EPS
|
||||
self.group_size = None
|
||||
self.norm_before_gate = True
|
||||
self.activation = "silu"
|
||||
self.activation = activation
|
||||
|
||||
def __call__(self, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor:
|
||||
return rmsnorm_fn(
|
||||
@@ -99,6 +99,7 @@ def _build_layer(
|
||||
dt_bias: torch.Tensor,
|
||||
conv_weight: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
output_gate_activation: str,
|
||||
):
|
||||
layer = types.SimpleNamespace(
|
||||
prefix=PREFIX,
|
||||
@@ -116,7 +117,7 @@ def _build_layer(
|
||||
dt_bias=dt_bias,
|
||||
conv1d=types.SimpleNamespace(weight=conv_weight, bias=None),
|
||||
kv_cache=(conv_state, ssm_state),
|
||||
norm=_TestGatedNorm(norm_weight),
|
||||
norm=_TestGatedNorm(norm_weight, output_gate_activation),
|
||||
layer_norm_epsilon=EPS,
|
||||
gdn_decode_kernel="cuda",
|
||||
)
|
||||
@@ -250,12 +251,14 @@ def test_fused_forward_uses_packed_entrypoint() -> None:
|
||||
pytest.param([128], [1], [-1], 0, id="pure-decode"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("output_gate_activation", ["silu", "sigmoid"])
|
||||
@torch.inference_mode()
|
||||
def test_fused_model_path_matches_reference(
|
||||
seq_lens: list[int],
|
||||
query_lens: list[int],
|
||||
draft_tokens: list[int],
|
||||
expected_fused_calls: int,
|
||||
output_gate_activation: str,
|
||||
) -> None:
|
||||
"""Fused MTP and its mixed/prefill/decode fallbacks match the reference."""
|
||||
torch.manual_seed(1)
|
||||
@@ -334,6 +337,7 @@ def test_fused_model_path_matches_reference(
|
||||
dt_bias,
|
||||
conv_weight,
|
||||
norm_weight,
|
||||
output_gate_activation,
|
||||
)
|
||||
reference_out = torch.zeros_like(output_gate)
|
||||
with patch.object(
|
||||
@@ -355,6 +359,7 @@ def test_fused_model_path_matches_reference(
|
||||
dt_bias,
|
||||
conv_weight,
|
||||
norm_weight,
|
||||
output_gate_activation,
|
||||
)
|
||||
context.no_compile_layers = {PREFIX: fused_layer}
|
||||
fused_out = torch.zeros_like(output_gate)
|
||||
|
||||
@@ -366,7 +366,7 @@ def test_preprocess_fused_align_matches_scalar_bookkeeping(monkeypatch, token_bi
|
||||
input_batch=scalar_case[1],
|
||||
requests=scalar_case[2],
|
||||
forward_context={},
|
||||
mamba_state_copy_funcs=(),
|
||||
mamba_state_copy_funcs={},
|
||||
copy_bufs=scalar_copy_bufs,
|
||||
)
|
||||
|
||||
@@ -384,7 +384,7 @@ def test_preprocess_fused_align_matches_scalar_bookkeeping(monkeypatch, token_bi
|
||||
input_batch=fused_case[1],
|
||||
requests=fused_case[2],
|
||||
forward_context={},
|
||||
mamba_state_copy_funcs=(),
|
||||
mamba_state_copy_funcs={},
|
||||
copy_bufs=fused_copy_bufs,
|
||||
align_ctx=ctx,
|
||||
)
|
||||
|
||||
@@ -351,3 +351,107 @@ def test_w8a8_block_fp8_deep_gemm_fused_moe(M, N, K, E, topk, seed, monkeypatch)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
torch.testing.assert_close(out, ref_out, atol=0.035, rtol=0.035)
|
||||
|
||||
|
||||
def _dequant_block_fp8(
|
||||
w_fp8: torch.Tensor, w_s: torch.Tensor, block_shape: list[int]
|
||||
) -> torch.Tensor:
|
||||
"""Dequantize a [N, K] fp8 weight with [Nb, Kb] block scales to fp32."""
|
||||
n, k = w_fp8.shape
|
||||
bn, bk = block_shape
|
||||
s = w_s.repeat_interleave(bn, dim=0).repeat_interleave(bk, dim=1)
|
||||
return w_fp8.to(torch.float32) * s[:n, :k]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_rank", [0, 1, 2, 3])
|
||||
@torch.inference_mode()
|
||||
def test_w8a8_block_fp8_fused_moe_refined_block_scales(tp_rank, workspace_init):
|
||||
"""TP-misaligned blockwise FP8 MoE (e.g. Qwen4Exp, intermediate
|
||||
640 at TP=4 -> 160 per rank, not divisible by the checkpoint's 128 block).
|
||||
|
||||
The per-rank scale grid is refined from 128 to 32 (lossless: each 32-block
|
||||
lies inside one global 128-block) so the Triton kernel can consume exact
|
||||
per-shard scales. Simulates one TP rank's shard and checks:
|
||||
1. the refined+sharded scales dequantize exactly like the global scales;
|
||||
2. the Triton fused MoE kernel with block_shape=[32, 32] matches the
|
||||
native-torch blockwise reference on the same shard.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
dtype = torch.bfloat16
|
||||
M, E, topk = 8, 4, 2
|
||||
hidden = 256
|
||||
inter_full = 640
|
||||
tp_size = 4
|
||||
n_shard = inter_full // tp_size # 160
|
||||
ckpt_block = [128, 128]
|
||||
refined_block = [32, 32]
|
||||
factor = ckpt_block[0] // refined_block[0] # 4
|
||||
|
||||
a = torch.randn((M, hidden), dtype=dtype) / 10
|
||||
score = torch.randn((M, E), dtype=dtype)
|
||||
|
||||
(_, w1_full, w1_s_full, _), (_, w2_full, w2_s_full, _) = make_test_weights(
|
||||
E,
|
||||
inter_full,
|
||||
hidden,
|
||||
dtype,
|
||||
torch.float8_e4m3fn,
|
||||
per_out_ch_quant=False,
|
||||
block_shape=ckpt_block,
|
||||
)
|
||||
assert w1_s_full is not None
|
||||
assert w2_s_full is not None
|
||||
|
||||
# TP shard the weights (exact slicing, no scale involvement).
|
||||
lo, hi = tp_rank * n_shard, (tp_rank + 1) * n_shard
|
||||
w1 = torch.cat(
|
||||
[w1_full[:, lo:hi], w1_full[:, inter_full + lo : inter_full + hi]], dim=1
|
||||
)
|
||||
w2 = w2_full[:, :, lo:hi]
|
||||
|
||||
# Refine the global 128-block scales to 32 blocks, then take the shard's
|
||||
# slice -- mirrors the Fp8MoEMethod/RoutedExperts loading path.
|
||||
w1_s32 = w1_s_full.repeat_interleave(factor, dim=-2).repeat_interleave(
|
||||
factor, dim=-1
|
||||
)
|
||||
nb = n_shard // refined_block[0] # 5 local 32-blocks per projection
|
||||
gate_hi = w1_s32.shape[1] // 2
|
||||
w1_s = torch.cat(
|
||||
[
|
||||
w1_s32[:, tp_rank * nb : (tp_rank + 1) * nb],
|
||||
w1_s32[:, gate_hi + tp_rank * nb : gate_hi + (tp_rank + 1) * nb],
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
w2_s32 = w2_s_full.repeat_interleave(factor, dim=-2).repeat_interleave(
|
||||
factor, dim=-1
|
||||
)
|
||||
w2_s = w2_s32[:, :, tp_rank * nb : (tp_rank + 1) * nb]
|
||||
|
||||
# The refined per-shard scales must reproduce the global-scale dequant
|
||||
# exactly (dequant full-width, then slice to the shard).
|
||||
for e in range(E):
|
||||
d32 = _dequant_block_fp8(w1[e, :n_shard], w1_s[e, :nb], refined_block)
|
||||
d128 = _dequant_block_fp8(w1_full[e], w1_s_full[e], ckpt_block)[lo:hi]
|
||||
assert torch.equal(d32, d128)
|
||||
d32 = _dequant_block_fp8(w2[e], w2_s[e], refined_block)
|
||||
d128 = _dequant_block_fp8(w2_full[e], w2_s_full[e], ckpt_block)[:, lo:hi]
|
||||
assert torch.equal(d32, d128)
|
||||
|
||||
quant_config = fp8_w8a8_moe_quant_config(
|
||||
w1_scale=w1_s,
|
||||
w2_scale=w2_s,
|
||||
block_shape=refined_block,
|
||||
)
|
||||
|
||||
topk_weights, topk_ids, _ = fused_topk(a, score.float(), topk, False)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
ref_out = torch_w8a8_block_fp8_moe(
|
||||
a, w1, w2, w1_s, w2_s, topk_weights, topk_ids, refined_block
|
||||
)
|
||||
out = fused_experts(
|
||||
a, w1, w2, topk_weights, topk_ids, quant_config=quant_config
|
||||
)
|
||||
|
||||
torch.testing.assert_close(out, ref_out, atol=0.035, rtol=0.035)
|
||||
|
||||
@@ -28,8 +28,9 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import (
|
||||
TrtLlmBf16ExpertsModular,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
convert_moe_weights_to_flashinfer_trtllm_block_layout,
|
||||
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
UnquantizedMoeBackend,
|
||||
convert_to_unquantized_kernel_format,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
|
||||
@@ -46,9 +47,8 @@ if pytest and (
|
||||
)
|
||||
|
||||
# (m, n, k) = (tokens, intermediate_size_per_partition, hidden_dim).
|
||||
# Covers larger NvFP4-like shapes while keeping BF16's FlashInfer TRTLLM
|
||||
# intermediate-size multiple-of-128 requirement.
|
||||
MNK_FACTORS = [
|
||||
(2, 160, 2560),
|
||||
(2, 1024, 1024),
|
||||
(64, 2048, 1536),
|
||||
(64, 1024, 4096),
|
||||
@@ -97,11 +97,15 @@ def test_trtllm_bf16_moe_modular_no_graph(
|
||||
max_num_tokens=next_power_of_2(m),
|
||||
)
|
||||
|
||||
trtllm_w1, trtllm_w2 = convert_moe_weights_to_flashinfer_trtllm_block_layout(
|
||||
{},
|
||||
trtllm_w1, trtllm_w2 = convert_to_unquantized_kernel_format(
|
||||
UnquantizedMoeBackend.FLASHINFER_TRTLLM,
|
||||
moe_config,
|
||||
w1,
|
||||
w2,
|
||||
)
|
||||
expected_n = (n + 127) // 128 * 128
|
||||
assert moe_config.intermediate_size_per_partition == expected_n
|
||||
assert trtllm_w2.numel() == e * k * expected_n
|
||||
|
||||
trtllm_experts = mk.FusedMoEKernel(
|
||||
maybe_make_prepare_finalize(
|
||||
|
||||
@@ -267,6 +267,7 @@ def test_fused_post_conv_l0():
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("output_gate_activation", ["silu", "sigmoid"])
|
||||
@torch.inference_mode()
|
||||
def test_fused_gdn_decode_post_conv_mtp_head_ratios(
|
||||
head_ratio: int,
|
||||
@@ -274,6 +275,7 @@ def test_fused_gdn_decode_post_conv_mtp_head_ratios(
|
||||
query_lengths: tuple[int, ...],
|
||||
state_dtype: torch.dtype,
|
||||
norm_dtype: torch.dtype,
|
||||
output_gate_activation: str,
|
||||
) -> None:
|
||||
if torch.cuda.get_device_capability() < (8, 0):
|
||||
pytest.skip("fused GDN decode MTP requires compute capability 8.0+")
|
||||
@@ -357,7 +359,7 @@ def test_fused_gdn_decode_post_conv_mtp_head_ratios(
|
||||
z=output_gate,
|
||||
eps=eps,
|
||||
norm_before_gate=True,
|
||||
activation="silu",
|
||||
activation=output_gate_activation,
|
||||
)
|
||||
actual = ops.fused_gdn_decode_post_conv_mtp(
|
||||
mixed_qkv=mixed_qkv,
|
||||
@@ -374,6 +376,7 @@ def test_fused_gdn_decode_post_conv_mtp_head_ratios(
|
||||
out=torch.empty_like(output_gate),
|
||||
scale=scale,
|
||||
norm_eps=eps,
|
||||
output_gate_activation=output_gate_activation,
|
||||
)
|
||||
|
||||
output_error = (actual.float() - expected.float()).norm()
|
||||
|
||||
@@ -18,8 +18,12 @@ from vllm.model_executor.layers.fused_moe.layer import determine_expert_counts
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
|
||||
from vllm.model_executor.layers.quantization.utils.config_utils import (
|
||||
get_quark_ocp_mx_group_size,
|
||||
is_shared_expert_quant_fse_compatible,
|
||||
)
|
||||
from vllm.model_executor.models.qwen3_next import (
|
||||
_should_replicate_misaligned_shared_expert,
|
||||
)
|
||||
from vllm.model_executor.models.utils import PPMissingLayer
|
||||
from vllm.models.deepseek_v4 import quant_config as deepseek_v4_quant_config
|
||||
from vllm.models.minimax_m3.amd import model as minimax_m3_model
|
||||
@@ -605,6 +609,82 @@ def test_quark_shared_expert_fse_compatibility(
|
||||
)
|
||||
|
||||
|
||||
def test_quark_ocp_mx_group_size_for_layer() -> None:
|
||||
quant_config = QuarkConfig({**_QUARK_FSE_CONFIG, "exclude": []})
|
||||
assert (
|
||||
get_quark_ocp_mx_group_size(
|
||||
quant_config,
|
||||
"model.layers.0.mlp.shared_expert.down_proj",
|
||||
)
|
||||
== 32
|
||||
)
|
||||
|
||||
quant_config.quant_config["exclude"] = ["model.layers.0.mlp.shared_expert_gate"]
|
||||
assert (
|
||||
get_quark_ocp_mx_group_size(
|
||||
quant_config,
|
||||
"model.layers.0.mlp.shared_expert.down_proj",
|
||||
)
|
||||
== 32
|
||||
)
|
||||
|
||||
quant_config.quant_config["exclude"] = [
|
||||
"model.layers.0.mlp.shared_expert.down_proj"
|
||||
]
|
||||
assert (
|
||||
get_quark_ocp_mx_group_size(
|
||||
quant_config,
|
||||
"model.layers.0.mlp.shared_expert.down_proj",
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [1, 2, 4])
|
||||
def test_quark_ocp_mx_shared_expert_uses_supported_tp_shards(tp_size: int) -> None:
|
||||
assert not _should_replicate_misaligned_shared_expert(
|
||||
intermediate_size=640,
|
||||
tp_size=tp_size,
|
||||
group_size=32,
|
||||
enable_expert_parallel=False,
|
||||
is_sequence_parallel=False,
|
||||
)
|
||||
|
||||
|
||||
def test_quark_ocp_mx_shared_expert_rejects_plain_tp8() -> None:
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="TP size 8 produces a partition of 80",
|
||||
):
|
||||
_should_replicate_misaligned_shared_expert(
|
||||
intermediate_size=640,
|
||||
tp_size=8,
|
||||
group_size=32,
|
||||
enable_expert_parallel=False,
|
||||
is_sequence_parallel=False,
|
||||
)
|
||||
|
||||
|
||||
def test_quark_ocp_mx_shared_expert_is_replicated_for_tep8() -> None:
|
||||
assert _should_replicate_misaligned_shared_expert(
|
||||
intermediate_size=640,
|
||||
tp_size=8,
|
||||
group_size=32,
|
||||
enable_expert_parallel=True,
|
||||
is_sequence_parallel=False,
|
||||
)
|
||||
|
||||
|
||||
def test_quark_ocp_mx_shared_expert_is_replicated_when_not_tp_divisible() -> None:
|
||||
assert _should_replicate_misaligned_shared_expert(
|
||||
intermediate_size=672,
|
||||
tp_size=8,
|
||||
group_size=32,
|
||||
enable_expert_parallel=True,
|
||||
is_sequence_parallel=False,
|
||||
)
|
||||
|
||||
|
||||
def test_quark_shared_expert_fse_requires_matching_layer_quant_configs() -> None:
|
||||
global_quant_config = {"weight": {"dtype": "fp4"}}
|
||||
quant_config = QuarkConfig(
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,200 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.speculative import SpeculativeConfig
|
||||
from vllm.model_executor.models.config import (
|
||||
Qwen3_5ForConditionalGenerationConfig,
|
||||
Qwen4ExpForConditionalGenerationConfig,
|
||||
)
|
||||
from vllm.models.qwen4_exp.config import (
|
||||
Qwen4ExpConfig,
|
||||
Qwen4ExpTextConfig,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia.model_state import Qwen4ExpModelState
|
||||
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
|
||||
|
||||
|
||||
def _text_config(**kwargs) -> Qwen4ExpTextConfig:
|
||||
values = {
|
||||
"vocab_size": 64,
|
||||
"hidden_size": 16,
|
||||
"intermediate_size": 32,
|
||||
"num_hidden_layers": 2,
|
||||
"num_attention_heads": 2,
|
||||
"num_key_value_heads": 1,
|
||||
"head_dim": 8,
|
||||
"layer_types": ["linear_attention", "full_attention"],
|
||||
"linear_num_key_heads": 2,
|
||||
"linear_num_value_heads": 2,
|
||||
"linear_key_head_dim": 8,
|
||||
"linear_value_head_dim": 8,
|
||||
"num_experts": 0,
|
||||
"hc_count": 2,
|
||||
"hc_lowrank": 4,
|
||||
"ple_layer_ids": [1],
|
||||
"mtp_num_hidden_layers": 1,
|
||||
"mtp": {"hybrid": True},
|
||||
}
|
||||
values.update(kwargs)
|
||||
return Qwen4ExpTextConfig(**values)
|
||||
|
||||
|
||||
def test_qwen4_exp_mtp_returns_sample_and_multi_streams() -> None:
|
||||
from vllm.models.qwen4_exp.nvidia.mtp import (
|
||||
Qwen4ExpMultiTokenPredictor,
|
||||
)
|
||||
|
||||
model = object.__new__(Qwen4ExpMultiTokenPredictor)
|
||||
torch.nn.Module.__init__(model)
|
||||
model.hc_count = 2
|
||||
model.hidden_size = 4
|
||||
model.num_mtp_layers = 1
|
||||
model.layers = [
|
||||
lambda **kwargs: (
|
||||
kwargs["hidden_states"],
|
||||
kwargs["hidden_states"],
|
||||
torch.zeros(kwargs["hidden_states"].shape[0], 2),
|
||||
),
|
||||
]
|
||||
model.hyper_connection_mixer = SimpleNamespace(
|
||||
combine_and_mix=lambda hidden_states, block_output, injection: (
|
||||
hidden_states,
|
||||
hidden_states.unflatten(-1, (2, 4)).mean(-2),
|
||||
None,
|
||||
),
|
||||
)
|
||||
multi_hidden = torch.arange(16, dtype=torch.float32).reshape(2, 8)
|
||||
pp_group = SimpleNamespace(is_first_rank=False, is_last_rank=True)
|
||||
|
||||
with patch(
|
||||
"vllm.models.qwen4_exp.nvidia.mtp.get_pp_group",
|
||||
return_value=pp_group,
|
||||
):
|
||||
sample_hidden, returned_multi_hidden = model.forward(
|
||||
input_ids=None,
|
||||
positions=torch.arange(2),
|
||||
intermediate_tensors={"hidden_states": multi_hidden},
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
sample_hidden,
|
||||
multi_hidden.unflatten(-1, (2, 4)).mean(dim=-2),
|
||||
)
|
||||
assert returned_multi_hidden is multi_hidden
|
||||
|
||||
|
||||
@pytest.mark.parametrize("wrapped_config", [False, True])
|
||||
def test_qwen4_exp_mtp_override_sets_draft_config(
|
||||
wrapped_config: bool,
|
||||
) -> None:
|
||||
text_config = _text_config(
|
||||
architectures=["Qwen4ExpForCausalLM"],
|
||||
index_share_for_mtp_iteration=True,
|
||||
)
|
||||
config = (
|
||||
Qwen4ExpConfig(
|
||||
architectures=["Qwen4ExpForConditionalGeneration"],
|
||||
text_config=text_config,
|
||||
)
|
||||
if wrapped_config
|
||||
else text_config
|
||||
)
|
||||
|
||||
draft_config = SpeculativeConfig.hf_config_override(config)
|
||||
|
||||
assert draft_config.index_share_for_mtp_iteration is True
|
||||
assert draft_config.model_type == "qwen4_exp_mtp"
|
||||
assert draft_config.architectures == ["Qwen4ExpMTP"]
|
||||
assert draft_config.hc_mult == 2
|
||||
assert draft_config.n_predict == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ple_layer_ids", [[1], []])
|
||||
def test_qwen4_exp_rejects_pipeline_parallel_only_with_ple(ple_layer_ids) -> None:
|
||||
"""PLE needs raw input_ids, which non-first pipeline ranks never see. The
|
||||
rest of the architecture is PP-capable, so the refusal must be conditional
|
||||
-- and must land before the engine spends time loading weights."""
|
||||
vllm_config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
hf_text_config=_text_config(ple_layer_ids=ple_layer_ids),
|
||||
multimodal_config=None,
|
||||
),
|
||||
parallel_config=SimpleNamespace(
|
||||
pipeline_parallel_size=2, enable_dbo=False, ubatch_size=1
|
||||
),
|
||||
speculative_config=None,
|
||||
)
|
||||
with patch.object(
|
||||
Qwen3_5ForConditionalGenerationConfig, "verify_and_update_config"
|
||||
):
|
||||
if ple_layer_ids:
|
||||
with pytest.raises(NotImplementedError, match="pipeline_parallel_size=1"):
|
||||
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(
|
||||
vllm_config
|
||||
)
|
||||
else:
|
||||
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(vllm_config)
|
||||
|
||||
|
||||
def test_qwen4_exp_model_state_prepares_ngram_context() -> None:
|
||||
model_state = object.__new__(Qwen4ExpModelState)
|
||||
model_state.uses_ngram_embedding = True
|
||||
model_state.ngram_context_len = 3
|
||||
model_state.ngram_eos_token_id = 99
|
||||
model_state.ngram_context = torch.empty((4, 3), dtype=torch.int32)
|
||||
model_state.ngram_context_offsets = torch.arange(-3, 0, dtype=torch.int64)
|
||||
model_state.ple_query_start_loc = torch.empty(5, dtype=torch.int32)
|
||||
|
||||
input_batch = SimpleNamespace(
|
||||
num_reqs=2,
|
||||
num_reqs_after_padding=3,
|
||||
idx_mapping=torch.tensor([1, 0]),
|
||||
query_start_loc=torch.tensor([0, 2, 3, 3], dtype=torch.int32),
|
||||
)
|
||||
req_states = SimpleNamespace(
|
||||
num_computed_tokens=SimpleNamespace(gpu=torch.tensor([3, 1])),
|
||||
all_token_ids=SimpleNamespace(
|
||||
gpu=torch.tensor([[1, 2, 3, 4], [20, 21, 22, 23]], dtype=torch.int32)
|
||||
),
|
||||
)
|
||||
|
||||
with patch.object(MambaHybridModelState, "prepare_inputs", return_value={}):
|
||||
model_inputs = model_state.prepare_inputs(input_batch, req_states)
|
||||
|
||||
torch.testing.assert_close(
|
||||
model_inputs["query_start_loc"],
|
||||
torch.tensor([0, 2, 3, 3], dtype=torch.int32),
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
model_inputs["ngram_context"],
|
||||
torch.tensor([[99, 99, 20], [1, 2, 3], [99, 99, 99]], dtype=torch.int32),
|
||||
)
|
||||
|
||||
|
||||
def test_qwen4_exp_model_state_prepares_stable_dummy_ngram_inputs() -> None:
|
||||
model_state = object.__new__(Qwen4ExpModelState)
|
||||
model_state.uses_ngram_embedding = True
|
||||
model_state.ngram_eos_token_id = 99
|
||||
model_state.ngram_context = torch.empty((4, 3), dtype=torch.int32)
|
||||
model_state.ple_query_start_loc = torch.empty(5, dtype=torch.int32)
|
||||
|
||||
with patch.object(MambaHybridModelState, "prepare_dummy_inputs", return_value={}):
|
||||
first = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=8)
|
||||
query_start_loc_ptr = first["query_start_loc"].data_ptr()
|
||||
ngram_context_ptr = first["ngram_context"].data_ptr()
|
||||
second = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=8)
|
||||
|
||||
torch.testing.assert_close(
|
||||
second["query_start_loc"], torch.tensor([0, 2, 5, 8], dtype=torch.int32)
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
second["ngram_context"], torch.full((3, 3), 99, dtype=torch.int32)
|
||||
)
|
||||
assert second["query_start_loc"].data_ptr() == query_start_loc_ptr
|
||||
assert second["ngram_context"].data_ptr() == ngram_context_ptr
|
||||
@@ -0,0 +1,94 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.qwen4_exp.nvidia.ops.hc import (
|
||||
grouped_gemma_rmsnorm,
|
||||
hc_combine,
|
||||
hc_combine_norm,
|
||||
hc_gate_mix,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda() or not HAS_TRITON,
|
||||
reason="HC kernels require CUDA and Triton",
|
||||
)
|
||||
|
||||
HC = 4
|
||||
HIDDEN_SIZE = 2560
|
||||
HYPER_HIDDEN_SIZE = HC * HIDDEN_SIZE
|
||||
EPS = 1e-6
|
||||
|
||||
|
||||
def test_grouped_gemma_rmsnorm() -> None:
|
||||
torch.manual_seed(0)
|
||||
x = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
actual = grouped_gemma_rmsnorm(x, weight, EPS, HC)
|
||||
|
||||
grouped = x.float().unflatten(-1, (HC, HIDDEN_SIZE))
|
||||
variance = grouped.square().mean(-1, keepdim=True)
|
||||
expected = grouped * torch.rsqrt(variance + EPS)
|
||||
expected = expected.flatten(-2) * (1.0 + weight.float())
|
||||
torch.testing.assert_close(actual, expected.to(torch.bfloat16))
|
||||
|
||||
|
||||
def test_hc_gate_mix() -> None:
|
||||
torch.manual_seed(0)
|
||||
x = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
gate = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
actual = hc_gate_mix(x, gate, HC)
|
||||
expected = (
|
||||
torch.sigmoid(gate.float().unflatten(-1, (HC, HIDDEN_SIZE)))
|
||||
* x.float().unflatten(-1, (HC, HIDDEN_SIZE))
|
||||
).mean(-2)
|
||||
|
||||
torch.testing.assert_close(actual, expected.to(torch.bfloat16))
|
||||
|
||||
|
||||
def test_hc_combine() -> None:
|
||||
torch.manual_seed(0)
|
||||
block_output = torch.randn(2, HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
residual = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
injection = torch.randn(2, HC, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
actual = hc_combine(residual, block_output, injection, HC)
|
||||
injection_weight = 2.0 * torch.sigmoid(injection.float() / HC)
|
||||
expected = residual.float().unflatten(-1, (HC, HIDDEN_SIZE))
|
||||
expected = expected + block_output.float().unsqueeze(
|
||||
-2
|
||||
) * injection_weight.unsqueeze(-1)
|
||||
|
||||
torch.testing.assert_close(actual, expected.flatten(-2).to(torch.bfloat16))
|
||||
|
||||
|
||||
def test_hc_combine_norm() -> None:
|
||||
torch.manual_seed(0)
|
||||
block_output = torch.randn(2, HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
residual = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
injection = torch.randn(2, HC, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
actual, actual_norm = hc_combine_norm(
|
||||
residual, block_output, injection, weight, EPS, HC
|
||||
)
|
||||
|
||||
injection_weight = 2.0 * torch.sigmoid(injection.float() / HC)
|
||||
expected = residual.float().unflatten(-1, (HC, HIDDEN_SIZE))
|
||||
expected = expected + block_output.float().unsqueeze(
|
||||
-2
|
||||
) * injection_weight.unsqueeze(-1)
|
||||
expected = expected.flatten(-2).to(residual.dtype)
|
||||
grouped = expected.float().unflatten(-1, (HC, HIDDEN_SIZE))
|
||||
variance = grouped.square().mean(-1, keepdim=True)
|
||||
expected_norm = grouped * torch.rsqrt(variance + EPS)
|
||||
expected_norm = expected_norm.flatten(-2) * (1.0 + weight.float())
|
||||
|
||||
torch.testing.assert_close(actual, expected)
|
||||
torch.testing.assert_close(actual_norm, expected_norm.to(torch.bfloat16))
|
||||
@@ -0,0 +1,401 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from functools import partial
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.nn import functional as F
|
||||
|
||||
import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module
|
||||
import vllm.model_executor.parameter as parameter_module
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.models.qwen4_exp.common.ple import (
|
||||
PLEShardOverlap,
|
||||
compute_ple_shard_overlap,
|
||||
copy_ple_embedding_shard_,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia import ple_layer as ple_layer_module
|
||||
from vllm.models.qwen4_exp.nvidia.ple_layer import (
|
||||
Qwen4ExpNGramEmbedding,
|
||||
Qwen4ExpPLEFp8EmbeddingMethod,
|
||||
Qwen4ExpPLELayer,
|
||||
_get_ple_embedding_quant_method,
|
||||
)
|
||||
|
||||
|
||||
def _make_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding:
|
||||
module = Qwen4ExpNGramEmbedding.__new__(Qwen4ExpNGramEmbedding)
|
||||
nn.Module.__init__(module)
|
||||
module.split_ngram_parts = 2
|
||||
module.register_buffer("layer_multipliers", torch.zeros(1, dtype=torch.long))
|
||||
module.register_buffer("ngram_heads_offsets", torch.zeros(1, dtype=torch.long))
|
||||
module.register_buffer("ngram_heads_vocab_sizes", torch.zeros(1, dtype=torch.long))
|
||||
module.ngram_embedding = SimpleNamespace(
|
||||
org_vocab_size=8,
|
||||
embedding_dim=2,
|
||||
weight=nn.Parameter(torch.full((4, 2), -1.0)),
|
||||
shard_indices=SimpleNamespace(
|
||||
org_vocab_start_index=2,
|
||||
org_vocab_end_index=6,
|
||||
),
|
||||
)
|
||||
_set_test_embedding_weight_loader(module.ngram_embedding)
|
||||
return module
|
||||
|
||||
|
||||
def _set_test_embedding_weight_loader(embedding) -> None:
|
||||
embedding.weight.weight_loader = partial(
|
||||
copy_ple_embedding_shard_,
|
||||
tp_start=embedding.shard_indices.org_vocab_start_index,
|
||||
tp_end=embedding.shard_indices.org_vocab_end_index,
|
||||
)
|
||||
|
||||
|
||||
def _make_fp8_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding:
|
||||
module = _make_ngram_embedding_for_load_test()
|
||||
embedding = nn.Module()
|
||||
embedding.org_vocab_size = 8
|
||||
embedding.embedding_dim = 2
|
||||
embedding.shard_indices = SimpleNamespace(
|
||||
org_vocab_start_index=2,
|
||||
org_vocab_end_index=6,
|
||||
)
|
||||
embedding.register_parameter(
|
||||
"weight",
|
||||
nn.Parameter(
|
||||
torch.full((4, 2), -1.0).to(torch.float8_e4m3fn),
|
||||
requires_grad=False,
|
||||
),
|
||||
)
|
||||
embedding.register_parameter(
|
||||
"weight_scale",
|
||||
nn.Parameter(torch.zeros(1, dtype=torch.bfloat16), requires_grad=False),
|
||||
)
|
||||
_set_test_embedding_weight_loader(embedding)
|
||||
module.ngram_embedding = embedding
|
||||
return module
|
||||
|
||||
|
||||
def test_ple_shard_overlap_and_copy() -> None:
|
||||
overlap = compute_ple_shard_overlap(
|
||||
checkpoint_start=2, checkpoint_rows=5, tp_start=4, tp_end=8
|
||||
)
|
||||
assert overlap == PLEShardOverlap(source_start=2, destination_start=0, row_count=3)
|
||||
|
||||
destination = torch.full((4, 2), -1.0)
|
||||
loaded = torch.arange(10, dtype=torch.float64).reshape(5, 2)
|
||||
copied = copy_ple_embedding_shard_(
|
||||
destination,
|
||||
loaded,
|
||||
checkpoint_start=2,
|
||||
tp_start=4,
|
||||
tp_end=8,
|
||||
)
|
||||
|
||||
assert copied == 3
|
||||
torch.testing.assert_close(destination[:3], loaded[2:5].float())
|
||||
torch.testing.assert_close(destination[3], torch.tensor([-1.0, -1.0]))
|
||||
|
||||
|
||||
def test_ple_shard_copy_is_a_noop_without_overlap() -> None:
|
||||
destination = torch.ones(4, 2)
|
||||
copied = copy_ple_embedding_shard_(
|
||||
destination,
|
||||
torch.zeros(2, 2),
|
||||
checkpoint_start=10,
|
||||
tp_start=4,
|
||||
tp_end=8,
|
||||
)
|
||||
|
||||
assert copied == 0
|
||||
assert torch.equal(destination, torch.ones_like(destination))
|
||||
|
||||
|
||||
def test_ngram_embedding_loads_shards_and_ignores_legacy_token_lookup() -> None:
|
||||
module = _make_ngram_embedding_for_load_test()
|
||||
shard_0 = torch.arange(8, dtype=torch.float32).reshape(4, 2)
|
||||
shard_1 = torch.arange(8, 16, dtype=torch.float32).reshape(4, 2)
|
||||
|
||||
loaded = module.load_weights(
|
||||
[
|
||||
("ngram_embedding.shard_0.weight", shard_0),
|
||||
("ngram_embedding.shard_1.weight", shard_1),
|
||||
("token_lookup", torch.tensor([2, 1, 0])),
|
||||
]
|
||||
)
|
||||
|
||||
assert loaded == {"ngram_embedding.weight"}
|
||||
torch.testing.assert_close(
|
||||
module.ngram_embedding.weight,
|
||||
torch.cat((shard_0[2:4], shard_1[0:2])),
|
||||
)
|
||||
|
||||
|
||||
def test_ngram_embedding_loads_fp8_shards_and_global_scale() -> None:
|
||||
module = _make_fp8_ngram_embedding_for_load_test()
|
||||
shard_0 = torch.arange(8, dtype=torch.float32).reshape(4, 2).to(torch.float8_e4m3fn)
|
||||
shard_1 = (
|
||||
torch.arange(8, 16, dtype=torch.float32).reshape(4, 2).to(torch.float8_e4m3fn)
|
||||
)
|
||||
weight_scale = torch.tensor([0.25], dtype=torch.bfloat16)
|
||||
|
||||
loaded = module.load_weights(
|
||||
[
|
||||
("ngram_embedding.shard_0.weight", shard_0),
|
||||
("ngram_embedding.shard_1.weight", shard_1),
|
||||
("ngram_embedding.weight_scale", weight_scale),
|
||||
]
|
||||
)
|
||||
|
||||
assert loaded == {"ngram_embedding.weight", "ngram_embedding.weight_scale"}
|
||||
assert module.ngram_embedding.weight.dtype == torch.float8_e4m3fn
|
||||
assert torch.equal(
|
||||
module.ngram_embedding.weight.float(),
|
||||
torch.cat((shard_0[2:4], shard_1[0:2])).float(),
|
||||
)
|
||||
assert torch.equal(module.ngram_embedding.weight_scale, weight_scale)
|
||||
|
||||
|
||||
def _make_fp8_embedding_layer(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> embedding_module.VocabParallelEmbedding:
|
||||
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
embedding_module, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
)
|
||||
monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
parameter_module, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
)
|
||||
method = Qwen4ExpPLEFp8EmbeddingMethod()
|
||||
layer = embedding_module.VocabParallelEmbedding(
|
||||
3,
|
||||
2,
|
||||
params_dtype=torch.bfloat16,
|
||||
padding_size=1,
|
||||
quant_method=method,
|
||||
)
|
||||
weight = torch.tensor([[1.0, 2.0], [4.0, 8.0], [16.0, 32.0]])
|
||||
layer.weight.data.copy_(weight.to(torch.float8_e4m3fn))
|
||||
layer.weight_scale.data.copy_(torch.tensor([0.25], dtype=torch.bfloat16))
|
||||
return layer
|
||||
|
||||
|
||||
def test_ple_fp8_embedding_dequantizes_in_ple_layer(monkeypatch) -> None:
|
||||
layer = _make_fp8_embedding_layer(monkeypatch)
|
||||
quantized_output = layer(torch.tensor([2, 0]))
|
||||
ple_layer = Qwen4ExpPLELayer.__new__(Qwen4ExpPLELayer)
|
||||
nn.Module.__init__(ple_layer)
|
||||
ple_layer.ple_embedding = nn.Module()
|
||||
ple_layer.ple_embedding.ngram_embedding = layer
|
||||
|
||||
output = ple_layer._dequantize_embeddings(
|
||||
quantized_output,
|
||||
torch.bfloat16,
|
||||
)
|
||||
|
||||
assert layer.weight.dtype == torch.float8_e4m3fn
|
||||
assert layer.weight_scale.dtype == torch.bfloat16
|
||||
assert quantized_output.dtype == torch.float8_e4m3fn
|
||||
assert output.dtype == torch.bfloat16
|
||||
weight = torch.tensor([[1.0, 2.0], [4.0, 8.0], [16.0, 32.0]])
|
||||
torch.testing.assert_close(output, (weight[[2, 0]] * 0.25).bfloat16())
|
||||
|
||||
|
||||
def test_ple_fp8_embedding_uses_int8_for_tp_reduce(monkeypatch) -> None:
|
||||
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
embedding_module, "get_tensor_model_parallel_world_size", lambda: 2
|
||||
)
|
||||
monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
parameter_module, "get_tensor_model_parallel_world_size", lambda: 2
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
embedding_module,
|
||||
"get_masked_input_and_mask",
|
||||
lambda *args: (
|
||||
torch.tensor([0, 0]),
|
||||
torch.tensor([False, True]),
|
||||
),
|
||||
)
|
||||
reduced_dtypes = []
|
||||
|
||||
def all_reduce(tensor: torch.Tensor) -> torch.Tensor:
|
||||
reduced_dtypes.append(tensor.dtype)
|
||||
return tensor.clone()
|
||||
|
||||
monkeypatch.setattr(
|
||||
embedding_module,
|
||||
"tensor_model_parallel_all_reduce",
|
||||
all_reduce,
|
||||
)
|
||||
layer = embedding_module.VocabParallelEmbedding(
|
||||
4,
|
||||
2,
|
||||
params_dtype=torch.bfloat16,
|
||||
padding_size=1,
|
||||
quant_method=Qwen4ExpPLEFp8EmbeddingMethod(),
|
||||
)
|
||||
layer.weight.data.copy_(
|
||||
torch.tensor([[1.0, 2.0], [4.0, 8.0]]).to(torch.float8_e4m3fn)
|
||||
)
|
||||
|
||||
output = layer(torch.tensor([0, 2]))
|
||||
|
||||
assert reduced_dtypes == [torch.int8]
|
||||
assert output.dtype == torch.float8_e4m3fn
|
||||
torch.testing.assert_close(output[0].float(), layer.weight[0].float())
|
||||
assert torch.count_nonzero(output[1].float()) == 0
|
||||
|
||||
|
||||
def test_ple_fp8_embedding_respects_checkpoint_shard_exclusions() -> None:
|
||||
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
|
||||
quant_config = Fp8Config(
|
||||
is_checkpoint_fp8_serialized=True,
|
||||
ignored_layers=[],
|
||||
weight_block_size=[128, 128],
|
||||
)
|
||||
assert isinstance(
|
||||
_get_ple_embedding_quant_method(quant_config, prefix),
|
||||
Qwen4ExpPLEFp8EmbeddingMethod,
|
||||
)
|
||||
|
||||
quant_config.ignored_layers = [f"{prefix}.shard_0"]
|
||||
assert _get_ple_embedding_quant_method(quant_config, prefix) is None
|
||||
|
||||
|
||||
def test_ple_ngram_ids_custom_op_uses_current_request_layout(monkeypatch) -> None:
|
||||
class RuntimeNGramEmbedding(nn.Module):
|
||||
def compute_ngram_ids(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
ngram_context: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
del input_ids, ngram_context
|
||||
num_reqs = query_start_loc.numel() - 1
|
||||
return torch.full((4, 2), num_reqs, dtype=torch.long)
|
||||
|
||||
layer = Qwen4ExpPLELayer.__new__(Qwen4ExpPLELayer)
|
||||
nn.Module.__init__(layer)
|
||||
layer.ple_embedding = RuntimeNGramEmbedding()
|
||||
monkeypatch.setattr(
|
||||
ple_layer_module,
|
||||
"get_forward_context",
|
||||
lambda: SimpleNamespace(no_compile_layers={"ple": layer}),
|
||||
)
|
||||
input_ids = torch.arange(4)
|
||||
ngram_context = torch.zeros(2, 2, dtype=torch.long)
|
||||
output = torch.empty(4, 2, dtype=torch.long)
|
||||
|
||||
ple_layer_module.qwen4_exp_compute_ple_ngram_ids(
|
||||
input_ids,
|
||||
torch.tensor([0, 4]),
|
||||
ngram_context,
|
||||
output,
|
||||
"ple",
|
||||
)
|
||||
assert torch.equal(output, torch.ones_like(output))
|
||||
|
||||
ple_layer_module.qwen4_exp_compute_ple_ngram_ids(
|
||||
input_ids,
|
||||
torch.tensor([0, 2, 4]),
|
||||
ngram_context,
|
||||
output,
|
||||
"ple",
|
||||
)
|
||||
assert torch.equal(output, torch.full_like(output, 2))
|
||||
|
||||
|
||||
def test_dilated_ple_spec_state_rolls_back_before_next_forward() -> None:
|
||||
module = Qwen4ExpPLELayer.__new__(Qwen4ExpPLELayer)
|
||||
nn.Module.__init__(module)
|
||||
module.conv_state_len = 6
|
||||
module.short_conv_dilation = 2
|
||||
|
||||
conv_weights = torch.tensor([[0.25, -0.5, 0.75, 1.0]])
|
||||
conv_state = torch.zeros(2, 1, 9)
|
||||
conv_state[1] = torch.arange(1, 10, dtype=torch.float32).reshape(1, 9)
|
||||
first_inputs = torch.tensor([[10.0], [20.0], [30.0], [40.0]])
|
||||
initial_state = conv_state[1:].clone()
|
||||
first_history = torch.cat(
|
||||
(initial_state[..., : module.conv_state_len], first_inputs.T.unsqueeze(0)),
|
||||
dim=-1,
|
||||
)
|
||||
|
||||
graph_padded_inputs = F.pad(first_inputs, (0, 0, 0, 4))
|
||||
first_output = module._short_conv_dilated_spec_batched(
|
||||
graph_padded_inputs,
|
||||
conv_state,
|
||||
conv_weights,
|
||||
torch.tensor([1, 0]),
|
||||
torch.tensor([0, 4, 4]),
|
||||
torch.tensor([1, 0]),
|
||||
spec_query_len=4,
|
||||
)
|
||||
|
||||
expected_first_output = F.silu(
|
||||
F.conv1d(
|
||||
first_history,
|
||||
conv_weights.unsqueeze(1),
|
||||
groups=1,
|
||||
dilation=module.short_conv_dilation,
|
||||
)
|
||||
).transpose(1, 2)[0]
|
||||
expected_first_state = first_history[..., 1:10]
|
||||
torch.testing.assert_close(first_output[:4], expected_first_output)
|
||||
assert torch.count_nonzero(first_output[4:]) == 0
|
||||
assert torch.count_nonzero(conv_state[0]) == 0
|
||||
torch.testing.assert_close(conv_state[1:], expected_first_state)
|
||||
|
||||
second_inputs = torch.tensor([[50.0], [60.0]])
|
||||
rollback_state = expected_first_state[..., 1:7]
|
||||
padded_second_inputs = F.pad(second_inputs.T.unsqueeze(0), (0, 2))
|
||||
second_history = torch.cat((rollback_state, padded_second_inputs), dim=-1)
|
||||
expected_second_state = expected_first_state.clone()
|
||||
expected_second_state[..., :7] = second_history[..., 1:8]
|
||||
|
||||
second_output = module._short_conv_dilated_spec_batched(
|
||||
second_inputs,
|
||||
conv_state,
|
||||
conv_weights,
|
||||
torch.tensor([1]),
|
||||
torch.tensor([0, 2]),
|
||||
torch.tensor([2]),
|
||||
spec_query_len=4,
|
||||
)
|
||||
|
||||
expected_second_output = F.silu(
|
||||
F.conv1d(
|
||||
second_history,
|
||||
conv_weights.unsqueeze(1),
|
||||
groups=1,
|
||||
dilation=module.short_conv_dilation,
|
||||
)
|
||||
).transpose(1, 2)[0, :2]
|
||||
torch.testing.assert_close(second_output, expected_second_output)
|
||||
torch.testing.assert_close(conv_state[1:], expected_second_state)
|
||||
|
||||
|
||||
def test_ple_short_conv_uses_fallback_when_profile_metadata_is_omitted(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
module = Qwen4ExpPLELayer.__new__(Qwen4ExpPLELayer)
|
||||
nn.Module.__init__(module)
|
||||
module.prefix = "model.layers.1.ple"
|
||||
inputs = torch.arange(6, dtype=torch.float32).reshape(3, 2)
|
||||
expected = inputs + 1
|
||||
monkeypatch.setattr(module, "_short_conv_fallback", lambda _: expected)
|
||||
monkeypatch.setattr(
|
||||
ple_layer_module,
|
||||
"get_forward_context",
|
||||
lambda: SimpleNamespace(attn_metadata={"model.layers.0.self_attn": object()}),
|
||||
)
|
||||
|
||||
output = module._short_conv(inputs)
|
||||
|
||||
assert output is expected
|
||||
@@ -0,0 +1,297 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
|
||||
from vllm.models.qwen4_exp.amd import (
|
||||
model as _qwen4_exp_model, # noqa: F401
|
||||
)
|
||||
from vllm.models.qwen4_exp.amd import ple_layer as ple_layer_module
|
||||
from vllm.models.qwen4_exp.amd.indexer_qsa import (
|
||||
apply_qsa_rmsnorm,
|
||||
apply_qsa_rope,
|
||||
)
|
||||
from vllm.models.qwen4_exp.amd.ops import qsa as qsa_ops
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="AMD QSA requires ROCm",
|
||||
)
|
||||
|
||||
requires_qsa_kernels = pytest.mark.skipif(
|
||||
not HAS_TRITON,
|
||||
reason="AMD QSA kernels require Triton",
|
||||
)
|
||||
|
||||
|
||||
def test_ple_ngram_embedding_custom_op_uses_resident_weight(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
layer_name = "model.layers.0.ple"
|
||||
layer = ple_layer_module.Qwen4ExpPLELayer.__new__(ple_layer_module.Qwen4ExpPLELayer)
|
||||
torch.nn.Module.__init__(layer)
|
||||
layer.ple_embedding = torch.nn.Module()
|
||||
layer.ple_embedding.ngram_embedding = torch.nn.Embedding(8, 3)
|
||||
context = SimpleNamespace(no_compile_layers={layer_name: layer})
|
||||
monkeypatch.setattr(ple_layer_module, "get_forward_context", lambda: context)
|
||||
|
||||
ngram_ids = torch.tensor([[0, 1], [2, 3]])
|
||||
output = torch.empty(2, 6)
|
||||
ple_layer_module.qwen4_exp_amd_ple_ngram_embedding(
|
||||
ngram_ids,
|
||||
output,
|
||||
layer_name,
|
||||
)
|
||||
|
||||
expected = layer.ple_embedding.ngram_embedding(ngram_ids).flatten(-2)
|
||||
torch.testing.assert_close(output, expected)
|
||||
|
||||
|
||||
def _qsa_sparse_paged_attention_reference(
|
||||
q: torch.Tensor,
|
||||
k_cache: torch.Tensor,
|
||||
v_cache: torch.Tensor,
|
||||
logical_indices: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
token_to_req: torch.Tensor,
|
||||
softmax_scale: float,
|
||||
) -> torch.Tensor:
|
||||
output = torch.zeros_like(q)
|
||||
repeats = q.shape[1] // k_cache.shape[2]
|
||||
page_size = k_cache.shape[1]
|
||||
for row in range(q.shape[0]):
|
||||
logical = logical_indices[row]
|
||||
logical = logical[logical >= 0].long()
|
||||
if not logical.numel():
|
||||
continue
|
||||
request = token_to_req[row].long()
|
||||
pages = block_table[request, logical // page_size].long()
|
||||
offsets = logical % page_size
|
||||
keys = k_cache[pages, offsets].repeat_interleave(repeats, dim=1)
|
||||
values = v_cache[pages, offsets].repeat_interleave(repeats, dim=1)
|
||||
scores = torch.einsum("hd,khd->hk", q[row].float(), keys.float())
|
||||
probabilities = torch.softmax(scores * softmax_scale, dim=-1)
|
||||
output[row] = torch.einsum("hk,khd->hd", probabilities, values.float()).to(
|
||||
q.dtype
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
def test_qsa_rope_uses_platform_dispatch() -> None:
|
||||
tensor = torch.arange(16, dtype=torch.float32).reshape(2, 2, 4)
|
||||
positions = torch.tensor([0, 1])
|
||||
calls = []
|
||||
|
||||
def apply_rotary_emb(
|
||||
rotary_input: torch.Tensor,
|
||||
cos: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
calls.append((rotary_input, cos, sin))
|
||||
return rotary_input + 1
|
||||
|
||||
rotary_emb = SimpleNamespace(
|
||||
rotary_dim=2,
|
||||
apply_rotary_emb=apply_rotary_emb,
|
||||
_match_cos_sin_cache_dtype=lambda _: torch.zeros(2, 4),
|
||||
)
|
||||
|
||||
output = apply_qsa_rope(rotary_emb, positions, tensor)
|
||||
|
||||
assert len(calls) == 1
|
||||
torch.testing.assert_close(output[..., :2], tensor[..., :2] + 1)
|
||||
torch.testing.assert_close(output[..., 2:], tensor[..., 2:])
|
||||
|
||||
|
||||
def test_qsa_rmsnorm_uses_portable_implementation(default_vllm_config) -> None:
|
||||
norm = GemmaRMSNorm(4, eps=1e-6)
|
||||
norm.weight.data.copy_(torch.tensor([0.1, -0.2, 0.3, -0.4]))
|
||||
tensor = torch.arange(8, dtype=torch.float32).reshape(2, 4)
|
||||
output = apply_qsa_rmsnorm(norm, tensor)
|
||||
|
||||
torch.testing.assert_close(output, norm.forward_native(tensor))
|
||||
|
||||
|
||||
def test_qsa_selection_uses_portable_topk_on_rocm(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
rows = 2
|
||||
token_topk = 8
|
||||
compress_ratio = 4
|
||||
block_topk = token_topk // compress_ratio
|
||||
blocks = torch.empty((rows, block_topk), dtype=torch.int32)
|
||||
visible_blocks = torch.tensor([4, 6], dtype=torch.int32)
|
||||
logits = torch.empty((rows, 8), dtype=torch.float32)
|
||||
selection_output = torch.empty(
|
||||
(rows, token_topk + compress_ratio - 1), dtype=torch.int32
|
||||
)
|
||||
topk_call: dict[str, Any] = {}
|
||||
|
||||
monkeypatch.setattr(
|
||||
qsa_ops,
|
||||
"qsa_mqa_paged",
|
||||
lambda *args, **kwargs: (logits, visible_blocks),
|
||||
)
|
||||
monkeypatch.setattr(qsa_ops.current_platform, "is_cuda", lambda: False)
|
||||
|
||||
def top_k_per_row_decode(
|
||||
topk_logits,
|
||||
next_n,
|
||||
seq_lens,
|
||||
raw_topk_indices,
|
||||
num_rows,
|
||||
stride0,
|
||||
stride1,
|
||||
topk_tokens,
|
||||
) -> None:
|
||||
topk_call.update(
|
||||
logits=topk_logits,
|
||||
next_n=next_n,
|
||||
seq_lens=seq_lens,
|
||||
raw_topk_indices=raw_topk_indices,
|
||||
num_rows=num_rows,
|
||||
strides=(stride0, stride1),
|
||||
topk_tokens=topk_tokens,
|
||||
)
|
||||
raw_topk_indices.zero_()
|
||||
|
||||
def expand_qsa_block_indices(*args) -> None:
|
||||
args[-1].fill_(-1)
|
||||
|
||||
monkeypatch.setattr(qsa_ops.ops, "top_k_per_row_decode", top_k_per_row_decode)
|
||||
monkeypatch.setattr(
|
||||
qsa_ops,
|
||||
"expand_qsa_block_indices_cuda",
|
||||
expand_qsa_block_indices,
|
||||
)
|
||||
|
||||
output = qsa_ops.qsa_select_paged_tokens(
|
||||
torch.empty((rows, 1, 1)),
|
||||
torch.empty((1, 4, 1, 1)),
|
||||
torch.empty((1, 2), dtype=torch.int32),
|
||||
torch.zeros(rows, dtype=torch.int32),
|
||||
torch.arange(rows, dtype=torch.int32),
|
||||
torch.tensor([8], dtype=torch.int32),
|
||||
token_topk,
|
||||
compress_ratio,
|
||||
selection_output,
|
||||
)
|
||||
|
||||
assert output is selection_output
|
||||
assert torch.all(output == -1)
|
||||
assert topk_call["logits"] is logits
|
||||
assert topk_call["next_n"] == 1
|
||||
assert topk_call["seq_lens"] is visible_blocks
|
||||
assert topk_call["raw_topk_indices"].shape == blocks.shape
|
||||
assert topk_call["num_rows"] == rows
|
||||
assert topk_call["strides"] == (logits.stride(0), logits.stride(1))
|
||||
assert topk_call["topk_tokens"] == block_topk
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
@pytest.mark.parametrize(
|
||||
("num_rows", "num_query_heads", "num_kv_heads", "page_size"),
|
||||
[
|
||||
pytest.param(1, 24, 2, 1792, id="tp1_split64"),
|
||||
pytest.param(16, 12, 1, 1792, id="tp2_split32"),
|
||||
pytest.param(32, 6, 1, 1024, id="tp4_split8"),
|
||||
pytest.param(257, 6, 1, 1024, id="tp4_split4"),
|
||||
pytest.param(513, 6, 1, 1024, id="tp4_split1"),
|
||||
],
|
||||
)
|
||||
def test_qsa_sparse_paged_attention_matches_reference(
|
||||
num_rows: int,
|
||||
num_query_heads: int,
|
||||
num_kv_heads: int,
|
||||
page_size: int,
|
||||
) -> None:
|
||||
torch.manual_seed(2)
|
||||
head_dim = 256
|
||||
num_requests = 2
|
||||
num_selected_pages = 64
|
||||
num_pages_per_request = num_selected_pages + 1
|
||||
num_cache_blocks = num_requests * num_pages_per_request
|
||||
indexer_budget = 2048
|
||||
indexer_compress_ratio = 4
|
||||
selection_width = indexer_budget + indexer_compress_ratio - 1
|
||||
q = torch.randn(
|
||||
num_rows, num_query_heads, head_dim, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
kv_cache = torch.randn(
|
||||
num_cache_blocks,
|
||||
page_size,
|
||||
num_kv_heads,
|
||||
2 * head_dim,
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
k_cache, v_cache = kv_cache.split(head_dim, dim=-1)
|
||||
block_table = (
|
||||
torch.randperm(num_cache_blocks, device="cuda")
|
||||
.reshape(num_requests, num_pages_per_request)
|
||||
.to(torch.int32)
|
||||
)
|
||||
rows_per_request = math.ceil(num_rows / num_requests)
|
||||
row_indices = torch.arange(num_rows, device="cuda", dtype=torch.int32)
|
||||
token_to_req = row_indices // rows_per_request
|
||||
request_row_counts = torch.tensor(
|
||||
[rows_per_request, num_rows - rows_per_request],
|
||||
device="cuda",
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
context_length = num_pages_per_request * page_size - 1
|
||||
block_topk = indexer_budget // indexer_compress_ratio
|
||||
compressed_blocks_per_page = page_size // indexer_compress_ratio
|
||||
selection = torch.arange(block_topk, device="cuda")
|
||||
selected_pages = selection % num_selected_pages
|
||||
selected_offsets = selection // num_selected_pages
|
||||
row_shifts = 2 * row_indices.unsqueeze(1)
|
||||
selected_offsets = (selected_offsets + row_shifts) % compressed_blocks_per_page
|
||||
block_indices = (selected_pages * compressed_blocks_per_page + selected_offsets).to(
|
||||
torch.int32
|
||||
)
|
||||
rows_within_request = row_indices % rows_per_request
|
||||
query_positions = (
|
||||
context_length - request_row_counts[token_to_req.long()] + rows_within_request
|
||||
).to(torch.int64)
|
||||
sequence_lengths = torch.full(
|
||||
(num_requests,), context_length, device="cuda", dtype=torch.int32
|
||||
)
|
||||
logical_indices = qsa_ops.expand_qsa_block_indices_cuda(
|
||||
block_indices,
|
||||
query_positions,
|
||||
sequence_lengths,
|
||||
token_to_req,
|
||||
indexer_compress_ratio,
|
||||
indexer_budget,
|
||||
)
|
||||
assert logical_indices.shape == (num_rows, selection_width)
|
||||
|
||||
actual = qsa_ops.qsa_sparse_paged_attention(
|
||||
q,
|
||||
k_cache,
|
||||
v_cache,
|
||||
logical_indices,
|
||||
block_table,
|
||||
token_to_req,
|
||||
)
|
||||
expected = _qsa_sparse_paged_attention_reference(
|
||||
q,
|
||||
k_cache,
|
||||
v_cache,
|
||||
logical_indices,
|
||||
block_table,
|
||||
token_to_req,
|
||||
q.shape[-1] ** -0.5,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, rtol=2e-2, atol=2e-2)
|
||||
@@ -0,0 +1,297 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Correctness tests for the fused QSA pre-indexer."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.qwen4_exp.common.qsa_cache import (
|
||||
canonical_qsa_rope_positions,
|
||||
circular_qsa_slot_mapping,
|
||||
compressed_qsa_slot_mapping,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia.indexer_qsa import apply_qsa_rope
|
||||
from vllm.models.qwen4_exp.nvidia.ops.qsa import (
|
||||
qsa_compress_groups_with_ratio,
|
||||
qsa_store_cache_rows,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia.ops.qsa_pre_indexer import (
|
||||
qsa_pre_indexer,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
|
||||
requires_qsa_kernels = pytest.mark.skipif(
|
||||
not current_platform.is_cuda() or not HAS_TRITON,
|
||||
reason="QSA kernels require CUDA and Triton",
|
||||
)
|
||||
|
||||
HQ, D = 4, 128
|
||||
CR = 4
|
||||
MROPE_SECTION = (11, 11, 10)
|
||||
EPS = 1e-6
|
||||
BLOCK_SIZE = 16
|
||||
COMP_PAGE = BLOCK_SIZE // CR
|
||||
ROPE_POS_OFFSET = D
|
||||
RTOL = 1.6e-2
|
||||
ATOL = 1e-2
|
||||
MIXED_BATCH = ([260, 259, 138], [1, 1, 37], [8, 8, 8])
|
||||
|
||||
|
||||
def _make_block_table(block_counts):
|
||||
num_blocks = sum(block_counts)
|
||||
table = torch.full((len(block_counts), max(block_counts)), -1, dtype=torch.int32)
|
||||
physical_blocks = torch.randperm(num_blocks)
|
||||
offset = 0
|
||||
for request, count in enumerate(block_counts):
|
||||
table[request, :count] = physical_blocks[offset : offset + count]
|
||||
offset += count
|
||||
return table, num_blocks
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
@pytest.mark.parametrize(
|
||||
"mrope,is_2d_positions,cache_rope_positions,state_size,seq_lens,query_lens,history_lens",
|
||||
[
|
||||
pytest.param(True, True, True, 4, *MIXED_BATCH, id="mrope"),
|
||||
pytest.param(False, False, False, 4, *MIXED_BATCH, id="text"),
|
||||
pytest.param(True, False, True, 8, *MIXED_BATCH, id="mrope-model-1d"),
|
||||
pytest.param(True, True, False, 4, *MIXED_BATCH, id="mrope-no-position-cache"),
|
||||
pytest.param(
|
||||
True,
|
||||
False,
|
||||
False,
|
||||
8,
|
||||
*MIXED_BATCH,
|
||||
id="mrope-model-1d-no-position-cache",
|
||||
),
|
||||
pytest.param(
|
||||
False, False, True, 4, *MIXED_BATCH, id="text-with-position-cache"
|
||||
),
|
||||
pytest.param(True, True, True, 4, [37], [37], [0], id="fresh"),
|
||||
pytest.param(True, True, True, 4, [4097], [4097], [0], id="tiled"),
|
||||
],
|
||||
)
|
||||
def test_qsa_fused_pre_indexer_matches_unfused(
|
||||
mrope,
|
||||
is_2d_positions,
|
||||
cache_rope_positions,
|
||||
state_size,
|
||||
seq_lens,
|
||||
query_lens,
|
||||
history_lens,
|
||||
) -> None:
|
||||
from flashinfer.norm import gemma_rmsnorm
|
||||
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
|
||||
device = "cuda"
|
||||
rope_params = {
|
||||
"partial_rotary_factor": 0.25,
|
||||
"rope_theta": 10000000,
|
||||
"rope_type": "default",
|
||||
}
|
||||
if mrope:
|
||||
rope_params["mrope_interleaved"] = True
|
||||
rope_params["mrope_section"] = list(MROPE_SECTION)
|
||||
with torch.device(device):
|
||||
rope = get_rope(
|
||||
head_size=256,
|
||||
max_position=32768,
|
||||
rope_parameters=rope_params,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
token_to_req = torch.cat(
|
||||
[
|
||||
torch.full((length,), request, dtype=torch.int32)
|
||||
for request, length in enumerate(query_lens)
|
||||
]
|
||||
).to(device)
|
||||
logical_positions = torch.cat(
|
||||
[
|
||||
torch.arange(seq_len - query_len, seq_len, dtype=torch.int64)
|
||||
for seq_len, query_len in zip(seq_lens, query_lens)
|
||||
]
|
||||
).to(device)
|
||||
num_tokens = logical_positions.numel()
|
||||
query_start_loc = torch.tensor(
|
||||
[0, *torch.tensor(query_lens).cumsum(0).tolist()], dtype=torch.int32
|
||||
).to(device)
|
||||
positions = (
|
||||
torch.stack(
|
||||
[
|
||||
logical_positions,
|
||||
logical_positions // 7 + 3,
|
||||
logical_positions // 13 + 11,
|
||||
]
|
||||
)
|
||||
if is_2d_positions
|
||||
else logical_positions
|
||||
)
|
||||
position_rows = (
|
||||
canonical_qsa_rope_positions(positions)
|
||||
if cache_rope_positions
|
||||
else logical_positions.view(-1, 1, 1).expand(-1, 1, 3)
|
||||
)
|
||||
|
||||
compressed_block_counts = [
|
||||
(seq_len // CR + COMP_PAGE - 1) // COMP_PAGE for seq_len in seq_lens
|
||||
]
|
||||
raw_block_table, num_raw_blocks = _make_block_table([1] * len(seq_lens))
|
||||
compressed_block_table, num_compressed_blocks = _make_block_table(
|
||||
compressed_block_counts
|
||||
)
|
||||
raw_block_table = raw_block_table.to(device)
|
||||
raw_slots = circular_qsa_slot_mapping(
|
||||
raw_block_table,
|
||||
token_to_req,
|
||||
logical_positions,
|
||||
state_size,
|
||||
query_start_loc,
|
||||
)
|
||||
compressed_slots = compressed_qsa_slot_mapping(
|
||||
compressed_block_table.to(device),
|
||||
token_to_req,
|
||||
logical_positions,
|
||||
COMP_PAGE,
|
||||
CR,
|
||||
)
|
||||
group_counts = torch.tensor(
|
||||
[
|
||||
seq_len // CR - (seq_len - query_len) // CR
|
||||
for seq_len, query_len in zip(seq_lens, query_lens)
|
||||
],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
k_work_counts = torch.maximum(group_counts, torch.ones_like(group_counts))
|
||||
k_start_loc = torch.cat([k_work_counts.new_zeros(1), k_work_counts.cumsum(0)])
|
||||
work_requests = torch.repeat_interleave(
|
||||
torch.arange(len(query_lens), dtype=torch.int32, device=device),
|
||||
k_work_counts,
|
||||
)
|
||||
local_work = torch.arange(
|
||||
int(k_start_loc[-1]), dtype=torch.int32, device=device
|
||||
) - torch.repeat_interleave(k_start_loc[:-1], k_work_counts)
|
||||
max_k_work = (num_tokens + (CR - 1) * len(query_lens)) // CR
|
||||
k_work_metadata = torch.full((max_k_work, 2), -1, dtype=torch.int32, device=device)
|
||||
k_work_metadata[: work_requests.numel()] = torch.stack(
|
||||
(work_requests, local_work), dim=1
|
||||
)
|
||||
|
||||
raw_width = D + 12 if cache_rope_positions else D
|
||||
# Match vLLM's padded-page cache layout: rows are contiguous, while physical
|
||||
# blocks have a larger stride than their logical contents.
|
||||
raw_page_elements = state_size * raw_width
|
||||
fused_raw_storage = torch.zeros(
|
||||
num_raw_blocks,
|
||||
raw_page_elements + 16,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
fused_raw = torch.as_strided(
|
||||
fused_raw_storage,
|
||||
(num_raw_blocks, state_size, 1, raw_width),
|
||||
(raw_page_elements + 16, raw_width, raw_width, 1),
|
||||
)
|
||||
for request, history_len in enumerate(history_lens):
|
||||
history_end = seq_lens[request] - query_lens[request]
|
||||
for position in range(history_end - history_len, history_end):
|
||||
block = int(raw_block_table[request, 0])
|
||||
row = fused_raw[block, position % state_size, 0]
|
||||
row[:D] = torch.randn(D, dtype=torch.bfloat16, device=device)
|
||||
if cache_rope_positions:
|
||||
row[ROPE_POS_OFFSET:].view(torch.int64).copy_(
|
||||
torch.tensor(
|
||||
[position, position // 7 + 3, position // 13 + 11],
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
)
|
||||
)
|
||||
unfused_raw = fused_raw.clone()
|
||||
compressed_page_elements = COMP_PAGE * D
|
||||
fused_compressed_storage = torch.zeros(
|
||||
num_compressed_blocks,
|
||||
compressed_page_elements + 16,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
fused_compressed = torch.as_strided(
|
||||
fused_compressed_storage,
|
||||
(num_compressed_blocks, COMP_PAGE, 1, D),
|
||||
(compressed_page_elements + 16, D, D, 1),
|
||||
)
|
||||
unfused_compressed = fused_compressed.clone()
|
||||
|
||||
projected_qk = torch.randn(
|
||||
num_tokens, (HQ + 1) * D, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
q_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
|
||||
k_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
|
||||
|
||||
fused_query = torch.empty(num_tokens, HQ, D, dtype=torch.bfloat16, device=device)
|
||||
qsa_pre_indexer(
|
||||
projected_qk[:, : HQ * D],
|
||||
projected_qk[:, HQ * D :],
|
||||
positions,
|
||||
rope.cos_sin_cache,
|
||||
q_weight,
|
||||
k_weight,
|
||||
EPS,
|
||||
fused_query,
|
||||
fused_raw,
|
||||
raw_slots,
|
||||
raw_block_table,
|
||||
query_start_loc,
|
||||
logical_positions,
|
||||
fused_compressed,
|
||||
compressed_slots,
|
||||
k_work_metadata,
|
||||
compress_ratio=CR,
|
||||
mrope_section=MROPE_SECTION if mrope else None,
|
||||
rope_pos_offset=ROPE_POS_OFFSET if cache_rope_positions else None,
|
||||
)
|
||||
|
||||
unfused_query = projected_qk[:, : HQ * D].reshape(num_tokens, HQ, D)
|
||||
unfused_query = gemma_rmsnorm(
|
||||
unfused_query.reshape(-1, D), q_weight, EPS
|
||||
).reshape_as(unfused_query)
|
||||
unfused_query = apply_qsa_rope(rope, positions, unfused_query)
|
||||
|
||||
raw_keys = unfused_raw[..., :D]
|
||||
rope_positions = (
|
||||
unfused_raw[..., ROPE_POS_OFFSET:].view(torch.int64)
|
||||
if cache_rope_positions
|
||||
else None
|
||||
)
|
||||
pooled, first_positions = qsa_compress_groups_with_ratio(
|
||||
projected_qk[:, HQ * D :].reshape(-1, 1, D),
|
||||
position_rows,
|
||||
raw_keys,
|
||||
raw_block_table,
|
||||
token_to_req,
|
||||
query_start_loc,
|
||||
logical_positions,
|
||||
compressed_slots,
|
||||
CR,
|
||||
rope_positions,
|
||||
)
|
||||
compressed_rows = gemma_rmsnorm(pooled.reshape(-1, D), k_weight, EPS).reshape(
|
||||
-1, 1, D
|
||||
)
|
||||
group_positions = (
|
||||
first_positions.transpose(0, 1) if mrope else first_positions[:, 0]
|
||||
)
|
||||
compressed_rows = apply_qsa_rope(rope, group_positions, compressed_rows)
|
||||
qsa_store_cache_rows(unfused_compressed, compressed_slots, compressed_rows)
|
||||
qsa_store_cache_rows(raw_keys, raw_slots, projected_qk[:, HQ * D :])
|
||||
if rope_positions is not None:
|
||||
qsa_store_cache_rows(rope_positions, raw_slots, position_rows)
|
||||
|
||||
torch.testing.assert_close(fused_query, unfused_query, rtol=RTOL, atol=ATOL)
|
||||
assert torch.equal(fused_raw.view(torch.int16), unfused_raw.view(torch.int16))
|
||||
torch.testing.assert_close(
|
||||
fused_compressed, unfused_compressed, rtol=RTOL, atol=ATOL
|
||||
)
|
||||
@@ -0,0 +1,973 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.qwen4_exp.common import qsa_cache
|
||||
from vllm.models.qwen4_exp.common.qsa_cache import QSAMetadataBuilder
|
||||
from vllm.models.qwen4_exp.nvidia import indexer_qsa
|
||||
from vllm.models.qwen4_exp.nvidia import (
|
||||
model as _qwen4_exp_model, # noqa: F401
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia.ops import qsa as qsa_ops
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
|
||||
requires_qsa_kernels = pytest.mark.skipif(
|
||||
not current_platform.is_cuda() or not HAS_TRITON,
|
||||
reason="QSA kernels require CUDA and Triton",
|
||||
)
|
||||
|
||||
|
||||
def test_qsa_mtp_index_share_updates_cache_but_skips_selection(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
rows = torch.tensor([[3, 1, -1], [5, 2, 0]], dtype=torch.int32)
|
||||
raw_metadata = SimpleNamespace(
|
||||
num_actual_tokens=2,
|
||||
slot_mapping=torch.arange(2),
|
||||
block_table=torch.empty(0),
|
||||
query_start_loc=torch.arange(3),
|
||||
logical_positions=torch.arange(2),
|
||||
)
|
||||
compressed_metadata = SimpleNamespace(
|
||||
num_actual_tokens=2,
|
||||
slot_mapping=torch.arange(2),
|
||||
k_work_metadata=torch.empty(0),
|
||||
)
|
||||
updates = []
|
||||
selections = []
|
||||
indexer = SimpleNamespace(
|
||||
skip_topk=True,
|
||||
_metadata=lambda: (raw_metadata, compressed_metadata),
|
||||
index_qk_proj=lambda hidden: (torch.zeros(2, 2), None),
|
||||
index_n_heads=1,
|
||||
index_kv_heads=1,
|
||||
index_head_dim=1,
|
||||
raw_key_cache=SimpleNamespace(
|
||||
kv_cache=torch.empty(0),
|
||||
rope_position_cache=None,
|
||||
rope_position_offset=0,
|
||||
),
|
||||
compressed_key_cache=SimpleNamespace(kv_cache=torch.empty(0)),
|
||||
use_fused_pre_indexer=True,
|
||||
rotary_emb=SimpleNamespace(cos_sin_cache=torch.empty(0)),
|
||||
q_layernorm=SimpleNamespace(weight=torch.ones(1), variance_epsilon=1e-6),
|
||||
k_layernorm=SimpleNamespace(weight=torch.ones(1)),
|
||||
compress_ratio=2,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
indexer_qsa,
|
||||
"qsa_pre_indexer",
|
||||
lambda *args, **kwargs: updates.append((args, kwargs)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
qsa_ops,
|
||||
"qsa_select_paged_tokens",
|
||||
lambda *args, **kwargs: selections.append((args, kwargs)),
|
||||
)
|
||||
|
||||
actual = indexer_qsa.QSAIndexer.forward(
|
||||
indexer,
|
||||
torch.zeros(2, 4),
|
||||
torch.tensor([7, 8]),
|
||||
rows,
|
||||
)
|
||||
|
||||
assert actual is rows
|
||||
assert len(updates) == 1
|
||||
assert not selections
|
||||
|
||||
|
||||
def _qsa_mqa_paged_reference(
|
||||
q: torch.Tensor,
|
||||
k_cache: torch.Tensor,
|
||||
page_table: torch.Tensor,
|
||||
token_to_req: torch.Tensor,
|
||||
visible_lengths: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
pages = page_table.index_select(0, token_to_req.long()).long()
|
||||
keys = k_cache[pages, :, 0, :].flatten(1, 2)
|
||||
scores = torch.einsum("rhd,rnd->rnh", q.float(), keys.float())
|
||||
logits = torch.relu(scores).sum(dim=-1) / math.sqrt(q.shape[-1])
|
||||
positions = torch.arange(keys.shape[1], device=q.device).unsqueeze(0)
|
||||
return logits.masked_fill(positions >= visible_lengths.unsqueeze(1), -torch.inf)
|
||||
|
||||
|
||||
def _qsa_relative_topk_reference(
|
||||
logits: torch.Tensor,
|
||||
row_starts: torch.Tensor,
|
||||
row_ends: torch.Tensor,
|
||||
topk: int,
|
||||
) -> torch.Tensor:
|
||||
output = torch.full(
|
||||
(logits.shape[0], topk), -1, dtype=torch.int32, device=logits.device
|
||||
)
|
||||
for row in range(logits.shape[0]):
|
||||
start = int(row_starts[row].item())
|
||||
length = int((row_ends[row] - row_starts[row]).item())
|
||||
width = min(length, topk)
|
||||
if width:
|
||||
output[row, :width] = torch.topk(
|
||||
logits[row, start : start + length], width
|
||||
).indices.to(torch.int32)
|
||||
return output
|
||||
|
||||
|
||||
def _expand_qsa_indices_reference(
|
||||
block_indices: torch.Tensor,
|
||||
query_positions: torch.Tensor,
|
||||
sequence_lengths: torch.Tensor,
|
||||
compress_ratio: int,
|
||||
token_topk: int,
|
||||
) -> torch.Tensor:
|
||||
rows = block_indices.shape[0]
|
||||
block_topk = token_topk // compress_ratio
|
||||
output_width = token_topk + compress_ratio - 1
|
||||
offsets = torch.arange(compress_ratio, device=block_indices.device)
|
||||
blocks = block_indices.long()
|
||||
expanded = blocks.unsqueeze(-1) * compress_ratio + offsets
|
||||
expanded = torch.where(
|
||||
blocks.unsqueeze(-1) >= 0, expanded, torch.full_like(expanded, -1)
|
||||
).reshape(rows, block_topk * compress_ratio)
|
||||
expanded = expanded[:, :token_topk]
|
||||
expanded = torch.where(
|
||||
(expanded >= 0) & (expanded < sequence_lengths.unsqueeze(1)),
|
||||
expanded,
|
||||
torch.full_like(expanded, -1),
|
||||
)
|
||||
|
||||
tail_offsets = torch.arange(compress_ratio - 1, device=block_indices.device)
|
||||
visible_tokens = query_positions + 1
|
||||
tail_start = visible_tokens // compress_ratio * compress_ratio
|
||||
tail = tail_start.unsqueeze(1) + tail_offsets.unsqueeze(0)
|
||||
tail_count = (visible_tokens - tail_start).unsqueeze(1)
|
||||
tail_valid = (tail_offsets.unsqueeze(0) < tail_count) & (
|
||||
tail < sequence_lengths.unsqueeze(1)
|
||||
)
|
||||
tail = torch.where(tail_valid, tail, torch.full_like(tail, -1))
|
||||
|
||||
result = torch.cat((expanded, tail), dim=1)
|
||||
order = torch.arange(output_width, device=result.device).expand(rows, -1)
|
||||
sort_key = torch.where(result >= 0, order, order + output_width)
|
||||
return result.gather(1, torch.argsort(sort_key, dim=1, stable=True)).to(torch.int32)
|
||||
|
||||
|
||||
def _qsa_select_paged_tokens_reference(
|
||||
q: torch.Tensor,
|
||||
k_cache: torch.Tensor,
|
||||
page_table: torch.Tensor,
|
||||
token_to_req: torch.Tensor,
|
||||
query_positions: torch.Tensor,
|
||||
sequence_lengths: torch.Tensor,
|
||||
token_topk: int,
|
||||
compress_ratio: int,
|
||||
) -> torch.Tensor:
|
||||
row_sequence_lengths = sequence_lengths.index_select(0, token_to_req.long())
|
||||
visible_blocks = torch.minimum(
|
||||
(query_positions + 1) // compress_ratio,
|
||||
row_sequence_lengths // compress_ratio,
|
||||
).to(torch.int32)
|
||||
logits = _qsa_mqa_paged_reference(
|
||||
q,
|
||||
k_cache,
|
||||
page_table,
|
||||
token_to_req,
|
||||
visible_blocks,
|
||||
)
|
||||
starts = torch.zeros_like(visible_blocks)
|
||||
blocks = _qsa_relative_topk_reference(
|
||||
logits,
|
||||
starts,
|
||||
visible_blocks,
|
||||
token_topk // compress_ratio,
|
||||
)
|
||||
return _expand_qsa_indices_reference(
|
||||
blocks,
|
||||
query_positions,
|
||||
row_sequence_lengths,
|
||||
compress_ratio,
|
||||
token_topk,
|
||||
)
|
||||
|
||||
|
||||
def _qsa_sparse_paged_attention_reference(
|
||||
q: torch.Tensor,
|
||||
k_cache: torch.Tensor,
|
||||
v_cache: torch.Tensor,
|
||||
logical_indices: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
token_to_req: torch.Tensor,
|
||||
softmax_scale: float,
|
||||
) -> torch.Tensor:
|
||||
output = torch.zeros_like(q)
|
||||
repeats = q.shape[1] // k_cache.shape[2]
|
||||
page_size = k_cache.shape[1]
|
||||
for row in range(q.shape[0]):
|
||||
logical = logical_indices[row]
|
||||
logical = logical[logical >= 0].long()
|
||||
if not logical.numel():
|
||||
continue
|
||||
request = token_to_req[row].long()
|
||||
pages = block_table[request, logical // page_size].long()
|
||||
offsets = logical % page_size
|
||||
keys = k_cache[pages, offsets].repeat_interleave(repeats, dim=1)
|
||||
values = v_cache[pages, offsets].repeat_interleave(repeats, dim=1)
|
||||
scores = torch.einsum("hd,khd->hk", q[row].float(), keys.float())
|
||||
probabilities = torch.softmax(scores * softmax_scale, dim=-1)
|
||||
output[row] = torch.einsum("hk,khd->hd", probabilities, values.float()).to(
|
||||
q.dtype
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
def test_qsa_side_metadata_marks_cudagraph_padding_inert() -> None:
|
||||
device = torch.device("cuda")
|
||||
builder = QSAMetadataBuilder.__new__(QSAMetadataBuilder)
|
||||
builder.compress_ratio = 1
|
||||
builder.is_circular_buffer = False
|
||||
builder.storage_block_size = 64
|
||||
builder.token_to_req_buffer = torch.empty(16, dtype=torch.int32, device=device)
|
||||
builder.slot_mapping_buffer = torch.empty(16, dtype=torch.int64, device=device)
|
||||
builder.logical_positions_buffer = torch.empty(16, dtype=torch.int64, device=device)
|
||||
builder.k_work_metadata_buffer = torch.empty(0, 2, dtype=torch.int32, device=device)
|
||||
query_start_loc = torch.tensor([0, 4, 8, 12, 12], dtype=torch.int32, device=device)
|
||||
token_to_req = torch.tensor([0] * 4 + [1] * 4 + [2] * 4 + [0] * 4, device=device)
|
||||
common = SimpleNamespace(
|
||||
num_actual_tokens=16,
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc.cpu(),
|
||||
seq_lens=torch.tensor([68, 68, 68, 0], dtype=torch.int32, device=device),
|
||||
slot_mapping=torch.tensor(list(range(12)) + [-1] * 4, device=device),
|
||||
block_table_tensor=torch.empty((4, 0), dtype=torch.int32, device=device),
|
||||
token_to_req_indices=lambda buffer: buffer.copy_(token_to_req),
|
||||
)
|
||||
|
||||
metadata = builder.build(0, common)
|
||||
|
||||
assert metadata.logical_positions.tolist() == [
|
||||
64,
|
||||
65,
|
||||
66,
|
||||
67,
|
||||
64,
|
||||
65,
|
||||
66,
|
||||
67,
|
||||
64,
|
||||
65,
|
||||
66,
|
||||
67,
|
||||
-1,
|
||||
-1,
|
||||
-1,
|
||||
-1,
|
||||
]
|
||||
assert metadata.slot_mapping.tolist() == list(range(12)) + [-1] * 4
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
def test_qsa_circular_buffer_metadata_keeps_only_each_requests_suffix() -> None:
|
||||
device = torch.device("cuda")
|
||||
builder = QSAMetadataBuilder.__new__(QSAMetadataBuilder)
|
||||
builder.compress_ratio = 4
|
||||
builder.is_circular_buffer = True
|
||||
builder.kv_cache_spec = SimpleNamespace(block_size=4)
|
||||
builder.storage_block_size = 4
|
||||
builder.token_to_req_buffer = torch.empty(16, dtype=torch.int32, device=device)
|
||||
builder.slot_mapping_buffer = torch.empty(16, dtype=torch.int64, device=device)
|
||||
builder.logical_positions_buffer = torch.empty(16, dtype=torch.int64, device=device)
|
||||
builder.k_work_metadata_buffer = torch.empty(0, 2, dtype=torch.int32, device=device)
|
||||
query_start_loc = torch.tensor([0, 7, 13, 13], dtype=torch.int32, device=device)
|
||||
token_to_req = torch.tensor([0] * 7 + [1] * 6 + [0] * 3, device=device)
|
||||
block_table = torch.tensor([[1], [0], [2]], dtype=torch.int32, device=device)
|
||||
common = SimpleNamespace(
|
||||
num_actual_tokens=16,
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc.cpu(),
|
||||
seq_lens=torch.tensor([9, 11, 0], dtype=torch.int32, device=device),
|
||||
slot_mapping=torch.full((16,), -1, dtype=torch.int64, device=device),
|
||||
block_table_tensor=block_table,
|
||||
token_to_req_indices=lambda buffer: buffer.copy_(token_to_req),
|
||||
)
|
||||
|
||||
metadata = builder.build(0, common)
|
||||
expected = [
|
||||
-1,
|
||||
-1,
|
||||
-1,
|
||||
5,
|
||||
6,
|
||||
7,
|
||||
4,
|
||||
-1,
|
||||
-1,
|
||||
3,
|
||||
0,
|
||||
1,
|
||||
2,
|
||||
-1,
|
||||
-1,
|
||||
-1,
|
||||
]
|
||||
|
||||
assert metadata.slot_mapping.tolist() == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("chunk_start", list(range(8)))
|
||||
def test_qsa_circular_buffer_survives_one_speculative_step(chunk_start: int) -> None:
|
||||
"""A speculative step must not overwrite the open group's committed keys.
|
||||
|
||||
The step stores every row it computes, drafts included, before acceptance
|
||||
is known, while the next step still reads the earlier members of the group
|
||||
being compressed from the ring. A ring sized at the compression ratio makes
|
||||
those rows alias, so a rejected draft silently replaces a committed key.
|
||||
"""
|
||||
compress_ratio = 4
|
||||
num_spec = 3
|
||||
capacity = compress_ratio * -(-(compress_ratio + num_spec) // compress_ratio)
|
||||
query_len = num_spec + 1
|
||||
|
||||
slots = qsa_cache.circular_qsa_slot_mapping(
|
||||
torch.tensor([[0]], dtype=torch.int32),
|
||||
torch.zeros(query_len, dtype=torch.int32),
|
||||
torch.arange(chunk_start, chunk_start + query_len),
|
||||
capacity,
|
||||
query_start_loc=torch.tensor([0, query_len], dtype=torch.int32),
|
||||
)
|
||||
|
||||
committed = torch.arange(chunk_start - chunk_start % compress_ratio, chunk_start)
|
||||
assert set(slots.tolist()).isdisjoint((committed % capacity).tolist())
|
||||
|
||||
|
||||
def _qsa_key_cache(block_size: int, compress_ratio: int) -> qsa_cache.QSAKeyStateCache:
|
||||
return qsa_cache.QSAKeyStateCache(
|
||||
head_size=64,
|
||||
dtype=torch.bfloat16,
|
||||
cache_config=SimpleNamespace(block_size=block_size),
|
||||
prefix=f"raw.{block_size}.{compress_ratio}",
|
||||
vllm_config=SimpleNamespace(
|
||||
compilation_config=SimpleNamespace(static_forward_context={})
|
||||
),
|
||||
compress_ratio=compress_ratio,
|
||||
)
|
||||
|
||||
|
||||
def test_qsa_state_caches_adapt_the_unified_logical_layout() -> None:
|
||||
raw_cache = _qsa_key_cache(block_size=32, compress_ratio=4)
|
||||
compressed_cache = qsa_cache.QSACompressedKeyCache(
|
||||
head_size=64,
|
||||
dtype=torch.bfloat16,
|
||||
cache_config=SimpleNamespace(block_size=32),
|
||||
prefix="compressed.bind",
|
||||
vllm_config=SimpleNamespace(
|
||||
compilation_config=SimpleNamespace(static_forward_context={})
|
||||
),
|
||||
compress_ratio=4,
|
||||
)
|
||||
raw_view = torch.empty(2, 1, 8, 64, dtype=torch.bfloat16)
|
||||
compressed_view = torch.empty(2, 1, 8, 64, dtype=torch.bfloat16)
|
||||
|
||||
raw_cache.bind_kv_cache(raw_view)
|
||||
compressed_cache.bind_kv_cache(compressed_view)
|
||||
|
||||
assert raw_cache.kv_cache.shape == (2, 8, 1, 64)
|
||||
assert compressed_cache.kv_cache.shape == (2, 8, 1, 64)
|
||||
assert raw_cache.kv_cache.data_ptr() == raw_view.data_ptr()
|
||||
assert compressed_cache.kv_cache.data_ptr() == compressed_view.data_ptr()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("compress_ratio", "num_spec", "expected"),
|
||||
[(4, 0, 4), (4, 1, 8), (4, 3, 8), (4, 4, 8), (4, 5, 12), (2, 3, 6)],
|
||||
)
|
||||
def test_qsa_ring_capacity_covers_one_speculative_step(
|
||||
compress_ratio: int, num_spec: int, expected: int
|
||||
) -> None:
|
||||
"""Capacity spans the open group plus one speculative step, in whole groups."""
|
||||
spec = _qsa_key_cache(
|
||||
block_size=48, compress_ratio=compress_ratio
|
||||
).get_kv_cache_spec(SimpleNamespace(num_speculative_tokens=num_spec))
|
||||
assert spec.block_size == expected
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
def test_qsa_compressed_metadata_keeps_dummy_slots_inert() -> None:
|
||||
device = torch.device("cuda")
|
||||
builder = QSAMetadataBuilder.__new__(QSAMetadataBuilder)
|
||||
builder.compress_ratio = 4
|
||||
builder.is_circular_buffer = False
|
||||
builder.storage_block_size = 16
|
||||
builder.token_to_req_buffer = torch.empty(8, dtype=torch.int32, device=device)
|
||||
builder.slot_mapping_buffer = torch.empty(8, dtype=torch.int64, device=device)
|
||||
builder.logical_positions_buffer = torch.empty(8, dtype=torch.int64, device=device)
|
||||
# Simulate max_num_seqs exceeding the three live requests below.
|
||||
builder.request_capacity = 8
|
||||
builder.k_work_metadata_buffer = torch.empty(4, 2, dtype=torch.int32, device=device)
|
||||
query_start_loc = torch.tensor([0, 3, 3, 8], dtype=torch.int32, device=device)
|
||||
token_to_req = torch.tensor(
|
||||
[0, 0, 0, 2, 2, 2, 2, 2], dtype=torch.int32, device=device
|
||||
)
|
||||
common = SimpleNamespace(
|
||||
num_actual_tokens=8,
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc.cpu(),
|
||||
seq_lens=torch.tensor([7, 0, 12], dtype=torch.int32, device=device),
|
||||
slot_mapping=torch.full((8,), -1, dtype=torch.int64, device=device),
|
||||
block_table_tensor=torch.zeros((3, 1), dtype=torch.int32, device=device),
|
||||
token_to_req_indices=lambda buffer: buffer.copy_(token_to_req),
|
||||
)
|
||||
|
||||
metadata = builder.build(0, common)
|
||||
|
||||
assert metadata.slot_mapping.tolist() == [-1] * 8
|
||||
assert metadata.k_work_metadata.tolist() == [[0, 0], [2, 0], [2, 1], [-1, -1]]
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
@pytest.mark.parametrize("compress_ratio", [1, 4])
|
||||
@pytest.mark.parametrize("num_reqs", [2, 3, 4, 7, 8, 9])
|
||||
def test_qsa_triton_metadata_matches_pytorch(
|
||||
compress_ratio: int, num_reqs: int
|
||||
) -> None:
|
||||
device = torch.device("cuda")
|
||||
num_tokens = 8
|
||||
query_start_loc = torch.tensor(
|
||||
[0, 3, *([3] * (num_reqs - 2)), 8], dtype=torch.int32, device=device
|
||||
)
|
||||
token_to_req = torch.tensor(
|
||||
[0, 0, 0, *([num_reqs - 1] * 5)],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
block_table_rows = torch.tensor(
|
||||
[
|
||||
[4, -1, 8, -1, 12, -1],
|
||||
[1, -1, 2, -1, 3, -1],
|
||||
[7, -1, 9, -1, 11, -1],
|
||||
],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
block_table_storage = block_table_rows[
|
||||
torch.arange(num_reqs, device=device) % block_table_rows.shape[0]
|
||||
]
|
||||
seq_lens = torch.zeros(num_reqs, dtype=torch.int32, device=device)
|
||||
seq_lens[0] = 10
|
||||
seq_lens[-1] = 20
|
||||
common = SimpleNamespace(
|
||||
num_actual_tokens=num_tokens,
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc.cpu(),
|
||||
seq_lens=seq_lens,
|
||||
slot_mapping=torch.tensor(
|
||||
[0, 1, -1, 3, 4, -1, -1, -1], dtype=torch.int64, device=device
|
||||
),
|
||||
block_table_tensor=block_table_storage[:, ::2],
|
||||
token_to_req_indices=lambda buffer: buffer.copy_(token_to_req),
|
||||
)
|
||||
|
||||
def make_buffers() -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
return (
|
||||
torch.empty(num_tokens, dtype=torch.int32, device=device),
|
||||
torch.empty(num_tokens, dtype=torch.int64, device=device),
|
||||
torch.empty(num_tokens, dtype=torch.int64, device=device),
|
||||
)
|
||||
|
||||
max_num_work = (
|
||||
(num_tokens + (compress_ratio - 1) * num_reqs) // compress_ratio
|
||||
if compress_ratio != 1
|
||||
else 0
|
||||
)
|
||||
actual_k_work = (
|
||||
torch.empty(max_num_work, 2, dtype=torch.int32, device=device)
|
||||
if max_num_work
|
||||
else None
|
||||
)
|
||||
actual_buffers = make_buffers()
|
||||
actual = qsa_cache.build_qsa_metadata_triton(
|
||||
common,
|
||||
*actual_buffers,
|
||||
storage_block_size=2,
|
||||
compress_ratio=compress_ratio,
|
||||
k_work_metadata_buffer=actual_k_work,
|
||||
request_capacity=num_reqs,
|
||||
)
|
||||
|
||||
expected_k_work = (
|
||||
torch.empty_like(actual_k_work) if actual_k_work is not None else None
|
||||
)
|
||||
expected_buffers = make_buffers()
|
||||
expected = qsa_cache._build_qsa_metadata_torch(
|
||||
common,
|
||||
*expected_buffers,
|
||||
storage_block_size=2,
|
||||
compress_ratio=compress_ratio,
|
||||
k_work_metadata_buffer=expected_k_work,
|
||||
request_capacity=num_reqs,
|
||||
)
|
||||
|
||||
for actual_tensor, expected_tensor in zip(actual, expected):
|
||||
torch.testing.assert_close(actual_tensor, expected_tensor)
|
||||
if actual_k_work is not None:
|
||||
torch.testing.assert_close(actual_k_work, expected_k_work)
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
def test_qsa_fused_metadata_matches_pytorch_for_large_padded_prefill() -> None:
|
||||
device = torch.device("cuda")
|
||||
num_mapped_tokens = 4096
|
||||
num_tokens = 4224
|
||||
query_start_loc = torch.tensor(
|
||||
[0, num_mapped_tokens], dtype=torch.int32, device=device
|
||||
)
|
||||
common = SimpleNamespace(
|
||||
num_actual_tokens=num_tokens,
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc.cpu(),
|
||||
seq_lens=torch.tensor(
|
||||
[num_mapped_tokens + 32], dtype=torch.int32, device=device
|
||||
),
|
||||
block_table_tensor=torch.arange(256, dtype=torch.int32, device=device)[None],
|
||||
slot_mapping=torch.tensor(
|
||||
[0] * num_mapped_tokens + [-1] * (num_tokens - num_mapped_tokens),
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
),
|
||||
token_to_req_indices=lambda buffer: buffer.zero_(),
|
||||
)
|
||||
|
||||
def make_buffers() -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
return (
|
||||
torch.empty(num_tokens, dtype=torch.int32, device=device),
|
||||
torch.empty(num_tokens, dtype=torch.int64, device=device),
|
||||
torch.empty(num_tokens, dtype=torch.int64, device=device),
|
||||
)
|
||||
|
||||
max_num_work = (num_tokens + 3) // 4
|
||||
actual_k_work = torch.empty(max_num_work, 2, dtype=torch.int32, device=device)
|
||||
expected_k_work = torch.empty_like(actual_k_work)
|
||||
actual = qsa_cache.build_qsa_metadata_triton(
|
||||
common,
|
||||
*make_buffers(),
|
||||
storage_block_size=8,
|
||||
compress_ratio=4,
|
||||
k_work_metadata_buffer=actual_k_work,
|
||||
)
|
||||
expected = qsa_cache._build_qsa_metadata_torch(
|
||||
common,
|
||||
*make_buffers(),
|
||||
storage_block_size=8,
|
||||
compress_ratio=4,
|
||||
k_work_metadata_buffer=expected_k_work,
|
||||
)
|
||||
|
||||
for actual_tensor, expected_tensor in zip(actual, expected):
|
||||
torch.testing.assert_close(actual_tensor, expected_tensor)
|
||||
torch.testing.assert_close(actual_k_work, expected_k_work)
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
@pytest.mark.parametrize(
|
||||
"num_rows",
|
||||
[
|
||||
pytest.param(3, id="one_tile_per_program"),
|
||||
pytest.param(33, id="looped_tiles"),
|
||||
],
|
||||
)
|
||||
def test_qsa_mqa_paged_matches_test_reference(num_rows: int) -> None:
|
||||
torch.manual_seed(1)
|
||||
head_dim = 128
|
||||
q = torch.randn(num_rows, 4, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
cache = torch.randn(40, 4, 1, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
page_table = torch.randperm(40, device="cuda", dtype=torch.int32).reshape(2, 20)
|
||||
request_rows = [(num_rows + 1) // 2, num_rows // 2]
|
||||
token_to_req = torch.repeat_interleave(
|
||||
torch.arange(2, device="cuda", dtype=torch.int32),
|
||||
torch.tensor(request_rows, device="cuda"),
|
||||
)
|
||||
sequence_length_values = [320, 264]
|
||||
sequence_lengths = torch.tensor(
|
||||
sequence_length_values, device="cuda", dtype=torch.int32
|
||||
)
|
||||
query_positions = torch.cat(
|
||||
[
|
||||
torch.arange(length - rows, length, device="cuda", dtype=torch.int32)
|
||||
for rows, length in zip(request_rows, sequence_length_values, strict=True)
|
||||
]
|
||||
)
|
||||
compress_ratio = 4
|
||||
visible_lengths = (query_positions + 1) // compress_ratio
|
||||
|
||||
actual, actual_visible_blocks = qsa_ops.qsa_mqa_paged(
|
||||
q,
|
||||
cache,
|
||||
page_table,
|
||||
token_to_req,
|
||||
query_positions,
|
||||
sequence_lengths,
|
||||
compress_ratio=compress_ratio,
|
||||
)
|
||||
expected = _qsa_mqa_paged_reference(
|
||||
q, cache, page_table, token_to_req, visible_lengths
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual_visible_blocks, visible_lengths)
|
||||
# Top-k is bounded by visible_blocks; later columns are intentionally unwritten.
|
||||
columns = torch.arange(actual.shape[1], device=actual.device)
|
||||
visible = columns[None, :] < actual_visible_blocks[:, None]
|
||||
torch.testing.assert_close(actual[visible], expected[visible], rtol=1e-3, atol=1e-3)
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
def test_qsa_block_expansion_matches_test_reference() -> None:
|
||||
blocks = torch.tensor([[0, -1], [1, 0]], device="cuda", dtype=torch.int32)
|
||||
query_positions = torch.tensor([5, 10], device="cuda")
|
||||
sequence_lengths = torch.tensor([6, 11], device="cuda")
|
||||
token_to_req = torch.tensor([0, 1], device="cuda", dtype=torch.int32)
|
||||
|
||||
actual = qsa_ops.expand_qsa_block_indices_cuda(
|
||||
blocks,
|
||||
query_positions,
|
||||
sequence_lengths,
|
||||
token_to_req,
|
||||
compress_ratio=4,
|
||||
token_topk=8,
|
||||
)
|
||||
expected = _expand_qsa_indices_reference(
|
||||
blocks,
|
||||
query_positions,
|
||||
sequence_lengths,
|
||||
compress_ratio=4,
|
||||
token_topk=8,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
@pytest.mark.parametrize(
|
||||
("num_rows", "num_query_heads", "num_kv_heads", "page_size"),
|
||||
[
|
||||
# Kernel-visible pages with --block-size 256 and hybrid-cache alignment.
|
||||
pytest.param(1, 24, 2, 1792, id="tp1_split64"),
|
||||
pytest.param(16, 12, 1, 1792, id="tp2_split32"),
|
||||
pytest.param(32, 6, 1, 1024, id="tp4_split8"),
|
||||
pytest.param(257, 6, 1, 1024, id="tp4_split4"),
|
||||
pytest.param(513, 6, 1, 1024, id="tp4_split1"),
|
||||
],
|
||||
)
|
||||
def test_qsa_sparse_paged_attention_matches_test_reference(
|
||||
num_rows: int,
|
||||
num_query_heads: int,
|
||||
num_kv_heads: int,
|
||||
page_size: int,
|
||||
) -> None:
|
||||
torch.manual_seed(2)
|
||||
head_dim = 256
|
||||
num_requests = 2
|
||||
num_selected_pages = 64
|
||||
# Keep the newest page outside the synthetic top-k as causal headroom.
|
||||
num_pages_per_request = num_selected_pages + 1
|
||||
num_cache_blocks = num_requests * num_pages_per_request
|
||||
indexer_budget = 2048
|
||||
indexer_compress_ratio = 4
|
||||
selection_width = indexer_budget + indexer_compress_ratio - 1
|
||||
q = torch.randn(
|
||||
num_rows, num_query_heads, head_dim, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
kv_cache = torch.randn(
|
||||
num_cache_blocks,
|
||||
page_size,
|
||||
num_kv_heads,
|
||||
2 * head_dim,
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
k_cache, v_cache = kv_cache.split(head_dim, dim=-1)
|
||||
block_table = (
|
||||
torch.randperm(num_cache_blocks, device="cuda")
|
||||
.reshape(num_requests, num_pages_per_request)
|
||||
.to(torch.int32)
|
||||
)
|
||||
rows_per_request = math.ceil(num_rows / num_requests)
|
||||
row_indices = torch.arange(num_rows, device="cuda", dtype=torch.int32)
|
||||
token_to_req = row_indices // rows_per_request
|
||||
request_row_counts = torch.tensor(
|
||||
[rows_per_request, num_rows - rows_per_request],
|
||||
device="cuda",
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
context_length = num_pages_per_request * page_size - 1
|
||||
block_topk = indexer_budget // indexer_compress_ratio
|
||||
compressed_blocks_per_page = page_size // indexer_compress_ratio
|
||||
selection = torch.arange(block_topk, device="cuda")
|
||||
selected_pages = selection % num_selected_pages
|
||||
selected_offsets = selection // num_selected_pages
|
||||
row_shifts = 2 * row_indices.unsqueeze(1)
|
||||
# Eight blocks per page; adjacent rows overlap by six of those eight.
|
||||
selected_offsets = (selected_offsets + row_shifts) % compressed_blocks_per_page
|
||||
block_indices = (selected_pages * compressed_blocks_per_page + selected_offsets).to(
|
||||
torch.int32
|
||||
)
|
||||
rows_within_request = row_indices % rows_per_request
|
||||
query_positions = (
|
||||
context_length - request_row_counts[token_to_req.long()] + rows_within_request
|
||||
).to(torch.int64)
|
||||
sequence_lengths = torch.full(
|
||||
(num_requests,), context_length, device="cuda", dtype=torch.int32
|
||||
)
|
||||
logical_indices = qsa_ops.expand_qsa_block_indices_cuda(
|
||||
block_indices,
|
||||
query_positions,
|
||||
sequence_lengths,
|
||||
token_to_req,
|
||||
indexer_compress_ratio,
|
||||
indexer_budget,
|
||||
)
|
||||
assert logical_indices.shape == (num_rows, selection_width)
|
||||
scale = q.shape[-1] ** -0.5
|
||||
|
||||
actual = qsa_ops.qsa_sparse_paged_attention(
|
||||
q,
|
||||
k_cache,
|
||||
v_cache,
|
||||
logical_indices,
|
||||
block_table,
|
||||
token_to_req,
|
||||
)
|
||||
expected = _qsa_sparse_paged_attention_reference(
|
||||
q,
|
||||
k_cache,
|
||||
v_cache,
|
||||
logical_indices,
|
||||
block_table,
|
||||
token_to_req,
|
||||
scale,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, rtol=2e-2, atol=2e-2)
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
def test_qsa_selection_chunks_workspace_and_matches_test_reference(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
workspace_init,
|
||||
) -> None:
|
||||
rows, keys, heads, head_dim = 65, 640, 4, 16
|
||||
token_topk, compress_ratio = 2048, 4
|
||||
torch.manual_seed(3)
|
||||
q = torch.randn(rows, heads, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
cache = torch.randn(40, 16, 1, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
page_table = torch.randperm(40, device="cuda", dtype=torch.int32).unsqueeze(0)
|
||||
token_to_req = torch.zeros(rows, device="cuda", dtype=torch.int32)
|
||||
query_positions = torch.full((rows,), 2559, device="cuda", dtype=torch.int32)
|
||||
sequence_lengths = torch.tensor([2560], device="cuda", dtype=torch.int32)
|
||||
monkeypatch.setattr(qsa_ops, "_LOGITS_WORKSPACE_BYTES", 32 * keys * 4)
|
||||
original_score = qsa_ops.qsa_mqa_paged
|
||||
scored_row_counts = []
|
||||
|
||||
def record_score(query: torch.Tensor, *args, **kwargs):
|
||||
scored_row_counts.append(query.shape[0])
|
||||
return original_score(query, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(qsa_ops, "qsa_mqa_paged", record_score)
|
||||
|
||||
actual = qsa_ops.qsa_select_paged_tokens(
|
||||
q,
|
||||
cache,
|
||||
page_table,
|
||||
token_to_req,
|
||||
query_positions,
|
||||
sequence_lengths,
|
||||
token_topk,
|
||||
compress_ratio,
|
||||
)
|
||||
expected = _qsa_select_paged_tokens_reference(
|
||||
q,
|
||||
cache,
|
||||
page_table,
|
||||
token_to_req,
|
||||
query_positions,
|
||||
sequence_lengths,
|
||||
token_topk,
|
||||
compress_ratio,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual.sort().values, expected.sort().values)
|
||||
assert scored_row_counts == [32, 32, 1]
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
def test_qsa_selection_handles_no_complete_compressed_blocks(workspace_init) -> None:
|
||||
q = torch.zeros(2, 4, 8, device="cuda", dtype=torch.bfloat16)
|
||||
cache = torch.zeros(1, 16, 1, 8, device="cuda", dtype=torch.bfloat16)
|
||||
page_table = torch.zeros(1, 1, device="cuda", dtype=torch.int32)
|
||||
token_to_req = torch.zeros(2, device="cuda", dtype=torch.int32)
|
||||
query_positions = torch.tensor([1, 2], device="cuda", dtype=torch.int32)
|
||||
sequence_lengths = torch.tensor([3], device="cuda", dtype=torch.int32)
|
||||
|
||||
selected = qsa_ops.qsa_select_paged_tokens(
|
||||
q,
|
||||
cache,
|
||||
page_table,
|
||||
token_to_req,
|
||||
query_positions,
|
||||
sequence_lengths,
|
||||
token_topk=2048,
|
||||
compress_ratio=4,
|
||||
)
|
||||
|
||||
assert selected[0, :2].tolist() == [0, 1]
|
||||
assert selected[1, :3].tolist() == [0, 1, 2]
|
||||
assert torch.all(selected[0, 2:] == -1)
|
||||
assert torch.all(selected[1, 3:] == -1)
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
def test_qsa_streaming_compression_and_compressor_state_store_match_reference() -> None:
|
||||
head_dim = 8
|
||||
current_pairs = [
|
||||
*((0, position) for position in range(2, 9)),
|
||||
*((1, position) for position in range(5, 11)),
|
||||
]
|
||||
|
||||
def key_row(request: int, position: int) -> torch.Tensor:
|
||||
return (
|
||||
torch.arange(head_dim, dtype=torch.float32) + request * 1000 + position * 10
|
||||
)
|
||||
|
||||
def position_row(request: int, position: int) -> torch.Tensor:
|
||||
return torch.tensor(
|
||||
[
|
||||
request * 1000 + position,
|
||||
request * 1000 + position + 100,
|
||||
request * 1000 + position + 200,
|
||||
],
|
||||
dtype=torch.int64,
|
||||
)
|
||||
|
||||
raw_keys = (
|
||||
torch.stack([key_row(request, position) for request, position in current_pairs])
|
||||
.unsqueeze(1)
|
||||
.to(device="cuda", dtype=torch.bfloat16)
|
||||
)
|
||||
raw_positions = (
|
||||
torch.stack(
|
||||
[position_row(request, position) for request, position in current_pairs]
|
||||
)
|
||||
.unsqueeze(1)
|
||||
.to(device="cuda")
|
||||
)
|
||||
token_to_req = torch.tensor(
|
||||
[request for request, _ in current_pairs],
|
||||
dtype=torch.int32,
|
||||
device="cuda",
|
||||
)
|
||||
logical_positions = torch.tensor(
|
||||
[position for _, position in current_pairs],
|
||||
dtype=torch.int64,
|
||||
device="cuda",
|
||||
)
|
||||
query_start_loc = torch.tensor([0, 7, 13], dtype=torch.int32, device="cuda")
|
||||
compressor_state_block_table = torch.tensor(
|
||||
[[1], [0]], dtype=torch.int32, device="cuda"
|
||||
)
|
||||
compressor_state_cache = torch.zeros(
|
||||
2, 4, 1, head_dim, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
rope_cache = torch.zeros(2, 4, 1, 3, dtype=torch.int64, device="cuda")
|
||||
for request, position, block in ((0, 0, 1), (0, 1, 1), (1, 4, 0)):
|
||||
compressor_state_cache[block, position % 4, 0] = key_row(request, position).to(
|
||||
device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
rope_cache[block, position % 4, 0] = position_row(request, position).to("cuda")
|
||||
|
||||
compressed_slots = torch.full(
|
||||
(len(current_pairs),), -1, dtype=torch.int64, device="cuda"
|
||||
)
|
||||
valid_rows = torch.tensor([1, 5, 9], dtype=torch.int64, device="cuda")
|
||||
compressed_slots[valid_rows] = torch.arange(3, device="cuda")
|
||||
pooled, first_positions = qsa_ops.qsa_compress_groups_with_ratio(
|
||||
raw_keys,
|
||||
raw_positions,
|
||||
compressor_state_cache,
|
||||
compressor_state_block_table,
|
||||
token_to_req,
|
||||
query_start_loc,
|
||||
logical_positions,
|
||||
compressed_slots,
|
||||
compress_ratio=4,
|
||||
rope_cache=rope_cache,
|
||||
)
|
||||
pooled_without_rope, scalar_first_positions = (
|
||||
qsa_ops.qsa_compress_groups_with_ratio(
|
||||
raw_keys,
|
||||
raw_positions,
|
||||
compressor_state_cache,
|
||||
compressor_state_block_table,
|
||||
token_to_req,
|
||||
query_start_loc,
|
||||
logical_positions,
|
||||
compressed_slots,
|
||||
compress_ratio=4,
|
||||
)
|
||||
)
|
||||
|
||||
groups = [
|
||||
[(0, position) for position in range(0, 4)],
|
||||
[(0, position) for position in range(4, 8)],
|
||||
[(1, position) for position in range(4, 8)],
|
||||
]
|
||||
expected_pooled = (
|
||||
torch.stack(
|
||||
[
|
||||
torch.stack([key_row(*pair) for pair in group]).mean(dim=0)
|
||||
for group in groups
|
||||
]
|
||||
)
|
||||
.unsqueeze(1)
|
||||
.to(device="cuda", dtype=torch.bfloat16)
|
||||
)
|
||||
expected_positions = torch.stack(
|
||||
[position_row(0, 0), position_row(0, 4), position_row(1, 4)]
|
||||
).to("cuda")
|
||||
expected_scalar_positions = torch.tensor(
|
||||
[[0, 0, 0], [4, 4, 4], [4, 4, 4]],
|
||||
dtype=torch.int64,
|
||||
device="cuda",
|
||||
)
|
||||
|
||||
torch.testing.assert_close(pooled[valid_rows], expected_pooled)
|
||||
torch.testing.assert_close(pooled_without_rope[valid_rows], expected_pooled)
|
||||
torch.testing.assert_close(first_positions[valid_rows], expected_positions)
|
||||
torch.testing.assert_close(
|
||||
scalar_first_positions[valid_rows], expected_scalar_positions
|
||||
)
|
||||
|
||||
compressor_state_slots = torch.tensor(
|
||||
[-1, -1, -1, 5, 6, 7, 4, -1, -1, 3, 0, 1, 2],
|
||||
dtype=torch.int64,
|
||||
device="cuda",
|
||||
)
|
||||
qsa_ops.qsa_store_cache_rows(
|
||||
compressor_state_cache, compressor_state_slots, raw_keys
|
||||
)
|
||||
qsa_ops.qsa_store_cache_rows(rope_cache, compressor_state_slots, raw_positions)
|
||||
for request, positions, block in ((0, range(5, 9), 1), (1, range(7, 11), 0)):
|
||||
for position in positions:
|
||||
torch.testing.assert_close(
|
||||
compressor_state_cache[block, position % 4, 0],
|
||||
key_row(request, position).to(device="cuda", dtype=torch.bfloat16),
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
rope_cache[block, position % 4, 0],
|
||||
position_row(request, position).to("cuda"),
|
||||
)
|
||||
@@ -508,6 +508,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
extras={"native-prefix": "imdatta0/small_qwen3_5_20b"},
|
||||
max_model_len=4096,
|
||||
),
|
||||
"Qwen4ExpForCausalLM": _HfExamplesInfo("", is_available_online=False),
|
||||
"MellumForCausalLM": _HfExamplesInfo("JetBrains/Mellum2-12B-A2.5B-Base"),
|
||||
"Qwen3NextForCausalLM": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-Next-80B-A3B-Instruct",
|
||||
@@ -1348,6 +1349,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"Qwen/Qwen3.5-35B-A3B",
|
||||
max_model_len=4096,
|
||||
),
|
||||
"Qwen4ExpForConditionalGeneration": _HfExamplesInfo(
|
||||
"",
|
||||
is_available_online=False,
|
||||
),
|
||||
"Qwen3OmniMoeForConditionalGeneration": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
max_model_len=4096,
|
||||
@@ -1797,6 +1802,7 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
"Qwen/Qwen3.5-35B-A3B",
|
||||
speculative_model="Qwen/Qwen3.5-35B-A3B",
|
||||
),
|
||||
"Qwen4ExpMTP": _HfExamplesInfo("", is_available_online=False),
|
||||
"Step3p5MTP": _HfExamplesInfo(
|
||||
"stepfun-ai/Step-3.5-Flash",
|
||||
speculative_model="stepfun-ai/Step-3.5-Flash",
|
||||
|
||||
@@ -6,6 +6,7 @@ Run `pytest tests/quantization/test_fp8.py --forked`.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
@@ -31,6 +32,10 @@ from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import (
|
||||
Fp8PerTensorOnlineLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils import flashinfer_utils
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
prepare_fp8_moe_layer_for_fi,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
process_fp8_input_tensor_strategy_moe,
|
||||
)
|
||||
@@ -50,6 +55,45 @@ MODELS = [
|
||||
]
|
||||
|
||||
|
||||
def test_prepare_gated_trtllm_fp8_moe_weights_pads_each_projection(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
flashinfer_utils,
|
||||
"rotate_weights_for_fi_trtllm_fp8_per_tensor_moe",
|
||||
lambda *args: None,
|
||||
)
|
||||
intermediate = 17
|
||||
padded_intermediate = 32
|
||||
hidden_size = 4
|
||||
gate = torch.ones((1, intermediate, hidden_size), dtype=torch.float8_e4m3fn)
|
||||
up = torch.full_like(gate, 2)
|
||||
w13 = torch.cat((gate, up), dim=1)
|
||||
w2 = torch.ones((1, hidden_size, intermediate), dtype=torch.float8_e4m3fn)
|
||||
layer = SimpleNamespace(
|
||||
activation=SimpleNamespace(is_gated=True),
|
||||
moe_config=SimpleNamespace(
|
||||
is_act_and_mul=True,
|
||||
intermediate_size_per_partition=intermediate,
|
||||
),
|
||||
)
|
||||
|
||||
padded_w31, _, _, _ = prepare_fp8_moe_layer_for_fi(
|
||||
layer,
|
||||
w13,
|
||||
w2,
|
||||
w13_scale=torch.ones(1),
|
||||
w13_input_scale=torch.ones(1),
|
||||
w2_scale=torch.ones(1),
|
||||
w2_input_scale=torch.ones(1),
|
||||
is_trtllm=True,
|
||||
)
|
||||
|
||||
expected = w13.new_zeros((1, 2 * padded_intermediate, hidden_size))
|
||||
expected[:, :intermediate] = up
|
||||
expected[:, padded_intermediate : padded_intermediate + intermediate] = gate
|
||||
assert layer.moe_config.intermediate_size_per_partition == padded_intermediate
|
||||
assert torch.equal(padded_w31, expected)
|
||||
|
||||
|
||||
def test_static_fp8_moe_input_scales_remain_scalar() -> None:
|
||||
a1_scale, a2_scale = process_fp8_input_tensor_strategy_moe(
|
||||
torch.tensor([0.25, 0.5]),
|
||||
|
||||
@@ -16,6 +16,7 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
Mxfp4MoeBackend,
|
||||
_requires_qwen38_tep8_emulation,
|
||||
select_mxfp4_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
@@ -72,6 +73,91 @@ def _make_w4a4_moe_config(moe_backend: str = "auto") -> FusedMoEConfig:
|
||||
)
|
||||
|
||||
|
||||
def _make_qwen38_tep8_moe_config() -> FusedMoEConfig:
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
|
||||
return FusedMoEConfig(
|
||||
num_experts=512,
|
||||
experts_per_token=9,
|
||||
hidden_dim=2560,
|
||||
intermediate_size=640,
|
||||
num_local_experts=64,
|
||||
num_logical_experts=512,
|
||||
moe_parallel_config=FusedMoEParallelConfig(
|
||||
tp_size=1,
|
||||
tp_rank=0,
|
||||
pcp_size=1,
|
||||
pcp_rank=0,
|
||||
dp_size=1,
|
||||
dp_rank=0,
|
||||
ep_size=8,
|
||||
ep_rank=0,
|
||||
sp_size=1,
|
||||
use_ep=True,
|
||||
all2all_backend="allgather_reducescatter",
|
||||
enable_eplb=False,
|
||||
),
|
||||
activation=MoEActivation.SILU,
|
||||
in_dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
routing_method=RoutingMethodType.Renormalize,
|
||||
)
|
||||
|
||||
|
||||
def test_qwen38_tep8_requires_emulation_only_on_gfx950(monkeypatch):
|
||||
import vllm.model_executor.layers.fused_moe.oracle.mxfp4 as mxfp4_oracle
|
||||
|
||||
config = _make_qwen38_tep8_moe_config()
|
||||
monkeypatch.setattr(current_platform, "is_rocm", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: True)
|
||||
|
||||
assert _requires_qwen38_tep8_emulation(config, kMxfp4Dynamic)
|
||||
|
||||
config.moe_parallel_config.ep_size = 4
|
||||
config.num_local_experts = 128
|
||||
assert not _requires_qwen38_tep8_emulation(config, kMxfp4Dynamic)
|
||||
|
||||
config = _make_qwen38_tep8_moe_config()
|
||||
monkeypatch.setattr(mxfp4_oracle.current_platform, "is_rocm", lambda: False)
|
||||
assert not _requires_qwen38_tep8_emulation(config, kMxfp4Dynamic)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"requested_backend,expected_backend",
|
||||
[
|
||||
("auto", Mxfp4MoeBackend.EMULATION),
|
||||
("aiter", Mxfp4MoeBackend.AITER_MXFP4_MXFP4),
|
||||
],
|
||||
)
|
||||
def test_qwen38_tep8_auto_fallback_respects_explicit_backend(
|
||||
requested_backend,
|
||||
expected_backend,
|
||||
monkeypatch,
|
||||
):
|
||||
import vllm.model_executor.layers.fused_moe.oracle.mxfp4 as mxfp4_oracle
|
||||
|
||||
class SupportedExperts:
|
||||
@staticmethod
|
||||
def is_supported_config(*args, **kwargs):
|
||||
return True, None
|
||||
|
||||
config = _make_qwen38_tep8_moe_config()
|
||||
config.moe_backend = requested_backend
|
||||
monkeypatch.setattr(current_platform, "is_rocm", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: True)
|
||||
monkeypatch.setattr(mxfp4_oracle, "_user_moe_activation_override", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
mxfp4_oracle, "backend_to_kernel_cls", lambda backend: [SupportedExperts]
|
||||
)
|
||||
|
||||
backend, experts_cls = select_mxfp4_moe_backend(
|
||||
config, activation_key=kMxfp4Dynamic
|
||||
)
|
||||
|
||||
assert backend == expected_backend
|
||||
assert experts_cls is SupportedExperts
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mxfp4_oracle_config():
|
||||
"""Stub the config the oracle reads (``model_config.quantization_config``)
|
||||
|
||||
@@ -604,3 +604,24 @@ def test_online_mxfp4_dense_matches_quark(
|
||||
keys=("weight", "weight_scale"),
|
||||
packed_weight_keys=("weight",),
|
||||
)
|
||||
|
||||
|
||||
def test_quark_ocp_mx_dense_rejects_unaligned_input_partition():
|
||||
scheme = object.__new__(QuarkOCP_MX)
|
||||
layer = torch.nn.Module()
|
||||
layer.prefix = "model.layers.0.mlp.shared_expert.down_proj"
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match=(
|
||||
"input size per partition of 80, which must be divisible by "
|
||||
"the OCP MX group size 32"
|
||||
),
|
||||
):
|
||||
scheme.create_weights(
|
||||
layer=layer,
|
||||
output_partition_sizes=[2560],
|
||||
input_size_per_partition=80,
|
||||
params_dtype=torch.bfloat16,
|
||||
weight_loader=lambda *_: None,
|
||||
)
|
||||
|
||||
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
|
||||
prepare_nvfp4_moe_layer_for_fi_or_cutlass,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
align_fp4_moe_weights_for_fi,
|
||||
align_trtllm_fp4_moe_hidden_dim_for_fi,
|
||||
)
|
||||
|
||||
@@ -107,3 +108,53 @@ def test_align_trtllm_fp4_moe_hidden_dim_pads_to_256_multiple():
|
||||
assert torch.count_nonzero(out_w13_scale[:, :, hidden_dim // 16 :]) == 0
|
||||
assert torch.count_nonzero(out_w2[:, hidden_dim:, :]) == 0
|
||||
assert torch.count_nonzero(out_w2_scale[:, hidden_dim:, :]) == 0
|
||||
|
||||
|
||||
def test_align_trtllm_fp4_moe_intermediate_pads_gate_and_up_separately():
|
||||
# Qwen3.8-Flash-Next has intermediate_size=640. At TP4 each rank owns
|
||||
# 160 values, which TRTLLM pads to the next 64-value boundary (192).
|
||||
intermediate = 160
|
||||
padded_intermediate = 192
|
||||
hidden_dim = 32
|
||||
|
||||
gate = torch.ones((1, intermediate, hidden_dim // 2), dtype=torch.uint8)
|
||||
up = torch.full_like(gate, 2)
|
||||
w13 = torch.cat((gate, up), dim=1)
|
||||
gate_scale = torch.full((1, intermediate, hidden_dim // 16), 3, dtype=torch.uint8)
|
||||
up_scale = torch.full_like(gate_scale, 4)
|
||||
w13_scale = torch.cat((gate_scale, up_scale), dim=1)
|
||||
w2 = torch.full((1, hidden_dim, intermediate // 2), 5, dtype=torch.uint8)
|
||||
w2_scale = torch.full((1, hidden_dim, intermediate // 16), 6, dtype=torch.uint8)
|
||||
|
||||
out_w13, out_w13_scale, out_w2, out_w2_scale, out_intermediate = (
|
||||
align_fp4_moe_weights_for_fi(
|
||||
w13,
|
||||
w13_scale,
|
||||
w2,
|
||||
w2_scale,
|
||||
is_act_and_mul=True,
|
||||
min_alignment=64,
|
||||
)
|
||||
)
|
||||
|
||||
assert out_intermediate == padded_intermediate
|
||||
torch.testing.assert_close(out_w13[:, :intermediate], gate)
|
||||
torch.testing.assert_close(
|
||||
out_w13[:, padded_intermediate : padded_intermediate + intermediate], up
|
||||
)
|
||||
torch.testing.assert_close(out_w13_scale[:, :intermediate], gate_scale)
|
||||
torch.testing.assert_close(
|
||||
out_w13_scale[:, padded_intermediate : padded_intermediate + intermediate],
|
||||
up_scale,
|
||||
)
|
||||
|
||||
assert torch.count_nonzero(out_w13[:, intermediate:padded_intermediate]) == 0
|
||||
assert torch.count_nonzero(out_w13[:, padded_intermediate + intermediate :]) == 0
|
||||
assert torch.count_nonzero(out_w13_scale[:, intermediate:padded_intermediate]) == 0
|
||||
assert (
|
||||
torch.count_nonzero(out_w13_scale[:, padded_intermediate + intermediate :]) == 0
|
||||
)
|
||||
torch.testing.assert_close(out_w2[:, :, : intermediate // 2], w2)
|
||||
torch.testing.assert_close(out_w2_scale[:, :, : intermediate // 16], w2_scale)
|
||||
assert torch.count_nonzero(out_w2[:, :, intermediate // 2 :]) == 0
|
||||
assert torch.count_nonzero(out_w2_scale[:, :, intermediate // 16 :]) == 0
|
||||
|
||||
@@ -9,23 +9,34 @@ block dim (a contiguous region per layer) or inside it (all layers' pages within
|
||||
block); the allocation is the same either way.
|
||||
"""
|
||||
|
||||
from dataclasses import replace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import CacheConfig
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheManager
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
_get_kv_cache_bytes_per_block,
|
||||
_get_packed_kv_cache_groups,
|
||||
_pool_bytes_per_block,
|
||||
generate_scheduler_kv_cache_config,
|
||||
get_kv_cache_config_from_groups,
|
||||
get_kv_cache_groups,
|
||||
resolve_kv_cache_block_sizes,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheLayout,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowMLASpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
iter_layer_specs,
|
||||
)
|
||||
from vllm.v1.worker.utils import allocate_kv_cache
|
||||
|
||||
@@ -86,6 +97,307 @@ def _bind(config, layout: str):
|
||||
return allocate_kv_cache(config, torch.device("cpu"), KVCacheLayout[layout], None)
|
||||
|
||||
|
||||
MAIN_KV_PAGE_BYTES = 2_048
|
||||
COMPRESSED_PAGE_BYTES = 128
|
||||
NUM_CACHE_TUPLES = 3
|
||||
BYTES_PER_BLOCK = NUM_CACHE_TUPLES * (MAIN_KV_PAGE_BYTES + COMPRESSED_PAGE_BYTES)
|
||||
|
||||
|
||||
def _main_kv_name(layer_index: int) -> str:
|
||||
return f"model.layers.{layer_index}.self_attn"
|
||||
|
||||
|
||||
def _compressed_name(layer_index: int) -> str:
|
||||
return f"model.layers.{layer_index}.self_attn.indexer.compressed_key_cache"
|
||||
|
||||
|
||||
def _compressor_state_name(layer_index: int) -> str:
|
||||
return f"model.layers.{layer_index}.self_attn.indexer.raw_key_cache"
|
||||
|
||||
|
||||
def _make_csa_linear_specs(
|
||||
num_mamba: int = 7,
|
||||
num_tuples: int = NUM_CACHE_TUPLES,
|
||||
) -> dict[str, KVCacheSpec]:
|
||||
specs: dict[str, KVCacheSpec] = {}
|
||||
for layer_index in range(num_mamba):
|
||||
specs[f"model.layers.{layer_index}.linear_attn"] = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((32,),),
|
||||
dtypes=(torch.bfloat16,),
|
||||
)
|
||||
specs["model.layers.0.ple"] = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((24,),),
|
||||
dtypes=(torch.bfloat16,),
|
||||
tp_replicated=True,
|
||||
)
|
||||
for layer_index in range(num_tuples):
|
||||
specs[_main_kv_name(layer_index)] = FullAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=2,
|
||||
head_size=16,
|
||||
head_size_v=16,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
specs[_compressed_name(layer_index)] = MLAAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=16,
|
||||
dtype=torch.bfloat16,
|
||||
tokens_per_state=4,
|
||||
)
|
||||
specs[_compressor_state_name(layer_index)] = CircularBufferSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=1,
|
||||
head_size=8,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
return specs
|
||||
|
||||
|
||||
def _shared_layout_config():
|
||||
config = _mock_vllm_config("BLNHC")
|
||||
config.scheduler_config.disable_hybrid_kv_cache_manager = False
|
||||
config.speculative_config = None
|
||||
config.model_config.max_model_len = 16
|
||||
config.model_config.get_total_num_hidden_layers.return_value = 64
|
||||
config.model_config.get_total_num_kv_heads.return_value = 2
|
||||
config.model_config.get_num_kv_heads.return_value = 2
|
||||
config.parallel_config.pipeline_parallel_size = 1
|
||||
config.parallel_config.decode_context_parallel_size = 1
|
||||
config.cache_config.block_size = 16
|
||||
config.cache_config.enable_prefix_caching = False
|
||||
config.cache_config.prefix_match_unit = None
|
||||
config.cache_config.mamba_cache_mode = "none"
|
||||
return config
|
||||
|
||||
|
||||
class TestCSALinearGrouping:
|
||||
"""A CSA + linear-attention model (sparse attention with a compressor ring,
|
||||
plus sharded GDN and one TP-replicated PLE state) goes through the generic
|
||||
packed-group path; no model-specific branch is involved."""
|
||||
|
||||
@staticmethod
|
||||
def _mamba_groups(groups):
|
||||
"""(group, per-layer mamba spec) for every group holding mamba state."""
|
||||
out = []
|
||||
for group in groups:
|
||||
spec = next(iter(iter_layer_specs(group.kv_cache_spec)))
|
||||
if isinstance(spec, MambaSpec):
|
||||
out.append((group, spec))
|
||||
return out
|
||||
|
||||
def test_replicated_state_gets_its_own_group(self):
|
||||
"""The PLE state is TP-replicated and a different size, so it must not
|
||||
share a manager group with the sharded GDN states (the NIXL worker
|
||||
requires exactly one single-layer replicated group)."""
|
||||
groups = get_kv_cache_groups(_shared_layout_config(), _make_csa_linear_specs())
|
||||
|
||||
mamba_groups = self._mamba_groups(groups)
|
||||
replicated = [g for g, spec in mamba_groups if spec.tp_replicated]
|
||||
assert len(replicated) == 1
|
||||
assert replicated[0].layer_names == ["model.layers.0.ple"]
|
||||
sharded = [g for g, spec in mamba_groups if not spec.tp_replicated]
|
||||
assert sharded, "GDN states must keep their own groups"
|
||||
assert sorted(n for g in sharded for n in g.layer_names) == sorted(
|
||||
f"model.layers.{i}.linear_attn" for i in range(7)
|
||||
)
|
||||
|
||||
def test_roles_land_in_separate_groups(self):
|
||||
groups = get_kv_cache_groups(_shared_layout_config(), _make_csa_linear_specs())
|
||||
|
||||
owner = next(g for g in groups if _main_kv_name(0) in g.layer_names)
|
||||
assert sorted(owner.layer_names) == sorted(
|
||||
[
|
||||
*(_main_kv_name(i) for i in range(NUM_CACHE_TUPLES)),
|
||||
*(_compressed_name(i) for i in range(NUM_CACHE_TUPLES)),
|
||||
]
|
||||
)
|
||||
assert owner.kv_cache_spec.prefix_cacheable
|
||||
|
||||
scratch = next(g for g in groups if _compressor_state_name(0) in g.layer_names)
|
||||
assert scratch.layer_names == [
|
||||
_compressor_state_name(i) for i in range(NUM_CACHE_TUPLES)
|
||||
]
|
||||
assert not scratch.kv_cache_spec.prefix_cacheable
|
||||
|
||||
def test_every_group_fits_one_packed_block(self):
|
||||
config = _shared_layout_config()
|
||||
groups = get_kv_cache_groups(config, _make_csa_linear_specs())
|
||||
bytes_per_block = _get_kv_cache_bytes_per_block(groups)
|
||||
|
||||
pages = _pages(groups)
|
||||
for group in groups:
|
||||
assert sum(pages[n] for n in group.layer_names) <= bytes_per_block
|
||||
|
||||
kv_cache_config = get_kv_cache_config_from_groups(
|
||||
config, groups, available_memory=bytes_per_block * 32
|
||||
)
|
||||
assert kv_cache_config.num_blocks == 32
|
||||
# Groups overlay from byte 0 of each block, so a layer never addresses
|
||||
# past the block it belongs to.
|
||||
for tensor in kv_cache_config.kv_cache_tensors:
|
||||
assert tensor.block_stride == bytes_per_block
|
||||
assert tensor.offset < bytes_per_block
|
||||
|
||||
def test_scratch_group_survives_computed_block_truncation(self):
|
||||
"""The scratch group contributes no computed blocks, so truncating a
|
||||
lookup result must skip it: its block size is the ring capacity, which
|
||||
neither divides the hit length nor bounds the (empty) block list."""
|
||||
config = _shared_layout_config()
|
||||
config.cache_config.enable_prefix_caching = True
|
||||
groups = get_kv_cache_groups(config, _make_csa_linear_specs())
|
||||
kv_cache_config = get_kv_cache_config_from_groups(
|
||||
config, groups, available_memory=BYTES_PER_BLOCK * 64
|
||||
)
|
||||
scheduler_config = generate_scheduler_kv_cache_config([kv_cache_config])
|
||||
manager = KVCacheManager(
|
||||
scheduler_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=16,
|
||||
scheduler_block_size=16,
|
||||
)
|
||||
blocks = manager.create_kv_cache_blocks(
|
||||
tuple(
|
||||
manager.block_pool.get_new_blocks(3)
|
||||
if group.kv_cache_spec.prefix_cacheable
|
||||
else []
|
||||
for group in scheduler_config.kv_cache_groups
|
||||
)
|
||||
)
|
||||
truncated = manager.truncate_computed_blocks(blocks, 48)
|
||||
scratch_index = next(
|
||||
i
|
||||
for i, group in enumerate(scheduler_config.kv_cache_groups)
|
||||
if not group.kv_cache_spec.prefix_cacheable
|
||||
)
|
||||
assert truncated.blocks[scratch_index] == []
|
||||
|
||||
def test_prefix_hits_respect_compression_alignment(self):
|
||||
config = _shared_layout_config()
|
||||
config.cache_config.enable_prefix_caching = True
|
||||
config.cache_config.prefix_match_unit = 16
|
||||
config.cache_config.mamba_cache_mode = "align"
|
||||
specs = {
|
||||
name: replace(spec, mamba_cache_mode="align")
|
||||
if isinstance(spec, MambaSpec)
|
||||
else spec
|
||||
for name, spec in _make_csa_linear_specs(num_tuples=1).items()
|
||||
}
|
||||
groups = get_kv_cache_groups(config, specs)
|
||||
kv_cache_config = get_kv_cache_config_from_groups(
|
||||
config, groups, available_memory=8 * MAIN_KV_PAGE_BYTES
|
||||
)
|
||||
assert resolve_kv_cache_block_sizes(kv_cache_config, config) == (16, 16)
|
||||
|
||||
# 2 tokens is not a multiple of the compression ratio 4: a prefix hit
|
||||
# could land inside a partially filled compressed state.
|
||||
config.cache_config.prefix_match_unit = 2
|
||||
with pytest.raises(ValueError, match="per-state compression"):
|
||||
resolve_kv_cache_block_sizes(kv_cache_config, config)
|
||||
|
||||
def test_scratch_ring_does_not_drag_hash_granularity(self):
|
||||
config = _shared_layout_config()
|
||||
config.cache_config.enable_prefix_caching = True
|
||||
config.cache_config.mamba_cache_mode = "align"
|
||||
specs = {
|
||||
name: replace(spec, mamba_cache_mode="align")
|
||||
if isinstance(spec, MambaSpec)
|
||||
else spec
|
||||
for name, spec in _make_csa_linear_specs(num_tuples=1).items()
|
||||
}
|
||||
groups = get_kv_cache_groups(config, specs)
|
||||
kv_cache_config = get_kv_cache_config_from_groups(
|
||||
config, groups, available_memory=8 * MAIN_KV_PAGE_BYTES
|
||||
)
|
||||
# The hash granularity is the GCD over prefix-cacheable groups only;
|
||||
# the 4-token scratch ring is excluded (it would drag it to 4).
|
||||
assert resolve_kv_cache_block_sizes(kv_cache_config, config) == (16, 16)
|
||||
|
||||
def test_compressed_attention_hashes_can_be_finer_than_cache_hits(self):
|
||||
config = _shared_layout_config()
|
||||
config.cache_config.enable_prefix_caching = True
|
||||
specs = {
|
||||
"compressed.4": MLAAttentionSpec(
|
||||
block_size=256,
|
||||
num_kv_heads=1,
|
||||
head_size=16,
|
||||
dtype=torch.bfloat16,
|
||||
tokens_per_state=4,
|
||||
),
|
||||
"compressed.128": MLAAttentionSpec(
|
||||
block_size=256,
|
||||
num_kv_heads=1,
|
||||
head_size=16,
|
||||
dtype=torch.bfloat16,
|
||||
tokens_per_state=128,
|
||||
),
|
||||
"compressor_state.4": SlidingWindowMLASpec(
|
||||
block_size=4,
|
||||
num_kv_heads=1,
|
||||
head_size=8,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=8,
|
||||
),
|
||||
}
|
||||
groups = get_kv_cache_groups(config, specs)
|
||||
kv_cache_config = get_kv_cache_config_from_groups(
|
||||
config, groups, available_memory=8 * MAIN_KV_PAGE_BYTES
|
||||
)
|
||||
|
||||
# Hashes are computed every 4 tokens, but without an align-mode Mamba
|
||||
# group cache hits remain on the 256-token scheduler boundary.
|
||||
assert resolve_kv_cache_block_sizes(kv_cache_config, config) == (256, 4)
|
||||
|
||||
@pytest.mark.parametrize("wide", ["unbalanced_attention", "unsplittable_state"])
|
||||
def test_mamba_split_measures_the_block_the_other_groups_already_force(self, wide):
|
||||
"""Whatever fixes the block stride -- a bucket with unequal layer counts
|
||||
per page size, which is emitted whole, or a single-layer state that
|
||||
cannot be split at all -- the mamba layers must be sized against it.
|
||||
Sizing them against a narrower bucket splits them past what the block
|
||||
already fits, spending a pool block per extra group for no saving."""
|
||||
config = _mock_vllm_config("BLNHC")
|
||||
config.speculative_config = None
|
||||
specs = {}
|
||||
if wide == "unbalanced_attention":
|
||||
for i in range(3):
|
||||
specs[f"wide.{i}"] = _mla(1024)
|
||||
specs["wide.odd"] = _mla(800)
|
||||
else:
|
||||
specs["wide.state"] = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((51_200,),),
|
||||
dtypes=(torch.bfloat16,),
|
||||
tp_replicated=True,
|
||||
)
|
||||
# Mixed and balanced, but far narrower than the bucket above.
|
||||
for i in range(10):
|
||||
specs[f"narrow.a.{i}"] = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=8, dtype=torch.uint8
|
||||
)
|
||||
specs[f"narrow.b.{i}"] = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=16, dtype=torch.uint8
|
||||
)
|
||||
for i in range(100):
|
||||
specs[f"gdn.{i}"] = MambaSpec(
|
||||
block_size=16, shapes=((512,),), dtypes=(torch.bfloat16,)
|
||||
)
|
||||
|
||||
groups = _get_packed_kv_cache_groups(config, specs)
|
||||
gdn = [g for g in groups if g.layer_names[0].startswith("gdn.")]
|
||||
|
||||
assert _get_kv_cache_bytes_per_block(groups) == sum(
|
||||
specs[name].page_size_bytes for name in specs if name.startswith("wide.")
|
||||
)
|
||||
# That block holds every GDN state at once, so the repeat pattern alone
|
||||
# decides the split; the cap must not add groups on top of it.
|
||||
assert len(gdn) == 10
|
||||
|
||||
|
||||
class TestDensePacking:
|
||||
def test_bytes_per_block_is_largest_group(self):
|
||||
groups, g1, g2 = _mixed_page_groups()
|
||||
|
||||
@@ -12,7 +12,7 @@ import pytest
|
||||
import torch
|
||||
|
||||
import vllm.v1.core.kv_cache_utils as kv_cache_utils
|
||||
from vllm.config import ModelConfig, SchedulerConfig, VllmConfig
|
||||
from vllm.config import CacheConfig, ModelConfig, SchedulerConfig, VllmConfig
|
||||
from vllm.config.kv_events import KVEventsConfig
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.multimodal.inputs import (
|
||||
@@ -37,7 +37,6 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
get_kv_cache_groups,
|
||||
get_max_concurrency_for_kv_cache_config,
|
||||
get_request_block_hasher,
|
||||
group_and_unify_kv_cache_specs,
|
||||
hash_block_tokens,
|
||||
init_none_hash,
|
||||
is_kv_cache_spec_uniform,
|
||||
@@ -2226,38 +2225,6 @@ def new_swa_mla_spec(head_size=576, sliding_window=128, model_version=None):
|
||||
)
|
||||
|
||||
|
||||
def test_group_and_unify_kv_cache_specs_no_swa_mla_returns_none():
|
||||
# Without any SlidingWindowMLASpec the function does not apply.
|
||||
specs = {"mla.0": new_mla_spec(), "mla.1": new_mla_spec()}
|
||||
assert group_and_unify_kv_cache_specs(specs) is None
|
||||
|
||||
|
||||
def test_group_and_unify_kv_cache_specs_uniform_page_size_returns_none():
|
||||
# A non-DeepseekV4 model that mixes full MLA and sliding-window MLA layers
|
||||
# with a uniform page size must not fall into the DeepseekV4 tuple-packing
|
||||
# path; it should defer to the generic uniform-page-size grouping instead.
|
||||
mla_spec = new_mla_spec()
|
||||
swa_spec = new_swa_mla_spec()
|
||||
assert mla_spec.page_size_bytes == swa_spec.page_size_bytes
|
||||
specs = {"mla.0": mla_spec, "mla.1": new_mla_spec(), "swa.0": swa_spec}
|
||||
assert group_and_unify_kv_cache_specs(specs) is None
|
||||
|
||||
|
||||
def test_group_and_unify_kv_cache_specs_mixed_page_size_groups():
|
||||
# DeepseekV4-style: differing page sizes across MLA and sliding-window MLA
|
||||
# layers do require tuple packing, so grouping must still be produced.
|
||||
mla_spec = new_mla_spec()
|
||||
swa_spec = new_swa_mla_spec(head_size=1024)
|
||||
assert mla_spec.page_size_bytes != swa_spec.page_size_bytes
|
||||
specs = {"mla.0": mla_spec, "mla.1": new_mla_spec(), "swa.0": swa_spec}
|
||||
grouped = group_and_unify_kv_cache_specs(specs)
|
||||
assert grouped is not None
|
||||
# One MLA group plus one sliding-window MLA group.
|
||||
assert len(grouped) == 2
|
||||
layer_names = {name for g in grouped for name in g.kv_cache_specs}
|
||||
assert layer_names == {"mla.0", "mla.1", "swa.0"}
|
||||
|
||||
|
||||
def new_indexer_mla_spec(block_size=16):
|
||||
# Sparse-attention indexer k_cache: an MLAAttentionSpec with a much smaller
|
||||
# page size than the main MLA attention (uint8, small head), so their pages
|
||||
@@ -2270,10 +2237,30 @@ def new_indexer_mla_spec(block_size=16):
|
||||
)
|
||||
|
||||
|
||||
def test_mixed_page_size_groups_use_spec_compatibility():
|
||||
specs = {}
|
||||
for i in range(3):
|
||||
specs[f"mla.{i}"] = new_mla_spec()
|
||||
specs[f"indexer.{i}"] = new_indexer_mla_spec()
|
||||
specs.update({f"swa.{i}": new_swa_mla_spec(head_size=1024) for i in range(5)})
|
||||
|
||||
config = _grouping_config()
|
||||
config.cache_config = CacheConfig()
|
||||
config.cache_config.kv_cache_layout = "BLNHC"
|
||||
groups = get_kv_cache_groups(config, specs)
|
||||
|
||||
assert len(groups) == 3
|
||||
assert {name for group in groups for name in group.layer_names} == set(specs)
|
||||
assert sorted(len(group.layer_names) for group in groups) == [2, 3, 6]
|
||||
|
||||
|
||||
def _grouping_config():
|
||||
cache_config = CacheConfig()
|
||||
cache_config.kv_cache_layout = "LBNHC"
|
||||
return SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
|
||||
speculative_config=None,
|
||||
cache_config=cache_config,
|
||||
)
|
||||
|
||||
|
||||
@@ -3281,10 +3268,16 @@ def test_iter_layer_specs_returns_group_members():
|
||||
assert list(iter_layer_specs(wrapped)) == [full, mla]
|
||||
|
||||
|
||||
def _spec_decode_grouping_config(method="dspark"):
|
||||
def _spec_decode_grouping_config(method="dspark", model_type=None):
|
||||
"""Grouping config with an EAGLE-family speculative method enabled."""
|
||||
return SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
|
||||
cache_config=SimpleNamespace(
|
||||
get_resolved_kv_cache_layout=lambda: SimpleNamespace(
|
||||
is_block_outermost=True
|
||||
)
|
||||
),
|
||||
model_config=SimpleNamespace(hf_config=SimpleNamespace(model_type=model_type)),
|
||||
speculative_config=SimpleNamespace(
|
||||
method=method,
|
||||
use_eagle=lambda: True,
|
||||
@@ -3322,13 +3315,12 @@ def test_draft_group_annotated_on_hybrid_general_path():
|
||||
|
||||
flagged = [g for g in groups if g.is_eagle_group]
|
||||
assert len(flagged) == 1
|
||||
assert flagged[0].layer_names == ["draft.attn.0"]
|
||||
assert "draft.attn.0" in flagged[0].layer_names
|
||||
|
||||
|
||||
def test_mamba_groups_never_flagged_even_when_draft_shares_a_group():
|
||||
# MLAAttentionSpec.merge ORs non_causal_multi_token_decode rather than
|
||||
# requiring equality, so a draft layer can share a group with target
|
||||
# layers; that combined group still holds volatile draft KV and must be
|
||||
# Packed uniform-type groups can contain distinct target and draft layer
|
||||
# specs; the combined group still holds volatile draft KV and must be
|
||||
# flagged. What must never happen is a Mamba group being flagged: that
|
||||
# widens its lookup window to two consecutive chunks, which align-mode
|
||||
# checkpointing never produces, zeroing every lookup.
|
||||
@@ -3347,10 +3339,8 @@ def test_mamba_groups_never_flagged_even_when_draft_shares_a_group():
|
||||
def test_draft_group_not_annotated_without_spec_decode():
|
||||
# The marker alone must not flag anything; the eagle semantics only apply
|
||||
# when a speculative method is actually enabled.
|
||||
config = SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
|
||||
speculative_config=None,
|
||||
)
|
||||
config = _spec_decode_grouping_config()
|
||||
config.speculative_config = None
|
||||
groups = get_kv_cache_groups(config, _hybrid_specs_with_draft(draft=True))
|
||||
|
||||
assert not any(g.is_eagle_group for g in groups)
|
||||
@@ -3396,12 +3386,13 @@ def _deepseek_v4_specs(model_version="deepseek_v4"):
|
||||
}
|
||||
|
||||
|
||||
def test_deepseek_v4_draft_group_annotated_on_group_and_unify_path():
|
||||
def test_deepseek_v4_draft_group_annotated_on_packed_path():
|
||||
# DeepseekV4's MTP block reuses the target's decoder layer, so its spec
|
||||
# carries no draft marker and only the positional rule can find it. This
|
||||
# pins the pre-existing behaviour that the unified annotator must preserve.
|
||||
groups = get_kv_cache_groups(
|
||||
_spec_decode_grouping_config(method="mtp"), _deepseek_v4_specs()
|
||||
_spec_decode_grouping_config(method="mtp", model_type="deepseek_v4"),
|
||||
_deepseek_v4_specs(model_version=None),
|
||||
)
|
||||
|
||||
flagged = [g for g in groups if g.is_eagle_group]
|
||||
@@ -3409,13 +3400,13 @@ def test_deepseek_v4_draft_group_annotated_on_group_and_unify_path():
|
||||
assert "model.layers.3.self_attn.attn" in flagged[0].layer_names
|
||||
|
||||
|
||||
def test_deepseek_v4_annotation_requires_model_version():
|
||||
def test_deepseek_v4_annotation_requires_model_type():
|
||||
# The positional rule is only sound for DeepseekV4, where the draft layer
|
||||
# is known to be registered last. Without that model gate nothing may be
|
||||
# flagged, however the grouping happens to fall out.
|
||||
groups = get_kv_cache_groups(
|
||||
_spec_decode_grouping_config(method="mtp"),
|
||||
_deepseek_v4_specs(model_version=None),
|
||||
_spec_decode_grouping_config(method="mtp", model_type="other"),
|
||||
_deepseek_v4_specs(),
|
||||
)
|
||||
|
||||
assert not any(g.is_eagle_group for g in groups)
|
||||
|
||||
@@ -1761,6 +1761,52 @@ def test_spec_decode_padding_skipped_for_diffusion():
|
||||
assert r2.request_id not in out.scheduled_spec_decode_tokens
|
||||
|
||||
|
||||
@pytest.mark.parametrize("aligned_tokens", [1, 2, 3])
|
||||
def test_spec_decode_padding_dropped_when_recurrent_alignment_clips(
|
||||
aligned_tokens: int,
|
||||
):
|
||||
"""A clipped padded request must drop its padding, not shorten it.
|
||||
|
||||
The sampler derives a request's row window from its draft count, so a
|
||||
padded request that keeps fewer than 1 + num_spec query rows would select
|
||||
the preceding request's hidden states.
|
||||
"""
|
||||
num_spec = 3
|
||||
scheduler = create_scheduler(
|
||||
num_speculative_tokens=num_spec,
|
||||
enable_prefix_caching=True,
|
||||
block_size=16,
|
||||
)
|
||||
r1, r2 = create_requests(
|
||||
num_requests=2, num_tokens=33, same_prompt=True, max_tokens=16
|
||||
)
|
||||
|
||||
scheduler.add_request(r1)
|
||||
out = scheduler.schedule()
|
||||
assert out.num_scheduled_tokens[r1.request_id] == 33
|
||||
_model_output(scheduler, out, [[100]])
|
||||
scheduler.update_draft_token_ids(DraftTokenIds([r1.request_id], [[1, 2, 3]]))
|
||||
|
||||
# Model the recurrent align split that clips a padded 1+3 prompt tail at
|
||||
# the next cache boundary. The real QSA block size produces all three cases
|
||||
# depending on prompt length modulo four.
|
||||
scheduler.need_mamba_block_aligned_split = True
|
||||
original_split = scheduler._mamba_block_aligned_split
|
||||
|
||||
def align_prompt_tail(request, num_new_tokens, *args):
|
||||
if request is r2:
|
||||
return aligned_tokens
|
||||
return original_split(request, num_new_tokens, *args)
|
||||
|
||||
scheduler._mamba_block_aligned_split = align_prompt_tail
|
||||
scheduler.add_request(r2)
|
||||
out = scheduler.schedule()
|
||||
|
||||
assert out.scheduled_spec_decode_tokens[r1.request_id] == [1, 2, 3]
|
||||
assert out.num_scheduled_tokens[r2.request_id] == 1
|
||||
assert r2.request_id not in out.scheduled_spec_decode_tokens
|
||||
|
||||
|
||||
def test_spec_decode_padding_skipped_with_prefill_in_batch():
|
||||
"""Padding is skipped when the batch contains a prefill chunk: the batch is
|
||||
already mixed/non-uniform, so padding a new decode request buys nothing.
|
||||
|
||||
@@ -14,6 +14,7 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
)
|
||||
from vllm.v1.core.single_type_kv_cache_manager import (
|
||||
ChunkedLocalAttentionManager,
|
||||
CircularBufferManager,
|
||||
FullAttentionManager,
|
||||
MambaManager,
|
||||
RSWAManager,
|
||||
@@ -21,6 +22,7 @@ from vllm.v1.core.single_type_kv_cache_manager import (
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
ChunkedLocalAttentionSpec,
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
MambaSpec,
|
||||
RSWASpec,
|
||||
@@ -136,6 +138,67 @@ def get_chunked_local_attention_manager(
|
||||
)
|
||||
|
||||
|
||||
def test_circular_buffer_allocates_one_block_for_the_request_lifetime():
|
||||
block_size = 4
|
||||
spec = CircularBufferSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=8,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
block_pool = BlockPool(
|
||||
num_gpu_blocks=10, enable_caching=True, hash_block_size=block_size
|
||||
)
|
||||
manager = CircularBufferManager(
|
||||
spec,
|
||||
block_pool=block_pool,
|
||||
enable_caching=True,
|
||||
kv_cache_group_id=0,
|
||||
scheduler_block_size=block_size,
|
||||
needs_kv_cache_zeroing=True,
|
||||
max_admission_blocks_per_request=1,
|
||||
)
|
||||
request_id = "request"
|
||||
|
||||
assert (
|
||||
manager.get_num_blocks_to_allocate(
|
||||
request_id,
|
||||
num_tokens=128,
|
||||
new_computed_blocks=[block_pool.blocks[1]],
|
||||
total_computed_tokens=0,
|
||||
num_local_computed_tokens=0,
|
||||
num_tokens_main_model=128,
|
||||
)
|
||||
== 1
|
||||
)
|
||||
blocks = manager.allocate_new_blocks(
|
||||
request_id, num_tokens=128, num_tokens_main_model=128
|
||||
)
|
||||
assert len(blocks) == 1
|
||||
assert not manager.records_new_block_ids
|
||||
assert manager.take_new_block_ids() == []
|
||||
|
||||
for num_tokens in (256, 1024):
|
||||
assert (
|
||||
manager.get_num_blocks_to_allocate(
|
||||
request_id,
|
||||
num_tokens,
|
||||
[],
|
||||
total_computed_tokens=num_tokens - 1,
|
||||
num_local_computed_tokens=0,
|
||||
num_tokens_main_model=num_tokens,
|
||||
)
|
||||
== 0
|
||||
)
|
||||
assert manager.allocate_new_blocks(request_id, num_tokens, num_tokens) == []
|
||||
|
||||
manager.cache_blocks(request_id, 1024)
|
||||
manager.remove_skipped_blocks(request_id, 1024)
|
||||
assert manager.get_num_common_prefix_blocks(request_id) == 0
|
||||
assert manager.get_num_skipped_tokens(1024) == 0
|
||||
assert manager.req_to_blocks[request_id] == blocks
|
||||
|
||||
|
||||
def test_sliding_window_records_new_blocks_for_zeroing():
|
||||
block_size = 2
|
||||
spec = SlidingWindowSpec(
|
||||
|
||||
@@ -234,7 +234,8 @@ def get_fake_execute_model_fn(original_execute_model_fn: Callable):
|
||||
iter(scheduler_output.num_scheduled_tokens.values())
|
||||
)
|
||||
assert num_scheduled_tokens == cur_step_action.num_scheduled_tokens
|
||||
mamba_group_ids, mamba_spec = get_mamba_groups(self.kv_cache_config)
|
||||
mamba_groups = get_mamba_groups(self.kv_cache_config)
|
||||
mamba_spec, mamba_group_ids = next(iter(mamba_groups.items()))
|
||||
mamba_group_id = mamba_group_ids[0]
|
||||
mamba_layer_name = self.kv_cache_config.kv_cache_groups[
|
||||
mamba_group_id
|
||||
@@ -321,7 +322,8 @@ def get_fake_process_mamba_fn(
|
||||
assert len(copy_info[0]) == len(copy_info[1]) == len(copy_info[2]) == 0
|
||||
else:
|
||||
assert len(copy_info[0]) == len(copy_info[1]) == len(copy_info[2]) == 2
|
||||
mamba_group_ids, mamba_spec = get_mamba_groups(kv_cache_config)
|
||||
mamba_groups = get_mamba_groups(kv_cache_config)
|
||||
mamba_spec, mamba_group_ids = next(iter(mamba_groups.items()))
|
||||
mamba_group_id = mamba_group_ids[0]
|
||||
mamba_layer_name = kv_cache_config.kv_cache_groups[
|
||||
mamba_group_id
|
||||
@@ -936,12 +938,12 @@ def _run_mamba_prefix_cache_mrv1(
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
@create_new_process_for_each_test("spawn")
|
||||
def test_mamba_prefix_cache_mrv1(monkeypatch: pytest.MonkeyPatch):
|
||||
_run_mamba_prefix_cache_mrv1(monkeypatch, async_scheduling=False)
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
@create_new_process_for_each_test("spawn")
|
||||
def test_mamba_prefix_cache_mrv1_async(monkeypatch: pytest.MonkeyPatch):
|
||||
_run_mamba_prefix_cache_mrv1(monkeypatch, async_scheduling=True)
|
||||
|
||||
@@ -975,11 +977,12 @@ def _run_mamba_prefix_cache_mrv2(
|
||||
forward_context = (
|
||||
model_state.vllm_config.compilation_config.static_forward_context
|
||||
)
|
||||
group_ids, _ = get_mamba_groups(kv_cache_config)
|
||||
for group_id in group_ids:
|
||||
block_table = block_tables[group_id]
|
||||
for layer_name in kv_cache_config.kv_cache_groups[group_id].layer_names:
|
||||
yield forward_context[layer_name].kv_cache[-1], block_table
|
||||
mamba_groups = get_mamba_groups(kv_cache_config)
|
||||
for group_ids in mamba_groups.values():
|
||||
for group_id in group_ids:
|
||||
block_table = block_tables[group_id]
|
||||
for layer_name in kv_cache_config.kv_cache_groups[group_id].layer_names:
|
||||
yield forward_context[layer_name].kv_cache[-1], block_table
|
||||
|
||||
def temporal_block(temporal_state, block_table, col):
|
||||
# Resolving the block id for assertions is a deliberate D2H.
|
||||
|
||||
@@ -15,6 +15,8 @@ from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
HiddenStateCacheSpec,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheLayout,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowMLASpec,
|
||||
)
|
||||
@@ -99,12 +101,12 @@ def test_get_block_size_falls_back_to_cache_config_when_no_kv_cache_config():
|
||||
assert block_size == 16
|
||||
|
||||
|
||||
# ---- MLA-verifier absorption ------------------------------------------------
|
||||
# ---- Packed MLA grouping ----------------------------------------------------
|
||||
|
||||
|
||||
def test_find_group_id_errors_clearly_when_absorbed_by_mla_swa_verifier():
|
||||
# HiddenStateCacheSpec subclasses MLAAttentionSpec, so an MLA + sliding-
|
||||
# window MLA verifier absorbs it into the MLA group instead of isolating it.
|
||||
def test_hidden_state_group_isolated_from_packed_mla_groups():
|
||||
# HiddenStateCacheSpec subclasses MLAAttentionSpec, but grouping must pull
|
||||
# it out before packing compatible MLA cache specs.
|
||||
dt = torch.bfloat16
|
||||
spec = {
|
||||
"layers.0.mla": MLAAttentionSpec(
|
||||
@@ -116,11 +118,54 @@ def test_find_group_id_errors_clearly_when_absorbed_by_mla_swa_verifier():
|
||||
"cache_only_layers.61": _hidden(64),
|
||||
}
|
||||
vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(
|
||||
get_resolved_kv_cache_layout=lambda: KVCacheLayout.BLHNC
|
||||
),
|
||||
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
|
||||
speculative_config=None,
|
||||
)
|
||||
groups = get_kv_cache_groups(vllm_config, spec)
|
||||
assert not any(isinstance(g.kv_cache_spec, HiddenStateCacheSpec) for g in groups)
|
||||
hidden_group_ids = [
|
||||
i
|
||||
for i, group in enumerate(groups)
|
||||
if isinstance(group.kv_cache_spec, HiddenStateCacheSpec)
|
||||
]
|
||||
assert len(hidden_group_ids) == 1
|
||||
cfg = SimpleNamespace(kv_cache_groups=groups)
|
||||
with pytest.raises(ValueError, match="MLA verifiers are unsupported"):
|
||||
ExampleHiddenStatesConnector._find_cache_kv_group_id(cfg)
|
||||
assert (
|
||||
ExampleHiddenStatesConnector._find_cache_kv_group_id(cfg) == hidden_group_ids[0]
|
||||
)
|
||||
|
||||
|
||||
def test_hidden_state_group_isolated_from_packed_mixed_page_groups():
|
||||
# Packed grouping keeps groups with unequal page sizes (blocks are strided
|
||||
# by the widest group), so the hidden group is appended without padding.
|
||||
hidden = _hidden(16)
|
||||
spec = {
|
||||
"layers.0.attn": _full(16),
|
||||
"layers.1.mamba": MambaSpec(
|
||||
block_size=16, shapes=((1024,),), dtypes=(torch.float32,)
|
||||
),
|
||||
"cache_only_layers.61": hidden,
|
||||
}
|
||||
vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(
|
||||
get_resolved_kv_cache_layout=lambda: KVCacheLayout.BLHNC
|
||||
),
|
||||
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
|
||||
speculative_config=None,
|
||||
)
|
||||
groups = get_kv_cache_groups(vllm_config, spec)
|
||||
page_sizes = {g.kv_cache_spec.page_size_bytes for g in groups}
|
||||
assert len(page_sizes) > 1, "spec must exercise the packed grouping path"
|
||||
hidden_group_ids = [
|
||||
i
|
||||
for i, group in enumerate(groups)
|
||||
if isinstance(group.kv_cache_spec, HiddenStateCacheSpec)
|
||||
]
|
||||
assert len(hidden_group_ids) == 1
|
||||
assert groups[hidden_group_ids[0]].kv_cache_spec == hidden
|
||||
cfg = SimpleNamespace(kv_cache_groups=groups)
|
||||
assert (
|
||||
ExampleHiddenStatesConnector._find_cache_kv_group_id(cfg) == hidden_group_ids[0]
|
||||
)
|
||||
|
||||
@@ -865,6 +865,10 @@ def _make_mock_worker_for_desc_ids(
|
||||
worker._group_spec_types = group_spec_types
|
||||
worker.block_len_per_layer = block_len_per_layer or [100]
|
||||
worker._conv_decomp = None
|
||||
worker._is_csa_linear = False
|
||||
worker._ssm_region_indices = [0] if has_mamba else []
|
||||
worker._ple_group_index = None
|
||||
worker._ple_region_index = None
|
||||
if has_mamba:
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import ( # noqa: E501
|
||||
MambaConvSplitInfo,
|
||||
@@ -1823,6 +1827,7 @@ def test_push_write_hybrid_mla_replicates_attention():
|
||||
worker.shutdown = lambda: None # skeleton worker: silence __del__
|
||||
worker.use_mla = True
|
||||
worker._has_mamba = True
|
||||
worker._is_csa_linear = False
|
||||
worker._group_spec_types = (MLAAttentionSpec, MambaSpec)
|
||||
worker.transfer_topo = MagicMock()
|
||||
worker.transfer_topo.tp_ratio.return_value = -2
|
||||
|
||||
@@ -645,3 +645,302 @@ def test_mismatched_mla_kernel_page_rejected_for_mla_hybrid():
|
||||
meta_r.block_lens = [x // 2 for x in worker.block_len_per_layer]
|
||||
with pytest.raises((AssertionError, RuntimeError)):
|
||||
worker.add_remote_agent(meta_r, remote_tp_rank=0, remote_tp_size=2)
|
||||
|
||||
|
||||
def _make_csa_linear_ple_worker(scratch_aliases: str = "compressed"):
|
||||
"""``scratch_aliases`` selects which pages the compressor ring overlays."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker as bw
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
|
||||
NixlConnectorWorker,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
|
||||
main_kv_specs = {
|
||||
f"main_kv.{index}": FullAttentionSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=1,
|
||||
head_size=16,
|
||||
dtype=torch.float16,
|
||||
)
|
||||
for index in range(2)
|
||||
}
|
||||
compressed_specs = {
|
||||
f"compressed.{index}": MLAAttentionSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=1,
|
||||
head_size=64,
|
||||
dtype=torch.float16,
|
||||
tokens_per_state=2,
|
||||
)
|
||||
for index in range(2)
|
||||
}
|
||||
compressor_state_specs = {
|
||||
f"compressor_state.{index}": CircularBufferSpec(
|
||||
block_size=2,
|
||||
num_kv_heads=1,
|
||||
head_size=32,
|
||||
dtype=torch.float16,
|
||||
)
|
||||
for index in range(2)
|
||||
}
|
||||
sharded_mamba_spec = MambaSpec(
|
||||
block_size=1,
|
||||
shapes=((8, 3), (1, 4, 4)),
|
||||
dtypes=(torch.float16, torch.float32),
|
||||
page_size_padded=256,
|
||||
mamba_type=MambaAttentionBackendEnum.GDN_ATTN,
|
||||
)
|
||||
ple_spec = MambaSpec(
|
||||
block_size=1,
|
||||
shapes=((6, 3),),
|
||||
dtypes=(torch.float16,),
|
||||
page_size_padded=256,
|
||||
mamba_type=MambaAttentionBackendEnum.SHORT_CONV,
|
||||
tp_replicated=True,
|
||||
)
|
||||
compressed_sparse_specs = {
|
||||
name: spec
|
||||
for index in range(2)
|
||||
for name, spec in (
|
||||
(f"main_kv.{index}", main_kv_specs[f"main_kv.{index}"]),
|
||||
(f"compressed.{index}", compressed_specs[f"compressed.{index}"]),
|
||||
)
|
||||
}
|
||||
|
||||
def group(specs):
|
||||
uniform = UniformTypeKVCacheSpecs.from_specs(specs)
|
||||
assert uniform is not None
|
||||
return KVCacheGroupSpec(list(specs), uniform)
|
||||
|
||||
tensor_regions: tuple[tuple[str, ...], ...]
|
||||
if scratch_aliases == "compressed":
|
||||
tensor_regions = (
|
||||
("main_kv.0", "mamba.sharded", "mamba.ple"),
|
||||
("main_kv.1",),
|
||||
("compressed.0", "compressor_state.0"),
|
||||
("compressed.1", "compressor_state.1"),
|
||||
)
|
||||
else:
|
||||
assert scratch_aliases == "main_kv"
|
||||
tensor_regions = (
|
||||
("main_kv.0", "mamba.sharded", "mamba.ple", "compressor_state.0"),
|
||||
("main_kv.1", "compressor_state.1"),
|
||||
("compressed.0",),
|
||||
("compressed.1",),
|
||||
)
|
||||
region_size = 512
|
||||
page_size = 256
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=2,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=len(tensor_regions) * region_size,
|
||||
layers=[layer_name],
|
||||
layer_stride=region_size,
|
||||
block_stride=page_size,
|
||||
offset=region_index * region_size,
|
||||
)
|
||||
for region_index, layer_names in enumerate(tensor_regions)
|
||||
for layer_name in layer_names
|
||||
],
|
||||
kv_cache_groups=[
|
||||
group(compressed_sparse_specs),
|
||||
group(compressor_state_specs),
|
||||
KVCacheGroupSpec(["mamba.sharded"], sharded_mamba_spec),
|
||||
KVCacheGroupSpec(["mamba.ple"], ple_spec),
|
||||
],
|
||||
)
|
||||
|
||||
vllm_config = create_vllm_config(block_size=4)
|
||||
vllm_config.cache_config.enable_prefix_caching = False
|
||||
vllm_config.kv_transfer_config.kv_buffer_device = "cuda"
|
||||
fake_backend = MagicMock()
|
||||
fake_backend.get_supported_kernel_block_sizes.return_value = [4]
|
||||
fake_backend.get_name.return_value = "TEST_ATTN"
|
||||
fake_backend.full_cls_name.return_value = "test.AttentionBackend"
|
||||
fake_platform = MagicMock()
|
||||
fake_platform.device_type = "cuda"
|
||||
fake_platform.get_nixl_memory_type.return_value = "VRAM"
|
||||
|
||||
with (
|
||||
patch.object(bw, "NixlWrapper", _RecordingNixl),
|
||||
patch.object(bw, "get_tensor_model_parallel_rank", return_value=0),
|
||||
patch.object(bw, "get_tensor_model_parallel_world_size", return_value=1),
|
||||
patch.object(bw, "get_current_attn_backends", return_value=[fake_backend]),
|
||||
patch.object(bw, "current_platform", fake_platform),
|
||||
patch(
|
||||
"vllm.model_executor.layers.mamba.mamba_utils.get_conv_state_layout",
|
||||
return_value="DS",
|
||||
),
|
||||
set_current_vllm_config(vllm_config),
|
||||
):
|
||||
worker = NixlConnectorWorker(vllm_config, "local-engine", kv_cache_config)
|
||||
tensors = [torch.zeros((2, 256), dtype=torch.uint8) for _ in range(4)]
|
||||
worker.register_kv_caches(
|
||||
{
|
||||
layer_name: tensors[region_index]
|
||||
for region_index, layer_names in enumerate(tensor_regions)
|
||||
for layer_name in layer_names
|
||||
}
|
||||
)
|
||||
return worker
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_csa_linear_registration_discovers_shared_regions():
|
||||
worker = _make_csa_linear_ple_worker()
|
||||
|
||||
assert worker._ssm_region_indices == [0]
|
||||
assert worker._ple_group_index == 3
|
||||
assert worker._ple_region_index == 0
|
||||
assert worker._region_is_mla == [False, False, True, True]
|
||||
assert worker._scratch_region_indices == [2, 3]
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_csa_linear_scratch_descs_follow_the_pages_they_overlay():
|
||||
"""The scratch ring is addressed by the regions it actually registered in,
|
||||
not by inferring that it must sit on the MLA pages: overlaying the main KV
|
||||
pages instead moves its descriptors with it."""
|
||||
worker = _make_csa_linear_ple_worker(scratch_aliases="main_kv")
|
||||
|
||||
# Same MLA regions as before, but the ring now lives in the non-MLA ones.
|
||||
assert worker._region_is_mla == [False, False, True, True]
|
||||
assert worker._scratch_region_indices == [0, 1]
|
||||
|
||||
desc_ids = worker._compute_desc_ids(
|
||||
block_ids=([1], [0], [1], [1]),
|
||||
dst_num_blocks=2,
|
||||
block_size_ratio=None,
|
||||
physical_blocks_per_logical=1,
|
||||
)
|
||||
# Scratch group (block 0) now maps to regions 0 and 1 -> descs 0 and 2,
|
||||
# where the compressed-page placement gave 4 and 6.
|
||||
assert desc_ids.tolist()[4:6] == [0, 2]
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_csa_linear_selects_compressed_sparse_compressor_state_and_mamba_regions():
|
||||
worker = _make_csa_linear_ple_worker()
|
||||
|
||||
desc_ids = worker._compute_desc_ids(
|
||||
block_ids=([1], [0], [1], [1]),
|
||||
dst_num_blocks=2,
|
||||
block_size_ratio=None,
|
||||
physical_blocks_per_logical=1,
|
||||
)
|
||||
assert desc_ids.tolist() == [
|
||||
1,
|
||||
3,
|
||||
5,
|
||||
7,
|
||||
4,
|
||||
6,
|
||||
9,
|
||||
11,
|
||||
13,
|
||||
15,
|
||||
17,
|
||||
]
|
||||
|
||||
mamba_only = worker._compute_desc_ids(
|
||||
block_ids=([], [], [1], [1]),
|
||||
dst_num_blocks=2,
|
||||
block_size_ratio=None,
|
||||
physical_blocks_per_logical=1,
|
||||
)
|
||||
assert mamba_only.tolist() == [9, 11, 13, 15, 17]
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_csa_linear_ple_descriptor_is_not_split():
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
|
||||
TPMapping,
|
||||
)
|
||||
|
||||
worker = _make_csa_linear_ple_worker()
|
||||
bases = [0x10000, 0x20000, 0x30000, 0x40000]
|
||||
attention = worker._build_fa_local(bases, block_size_ratio=1)
|
||||
mamba = worker._build_mamba_local(bases)
|
||||
assert attention.shape == (8, 3)
|
||||
assert mamba.shape == (10, 3)
|
||||
assert mamba[-2:, 0].tolist() == [0x10000, 0x10100]
|
||||
assert mamba[-2:, 1].tolist() == [256, 256]
|
||||
|
||||
mapping = TPMapping(
|
||||
source_ranks_per_group=((0,), (0,), (0, 1), (0, 1)),
|
||||
all_source_ranks=(0, 1),
|
||||
rank_to_attention_slot={0: 0, 1: 0},
|
||||
rank_offset_factor=0,
|
||||
)
|
||||
splits = list(
|
||||
worker._build_local_splits_from_plan(
|
||||
mapping,
|
||||
np.concatenate([attention, mamba]),
|
||||
num_fa_descs=len(attention),
|
||||
)
|
||||
)
|
||||
assert len(splits) == 2
|
||||
assert [row[1] for row in splits[0][-2:]] == [256, 256]
|
||||
assert [row[1] for row in splits[1][-2:]] == [256, 256]
|
||||
assert splits[0][-3][1] * 2 == mamba[-3, 1]
|
||||
assert splits[1][-3][1] * 2 == mamba[-3, 1]
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_csa_linear_remote_ple_is_copied_whole():
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
NixlAgentMetadata,
|
||||
)
|
||||
|
||||
worker = _make_csa_linear_ple_worker()
|
||||
metadata = NixlAgentMetadata(
|
||||
engine_id="remote",
|
||||
agent_metadata=b"agent",
|
||||
kv_caches_base_addr=[
|
||||
0x10000,
|
||||
0x20000,
|
||||
0x30000,
|
||||
0x40000,
|
||||
],
|
||||
device_id=0,
|
||||
num_blocks=2,
|
||||
block_lens=[256] * 4,
|
||||
block_strides=[256] * 4,
|
||||
kv_cache_layout="HND",
|
||||
block_size=4,
|
||||
ssm_sizes=(24, 32),
|
||||
attn_backend_name="test",
|
||||
physical_blocks_per_logical_kv_block=1,
|
||||
)
|
||||
descriptors = worker._build_mamba_remote(
|
||||
metadata,
|
||||
tp_ratio=-2,
|
||||
transfer_info=SimpleNamespace(remote_physical_blocks_per_logical=1),
|
||||
)
|
||||
assert descriptors.shape == (10, 3)
|
||||
assert descriptors[-2:, 0].tolist() == [0x10000, 0x10100]
|
||||
assert descriptors[-2:, 1].tolist() == [256, 256]
|
||||
|
||||
metadata.block_lens[0] = 128
|
||||
with pytest.raises(ValueError, match="PLE pages require identical"):
|
||||
worker._build_mamba_remote(
|
||||
metadata,
|
||||
tp_ratio=-2,
|
||||
transfer_info=SimpleNamespace(remote_physical_blocks_per_logical=1),
|
||||
)
|
||||
|
||||
@@ -359,6 +359,7 @@ class _StubWriterWorker(NixlPushConnectorWorker):
|
||||
w._physical_blocks_per_logical_kv_block = 1
|
||||
# Single non-hybrid attention group, matching the stub block id lists.
|
||||
w._has_mamba = False
|
||||
w._is_csa_linear = False
|
||||
w._group_spec_types = (FullAttentionSpec,)
|
||||
w._engine_ttl = 0.0
|
||||
w._engine_last_active = {}
|
||||
@@ -1198,6 +1199,75 @@ class TestPushWriterMlaReplication:
|
||||
# engine thread never sees a partial set and double-frees the request.
|
||||
assert sorted(w._sending_transfers["p-req"]) == [1000, 1001]
|
||||
|
||||
def test_csa_linear_writes_every_group_to_each_target(self):
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
RemoteMeta,
|
||||
ReqMeta,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
|
||||
TPMapping,
|
||||
)
|
||||
|
||||
engine_id = "decode-engine"
|
||||
w = _StubWriterWorker.fresh()
|
||||
w.world_size = 2
|
||||
w.use_mla = False
|
||||
w._has_mamba = True
|
||||
w._is_csa_linear = True
|
||||
w.nixl_wrapper = MagicMock()
|
||||
w.transfer_topo = MagicMock()
|
||||
w.transfer_topo.get_engine_info.return_value = SimpleNamespace(
|
||||
remote_tp_size=4,
|
||||
remote_block_size=16,
|
||||
remote_physical_blocks_per_logical=1,
|
||||
)
|
||||
w.transfer_topo.tp_ratio.return_value = -2
|
||||
w.transfer_topo.handshake_target_ranks.return_value = (0, 1)
|
||||
w.tp_mappings = {
|
||||
engine_id: TPMapping(
|
||||
source_ranks_per_group=((0,), (0,), (0, 1), (0, 1)),
|
||||
all_source_ranks=(0, 1),
|
||||
rank_to_attention_slot={0: 0, 1: 0},
|
||||
rank_offset_factor=0,
|
||||
)
|
||||
}
|
||||
w._logical_to_kernel_block_ids = lambda block_ids, ratio: block_ids
|
||||
w.dst_xfer_side_handles = {engine_id: {0: 1000, 1: 1001}}
|
||||
w.src_xfer_handles_by_tp_ratio = {(-2, 16): [2000, 2001]}
|
||||
|
||||
writes = []
|
||||
|
||||
def _fake_xfer(**kwargs):
|
||||
writes.append(kwargs)
|
||||
return 3000 + kwargs["read_spec"].remote_rank
|
||||
|
||||
w._xfer_blocks = _fake_xfer
|
||||
block_ids = ([10], [20], [30], [40])
|
||||
meta = ReqMeta(
|
||||
local_block_ids=block_ids,
|
||||
local_physical_block_ids=block_ids,
|
||||
tp_size=2,
|
||||
remote=RemoteMeta(
|
||||
block_ids=([50], [60], [70], [80]),
|
||||
host="",
|
||||
port=0,
|
||||
engine_id=engine_id,
|
||||
request_id="d-req",
|
||||
),
|
||||
)
|
||||
|
||||
w._xfer_blocks_for_req(req_id="p-req", meta=meta)
|
||||
|
||||
assert [write["read_spec"].remote_rank for write in writes] == [0, 1]
|
||||
assert [write["read_spec"].local_block_ids for write in writes] == [
|
||||
list(block_ids),
|
||||
list(block_ids),
|
||||
]
|
||||
assert [write["local_xfer_side_handle"] for write in writes] == [2000, 2001]
|
||||
assert sorted(w._sending_transfers["p-req"]) == [3000, 3001]
|
||||
|
||||
|
||||
class TestPushPrefixCaching:
|
||||
"""Partial prefix-cache hit on D: D preallocates only its *uncomputed*
|
||||
|
||||
@@ -154,6 +154,9 @@ def _make_mock_worker_for_splits(group_spec_types):
|
||||
worker.block_len_per_layer = []
|
||||
worker.num_regions = 0
|
||||
worker._region_is_mla = []
|
||||
worker._conv_decomp = SimpleNamespace(local_conv_offsets=())
|
||||
worker._ssm_region_indices = [0] if MambaSpec in group_spec_types else []
|
||||
worker._ple_region_index = None
|
||||
return worker
|
||||
|
||||
|
||||
@@ -282,3 +285,31 @@ class TestMambaPlanSplitHandles:
|
||||
|
||||
with pytest.raises(AssertionError, match="Head-sharded"):
|
||||
list(worker._build_local_splits_from_plan(plan, src_blocks_data, 2, 2))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("total_kv_heads", "local_tp", "remote_tp", "compatible"),
|
||||
[
|
||||
# 2 heads: TP1 packs both heads into a page, TP >= 2 packs one.
|
||||
(2, 1, 2, False),
|
||||
(2, 2, 1, False),
|
||||
(2, 1, 1, True),
|
||||
(2, 2, 4, True),
|
||||
(2, 4, 2, True),
|
||||
# 4 heads: the boundary moves with the head count.
|
||||
(4, 2, 4, False),
|
||||
(4, 2, 2, True),
|
||||
(4, 4, 8, True),
|
||||
],
|
||||
)
|
||||
def test_csa_linear_tp_layout_boundary(total_kv_heads, local_tp, remote_tp, compatible):
|
||||
worker = object.__new__(NixlConnectorWorker)
|
||||
worker.world_size = local_tp
|
||||
worker._is_csa_linear = True
|
||||
worker.transfer_topo = SimpleNamespace(total_num_kv_heads=total_kv_heads)
|
||||
|
||||
if compatible:
|
||||
worker._validate_csa_linear_tp_layout(remote_tp)
|
||||
else:
|
||||
with pytest.raises(ValueError, match="KV-head sharding boundary"):
|
||||
worker._validate_csa_linear_tp_layout(remote_tp)
|
||||
|
||||
@@ -104,6 +104,47 @@ def _make_speculator(
|
||||
return speculator
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("hc_mult", "expected"), [(None, 64), (4, 256)])
|
||||
def test_speculator_uses_draft_model_hidden_size(monkeypatch, hc_mult, expected):
|
||||
# Qwen4Exp targets expose multi-stream HC residuals to the drafter.
|
||||
monkeypatch.setattr(base_spec_module, "_target_feeds_hc_residual", lambda _: True)
|
||||
hf_config = SimpleNamespace()
|
||||
if hc_mult is not None:
|
||||
hf_config.hc_mult = hc_mult
|
||||
draft_model_config = SimpleNamespace(
|
||||
hf_config=hf_config,
|
||||
get_hidden_size=lambda: 64,
|
||||
get_vocab_size=lambda: 32,
|
||||
)
|
||||
speculative_config = SimpleNamespace(
|
||||
method="mtp",
|
||||
num_speculative_tokens=3,
|
||||
draft_model_config=draft_model_config,
|
||||
use_local_argmax_reduction=False,
|
||||
draft_sample_method="greedy",
|
||||
)
|
||||
vllm_config = SimpleNamespace(
|
||||
speculative_config=speculative_config,
|
||||
scheduler_config=SimpleNamespace(
|
||||
max_num_seqs=2,
|
||||
max_num_batched_tokens=8,
|
||||
),
|
||||
model_config=SimpleNamespace(
|
||||
max_model_len=32,
|
||||
dtype=torch.float32,
|
||||
use_fp64_gumbel=False,
|
||||
),
|
||||
parallel_config=SimpleNamespace(
|
||||
data_parallel_size=1,
|
||||
data_parallel_rank=0,
|
||||
),
|
||||
)
|
||||
|
||||
speculator = _TestSpeculator(vllm_config, torch.device("cpu"))
|
||||
|
||||
assert speculator.hidden_size == expected
|
||||
|
||||
|
||||
def test_mm_support_configured_after_model_load(monkeypatch):
|
||||
target_model_config = object()
|
||||
draft_model_config = object()
|
||||
@@ -197,9 +238,11 @@ def test_load_model_disables_mm_support_for_text_only_drafter(monkeypatch):
|
||||
|
||||
assert not speculator.supports_mm_inputs
|
||||
assert warning_messages == [
|
||||
"Draft model _TextOnlyDraftModel does not support external multimodal "
|
||||
"embeddings. Embeddings from the target model will not be passed to the "
|
||||
"drafter; using text-only draft inputs instead."
|
||||
(
|
||||
"Draft model _TextOnlyDraftModel does not support external multimodal "
|
||||
"embeddings. Embeddings from the target model will not be passed to the "
|
||||
"drafter; using text-only draft inputs instead."
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -132,6 +132,42 @@ def test_block_tables_apply_staged_writes_single_group():
|
||||
)
|
||||
|
||||
|
||||
def test_block_tables_skip_custom_slot_mapping_groups():
|
||||
device = torch.device("cuda")
|
||||
block_tables = BlockTables(
|
||||
block_sizes=[8, 262144],
|
||||
max_num_reqs=1,
|
||||
max_num_batched_tokens=4,
|
||||
max_num_blocks_per_group=[1, 1],
|
||||
device=device,
|
||||
kernel_block_sizes=[8, 262144],
|
||||
slot_mapping_enabled=[False, True],
|
||||
)
|
||||
block_tables.append_block_ids(
|
||||
req_index=0,
|
||||
new_block_ids=([7], [12]),
|
||||
overwrite=True,
|
||||
)
|
||||
block_tables.apply_staged_writes()
|
||||
|
||||
idx_mapping = torch.tensor([0], dtype=torch.int32, device=device)
|
||||
query_start_loc = torch.tensor([0, 2], dtype=torch.int32, device=device)
|
||||
positions = torch.tensor([153797, 165757], dtype=torch.int64, device=device)
|
||||
slot_mappings = block_tables.compute_slot_mappings(
|
||||
idx_mapping,
|
||||
query_start_loc,
|
||||
positions,
|
||||
num_tokens_padded=2,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
assert slot_mappings[0].tolist() == [-1, -1]
|
||||
assert slot_mappings[1].tolist() == [
|
||||
12 * 262144 + 153797,
|
||||
12 * 262144 + 165757,
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cp_rank", range(4))
|
||||
def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
|
||||
"""DCP interleave is expressed in logical-block token coordinates."""
|
||||
|
||||
@@ -8,15 +8,107 @@ import torch
|
||||
|
||||
import vllm.v1.worker.gpu.model_runner as model_runner_module
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
|
||||
|
||||
|
||||
def test_qsa_circular_group_uses_custom_slot_mapping(monkeypatch):
|
||||
runner = GPUModelRunner.__new__(GPUModelRunner)
|
||||
runner.max_model_len = 262144
|
||||
runner.is_encoder_decoder = False
|
||||
runner.dcp_size = 1
|
||||
runner.dcp_rank = 0
|
||||
runner.cp_interleave = 1
|
||||
runner.cache_config = SimpleNamespace(enable_prefix_caching=True)
|
||||
parallel_config = SimpleNamespace(
|
||||
decode_context_parallel_size=1,
|
||||
cp_kv_cache_interleave_size=1,
|
||||
)
|
||||
runner.parallel_config = parallel_config
|
||||
runner.vllm_config = SimpleNamespace(
|
||||
parallel_config=parallel_config,
|
||||
cache_config=SimpleNamespace(mamba_cache_mode="none"),
|
||||
)
|
||||
runner.model_state = SimpleNamespace(
|
||||
get_additional_cg_support=lambda: (),
|
||||
num_new_sampled_tokens_per_step=1,
|
||||
)
|
||||
runner.speculator = None
|
||||
runner.req_states = []
|
||||
runner.input_buffers = SimpleNamespace(query_start_loc=None)
|
||||
runner.vocab_size = 1
|
||||
runner.max_num_reqs = 1
|
||||
runner.max_num_tokens = 2
|
||||
runner.device = torch.device("cuda")
|
||||
|
||||
raw_spec = CircularBufferSpec(
|
||||
block_size=8,
|
||||
num_kv_heads=1,
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
compressed_spec = FullAttentionSpec(
|
||||
block_size=262144,
|
||||
num_kv_heads=1,
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=1,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
layer_names=["raw"],
|
||||
kv_cache_spec=UniformTypeKVCacheSpecs(
|
||||
block_size=8,
|
||||
kv_cache_specs={"raw": raw_spec},
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(layer_names=["compressed"], kv_cache_spec=compressed_spec),
|
||||
],
|
||||
)
|
||||
|
||||
class FakeAttnCGSupport:
|
||||
def narrow(self, *args):
|
||||
return self
|
||||
|
||||
attn_cg_support = FakeAttnCGSupport()
|
||||
monkeypatch.setattr(
|
||||
model_runner_module,
|
||||
"init_attn_backend",
|
||||
lambda *args: ([], attn_cg_support, [8, 262144]),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
model_runner_module,
|
||||
"maybe_create_adaptive_verification_manager",
|
||||
lambda **kwargs: None,
|
||||
)
|
||||
|
||||
captured = {}
|
||||
|
||||
class BlockTablesCaptured(Exception):
|
||||
pass
|
||||
|
||||
def capture_block_tables(**kwargs):
|
||||
captured.update(kwargs)
|
||||
raise BlockTablesCaptured
|
||||
|
||||
monkeypatch.setattr(model_runner_module, "BlockTables", capture_block_tables)
|
||||
|
||||
with pytest.raises(BlockTablesCaptured):
|
||||
runner.initialize_kv_cache(kv_cache_config)
|
||||
|
||||
assert captured["max_num_blocks_per_group"] == [1, 1]
|
||||
assert captured["slot_mapping_enabled"] == [False, True]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mamba_cache_mode", "num_speculative_blocks", "expected"),
|
||||
[
|
||||
|
||||
@@ -20,6 +20,7 @@ from vllm.config.vllm import VllmConfig
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
@@ -53,6 +54,18 @@ def _attention_group() -> KVCacheGroupSpec:
|
||||
)
|
||||
|
||||
|
||||
def _circular_group() -> KVCacheGroupSpec:
|
||||
return KVCacheGroupSpec(
|
||||
["circular"],
|
||||
CircularBufferSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _mamba_group(mamba_cache_mode: str) -> KVCacheGroupSpec:
|
||||
# Name carries the mode so several groups can coexist in one config.
|
||||
return KVCacheGroupSpec(
|
||||
@@ -212,8 +225,7 @@ def test_warmup_reserves_mamba_speculative_blocks(mamba_cache_mode):
|
||||
|
||||
|
||||
def _hybrid_kv_cache_config(num_blocks: int) -> KVCacheConfig:
|
||||
"""Full attention plus one Mamba group per cache mode, so a single manager
|
||||
exercises every branch `_reserved_block_count` has.
|
||||
"""Attention, circular, and Mamba groups exercise every reservation branch.
|
||||
|
||||
"none" and "all" reach the same branch of both `_reserved_block_count` and
|
||||
`MambaManager`, which tests only for "align". They are still both listed:
|
||||
@@ -226,6 +238,7 @@ def _hybrid_kv_cache_config(num_blocks: int) -> KVCacheConfig:
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
_attention_group(),
|
||||
_circular_group(),
|
||||
_mamba_group("none"),
|
||||
_mamba_group("all"),
|
||||
_mamba_group("align"),
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
@@ -10,11 +9,20 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateCopyFunc,
|
||||
MambaStateCopyFuncsByType,
|
||||
get_conv_copy_spec,
|
||||
get_temporal_copy_spec,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheGroupSpec, MambaSpec
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
|
||||
from vllm.v1.worker.mamba_utils import (
|
||||
MambaCopyBuffers,
|
||||
MambaSpecDecodeGPUContext,
|
||||
@@ -22,17 +30,21 @@ from vllm.v1.worker.mamba_utils import (
|
||||
batch_memcpy,
|
||||
collect_mamba_copy_meta,
|
||||
do_mamba_copy_block,
|
||||
get_mamba_groups,
|
||||
preprocess_mamba,
|
||||
stage_postprocess_inputs_to_gpu,
|
||||
)
|
||||
|
||||
MambaStateCopyFunc = Callable[..., Any]
|
||||
|
||||
# Conv + temporal copy specs, in the order the tests' MambaSpec shapes expect.
|
||||
_COPY_FUNCS: tuple[MambaStateCopyFunc, ...] = (
|
||||
_DEFAULT_COPY_FUNCS: tuple[MambaStateCopyFunc, ...] = (
|
||||
get_conv_copy_spec,
|
||||
get_temporal_copy_spec,
|
||||
)
|
||||
_COPY_FUNCS: MambaStateCopyFuncsByType = {
|
||||
MambaAttentionBackendEnum.MAMBA2: _DEFAULT_COPY_FUNCS,
|
||||
MambaAttentionBackendEnum.GDN_ATTN: _DEFAULT_COPY_FUNCS,
|
||||
MambaAttentionBackendEnum.SHORT_CONV: (get_conv_copy_spec,),
|
||||
}
|
||||
|
||||
|
||||
def postprocess_mamba(
|
||||
@@ -41,7 +53,7 @@ def postprocess_mamba(
|
||||
input_batch: Any,
|
||||
requests: dict[str, Any],
|
||||
forward_context: dict[str, Any],
|
||||
mamba_state_copy_funcs: tuple[MambaStateCopyFunc, ...],
|
||||
mamba_state_copy_funcs: MambaStateCopyFuncsByType,
|
||||
copy_bufs: "MambaCopyBuffers",
|
||||
):
|
||||
"""CPU reference for the align-mode postprocess.
|
||||
@@ -147,7 +159,7 @@ def test_resumed_req_ids_cleared_from_mamba_state_idx():
|
||||
input_batch,
|
||||
{}, # requests
|
||||
{}, # forward_context
|
||||
(), # mamba_state_copy_funcs
|
||||
{}, # mamba_state_copy_funcs
|
||||
copy_bufs,
|
||||
)
|
||||
|
||||
@@ -430,7 +442,10 @@ def _make_requests(
|
||||
|
||||
|
||||
def _make_copy_bufs(
|
||||
cfg: _TestConfig, kv_cache_config: KVCacheConfig, device: torch.device
|
||||
cfg: _TestConfig,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
device: torch.device,
|
||||
copy_funcs: MambaStateCopyFuncsByType = _COPY_FUNCS,
|
||||
) -> MambaCopyBuffers:
|
||||
"""Create MambaCopyBuffers for the Python path."""
|
||||
|
||||
@@ -440,13 +455,16 @@ def _make_copy_bufs(
|
||||
return MambaCopyBuffers.create(
|
||||
max_num_reqs=cfg.max_num_reqs,
|
||||
kv_cache_config=kv_cache_config,
|
||||
copy_funcs=(get_conv_copy_spec, get_temporal_copy_spec),
|
||||
copy_funcs=copy_funcs,
|
||||
make_buffer=make_buffer,
|
||||
)
|
||||
|
||||
|
||||
def _make_gpu_ctx(
|
||||
cfg: _TestConfig, kv_cache_config: KVCacheConfig, device: torch.device
|
||||
cfg: _TestConfig,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
device: torch.device,
|
||||
copy_funcs: MambaStateCopyFuncsByType = _COPY_FUNCS,
|
||||
) -> MambaSpecDecodeGPUContext:
|
||||
"""Create MambaSpecDecodeGPUContext for the GPU path."""
|
||||
|
||||
@@ -456,12 +474,170 @@ def _make_gpu_ctx(
|
||||
return MambaSpecDecodeGPUContext.create(
|
||||
max_num_reqs=cfg.max_num_reqs,
|
||||
kv_cache_config=kv_cache_config,
|
||||
num_state_types=2,
|
||||
copy_funcs=copy_funcs,
|
||||
device=device,
|
||||
make_buffer=make_buffer,
|
||||
)
|
||||
|
||||
|
||||
def test_mamba_groups_support_different_state_specs():
|
||||
gdn_spec = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((4, 4), (2, 4, 4)),
|
||||
dtypes=(torch.float16, torch.float32),
|
||||
mamba_type=MambaAttentionBackendEnum.GDN_ATTN,
|
||||
mamba_cache_mode="align",
|
||||
num_speculative_blocks=3,
|
||||
)
|
||||
short_conv_spec = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((12, 8),),
|
||||
dtypes=(torch.float16,),
|
||||
mamba_type=MambaAttentionBackendEnum.SHORT_CONV,
|
||||
mamba_cache_mode="align",
|
||||
num_speculative_blocks=3,
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=8,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["gdn.0", "gdn.1"], gdn_spec),
|
||||
KVCacheGroupSpec(["ple.0"], short_conv_spec),
|
||||
],
|
||||
)
|
||||
|
||||
mamba_groups = get_mamba_groups(kv_cache_config)
|
||||
assert mamba_groups == {gdn_spec: [0], short_conv_spec: [1]}
|
||||
|
||||
model_state = object.__new__(MambaHybridModelState)
|
||||
model_state._mamba_group_ids = []
|
||||
model_state._mamba_spec = None
|
||||
group_ids, representative_spec = model_state._get_mamba_group_info(kv_cache_config)
|
||||
assert group_ids == [0, 1]
|
||||
assert representative_spec is gdn_spec
|
||||
|
||||
cfg = _TestConfig()
|
||||
copy_bufs = _make_copy_bufs(cfg, kv_cache_config, torch.device("cpu"))
|
||||
assert copy_bufs.src_ptrs.cpu.numel() == cfg.max_num_reqs * 5
|
||||
assert copy_bufs.dst_ptrs.cpu.numel() == cfg.max_num_reqs * 5
|
||||
assert copy_bufs.sizes.cpu.numel() == cfg.max_num_reqs * 5
|
||||
|
||||
ctx = _make_gpu_ctx(cfg, kv_cache_config, torch.device("cpu"))
|
||||
assert ctx.num_states == 5
|
||||
|
||||
forward_context = {}
|
||||
gdn_spec_layer_names = ["gdn.0", "gdn.1"]
|
||||
for layer_name in gdn_spec_layer_names:
|
||||
attention = MagicMock()
|
||||
attention.kv_cache = (
|
||||
torch.empty(8, 4, 4, dtype=torch.float16),
|
||||
torch.empty(8, 2, 4, 4, dtype=torch.float32),
|
||||
)
|
||||
forward_context[layer_name] = attention
|
||||
ple_attention = MagicMock()
|
||||
ple_attention.kv_cache = (torch.empty(8, 12, 8, dtype=torch.float16),)
|
||||
forward_context["ple.0"] = ple_attention
|
||||
assert gdn_spec_layer_names == kv_cache_config.kv_cache_groups[0].layer_names
|
||||
|
||||
block_tables = [
|
||||
torch.zeros(4, 8, dtype=torch.int32),
|
||||
torch.zeros(4, 8, dtype=torch.int32),
|
||||
]
|
||||
ctx.initialize_from_forward_context(
|
||||
kv_cache_config,
|
||||
forward_context,
|
||||
_COPY_FUNCS,
|
||||
block_tables,
|
||||
)
|
||||
assert ctx.is_initialized
|
||||
assert ctx.state_group_indices.tolist() == [0, 0, 0, 0, 1]
|
||||
assert ctx.state_conv_widths.tolist() == [4, 0, 4, 0, 12]
|
||||
|
||||
|
||||
def test_mamba_groups_support_mixed_specs_in_uniform_group():
|
||||
gdn_spec = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((4, 4), (2, 4, 4)),
|
||||
dtypes=(torch.float16, torch.float32),
|
||||
mamba_type=MambaAttentionBackendEnum.GDN_ATTN,
|
||||
mamba_cache_mode="align",
|
||||
num_speculative_blocks=3,
|
||||
)
|
||||
short_conv_spec = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((12, 8),),
|
||||
dtypes=(torch.float16,),
|
||||
mamba_type=MambaAttentionBackendEnum.SHORT_CONV,
|
||||
mamba_cache_mode="align",
|
||||
num_speculative_blocks=3,
|
||||
)
|
||||
layer_specs = {
|
||||
"gdn.0": gdn_spec,
|
||||
"gdn.1": gdn_spec,
|
||||
"ple.0": short_conv_spec,
|
||||
}
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=8,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
list(layer_specs),
|
||||
UniformTypeKVCacheSpecs(
|
||||
block_size=16,
|
||||
kv_cache_specs=layer_specs,
|
||||
),
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assert kv_cache_config.has_mamba_layers
|
||||
assert get_mamba_groups(kv_cache_config) == {
|
||||
gdn_spec: [0],
|
||||
short_conv_spec: [0],
|
||||
}
|
||||
|
||||
# The group is listed once per spec, so its id must not be repeated.
|
||||
model_state = object.__new__(MambaHybridModelState)
|
||||
model_state._mamba_group_ids = []
|
||||
model_state._mamba_spec = None
|
||||
group_ids, representative_spec = model_state._get_mamba_group_info(kv_cache_config)
|
||||
assert group_ids == [0]
|
||||
assert representative_spec is gdn_spec
|
||||
|
||||
cfg = _TestConfig()
|
||||
copy_bufs = _make_copy_bufs(cfg, kv_cache_config, torch.device("cpu"))
|
||||
assert copy_bufs.mamba_group_ids == [0]
|
||||
assert copy_bufs.src_ptrs.cpu.numel() == cfg.max_num_reqs * 5
|
||||
|
||||
ctx = _make_gpu_ctx(cfg, kv_cache_config, torch.device("cpu"))
|
||||
assert ctx.mamba_group_ids == [0]
|
||||
assert ctx.num_states == 5
|
||||
|
||||
forward_context = {
|
||||
"gdn.0": MagicMock(
|
||||
kv_cache=(
|
||||
torch.empty(8, 4, 4, dtype=torch.float16),
|
||||
torch.empty(8, 2, 4, 4, dtype=torch.float32),
|
||||
)
|
||||
),
|
||||
"gdn.1": MagicMock(
|
||||
kv_cache=(
|
||||
torch.empty(8, 4, 4, dtype=torch.float16),
|
||||
torch.empty(8, 2, 4, 4, dtype=torch.float32),
|
||||
)
|
||||
),
|
||||
"ple.0": MagicMock(kv_cache=(torch.empty(8, 12, 8, dtype=torch.float16),)),
|
||||
}
|
||||
ctx.initialize_from_forward_context(
|
||||
kv_cache_config,
|
||||
forward_context,
|
||||
_COPY_FUNCS,
|
||||
[torch.zeros(4, 8, dtype=torch.int32)],
|
||||
)
|
||||
assert ctx.state_group_indices.tolist() == [0, 0, 0, 0, 0]
|
||||
assert ctx.state_conv_widths.tolist() == [4, 0, 4, 0, 12]
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# stage_postprocess_inputs_to_gpu: single-pass staging into pinned views
|
||||
# -----------------------------------------------------------------------------
|
||||
@@ -620,7 +796,7 @@ def _run_gpu_postprocess(
|
||||
*,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
forward_context: dict[str, Any],
|
||||
copy_funcs: tuple,
|
||||
copy_funcs: MambaStateCopyFuncsByType,
|
||||
block_table: torch.Tensor,
|
||||
req_ids: list[str],
|
||||
num_accepted_tokens: list[int],
|
||||
|
||||
Reference in New Issue
Block a user