[Model] Support Qwen3.8-Flash-Next (#53896)

Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com>
Signed-off-by: andyluo7 <andy.luo@amd.com>
Signed-off-by: lishunyang12 <lishunyang03@gmail.com>
Signed-off-by: Harry Huang <huanghaoyan.hhy@alibaba-inc.com>
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: KungYork <firelikest@163.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: andyluo7 <43718156+andyluo7@users.noreply.github.com>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: andyluo7 <andy.luo@amd.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: lishunyang12 <lishunyang03@gmail.com>
Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Harry Huang
2026-08-31 05:57:56 +00:00
committed by GitHub
co-authored by zjy0516 KungYork Thien Tran Yifan Qiao andyluo7 Isotr0py andyluo7 OpenAI Codex lishunyang12 Vadim Gimpelson Isotr0py Lucas Wilkinson Claude Opus 5 OpenAI Codex
parent 687db59744
commit e126687a9a
124 changed files with 19424 additions and 450 deletions
+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+200
View File
@@ -0,0 +1,200 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from unittest.mock import patch
import pytest
import torch
from vllm.config.speculative import SpeculativeConfig
from vllm.model_executor.models.config import (
Qwen3_5ForConditionalGenerationConfig,
Qwen4ExpForConditionalGenerationConfig,
)
from vllm.models.qwen4_exp.config import (
Qwen4ExpConfig,
Qwen4ExpTextConfig,
)
from vllm.models.qwen4_exp.nvidia.model_state import Qwen4ExpModelState
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
def _text_config(**kwargs) -> Qwen4ExpTextConfig:
values = {
"vocab_size": 64,
"hidden_size": 16,
"intermediate_size": 32,
"num_hidden_layers": 2,
"num_attention_heads": 2,
"num_key_value_heads": 1,
"head_dim": 8,
"layer_types": ["linear_attention", "full_attention"],
"linear_num_key_heads": 2,
"linear_num_value_heads": 2,
"linear_key_head_dim": 8,
"linear_value_head_dim": 8,
"num_experts": 0,
"hc_count": 2,
"hc_lowrank": 4,
"ple_layer_ids": [1],
"mtp_num_hidden_layers": 1,
"mtp": {"hybrid": True},
}
values.update(kwargs)
return Qwen4ExpTextConfig(**values)
def test_qwen4_exp_mtp_returns_sample_and_multi_streams() -> None:
from vllm.models.qwen4_exp.nvidia.mtp import (
Qwen4ExpMultiTokenPredictor,
)
model = object.__new__(Qwen4ExpMultiTokenPredictor)
torch.nn.Module.__init__(model)
model.hc_count = 2
model.hidden_size = 4
model.num_mtp_layers = 1
model.layers = [
lambda **kwargs: (
kwargs["hidden_states"],
kwargs["hidden_states"],
torch.zeros(kwargs["hidden_states"].shape[0], 2),
),
]
model.hyper_connection_mixer = SimpleNamespace(
combine_and_mix=lambda hidden_states, block_output, injection: (
hidden_states,
hidden_states.unflatten(-1, (2, 4)).mean(-2),
None,
),
)
multi_hidden = torch.arange(16, dtype=torch.float32).reshape(2, 8)
pp_group = SimpleNamespace(is_first_rank=False, is_last_rank=True)
with patch(
"vllm.models.qwen4_exp.nvidia.mtp.get_pp_group",
return_value=pp_group,
):
sample_hidden, returned_multi_hidden = model.forward(
input_ids=None,
positions=torch.arange(2),
intermediate_tensors={"hidden_states": multi_hidden},
)
torch.testing.assert_close(
sample_hidden,
multi_hidden.unflatten(-1, (2, 4)).mean(dim=-2),
)
assert returned_multi_hidden is multi_hidden
@pytest.mark.parametrize("wrapped_config", [False, True])
def test_qwen4_exp_mtp_override_sets_draft_config(
wrapped_config: bool,
) -> None:
text_config = _text_config(
architectures=["Qwen4ExpForCausalLM"],
index_share_for_mtp_iteration=True,
)
config = (
Qwen4ExpConfig(
architectures=["Qwen4ExpForConditionalGeneration"],
text_config=text_config,
)
if wrapped_config
else text_config
)
draft_config = SpeculativeConfig.hf_config_override(config)
assert draft_config.index_share_for_mtp_iteration is True
assert draft_config.model_type == "qwen4_exp_mtp"
assert draft_config.architectures == ["Qwen4ExpMTP"]
assert draft_config.hc_mult == 2
assert draft_config.n_predict == 1
@pytest.mark.parametrize("ple_layer_ids", [[1], []])
def test_qwen4_exp_rejects_pipeline_parallel_only_with_ple(ple_layer_ids) -> None:
"""PLE needs raw input_ids, which non-first pipeline ranks never see. The
rest of the architecture is PP-capable, so the refusal must be conditional
-- and must land before the engine spends time loading weights."""
vllm_config = SimpleNamespace(
model_config=SimpleNamespace(
hf_text_config=_text_config(ple_layer_ids=ple_layer_ids),
multimodal_config=None,
),
parallel_config=SimpleNamespace(
pipeline_parallel_size=2, enable_dbo=False, ubatch_size=1
),
speculative_config=None,
)
with patch.object(
Qwen3_5ForConditionalGenerationConfig, "verify_and_update_config"
):
if ple_layer_ids:
with pytest.raises(NotImplementedError, match="pipeline_parallel_size=1"):
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(
vllm_config
)
else:
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(vllm_config)
def test_qwen4_exp_model_state_prepares_ngram_context() -> None:
model_state = object.__new__(Qwen4ExpModelState)
model_state.uses_ngram_embedding = True
model_state.ngram_context_len = 3
model_state.ngram_eos_token_id = 99
model_state.ngram_context = torch.empty((4, 3), dtype=torch.int32)
model_state.ngram_context_offsets = torch.arange(-3, 0, dtype=torch.int64)
model_state.ple_query_start_loc = torch.empty(5, dtype=torch.int32)
input_batch = SimpleNamespace(
num_reqs=2,
num_reqs_after_padding=3,
idx_mapping=torch.tensor([1, 0]),
query_start_loc=torch.tensor([0, 2, 3, 3], dtype=torch.int32),
)
req_states = SimpleNamespace(
num_computed_tokens=SimpleNamespace(gpu=torch.tensor([3, 1])),
all_token_ids=SimpleNamespace(
gpu=torch.tensor([[1, 2, 3, 4], [20, 21, 22, 23]], dtype=torch.int32)
),
)
with patch.object(MambaHybridModelState, "prepare_inputs", return_value={}):
model_inputs = model_state.prepare_inputs(input_batch, req_states)
torch.testing.assert_close(
model_inputs["query_start_loc"],
torch.tensor([0, 2, 3, 3], dtype=torch.int32),
)
torch.testing.assert_close(
model_inputs["ngram_context"],
torch.tensor([[99, 99, 20], [1, 2, 3], [99, 99, 99]], dtype=torch.int32),
)
def test_qwen4_exp_model_state_prepares_stable_dummy_ngram_inputs() -> None:
model_state = object.__new__(Qwen4ExpModelState)
model_state.uses_ngram_embedding = True
model_state.ngram_eos_token_id = 99
model_state.ngram_context = torch.empty((4, 3), dtype=torch.int32)
model_state.ple_query_start_loc = torch.empty(5, dtype=torch.int32)
with patch.object(MambaHybridModelState, "prepare_dummy_inputs", return_value={}):
first = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=8)
query_start_loc_ptr = first["query_start_loc"].data_ptr()
ngram_context_ptr = first["ngram_context"].data_ptr()
second = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=8)
torch.testing.assert_close(
second["query_start_loc"], torch.tensor([0, 2, 5, 8], dtype=torch.int32)
)
torch.testing.assert_close(
second["ngram_context"], torch.full((3, 3), 99, dtype=torch.int32)
)
assert second["query_start_loc"].data_ptr() == query_start_loc_ptr
assert second["ngram_context"].data_ptr() == ngram_context_ptr
+94
View File
@@ -0,0 +1,94 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.models.qwen4_exp.nvidia.ops.hc import (
grouped_gemma_rmsnorm,
hc_combine,
hc_combine_norm,
hc_gate_mix,
)
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda() or not HAS_TRITON,
reason="HC kernels require CUDA and Triton",
)
HC = 4
HIDDEN_SIZE = 2560
HYPER_HIDDEN_SIZE = HC * HIDDEN_SIZE
EPS = 1e-6
def test_grouped_gemma_rmsnorm() -> None:
torch.manual_seed(0)
x = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
actual = grouped_gemma_rmsnorm(x, weight, EPS, HC)
grouped = x.float().unflatten(-1, (HC, HIDDEN_SIZE))
variance = grouped.square().mean(-1, keepdim=True)
expected = grouped * torch.rsqrt(variance + EPS)
expected = expected.flatten(-2) * (1.0 + weight.float())
torch.testing.assert_close(actual, expected.to(torch.bfloat16))
def test_hc_gate_mix() -> None:
torch.manual_seed(0)
x = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
gate = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
actual = hc_gate_mix(x, gate, HC)
expected = (
torch.sigmoid(gate.float().unflatten(-1, (HC, HIDDEN_SIZE)))
* x.float().unflatten(-1, (HC, HIDDEN_SIZE))
).mean(-2)
torch.testing.assert_close(actual, expected.to(torch.bfloat16))
def test_hc_combine() -> None:
torch.manual_seed(0)
block_output = torch.randn(2, HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
residual = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
injection = torch.randn(2, HC, dtype=torch.bfloat16, device="cuda")
actual = hc_combine(residual, block_output, injection, HC)
injection_weight = 2.0 * torch.sigmoid(injection.float() / HC)
expected = residual.float().unflatten(-1, (HC, HIDDEN_SIZE))
expected = expected + block_output.float().unsqueeze(
-2
) * injection_weight.unsqueeze(-1)
torch.testing.assert_close(actual, expected.flatten(-2).to(torch.bfloat16))
def test_hc_combine_norm() -> None:
torch.manual_seed(0)
block_output = torch.randn(2, HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
residual = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
injection = torch.randn(2, HC, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
actual, actual_norm = hc_combine_norm(
residual, block_output, injection, weight, EPS, HC
)
injection_weight = 2.0 * torch.sigmoid(injection.float() / HC)
expected = residual.float().unflatten(-1, (HC, HIDDEN_SIZE))
expected = expected + block_output.float().unsqueeze(
-2
) * injection_weight.unsqueeze(-1)
expected = expected.flatten(-2).to(residual.dtype)
grouped = expected.float().unflatten(-1, (HC, HIDDEN_SIZE))
variance = grouped.square().mean(-1, keepdim=True)
expected_norm = grouped * torch.rsqrt(variance + EPS)
expected_norm = expected_norm.flatten(-2) * (1.0 + weight.float())
torch.testing.assert_close(actual, expected)
torch.testing.assert_close(actual_norm, expected_norm.to(torch.bfloat16))
+401
View File
@@ -0,0 +1,401 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from functools import partial
from types import SimpleNamespace
import pytest
import torch
from torch import nn
from torch.nn import functional as F
import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module
import vllm.model_executor.parameter as parameter_module
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.models.qwen4_exp.common.ple import (
PLEShardOverlap,
compute_ple_shard_overlap,
copy_ple_embedding_shard_,
)
from vllm.models.qwen4_exp.nvidia import ple_layer as ple_layer_module
from vllm.models.qwen4_exp.nvidia.ple_layer import (
Qwen4ExpNGramEmbedding,
Qwen4ExpPLEFp8EmbeddingMethod,
Qwen4ExpPLELayer,
_get_ple_embedding_quant_method,
)
def _make_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding:
module = Qwen4ExpNGramEmbedding.__new__(Qwen4ExpNGramEmbedding)
nn.Module.__init__(module)
module.split_ngram_parts = 2
module.register_buffer("layer_multipliers", torch.zeros(1, dtype=torch.long))
module.register_buffer("ngram_heads_offsets", torch.zeros(1, dtype=torch.long))
module.register_buffer("ngram_heads_vocab_sizes", torch.zeros(1, dtype=torch.long))
module.ngram_embedding = SimpleNamespace(
org_vocab_size=8,
embedding_dim=2,
weight=nn.Parameter(torch.full((4, 2), -1.0)),
shard_indices=SimpleNamespace(
org_vocab_start_index=2,
org_vocab_end_index=6,
),
)
_set_test_embedding_weight_loader(module.ngram_embedding)
return module
def _set_test_embedding_weight_loader(embedding) -> None:
embedding.weight.weight_loader = partial(
copy_ple_embedding_shard_,
tp_start=embedding.shard_indices.org_vocab_start_index,
tp_end=embedding.shard_indices.org_vocab_end_index,
)
def _make_fp8_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding:
module = _make_ngram_embedding_for_load_test()
embedding = nn.Module()
embedding.org_vocab_size = 8
embedding.embedding_dim = 2
embedding.shard_indices = SimpleNamespace(
org_vocab_start_index=2,
org_vocab_end_index=6,
)
embedding.register_parameter(
"weight",
nn.Parameter(
torch.full((4, 2), -1.0).to(torch.float8_e4m3fn),
requires_grad=False,
),
)
embedding.register_parameter(
"weight_scale",
nn.Parameter(torch.zeros(1, dtype=torch.bfloat16), requires_grad=False),
)
_set_test_embedding_weight_loader(embedding)
module.ngram_embedding = embedding
return module
def test_ple_shard_overlap_and_copy() -> None:
overlap = compute_ple_shard_overlap(
checkpoint_start=2, checkpoint_rows=5, tp_start=4, tp_end=8
)
assert overlap == PLEShardOverlap(source_start=2, destination_start=0, row_count=3)
destination = torch.full((4, 2), -1.0)
loaded = torch.arange(10, dtype=torch.float64).reshape(5, 2)
copied = copy_ple_embedding_shard_(
destination,
loaded,
checkpoint_start=2,
tp_start=4,
tp_end=8,
)
assert copied == 3
torch.testing.assert_close(destination[:3], loaded[2:5].float())
torch.testing.assert_close(destination[3], torch.tensor([-1.0, -1.0]))
def test_ple_shard_copy_is_a_noop_without_overlap() -> None:
destination = torch.ones(4, 2)
copied = copy_ple_embedding_shard_(
destination,
torch.zeros(2, 2),
checkpoint_start=10,
tp_start=4,
tp_end=8,
)
assert copied == 0
assert torch.equal(destination, torch.ones_like(destination))
def test_ngram_embedding_loads_shards_and_ignores_legacy_token_lookup() -> None:
module = _make_ngram_embedding_for_load_test()
shard_0 = torch.arange(8, dtype=torch.float32).reshape(4, 2)
shard_1 = torch.arange(8, 16, dtype=torch.float32).reshape(4, 2)
loaded = module.load_weights(
[
("ngram_embedding.shard_0.weight", shard_0),
("ngram_embedding.shard_1.weight", shard_1),
("token_lookup", torch.tensor([2, 1, 0])),
]
)
assert loaded == {"ngram_embedding.weight"}
torch.testing.assert_close(
module.ngram_embedding.weight,
torch.cat((shard_0[2:4], shard_1[0:2])),
)
def test_ngram_embedding_loads_fp8_shards_and_global_scale() -> None:
module = _make_fp8_ngram_embedding_for_load_test()
shard_0 = torch.arange(8, dtype=torch.float32).reshape(4, 2).to(torch.float8_e4m3fn)
shard_1 = (
torch.arange(8, 16, dtype=torch.float32).reshape(4, 2).to(torch.float8_e4m3fn)
)
weight_scale = torch.tensor([0.25], dtype=torch.bfloat16)
loaded = module.load_weights(
[
("ngram_embedding.shard_0.weight", shard_0),
("ngram_embedding.shard_1.weight", shard_1),
("ngram_embedding.weight_scale", weight_scale),
]
)
assert loaded == {"ngram_embedding.weight", "ngram_embedding.weight_scale"}
assert module.ngram_embedding.weight.dtype == torch.float8_e4m3fn
assert torch.equal(
module.ngram_embedding.weight.float(),
torch.cat((shard_0[2:4], shard_1[0:2])).float(),
)
assert torch.equal(module.ngram_embedding.weight_scale, weight_scale)
def _make_fp8_embedding_layer(
monkeypatch: pytest.MonkeyPatch,
) -> embedding_module.VocabParallelEmbedding:
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
embedding_module, "get_tensor_model_parallel_world_size", lambda: 1
)
monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
parameter_module, "get_tensor_model_parallel_world_size", lambda: 1
)
method = Qwen4ExpPLEFp8EmbeddingMethod()
layer = embedding_module.VocabParallelEmbedding(
3,
2,
params_dtype=torch.bfloat16,
padding_size=1,
quant_method=method,
)
weight = torch.tensor([[1.0, 2.0], [4.0, 8.0], [16.0, 32.0]])
layer.weight.data.copy_(weight.to(torch.float8_e4m3fn))
layer.weight_scale.data.copy_(torch.tensor([0.25], dtype=torch.bfloat16))
return layer
def test_ple_fp8_embedding_dequantizes_in_ple_layer(monkeypatch) -> None:
layer = _make_fp8_embedding_layer(monkeypatch)
quantized_output = layer(torch.tensor([2, 0]))
ple_layer = Qwen4ExpPLELayer.__new__(Qwen4ExpPLELayer)
nn.Module.__init__(ple_layer)
ple_layer.ple_embedding = nn.Module()
ple_layer.ple_embedding.ngram_embedding = layer
output = ple_layer._dequantize_embeddings(
quantized_output,
torch.bfloat16,
)
assert layer.weight.dtype == torch.float8_e4m3fn
assert layer.weight_scale.dtype == torch.bfloat16
assert quantized_output.dtype == torch.float8_e4m3fn
assert output.dtype == torch.bfloat16
weight = torch.tensor([[1.0, 2.0], [4.0, 8.0], [16.0, 32.0]])
torch.testing.assert_close(output, (weight[[2, 0]] * 0.25).bfloat16())
def test_ple_fp8_embedding_uses_int8_for_tp_reduce(monkeypatch) -> None:
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
embedding_module, "get_tensor_model_parallel_world_size", lambda: 2
)
monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
parameter_module, "get_tensor_model_parallel_world_size", lambda: 2
)
monkeypatch.setattr(
embedding_module,
"get_masked_input_and_mask",
lambda *args: (
torch.tensor([0, 0]),
torch.tensor([False, True]),
),
)
reduced_dtypes = []
def all_reduce(tensor: torch.Tensor) -> torch.Tensor:
reduced_dtypes.append(tensor.dtype)
return tensor.clone()
monkeypatch.setattr(
embedding_module,
"tensor_model_parallel_all_reduce",
all_reduce,
)
layer = embedding_module.VocabParallelEmbedding(
4,
2,
params_dtype=torch.bfloat16,
padding_size=1,
quant_method=Qwen4ExpPLEFp8EmbeddingMethod(),
)
layer.weight.data.copy_(
torch.tensor([[1.0, 2.0], [4.0, 8.0]]).to(torch.float8_e4m3fn)
)
output = layer(torch.tensor([0, 2]))
assert reduced_dtypes == [torch.int8]
assert output.dtype == torch.float8_e4m3fn
torch.testing.assert_close(output[0].float(), layer.weight[0].float())
assert torch.count_nonzero(output[1].float()) == 0
def test_ple_fp8_embedding_respects_checkpoint_shard_exclusions() -> None:
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
quant_config = Fp8Config(
is_checkpoint_fp8_serialized=True,
ignored_layers=[],
weight_block_size=[128, 128],
)
assert isinstance(
_get_ple_embedding_quant_method(quant_config, prefix),
Qwen4ExpPLEFp8EmbeddingMethod,
)
quant_config.ignored_layers = [f"{prefix}.shard_0"]
assert _get_ple_embedding_quant_method(quant_config, prefix) is None
def test_ple_ngram_ids_custom_op_uses_current_request_layout(monkeypatch) -> None:
class RuntimeNGramEmbedding(nn.Module):
def compute_ngram_ids(
self,
input_ids: torch.Tensor,
query_start_loc: torch.Tensor,
ngram_context: torch.Tensor,
) -> torch.Tensor:
del input_ids, ngram_context
num_reqs = query_start_loc.numel() - 1
return torch.full((4, 2), num_reqs, dtype=torch.long)
layer = Qwen4ExpPLELayer.__new__(Qwen4ExpPLELayer)
nn.Module.__init__(layer)
layer.ple_embedding = RuntimeNGramEmbedding()
monkeypatch.setattr(
ple_layer_module,
"get_forward_context",
lambda: SimpleNamespace(no_compile_layers={"ple": layer}),
)
input_ids = torch.arange(4)
ngram_context = torch.zeros(2, 2, dtype=torch.long)
output = torch.empty(4, 2, dtype=torch.long)
ple_layer_module.qwen4_exp_compute_ple_ngram_ids(
input_ids,
torch.tensor([0, 4]),
ngram_context,
output,
"ple",
)
assert torch.equal(output, torch.ones_like(output))
ple_layer_module.qwen4_exp_compute_ple_ngram_ids(
input_ids,
torch.tensor([0, 2, 4]),
ngram_context,
output,
"ple",
)
assert torch.equal(output, torch.full_like(output, 2))
def test_dilated_ple_spec_state_rolls_back_before_next_forward() -> None:
module = Qwen4ExpPLELayer.__new__(Qwen4ExpPLELayer)
nn.Module.__init__(module)
module.conv_state_len = 6
module.short_conv_dilation = 2
conv_weights = torch.tensor([[0.25, -0.5, 0.75, 1.0]])
conv_state = torch.zeros(2, 1, 9)
conv_state[1] = torch.arange(1, 10, dtype=torch.float32).reshape(1, 9)
first_inputs = torch.tensor([[10.0], [20.0], [30.0], [40.0]])
initial_state = conv_state[1:].clone()
first_history = torch.cat(
(initial_state[..., : module.conv_state_len], first_inputs.T.unsqueeze(0)),
dim=-1,
)
graph_padded_inputs = F.pad(first_inputs, (0, 0, 0, 4))
first_output = module._short_conv_dilated_spec_batched(
graph_padded_inputs,
conv_state,
conv_weights,
torch.tensor([1, 0]),
torch.tensor([0, 4, 4]),
torch.tensor([1, 0]),
spec_query_len=4,
)
expected_first_output = F.silu(
F.conv1d(
first_history,
conv_weights.unsqueeze(1),
groups=1,
dilation=module.short_conv_dilation,
)
).transpose(1, 2)[0]
expected_first_state = first_history[..., 1:10]
torch.testing.assert_close(first_output[:4], expected_first_output)
assert torch.count_nonzero(first_output[4:]) == 0
assert torch.count_nonzero(conv_state[0]) == 0
torch.testing.assert_close(conv_state[1:], expected_first_state)
second_inputs = torch.tensor([[50.0], [60.0]])
rollback_state = expected_first_state[..., 1:7]
padded_second_inputs = F.pad(second_inputs.T.unsqueeze(0), (0, 2))
second_history = torch.cat((rollback_state, padded_second_inputs), dim=-1)
expected_second_state = expected_first_state.clone()
expected_second_state[..., :7] = second_history[..., 1:8]
second_output = module._short_conv_dilated_spec_batched(
second_inputs,
conv_state,
conv_weights,
torch.tensor([1]),
torch.tensor([0, 2]),
torch.tensor([2]),
spec_query_len=4,
)
expected_second_output = F.silu(
F.conv1d(
second_history,
conv_weights.unsqueeze(1),
groups=1,
dilation=module.short_conv_dilation,
)
).transpose(1, 2)[0, :2]
torch.testing.assert_close(second_output, expected_second_output)
torch.testing.assert_close(conv_state[1:], expected_second_state)
def test_ple_short_conv_uses_fallback_when_profile_metadata_is_omitted(
monkeypatch: pytest.MonkeyPatch,
) -> None:
module = Qwen4ExpPLELayer.__new__(Qwen4ExpPLELayer)
nn.Module.__init__(module)
module.prefix = "model.layers.1.ple"
inputs = torch.arange(6, dtype=torch.float32).reshape(3, 2)
expected = inputs + 1
monkeypatch.setattr(module, "_short_conv_fallback", lambda _: expected)
monkeypatch.setattr(
ple_layer_module,
"get_forward_context",
lambda: SimpleNamespace(attn_metadata={"model.layers.0.self_attn": object()}),
)
output = module._short_conv(inputs)
assert output is expected
+297
View File
@@ -0,0 +1,297 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from types import SimpleNamespace
from typing import Any
import pytest
import torch
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
from vllm.models.qwen4_exp.amd import (
model as _qwen4_exp_model, # noqa: F401
)
from vllm.models.qwen4_exp.amd import ple_layer as ple_layer_module
from vllm.models.qwen4_exp.amd.indexer_qsa import (
apply_qsa_rmsnorm,
apply_qsa_rope,
)
from vllm.models.qwen4_exp.amd.ops import qsa as qsa_ops
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(),
reason="AMD QSA requires ROCm",
)
requires_qsa_kernels = pytest.mark.skipif(
not HAS_TRITON,
reason="AMD QSA kernels require Triton",
)
def test_ple_ngram_embedding_custom_op_uses_resident_weight(
monkeypatch: pytest.MonkeyPatch,
) -> None:
layer_name = "model.layers.0.ple"
layer = ple_layer_module.Qwen4ExpPLELayer.__new__(ple_layer_module.Qwen4ExpPLELayer)
torch.nn.Module.__init__(layer)
layer.ple_embedding = torch.nn.Module()
layer.ple_embedding.ngram_embedding = torch.nn.Embedding(8, 3)
context = SimpleNamespace(no_compile_layers={layer_name: layer})
monkeypatch.setattr(ple_layer_module, "get_forward_context", lambda: context)
ngram_ids = torch.tensor([[0, 1], [2, 3]])
output = torch.empty(2, 6)
ple_layer_module.qwen4_exp_amd_ple_ngram_embedding(
ngram_ids,
output,
layer_name,
)
expected = layer.ple_embedding.ngram_embedding(ngram_ids).flatten(-2)
torch.testing.assert_close(output, expected)
def _qsa_sparse_paged_attention_reference(
q: torch.Tensor,
k_cache: torch.Tensor,
v_cache: torch.Tensor,
logical_indices: torch.Tensor,
block_table: torch.Tensor,
token_to_req: torch.Tensor,
softmax_scale: float,
) -> torch.Tensor:
output = torch.zeros_like(q)
repeats = q.shape[1] // k_cache.shape[2]
page_size = k_cache.shape[1]
for row in range(q.shape[0]):
logical = logical_indices[row]
logical = logical[logical >= 0].long()
if not logical.numel():
continue
request = token_to_req[row].long()
pages = block_table[request, logical // page_size].long()
offsets = logical % page_size
keys = k_cache[pages, offsets].repeat_interleave(repeats, dim=1)
values = v_cache[pages, offsets].repeat_interleave(repeats, dim=1)
scores = torch.einsum("hd,khd->hk", q[row].float(), keys.float())
probabilities = torch.softmax(scores * softmax_scale, dim=-1)
output[row] = torch.einsum("hk,khd->hd", probabilities, values.float()).to(
q.dtype
)
return output
def test_qsa_rope_uses_platform_dispatch() -> None:
tensor = torch.arange(16, dtype=torch.float32).reshape(2, 2, 4)
positions = torch.tensor([0, 1])
calls = []
def apply_rotary_emb(
rotary_input: torch.Tensor,
cos: torch.Tensor,
sin: torch.Tensor,
) -> torch.Tensor:
calls.append((rotary_input, cos, sin))
return rotary_input + 1
rotary_emb = SimpleNamespace(
rotary_dim=2,
apply_rotary_emb=apply_rotary_emb,
_match_cos_sin_cache_dtype=lambda _: torch.zeros(2, 4),
)
output = apply_qsa_rope(rotary_emb, positions, tensor)
assert len(calls) == 1
torch.testing.assert_close(output[..., :2], tensor[..., :2] + 1)
torch.testing.assert_close(output[..., 2:], tensor[..., 2:])
def test_qsa_rmsnorm_uses_portable_implementation(default_vllm_config) -> None:
norm = GemmaRMSNorm(4, eps=1e-6)
norm.weight.data.copy_(torch.tensor([0.1, -0.2, 0.3, -0.4]))
tensor = torch.arange(8, dtype=torch.float32).reshape(2, 4)
output = apply_qsa_rmsnorm(norm, tensor)
torch.testing.assert_close(output, norm.forward_native(tensor))
def test_qsa_selection_uses_portable_topk_on_rocm(
monkeypatch: pytest.MonkeyPatch,
) -> None:
rows = 2
token_topk = 8
compress_ratio = 4
block_topk = token_topk // compress_ratio
blocks = torch.empty((rows, block_topk), dtype=torch.int32)
visible_blocks = torch.tensor([4, 6], dtype=torch.int32)
logits = torch.empty((rows, 8), dtype=torch.float32)
selection_output = torch.empty(
(rows, token_topk + compress_ratio - 1), dtype=torch.int32
)
topk_call: dict[str, Any] = {}
monkeypatch.setattr(
qsa_ops,
"qsa_mqa_paged",
lambda *args, **kwargs: (logits, visible_blocks),
)
monkeypatch.setattr(qsa_ops.current_platform, "is_cuda", lambda: False)
def top_k_per_row_decode(
topk_logits,
next_n,
seq_lens,
raw_topk_indices,
num_rows,
stride0,
stride1,
topk_tokens,
) -> None:
topk_call.update(
logits=topk_logits,
next_n=next_n,
seq_lens=seq_lens,
raw_topk_indices=raw_topk_indices,
num_rows=num_rows,
strides=(stride0, stride1),
topk_tokens=topk_tokens,
)
raw_topk_indices.zero_()
def expand_qsa_block_indices(*args) -> None:
args[-1].fill_(-1)
monkeypatch.setattr(qsa_ops.ops, "top_k_per_row_decode", top_k_per_row_decode)
monkeypatch.setattr(
qsa_ops,
"expand_qsa_block_indices_cuda",
expand_qsa_block_indices,
)
output = qsa_ops.qsa_select_paged_tokens(
torch.empty((rows, 1, 1)),
torch.empty((1, 4, 1, 1)),
torch.empty((1, 2), dtype=torch.int32),
torch.zeros(rows, dtype=torch.int32),
torch.arange(rows, dtype=torch.int32),
torch.tensor([8], dtype=torch.int32),
token_topk,
compress_ratio,
selection_output,
)
assert output is selection_output
assert torch.all(output == -1)
assert topk_call["logits"] is logits
assert topk_call["next_n"] == 1
assert topk_call["seq_lens"] is visible_blocks
assert topk_call["raw_topk_indices"].shape == blocks.shape
assert topk_call["num_rows"] == rows
assert topk_call["strides"] == (logits.stride(0), logits.stride(1))
assert topk_call["topk_tokens"] == block_topk
@requires_qsa_kernels
@pytest.mark.parametrize(
("num_rows", "num_query_heads", "num_kv_heads", "page_size"),
[
pytest.param(1, 24, 2, 1792, id="tp1_split64"),
pytest.param(16, 12, 1, 1792, id="tp2_split32"),
pytest.param(32, 6, 1, 1024, id="tp4_split8"),
pytest.param(257, 6, 1, 1024, id="tp4_split4"),
pytest.param(513, 6, 1, 1024, id="tp4_split1"),
],
)
def test_qsa_sparse_paged_attention_matches_reference(
num_rows: int,
num_query_heads: int,
num_kv_heads: int,
page_size: int,
) -> None:
torch.manual_seed(2)
head_dim = 256
num_requests = 2
num_selected_pages = 64
num_pages_per_request = num_selected_pages + 1
num_cache_blocks = num_requests * num_pages_per_request
indexer_budget = 2048
indexer_compress_ratio = 4
selection_width = indexer_budget + indexer_compress_ratio - 1
q = torch.randn(
num_rows, num_query_heads, head_dim, device="cuda", dtype=torch.bfloat16
)
kv_cache = torch.randn(
num_cache_blocks,
page_size,
num_kv_heads,
2 * head_dim,
device="cuda",
dtype=torch.bfloat16,
)
k_cache, v_cache = kv_cache.split(head_dim, dim=-1)
block_table = (
torch.randperm(num_cache_blocks, device="cuda")
.reshape(num_requests, num_pages_per_request)
.to(torch.int32)
)
rows_per_request = math.ceil(num_rows / num_requests)
row_indices = torch.arange(num_rows, device="cuda", dtype=torch.int32)
token_to_req = row_indices // rows_per_request
request_row_counts = torch.tensor(
[rows_per_request, num_rows - rows_per_request],
device="cuda",
dtype=torch.int32,
)
context_length = num_pages_per_request * page_size - 1
block_topk = indexer_budget // indexer_compress_ratio
compressed_blocks_per_page = page_size // indexer_compress_ratio
selection = torch.arange(block_topk, device="cuda")
selected_pages = selection % num_selected_pages
selected_offsets = selection // num_selected_pages
row_shifts = 2 * row_indices.unsqueeze(1)
selected_offsets = (selected_offsets + row_shifts) % compressed_blocks_per_page
block_indices = (selected_pages * compressed_blocks_per_page + selected_offsets).to(
torch.int32
)
rows_within_request = row_indices % rows_per_request
query_positions = (
context_length - request_row_counts[token_to_req.long()] + rows_within_request
).to(torch.int64)
sequence_lengths = torch.full(
(num_requests,), context_length, device="cuda", dtype=torch.int32
)
logical_indices = qsa_ops.expand_qsa_block_indices_cuda(
block_indices,
query_positions,
sequence_lengths,
token_to_req,
indexer_compress_ratio,
indexer_budget,
)
assert logical_indices.shape == (num_rows, selection_width)
actual = qsa_ops.qsa_sparse_paged_attention(
q,
k_cache,
v_cache,
logical_indices,
block_table,
token_to_req,
)
expected = _qsa_sparse_paged_attention_reference(
q,
k_cache,
v_cache,
logical_indices,
block_table,
token_to_req,
q.shape[-1] ** -0.5,
)
torch.testing.assert_close(actual, expected, rtol=2e-2, atol=2e-2)
@@ -0,0 +1,297 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness tests for the fused QSA pre-indexer."""
import pytest
import torch
from vllm.models.qwen4_exp.common.qsa_cache import (
canonical_qsa_rope_positions,
circular_qsa_slot_mapping,
compressed_qsa_slot_mapping,
)
from vllm.models.qwen4_exp.nvidia.indexer_qsa import apply_qsa_rope
from vllm.models.qwen4_exp.nvidia.ops.qsa import (
qsa_compress_groups_with_ratio,
qsa_store_cache_rows,
)
from vllm.models.qwen4_exp.nvidia.ops.qsa_pre_indexer import (
qsa_pre_indexer,
)
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
requires_qsa_kernels = pytest.mark.skipif(
not current_platform.is_cuda() or not HAS_TRITON,
reason="QSA kernels require CUDA and Triton",
)
HQ, D = 4, 128
CR = 4
MROPE_SECTION = (11, 11, 10)
EPS = 1e-6
BLOCK_SIZE = 16
COMP_PAGE = BLOCK_SIZE // CR
ROPE_POS_OFFSET = D
RTOL = 1.6e-2
ATOL = 1e-2
MIXED_BATCH = ([260, 259, 138], [1, 1, 37], [8, 8, 8])
def _make_block_table(block_counts):
num_blocks = sum(block_counts)
table = torch.full((len(block_counts), max(block_counts)), -1, dtype=torch.int32)
physical_blocks = torch.randperm(num_blocks)
offset = 0
for request, count in enumerate(block_counts):
table[request, :count] = physical_blocks[offset : offset + count]
offset += count
return table, num_blocks
@requires_qsa_kernels
@pytest.mark.usefixtures("default_vllm_config")
@pytest.mark.parametrize(
"mrope,is_2d_positions,cache_rope_positions,state_size,seq_lens,query_lens,history_lens",
[
pytest.param(True, True, True, 4, *MIXED_BATCH, id="mrope"),
pytest.param(False, False, False, 4, *MIXED_BATCH, id="text"),
pytest.param(True, False, True, 8, *MIXED_BATCH, id="mrope-model-1d"),
pytest.param(True, True, False, 4, *MIXED_BATCH, id="mrope-no-position-cache"),
pytest.param(
True,
False,
False,
8,
*MIXED_BATCH,
id="mrope-model-1d-no-position-cache",
),
pytest.param(
False, False, True, 4, *MIXED_BATCH, id="text-with-position-cache"
),
pytest.param(True, True, True, 4, [37], [37], [0], id="fresh"),
pytest.param(True, True, True, 4, [4097], [4097], [0], id="tiled"),
],
)
def test_qsa_fused_pre_indexer_matches_unfused(
mrope,
is_2d_positions,
cache_rope_positions,
state_size,
seq_lens,
query_lens,
history_lens,
) -> None:
from flashinfer.norm import gemma_rmsnorm
from vllm.model_executor.layers.rotary_embedding import get_rope
device = "cuda"
rope_params = {
"partial_rotary_factor": 0.25,
"rope_theta": 10000000,
"rope_type": "default",
}
if mrope:
rope_params["mrope_interleaved"] = True
rope_params["mrope_section"] = list(MROPE_SECTION)
with torch.device(device):
rope = get_rope(
head_size=256,
max_position=32768,
rope_parameters=rope_params,
dtype=torch.bfloat16,
)
token_to_req = torch.cat(
[
torch.full((length,), request, dtype=torch.int32)
for request, length in enumerate(query_lens)
]
).to(device)
logical_positions = torch.cat(
[
torch.arange(seq_len - query_len, seq_len, dtype=torch.int64)
for seq_len, query_len in zip(seq_lens, query_lens)
]
).to(device)
num_tokens = logical_positions.numel()
query_start_loc = torch.tensor(
[0, *torch.tensor(query_lens).cumsum(0).tolist()], dtype=torch.int32
).to(device)
positions = (
torch.stack(
[
logical_positions,
logical_positions // 7 + 3,
logical_positions // 13 + 11,
]
)
if is_2d_positions
else logical_positions
)
position_rows = (
canonical_qsa_rope_positions(positions)
if cache_rope_positions
else logical_positions.view(-1, 1, 1).expand(-1, 1, 3)
)
compressed_block_counts = [
(seq_len // CR + COMP_PAGE - 1) // COMP_PAGE for seq_len in seq_lens
]
raw_block_table, num_raw_blocks = _make_block_table([1] * len(seq_lens))
compressed_block_table, num_compressed_blocks = _make_block_table(
compressed_block_counts
)
raw_block_table = raw_block_table.to(device)
raw_slots = circular_qsa_slot_mapping(
raw_block_table,
token_to_req,
logical_positions,
state_size,
query_start_loc,
)
compressed_slots = compressed_qsa_slot_mapping(
compressed_block_table.to(device),
token_to_req,
logical_positions,
COMP_PAGE,
CR,
)
group_counts = torch.tensor(
[
seq_len // CR - (seq_len - query_len) // CR
for seq_len, query_len in zip(seq_lens, query_lens)
],
dtype=torch.int32,
device=device,
)
k_work_counts = torch.maximum(group_counts, torch.ones_like(group_counts))
k_start_loc = torch.cat([k_work_counts.new_zeros(1), k_work_counts.cumsum(0)])
work_requests = torch.repeat_interleave(
torch.arange(len(query_lens), dtype=torch.int32, device=device),
k_work_counts,
)
local_work = torch.arange(
int(k_start_loc[-1]), dtype=torch.int32, device=device
) - torch.repeat_interleave(k_start_loc[:-1], k_work_counts)
max_k_work = (num_tokens + (CR - 1) * len(query_lens)) // CR
k_work_metadata = torch.full((max_k_work, 2), -1, dtype=torch.int32, device=device)
k_work_metadata[: work_requests.numel()] = torch.stack(
(work_requests, local_work), dim=1
)
raw_width = D + 12 if cache_rope_positions else D
# Match vLLM's padded-page cache layout: rows are contiguous, while physical
# blocks have a larger stride than their logical contents.
raw_page_elements = state_size * raw_width
fused_raw_storage = torch.zeros(
num_raw_blocks,
raw_page_elements + 16,
dtype=torch.bfloat16,
device=device,
)
fused_raw = torch.as_strided(
fused_raw_storage,
(num_raw_blocks, state_size, 1, raw_width),
(raw_page_elements + 16, raw_width, raw_width, 1),
)
for request, history_len in enumerate(history_lens):
history_end = seq_lens[request] - query_lens[request]
for position in range(history_end - history_len, history_end):
block = int(raw_block_table[request, 0])
row = fused_raw[block, position % state_size, 0]
row[:D] = torch.randn(D, dtype=torch.bfloat16, device=device)
if cache_rope_positions:
row[ROPE_POS_OFFSET:].view(torch.int64).copy_(
torch.tensor(
[position, position // 7 + 3, position // 13 + 11],
dtype=torch.int64,
device=device,
)
)
unfused_raw = fused_raw.clone()
compressed_page_elements = COMP_PAGE * D
fused_compressed_storage = torch.zeros(
num_compressed_blocks,
compressed_page_elements + 16,
dtype=torch.bfloat16,
device=device,
)
fused_compressed = torch.as_strided(
fused_compressed_storage,
(num_compressed_blocks, COMP_PAGE, 1, D),
(compressed_page_elements + 16, D, D, 1),
)
unfused_compressed = fused_compressed.clone()
projected_qk = torch.randn(
num_tokens, (HQ + 1) * D, dtype=torch.bfloat16, device=device
)
q_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
k_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
fused_query = torch.empty(num_tokens, HQ, D, dtype=torch.bfloat16, device=device)
qsa_pre_indexer(
projected_qk[:, : HQ * D],
projected_qk[:, HQ * D :],
positions,
rope.cos_sin_cache,
q_weight,
k_weight,
EPS,
fused_query,
fused_raw,
raw_slots,
raw_block_table,
query_start_loc,
logical_positions,
fused_compressed,
compressed_slots,
k_work_metadata,
compress_ratio=CR,
mrope_section=MROPE_SECTION if mrope else None,
rope_pos_offset=ROPE_POS_OFFSET if cache_rope_positions else None,
)
unfused_query = projected_qk[:, : HQ * D].reshape(num_tokens, HQ, D)
unfused_query = gemma_rmsnorm(
unfused_query.reshape(-1, D), q_weight, EPS
).reshape_as(unfused_query)
unfused_query = apply_qsa_rope(rope, positions, unfused_query)
raw_keys = unfused_raw[..., :D]
rope_positions = (
unfused_raw[..., ROPE_POS_OFFSET:].view(torch.int64)
if cache_rope_positions
else None
)
pooled, first_positions = qsa_compress_groups_with_ratio(
projected_qk[:, HQ * D :].reshape(-1, 1, D),
position_rows,
raw_keys,
raw_block_table,
token_to_req,
query_start_loc,
logical_positions,
compressed_slots,
CR,
rope_positions,
)
compressed_rows = gemma_rmsnorm(pooled.reshape(-1, D), k_weight, EPS).reshape(
-1, 1, D
)
group_positions = (
first_positions.transpose(0, 1) if mrope else first_positions[:, 0]
)
compressed_rows = apply_qsa_rope(rope, group_positions, compressed_rows)
qsa_store_cache_rows(unfused_compressed, compressed_slots, compressed_rows)
qsa_store_cache_rows(raw_keys, raw_slots, projected_qk[:, HQ * D :])
if rope_positions is not None:
qsa_store_cache_rows(rope_positions, raw_slots, position_rows)
torch.testing.assert_close(fused_query, unfused_query, rtol=RTOL, atol=ATOL)
assert torch.equal(fused_raw.view(torch.int16), unfused_raw.view(torch.int16))
torch.testing.assert_close(
fused_compressed, unfused_compressed, rtol=RTOL, atol=ATOL
)
@@ -0,0 +1,973 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from types import SimpleNamespace
import pytest
import torch
from vllm.models.qwen4_exp.common import qsa_cache
from vllm.models.qwen4_exp.common.qsa_cache import QSAMetadataBuilder
from vllm.models.qwen4_exp.nvidia import indexer_qsa
from vllm.models.qwen4_exp.nvidia import (
model as _qwen4_exp_model, # noqa: F401
)
from vllm.models.qwen4_exp.nvidia.ops import qsa as qsa_ops
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
requires_qsa_kernels = pytest.mark.skipif(
not current_platform.is_cuda() or not HAS_TRITON,
reason="QSA kernels require CUDA and Triton",
)
def test_qsa_mtp_index_share_updates_cache_but_skips_selection(
monkeypatch: pytest.MonkeyPatch,
) -> None:
rows = torch.tensor([[3, 1, -1], [5, 2, 0]], dtype=torch.int32)
raw_metadata = SimpleNamespace(
num_actual_tokens=2,
slot_mapping=torch.arange(2),
block_table=torch.empty(0),
query_start_loc=torch.arange(3),
logical_positions=torch.arange(2),
)
compressed_metadata = SimpleNamespace(
num_actual_tokens=2,
slot_mapping=torch.arange(2),
k_work_metadata=torch.empty(0),
)
updates = []
selections = []
indexer = SimpleNamespace(
skip_topk=True,
_metadata=lambda: (raw_metadata, compressed_metadata),
index_qk_proj=lambda hidden: (torch.zeros(2, 2), None),
index_n_heads=1,
index_kv_heads=1,
index_head_dim=1,
raw_key_cache=SimpleNamespace(
kv_cache=torch.empty(0),
rope_position_cache=None,
rope_position_offset=0,
),
compressed_key_cache=SimpleNamespace(kv_cache=torch.empty(0)),
use_fused_pre_indexer=True,
rotary_emb=SimpleNamespace(cos_sin_cache=torch.empty(0)),
q_layernorm=SimpleNamespace(weight=torch.ones(1), variance_epsilon=1e-6),
k_layernorm=SimpleNamespace(weight=torch.ones(1)),
compress_ratio=2,
)
monkeypatch.setattr(
indexer_qsa,
"qsa_pre_indexer",
lambda *args, **kwargs: updates.append((args, kwargs)),
)
monkeypatch.setattr(
qsa_ops,
"qsa_select_paged_tokens",
lambda *args, **kwargs: selections.append((args, kwargs)),
)
actual = indexer_qsa.QSAIndexer.forward(
indexer,
torch.zeros(2, 4),
torch.tensor([7, 8]),
rows,
)
assert actual is rows
assert len(updates) == 1
assert not selections
def _qsa_mqa_paged_reference(
q: torch.Tensor,
k_cache: torch.Tensor,
page_table: torch.Tensor,
token_to_req: torch.Tensor,
visible_lengths: torch.Tensor,
) -> torch.Tensor:
pages = page_table.index_select(0, token_to_req.long()).long()
keys = k_cache[pages, :, 0, :].flatten(1, 2)
scores = torch.einsum("rhd,rnd->rnh", q.float(), keys.float())
logits = torch.relu(scores).sum(dim=-1) / math.sqrt(q.shape[-1])
positions = torch.arange(keys.shape[1], device=q.device).unsqueeze(0)
return logits.masked_fill(positions >= visible_lengths.unsqueeze(1), -torch.inf)
def _qsa_relative_topk_reference(
logits: torch.Tensor,
row_starts: torch.Tensor,
row_ends: torch.Tensor,
topk: int,
) -> torch.Tensor:
output = torch.full(
(logits.shape[0], topk), -1, dtype=torch.int32, device=logits.device
)
for row in range(logits.shape[0]):
start = int(row_starts[row].item())
length = int((row_ends[row] - row_starts[row]).item())
width = min(length, topk)
if width:
output[row, :width] = torch.topk(
logits[row, start : start + length], width
).indices.to(torch.int32)
return output
def _expand_qsa_indices_reference(
block_indices: torch.Tensor,
query_positions: torch.Tensor,
sequence_lengths: torch.Tensor,
compress_ratio: int,
token_topk: int,
) -> torch.Tensor:
rows = block_indices.shape[0]
block_topk = token_topk // compress_ratio
output_width = token_topk + compress_ratio - 1
offsets = torch.arange(compress_ratio, device=block_indices.device)
blocks = block_indices.long()
expanded = blocks.unsqueeze(-1) * compress_ratio + offsets
expanded = torch.where(
blocks.unsqueeze(-1) >= 0, expanded, torch.full_like(expanded, -1)
).reshape(rows, block_topk * compress_ratio)
expanded = expanded[:, :token_topk]
expanded = torch.where(
(expanded >= 0) & (expanded < sequence_lengths.unsqueeze(1)),
expanded,
torch.full_like(expanded, -1),
)
tail_offsets = torch.arange(compress_ratio - 1, device=block_indices.device)
visible_tokens = query_positions + 1
tail_start = visible_tokens // compress_ratio * compress_ratio
tail = tail_start.unsqueeze(1) + tail_offsets.unsqueeze(0)
tail_count = (visible_tokens - tail_start).unsqueeze(1)
tail_valid = (tail_offsets.unsqueeze(0) < tail_count) & (
tail < sequence_lengths.unsqueeze(1)
)
tail = torch.where(tail_valid, tail, torch.full_like(tail, -1))
result = torch.cat((expanded, tail), dim=1)
order = torch.arange(output_width, device=result.device).expand(rows, -1)
sort_key = torch.where(result >= 0, order, order + output_width)
return result.gather(1, torch.argsort(sort_key, dim=1, stable=True)).to(torch.int32)
def _qsa_select_paged_tokens_reference(
q: torch.Tensor,
k_cache: torch.Tensor,
page_table: torch.Tensor,
token_to_req: torch.Tensor,
query_positions: torch.Tensor,
sequence_lengths: torch.Tensor,
token_topk: int,
compress_ratio: int,
) -> torch.Tensor:
row_sequence_lengths = sequence_lengths.index_select(0, token_to_req.long())
visible_blocks = torch.minimum(
(query_positions + 1) // compress_ratio,
row_sequence_lengths // compress_ratio,
).to(torch.int32)
logits = _qsa_mqa_paged_reference(
q,
k_cache,
page_table,
token_to_req,
visible_blocks,
)
starts = torch.zeros_like(visible_blocks)
blocks = _qsa_relative_topk_reference(
logits,
starts,
visible_blocks,
token_topk // compress_ratio,
)
return _expand_qsa_indices_reference(
blocks,
query_positions,
row_sequence_lengths,
compress_ratio,
token_topk,
)
def _qsa_sparse_paged_attention_reference(
q: torch.Tensor,
k_cache: torch.Tensor,
v_cache: torch.Tensor,
logical_indices: torch.Tensor,
block_table: torch.Tensor,
token_to_req: torch.Tensor,
softmax_scale: float,
) -> torch.Tensor:
output = torch.zeros_like(q)
repeats = q.shape[1] // k_cache.shape[2]
page_size = k_cache.shape[1]
for row in range(q.shape[0]):
logical = logical_indices[row]
logical = logical[logical >= 0].long()
if not logical.numel():
continue
request = token_to_req[row].long()
pages = block_table[request, logical // page_size].long()
offsets = logical % page_size
keys = k_cache[pages, offsets].repeat_interleave(repeats, dim=1)
values = v_cache[pages, offsets].repeat_interleave(repeats, dim=1)
scores = torch.einsum("hd,khd->hk", q[row].float(), keys.float())
probabilities = torch.softmax(scores * softmax_scale, dim=-1)
output[row] = torch.einsum("hk,khd->hd", probabilities, values.float()).to(
q.dtype
)
return output
@requires_qsa_kernels
def test_qsa_side_metadata_marks_cudagraph_padding_inert() -> None:
device = torch.device("cuda")
builder = QSAMetadataBuilder.__new__(QSAMetadataBuilder)
builder.compress_ratio = 1
builder.is_circular_buffer = False
builder.storage_block_size = 64
builder.token_to_req_buffer = torch.empty(16, dtype=torch.int32, device=device)
builder.slot_mapping_buffer = torch.empty(16, dtype=torch.int64, device=device)
builder.logical_positions_buffer = torch.empty(16, dtype=torch.int64, device=device)
builder.k_work_metadata_buffer = torch.empty(0, 2, dtype=torch.int32, device=device)
query_start_loc = torch.tensor([0, 4, 8, 12, 12], dtype=torch.int32, device=device)
token_to_req = torch.tensor([0] * 4 + [1] * 4 + [2] * 4 + [0] * 4, device=device)
common = SimpleNamespace(
num_actual_tokens=16,
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc.cpu(),
seq_lens=torch.tensor([68, 68, 68, 0], dtype=torch.int32, device=device),
slot_mapping=torch.tensor(list(range(12)) + [-1] * 4, device=device),
block_table_tensor=torch.empty((4, 0), dtype=torch.int32, device=device),
token_to_req_indices=lambda buffer: buffer.copy_(token_to_req),
)
metadata = builder.build(0, common)
assert metadata.logical_positions.tolist() == [
64,
65,
66,
67,
64,
65,
66,
67,
64,
65,
66,
67,
-1,
-1,
-1,
-1,
]
assert metadata.slot_mapping.tolist() == list(range(12)) + [-1] * 4
@requires_qsa_kernels
def test_qsa_circular_buffer_metadata_keeps_only_each_requests_suffix() -> None:
device = torch.device("cuda")
builder = QSAMetadataBuilder.__new__(QSAMetadataBuilder)
builder.compress_ratio = 4
builder.is_circular_buffer = True
builder.kv_cache_spec = SimpleNamespace(block_size=4)
builder.storage_block_size = 4
builder.token_to_req_buffer = torch.empty(16, dtype=torch.int32, device=device)
builder.slot_mapping_buffer = torch.empty(16, dtype=torch.int64, device=device)
builder.logical_positions_buffer = torch.empty(16, dtype=torch.int64, device=device)
builder.k_work_metadata_buffer = torch.empty(0, 2, dtype=torch.int32, device=device)
query_start_loc = torch.tensor([0, 7, 13, 13], dtype=torch.int32, device=device)
token_to_req = torch.tensor([0] * 7 + [1] * 6 + [0] * 3, device=device)
block_table = torch.tensor([[1], [0], [2]], dtype=torch.int32, device=device)
common = SimpleNamespace(
num_actual_tokens=16,
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc.cpu(),
seq_lens=torch.tensor([9, 11, 0], dtype=torch.int32, device=device),
slot_mapping=torch.full((16,), -1, dtype=torch.int64, device=device),
block_table_tensor=block_table,
token_to_req_indices=lambda buffer: buffer.copy_(token_to_req),
)
metadata = builder.build(0, common)
expected = [
-1,
-1,
-1,
5,
6,
7,
4,
-1,
-1,
3,
0,
1,
2,
-1,
-1,
-1,
]
assert metadata.slot_mapping.tolist() == expected
@pytest.mark.parametrize("chunk_start", list(range(8)))
def test_qsa_circular_buffer_survives_one_speculative_step(chunk_start: int) -> None:
"""A speculative step must not overwrite the open group's committed keys.
The step stores every row it computes, drafts included, before acceptance
is known, while the next step still reads the earlier members of the group
being compressed from the ring. A ring sized at the compression ratio makes
those rows alias, so a rejected draft silently replaces a committed key.
"""
compress_ratio = 4
num_spec = 3
capacity = compress_ratio * -(-(compress_ratio + num_spec) // compress_ratio)
query_len = num_spec + 1
slots = qsa_cache.circular_qsa_slot_mapping(
torch.tensor([[0]], dtype=torch.int32),
torch.zeros(query_len, dtype=torch.int32),
torch.arange(chunk_start, chunk_start + query_len),
capacity,
query_start_loc=torch.tensor([0, query_len], dtype=torch.int32),
)
committed = torch.arange(chunk_start - chunk_start % compress_ratio, chunk_start)
assert set(slots.tolist()).isdisjoint((committed % capacity).tolist())
def _qsa_key_cache(block_size: int, compress_ratio: int) -> qsa_cache.QSAKeyStateCache:
return qsa_cache.QSAKeyStateCache(
head_size=64,
dtype=torch.bfloat16,
cache_config=SimpleNamespace(block_size=block_size),
prefix=f"raw.{block_size}.{compress_ratio}",
vllm_config=SimpleNamespace(
compilation_config=SimpleNamespace(static_forward_context={})
),
compress_ratio=compress_ratio,
)
def test_qsa_state_caches_adapt_the_unified_logical_layout() -> None:
raw_cache = _qsa_key_cache(block_size=32, compress_ratio=4)
compressed_cache = qsa_cache.QSACompressedKeyCache(
head_size=64,
dtype=torch.bfloat16,
cache_config=SimpleNamespace(block_size=32),
prefix="compressed.bind",
vllm_config=SimpleNamespace(
compilation_config=SimpleNamespace(static_forward_context={})
),
compress_ratio=4,
)
raw_view = torch.empty(2, 1, 8, 64, dtype=torch.bfloat16)
compressed_view = torch.empty(2, 1, 8, 64, dtype=torch.bfloat16)
raw_cache.bind_kv_cache(raw_view)
compressed_cache.bind_kv_cache(compressed_view)
assert raw_cache.kv_cache.shape == (2, 8, 1, 64)
assert compressed_cache.kv_cache.shape == (2, 8, 1, 64)
assert raw_cache.kv_cache.data_ptr() == raw_view.data_ptr()
assert compressed_cache.kv_cache.data_ptr() == compressed_view.data_ptr()
@pytest.mark.parametrize(
("compress_ratio", "num_spec", "expected"),
[(4, 0, 4), (4, 1, 8), (4, 3, 8), (4, 4, 8), (4, 5, 12), (2, 3, 6)],
)
def test_qsa_ring_capacity_covers_one_speculative_step(
compress_ratio: int, num_spec: int, expected: int
) -> None:
"""Capacity spans the open group plus one speculative step, in whole groups."""
spec = _qsa_key_cache(
block_size=48, compress_ratio=compress_ratio
).get_kv_cache_spec(SimpleNamespace(num_speculative_tokens=num_spec))
assert spec.block_size == expected
@requires_qsa_kernels
def test_qsa_compressed_metadata_keeps_dummy_slots_inert() -> None:
device = torch.device("cuda")
builder = QSAMetadataBuilder.__new__(QSAMetadataBuilder)
builder.compress_ratio = 4
builder.is_circular_buffer = False
builder.storage_block_size = 16
builder.token_to_req_buffer = torch.empty(8, dtype=torch.int32, device=device)
builder.slot_mapping_buffer = torch.empty(8, dtype=torch.int64, device=device)
builder.logical_positions_buffer = torch.empty(8, dtype=torch.int64, device=device)
# Simulate max_num_seqs exceeding the three live requests below.
builder.request_capacity = 8
builder.k_work_metadata_buffer = torch.empty(4, 2, dtype=torch.int32, device=device)
query_start_loc = torch.tensor([0, 3, 3, 8], dtype=torch.int32, device=device)
token_to_req = torch.tensor(
[0, 0, 0, 2, 2, 2, 2, 2], dtype=torch.int32, device=device
)
common = SimpleNamespace(
num_actual_tokens=8,
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc.cpu(),
seq_lens=torch.tensor([7, 0, 12], dtype=torch.int32, device=device),
slot_mapping=torch.full((8,), -1, dtype=torch.int64, device=device),
block_table_tensor=torch.zeros((3, 1), dtype=torch.int32, device=device),
token_to_req_indices=lambda buffer: buffer.copy_(token_to_req),
)
metadata = builder.build(0, common)
assert metadata.slot_mapping.tolist() == [-1] * 8
assert metadata.k_work_metadata.tolist() == [[0, 0], [2, 0], [2, 1], [-1, -1]]
@requires_qsa_kernels
@pytest.mark.parametrize("compress_ratio", [1, 4])
@pytest.mark.parametrize("num_reqs", [2, 3, 4, 7, 8, 9])
def test_qsa_triton_metadata_matches_pytorch(
compress_ratio: int, num_reqs: int
) -> None:
device = torch.device("cuda")
num_tokens = 8
query_start_loc = torch.tensor(
[0, 3, *([3] * (num_reqs - 2)), 8], dtype=torch.int32, device=device
)
token_to_req = torch.tensor(
[0, 0, 0, *([num_reqs - 1] * 5)],
dtype=torch.int32,
device=device,
)
block_table_rows = torch.tensor(
[
[4, -1, 8, -1, 12, -1],
[1, -1, 2, -1, 3, -1],
[7, -1, 9, -1, 11, -1],
],
dtype=torch.int32,
device=device,
)
block_table_storage = block_table_rows[
torch.arange(num_reqs, device=device) % block_table_rows.shape[0]
]
seq_lens = torch.zeros(num_reqs, dtype=torch.int32, device=device)
seq_lens[0] = 10
seq_lens[-1] = 20
common = SimpleNamespace(
num_actual_tokens=num_tokens,
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc.cpu(),
seq_lens=seq_lens,
slot_mapping=torch.tensor(
[0, 1, -1, 3, 4, -1, -1, -1], dtype=torch.int64, device=device
),
block_table_tensor=block_table_storage[:, ::2],
token_to_req_indices=lambda buffer: buffer.copy_(token_to_req),
)
def make_buffers() -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
return (
torch.empty(num_tokens, dtype=torch.int32, device=device),
torch.empty(num_tokens, dtype=torch.int64, device=device),
torch.empty(num_tokens, dtype=torch.int64, device=device),
)
max_num_work = (
(num_tokens + (compress_ratio - 1) * num_reqs) // compress_ratio
if compress_ratio != 1
else 0
)
actual_k_work = (
torch.empty(max_num_work, 2, dtype=torch.int32, device=device)
if max_num_work
else None
)
actual_buffers = make_buffers()
actual = qsa_cache.build_qsa_metadata_triton(
common,
*actual_buffers,
storage_block_size=2,
compress_ratio=compress_ratio,
k_work_metadata_buffer=actual_k_work,
request_capacity=num_reqs,
)
expected_k_work = (
torch.empty_like(actual_k_work) if actual_k_work is not None else None
)
expected_buffers = make_buffers()
expected = qsa_cache._build_qsa_metadata_torch(
common,
*expected_buffers,
storage_block_size=2,
compress_ratio=compress_ratio,
k_work_metadata_buffer=expected_k_work,
request_capacity=num_reqs,
)
for actual_tensor, expected_tensor in zip(actual, expected):
torch.testing.assert_close(actual_tensor, expected_tensor)
if actual_k_work is not None:
torch.testing.assert_close(actual_k_work, expected_k_work)
@requires_qsa_kernels
def test_qsa_fused_metadata_matches_pytorch_for_large_padded_prefill() -> None:
device = torch.device("cuda")
num_mapped_tokens = 4096
num_tokens = 4224
query_start_loc = torch.tensor(
[0, num_mapped_tokens], dtype=torch.int32, device=device
)
common = SimpleNamespace(
num_actual_tokens=num_tokens,
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc.cpu(),
seq_lens=torch.tensor(
[num_mapped_tokens + 32], dtype=torch.int32, device=device
),
block_table_tensor=torch.arange(256, dtype=torch.int32, device=device)[None],
slot_mapping=torch.tensor(
[0] * num_mapped_tokens + [-1] * (num_tokens - num_mapped_tokens),
dtype=torch.int64,
device=device,
),
token_to_req_indices=lambda buffer: buffer.zero_(),
)
def make_buffers() -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
return (
torch.empty(num_tokens, dtype=torch.int32, device=device),
torch.empty(num_tokens, dtype=torch.int64, device=device),
torch.empty(num_tokens, dtype=torch.int64, device=device),
)
max_num_work = (num_tokens + 3) // 4
actual_k_work = torch.empty(max_num_work, 2, dtype=torch.int32, device=device)
expected_k_work = torch.empty_like(actual_k_work)
actual = qsa_cache.build_qsa_metadata_triton(
common,
*make_buffers(),
storage_block_size=8,
compress_ratio=4,
k_work_metadata_buffer=actual_k_work,
)
expected = qsa_cache._build_qsa_metadata_torch(
common,
*make_buffers(),
storage_block_size=8,
compress_ratio=4,
k_work_metadata_buffer=expected_k_work,
)
for actual_tensor, expected_tensor in zip(actual, expected):
torch.testing.assert_close(actual_tensor, expected_tensor)
torch.testing.assert_close(actual_k_work, expected_k_work)
@requires_qsa_kernels
@pytest.mark.parametrize(
"num_rows",
[
pytest.param(3, id="one_tile_per_program"),
pytest.param(33, id="looped_tiles"),
],
)
def test_qsa_mqa_paged_matches_test_reference(num_rows: int) -> None:
torch.manual_seed(1)
head_dim = 128
q = torch.randn(num_rows, 4, head_dim, device="cuda", dtype=torch.bfloat16)
cache = torch.randn(40, 4, 1, head_dim, device="cuda", dtype=torch.bfloat16)
page_table = torch.randperm(40, device="cuda", dtype=torch.int32).reshape(2, 20)
request_rows = [(num_rows + 1) // 2, num_rows // 2]
token_to_req = torch.repeat_interleave(
torch.arange(2, device="cuda", dtype=torch.int32),
torch.tensor(request_rows, device="cuda"),
)
sequence_length_values = [320, 264]
sequence_lengths = torch.tensor(
sequence_length_values, device="cuda", dtype=torch.int32
)
query_positions = torch.cat(
[
torch.arange(length - rows, length, device="cuda", dtype=torch.int32)
for rows, length in zip(request_rows, sequence_length_values, strict=True)
]
)
compress_ratio = 4
visible_lengths = (query_positions + 1) // compress_ratio
actual, actual_visible_blocks = qsa_ops.qsa_mqa_paged(
q,
cache,
page_table,
token_to_req,
query_positions,
sequence_lengths,
compress_ratio=compress_ratio,
)
expected = _qsa_mqa_paged_reference(
q, cache, page_table, token_to_req, visible_lengths
)
torch.testing.assert_close(actual_visible_blocks, visible_lengths)
# Top-k is bounded by visible_blocks; later columns are intentionally unwritten.
columns = torch.arange(actual.shape[1], device=actual.device)
visible = columns[None, :] < actual_visible_blocks[:, None]
torch.testing.assert_close(actual[visible], expected[visible], rtol=1e-3, atol=1e-3)
@requires_qsa_kernels
def test_qsa_block_expansion_matches_test_reference() -> None:
blocks = torch.tensor([[0, -1], [1, 0]], device="cuda", dtype=torch.int32)
query_positions = torch.tensor([5, 10], device="cuda")
sequence_lengths = torch.tensor([6, 11], device="cuda")
token_to_req = torch.tensor([0, 1], device="cuda", dtype=torch.int32)
actual = qsa_ops.expand_qsa_block_indices_cuda(
blocks,
query_positions,
sequence_lengths,
token_to_req,
compress_ratio=4,
token_topk=8,
)
expected = _expand_qsa_indices_reference(
blocks,
query_positions,
sequence_lengths,
compress_ratio=4,
token_topk=8,
)
torch.testing.assert_close(actual, expected)
@requires_qsa_kernels
@pytest.mark.parametrize(
("num_rows", "num_query_heads", "num_kv_heads", "page_size"),
[
# Kernel-visible pages with --block-size 256 and hybrid-cache alignment.
pytest.param(1, 24, 2, 1792, id="tp1_split64"),
pytest.param(16, 12, 1, 1792, id="tp2_split32"),
pytest.param(32, 6, 1, 1024, id="tp4_split8"),
pytest.param(257, 6, 1, 1024, id="tp4_split4"),
pytest.param(513, 6, 1, 1024, id="tp4_split1"),
],
)
def test_qsa_sparse_paged_attention_matches_test_reference(
num_rows: int,
num_query_heads: int,
num_kv_heads: int,
page_size: int,
) -> None:
torch.manual_seed(2)
head_dim = 256
num_requests = 2
num_selected_pages = 64
# Keep the newest page outside the synthetic top-k as causal headroom.
num_pages_per_request = num_selected_pages + 1
num_cache_blocks = num_requests * num_pages_per_request
indexer_budget = 2048
indexer_compress_ratio = 4
selection_width = indexer_budget + indexer_compress_ratio - 1
q = torch.randn(
num_rows, num_query_heads, head_dim, device="cuda", dtype=torch.bfloat16
)
kv_cache = torch.randn(
num_cache_blocks,
page_size,
num_kv_heads,
2 * head_dim,
device="cuda",
dtype=torch.bfloat16,
)
k_cache, v_cache = kv_cache.split(head_dim, dim=-1)
block_table = (
torch.randperm(num_cache_blocks, device="cuda")
.reshape(num_requests, num_pages_per_request)
.to(torch.int32)
)
rows_per_request = math.ceil(num_rows / num_requests)
row_indices = torch.arange(num_rows, device="cuda", dtype=torch.int32)
token_to_req = row_indices // rows_per_request
request_row_counts = torch.tensor(
[rows_per_request, num_rows - rows_per_request],
device="cuda",
dtype=torch.int32,
)
context_length = num_pages_per_request * page_size - 1
block_topk = indexer_budget // indexer_compress_ratio
compressed_blocks_per_page = page_size // indexer_compress_ratio
selection = torch.arange(block_topk, device="cuda")
selected_pages = selection % num_selected_pages
selected_offsets = selection // num_selected_pages
row_shifts = 2 * row_indices.unsqueeze(1)
# Eight blocks per page; adjacent rows overlap by six of those eight.
selected_offsets = (selected_offsets + row_shifts) % compressed_blocks_per_page
block_indices = (selected_pages * compressed_blocks_per_page + selected_offsets).to(
torch.int32
)
rows_within_request = row_indices % rows_per_request
query_positions = (
context_length - request_row_counts[token_to_req.long()] + rows_within_request
).to(torch.int64)
sequence_lengths = torch.full(
(num_requests,), context_length, device="cuda", dtype=torch.int32
)
logical_indices = qsa_ops.expand_qsa_block_indices_cuda(
block_indices,
query_positions,
sequence_lengths,
token_to_req,
indexer_compress_ratio,
indexer_budget,
)
assert logical_indices.shape == (num_rows, selection_width)
scale = q.shape[-1] ** -0.5
actual = qsa_ops.qsa_sparse_paged_attention(
q,
k_cache,
v_cache,
logical_indices,
block_table,
token_to_req,
)
expected = _qsa_sparse_paged_attention_reference(
q,
k_cache,
v_cache,
logical_indices,
block_table,
token_to_req,
scale,
)
torch.testing.assert_close(actual, expected, rtol=2e-2, atol=2e-2)
@requires_qsa_kernels
def test_qsa_selection_chunks_workspace_and_matches_test_reference(
monkeypatch: pytest.MonkeyPatch,
workspace_init,
) -> None:
rows, keys, heads, head_dim = 65, 640, 4, 16
token_topk, compress_ratio = 2048, 4
torch.manual_seed(3)
q = torch.randn(rows, heads, head_dim, device="cuda", dtype=torch.bfloat16)
cache = torch.randn(40, 16, 1, head_dim, device="cuda", dtype=torch.bfloat16)
page_table = torch.randperm(40, device="cuda", dtype=torch.int32).unsqueeze(0)
token_to_req = torch.zeros(rows, device="cuda", dtype=torch.int32)
query_positions = torch.full((rows,), 2559, device="cuda", dtype=torch.int32)
sequence_lengths = torch.tensor([2560], device="cuda", dtype=torch.int32)
monkeypatch.setattr(qsa_ops, "_LOGITS_WORKSPACE_BYTES", 32 * keys * 4)
original_score = qsa_ops.qsa_mqa_paged
scored_row_counts = []
def record_score(query: torch.Tensor, *args, **kwargs):
scored_row_counts.append(query.shape[0])
return original_score(query, *args, **kwargs)
monkeypatch.setattr(qsa_ops, "qsa_mqa_paged", record_score)
actual = qsa_ops.qsa_select_paged_tokens(
q,
cache,
page_table,
token_to_req,
query_positions,
sequence_lengths,
token_topk,
compress_ratio,
)
expected = _qsa_select_paged_tokens_reference(
q,
cache,
page_table,
token_to_req,
query_positions,
sequence_lengths,
token_topk,
compress_ratio,
)
torch.testing.assert_close(actual.sort().values, expected.sort().values)
assert scored_row_counts == [32, 32, 1]
@requires_qsa_kernels
def test_qsa_selection_handles_no_complete_compressed_blocks(workspace_init) -> None:
q = torch.zeros(2, 4, 8, device="cuda", dtype=torch.bfloat16)
cache = torch.zeros(1, 16, 1, 8, device="cuda", dtype=torch.bfloat16)
page_table = torch.zeros(1, 1, device="cuda", dtype=torch.int32)
token_to_req = torch.zeros(2, device="cuda", dtype=torch.int32)
query_positions = torch.tensor([1, 2], device="cuda", dtype=torch.int32)
sequence_lengths = torch.tensor([3], device="cuda", dtype=torch.int32)
selected = qsa_ops.qsa_select_paged_tokens(
q,
cache,
page_table,
token_to_req,
query_positions,
sequence_lengths,
token_topk=2048,
compress_ratio=4,
)
assert selected[0, :2].tolist() == [0, 1]
assert selected[1, :3].tolist() == [0, 1, 2]
assert torch.all(selected[0, 2:] == -1)
assert torch.all(selected[1, 3:] == -1)
@requires_qsa_kernels
def test_qsa_streaming_compression_and_compressor_state_store_match_reference() -> None:
head_dim = 8
current_pairs = [
*((0, position) for position in range(2, 9)),
*((1, position) for position in range(5, 11)),
]
def key_row(request: int, position: int) -> torch.Tensor:
return (
torch.arange(head_dim, dtype=torch.float32) + request * 1000 + position * 10
)
def position_row(request: int, position: int) -> torch.Tensor:
return torch.tensor(
[
request * 1000 + position,
request * 1000 + position + 100,
request * 1000 + position + 200,
],
dtype=torch.int64,
)
raw_keys = (
torch.stack([key_row(request, position) for request, position in current_pairs])
.unsqueeze(1)
.to(device="cuda", dtype=torch.bfloat16)
)
raw_positions = (
torch.stack(
[position_row(request, position) for request, position in current_pairs]
)
.unsqueeze(1)
.to(device="cuda")
)
token_to_req = torch.tensor(
[request for request, _ in current_pairs],
dtype=torch.int32,
device="cuda",
)
logical_positions = torch.tensor(
[position for _, position in current_pairs],
dtype=torch.int64,
device="cuda",
)
query_start_loc = torch.tensor([0, 7, 13], dtype=torch.int32, device="cuda")
compressor_state_block_table = torch.tensor(
[[1], [0]], dtype=torch.int32, device="cuda"
)
compressor_state_cache = torch.zeros(
2, 4, 1, head_dim, dtype=torch.bfloat16, device="cuda"
)
rope_cache = torch.zeros(2, 4, 1, 3, dtype=torch.int64, device="cuda")
for request, position, block in ((0, 0, 1), (0, 1, 1), (1, 4, 0)):
compressor_state_cache[block, position % 4, 0] = key_row(request, position).to(
device="cuda", dtype=torch.bfloat16
)
rope_cache[block, position % 4, 0] = position_row(request, position).to("cuda")
compressed_slots = torch.full(
(len(current_pairs),), -1, dtype=torch.int64, device="cuda"
)
valid_rows = torch.tensor([1, 5, 9], dtype=torch.int64, device="cuda")
compressed_slots[valid_rows] = torch.arange(3, device="cuda")
pooled, first_positions = qsa_ops.qsa_compress_groups_with_ratio(
raw_keys,
raw_positions,
compressor_state_cache,
compressor_state_block_table,
token_to_req,
query_start_loc,
logical_positions,
compressed_slots,
compress_ratio=4,
rope_cache=rope_cache,
)
pooled_without_rope, scalar_first_positions = (
qsa_ops.qsa_compress_groups_with_ratio(
raw_keys,
raw_positions,
compressor_state_cache,
compressor_state_block_table,
token_to_req,
query_start_loc,
logical_positions,
compressed_slots,
compress_ratio=4,
)
)
groups = [
[(0, position) for position in range(0, 4)],
[(0, position) for position in range(4, 8)],
[(1, position) for position in range(4, 8)],
]
expected_pooled = (
torch.stack(
[
torch.stack([key_row(*pair) for pair in group]).mean(dim=0)
for group in groups
]
)
.unsqueeze(1)
.to(device="cuda", dtype=torch.bfloat16)
)
expected_positions = torch.stack(
[position_row(0, 0), position_row(0, 4), position_row(1, 4)]
).to("cuda")
expected_scalar_positions = torch.tensor(
[[0, 0, 0], [4, 4, 4], [4, 4, 4]],
dtype=torch.int64,
device="cuda",
)
torch.testing.assert_close(pooled[valid_rows], expected_pooled)
torch.testing.assert_close(pooled_without_rope[valid_rows], expected_pooled)
torch.testing.assert_close(first_positions[valid_rows], expected_positions)
torch.testing.assert_close(
scalar_first_positions[valid_rows], expected_scalar_positions
)
compressor_state_slots = torch.tensor(
[-1, -1, -1, 5, 6, 7, 4, -1, -1, 3, 0, 1, 2],
dtype=torch.int64,
device="cuda",
)
qsa_ops.qsa_store_cache_rows(
compressor_state_cache, compressor_state_slots, raw_keys
)
qsa_ops.qsa_store_cache_rows(rope_cache, compressor_state_slots, raw_positions)
for request, positions, block in ((0, range(5, 9), 1), (1, range(7, 11), 0)):
for position in positions:
torch.testing.assert_close(
compressor_state_cache[block, position % 4, 0],
key_row(request, position).to(device="cuda", dtype=torch.bfloat16),
)
torch.testing.assert_close(
rope_cache[block, position % 4, 0],
position_row(request, position).to("cuda"),
)
+6
View File
@@ -508,6 +508,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
extras={"native-prefix": "imdatta0/small_qwen3_5_20b"},
max_model_len=4096,
),
"Qwen4ExpForCausalLM": _HfExamplesInfo("", is_available_online=False),
"MellumForCausalLM": _HfExamplesInfo("JetBrains/Mellum2-12B-A2.5B-Base"),
"Qwen3NextForCausalLM": _HfExamplesInfo(
"Qwen/Qwen3-Next-80B-A3B-Instruct",
@@ -1348,6 +1349,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"Qwen/Qwen3.5-35B-A3B",
max_model_len=4096,
),
"Qwen4ExpForConditionalGeneration": _HfExamplesInfo(
"",
is_available_online=False,
),
"Qwen3OmniMoeForConditionalGeneration": _HfExamplesInfo(
"Qwen/Qwen3-Omni-30B-A3B-Instruct",
max_model_len=4096,
@@ -1797,6 +1802,7 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
"Qwen/Qwen3.5-35B-A3B",
speculative_model="Qwen/Qwen3.5-35B-A3B",
),
"Qwen4ExpMTP": _HfExamplesInfo("", is_available_online=False),
"Step3p5MTP": _HfExamplesInfo(
"stepfun-ai/Step-3.5-Flash",
speculative_model="stepfun-ai/Step-3.5-Flash",