[Model] Support Qwen3.8-Flash-Next (#53896)
Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com> Signed-off-by: andyluo7 <andy.luo@amd.com> Signed-off-by: lishunyang12 <lishunyang03@gmail.com> Signed-off-by: Harry Huang <huanghaoyan.hhy@alibaba-inc.com> Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: KungYork <firelikest@163.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: andyluo7 <43718156+andyluo7@users.noreply.github.com> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: andyluo7 <andy.luo@amd.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: lishunyang12 <lishunyang03@gmail.com> Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
zjy0516
KungYork
Thien Tran
Yifan Qiao
andyluo7
Isotr0py
andyluo7
OpenAI Codex
lishunyang12
Vadim Gimpelson
Isotr0py
Lucas Wilkinson
Claude Opus 5
OpenAI Codex
parent
687db59744
commit
e126687a9a
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,200 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.speculative import SpeculativeConfig
|
||||
from vllm.model_executor.models.config import (
|
||||
Qwen3_5ForConditionalGenerationConfig,
|
||||
Qwen4ExpForConditionalGenerationConfig,
|
||||
)
|
||||
from vllm.models.qwen4_exp.config import (
|
||||
Qwen4ExpConfig,
|
||||
Qwen4ExpTextConfig,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia.model_state import Qwen4ExpModelState
|
||||
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
|
||||
|
||||
|
||||
def _text_config(**kwargs) -> Qwen4ExpTextConfig:
|
||||
values = {
|
||||
"vocab_size": 64,
|
||||
"hidden_size": 16,
|
||||
"intermediate_size": 32,
|
||||
"num_hidden_layers": 2,
|
||||
"num_attention_heads": 2,
|
||||
"num_key_value_heads": 1,
|
||||
"head_dim": 8,
|
||||
"layer_types": ["linear_attention", "full_attention"],
|
||||
"linear_num_key_heads": 2,
|
||||
"linear_num_value_heads": 2,
|
||||
"linear_key_head_dim": 8,
|
||||
"linear_value_head_dim": 8,
|
||||
"num_experts": 0,
|
||||
"hc_count": 2,
|
||||
"hc_lowrank": 4,
|
||||
"ple_layer_ids": [1],
|
||||
"mtp_num_hidden_layers": 1,
|
||||
"mtp": {"hybrid": True},
|
||||
}
|
||||
values.update(kwargs)
|
||||
return Qwen4ExpTextConfig(**values)
|
||||
|
||||
|
||||
def test_qwen4_exp_mtp_returns_sample_and_multi_streams() -> None:
|
||||
from vllm.models.qwen4_exp.nvidia.mtp import (
|
||||
Qwen4ExpMultiTokenPredictor,
|
||||
)
|
||||
|
||||
model = object.__new__(Qwen4ExpMultiTokenPredictor)
|
||||
torch.nn.Module.__init__(model)
|
||||
model.hc_count = 2
|
||||
model.hidden_size = 4
|
||||
model.num_mtp_layers = 1
|
||||
model.layers = [
|
||||
lambda **kwargs: (
|
||||
kwargs["hidden_states"],
|
||||
kwargs["hidden_states"],
|
||||
torch.zeros(kwargs["hidden_states"].shape[0], 2),
|
||||
),
|
||||
]
|
||||
model.hyper_connection_mixer = SimpleNamespace(
|
||||
combine_and_mix=lambda hidden_states, block_output, injection: (
|
||||
hidden_states,
|
||||
hidden_states.unflatten(-1, (2, 4)).mean(-2),
|
||||
None,
|
||||
),
|
||||
)
|
||||
multi_hidden = torch.arange(16, dtype=torch.float32).reshape(2, 8)
|
||||
pp_group = SimpleNamespace(is_first_rank=False, is_last_rank=True)
|
||||
|
||||
with patch(
|
||||
"vllm.models.qwen4_exp.nvidia.mtp.get_pp_group",
|
||||
return_value=pp_group,
|
||||
):
|
||||
sample_hidden, returned_multi_hidden = model.forward(
|
||||
input_ids=None,
|
||||
positions=torch.arange(2),
|
||||
intermediate_tensors={"hidden_states": multi_hidden},
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
sample_hidden,
|
||||
multi_hidden.unflatten(-1, (2, 4)).mean(dim=-2),
|
||||
)
|
||||
assert returned_multi_hidden is multi_hidden
|
||||
|
||||
|
||||
@pytest.mark.parametrize("wrapped_config", [False, True])
|
||||
def test_qwen4_exp_mtp_override_sets_draft_config(
|
||||
wrapped_config: bool,
|
||||
) -> None:
|
||||
text_config = _text_config(
|
||||
architectures=["Qwen4ExpForCausalLM"],
|
||||
index_share_for_mtp_iteration=True,
|
||||
)
|
||||
config = (
|
||||
Qwen4ExpConfig(
|
||||
architectures=["Qwen4ExpForConditionalGeneration"],
|
||||
text_config=text_config,
|
||||
)
|
||||
if wrapped_config
|
||||
else text_config
|
||||
)
|
||||
|
||||
draft_config = SpeculativeConfig.hf_config_override(config)
|
||||
|
||||
assert draft_config.index_share_for_mtp_iteration is True
|
||||
assert draft_config.model_type == "qwen4_exp_mtp"
|
||||
assert draft_config.architectures == ["Qwen4ExpMTP"]
|
||||
assert draft_config.hc_mult == 2
|
||||
assert draft_config.n_predict == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ple_layer_ids", [[1], []])
|
||||
def test_qwen4_exp_rejects_pipeline_parallel_only_with_ple(ple_layer_ids) -> None:
|
||||
"""PLE needs raw input_ids, which non-first pipeline ranks never see. The
|
||||
rest of the architecture is PP-capable, so the refusal must be conditional
|
||||
-- and must land before the engine spends time loading weights."""
|
||||
vllm_config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
hf_text_config=_text_config(ple_layer_ids=ple_layer_ids),
|
||||
multimodal_config=None,
|
||||
),
|
||||
parallel_config=SimpleNamespace(
|
||||
pipeline_parallel_size=2, enable_dbo=False, ubatch_size=1
|
||||
),
|
||||
speculative_config=None,
|
||||
)
|
||||
with patch.object(
|
||||
Qwen3_5ForConditionalGenerationConfig, "verify_and_update_config"
|
||||
):
|
||||
if ple_layer_ids:
|
||||
with pytest.raises(NotImplementedError, match="pipeline_parallel_size=1"):
|
||||
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(
|
||||
vllm_config
|
||||
)
|
||||
else:
|
||||
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(vllm_config)
|
||||
|
||||
|
||||
def test_qwen4_exp_model_state_prepares_ngram_context() -> None:
|
||||
model_state = object.__new__(Qwen4ExpModelState)
|
||||
model_state.uses_ngram_embedding = True
|
||||
model_state.ngram_context_len = 3
|
||||
model_state.ngram_eos_token_id = 99
|
||||
model_state.ngram_context = torch.empty((4, 3), dtype=torch.int32)
|
||||
model_state.ngram_context_offsets = torch.arange(-3, 0, dtype=torch.int64)
|
||||
model_state.ple_query_start_loc = torch.empty(5, dtype=torch.int32)
|
||||
|
||||
input_batch = SimpleNamespace(
|
||||
num_reqs=2,
|
||||
num_reqs_after_padding=3,
|
||||
idx_mapping=torch.tensor([1, 0]),
|
||||
query_start_loc=torch.tensor([0, 2, 3, 3], dtype=torch.int32),
|
||||
)
|
||||
req_states = SimpleNamespace(
|
||||
num_computed_tokens=SimpleNamespace(gpu=torch.tensor([3, 1])),
|
||||
all_token_ids=SimpleNamespace(
|
||||
gpu=torch.tensor([[1, 2, 3, 4], [20, 21, 22, 23]], dtype=torch.int32)
|
||||
),
|
||||
)
|
||||
|
||||
with patch.object(MambaHybridModelState, "prepare_inputs", return_value={}):
|
||||
model_inputs = model_state.prepare_inputs(input_batch, req_states)
|
||||
|
||||
torch.testing.assert_close(
|
||||
model_inputs["query_start_loc"],
|
||||
torch.tensor([0, 2, 3, 3], dtype=torch.int32),
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
model_inputs["ngram_context"],
|
||||
torch.tensor([[99, 99, 20], [1, 2, 3], [99, 99, 99]], dtype=torch.int32),
|
||||
)
|
||||
|
||||
|
||||
def test_qwen4_exp_model_state_prepares_stable_dummy_ngram_inputs() -> None:
|
||||
model_state = object.__new__(Qwen4ExpModelState)
|
||||
model_state.uses_ngram_embedding = True
|
||||
model_state.ngram_eos_token_id = 99
|
||||
model_state.ngram_context = torch.empty((4, 3), dtype=torch.int32)
|
||||
model_state.ple_query_start_loc = torch.empty(5, dtype=torch.int32)
|
||||
|
||||
with patch.object(MambaHybridModelState, "prepare_dummy_inputs", return_value={}):
|
||||
first = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=8)
|
||||
query_start_loc_ptr = first["query_start_loc"].data_ptr()
|
||||
ngram_context_ptr = first["ngram_context"].data_ptr()
|
||||
second = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=8)
|
||||
|
||||
torch.testing.assert_close(
|
||||
second["query_start_loc"], torch.tensor([0, 2, 5, 8], dtype=torch.int32)
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
second["ngram_context"], torch.full((3, 3), 99, dtype=torch.int32)
|
||||
)
|
||||
assert second["query_start_loc"].data_ptr() == query_start_loc_ptr
|
||||
assert second["ngram_context"].data_ptr() == ngram_context_ptr
|
||||
@@ -0,0 +1,94 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.qwen4_exp.nvidia.ops.hc import (
|
||||
grouped_gemma_rmsnorm,
|
||||
hc_combine,
|
||||
hc_combine_norm,
|
||||
hc_gate_mix,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda() or not HAS_TRITON,
|
||||
reason="HC kernels require CUDA and Triton",
|
||||
)
|
||||
|
||||
HC = 4
|
||||
HIDDEN_SIZE = 2560
|
||||
HYPER_HIDDEN_SIZE = HC * HIDDEN_SIZE
|
||||
EPS = 1e-6
|
||||
|
||||
|
||||
def test_grouped_gemma_rmsnorm() -> None:
|
||||
torch.manual_seed(0)
|
||||
x = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
actual = grouped_gemma_rmsnorm(x, weight, EPS, HC)
|
||||
|
||||
grouped = x.float().unflatten(-1, (HC, HIDDEN_SIZE))
|
||||
variance = grouped.square().mean(-1, keepdim=True)
|
||||
expected = grouped * torch.rsqrt(variance + EPS)
|
||||
expected = expected.flatten(-2) * (1.0 + weight.float())
|
||||
torch.testing.assert_close(actual, expected.to(torch.bfloat16))
|
||||
|
||||
|
||||
def test_hc_gate_mix() -> None:
|
||||
torch.manual_seed(0)
|
||||
x = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
gate = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
actual = hc_gate_mix(x, gate, HC)
|
||||
expected = (
|
||||
torch.sigmoid(gate.float().unflatten(-1, (HC, HIDDEN_SIZE)))
|
||||
* x.float().unflatten(-1, (HC, HIDDEN_SIZE))
|
||||
).mean(-2)
|
||||
|
||||
torch.testing.assert_close(actual, expected.to(torch.bfloat16))
|
||||
|
||||
|
||||
def test_hc_combine() -> None:
|
||||
torch.manual_seed(0)
|
||||
block_output = torch.randn(2, HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
residual = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
injection = torch.randn(2, HC, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
actual = hc_combine(residual, block_output, injection, HC)
|
||||
injection_weight = 2.0 * torch.sigmoid(injection.float() / HC)
|
||||
expected = residual.float().unflatten(-1, (HC, HIDDEN_SIZE))
|
||||
expected = expected + block_output.float().unsqueeze(
|
||||
-2
|
||||
) * injection_weight.unsqueeze(-1)
|
||||
|
||||
torch.testing.assert_close(actual, expected.flatten(-2).to(torch.bfloat16))
|
||||
|
||||
|
||||
def test_hc_combine_norm() -> None:
|
||||
torch.manual_seed(0)
|
||||
block_output = torch.randn(2, HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
residual = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
injection = torch.randn(2, HC, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
actual, actual_norm = hc_combine_norm(
|
||||
residual, block_output, injection, weight, EPS, HC
|
||||
)
|
||||
|
||||
injection_weight = 2.0 * torch.sigmoid(injection.float() / HC)
|
||||
expected = residual.float().unflatten(-1, (HC, HIDDEN_SIZE))
|
||||
expected = expected + block_output.float().unsqueeze(
|
||||
-2
|
||||
) * injection_weight.unsqueeze(-1)
|
||||
expected = expected.flatten(-2).to(residual.dtype)
|
||||
grouped = expected.float().unflatten(-1, (HC, HIDDEN_SIZE))
|
||||
variance = grouped.square().mean(-1, keepdim=True)
|
||||
expected_norm = grouped * torch.rsqrt(variance + EPS)
|
||||
expected_norm = expected_norm.flatten(-2) * (1.0 + weight.float())
|
||||
|
||||
torch.testing.assert_close(actual, expected)
|
||||
torch.testing.assert_close(actual_norm, expected_norm.to(torch.bfloat16))
|
||||
@@ -0,0 +1,401 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from functools import partial
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.nn import functional as F
|
||||
|
||||
import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module
|
||||
import vllm.model_executor.parameter as parameter_module
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.models.qwen4_exp.common.ple import (
|
||||
PLEShardOverlap,
|
||||
compute_ple_shard_overlap,
|
||||
copy_ple_embedding_shard_,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia import ple_layer as ple_layer_module
|
||||
from vllm.models.qwen4_exp.nvidia.ple_layer import (
|
||||
Qwen4ExpNGramEmbedding,
|
||||
Qwen4ExpPLEFp8EmbeddingMethod,
|
||||
Qwen4ExpPLELayer,
|
||||
_get_ple_embedding_quant_method,
|
||||
)
|
||||
|
||||
|
||||
def _make_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding:
|
||||
module = Qwen4ExpNGramEmbedding.__new__(Qwen4ExpNGramEmbedding)
|
||||
nn.Module.__init__(module)
|
||||
module.split_ngram_parts = 2
|
||||
module.register_buffer("layer_multipliers", torch.zeros(1, dtype=torch.long))
|
||||
module.register_buffer("ngram_heads_offsets", torch.zeros(1, dtype=torch.long))
|
||||
module.register_buffer("ngram_heads_vocab_sizes", torch.zeros(1, dtype=torch.long))
|
||||
module.ngram_embedding = SimpleNamespace(
|
||||
org_vocab_size=8,
|
||||
embedding_dim=2,
|
||||
weight=nn.Parameter(torch.full((4, 2), -1.0)),
|
||||
shard_indices=SimpleNamespace(
|
||||
org_vocab_start_index=2,
|
||||
org_vocab_end_index=6,
|
||||
),
|
||||
)
|
||||
_set_test_embedding_weight_loader(module.ngram_embedding)
|
||||
return module
|
||||
|
||||
|
||||
def _set_test_embedding_weight_loader(embedding) -> None:
|
||||
embedding.weight.weight_loader = partial(
|
||||
copy_ple_embedding_shard_,
|
||||
tp_start=embedding.shard_indices.org_vocab_start_index,
|
||||
tp_end=embedding.shard_indices.org_vocab_end_index,
|
||||
)
|
||||
|
||||
|
||||
def _make_fp8_ngram_embedding_for_load_test() -> Qwen4ExpNGramEmbedding:
|
||||
module = _make_ngram_embedding_for_load_test()
|
||||
embedding = nn.Module()
|
||||
embedding.org_vocab_size = 8
|
||||
embedding.embedding_dim = 2
|
||||
embedding.shard_indices = SimpleNamespace(
|
||||
org_vocab_start_index=2,
|
||||
org_vocab_end_index=6,
|
||||
)
|
||||
embedding.register_parameter(
|
||||
"weight",
|
||||
nn.Parameter(
|
||||
torch.full((4, 2), -1.0).to(torch.float8_e4m3fn),
|
||||
requires_grad=False,
|
||||
),
|
||||
)
|
||||
embedding.register_parameter(
|
||||
"weight_scale",
|
||||
nn.Parameter(torch.zeros(1, dtype=torch.bfloat16), requires_grad=False),
|
||||
)
|
||||
_set_test_embedding_weight_loader(embedding)
|
||||
module.ngram_embedding = embedding
|
||||
return module
|
||||
|
||||
|
||||
def test_ple_shard_overlap_and_copy() -> None:
|
||||
overlap = compute_ple_shard_overlap(
|
||||
checkpoint_start=2, checkpoint_rows=5, tp_start=4, tp_end=8
|
||||
)
|
||||
assert overlap == PLEShardOverlap(source_start=2, destination_start=0, row_count=3)
|
||||
|
||||
destination = torch.full((4, 2), -1.0)
|
||||
loaded = torch.arange(10, dtype=torch.float64).reshape(5, 2)
|
||||
copied = copy_ple_embedding_shard_(
|
||||
destination,
|
||||
loaded,
|
||||
checkpoint_start=2,
|
||||
tp_start=4,
|
||||
tp_end=8,
|
||||
)
|
||||
|
||||
assert copied == 3
|
||||
torch.testing.assert_close(destination[:3], loaded[2:5].float())
|
||||
torch.testing.assert_close(destination[3], torch.tensor([-1.0, -1.0]))
|
||||
|
||||
|
||||
def test_ple_shard_copy_is_a_noop_without_overlap() -> None:
|
||||
destination = torch.ones(4, 2)
|
||||
copied = copy_ple_embedding_shard_(
|
||||
destination,
|
||||
torch.zeros(2, 2),
|
||||
checkpoint_start=10,
|
||||
tp_start=4,
|
||||
tp_end=8,
|
||||
)
|
||||
|
||||
assert copied == 0
|
||||
assert torch.equal(destination, torch.ones_like(destination))
|
||||
|
||||
|
||||
def test_ngram_embedding_loads_shards_and_ignores_legacy_token_lookup() -> None:
|
||||
module = _make_ngram_embedding_for_load_test()
|
||||
shard_0 = torch.arange(8, dtype=torch.float32).reshape(4, 2)
|
||||
shard_1 = torch.arange(8, 16, dtype=torch.float32).reshape(4, 2)
|
||||
|
||||
loaded = module.load_weights(
|
||||
[
|
||||
("ngram_embedding.shard_0.weight", shard_0),
|
||||
("ngram_embedding.shard_1.weight", shard_1),
|
||||
("token_lookup", torch.tensor([2, 1, 0])),
|
||||
]
|
||||
)
|
||||
|
||||
assert loaded == {"ngram_embedding.weight"}
|
||||
torch.testing.assert_close(
|
||||
module.ngram_embedding.weight,
|
||||
torch.cat((shard_0[2:4], shard_1[0:2])),
|
||||
)
|
||||
|
||||
|
||||
def test_ngram_embedding_loads_fp8_shards_and_global_scale() -> None:
|
||||
module = _make_fp8_ngram_embedding_for_load_test()
|
||||
shard_0 = torch.arange(8, dtype=torch.float32).reshape(4, 2).to(torch.float8_e4m3fn)
|
||||
shard_1 = (
|
||||
torch.arange(8, 16, dtype=torch.float32).reshape(4, 2).to(torch.float8_e4m3fn)
|
||||
)
|
||||
weight_scale = torch.tensor([0.25], dtype=torch.bfloat16)
|
||||
|
||||
loaded = module.load_weights(
|
||||
[
|
||||
("ngram_embedding.shard_0.weight", shard_0),
|
||||
("ngram_embedding.shard_1.weight", shard_1),
|
||||
("ngram_embedding.weight_scale", weight_scale),
|
||||
]
|
||||
)
|
||||
|
||||
assert loaded == {"ngram_embedding.weight", "ngram_embedding.weight_scale"}
|
||||
assert module.ngram_embedding.weight.dtype == torch.float8_e4m3fn
|
||||
assert torch.equal(
|
||||
module.ngram_embedding.weight.float(),
|
||||
torch.cat((shard_0[2:4], shard_1[0:2])).float(),
|
||||
)
|
||||
assert torch.equal(module.ngram_embedding.weight_scale, weight_scale)
|
||||
|
||||
|
||||
def _make_fp8_embedding_layer(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> embedding_module.VocabParallelEmbedding:
|
||||
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
embedding_module, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
)
|
||||
monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
parameter_module, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
)
|
||||
method = Qwen4ExpPLEFp8EmbeddingMethod()
|
||||
layer = embedding_module.VocabParallelEmbedding(
|
||||
3,
|
||||
2,
|
||||
params_dtype=torch.bfloat16,
|
||||
padding_size=1,
|
||||
quant_method=method,
|
||||
)
|
||||
weight = torch.tensor([[1.0, 2.0], [4.0, 8.0], [16.0, 32.0]])
|
||||
layer.weight.data.copy_(weight.to(torch.float8_e4m3fn))
|
||||
layer.weight_scale.data.copy_(torch.tensor([0.25], dtype=torch.bfloat16))
|
||||
return layer
|
||||
|
||||
|
||||
def test_ple_fp8_embedding_dequantizes_in_ple_layer(monkeypatch) -> None:
|
||||
layer = _make_fp8_embedding_layer(monkeypatch)
|
||||
quantized_output = layer(torch.tensor([2, 0]))
|
||||
ple_layer = Qwen4ExpPLELayer.__new__(Qwen4ExpPLELayer)
|
||||
nn.Module.__init__(ple_layer)
|
||||
ple_layer.ple_embedding = nn.Module()
|
||||
ple_layer.ple_embedding.ngram_embedding = layer
|
||||
|
||||
output = ple_layer._dequantize_embeddings(
|
||||
quantized_output,
|
||||
torch.bfloat16,
|
||||
)
|
||||
|
||||
assert layer.weight.dtype == torch.float8_e4m3fn
|
||||
assert layer.weight_scale.dtype == torch.bfloat16
|
||||
assert quantized_output.dtype == torch.float8_e4m3fn
|
||||
assert output.dtype == torch.bfloat16
|
||||
weight = torch.tensor([[1.0, 2.0], [4.0, 8.0], [16.0, 32.0]])
|
||||
torch.testing.assert_close(output, (weight[[2, 0]] * 0.25).bfloat16())
|
||||
|
||||
|
||||
def test_ple_fp8_embedding_uses_int8_for_tp_reduce(monkeypatch) -> None:
|
||||
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
embedding_module, "get_tensor_model_parallel_world_size", lambda: 2
|
||||
)
|
||||
monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
parameter_module, "get_tensor_model_parallel_world_size", lambda: 2
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
embedding_module,
|
||||
"get_masked_input_and_mask",
|
||||
lambda *args: (
|
||||
torch.tensor([0, 0]),
|
||||
torch.tensor([False, True]),
|
||||
),
|
||||
)
|
||||
reduced_dtypes = []
|
||||
|
||||
def all_reduce(tensor: torch.Tensor) -> torch.Tensor:
|
||||
reduced_dtypes.append(tensor.dtype)
|
||||
return tensor.clone()
|
||||
|
||||
monkeypatch.setattr(
|
||||
embedding_module,
|
||||
"tensor_model_parallel_all_reduce",
|
||||
all_reduce,
|
||||
)
|
||||
layer = embedding_module.VocabParallelEmbedding(
|
||||
4,
|
||||
2,
|
||||
params_dtype=torch.bfloat16,
|
||||
padding_size=1,
|
||||
quant_method=Qwen4ExpPLEFp8EmbeddingMethod(),
|
||||
)
|
||||
layer.weight.data.copy_(
|
||||
torch.tensor([[1.0, 2.0], [4.0, 8.0]]).to(torch.float8_e4m3fn)
|
||||
)
|
||||
|
||||
output = layer(torch.tensor([0, 2]))
|
||||
|
||||
assert reduced_dtypes == [torch.int8]
|
||||
assert output.dtype == torch.float8_e4m3fn
|
||||
torch.testing.assert_close(output[0].float(), layer.weight[0].float())
|
||||
assert torch.count_nonzero(output[1].float()) == 0
|
||||
|
||||
|
||||
def test_ple_fp8_embedding_respects_checkpoint_shard_exclusions() -> None:
|
||||
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
|
||||
quant_config = Fp8Config(
|
||||
is_checkpoint_fp8_serialized=True,
|
||||
ignored_layers=[],
|
||||
weight_block_size=[128, 128],
|
||||
)
|
||||
assert isinstance(
|
||||
_get_ple_embedding_quant_method(quant_config, prefix),
|
||||
Qwen4ExpPLEFp8EmbeddingMethod,
|
||||
)
|
||||
|
||||
quant_config.ignored_layers = [f"{prefix}.shard_0"]
|
||||
assert _get_ple_embedding_quant_method(quant_config, prefix) is None
|
||||
|
||||
|
||||
def test_ple_ngram_ids_custom_op_uses_current_request_layout(monkeypatch) -> None:
|
||||
class RuntimeNGramEmbedding(nn.Module):
|
||||
def compute_ngram_ids(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
ngram_context: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
del input_ids, ngram_context
|
||||
num_reqs = query_start_loc.numel() - 1
|
||||
return torch.full((4, 2), num_reqs, dtype=torch.long)
|
||||
|
||||
layer = Qwen4ExpPLELayer.__new__(Qwen4ExpPLELayer)
|
||||
nn.Module.__init__(layer)
|
||||
layer.ple_embedding = RuntimeNGramEmbedding()
|
||||
monkeypatch.setattr(
|
||||
ple_layer_module,
|
||||
"get_forward_context",
|
||||
lambda: SimpleNamespace(no_compile_layers={"ple": layer}),
|
||||
)
|
||||
input_ids = torch.arange(4)
|
||||
ngram_context = torch.zeros(2, 2, dtype=torch.long)
|
||||
output = torch.empty(4, 2, dtype=torch.long)
|
||||
|
||||
ple_layer_module.qwen4_exp_compute_ple_ngram_ids(
|
||||
input_ids,
|
||||
torch.tensor([0, 4]),
|
||||
ngram_context,
|
||||
output,
|
||||
"ple",
|
||||
)
|
||||
assert torch.equal(output, torch.ones_like(output))
|
||||
|
||||
ple_layer_module.qwen4_exp_compute_ple_ngram_ids(
|
||||
input_ids,
|
||||
torch.tensor([0, 2, 4]),
|
||||
ngram_context,
|
||||
output,
|
||||
"ple",
|
||||
)
|
||||
assert torch.equal(output, torch.full_like(output, 2))
|
||||
|
||||
|
||||
def test_dilated_ple_spec_state_rolls_back_before_next_forward() -> None:
|
||||
module = Qwen4ExpPLELayer.__new__(Qwen4ExpPLELayer)
|
||||
nn.Module.__init__(module)
|
||||
module.conv_state_len = 6
|
||||
module.short_conv_dilation = 2
|
||||
|
||||
conv_weights = torch.tensor([[0.25, -0.5, 0.75, 1.0]])
|
||||
conv_state = torch.zeros(2, 1, 9)
|
||||
conv_state[1] = torch.arange(1, 10, dtype=torch.float32).reshape(1, 9)
|
||||
first_inputs = torch.tensor([[10.0], [20.0], [30.0], [40.0]])
|
||||
initial_state = conv_state[1:].clone()
|
||||
first_history = torch.cat(
|
||||
(initial_state[..., : module.conv_state_len], first_inputs.T.unsqueeze(0)),
|
||||
dim=-1,
|
||||
)
|
||||
|
||||
graph_padded_inputs = F.pad(first_inputs, (0, 0, 0, 4))
|
||||
first_output = module._short_conv_dilated_spec_batched(
|
||||
graph_padded_inputs,
|
||||
conv_state,
|
||||
conv_weights,
|
||||
torch.tensor([1, 0]),
|
||||
torch.tensor([0, 4, 4]),
|
||||
torch.tensor([1, 0]),
|
||||
spec_query_len=4,
|
||||
)
|
||||
|
||||
expected_first_output = F.silu(
|
||||
F.conv1d(
|
||||
first_history,
|
||||
conv_weights.unsqueeze(1),
|
||||
groups=1,
|
||||
dilation=module.short_conv_dilation,
|
||||
)
|
||||
).transpose(1, 2)[0]
|
||||
expected_first_state = first_history[..., 1:10]
|
||||
torch.testing.assert_close(first_output[:4], expected_first_output)
|
||||
assert torch.count_nonzero(first_output[4:]) == 0
|
||||
assert torch.count_nonzero(conv_state[0]) == 0
|
||||
torch.testing.assert_close(conv_state[1:], expected_first_state)
|
||||
|
||||
second_inputs = torch.tensor([[50.0], [60.0]])
|
||||
rollback_state = expected_first_state[..., 1:7]
|
||||
padded_second_inputs = F.pad(second_inputs.T.unsqueeze(0), (0, 2))
|
||||
second_history = torch.cat((rollback_state, padded_second_inputs), dim=-1)
|
||||
expected_second_state = expected_first_state.clone()
|
||||
expected_second_state[..., :7] = second_history[..., 1:8]
|
||||
|
||||
second_output = module._short_conv_dilated_spec_batched(
|
||||
second_inputs,
|
||||
conv_state,
|
||||
conv_weights,
|
||||
torch.tensor([1]),
|
||||
torch.tensor([0, 2]),
|
||||
torch.tensor([2]),
|
||||
spec_query_len=4,
|
||||
)
|
||||
|
||||
expected_second_output = F.silu(
|
||||
F.conv1d(
|
||||
second_history,
|
||||
conv_weights.unsqueeze(1),
|
||||
groups=1,
|
||||
dilation=module.short_conv_dilation,
|
||||
)
|
||||
).transpose(1, 2)[0, :2]
|
||||
torch.testing.assert_close(second_output, expected_second_output)
|
||||
torch.testing.assert_close(conv_state[1:], expected_second_state)
|
||||
|
||||
|
||||
def test_ple_short_conv_uses_fallback_when_profile_metadata_is_omitted(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
module = Qwen4ExpPLELayer.__new__(Qwen4ExpPLELayer)
|
||||
nn.Module.__init__(module)
|
||||
module.prefix = "model.layers.1.ple"
|
||||
inputs = torch.arange(6, dtype=torch.float32).reshape(3, 2)
|
||||
expected = inputs + 1
|
||||
monkeypatch.setattr(module, "_short_conv_fallback", lambda _: expected)
|
||||
monkeypatch.setattr(
|
||||
ple_layer_module,
|
||||
"get_forward_context",
|
||||
lambda: SimpleNamespace(attn_metadata={"model.layers.0.self_attn": object()}),
|
||||
)
|
||||
|
||||
output = module._short_conv(inputs)
|
||||
|
||||
assert output is expected
|
||||
@@ -0,0 +1,297 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
|
||||
from vllm.models.qwen4_exp.amd import (
|
||||
model as _qwen4_exp_model, # noqa: F401
|
||||
)
|
||||
from vllm.models.qwen4_exp.amd import ple_layer as ple_layer_module
|
||||
from vllm.models.qwen4_exp.amd.indexer_qsa import (
|
||||
apply_qsa_rmsnorm,
|
||||
apply_qsa_rope,
|
||||
)
|
||||
from vllm.models.qwen4_exp.amd.ops import qsa as qsa_ops
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="AMD QSA requires ROCm",
|
||||
)
|
||||
|
||||
requires_qsa_kernels = pytest.mark.skipif(
|
||||
not HAS_TRITON,
|
||||
reason="AMD QSA kernels require Triton",
|
||||
)
|
||||
|
||||
|
||||
def test_ple_ngram_embedding_custom_op_uses_resident_weight(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
layer_name = "model.layers.0.ple"
|
||||
layer = ple_layer_module.Qwen4ExpPLELayer.__new__(ple_layer_module.Qwen4ExpPLELayer)
|
||||
torch.nn.Module.__init__(layer)
|
||||
layer.ple_embedding = torch.nn.Module()
|
||||
layer.ple_embedding.ngram_embedding = torch.nn.Embedding(8, 3)
|
||||
context = SimpleNamespace(no_compile_layers={layer_name: layer})
|
||||
monkeypatch.setattr(ple_layer_module, "get_forward_context", lambda: context)
|
||||
|
||||
ngram_ids = torch.tensor([[0, 1], [2, 3]])
|
||||
output = torch.empty(2, 6)
|
||||
ple_layer_module.qwen4_exp_amd_ple_ngram_embedding(
|
||||
ngram_ids,
|
||||
output,
|
||||
layer_name,
|
||||
)
|
||||
|
||||
expected = layer.ple_embedding.ngram_embedding(ngram_ids).flatten(-2)
|
||||
torch.testing.assert_close(output, expected)
|
||||
|
||||
|
||||
def _qsa_sparse_paged_attention_reference(
|
||||
q: torch.Tensor,
|
||||
k_cache: torch.Tensor,
|
||||
v_cache: torch.Tensor,
|
||||
logical_indices: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
token_to_req: torch.Tensor,
|
||||
softmax_scale: float,
|
||||
) -> torch.Tensor:
|
||||
output = torch.zeros_like(q)
|
||||
repeats = q.shape[1] // k_cache.shape[2]
|
||||
page_size = k_cache.shape[1]
|
||||
for row in range(q.shape[0]):
|
||||
logical = logical_indices[row]
|
||||
logical = logical[logical >= 0].long()
|
||||
if not logical.numel():
|
||||
continue
|
||||
request = token_to_req[row].long()
|
||||
pages = block_table[request, logical // page_size].long()
|
||||
offsets = logical % page_size
|
||||
keys = k_cache[pages, offsets].repeat_interleave(repeats, dim=1)
|
||||
values = v_cache[pages, offsets].repeat_interleave(repeats, dim=1)
|
||||
scores = torch.einsum("hd,khd->hk", q[row].float(), keys.float())
|
||||
probabilities = torch.softmax(scores * softmax_scale, dim=-1)
|
||||
output[row] = torch.einsum("hk,khd->hd", probabilities, values.float()).to(
|
||||
q.dtype
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
def test_qsa_rope_uses_platform_dispatch() -> None:
|
||||
tensor = torch.arange(16, dtype=torch.float32).reshape(2, 2, 4)
|
||||
positions = torch.tensor([0, 1])
|
||||
calls = []
|
||||
|
||||
def apply_rotary_emb(
|
||||
rotary_input: torch.Tensor,
|
||||
cos: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
calls.append((rotary_input, cos, sin))
|
||||
return rotary_input + 1
|
||||
|
||||
rotary_emb = SimpleNamespace(
|
||||
rotary_dim=2,
|
||||
apply_rotary_emb=apply_rotary_emb,
|
||||
_match_cos_sin_cache_dtype=lambda _: torch.zeros(2, 4),
|
||||
)
|
||||
|
||||
output = apply_qsa_rope(rotary_emb, positions, tensor)
|
||||
|
||||
assert len(calls) == 1
|
||||
torch.testing.assert_close(output[..., :2], tensor[..., :2] + 1)
|
||||
torch.testing.assert_close(output[..., 2:], tensor[..., 2:])
|
||||
|
||||
|
||||
def test_qsa_rmsnorm_uses_portable_implementation(default_vllm_config) -> None:
|
||||
norm = GemmaRMSNorm(4, eps=1e-6)
|
||||
norm.weight.data.copy_(torch.tensor([0.1, -0.2, 0.3, -0.4]))
|
||||
tensor = torch.arange(8, dtype=torch.float32).reshape(2, 4)
|
||||
output = apply_qsa_rmsnorm(norm, tensor)
|
||||
|
||||
torch.testing.assert_close(output, norm.forward_native(tensor))
|
||||
|
||||
|
||||
def test_qsa_selection_uses_portable_topk_on_rocm(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
rows = 2
|
||||
token_topk = 8
|
||||
compress_ratio = 4
|
||||
block_topk = token_topk // compress_ratio
|
||||
blocks = torch.empty((rows, block_topk), dtype=torch.int32)
|
||||
visible_blocks = torch.tensor([4, 6], dtype=torch.int32)
|
||||
logits = torch.empty((rows, 8), dtype=torch.float32)
|
||||
selection_output = torch.empty(
|
||||
(rows, token_topk + compress_ratio - 1), dtype=torch.int32
|
||||
)
|
||||
topk_call: dict[str, Any] = {}
|
||||
|
||||
monkeypatch.setattr(
|
||||
qsa_ops,
|
||||
"qsa_mqa_paged",
|
||||
lambda *args, **kwargs: (logits, visible_blocks),
|
||||
)
|
||||
monkeypatch.setattr(qsa_ops.current_platform, "is_cuda", lambda: False)
|
||||
|
||||
def top_k_per_row_decode(
|
||||
topk_logits,
|
||||
next_n,
|
||||
seq_lens,
|
||||
raw_topk_indices,
|
||||
num_rows,
|
||||
stride0,
|
||||
stride1,
|
||||
topk_tokens,
|
||||
) -> None:
|
||||
topk_call.update(
|
||||
logits=topk_logits,
|
||||
next_n=next_n,
|
||||
seq_lens=seq_lens,
|
||||
raw_topk_indices=raw_topk_indices,
|
||||
num_rows=num_rows,
|
||||
strides=(stride0, stride1),
|
||||
topk_tokens=topk_tokens,
|
||||
)
|
||||
raw_topk_indices.zero_()
|
||||
|
||||
def expand_qsa_block_indices(*args) -> None:
|
||||
args[-1].fill_(-1)
|
||||
|
||||
monkeypatch.setattr(qsa_ops.ops, "top_k_per_row_decode", top_k_per_row_decode)
|
||||
monkeypatch.setattr(
|
||||
qsa_ops,
|
||||
"expand_qsa_block_indices_cuda",
|
||||
expand_qsa_block_indices,
|
||||
)
|
||||
|
||||
output = qsa_ops.qsa_select_paged_tokens(
|
||||
torch.empty((rows, 1, 1)),
|
||||
torch.empty((1, 4, 1, 1)),
|
||||
torch.empty((1, 2), dtype=torch.int32),
|
||||
torch.zeros(rows, dtype=torch.int32),
|
||||
torch.arange(rows, dtype=torch.int32),
|
||||
torch.tensor([8], dtype=torch.int32),
|
||||
token_topk,
|
||||
compress_ratio,
|
||||
selection_output,
|
||||
)
|
||||
|
||||
assert output is selection_output
|
||||
assert torch.all(output == -1)
|
||||
assert topk_call["logits"] is logits
|
||||
assert topk_call["next_n"] == 1
|
||||
assert topk_call["seq_lens"] is visible_blocks
|
||||
assert topk_call["raw_topk_indices"].shape == blocks.shape
|
||||
assert topk_call["num_rows"] == rows
|
||||
assert topk_call["strides"] == (logits.stride(0), logits.stride(1))
|
||||
assert topk_call["topk_tokens"] == block_topk
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
@pytest.mark.parametrize(
|
||||
("num_rows", "num_query_heads", "num_kv_heads", "page_size"),
|
||||
[
|
||||
pytest.param(1, 24, 2, 1792, id="tp1_split64"),
|
||||
pytest.param(16, 12, 1, 1792, id="tp2_split32"),
|
||||
pytest.param(32, 6, 1, 1024, id="tp4_split8"),
|
||||
pytest.param(257, 6, 1, 1024, id="tp4_split4"),
|
||||
pytest.param(513, 6, 1, 1024, id="tp4_split1"),
|
||||
],
|
||||
)
|
||||
def test_qsa_sparse_paged_attention_matches_reference(
|
||||
num_rows: int,
|
||||
num_query_heads: int,
|
||||
num_kv_heads: int,
|
||||
page_size: int,
|
||||
) -> None:
|
||||
torch.manual_seed(2)
|
||||
head_dim = 256
|
||||
num_requests = 2
|
||||
num_selected_pages = 64
|
||||
num_pages_per_request = num_selected_pages + 1
|
||||
num_cache_blocks = num_requests * num_pages_per_request
|
||||
indexer_budget = 2048
|
||||
indexer_compress_ratio = 4
|
||||
selection_width = indexer_budget + indexer_compress_ratio - 1
|
||||
q = torch.randn(
|
||||
num_rows, num_query_heads, head_dim, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
kv_cache = torch.randn(
|
||||
num_cache_blocks,
|
||||
page_size,
|
||||
num_kv_heads,
|
||||
2 * head_dim,
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
k_cache, v_cache = kv_cache.split(head_dim, dim=-1)
|
||||
block_table = (
|
||||
torch.randperm(num_cache_blocks, device="cuda")
|
||||
.reshape(num_requests, num_pages_per_request)
|
||||
.to(torch.int32)
|
||||
)
|
||||
rows_per_request = math.ceil(num_rows / num_requests)
|
||||
row_indices = torch.arange(num_rows, device="cuda", dtype=torch.int32)
|
||||
token_to_req = row_indices // rows_per_request
|
||||
request_row_counts = torch.tensor(
|
||||
[rows_per_request, num_rows - rows_per_request],
|
||||
device="cuda",
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
context_length = num_pages_per_request * page_size - 1
|
||||
block_topk = indexer_budget // indexer_compress_ratio
|
||||
compressed_blocks_per_page = page_size // indexer_compress_ratio
|
||||
selection = torch.arange(block_topk, device="cuda")
|
||||
selected_pages = selection % num_selected_pages
|
||||
selected_offsets = selection // num_selected_pages
|
||||
row_shifts = 2 * row_indices.unsqueeze(1)
|
||||
selected_offsets = (selected_offsets + row_shifts) % compressed_blocks_per_page
|
||||
block_indices = (selected_pages * compressed_blocks_per_page + selected_offsets).to(
|
||||
torch.int32
|
||||
)
|
||||
rows_within_request = row_indices % rows_per_request
|
||||
query_positions = (
|
||||
context_length - request_row_counts[token_to_req.long()] + rows_within_request
|
||||
).to(torch.int64)
|
||||
sequence_lengths = torch.full(
|
||||
(num_requests,), context_length, device="cuda", dtype=torch.int32
|
||||
)
|
||||
logical_indices = qsa_ops.expand_qsa_block_indices_cuda(
|
||||
block_indices,
|
||||
query_positions,
|
||||
sequence_lengths,
|
||||
token_to_req,
|
||||
indexer_compress_ratio,
|
||||
indexer_budget,
|
||||
)
|
||||
assert logical_indices.shape == (num_rows, selection_width)
|
||||
|
||||
actual = qsa_ops.qsa_sparse_paged_attention(
|
||||
q,
|
||||
k_cache,
|
||||
v_cache,
|
||||
logical_indices,
|
||||
block_table,
|
||||
token_to_req,
|
||||
)
|
||||
expected = _qsa_sparse_paged_attention_reference(
|
||||
q,
|
||||
k_cache,
|
||||
v_cache,
|
||||
logical_indices,
|
||||
block_table,
|
||||
token_to_req,
|
||||
q.shape[-1] ** -0.5,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, rtol=2e-2, atol=2e-2)
|
||||
@@ -0,0 +1,297 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Correctness tests for the fused QSA pre-indexer."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.qwen4_exp.common.qsa_cache import (
|
||||
canonical_qsa_rope_positions,
|
||||
circular_qsa_slot_mapping,
|
||||
compressed_qsa_slot_mapping,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia.indexer_qsa import apply_qsa_rope
|
||||
from vllm.models.qwen4_exp.nvidia.ops.qsa import (
|
||||
qsa_compress_groups_with_ratio,
|
||||
qsa_store_cache_rows,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia.ops.qsa_pre_indexer import (
|
||||
qsa_pre_indexer,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
|
||||
requires_qsa_kernels = pytest.mark.skipif(
|
||||
not current_platform.is_cuda() or not HAS_TRITON,
|
||||
reason="QSA kernels require CUDA and Triton",
|
||||
)
|
||||
|
||||
HQ, D = 4, 128
|
||||
CR = 4
|
||||
MROPE_SECTION = (11, 11, 10)
|
||||
EPS = 1e-6
|
||||
BLOCK_SIZE = 16
|
||||
COMP_PAGE = BLOCK_SIZE // CR
|
||||
ROPE_POS_OFFSET = D
|
||||
RTOL = 1.6e-2
|
||||
ATOL = 1e-2
|
||||
MIXED_BATCH = ([260, 259, 138], [1, 1, 37], [8, 8, 8])
|
||||
|
||||
|
||||
def _make_block_table(block_counts):
|
||||
num_blocks = sum(block_counts)
|
||||
table = torch.full((len(block_counts), max(block_counts)), -1, dtype=torch.int32)
|
||||
physical_blocks = torch.randperm(num_blocks)
|
||||
offset = 0
|
||||
for request, count in enumerate(block_counts):
|
||||
table[request, :count] = physical_blocks[offset : offset + count]
|
||||
offset += count
|
||||
return table, num_blocks
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
@pytest.mark.parametrize(
|
||||
"mrope,is_2d_positions,cache_rope_positions,state_size,seq_lens,query_lens,history_lens",
|
||||
[
|
||||
pytest.param(True, True, True, 4, *MIXED_BATCH, id="mrope"),
|
||||
pytest.param(False, False, False, 4, *MIXED_BATCH, id="text"),
|
||||
pytest.param(True, False, True, 8, *MIXED_BATCH, id="mrope-model-1d"),
|
||||
pytest.param(True, True, False, 4, *MIXED_BATCH, id="mrope-no-position-cache"),
|
||||
pytest.param(
|
||||
True,
|
||||
False,
|
||||
False,
|
||||
8,
|
||||
*MIXED_BATCH,
|
||||
id="mrope-model-1d-no-position-cache",
|
||||
),
|
||||
pytest.param(
|
||||
False, False, True, 4, *MIXED_BATCH, id="text-with-position-cache"
|
||||
),
|
||||
pytest.param(True, True, True, 4, [37], [37], [0], id="fresh"),
|
||||
pytest.param(True, True, True, 4, [4097], [4097], [0], id="tiled"),
|
||||
],
|
||||
)
|
||||
def test_qsa_fused_pre_indexer_matches_unfused(
|
||||
mrope,
|
||||
is_2d_positions,
|
||||
cache_rope_positions,
|
||||
state_size,
|
||||
seq_lens,
|
||||
query_lens,
|
||||
history_lens,
|
||||
) -> None:
|
||||
from flashinfer.norm import gemma_rmsnorm
|
||||
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
|
||||
device = "cuda"
|
||||
rope_params = {
|
||||
"partial_rotary_factor": 0.25,
|
||||
"rope_theta": 10000000,
|
||||
"rope_type": "default",
|
||||
}
|
||||
if mrope:
|
||||
rope_params["mrope_interleaved"] = True
|
||||
rope_params["mrope_section"] = list(MROPE_SECTION)
|
||||
with torch.device(device):
|
||||
rope = get_rope(
|
||||
head_size=256,
|
||||
max_position=32768,
|
||||
rope_parameters=rope_params,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
token_to_req = torch.cat(
|
||||
[
|
||||
torch.full((length,), request, dtype=torch.int32)
|
||||
for request, length in enumerate(query_lens)
|
||||
]
|
||||
).to(device)
|
||||
logical_positions = torch.cat(
|
||||
[
|
||||
torch.arange(seq_len - query_len, seq_len, dtype=torch.int64)
|
||||
for seq_len, query_len in zip(seq_lens, query_lens)
|
||||
]
|
||||
).to(device)
|
||||
num_tokens = logical_positions.numel()
|
||||
query_start_loc = torch.tensor(
|
||||
[0, *torch.tensor(query_lens).cumsum(0).tolist()], dtype=torch.int32
|
||||
).to(device)
|
||||
positions = (
|
||||
torch.stack(
|
||||
[
|
||||
logical_positions,
|
||||
logical_positions // 7 + 3,
|
||||
logical_positions // 13 + 11,
|
||||
]
|
||||
)
|
||||
if is_2d_positions
|
||||
else logical_positions
|
||||
)
|
||||
position_rows = (
|
||||
canonical_qsa_rope_positions(positions)
|
||||
if cache_rope_positions
|
||||
else logical_positions.view(-1, 1, 1).expand(-1, 1, 3)
|
||||
)
|
||||
|
||||
compressed_block_counts = [
|
||||
(seq_len // CR + COMP_PAGE - 1) // COMP_PAGE for seq_len in seq_lens
|
||||
]
|
||||
raw_block_table, num_raw_blocks = _make_block_table([1] * len(seq_lens))
|
||||
compressed_block_table, num_compressed_blocks = _make_block_table(
|
||||
compressed_block_counts
|
||||
)
|
||||
raw_block_table = raw_block_table.to(device)
|
||||
raw_slots = circular_qsa_slot_mapping(
|
||||
raw_block_table,
|
||||
token_to_req,
|
||||
logical_positions,
|
||||
state_size,
|
||||
query_start_loc,
|
||||
)
|
||||
compressed_slots = compressed_qsa_slot_mapping(
|
||||
compressed_block_table.to(device),
|
||||
token_to_req,
|
||||
logical_positions,
|
||||
COMP_PAGE,
|
||||
CR,
|
||||
)
|
||||
group_counts = torch.tensor(
|
||||
[
|
||||
seq_len // CR - (seq_len - query_len) // CR
|
||||
for seq_len, query_len in zip(seq_lens, query_lens)
|
||||
],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
k_work_counts = torch.maximum(group_counts, torch.ones_like(group_counts))
|
||||
k_start_loc = torch.cat([k_work_counts.new_zeros(1), k_work_counts.cumsum(0)])
|
||||
work_requests = torch.repeat_interleave(
|
||||
torch.arange(len(query_lens), dtype=torch.int32, device=device),
|
||||
k_work_counts,
|
||||
)
|
||||
local_work = torch.arange(
|
||||
int(k_start_loc[-1]), dtype=torch.int32, device=device
|
||||
) - torch.repeat_interleave(k_start_loc[:-1], k_work_counts)
|
||||
max_k_work = (num_tokens + (CR - 1) * len(query_lens)) // CR
|
||||
k_work_metadata = torch.full((max_k_work, 2), -1, dtype=torch.int32, device=device)
|
||||
k_work_metadata[: work_requests.numel()] = torch.stack(
|
||||
(work_requests, local_work), dim=1
|
||||
)
|
||||
|
||||
raw_width = D + 12 if cache_rope_positions else D
|
||||
# Match vLLM's padded-page cache layout: rows are contiguous, while physical
|
||||
# blocks have a larger stride than their logical contents.
|
||||
raw_page_elements = state_size * raw_width
|
||||
fused_raw_storage = torch.zeros(
|
||||
num_raw_blocks,
|
||||
raw_page_elements + 16,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
fused_raw = torch.as_strided(
|
||||
fused_raw_storage,
|
||||
(num_raw_blocks, state_size, 1, raw_width),
|
||||
(raw_page_elements + 16, raw_width, raw_width, 1),
|
||||
)
|
||||
for request, history_len in enumerate(history_lens):
|
||||
history_end = seq_lens[request] - query_lens[request]
|
||||
for position in range(history_end - history_len, history_end):
|
||||
block = int(raw_block_table[request, 0])
|
||||
row = fused_raw[block, position % state_size, 0]
|
||||
row[:D] = torch.randn(D, dtype=torch.bfloat16, device=device)
|
||||
if cache_rope_positions:
|
||||
row[ROPE_POS_OFFSET:].view(torch.int64).copy_(
|
||||
torch.tensor(
|
||||
[position, position // 7 + 3, position // 13 + 11],
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
)
|
||||
)
|
||||
unfused_raw = fused_raw.clone()
|
||||
compressed_page_elements = COMP_PAGE * D
|
||||
fused_compressed_storage = torch.zeros(
|
||||
num_compressed_blocks,
|
||||
compressed_page_elements + 16,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
fused_compressed = torch.as_strided(
|
||||
fused_compressed_storage,
|
||||
(num_compressed_blocks, COMP_PAGE, 1, D),
|
||||
(compressed_page_elements + 16, D, D, 1),
|
||||
)
|
||||
unfused_compressed = fused_compressed.clone()
|
||||
|
||||
projected_qk = torch.randn(
|
||||
num_tokens, (HQ + 1) * D, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
q_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
|
||||
k_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
|
||||
|
||||
fused_query = torch.empty(num_tokens, HQ, D, dtype=torch.bfloat16, device=device)
|
||||
qsa_pre_indexer(
|
||||
projected_qk[:, : HQ * D],
|
||||
projected_qk[:, HQ * D :],
|
||||
positions,
|
||||
rope.cos_sin_cache,
|
||||
q_weight,
|
||||
k_weight,
|
||||
EPS,
|
||||
fused_query,
|
||||
fused_raw,
|
||||
raw_slots,
|
||||
raw_block_table,
|
||||
query_start_loc,
|
||||
logical_positions,
|
||||
fused_compressed,
|
||||
compressed_slots,
|
||||
k_work_metadata,
|
||||
compress_ratio=CR,
|
||||
mrope_section=MROPE_SECTION if mrope else None,
|
||||
rope_pos_offset=ROPE_POS_OFFSET if cache_rope_positions else None,
|
||||
)
|
||||
|
||||
unfused_query = projected_qk[:, : HQ * D].reshape(num_tokens, HQ, D)
|
||||
unfused_query = gemma_rmsnorm(
|
||||
unfused_query.reshape(-1, D), q_weight, EPS
|
||||
).reshape_as(unfused_query)
|
||||
unfused_query = apply_qsa_rope(rope, positions, unfused_query)
|
||||
|
||||
raw_keys = unfused_raw[..., :D]
|
||||
rope_positions = (
|
||||
unfused_raw[..., ROPE_POS_OFFSET:].view(torch.int64)
|
||||
if cache_rope_positions
|
||||
else None
|
||||
)
|
||||
pooled, first_positions = qsa_compress_groups_with_ratio(
|
||||
projected_qk[:, HQ * D :].reshape(-1, 1, D),
|
||||
position_rows,
|
||||
raw_keys,
|
||||
raw_block_table,
|
||||
token_to_req,
|
||||
query_start_loc,
|
||||
logical_positions,
|
||||
compressed_slots,
|
||||
CR,
|
||||
rope_positions,
|
||||
)
|
||||
compressed_rows = gemma_rmsnorm(pooled.reshape(-1, D), k_weight, EPS).reshape(
|
||||
-1, 1, D
|
||||
)
|
||||
group_positions = (
|
||||
first_positions.transpose(0, 1) if mrope else first_positions[:, 0]
|
||||
)
|
||||
compressed_rows = apply_qsa_rope(rope, group_positions, compressed_rows)
|
||||
qsa_store_cache_rows(unfused_compressed, compressed_slots, compressed_rows)
|
||||
qsa_store_cache_rows(raw_keys, raw_slots, projected_qk[:, HQ * D :])
|
||||
if rope_positions is not None:
|
||||
qsa_store_cache_rows(rope_positions, raw_slots, position_rows)
|
||||
|
||||
torch.testing.assert_close(fused_query, unfused_query, rtol=RTOL, atol=ATOL)
|
||||
assert torch.equal(fused_raw.view(torch.int16), unfused_raw.view(torch.int16))
|
||||
torch.testing.assert_close(
|
||||
fused_compressed, unfused_compressed, rtol=RTOL, atol=ATOL
|
||||
)
|
||||
@@ -0,0 +1,973 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.qwen4_exp.common import qsa_cache
|
||||
from vllm.models.qwen4_exp.common.qsa_cache import QSAMetadataBuilder
|
||||
from vllm.models.qwen4_exp.nvidia import indexer_qsa
|
||||
from vllm.models.qwen4_exp.nvidia import (
|
||||
model as _qwen4_exp_model, # noqa: F401
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia.ops import qsa as qsa_ops
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
|
||||
requires_qsa_kernels = pytest.mark.skipif(
|
||||
not current_platform.is_cuda() or not HAS_TRITON,
|
||||
reason="QSA kernels require CUDA and Triton",
|
||||
)
|
||||
|
||||
|
||||
def test_qsa_mtp_index_share_updates_cache_but_skips_selection(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
rows = torch.tensor([[3, 1, -1], [5, 2, 0]], dtype=torch.int32)
|
||||
raw_metadata = SimpleNamespace(
|
||||
num_actual_tokens=2,
|
||||
slot_mapping=torch.arange(2),
|
||||
block_table=torch.empty(0),
|
||||
query_start_loc=torch.arange(3),
|
||||
logical_positions=torch.arange(2),
|
||||
)
|
||||
compressed_metadata = SimpleNamespace(
|
||||
num_actual_tokens=2,
|
||||
slot_mapping=torch.arange(2),
|
||||
k_work_metadata=torch.empty(0),
|
||||
)
|
||||
updates = []
|
||||
selections = []
|
||||
indexer = SimpleNamespace(
|
||||
skip_topk=True,
|
||||
_metadata=lambda: (raw_metadata, compressed_metadata),
|
||||
index_qk_proj=lambda hidden: (torch.zeros(2, 2), None),
|
||||
index_n_heads=1,
|
||||
index_kv_heads=1,
|
||||
index_head_dim=1,
|
||||
raw_key_cache=SimpleNamespace(
|
||||
kv_cache=torch.empty(0),
|
||||
rope_position_cache=None,
|
||||
rope_position_offset=0,
|
||||
),
|
||||
compressed_key_cache=SimpleNamespace(kv_cache=torch.empty(0)),
|
||||
use_fused_pre_indexer=True,
|
||||
rotary_emb=SimpleNamespace(cos_sin_cache=torch.empty(0)),
|
||||
q_layernorm=SimpleNamespace(weight=torch.ones(1), variance_epsilon=1e-6),
|
||||
k_layernorm=SimpleNamespace(weight=torch.ones(1)),
|
||||
compress_ratio=2,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
indexer_qsa,
|
||||
"qsa_pre_indexer",
|
||||
lambda *args, **kwargs: updates.append((args, kwargs)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
qsa_ops,
|
||||
"qsa_select_paged_tokens",
|
||||
lambda *args, **kwargs: selections.append((args, kwargs)),
|
||||
)
|
||||
|
||||
actual = indexer_qsa.QSAIndexer.forward(
|
||||
indexer,
|
||||
torch.zeros(2, 4),
|
||||
torch.tensor([7, 8]),
|
||||
rows,
|
||||
)
|
||||
|
||||
assert actual is rows
|
||||
assert len(updates) == 1
|
||||
assert not selections
|
||||
|
||||
|
||||
def _qsa_mqa_paged_reference(
|
||||
q: torch.Tensor,
|
||||
k_cache: torch.Tensor,
|
||||
page_table: torch.Tensor,
|
||||
token_to_req: torch.Tensor,
|
||||
visible_lengths: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
pages = page_table.index_select(0, token_to_req.long()).long()
|
||||
keys = k_cache[pages, :, 0, :].flatten(1, 2)
|
||||
scores = torch.einsum("rhd,rnd->rnh", q.float(), keys.float())
|
||||
logits = torch.relu(scores).sum(dim=-1) / math.sqrt(q.shape[-1])
|
||||
positions = torch.arange(keys.shape[1], device=q.device).unsqueeze(0)
|
||||
return logits.masked_fill(positions >= visible_lengths.unsqueeze(1), -torch.inf)
|
||||
|
||||
|
||||
def _qsa_relative_topk_reference(
|
||||
logits: torch.Tensor,
|
||||
row_starts: torch.Tensor,
|
||||
row_ends: torch.Tensor,
|
||||
topk: int,
|
||||
) -> torch.Tensor:
|
||||
output = torch.full(
|
||||
(logits.shape[0], topk), -1, dtype=torch.int32, device=logits.device
|
||||
)
|
||||
for row in range(logits.shape[0]):
|
||||
start = int(row_starts[row].item())
|
||||
length = int((row_ends[row] - row_starts[row]).item())
|
||||
width = min(length, topk)
|
||||
if width:
|
||||
output[row, :width] = torch.topk(
|
||||
logits[row, start : start + length], width
|
||||
).indices.to(torch.int32)
|
||||
return output
|
||||
|
||||
|
||||
def _expand_qsa_indices_reference(
|
||||
block_indices: torch.Tensor,
|
||||
query_positions: torch.Tensor,
|
||||
sequence_lengths: torch.Tensor,
|
||||
compress_ratio: int,
|
||||
token_topk: int,
|
||||
) -> torch.Tensor:
|
||||
rows = block_indices.shape[0]
|
||||
block_topk = token_topk // compress_ratio
|
||||
output_width = token_topk + compress_ratio - 1
|
||||
offsets = torch.arange(compress_ratio, device=block_indices.device)
|
||||
blocks = block_indices.long()
|
||||
expanded = blocks.unsqueeze(-1) * compress_ratio + offsets
|
||||
expanded = torch.where(
|
||||
blocks.unsqueeze(-1) >= 0, expanded, torch.full_like(expanded, -1)
|
||||
).reshape(rows, block_topk * compress_ratio)
|
||||
expanded = expanded[:, :token_topk]
|
||||
expanded = torch.where(
|
||||
(expanded >= 0) & (expanded < sequence_lengths.unsqueeze(1)),
|
||||
expanded,
|
||||
torch.full_like(expanded, -1),
|
||||
)
|
||||
|
||||
tail_offsets = torch.arange(compress_ratio - 1, device=block_indices.device)
|
||||
visible_tokens = query_positions + 1
|
||||
tail_start = visible_tokens // compress_ratio * compress_ratio
|
||||
tail = tail_start.unsqueeze(1) + tail_offsets.unsqueeze(0)
|
||||
tail_count = (visible_tokens - tail_start).unsqueeze(1)
|
||||
tail_valid = (tail_offsets.unsqueeze(0) < tail_count) & (
|
||||
tail < sequence_lengths.unsqueeze(1)
|
||||
)
|
||||
tail = torch.where(tail_valid, tail, torch.full_like(tail, -1))
|
||||
|
||||
result = torch.cat((expanded, tail), dim=1)
|
||||
order = torch.arange(output_width, device=result.device).expand(rows, -1)
|
||||
sort_key = torch.where(result >= 0, order, order + output_width)
|
||||
return result.gather(1, torch.argsort(sort_key, dim=1, stable=True)).to(torch.int32)
|
||||
|
||||
|
||||
def _qsa_select_paged_tokens_reference(
|
||||
q: torch.Tensor,
|
||||
k_cache: torch.Tensor,
|
||||
page_table: torch.Tensor,
|
||||
token_to_req: torch.Tensor,
|
||||
query_positions: torch.Tensor,
|
||||
sequence_lengths: torch.Tensor,
|
||||
token_topk: int,
|
||||
compress_ratio: int,
|
||||
) -> torch.Tensor:
|
||||
row_sequence_lengths = sequence_lengths.index_select(0, token_to_req.long())
|
||||
visible_blocks = torch.minimum(
|
||||
(query_positions + 1) // compress_ratio,
|
||||
row_sequence_lengths // compress_ratio,
|
||||
).to(torch.int32)
|
||||
logits = _qsa_mqa_paged_reference(
|
||||
q,
|
||||
k_cache,
|
||||
page_table,
|
||||
token_to_req,
|
||||
visible_blocks,
|
||||
)
|
||||
starts = torch.zeros_like(visible_blocks)
|
||||
blocks = _qsa_relative_topk_reference(
|
||||
logits,
|
||||
starts,
|
||||
visible_blocks,
|
||||
token_topk // compress_ratio,
|
||||
)
|
||||
return _expand_qsa_indices_reference(
|
||||
blocks,
|
||||
query_positions,
|
||||
row_sequence_lengths,
|
||||
compress_ratio,
|
||||
token_topk,
|
||||
)
|
||||
|
||||
|
||||
def _qsa_sparse_paged_attention_reference(
|
||||
q: torch.Tensor,
|
||||
k_cache: torch.Tensor,
|
||||
v_cache: torch.Tensor,
|
||||
logical_indices: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
token_to_req: torch.Tensor,
|
||||
softmax_scale: float,
|
||||
) -> torch.Tensor:
|
||||
output = torch.zeros_like(q)
|
||||
repeats = q.shape[1] // k_cache.shape[2]
|
||||
page_size = k_cache.shape[1]
|
||||
for row in range(q.shape[0]):
|
||||
logical = logical_indices[row]
|
||||
logical = logical[logical >= 0].long()
|
||||
if not logical.numel():
|
||||
continue
|
||||
request = token_to_req[row].long()
|
||||
pages = block_table[request, logical // page_size].long()
|
||||
offsets = logical % page_size
|
||||
keys = k_cache[pages, offsets].repeat_interleave(repeats, dim=1)
|
||||
values = v_cache[pages, offsets].repeat_interleave(repeats, dim=1)
|
||||
scores = torch.einsum("hd,khd->hk", q[row].float(), keys.float())
|
||||
probabilities = torch.softmax(scores * softmax_scale, dim=-1)
|
||||
output[row] = torch.einsum("hk,khd->hd", probabilities, values.float()).to(
|
||||
q.dtype
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
def test_qsa_side_metadata_marks_cudagraph_padding_inert() -> None:
|
||||
device = torch.device("cuda")
|
||||
builder = QSAMetadataBuilder.__new__(QSAMetadataBuilder)
|
||||
builder.compress_ratio = 1
|
||||
builder.is_circular_buffer = False
|
||||
builder.storage_block_size = 64
|
||||
builder.token_to_req_buffer = torch.empty(16, dtype=torch.int32, device=device)
|
||||
builder.slot_mapping_buffer = torch.empty(16, dtype=torch.int64, device=device)
|
||||
builder.logical_positions_buffer = torch.empty(16, dtype=torch.int64, device=device)
|
||||
builder.k_work_metadata_buffer = torch.empty(0, 2, dtype=torch.int32, device=device)
|
||||
query_start_loc = torch.tensor([0, 4, 8, 12, 12], dtype=torch.int32, device=device)
|
||||
token_to_req = torch.tensor([0] * 4 + [1] * 4 + [2] * 4 + [0] * 4, device=device)
|
||||
common = SimpleNamespace(
|
||||
num_actual_tokens=16,
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc.cpu(),
|
||||
seq_lens=torch.tensor([68, 68, 68, 0], dtype=torch.int32, device=device),
|
||||
slot_mapping=torch.tensor(list(range(12)) + [-1] * 4, device=device),
|
||||
block_table_tensor=torch.empty((4, 0), dtype=torch.int32, device=device),
|
||||
token_to_req_indices=lambda buffer: buffer.copy_(token_to_req),
|
||||
)
|
||||
|
||||
metadata = builder.build(0, common)
|
||||
|
||||
assert metadata.logical_positions.tolist() == [
|
||||
64,
|
||||
65,
|
||||
66,
|
||||
67,
|
||||
64,
|
||||
65,
|
||||
66,
|
||||
67,
|
||||
64,
|
||||
65,
|
||||
66,
|
||||
67,
|
||||
-1,
|
||||
-1,
|
||||
-1,
|
||||
-1,
|
||||
]
|
||||
assert metadata.slot_mapping.tolist() == list(range(12)) + [-1] * 4
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
def test_qsa_circular_buffer_metadata_keeps_only_each_requests_suffix() -> None:
|
||||
device = torch.device("cuda")
|
||||
builder = QSAMetadataBuilder.__new__(QSAMetadataBuilder)
|
||||
builder.compress_ratio = 4
|
||||
builder.is_circular_buffer = True
|
||||
builder.kv_cache_spec = SimpleNamespace(block_size=4)
|
||||
builder.storage_block_size = 4
|
||||
builder.token_to_req_buffer = torch.empty(16, dtype=torch.int32, device=device)
|
||||
builder.slot_mapping_buffer = torch.empty(16, dtype=torch.int64, device=device)
|
||||
builder.logical_positions_buffer = torch.empty(16, dtype=torch.int64, device=device)
|
||||
builder.k_work_metadata_buffer = torch.empty(0, 2, dtype=torch.int32, device=device)
|
||||
query_start_loc = torch.tensor([0, 7, 13, 13], dtype=torch.int32, device=device)
|
||||
token_to_req = torch.tensor([0] * 7 + [1] * 6 + [0] * 3, device=device)
|
||||
block_table = torch.tensor([[1], [0], [2]], dtype=torch.int32, device=device)
|
||||
common = SimpleNamespace(
|
||||
num_actual_tokens=16,
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc.cpu(),
|
||||
seq_lens=torch.tensor([9, 11, 0], dtype=torch.int32, device=device),
|
||||
slot_mapping=torch.full((16,), -1, dtype=torch.int64, device=device),
|
||||
block_table_tensor=block_table,
|
||||
token_to_req_indices=lambda buffer: buffer.copy_(token_to_req),
|
||||
)
|
||||
|
||||
metadata = builder.build(0, common)
|
||||
expected = [
|
||||
-1,
|
||||
-1,
|
||||
-1,
|
||||
5,
|
||||
6,
|
||||
7,
|
||||
4,
|
||||
-1,
|
||||
-1,
|
||||
3,
|
||||
0,
|
||||
1,
|
||||
2,
|
||||
-1,
|
||||
-1,
|
||||
-1,
|
||||
]
|
||||
|
||||
assert metadata.slot_mapping.tolist() == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("chunk_start", list(range(8)))
|
||||
def test_qsa_circular_buffer_survives_one_speculative_step(chunk_start: int) -> None:
|
||||
"""A speculative step must not overwrite the open group's committed keys.
|
||||
|
||||
The step stores every row it computes, drafts included, before acceptance
|
||||
is known, while the next step still reads the earlier members of the group
|
||||
being compressed from the ring. A ring sized at the compression ratio makes
|
||||
those rows alias, so a rejected draft silently replaces a committed key.
|
||||
"""
|
||||
compress_ratio = 4
|
||||
num_spec = 3
|
||||
capacity = compress_ratio * -(-(compress_ratio + num_spec) // compress_ratio)
|
||||
query_len = num_spec + 1
|
||||
|
||||
slots = qsa_cache.circular_qsa_slot_mapping(
|
||||
torch.tensor([[0]], dtype=torch.int32),
|
||||
torch.zeros(query_len, dtype=torch.int32),
|
||||
torch.arange(chunk_start, chunk_start + query_len),
|
||||
capacity,
|
||||
query_start_loc=torch.tensor([0, query_len], dtype=torch.int32),
|
||||
)
|
||||
|
||||
committed = torch.arange(chunk_start - chunk_start % compress_ratio, chunk_start)
|
||||
assert set(slots.tolist()).isdisjoint((committed % capacity).tolist())
|
||||
|
||||
|
||||
def _qsa_key_cache(block_size: int, compress_ratio: int) -> qsa_cache.QSAKeyStateCache:
|
||||
return qsa_cache.QSAKeyStateCache(
|
||||
head_size=64,
|
||||
dtype=torch.bfloat16,
|
||||
cache_config=SimpleNamespace(block_size=block_size),
|
||||
prefix=f"raw.{block_size}.{compress_ratio}",
|
||||
vllm_config=SimpleNamespace(
|
||||
compilation_config=SimpleNamespace(static_forward_context={})
|
||||
),
|
||||
compress_ratio=compress_ratio,
|
||||
)
|
||||
|
||||
|
||||
def test_qsa_state_caches_adapt_the_unified_logical_layout() -> None:
|
||||
raw_cache = _qsa_key_cache(block_size=32, compress_ratio=4)
|
||||
compressed_cache = qsa_cache.QSACompressedKeyCache(
|
||||
head_size=64,
|
||||
dtype=torch.bfloat16,
|
||||
cache_config=SimpleNamespace(block_size=32),
|
||||
prefix="compressed.bind",
|
||||
vllm_config=SimpleNamespace(
|
||||
compilation_config=SimpleNamespace(static_forward_context={})
|
||||
),
|
||||
compress_ratio=4,
|
||||
)
|
||||
raw_view = torch.empty(2, 1, 8, 64, dtype=torch.bfloat16)
|
||||
compressed_view = torch.empty(2, 1, 8, 64, dtype=torch.bfloat16)
|
||||
|
||||
raw_cache.bind_kv_cache(raw_view)
|
||||
compressed_cache.bind_kv_cache(compressed_view)
|
||||
|
||||
assert raw_cache.kv_cache.shape == (2, 8, 1, 64)
|
||||
assert compressed_cache.kv_cache.shape == (2, 8, 1, 64)
|
||||
assert raw_cache.kv_cache.data_ptr() == raw_view.data_ptr()
|
||||
assert compressed_cache.kv_cache.data_ptr() == compressed_view.data_ptr()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("compress_ratio", "num_spec", "expected"),
|
||||
[(4, 0, 4), (4, 1, 8), (4, 3, 8), (4, 4, 8), (4, 5, 12), (2, 3, 6)],
|
||||
)
|
||||
def test_qsa_ring_capacity_covers_one_speculative_step(
|
||||
compress_ratio: int, num_spec: int, expected: int
|
||||
) -> None:
|
||||
"""Capacity spans the open group plus one speculative step, in whole groups."""
|
||||
spec = _qsa_key_cache(
|
||||
block_size=48, compress_ratio=compress_ratio
|
||||
).get_kv_cache_spec(SimpleNamespace(num_speculative_tokens=num_spec))
|
||||
assert spec.block_size == expected
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
def test_qsa_compressed_metadata_keeps_dummy_slots_inert() -> None:
|
||||
device = torch.device("cuda")
|
||||
builder = QSAMetadataBuilder.__new__(QSAMetadataBuilder)
|
||||
builder.compress_ratio = 4
|
||||
builder.is_circular_buffer = False
|
||||
builder.storage_block_size = 16
|
||||
builder.token_to_req_buffer = torch.empty(8, dtype=torch.int32, device=device)
|
||||
builder.slot_mapping_buffer = torch.empty(8, dtype=torch.int64, device=device)
|
||||
builder.logical_positions_buffer = torch.empty(8, dtype=torch.int64, device=device)
|
||||
# Simulate max_num_seqs exceeding the three live requests below.
|
||||
builder.request_capacity = 8
|
||||
builder.k_work_metadata_buffer = torch.empty(4, 2, dtype=torch.int32, device=device)
|
||||
query_start_loc = torch.tensor([0, 3, 3, 8], dtype=torch.int32, device=device)
|
||||
token_to_req = torch.tensor(
|
||||
[0, 0, 0, 2, 2, 2, 2, 2], dtype=torch.int32, device=device
|
||||
)
|
||||
common = SimpleNamespace(
|
||||
num_actual_tokens=8,
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc.cpu(),
|
||||
seq_lens=torch.tensor([7, 0, 12], dtype=torch.int32, device=device),
|
||||
slot_mapping=torch.full((8,), -1, dtype=torch.int64, device=device),
|
||||
block_table_tensor=torch.zeros((3, 1), dtype=torch.int32, device=device),
|
||||
token_to_req_indices=lambda buffer: buffer.copy_(token_to_req),
|
||||
)
|
||||
|
||||
metadata = builder.build(0, common)
|
||||
|
||||
assert metadata.slot_mapping.tolist() == [-1] * 8
|
||||
assert metadata.k_work_metadata.tolist() == [[0, 0], [2, 0], [2, 1], [-1, -1]]
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
@pytest.mark.parametrize("compress_ratio", [1, 4])
|
||||
@pytest.mark.parametrize("num_reqs", [2, 3, 4, 7, 8, 9])
|
||||
def test_qsa_triton_metadata_matches_pytorch(
|
||||
compress_ratio: int, num_reqs: int
|
||||
) -> None:
|
||||
device = torch.device("cuda")
|
||||
num_tokens = 8
|
||||
query_start_loc = torch.tensor(
|
||||
[0, 3, *([3] * (num_reqs - 2)), 8], dtype=torch.int32, device=device
|
||||
)
|
||||
token_to_req = torch.tensor(
|
||||
[0, 0, 0, *([num_reqs - 1] * 5)],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
block_table_rows = torch.tensor(
|
||||
[
|
||||
[4, -1, 8, -1, 12, -1],
|
||||
[1, -1, 2, -1, 3, -1],
|
||||
[7, -1, 9, -1, 11, -1],
|
||||
],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
block_table_storage = block_table_rows[
|
||||
torch.arange(num_reqs, device=device) % block_table_rows.shape[0]
|
||||
]
|
||||
seq_lens = torch.zeros(num_reqs, dtype=torch.int32, device=device)
|
||||
seq_lens[0] = 10
|
||||
seq_lens[-1] = 20
|
||||
common = SimpleNamespace(
|
||||
num_actual_tokens=num_tokens,
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc.cpu(),
|
||||
seq_lens=seq_lens,
|
||||
slot_mapping=torch.tensor(
|
||||
[0, 1, -1, 3, 4, -1, -1, -1], dtype=torch.int64, device=device
|
||||
),
|
||||
block_table_tensor=block_table_storage[:, ::2],
|
||||
token_to_req_indices=lambda buffer: buffer.copy_(token_to_req),
|
||||
)
|
||||
|
||||
def make_buffers() -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
return (
|
||||
torch.empty(num_tokens, dtype=torch.int32, device=device),
|
||||
torch.empty(num_tokens, dtype=torch.int64, device=device),
|
||||
torch.empty(num_tokens, dtype=torch.int64, device=device),
|
||||
)
|
||||
|
||||
max_num_work = (
|
||||
(num_tokens + (compress_ratio - 1) * num_reqs) // compress_ratio
|
||||
if compress_ratio != 1
|
||||
else 0
|
||||
)
|
||||
actual_k_work = (
|
||||
torch.empty(max_num_work, 2, dtype=torch.int32, device=device)
|
||||
if max_num_work
|
||||
else None
|
||||
)
|
||||
actual_buffers = make_buffers()
|
||||
actual = qsa_cache.build_qsa_metadata_triton(
|
||||
common,
|
||||
*actual_buffers,
|
||||
storage_block_size=2,
|
||||
compress_ratio=compress_ratio,
|
||||
k_work_metadata_buffer=actual_k_work,
|
||||
request_capacity=num_reqs,
|
||||
)
|
||||
|
||||
expected_k_work = (
|
||||
torch.empty_like(actual_k_work) if actual_k_work is not None else None
|
||||
)
|
||||
expected_buffers = make_buffers()
|
||||
expected = qsa_cache._build_qsa_metadata_torch(
|
||||
common,
|
||||
*expected_buffers,
|
||||
storage_block_size=2,
|
||||
compress_ratio=compress_ratio,
|
||||
k_work_metadata_buffer=expected_k_work,
|
||||
request_capacity=num_reqs,
|
||||
)
|
||||
|
||||
for actual_tensor, expected_tensor in zip(actual, expected):
|
||||
torch.testing.assert_close(actual_tensor, expected_tensor)
|
||||
if actual_k_work is not None:
|
||||
torch.testing.assert_close(actual_k_work, expected_k_work)
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
def test_qsa_fused_metadata_matches_pytorch_for_large_padded_prefill() -> None:
|
||||
device = torch.device("cuda")
|
||||
num_mapped_tokens = 4096
|
||||
num_tokens = 4224
|
||||
query_start_loc = torch.tensor(
|
||||
[0, num_mapped_tokens], dtype=torch.int32, device=device
|
||||
)
|
||||
common = SimpleNamespace(
|
||||
num_actual_tokens=num_tokens,
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc.cpu(),
|
||||
seq_lens=torch.tensor(
|
||||
[num_mapped_tokens + 32], dtype=torch.int32, device=device
|
||||
),
|
||||
block_table_tensor=torch.arange(256, dtype=torch.int32, device=device)[None],
|
||||
slot_mapping=torch.tensor(
|
||||
[0] * num_mapped_tokens + [-1] * (num_tokens - num_mapped_tokens),
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
),
|
||||
token_to_req_indices=lambda buffer: buffer.zero_(),
|
||||
)
|
||||
|
||||
def make_buffers() -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
return (
|
||||
torch.empty(num_tokens, dtype=torch.int32, device=device),
|
||||
torch.empty(num_tokens, dtype=torch.int64, device=device),
|
||||
torch.empty(num_tokens, dtype=torch.int64, device=device),
|
||||
)
|
||||
|
||||
max_num_work = (num_tokens + 3) // 4
|
||||
actual_k_work = torch.empty(max_num_work, 2, dtype=torch.int32, device=device)
|
||||
expected_k_work = torch.empty_like(actual_k_work)
|
||||
actual = qsa_cache.build_qsa_metadata_triton(
|
||||
common,
|
||||
*make_buffers(),
|
||||
storage_block_size=8,
|
||||
compress_ratio=4,
|
||||
k_work_metadata_buffer=actual_k_work,
|
||||
)
|
||||
expected = qsa_cache._build_qsa_metadata_torch(
|
||||
common,
|
||||
*make_buffers(),
|
||||
storage_block_size=8,
|
||||
compress_ratio=4,
|
||||
k_work_metadata_buffer=expected_k_work,
|
||||
)
|
||||
|
||||
for actual_tensor, expected_tensor in zip(actual, expected):
|
||||
torch.testing.assert_close(actual_tensor, expected_tensor)
|
||||
torch.testing.assert_close(actual_k_work, expected_k_work)
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
@pytest.mark.parametrize(
|
||||
"num_rows",
|
||||
[
|
||||
pytest.param(3, id="one_tile_per_program"),
|
||||
pytest.param(33, id="looped_tiles"),
|
||||
],
|
||||
)
|
||||
def test_qsa_mqa_paged_matches_test_reference(num_rows: int) -> None:
|
||||
torch.manual_seed(1)
|
||||
head_dim = 128
|
||||
q = torch.randn(num_rows, 4, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
cache = torch.randn(40, 4, 1, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
page_table = torch.randperm(40, device="cuda", dtype=torch.int32).reshape(2, 20)
|
||||
request_rows = [(num_rows + 1) // 2, num_rows // 2]
|
||||
token_to_req = torch.repeat_interleave(
|
||||
torch.arange(2, device="cuda", dtype=torch.int32),
|
||||
torch.tensor(request_rows, device="cuda"),
|
||||
)
|
||||
sequence_length_values = [320, 264]
|
||||
sequence_lengths = torch.tensor(
|
||||
sequence_length_values, device="cuda", dtype=torch.int32
|
||||
)
|
||||
query_positions = torch.cat(
|
||||
[
|
||||
torch.arange(length - rows, length, device="cuda", dtype=torch.int32)
|
||||
for rows, length in zip(request_rows, sequence_length_values, strict=True)
|
||||
]
|
||||
)
|
||||
compress_ratio = 4
|
||||
visible_lengths = (query_positions + 1) // compress_ratio
|
||||
|
||||
actual, actual_visible_blocks = qsa_ops.qsa_mqa_paged(
|
||||
q,
|
||||
cache,
|
||||
page_table,
|
||||
token_to_req,
|
||||
query_positions,
|
||||
sequence_lengths,
|
||||
compress_ratio=compress_ratio,
|
||||
)
|
||||
expected = _qsa_mqa_paged_reference(
|
||||
q, cache, page_table, token_to_req, visible_lengths
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual_visible_blocks, visible_lengths)
|
||||
# Top-k is bounded by visible_blocks; later columns are intentionally unwritten.
|
||||
columns = torch.arange(actual.shape[1], device=actual.device)
|
||||
visible = columns[None, :] < actual_visible_blocks[:, None]
|
||||
torch.testing.assert_close(actual[visible], expected[visible], rtol=1e-3, atol=1e-3)
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
def test_qsa_block_expansion_matches_test_reference() -> None:
|
||||
blocks = torch.tensor([[0, -1], [1, 0]], device="cuda", dtype=torch.int32)
|
||||
query_positions = torch.tensor([5, 10], device="cuda")
|
||||
sequence_lengths = torch.tensor([6, 11], device="cuda")
|
||||
token_to_req = torch.tensor([0, 1], device="cuda", dtype=torch.int32)
|
||||
|
||||
actual = qsa_ops.expand_qsa_block_indices_cuda(
|
||||
blocks,
|
||||
query_positions,
|
||||
sequence_lengths,
|
||||
token_to_req,
|
||||
compress_ratio=4,
|
||||
token_topk=8,
|
||||
)
|
||||
expected = _expand_qsa_indices_reference(
|
||||
blocks,
|
||||
query_positions,
|
||||
sequence_lengths,
|
||||
compress_ratio=4,
|
||||
token_topk=8,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
@pytest.mark.parametrize(
|
||||
("num_rows", "num_query_heads", "num_kv_heads", "page_size"),
|
||||
[
|
||||
# Kernel-visible pages with --block-size 256 and hybrid-cache alignment.
|
||||
pytest.param(1, 24, 2, 1792, id="tp1_split64"),
|
||||
pytest.param(16, 12, 1, 1792, id="tp2_split32"),
|
||||
pytest.param(32, 6, 1, 1024, id="tp4_split8"),
|
||||
pytest.param(257, 6, 1, 1024, id="tp4_split4"),
|
||||
pytest.param(513, 6, 1, 1024, id="tp4_split1"),
|
||||
],
|
||||
)
|
||||
def test_qsa_sparse_paged_attention_matches_test_reference(
|
||||
num_rows: int,
|
||||
num_query_heads: int,
|
||||
num_kv_heads: int,
|
||||
page_size: int,
|
||||
) -> None:
|
||||
torch.manual_seed(2)
|
||||
head_dim = 256
|
||||
num_requests = 2
|
||||
num_selected_pages = 64
|
||||
# Keep the newest page outside the synthetic top-k as causal headroom.
|
||||
num_pages_per_request = num_selected_pages + 1
|
||||
num_cache_blocks = num_requests * num_pages_per_request
|
||||
indexer_budget = 2048
|
||||
indexer_compress_ratio = 4
|
||||
selection_width = indexer_budget + indexer_compress_ratio - 1
|
||||
q = torch.randn(
|
||||
num_rows, num_query_heads, head_dim, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
kv_cache = torch.randn(
|
||||
num_cache_blocks,
|
||||
page_size,
|
||||
num_kv_heads,
|
||||
2 * head_dim,
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
k_cache, v_cache = kv_cache.split(head_dim, dim=-1)
|
||||
block_table = (
|
||||
torch.randperm(num_cache_blocks, device="cuda")
|
||||
.reshape(num_requests, num_pages_per_request)
|
||||
.to(torch.int32)
|
||||
)
|
||||
rows_per_request = math.ceil(num_rows / num_requests)
|
||||
row_indices = torch.arange(num_rows, device="cuda", dtype=torch.int32)
|
||||
token_to_req = row_indices // rows_per_request
|
||||
request_row_counts = torch.tensor(
|
||||
[rows_per_request, num_rows - rows_per_request],
|
||||
device="cuda",
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
context_length = num_pages_per_request * page_size - 1
|
||||
block_topk = indexer_budget // indexer_compress_ratio
|
||||
compressed_blocks_per_page = page_size // indexer_compress_ratio
|
||||
selection = torch.arange(block_topk, device="cuda")
|
||||
selected_pages = selection % num_selected_pages
|
||||
selected_offsets = selection // num_selected_pages
|
||||
row_shifts = 2 * row_indices.unsqueeze(1)
|
||||
# Eight blocks per page; adjacent rows overlap by six of those eight.
|
||||
selected_offsets = (selected_offsets + row_shifts) % compressed_blocks_per_page
|
||||
block_indices = (selected_pages * compressed_blocks_per_page + selected_offsets).to(
|
||||
torch.int32
|
||||
)
|
||||
rows_within_request = row_indices % rows_per_request
|
||||
query_positions = (
|
||||
context_length - request_row_counts[token_to_req.long()] + rows_within_request
|
||||
).to(torch.int64)
|
||||
sequence_lengths = torch.full(
|
||||
(num_requests,), context_length, device="cuda", dtype=torch.int32
|
||||
)
|
||||
logical_indices = qsa_ops.expand_qsa_block_indices_cuda(
|
||||
block_indices,
|
||||
query_positions,
|
||||
sequence_lengths,
|
||||
token_to_req,
|
||||
indexer_compress_ratio,
|
||||
indexer_budget,
|
||||
)
|
||||
assert logical_indices.shape == (num_rows, selection_width)
|
||||
scale = q.shape[-1] ** -0.5
|
||||
|
||||
actual = qsa_ops.qsa_sparse_paged_attention(
|
||||
q,
|
||||
k_cache,
|
||||
v_cache,
|
||||
logical_indices,
|
||||
block_table,
|
||||
token_to_req,
|
||||
)
|
||||
expected = _qsa_sparse_paged_attention_reference(
|
||||
q,
|
||||
k_cache,
|
||||
v_cache,
|
||||
logical_indices,
|
||||
block_table,
|
||||
token_to_req,
|
||||
scale,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, rtol=2e-2, atol=2e-2)
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
def test_qsa_selection_chunks_workspace_and_matches_test_reference(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
workspace_init,
|
||||
) -> None:
|
||||
rows, keys, heads, head_dim = 65, 640, 4, 16
|
||||
token_topk, compress_ratio = 2048, 4
|
||||
torch.manual_seed(3)
|
||||
q = torch.randn(rows, heads, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
cache = torch.randn(40, 16, 1, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
page_table = torch.randperm(40, device="cuda", dtype=torch.int32).unsqueeze(0)
|
||||
token_to_req = torch.zeros(rows, device="cuda", dtype=torch.int32)
|
||||
query_positions = torch.full((rows,), 2559, device="cuda", dtype=torch.int32)
|
||||
sequence_lengths = torch.tensor([2560], device="cuda", dtype=torch.int32)
|
||||
monkeypatch.setattr(qsa_ops, "_LOGITS_WORKSPACE_BYTES", 32 * keys * 4)
|
||||
original_score = qsa_ops.qsa_mqa_paged
|
||||
scored_row_counts = []
|
||||
|
||||
def record_score(query: torch.Tensor, *args, **kwargs):
|
||||
scored_row_counts.append(query.shape[0])
|
||||
return original_score(query, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(qsa_ops, "qsa_mqa_paged", record_score)
|
||||
|
||||
actual = qsa_ops.qsa_select_paged_tokens(
|
||||
q,
|
||||
cache,
|
||||
page_table,
|
||||
token_to_req,
|
||||
query_positions,
|
||||
sequence_lengths,
|
||||
token_topk,
|
||||
compress_ratio,
|
||||
)
|
||||
expected = _qsa_select_paged_tokens_reference(
|
||||
q,
|
||||
cache,
|
||||
page_table,
|
||||
token_to_req,
|
||||
query_positions,
|
||||
sequence_lengths,
|
||||
token_topk,
|
||||
compress_ratio,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual.sort().values, expected.sort().values)
|
||||
assert scored_row_counts == [32, 32, 1]
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
def test_qsa_selection_handles_no_complete_compressed_blocks(workspace_init) -> None:
|
||||
q = torch.zeros(2, 4, 8, device="cuda", dtype=torch.bfloat16)
|
||||
cache = torch.zeros(1, 16, 1, 8, device="cuda", dtype=torch.bfloat16)
|
||||
page_table = torch.zeros(1, 1, device="cuda", dtype=torch.int32)
|
||||
token_to_req = torch.zeros(2, device="cuda", dtype=torch.int32)
|
||||
query_positions = torch.tensor([1, 2], device="cuda", dtype=torch.int32)
|
||||
sequence_lengths = torch.tensor([3], device="cuda", dtype=torch.int32)
|
||||
|
||||
selected = qsa_ops.qsa_select_paged_tokens(
|
||||
q,
|
||||
cache,
|
||||
page_table,
|
||||
token_to_req,
|
||||
query_positions,
|
||||
sequence_lengths,
|
||||
token_topk=2048,
|
||||
compress_ratio=4,
|
||||
)
|
||||
|
||||
assert selected[0, :2].tolist() == [0, 1]
|
||||
assert selected[1, :3].tolist() == [0, 1, 2]
|
||||
assert torch.all(selected[0, 2:] == -1)
|
||||
assert torch.all(selected[1, 3:] == -1)
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
def test_qsa_streaming_compression_and_compressor_state_store_match_reference() -> None:
|
||||
head_dim = 8
|
||||
current_pairs = [
|
||||
*((0, position) for position in range(2, 9)),
|
||||
*((1, position) for position in range(5, 11)),
|
||||
]
|
||||
|
||||
def key_row(request: int, position: int) -> torch.Tensor:
|
||||
return (
|
||||
torch.arange(head_dim, dtype=torch.float32) + request * 1000 + position * 10
|
||||
)
|
||||
|
||||
def position_row(request: int, position: int) -> torch.Tensor:
|
||||
return torch.tensor(
|
||||
[
|
||||
request * 1000 + position,
|
||||
request * 1000 + position + 100,
|
||||
request * 1000 + position + 200,
|
||||
],
|
||||
dtype=torch.int64,
|
||||
)
|
||||
|
||||
raw_keys = (
|
||||
torch.stack([key_row(request, position) for request, position in current_pairs])
|
||||
.unsqueeze(1)
|
||||
.to(device="cuda", dtype=torch.bfloat16)
|
||||
)
|
||||
raw_positions = (
|
||||
torch.stack(
|
||||
[position_row(request, position) for request, position in current_pairs]
|
||||
)
|
||||
.unsqueeze(1)
|
||||
.to(device="cuda")
|
||||
)
|
||||
token_to_req = torch.tensor(
|
||||
[request for request, _ in current_pairs],
|
||||
dtype=torch.int32,
|
||||
device="cuda",
|
||||
)
|
||||
logical_positions = torch.tensor(
|
||||
[position for _, position in current_pairs],
|
||||
dtype=torch.int64,
|
||||
device="cuda",
|
||||
)
|
||||
query_start_loc = torch.tensor([0, 7, 13], dtype=torch.int32, device="cuda")
|
||||
compressor_state_block_table = torch.tensor(
|
||||
[[1], [0]], dtype=torch.int32, device="cuda"
|
||||
)
|
||||
compressor_state_cache = torch.zeros(
|
||||
2, 4, 1, head_dim, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
rope_cache = torch.zeros(2, 4, 1, 3, dtype=torch.int64, device="cuda")
|
||||
for request, position, block in ((0, 0, 1), (0, 1, 1), (1, 4, 0)):
|
||||
compressor_state_cache[block, position % 4, 0] = key_row(request, position).to(
|
||||
device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
rope_cache[block, position % 4, 0] = position_row(request, position).to("cuda")
|
||||
|
||||
compressed_slots = torch.full(
|
||||
(len(current_pairs),), -1, dtype=torch.int64, device="cuda"
|
||||
)
|
||||
valid_rows = torch.tensor([1, 5, 9], dtype=torch.int64, device="cuda")
|
||||
compressed_slots[valid_rows] = torch.arange(3, device="cuda")
|
||||
pooled, first_positions = qsa_ops.qsa_compress_groups_with_ratio(
|
||||
raw_keys,
|
||||
raw_positions,
|
||||
compressor_state_cache,
|
||||
compressor_state_block_table,
|
||||
token_to_req,
|
||||
query_start_loc,
|
||||
logical_positions,
|
||||
compressed_slots,
|
||||
compress_ratio=4,
|
||||
rope_cache=rope_cache,
|
||||
)
|
||||
pooled_without_rope, scalar_first_positions = (
|
||||
qsa_ops.qsa_compress_groups_with_ratio(
|
||||
raw_keys,
|
||||
raw_positions,
|
||||
compressor_state_cache,
|
||||
compressor_state_block_table,
|
||||
token_to_req,
|
||||
query_start_loc,
|
||||
logical_positions,
|
||||
compressed_slots,
|
||||
compress_ratio=4,
|
||||
)
|
||||
)
|
||||
|
||||
groups = [
|
||||
[(0, position) for position in range(0, 4)],
|
||||
[(0, position) for position in range(4, 8)],
|
||||
[(1, position) for position in range(4, 8)],
|
||||
]
|
||||
expected_pooled = (
|
||||
torch.stack(
|
||||
[
|
||||
torch.stack([key_row(*pair) for pair in group]).mean(dim=0)
|
||||
for group in groups
|
||||
]
|
||||
)
|
||||
.unsqueeze(1)
|
||||
.to(device="cuda", dtype=torch.bfloat16)
|
||||
)
|
||||
expected_positions = torch.stack(
|
||||
[position_row(0, 0), position_row(0, 4), position_row(1, 4)]
|
||||
).to("cuda")
|
||||
expected_scalar_positions = torch.tensor(
|
||||
[[0, 0, 0], [4, 4, 4], [4, 4, 4]],
|
||||
dtype=torch.int64,
|
||||
device="cuda",
|
||||
)
|
||||
|
||||
torch.testing.assert_close(pooled[valid_rows], expected_pooled)
|
||||
torch.testing.assert_close(pooled_without_rope[valid_rows], expected_pooled)
|
||||
torch.testing.assert_close(first_positions[valid_rows], expected_positions)
|
||||
torch.testing.assert_close(
|
||||
scalar_first_positions[valid_rows], expected_scalar_positions
|
||||
)
|
||||
|
||||
compressor_state_slots = torch.tensor(
|
||||
[-1, -1, -1, 5, 6, 7, 4, -1, -1, 3, 0, 1, 2],
|
||||
dtype=torch.int64,
|
||||
device="cuda",
|
||||
)
|
||||
qsa_ops.qsa_store_cache_rows(
|
||||
compressor_state_cache, compressor_state_slots, raw_keys
|
||||
)
|
||||
qsa_ops.qsa_store_cache_rows(rope_cache, compressor_state_slots, raw_positions)
|
||||
for request, positions, block in ((0, range(5, 9), 1), (1, range(7, 11), 0)):
|
||||
for position in positions:
|
||||
torch.testing.assert_close(
|
||||
compressor_state_cache[block, position % 4, 0],
|
||||
key_row(request, position).to(device="cuda", dtype=torch.bfloat16),
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
rope_cache[block, position % 4, 0],
|
||||
position_row(request, position).to("cuda"),
|
||||
)
|
||||
@@ -508,6 +508,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
extras={"native-prefix": "imdatta0/small_qwen3_5_20b"},
|
||||
max_model_len=4096,
|
||||
),
|
||||
"Qwen4ExpForCausalLM": _HfExamplesInfo("", is_available_online=False),
|
||||
"MellumForCausalLM": _HfExamplesInfo("JetBrains/Mellum2-12B-A2.5B-Base"),
|
||||
"Qwen3NextForCausalLM": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-Next-80B-A3B-Instruct",
|
||||
@@ -1348,6 +1349,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"Qwen/Qwen3.5-35B-A3B",
|
||||
max_model_len=4096,
|
||||
),
|
||||
"Qwen4ExpForConditionalGeneration": _HfExamplesInfo(
|
||||
"",
|
||||
is_available_online=False,
|
||||
),
|
||||
"Qwen3OmniMoeForConditionalGeneration": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
max_model_len=4096,
|
||||
@@ -1797,6 +1802,7 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
"Qwen/Qwen3.5-35B-A3B",
|
||||
speculative_model="Qwen/Qwen3.5-35B-A3B",
|
||||
),
|
||||
"Qwen4ExpMTP": _HfExamplesInfo("", is_available_online=False),
|
||||
"Step3p5MTP": _HfExamplesInfo(
|
||||
"stepfun-ai/Step-3.5-Flash",
|
||||
speculative_model="stepfun-ai/Step-3.5-Flash",
|
||||
|
||||
Reference in New Issue
Block a user