Merge remote-tracking branch 'upstream/main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-03 21:59:03 +07:00
1012 changed files with 61873 additions and 14499 deletions
+154 -102
View File
@@ -494,8 +494,10 @@ def _sd_conv_states(
return storage.transpose(1, 2)
def _maybe_pack_conv_weight(weight: torch.Tensor, is_vnni: bool) -> torch.Tensor:
return ops.causal_conv1d_weight_pack(weight) if is_vnni else weight
def _maybe_pack_conv_weight(
weight: torch.Tensor, is_weight_packed: bool
) -> torch.Tensor:
return ops.causal_conv1d_weight_pack(weight) if is_weight_packed else weight
@torch.inference_mode()
@@ -523,7 +525,7 @@ def test_causal_conv1d_update_cpu_preserves_sd_state(
bias=None,
silu_activation=False,
conv_state_indices=state_indices,
is_vnni=False,
is_weight_packed=False,
)
assert out is x
@@ -666,7 +668,10 @@ def test_spec_aware_nonspec_materializes_state_indices(
@torch.inference_mode()
@pytest.mark.parametrize(("has_amx", "has_arm_bf16"), [(True, False), (False, True)])
def test_spec_forward_prepares_native_conv_metadata(
has_amx: bool,
has_arm_bf16: bool,
monkeypatch: pytest.MonkeyPatch,
) -> None:
block_table = torch.tensor([[0, 1], [4, 5]], dtype=torch.int32)
@@ -687,14 +692,17 @@ def test_spec_forward_prepares_native_conv_metadata(
)
forwarded_indices = None
forwarded_counts = None
forwarded_is_weight_packed = None
def causal_conv1d_update_cpu(**kwargs):
nonlocal forwarded_counts, forwarded_indices
nonlocal forwarded_counts, forwarded_indices, forwarded_is_weight_packed
forwarded_indices = kwargs["conv_state_indices"]
forwarded_counts = kwargs["num_accepted_tokens"]
forwarded_is_weight_packed = kwargs["is_weight_packed"]
return kwargs["x"]
monkeypatch.setattr(torch.cpu, "_is_amx_tile_supported", lambda: True)
monkeypatch.setattr(torch.cpu, "_is_amx_tile_supported", lambda: has_amx)
monkeypatch.setattr(gdn_attention, "is_arm_bf16", lambda: has_arm_bf16)
monkeypatch.setattr(gdn_attention, "is_conv_state_dim_first", lambda: False)
monkeypatch.setattr(
gdn_attention.ops, "causal_conv1d_update_cpu", causal_conv1d_update_cpu
@@ -732,6 +740,7 @@ def test_spec_forward_prepares_native_conv_metadata(
assert actual is not None
assert actual.is_contiguous()
assert actual.dtype == torch.int32
assert forwarded_is_weight_packed is True
torch.testing.assert_close(actual, reference)
@@ -792,15 +801,15 @@ def test_causal_conv1d_torch_two_call_split(total_tokens: int, split: int) -> No
@pytest.mark.skipif(
not torch.cpu._is_amx_tile_supported(),
reason="requires AMX support",
not (torch.cpu._is_amx_tile_supported() or gdn_attention.is_arm_bf16()),
reason="native causal conv1d requires AMX or Arm BF16 support",
)
@torch.inference_mode()
def test_causal_conv1d_update_cpu_accepts_wide_state() -> None:
state_len = CONV_KERNEL - 1
wide_state_len = state_len + 5
batch_size = 3
is_vnni = True
is_weight_packed = True
x, weight, bias = _conv_inputs(batch_size)
conv_state_indices = torch.tensor([2, 0, 1], dtype=torch.int32)
@@ -813,7 +822,7 @@ def test_causal_conv1d_update_cpu_accepts_wide_state() -> None:
wide_state[:, :, state_len:].fill_(7)
wide_tail = wide_state[:, :, state_len:].clone()
conv_weight = _maybe_pack_conv_weight(weight, is_vnni)
conv_weight = _maybe_pack_conv_weight(weight, is_weight_packed)
out_narrow = ops.causal_conv1d_update_cpu(
x=x,
conv_states=narrow_state,
@@ -821,7 +830,7 @@ def test_causal_conv1d_update_cpu_accepts_wide_state() -> None:
bias=bias,
silu_activation=True,
conv_state_indices=conv_state_indices,
is_vnni=is_vnni,
is_weight_packed=is_weight_packed,
)
out_wide = ops.causal_conv1d_update_cpu(
x=x,
@@ -830,7 +839,7 @@ def test_causal_conv1d_update_cpu_accepts_wide_state() -> None:
bias=bias,
silu_activation=True,
conv_state_indices=conv_state_indices,
is_vnni=is_vnni,
is_weight_packed=is_weight_packed,
)
torch.testing.assert_close(out_wide, out_narrow, atol=1e-2, rtol=1e-2)
@@ -872,11 +881,18 @@ def _ref_causal_conv1d_update_cpu_multi(
@pytest.mark.skipif(
not torch.cpu._is_amx_tile_supported(),
reason="requires AMX support",
not (torch.cpu._is_amx_tile_supported() or gdn_attention.is_arm_bf16()),
reason="native causal conv1d requires AMX or Arm BF16 support",
)
@pytest.mark.parametrize(
("batch_size, seq_len, accepted_counts, has_bias, silu_activation, is_vnni"),
(
"batch_size, "
"seq_len, "
"accepted_counts, "
"has_bias, "
"silu_activation, "
"is_weight_packed"
),
[
(1, 1, [1], False, False, False),
(1, 1, [1], True, True, True),
@@ -893,7 +909,7 @@ def test_causal_conv1d_update_cpu_multi_token_matches_python(
accepted_counts: list[int],
has_bias: bool,
silu_activation: bool,
is_vnni: bool,
is_weight_packed: bool,
) -> None:
dim = 96
state_len = seq_len + 2
@@ -911,7 +927,7 @@ def test_causal_conv1d_update_cpu_multi_token_matches_python(
)
conv_states = conv_states_ref.clone()
conv_weight = _maybe_pack_conv_weight(weight, is_vnni)
conv_weight = _maybe_pack_conv_weight(weight, is_weight_packed)
out = ops.causal_conv1d_update_cpu(
x=x,
conv_states=conv_states,
@@ -919,7 +935,7 @@ def test_causal_conv1d_update_cpu_multi_token_matches_python(
bias=bias,
silu_activation=silu_activation,
conv_state_indices=conv_state_indices,
is_vnni=is_vnni,
is_weight_packed=is_weight_packed,
num_accepted_tokens=num_accepted_tokens,
)
ref_out = _ref_causal_conv1d_update_cpu_multi(
@@ -937,8 +953,8 @@ def test_causal_conv1d_update_cpu_multi_token_matches_python(
@pytest.mark.skipif(
not torch.cpu._is_amx_tile_supported(),
reason="requires AMX support",
not (torch.cpu._is_amx_tile_supported() or gdn_attention.is_arm_bf16()),
reason="native causal conv1d requires AMX or Arm BF16 support",
)
@pytest.mark.parametrize("num_accepted", [0, 17])
@torch.inference_mode()
@@ -961,14 +977,15 @@ def test_causal_conv1d_update_cpu_rejects_invalid_accepted_count(
bias=None,
silu_activation=True,
conv_state_indices=torch.tensor([0], dtype=torch.int32),
is_vnni=False,
is_weight_packed=False,
num_accepted_tokens=torch.tensor([num_accepted], dtype=torch.int32),
)
@pytest.mark.skipif(
not torch.cpu._is_avx512_bf16_supported(),
reason="causal_conv1d_fwd_cpu requires AVX-512BF16 (Intel Xeon or AMD EPYC)",
not (torch.cpu._is_avx512_bf16_supported() or gdn_attention.is_arm_bf16()),
reason="causal_conv1d_fwd_cpu requires AVX-512BF16"
" (Intel Xeon or AMD EPYC) or Arm BF16 support",
)
@pytest.mark.parametrize("total_tokens, split", TWO_CALL_SPLITS)
@torch.inference_mode()
@@ -977,13 +994,13 @@ def test_causal_conv1d_fwd_cpu_two_call_split(total_tokens: int, split: int) ->
matches the single-call result.
Regression test for ``causal_conv1d_fwd_varlen_kernel_impl`` (``conv.cpp``)
ignoring the carried conv state on continued chunks. Runs on any
AVX-512BF16 CPU since conv.cpp uses VDPBF16PS, not AMX tiles.
ignoring the carried conv state on continued chunks. Runs on aarch64 cpus with
bf16 support or any AVX-512BF16 CPU since conv.cpp uses VDPBF16PS, not AMX tiles.
"""
state_len = CONV_KERNEL - 1
x, weight, bias = _conv_inputs(total_tokens)
def amx(x_seg, conv_states, has_init):
def native(x_seg, conv_states, has_init):
seq = x_seg.shape[0]
return ops.causal_conv1d_fwd_cpu(
x=x_seg.transpose(0, 1), # [dim, seq]; stride(-2)==1 (view of [seq,dim])
@@ -994,30 +1011,30 @@ def test_causal_conv1d_fwd_cpu_two_call_split(total_tokens: int, split: int) ->
cache_indices=torch.tensor([0], dtype=torch.int32),
has_initial_state=torch.tensor([has_init]),
silu_activation=True,
is_vnni=False,
is_weight_packed=False,
).contiguous()
# conv_state layout passed by the AMX branch: [num_slots, dim, state_len].
cs_full = torch.zeros(1, CONV_DIM, state_len, dtype=x.dtype)
out_full = amx(x, cs_full, False)
out_full = native(x, cs_full, False)
cs_split = torch.zeros(1, CONV_DIM, state_len, dtype=x.dtype)
out1 = amx(x[:split], cs_split, False)
out2 = amx(x[split:], cs_split, True)
out1 = native(x[:split], cs_split, False)
out2 = native(x[split:], cs_split, True)
out_split = torch.cat([out1, out2], dim=1)
torch.testing.assert_close(out_split, out_full, atol=1e-2, rtol=1e-2)
@pytest.mark.skipif(
not torch.cpu._is_amx_tile_supported(),
reason="requires AMX support",
not (torch.cpu._is_amx_tile_supported() or gdn_attention.is_arm_bf16()),
reason="native causal conv1d requires AMX or Arm BF16 support",
)
@torch.inference_mode()
def test_causal_conv1d_fwd_cpu_accepts_wide_state() -> None:
state_len = CONV_KERNEL - 1
wide_state_len = state_len + 5
is_vnni = True
is_weight_packed = True
seq_lens = [CHUNK_SIZE - 1, CHUNK_SIZE + 5]
total_tokens = sum(seq_lens)
x, weight, bias = _conv_inputs(total_tokens)
@@ -1034,7 +1051,7 @@ def test_causal_conv1d_fwd_cpu_accepts_wide_state() -> None:
wide_state[:, :, state_len:].fill_(7)
wide_tail = wide_state[:, :, state_len:].clone()
conv_weight = _maybe_pack_conv_weight(weight, is_vnni)
conv_weight = _maybe_pack_conv_weight(weight, is_weight_packed)
out_narrow = ops.causal_conv1d_fwd_cpu(
x=x.transpose(0, 1),
weight=conv_weight,
@@ -1044,7 +1061,7 @@ def test_causal_conv1d_fwd_cpu_accepts_wide_state() -> None:
cache_indices=cache_indices,
has_initial_state=has_initial_state,
silu_activation=True,
is_vnni=is_vnni,
is_weight_packed=is_weight_packed,
)
out_wide = ops.causal_conv1d_fwd_cpu(
x=x.transpose(0, 1),
@@ -1055,7 +1072,7 @@ def test_causal_conv1d_fwd_cpu_accepts_wide_state() -> None:
cache_indices=cache_indices,
has_initial_state=has_initial_state,
silu_activation=True,
is_vnni=is_vnni,
is_weight_packed=is_weight_packed,
)
torch.testing.assert_close(out_wide, out_narrow, atol=1e-2, rtol=1e-2)
@@ -1100,15 +1117,23 @@ def test_batch_memcpy_cpu_fallback() -> None:
# ---------------------------------------------------------------------------
# C++ conv (conv.cpp) uses VDPBF16PS, not AMX tiles, so it runs on any
# AVX-512BF16 CPU; weight is VNNI-packed on this same predicate at load time.
# C++ conv (conv.cpp) uses AArch64BF16 or VDPBF16PS, not AMX tiles, so it can
# run on any AVX-512BF16 CPU; weight is packed on this same predicate at load time.
# ---------------------------------------------------------------------------
_HAS_AVX512_BF16 = torch.cpu._is_avx512_bf16_supported()
_STATE_LEN = CONV_KERNEL - 1
CONV_EQUIV_SEQ_LENS = [[1], [7], [64], [65], [1, 2, 3], [63, 64, 65], [128, 129]]
CONV_EQUIV_SEQ_LENS = [
[1],
[7],
[64],
[65],
[1, 2, 3],
[63, 64, 65],
[128, 129],
]
def _conv_fp32_oracle(x, weight, bias, seq_lens, activation="silu"):
@@ -1153,10 +1178,10 @@ def _run_prefill_torch(x, weight, bias, seq_lens):
return out.transpose(0, 1).contiguous(), conv_states
def _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=False):
def _run_prefill_cpp(x, weight, bias, seq_lens, is_weight_packed=False):
num_seqs = len(seq_lens)
packed_w = ops.causal_conv1d_weight_pack(weight) if is_vnni else weight
if is_vnni:
packed_w = ops.causal_conv1d_weight_pack(weight) if is_weight_packed else weight
if is_weight_packed:
# C++-branch layout: kv-cache "SD" [slots, state_len, dim] transposed to
# [slots, dim, state_len] (a non-contiguous view).
conv_state = torch.zeros(
@@ -1176,7 +1201,7 @@ def _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=False):
cache_indices=torch.arange(num_seqs, dtype=torch.int32),
has_initial_state=torch.zeros(num_seqs, dtype=torch.bool),
silu_activation=True,
is_vnni=is_vnni,
is_weight_packed=is_weight_packed,
)
return out.transpose(0, 1).contiguous(), conv_state
@@ -1245,7 +1270,7 @@ def test_causal_conv1d_fwd_cpu_ds_adapter_varlen_mixed_initial_state(
cache_indices=state_indices,
has_initial_state=has_initial_state,
silu_activation=True,
is_vnni=False,
is_weight_packed=False,
)
monkeypatch.setattr(gdn_attention, "is_conv_state_dim_first", lambda: True)
out_ds = gdn_attention._causal_conv1d_fwd_cpu(
@@ -1257,7 +1282,7 @@ def test_causal_conv1d_fwd_cpu_ds_adapter_varlen_mixed_initial_state(
cache_indices=state_indices,
has_initial_state=has_initial_state,
silu_activation=True,
is_vnni=False,
is_weight_packed=False,
)
torch.testing.assert_close(out_ds, out_sd)
@@ -1302,7 +1327,7 @@ def test_causal_conv1d_update_cpu_ds_adapter_wide_multi_token_copyback(
bias=None,
silu_activation=False,
conv_state_indices=state_indices,
is_vnni=False,
is_weight_packed=False,
num_accepted_tokens=num_accepted_tokens,
)
@@ -1321,7 +1346,8 @@ def test_causal_conv1d_update_cpu_ds_adapter_wide_multi_token_copyback(
@pytest.mark.skipif(
not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16"
not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()),
reason="C++ causal_conv1d requires AVX-512BF16 or Arm BF16 support",
)
@pytest.mark.parametrize("seq_lens", CONV_EQUIV_SEQ_LENS)
@torch.inference_mode()
@@ -1335,7 +1361,8 @@ def test_conv_cpp_matches_torch(seq_lens):
@pytest.mark.skipif(
not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16"
not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()),
reason="C++ causal_conv1d requires AVX-512BF16 or Arm BF16 support",
)
@pytest.mark.parametrize("seq_lens", CONV_EQUIV_SEQ_LENS)
@torch.inference_mode()
@@ -1354,89 +1381,114 @@ def test_conv_cpp_no_worse_than_torch_vs_fp32(seq_lens):
@pytest.mark.skipif(
not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16"
not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()),
reason="C++ causal_conv1d requires AVX-512BF16 or Arm BF16 support",
)
@pytest.mark.parametrize("seq_lens", CONV_EQUIV_SEQ_LENS)
@torch.inference_mode()
def test_conv_cpp_vnni_packed_matches_torch(seq_lens):
"""The exact runtime prefill sequence (VNNI-packed weight + SD-layout
conv_state view + is_vnni=True) must match the torch fallback. Validates
the packing + layout handoff on any AVX-512BF16 CPU."""
def test_conv_cpp_weight_packed_matches_torch(seq_lens):
"""The exact runtime prefill sequence (packed weight + SD-layout
conv_state view + is_weight_packed=True) must match the torch fallback. Validates
the native packing + layout handoff."""
x, weight, bias = _conv_inputs(sum(seq_lens))
out_torch, _ = _run_prefill_torch(x, weight, bias, seq_lens)
out_vnni, _ = _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=True)
torch.testing.assert_close(out_vnni, out_torch, atol=1e-2, rtol=1e-2)
out_weight_packed, _ = _run_prefill_cpp(
x, weight, bias, seq_lens, is_weight_packed=True
)
torch.testing.assert_close(out_weight_packed, out_torch, atol=1e-2, rtol=1e-2)
@pytest.mark.skipif(
not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16"
not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()),
reason="C++ causal_conv1d requires AVX-512BF16 or Arm BF16 support",
)
@pytest.mark.parametrize("batch", DECODE_BATCH_SIZES)
@pytest.mark.parametrize(
("dim", "has_bias", "silu_activation", "is_weight_packed"),
[
(32, False, False, False),
(32, True, True, True),
(96, False, True, True),
(96, True, False, False),
(8192, False, True, False),
(8192, True, True, True),
],
)
@torch.inference_mode()
def test_conv_update_cpp_matches_torch(batch):
"""Decode conv: causal_conv1d_update_cpu matches causal_conv1d_update_torch,
including the in-place conv_state update (the next-step handoff)."""
def test_conv_update_cpp_matches_torch(
batch, dim, has_bias, silu_activation, is_weight_packed
):
"""Decode preserves indexed, padded state across successive signed inputs."""
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
causal_conv1d_update_torch,
)
x = tensor_cache(batch * CONV_DIM, torch.bfloat16).view(batch, CONV_DIM)
weight = tensor_cache(CONV_DIM * CONV_KERNEL, torch.bfloat16).view(
CONV_DIM, CONV_KERNEL
)
bias = tensor_cache(CONV_DIM, torch.bfloat16)
conv_state = tensor_cache(batch * CONV_DIM * _STATE_LEN, torch.bfloat16).view(
batch, CONV_DIM, _STATE_LEN
)
torch.manual_seed(0)
weight = torch.randn(dim, CONV_KERNEL, dtype=torch.bfloat16)
bias = torch.randn(dim, dtype=torch.bfloat16) if has_bias else None
conv_weight = _maybe_pack_conv_weight(weight, is_weight_packed)
indices = torch.arange(batch, 0, -1, dtype=torch.int32)
storage_ref = torch.randn(batch + 2, _STATE_LEN + 2, dim, dtype=torch.bfloat16)
storage_cpp = storage_ref.clone()
cs_cpp = storage_cpp[:, :_STATE_LEN].transpose(1, 2)
cs_torch = conv_state.clone()
out_torch = causal_conv1d_update_torch(
x=x.unsqueeze(-1),
conv_state=cs_torch,
weight=weight,
bias=bias,
activation="silu",
).squeeze(-1)
cs_cpp = conv_state.clone()
out_cpp = ops.causal_conv1d_update_cpu(
x=x.contiguous(),
conv_states=cs_cpp,
weight=weight,
bias=bias,
silu_activation=True,
conv_state_indices=torch.arange(batch, dtype=torch.int32),
is_vnni=False,
)
torch.testing.assert_close(out_cpp, out_torch, atol=1e-2, rtol=1e-2)
torch.testing.assert_close(cs_cpp, cs_torch, atol=1e-2, rtol=1e-2)
for _ in range(3):
x = torch.randn(batch, dim, dtype=torch.bfloat16)
cs_torch = storage_ref[indices, :_STATE_LEN].transpose(1, 2).contiguous()
out_torch = causal_conv1d_update_torch(
x=x.unsqueeze(-1),
conv_state=cs_torch,
weight=weight,
bias=bias,
activation="silu" if silu_activation else None,
).squeeze(-1)
storage_ref[indices, :_STATE_LEN] = cs_torch.transpose(1, 2)
out_cpp = ops.causal_conv1d_update_cpu(
x=x,
conv_states=cs_cpp,
weight=conv_weight,
bias=bias,
silu_activation=silu_activation,
conv_state_indices=indices,
is_weight_packed=is_weight_packed,
)
torch.testing.assert_close(out_cpp, out_torch, atol=1e-2, rtol=1e-2)
torch.testing.assert_close(storage_cpp, storage_ref, atol=0, rtol=0)
@pytest.mark.skipif(
not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16"
not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()),
reason="C++ causal_conv1d requires AVX-512BF16 or Arm BF16 support",
)
@torch.inference_mode()
def test_conv_weight_pack_roundtrip_unpacked_matches():
"""`causal_conv1d_weight_pack` repacks the (dim, width) weight, so the C++ conv op
with packed weight + is_vnni=True must equal unpacked weight + is_vnni=False.
Guards the spec-decode contract: the runtime VNNI-packs `layer.conv1d.weight`
in place and stashes the original as `_cpu_unpacked_conv_weight` for the torch
spec-decode path, so both must produce identical math.
with packed weight + is_weight_packed=True must equal unpacked weight
+ is_weight_packed=False. Guards the spec-decode contract: the runtime packs
`layer.conv1d.weight` in place and stashes the original as
`_cpu_unpacked_conv_weight` for the torch spec-decode path, so both must produce
identical math.
"""
seq_lens = [7, 64, 65]
x, weight, bias = _conv_inputs(sum(seq_lens))
out_unpacked, _ = _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=False)
out_packed, _ = _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=True)
torch.testing.assert_close(out_packed, out_unpacked, atol=1e-2, rtol=1e-2)
out_unpacked, state_unpacked = _run_prefill_cpp(
x, weight, bias, seq_lens, is_weight_packed=False
)
out_packed, state_packed = _run_prefill_cpp(
x, weight, bias, seq_lens, is_weight_packed=True
)
torch.testing.assert_close(out_packed, out_unpacked, atol=0, rtol=0)
torch.testing.assert_close(state_packed, state_unpacked, atol=0, rtol=0)
@pytest.mark.skipif(
not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16"
not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()),
reason="C++ causal_conv1d requires AVX-512BF16 or AArch64 BF16",
)
@torch.inference_mode()
def test_spec_decode_unpacked_conv_weight_stash():
"""Spec-decode correctness: AVX-512BF16 CPUs VNNI-pack ``conv1d.weight`` in place
at load time and stash the original ``(dim, width)`` tensor as
"""Spec-decode correctness: AVX-512BF16 and AArch64 BF16 CPUs pack ``conv1d.weight``
in place at load time and stash the original ``(dim, width)`` tensor as
``_cpu_unpacked_conv_weight``; the spec-decode path must use that stash since
reading the packed weight directly produces garbage. Verifies the recovered
weight equals the original and that torch F.conv1d agrees with the C++ conv
@@ -1460,7 +1512,7 @@ def test_spec_decode_unpacked_conv_weight_stash():
# 1. The stash must exist and equal the original (dim, width) weight.
assert hasattr(conv, "_cpu_unpacked_conv_weight"), (
"dispatch_cpu_unquantized_gemm did not stash _cpu_unpacked_conv_weight "
"on an AVX-512BF16 CPU"
"on an AArch64-BF16 or AVX-512BF16 CPU"
)
stash = conv._cpu_unpacked_conv_weight
torch.testing.assert_close(stash, orig_2d, atol=0, rtol=0)
@@ -1469,10 +1521,10 @@ def test_spec_decode_unpacked_conv_weight_stash():
recovered = getattr(conv, "_cpu_unpacked_conv_weight", None)
assert recovered is not None
# conv.weight is now packed; using it directly (the bug) would differ.
packed_weight = conv.weight # [dim, 1, width], VNNI-packed contents
packed_weight = conv.weight # [dim, 1, width], packed contents
# 3. Spec-path torch conv with the recovered unpacked weight must match the
# nonspec C++ conv with the packed weight (is_vnni=True).
# nonspec C++ conv with the packed weight (is_weight_packed=True).
seq_lens = [7, 65]
total = sum(seq_lens)
x = tensor_cache(total * CONV_DIM, torch.bfloat16).view(total, CONV_DIM)
@@ -1494,7 +1546,7 @@ def test_spec_decode_unpacked_conv_weight_stash():
cache_indices=torch.arange(num_seqs, dtype=torch.int32),
has_initial_state=torch.zeros(num_seqs, dtype=torch.bool),
silu_activation=True,
is_vnni=True,
is_weight_packed=True,
)
.transpose(0, 1)
.contiguous()
@@ -88,11 +88,6 @@ def test_short_conv_forward_native_prefill(vllm_config):
state_indices_tensor_d=torch.empty((0, 1), dtype=torch.int32),
num_accepted_tokens=None,
query_start_loc_d=None,
block_idx_last_scheduled_token=None,
block_idx_first_scheduled_token_p=None,
block_idx_last_computed_token=None,
block_idx_last_scheduled_token_prev_step=None,
num_computed_tokens_p=None,
seq_lens=torch.tensor([5]),
)
@@ -145,11 +140,6 @@ def test_short_conv_forward_native_decode(vllm_config):
state_indices_tensor_d=state_indices_tensor_d,
num_accepted_tokens=None,
query_start_loc_d=torch.tensor([0, 1, 2], dtype=torch.int32),
block_idx_last_scheduled_token=None,
block_idx_first_scheduled_token_p=None,
block_idx_last_computed_token=None,
block_idx_last_scheduled_token_prev_step=None,
num_computed_tokens_p=None,
seq_lens=torch.tensor([1, 1]),
)
@@ -176,7 +176,7 @@ def test_forward_core_split_matches_unified(
builder = GDNAttentionMetadataBuilder(
# GDNAttentionMetadataBuilder declares kv_cache_spec as AttentionSpec but
# asserts isinstance(kv_cache_spec, MambaSpec). MambaSpec is correct here.
kv_cache_spec=MambaSpec( # type: ignore[arg-type]
kv_cache_spec=MambaSpec(
block_size=BLOCK_SIZE, shapes=((16, 64),), dtypes=(torch.float16,)
),
layer_names=[PREFIX],
+388 -5
View File
@@ -1,24 +1,30 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
import torch
from vllm.config.mamba import MambaBackendEnum, MambaConfig, MambaSSUAlgorithm
from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateShapeCalculator
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
FlashInferSSUBackend,
TritonSSUBackend,
commit_replayssm_ring_trackers,
get_mamba_ssu_backend,
initialize_mamba_ssu_backend,
reset_replayssm_ring_trackers,
selective_state_update,
selective_state_update_replayssm_flashinfer,
update_replayssm_ring_trackers,
)
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadata
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
@@ -32,6 +38,13 @@ try:
except ImportError:
HAS_FLASHINFER = False
try:
from flashinfer.mamba.checkpointing_ssu import CheckpointingSSURunner
HAS_FLASHINFER_CHECKPOINTING_SSU = CheckpointingSSURunner is not None
except ImportError:
HAS_FLASHINFER_CHECKPOINTING_SSU = False
@pytest.fixture(autouse=True)
def restore_backend_state():
@@ -47,6 +60,7 @@ def restore_backend_state():
def test_flashinfer_replayssm_ring_tracker_lifecycle():
ring_start = torch.zeros(2, dtype=torch.int32, device="cuda")
prev_num_accepted = torch.zeros(2, dtype=torch.int32, device="cuda")
prev_query_len = torch.zeros(2, dtype=torch.int32, device="cuda")
state_batch_indices = torch.tensor([1], dtype=torch.int32, device="cuda")
observed = []
@@ -54,6 +68,7 @@ def test_flashinfer_replayssm_ring_tracker_lifecycle():
update_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
logical_window=16,
ring_buffer_len=17,
@@ -69,9 +84,205 @@ def test_flashinfer_replayssm_ring_tracker_lifecycle():
reset_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
)
assert (ring_start[1].item(), prev_num_accepted[1].item()) == (0, 0)
assert (
ring_start[1].item(),
prev_num_accepted[1].item(),
prev_query_len[1].item(),
) == (0, 0, 0)
@pytest.mark.parametrize(
("accepted_sequence", "expected"),
[
pytest.param(
[4] * 22,
[
(0, 0, 4),
(0, 4, 4),
(0, 8, 4),
(0, 12, 4),
(0, 16, 4),
(16, 4, 4),
(16, 8, 4),
(16, 12, 4),
(16, 16, 4),
(12, 4, 4),
(12, 8, 4),
(12, 12, 4),
(12, 16, 4),
(8, 4, 4),
(8, 8, 4),
(8, 12, 4),
(8, 16, 4),
(4, 4, 4),
(4, 8, 4),
(4, 12, 4),
(4, 16, 4),
(0, 4, 4),
],
id="all-accepted",
),
pytest.param(
[4, 4, 0, 3, 4, 2, 4, 1],
[
(0, 0, 4),
(0, 4, 4),
(0, 4, 4),
(0, 7, 4),
(0, 11, 4),
(0, 13, 4),
(13, 4, 4),
(13, 5, 4),
],
id="mixed",
),
],
)
def test_replayssm_commit_tracker_acceptance_sequence(accepted_sequence, expected):
logical_window = 16
num_speculative_tokens = 3
query_len = 1 + num_speculative_tokens
ring_buffer_len = logical_window + 1 + num_speculative_tokens
ring_start = torch.zeros(2, dtype=torch.int32, device="cuda")
prev_num_accepted = torch.zeros(2, dtype=torch.int32, device="cuda")
prev_query_len = torch.zeros(2, dtype=torch.int32, device="cuda")
state_batch_indices = torch.tensor([1], dtype=torch.int32, device="cuda")
query_start_loc = torch.tensor([0, query_len], dtype=torch.int32, device="cuda")
observed = []
for accepted in accepted_sequence:
commit_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
torch.tensor([accepted], dtype=torch.int32, device="cuda"),
query_start_loc,
logical_window,
ring_buffer_len,
)
snapshot = (
ring_start[1].item(),
prev_num_accepted[1].item(),
prev_query_len[1].item(),
)
observed.append(snapshot)
assert snapshot[1] + snapshot[2] <= ring_buffer_len
assert observed == expected
def test_replayssm_resume_resets_commit_history():
ring_start = torch.tensor([0, 13], dtype=torch.int32, device="cuda")
prev_num_accepted = torch.tensor([0, 13], dtype=torch.int32, device="cuda")
prev_query_len = torch.tensor([0, 4], dtype=torch.int32, device="cuda")
state_batch_indices = torch.tensor([1], dtype=torch.int32, device="cuda")
reset_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
)
assert (
ring_start[1].item(),
prev_num_accepted[1].item(),
prev_query_len[1].item(),
) == (0, 0, 0)
commit_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
torch.tensor([3], dtype=torch.int32, device="cuda"),
torch.tensor([0, 4], dtype=torch.int32, device="cuda"),
logical_window=16,
ring_buffer_len=20,
)
assert (
ring_start[1].item(),
prev_num_accepted[1].item(),
prev_query_len[1].item(),
) == (0, 0, 4)
def test_replayssm_commit_tracker_ragged_query_lengths():
ring_start = torch.zeros(3, dtype=torch.int32, device="cuda")
prev_num_accepted = torch.zeros(3, dtype=torch.int32, device="cuda")
prev_query_len = torch.zeros(3, dtype=torch.int32, device="cuda")
state_batch_indices = torch.tensor([1, 2], dtype=torch.int32, device="cuda")
query_start_loc = torch.tensor([0, 4, 6], dtype=torch.int32, device="cuda")
observed = []
for accepted in ([4, 2], [3, 1]):
commit_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
torch.tensor(accepted, dtype=torch.int32, device="cuda"),
query_start_loc,
logical_window=16,
ring_buffer_len=20,
)
observed.append(
[
(
ring_start[slot].item(),
prev_num_accepted[slot].item(),
prev_query_len[slot].item(),
)
for slot in (1, 2)
]
)
assert observed == [[(0, 0, 4), (0, 0, 2)], [(0, 3, 4), (0, 1, 2)]]
@pytest.mark.parametrize("operation", ["commit", "reset"])
def test_replayssm_tracker_kernels_mask_invalid_slots(operation):
num_states = 3
ring_start = torch.tensor([11, 2, 33], dtype=torch.int32, device="cuda")
prev_num_accepted = torch.tensor([11, 3, 33], dtype=torch.int32, device="cuda")
prev_query_len = torch.tensor([11, 4, 33], dtype=torch.int32, device="cuda")
state_batch_indices = torch.tensor(
[-1, num_states, NULL_BLOCK_ID, 1], dtype=torch.int32, device="cuda"
)
if operation == "commit":
commit_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
torch.tensor([4, 4, 4, 2], dtype=torch.int32, device="cuda"),
torch.tensor([0, 4, 8, 12, 16], dtype=torch.int32, device="cuda"),
logical_window=16,
ring_buffer_len=20,
)
expected_valid = (2, 5, 4)
else:
reset_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
)
expected_valid = (0, 0, 0)
assert (
ring_start.tolist(),
prev_num_accepted.tolist(),
prev_query_len.tolist(),
) == (
[11, expected_valid[0], 33],
[11, expected_valid[1], 33],
[11, expected_valid[2], 33],
)
def _kv_cache_config_with_ssu(
@@ -231,14 +442,185 @@ def test_triton_basic_call():
assert not torch.isnan(out).any()
@pytest.mark.parametrize("layout", ["packed", "dense"])
def test_replayssm_flashinfer_call_forwards_mtp_layout(monkeypatch, layout):
import vllm.model_executor.layers.mamba.ops.ssu_dispatch as mod
kernel = Mock(return_value=torch.empty(0))
monkeypatch.setattr(mod, "_flashinfer_replayssm_kernel", kernel)
batch, max_seqlen, nheads, dim, dstate, ngroups = 2, 4, 2, 4, 8, 1
state = torch.empty(2, nheads, dim, dstate)
x_shape: tuple[int, ...]
B_shape: tuple[int, ...]
expected_x_shape: tuple[int, ...]
expected_B_shape: tuple[int, ...]
if layout == "packed":
x_shape = (6, nheads, dim)
B_shape = (6, ngroups, dstate)
expected_x_shape = (1, 6, nheads, dim)
expected_B_shape = (1, 6, ngroups, dstate)
cu_seqlens = torch.tensor([0, 4, 6], dtype=torch.int32)
kernel_max_seqlen = max_seqlen
else:
x_shape = (batch, max_seqlen, nheads, dim)
B_shape = (batch, max_seqlen, ngroups, dstate)
expected_x_shape = x_shape
expected_B_shape = B_shape
cu_seqlens = None
kernel_max_seqlen = None
x = torch.empty(x_shape)
dt = torch.empty_like(x)
A = torch.empty(nheads, dim, dstate)
B = torch.empty(B_shape)
C = torch.empty_like(B)
out = torch.empty_like(x)
x_cache = torch.empty(2, nheads, 20, dim)
dt_cache = torch.empty(2, nheads, 20)
B_cache = torch.empty(2, ngroups, 20, dstate)
ring_start = torch.zeros(2, dtype=torch.int32)
prev_num_accepted = torch.zeros(2, dtype=torch.int32)
prev_query_len = torch.zeros(2, dtype=torch.int32)
selective_state_update_replayssm_flashinfer(
state,
x,
dt,
A,
B,
C,
out,
x_cache,
B_cache,
dt_cache,
ring_start,
prev_num_accepted,
prev_query_len,
logical_window=16,
state_batch_indices=torch.tensor([0, 1], dtype=torch.int32),
cu_seqlens=cu_seqlens,
max_seqlen=kernel_max_seqlen,
update_trackers=False,
)
args = kernel.call_args.args
assert args[6].shape == expected_x_shape
assert args[7].shape == expected_x_shape
assert args[9].shape == expected_B_shape
assert args[10].shape == expected_B_shape
assert args[11].shape == expected_x_shape
assert kernel.call_args.kwargs["cu_seqlens"] is cu_seqlens
assert kernel.call_args.kwargs["max_seqlen"] == kernel_max_seqlen
@pytest.mark.parametrize(
("backend", "expected_ring_len"),
("query_start_loc", "expected_shape", "expected_max_seqlen"),
[
(MambaBackendEnum.TRITON, 16),
(MambaBackendEnum.FLASHINFER, 17),
pytest.param([0, 4, 8], (2, 4, 2, 4), None, id="dense"),
pytest.param([0, 4, 6], (6, 2, 4), 4, id="packed"),
],
)
def test_replayssm_physical_ring_shape(backend, expected_ring_len):
def test_replayssm_mixer_selects_mtp_layout(
monkeypatch, query_start_loc, expected_shape, expected_max_seqlen
):
import vllm.model_executor.layers.mamba.mamba_mixer2 as mod
mixer = MambaMixer2.__new__(MambaMixer2)
torch.nn.Module.__init__(mixer)
mixer.prefix = "mixer"
mixer.tped_intermediate_size = 0
mixer.tped_conv_size = 1
mixer.tped_dt_size = 2
mixer.num_heads = 2
mixer.head_dim = 4
mixer.n_groups = mixer.tp_size = 1
mixer.ssm_state_size = 8
mixer.num_spec = 3
mixer.use_replayssm = True
mixer.replayssm_buffer_len = 16
mixer._commits_replayssm_trackers = True
mixer._updates_replayssm_trackers = False
mixer.mamba_config = MambaConfig(backend=MambaBackendEnum.FLASHINFER)
mixer.cache_config = SimpleNamespace(mamba_block_size=16, mamba_cache_mode="none")
mixer.conv_weights = torch.empty(0)
mixer.conv1d = SimpleNamespace(bias=None)
mixer.activation = "silu"
mixer.A = torch.empty(2)
mixer.dt_bias = torch.empty(2)
mixer.D = torch.empty(2)
mixer._replayssm_ring_start = torch.zeros(3, dtype=torch.int32)
mixer._replayssm_prev_num_accepted = torch.zeros(3, dtype=torch.int32)
mixer._replayssm_prev_query_len = torch.zeros(3, dtype=torch.int32)
mixer.kv_cache = (
torch.empty(3, 1),
torch.empty(3, 2, 4, 8),
torch.empty(3, 2, 20, 4),
torch.empty(3, 2, 20),
torch.empty(3, 1, 20, 8),
)
num_decode_tokens = query_start_loc[-1]
query_start_loc_d = torch.tensor(query_start_loc, dtype=torch.int32)
metadata = Mamba2AttentionMetadata(
num_prefills=0,
num_prefill_tokens=0,
num_decodes=2,
num_decode_tokens=num_decode_tokens,
num_reqs=2,
has_initial_states_p=None,
query_start_loc_p=None,
state_indices_tensor_p=None,
state_indices_tensor_d=torch.tensor([[1], [2]], dtype=torch.int32),
query_start_loc_d=query_start_loc_d,
num_accepted_tokens=torch.tensor([4, 2], dtype=torch.int32),
seq_lens=torch.tensor([104, 102], dtype=torch.int32),
replayssm_scratch=(torch.empty(0), torch.empty(0), torch.empty(0)),
replayssm_state_indices_d=torch.tensor([1, 2], dtype=torch.int32),
)
def split_hidden_states_B_C(values):
tokens = values.size(0)
return (
torch.empty(tokens, 8),
torch.empty(tokens, 8),
torch.empty(tokens, 8),
)
mixer.split_hidden_states_B_C_fn = split_hidden_states_B_C
kernel = Mock()
monkeypatch.setattr(
mod,
"get_forward_context",
lambda: SimpleNamespace(attn_metadata={mixer.prefix: metadata}),
)
monkeypatch.setattr(mod, "commit_replayssm_ring_trackers", Mock())
monkeypatch.setattr(
mod, "causal_conv1d_update", lambda values, *args, **kwargs: values
)
monkeypatch.setattr(mod, "selective_state_update_replayssm_flashinfer", kernel)
mixer.conv_ssm_forward(
torch.empty(num_decode_tokens, 3), torch.empty(num_decode_tokens, 8)
)
assert kernel.call_args.args[1].shape == expected_shape
if expected_max_seqlen is None:
assert kernel.call_args.kwargs["cu_seqlens"] is None
else:
assert kernel.call_args.kwargs["cu_seqlens"] is query_start_loc_d
assert kernel.call_args.kwargs["max_seqlen"] == expected_max_seqlen
@pytest.mark.parametrize(
("backend", "num_speculative_tokens", "expected_ring_len"),
[
(MambaBackendEnum.TRITON, 0, 16),
(MambaBackendEnum.FLASHINFER, 0, 17),
(MambaBackendEnum.FLASHINFER, 3, 20),
],
)
def test_replayssm_physical_ring_shape(
backend, num_speculative_tokens, expected_ring_len
):
base_shapes = ((64, 3), (8, 4, 16))
shapes = MambaStateShapeCalculator.append_replayssm_ring(
@@ -247,6 +629,7 @@ def test_replayssm_physical_ring_shape(backend, expected_ring_len):
tp_world_size=2,
logical_window=16,
backend=backend,
num_speculative_tokens=num_speculative_tokens,
)
assert shapes[2:] == (