Merge remote-tracking branch 'upstream/main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -494,8 +494,10 @@ def _sd_conv_states(
|
||||
return storage.transpose(1, 2)
|
||||
|
||||
|
||||
def _maybe_pack_conv_weight(weight: torch.Tensor, is_vnni: bool) -> torch.Tensor:
|
||||
return ops.causal_conv1d_weight_pack(weight) if is_vnni else weight
|
||||
def _maybe_pack_conv_weight(
|
||||
weight: torch.Tensor, is_weight_packed: bool
|
||||
) -> torch.Tensor:
|
||||
return ops.causal_conv1d_weight_pack(weight) if is_weight_packed else weight
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
@@ -523,7 +525,7 @@ def test_causal_conv1d_update_cpu_preserves_sd_state(
|
||||
bias=None,
|
||||
silu_activation=False,
|
||||
conv_state_indices=state_indices,
|
||||
is_vnni=False,
|
||||
is_weight_packed=False,
|
||||
)
|
||||
|
||||
assert out is x
|
||||
@@ -666,7 +668,10 @@ def test_spec_aware_nonspec_materializes_state_indices(
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
@pytest.mark.parametrize(("has_amx", "has_arm_bf16"), [(True, False), (False, True)])
|
||||
def test_spec_forward_prepares_native_conv_metadata(
|
||||
has_amx: bool,
|
||||
has_arm_bf16: bool,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
block_table = torch.tensor([[0, 1], [4, 5]], dtype=torch.int32)
|
||||
@@ -687,14 +692,17 @@ def test_spec_forward_prepares_native_conv_metadata(
|
||||
)
|
||||
forwarded_indices = None
|
||||
forwarded_counts = None
|
||||
forwarded_is_weight_packed = None
|
||||
|
||||
def causal_conv1d_update_cpu(**kwargs):
|
||||
nonlocal forwarded_counts, forwarded_indices
|
||||
nonlocal forwarded_counts, forwarded_indices, forwarded_is_weight_packed
|
||||
forwarded_indices = kwargs["conv_state_indices"]
|
||||
forwarded_counts = kwargs["num_accepted_tokens"]
|
||||
forwarded_is_weight_packed = kwargs["is_weight_packed"]
|
||||
return kwargs["x"]
|
||||
|
||||
monkeypatch.setattr(torch.cpu, "_is_amx_tile_supported", lambda: True)
|
||||
monkeypatch.setattr(torch.cpu, "_is_amx_tile_supported", lambda: has_amx)
|
||||
monkeypatch.setattr(gdn_attention, "is_arm_bf16", lambda: has_arm_bf16)
|
||||
monkeypatch.setattr(gdn_attention, "is_conv_state_dim_first", lambda: False)
|
||||
monkeypatch.setattr(
|
||||
gdn_attention.ops, "causal_conv1d_update_cpu", causal_conv1d_update_cpu
|
||||
@@ -732,6 +740,7 @@ def test_spec_forward_prepares_native_conv_metadata(
|
||||
assert actual is not None
|
||||
assert actual.is_contiguous()
|
||||
assert actual.dtype == torch.int32
|
||||
assert forwarded_is_weight_packed is True
|
||||
torch.testing.assert_close(actual, reference)
|
||||
|
||||
|
||||
@@ -792,15 +801,15 @@ def test_causal_conv1d_torch_two_call_split(total_tokens: int, split: int) -> No
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cpu._is_amx_tile_supported(),
|
||||
reason="requires AMX support",
|
||||
not (torch.cpu._is_amx_tile_supported() or gdn_attention.is_arm_bf16()),
|
||||
reason="native causal conv1d requires AMX or Arm BF16 support",
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_causal_conv1d_update_cpu_accepts_wide_state() -> None:
|
||||
state_len = CONV_KERNEL - 1
|
||||
wide_state_len = state_len + 5
|
||||
batch_size = 3
|
||||
is_vnni = True
|
||||
is_weight_packed = True
|
||||
x, weight, bias = _conv_inputs(batch_size)
|
||||
conv_state_indices = torch.tensor([2, 0, 1], dtype=torch.int32)
|
||||
|
||||
@@ -813,7 +822,7 @@ def test_causal_conv1d_update_cpu_accepts_wide_state() -> None:
|
||||
wide_state[:, :, state_len:].fill_(7)
|
||||
wide_tail = wide_state[:, :, state_len:].clone()
|
||||
|
||||
conv_weight = _maybe_pack_conv_weight(weight, is_vnni)
|
||||
conv_weight = _maybe_pack_conv_weight(weight, is_weight_packed)
|
||||
out_narrow = ops.causal_conv1d_update_cpu(
|
||||
x=x,
|
||||
conv_states=narrow_state,
|
||||
@@ -821,7 +830,7 @@ def test_causal_conv1d_update_cpu_accepts_wide_state() -> None:
|
||||
bias=bias,
|
||||
silu_activation=True,
|
||||
conv_state_indices=conv_state_indices,
|
||||
is_vnni=is_vnni,
|
||||
is_weight_packed=is_weight_packed,
|
||||
)
|
||||
out_wide = ops.causal_conv1d_update_cpu(
|
||||
x=x,
|
||||
@@ -830,7 +839,7 @@ def test_causal_conv1d_update_cpu_accepts_wide_state() -> None:
|
||||
bias=bias,
|
||||
silu_activation=True,
|
||||
conv_state_indices=conv_state_indices,
|
||||
is_vnni=is_vnni,
|
||||
is_weight_packed=is_weight_packed,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(out_wide, out_narrow, atol=1e-2, rtol=1e-2)
|
||||
@@ -872,11 +881,18 @@ def _ref_causal_conv1d_update_cpu_multi(
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cpu._is_amx_tile_supported(),
|
||||
reason="requires AMX support",
|
||||
not (torch.cpu._is_amx_tile_supported() or gdn_attention.is_arm_bf16()),
|
||||
reason="native causal conv1d requires AMX or Arm BF16 support",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("batch_size, seq_len, accepted_counts, has_bias, silu_activation, is_vnni"),
|
||||
(
|
||||
"batch_size, "
|
||||
"seq_len, "
|
||||
"accepted_counts, "
|
||||
"has_bias, "
|
||||
"silu_activation, "
|
||||
"is_weight_packed"
|
||||
),
|
||||
[
|
||||
(1, 1, [1], False, False, False),
|
||||
(1, 1, [1], True, True, True),
|
||||
@@ -893,7 +909,7 @@ def test_causal_conv1d_update_cpu_multi_token_matches_python(
|
||||
accepted_counts: list[int],
|
||||
has_bias: bool,
|
||||
silu_activation: bool,
|
||||
is_vnni: bool,
|
||||
is_weight_packed: bool,
|
||||
) -> None:
|
||||
dim = 96
|
||||
state_len = seq_len + 2
|
||||
@@ -911,7 +927,7 @@ def test_causal_conv1d_update_cpu_multi_token_matches_python(
|
||||
)
|
||||
conv_states = conv_states_ref.clone()
|
||||
|
||||
conv_weight = _maybe_pack_conv_weight(weight, is_vnni)
|
||||
conv_weight = _maybe_pack_conv_weight(weight, is_weight_packed)
|
||||
out = ops.causal_conv1d_update_cpu(
|
||||
x=x,
|
||||
conv_states=conv_states,
|
||||
@@ -919,7 +935,7 @@ def test_causal_conv1d_update_cpu_multi_token_matches_python(
|
||||
bias=bias,
|
||||
silu_activation=silu_activation,
|
||||
conv_state_indices=conv_state_indices,
|
||||
is_vnni=is_vnni,
|
||||
is_weight_packed=is_weight_packed,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
)
|
||||
ref_out = _ref_causal_conv1d_update_cpu_multi(
|
||||
@@ -937,8 +953,8 @@ def test_causal_conv1d_update_cpu_multi_token_matches_python(
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cpu._is_amx_tile_supported(),
|
||||
reason="requires AMX support",
|
||||
not (torch.cpu._is_amx_tile_supported() or gdn_attention.is_arm_bf16()),
|
||||
reason="native causal conv1d requires AMX or Arm BF16 support",
|
||||
)
|
||||
@pytest.mark.parametrize("num_accepted", [0, 17])
|
||||
@torch.inference_mode()
|
||||
@@ -961,14 +977,15 @@ def test_causal_conv1d_update_cpu_rejects_invalid_accepted_count(
|
||||
bias=None,
|
||||
silu_activation=True,
|
||||
conv_state_indices=torch.tensor([0], dtype=torch.int32),
|
||||
is_vnni=False,
|
||||
is_weight_packed=False,
|
||||
num_accepted_tokens=torch.tensor([num_accepted], dtype=torch.int32),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cpu._is_avx512_bf16_supported(),
|
||||
reason="causal_conv1d_fwd_cpu requires AVX-512BF16 (Intel Xeon or AMD EPYC)",
|
||||
not (torch.cpu._is_avx512_bf16_supported() or gdn_attention.is_arm_bf16()),
|
||||
reason="causal_conv1d_fwd_cpu requires AVX-512BF16"
|
||||
" (Intel Xeon or AMD EPYC) or Arm BF16 support",
|
||||
)
|
||||
@pytest.mark.parametrize("total_tokens, split", TWO_CALL_SPLITS)
|
||||
@torch.inference_mode()
|
||||
@@ -977,13 +994,13 @@ def test_causal_conv1d_fwd_cpu_two_call_split(total_tokens: int, split: int) ->
|
||||
matches the single-call result.
|
||||
|
||||
Regression test for ``causal_conv1d_fwd_varlen_kernel_impl`` (``conv.cpp``)
|
||||
ignoring the carried conv state on continued chunks. Runs on any
|
||||
AVX-512BF16 CPU since conv.cpp uses VDPBF16PS, not AMX tiles.
|
||||
ignoring the carried conv state on continued chunks. Runs on aarch64 cpus with
|
||||
bf16 support or any AVX-512BF16 CPU since conv.cpp uses VDPBF16PS, not AMX tiles.
|
||||
"""
|
||||
state_len = CONV_KERNEL - 1
|
||||
x, weight, bias = _conv_inputs(total_tokens)
|
||||
|
||||
def amx(x_seg, conv_states, has_init):
|
||||
def native(x_seg, conv_states, has_init):
|
||||
seq = x_seg.shape[0]
|
||||
return ops.causal_conv1d_fwd_cpu(
|
||||
x=x_seg.transpose(0, 1), # [dim, seq]; stride(-2)==1 (view of [seq,dim])
|
||||
@@ -994,30 +1011,30 @@ def test_causal_conv1d_fwd_cpu_two_call_split(total_tokens: int, split: int) ->
|
||||
cache_indices=torch.tensor([0], dtype=torch.int32),
|
||||
has_initial_state=torch.tensor([has_init]),
|
||||
silu_activation=True,
|
||||
is_vnni=False,
|
||||
is_weight_packed=False,
|
||||
).contiguous()
|
||||
|
||||
# conv_state layout passed by the AMX branch: [num_slots, dim, state_len].
|
||||
cs_full = torch.zeros(1, CONV_DIM, state_len, dtype=x.dtype)
|
||||
out_full = amx(x, cs_full, False)
|
||||
out_full = native(x, cs_full, False)
|
||||
|
||||
cs_split = torch.zeros(1, CONV_DIM, state_len, dtype=x.dtype)
|
||||
out1 = amx(x[:split], cs_split, False)
|
||||
out2 = amx(x[split:], cs_split, True)
|
||||
out1 = native(x[:split], cs_split, False)
|
||||
out2 = native(x[split:], cs_split, True)
|
||||
out_split = torch.cat([out1, out2], dim=1)
|
||||
|
||||
torch.testing.assert_close(out_split, out_full, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cpu._is_amx_tile_supported(),
|
||||
reason="requires AMX support",
|
||||
not (torch.cpu._is_amx_tile_supported() or gdn_attention.is_arm_bf16()),
|
||||
reason="native causal conv1d requires AMX or Arm BF16 support",
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_causal_conv1d_fwd_cpu_accepts_wide_state() -> None:
|
||||
state_len = CONV_KERNEL - 1
|
||||
wide_state_len = state_len + 5
|
||||
is_vnni = True
|
||||
is_weight_packed = True
|
||||
seq_lens = [CHUNK_SIZE - 1, CHUNK_SIZE + 5]
|
||||
total_tokens = sum(seq_lens)
|
||||
x, weight, bias = _conv_inputs(total_tokens)
|
||||
@@ -1034,7 +1051,7 @@ def test_causal_conv1d_fwd_cpu_accepts_wide_state() -> None:
|
||||
wide_state[:, :, state_len:].fill_(7)
|
||||
wide_tail = wide_state[:, :, state_len:].clone()
|
||||
|
||||
conv_weight = _maybe_pack_conv_weight(weight, is_vnni)
|
||||
conv_weight = _maybe_pack_conv_weight(weight, is_weight_packed)
|
||||
out_narrow = ops.causal_conv1d_fwd_cpu(
|
||||
x=x.transpose(0, 1),
|
||||
weight=conv_weight,
|
||||
@@ -1044,7 +1061,7 @@ def test_causal_conv1d_fwd_cpu_accepts_wide_state() -> None:
|
||||
cache_indices=cache_indices,
|
||||
has_initial_state=has_initial_state,
|
||||
silu_activation=True,
|
||||
is_vnni=is_vnni,
|
||||
is_weight_packed=is_weight_packed,
|
||||
)
|
||||
out_wide = ops.causal_conv1d_fwd_cpu(
|
||||
x=x.transpose(0, 1),
|
||||
@@ -1055,7 +1072,7 @@ def test_causal_conv1d_fwd_cpu_accepts_wide_state() -> None:
|
||||
cache_indices=cache_indices,
|
||||
has_initial_state=has_initial_state,
|
||||
silu_activation=True,
|
||||
is_vnni=is_vnni,
|
||||
is_weight_packed=is_weight_packed,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(out_wide, out_narrow, atol=1e-2, rtol=1e-2)
|
||||
@@ -1100,15 +1117,23 @@ def test_batch_memcpy_cpu_fallback() -> None:
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# C++ conv (conv.cpp) uses VDPBF16PS, not AMX tiles, so it runs on any
|
||||
# AVX-512BF16 CPU; weight is VNNI-packed on this same predicate at load time.
|
||||
# C++ conv (conv.cpp) uses AArch64BF16 or VDPBF16PS, not AMX tiles, so it can
|
||||
# run on any AVX-512BF16 CPU; weight is packed on this same predicate at load time.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_HAS_AVX512_BF16 = torch.cpu._is_avx512_bf16_supported()
|
||||
|
||||
_STATE_LEN = CONV_KERNEL - 1
|
||||
|
||||
CONV_EQUIV_SEQ_LENS = [[1], [7], [64], [65], [1, 2, 3], [63, 64, 65], [128, 129]]
|
||||
CONV_EQUIV_SEQ_LENS = [
|
||||
[1],
|
||||
[7],
|
||||
[64],
|
||||
[65],
|
||||
[1, 2, 3],
|
||||
[63, 64, 65],
|
||||
[128, 129],
|
||||
]
|
||||
|
||||
|
||||
def _conv_fp32_oracle(x, weight, bias, seq_lens, activation="silu"):
|
||||
@@ -1153,10 +1178,10 @@ def _run_prefill_torch(x, weight, bias, seq_lens):
|
||||
return out.transpose(0, 1).contiguous(), conv_states
|
||||
|
||||
|
||||
def _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=False):
|
||||
def _run_prefill_cpp(x, weight, bias, seq_lens, is_weight_packed=False):
|
||||
num_seqs = len(seq_lens)
|
||||
packed_w = ops.causal_conv1d_weight_pack(weight) if is_vnni else weight
|
||||
if is_vnni:
|
||||
packed_w = ops.causal_conv1d_weight_pack(weight) if is_weight_packed else weight
|
||||
if is_weight_packed:
|
||||
# C++-branch layout: kv-cache "SD" [slots, state_len, dim] transposed to
|
||||
# [slots, dim, state_len] (a non-contiguous view).
|
||||
conv_state = torch.zeros(
|
||||
@@ -1176,7 +1201,7 @@ def _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=False):
|
||||
cache_indices=torch.arange(num_seqs, dtype=torch.int32),
|
||||
has_initial_state=torch.zeros(num_seqs, dtype=torch.bool),
|
||||
silu_activation=True,
|
||||
is_vnni=is_vnni,
|
||||
is_weight_packed=is_weight_packed,
|
||||
)
|
||||
return out.transpose(0, 1).contiguous(), conv_state
|
||||
|
||||
@@ -1245,7 +1270,7 @@ def test_causal_conv1d_fwd_cpu_ds_adapter_varlen_mixed_initial_state(
|
||||
cache_indices=state_indices,
|
||||
has_initial_state=has_initial_state,
|
||||
silu_activation=True,
|
||||
is_vnni=False,
|
||||
is_weight_packed=False,
|
||||
)
|
||||
monkeypatch.setattr(gdn_attention, "is_conv_state_dim_first", lambda: True)
|
||||
out_ds = gdn_attention._causal_conv1d_fwd_cpu(
|
||||
@@ -1257,7 +1282,7 @@ def test_causal_conv1d_fwd_cpu_ds_adapter_varlen_mixed_initial_state(
|
||||
cache_indices=state_indices,
|
||||
has_initial_state=has_initial_state,
|
||||
silu_activation=True,
|
||||
is_vnni=False,
|
||||
is_weight_packed=False,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(out_ds, out_sd)
|
||||
@@ -1302,7 +1327,7 @@ def test_causal_conv1d_update_cpu_ds_adapter_wide_multi_token_copyback(
|
||||
bias=None,
|
||||
silu_activation=False,
|
||||
conv_state_indices=state_indices,
|
||||
is_vnni=False,
|
||||
is_weight_packed=False,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
)
|
||||
|
||||
@@ -1321,7 +1346,8 @@ def test_causal_conv1d_update_cpu_ds_adapter_wide_multi_token_copyback(
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16"
|
||||
not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()),
|
||||
reason="C++ causal_conv1d requires AVX-512BF16 or Arm BF16 support",
|
||||
)
|
||||
@pytest.mark.parametrize("seq_lens", CONV_EQUIV_SEQ_LENS)
|
||||
@torch.inference_mode()
|
||||
@@ -1335,7 +1361,8 @@ def test_conv_cpp_matches_torch(seq_lens):
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16"
|
||||
not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()),
|
||||
reason="C++ causal_conv1d requires AVX-512BF16 or Arm BF16 support",
|
||||
)
|
||||
@pytest.mark.parametrize("seq_lens", CONV_EQUIV_SEQ_LENS)
|
||||
@torch.inference_mode()
|
||||
@@ -1354,89 +1381,114 @@ def test_conv_cpp_no_worse_than_torch_vs_fp32(seq_lens):
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16"
|
||||
not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()),
|
||||
reason="C++ causal_conv1d requires AVX-512BF16 or Arm BF16 support",
|
||||
)
|
||||
@pytest.mark.parametrize("seq_lens", CONV_EQUIV_SEQ_LENS)
|
||||
@torch.inference_mode()
|
||||
def test_conv_cpp_vnni_packed_matches_torch(seq_lens):
|
||||
"""The exact runtime prefill sequence (VNNI-packed weight + SD-layout
|
||||
conv_state view + is_vnni=True) must match the torch fallback. Validates
|
||||
the packing + layout handoff on any AVX-512BF16 CPU."""
|
||||
def test_conv_cpp_weight_packed_matches_torch(seq_lens):
|
||||
"""The exact runtime prefill sequence (packed weight + SD-layout
|
||||
conv_state view + is_weight_packed=True) must match the torch fallback. Validates
|
||||
the native packing + layout handoff."""
|
||||
x, weight, bias = _conv_inputs(sum(seq_lens))
|
||||
out_torch, _ = _run_prefill_torch(x, weight, bias, seq_lens)
|
||||
out_vnni, _ = _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=True)
|
||||
torch.testing.assert_close(out_vnni, out_torch, atol=1e-2, rtol=1e-2)
|
||||
out_weight_packed, _ = _run_prefill_cpp(
|
||||
x, weight, bias, seq_lens, is_weight_packed=True
|
||||
)
|
||||
torch.testing.assert_close(out_weight_packed, out_torch, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16"
|
||||
not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()),
|
||||
reason="C++ causal_conv1d requires AVX-512BF16 or Arm BF16 support",
|
||||
)
|
||||
@pytest.mark.parametrize("batch", DECODE_BATCH_SIZES)
|
||||
@pytest.mark.parametrize(
|
||||
("dim", "has_bias", "silu_activation", "is_weight_packed"),
|
||||
[
|
||||
(32, False, False, False),
|
||||
(32, True, True, True),
|
||||
(96, False, True, True),
|
||||
(96, True, False, False),
|
||||
(8192, False, True, False),
|
||||
(8192, True, True, True),
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_conv_update_cpp_matches_torch(batch):
|
||||
"""Decode conv: causal_conv1d_update_cpu matches causal_conv1d_update_torch,
|
||||
including the in-place conv_state update (the next-step handoff)."""
|
||||
def test_conv_update_cpp_matches_torch(
|
||||
batch, dim, has_bias, silu_activation, is_weight_packed
|
||||
):
|
||||
"""Decode preserves indexed, padded state across successive signed inputs."""
|
||||
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
|
||||
causal_conv1d_update_torch,
|
||||
)
|
||||
|
||||
x = tensor_cache(batch * CONV_DIM, torch.bfloat16).view(batch, CONV_DIM)
|
||||
weight = tensor_cache(CONV_DIM * CONV_KERNEL, torch.bfloat16).view(
|
||||
CONV_DIM, CONV_KERNEL
|
||||
)
|
||||
bias = tensor_cache(CONV_DIM, torch.bfloat16)
|
||||
conv_state = tensor_cache(batch * CONV_DIM * _STATE_LEN, torch.bfloat16).view(
|
||||
batch, CONV_DIM, _STATE_LEN
|
||||
)
|
||||
torch.manual_seed(0)
|
||||
weight = torch.randn(dim, CONV_KERNEL, dtype=torch.bfloat16)
|
||||
bias = torch.randn(dim, dtype=torch.bfloat16) if has_bias else None
|
||||
conv_weight = _maybe_pack_conv_weight(weight, is_weight_packed)
|
||||
indices = torch.arange(batch, 0, -1, dtype=torch.int32)
|
||||
storage_ref = torch.randn(batch + 2, _STATE_LEN + 2, dim, dtype=torch.bfloat16)
|
||||
storage_cpp = storage_ref.clone()
|
||||
cs_cpp = storage_cpp[:, :_STATE_LEN].transpose(1, 2)
|
||||
|
||||
cs_torch = conv_state.clone()
|
||||
out_torch = causal_conv1d_update_torch(
|
||||
x=x.unsqueeze(-1),
|
||||
conv_state=cs_torch,
|
||||
weight=weight,
|
||||
bias=bias,
|
||||
activation="silu",
|
||||
).squeeze(-1)
|
||||
|
||||
cs_cpp = conv_state.clone()
|
||||
out_cpp = ops.causal_conv1d_update_cpu(
|
||||
x=x.contiguous(),
|
||||
conv_states=cs_cpp,
|
||||
weight=weight,
|
||||
bias=bias,
|
||||
silu_activation=True,
|
||||
conv_state_indices=torch.arange(batch, dtype=torch.int32),
|
||||
is_vnni=False,
|
||||
)
|
||||
torch.testing.assert_close(out_cpp, out_torch, atol=1e-2, rtol=1e-2)
|
||||
torch.testing.assert_close(cs_cpp, cs_torch, atol=1e-2, rtol=1e-2)
|
||||
for _ in range(3):
|
||||
x = torch.randn(batch, dim, dtype=torch.bfloat16)
|
||||
cs_torch = storage_ref[indices, :_STATE_LEN].transpose(1, 2).contiguous()
|
||||
out_torch = causal_conv1d_update_torch(
|
||||
x=x.unsqueeze(-1),
|
||||
conv_state=cs_torch,
|
||||
weight=weight,
|
||||
bias=bias,
|
||||
activation="silu" if silu_activation else None,
|
||||
).squeeze(-1)
|
||||
storage_ref[indices, :_STATE_LEN] = cs_torch.transpose(1, 2)
|
||||
out_cpp = ops.causal_conv1d_update_cpu(
|
||||
x=x,
|
||||
conv_states=cs_cpp,
|
||||
weight=conv_weight,
|
||||
bias=bias,
|
||||
silu_activation=silu_activation,
|
||||
conv_state_indices=indices,
|
||||
is_weight_packed=is_weight_packed,
|
||||
)
|
||||
torch.testing.assert_close(out_cpp, out_torch, atol=1e-2, rtol=1e-2)
|
||||
torch.testing.assert_close(storage_cpp, storage_ref, atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16"
|
||||
not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()),
|
||||
reason="C++ causal_conv1d requires AVX-512BF16 or Arm BF16 support",
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_conv_weight_pack_roundtrip_unpacked_matches():
|
||||
"""`causal_conv1d_weight_pack` repacks the (dim, width) weight, so the C++ conv op
|
||||
with packed weight + is_vnni=True must equal unpacked weight + is_vnni=False.
|
||||
Guards the spec-decode contract: the runtime VNNI-packs `layer.conv1d.weight`
|
||||
in place and stashes the original as `_cpu_unpacked_conv_weight` for the torch
|
||||
spec-decode path, so both must produce identical math.
|
||||
with packed weight + is_weight_packed=True must equal unpacked weight
|
||||
+ is_weight_packed=False. Guards the spec-decode contract: the runtime packs
|
||||
`layer.conv1d.weight` in place and stashes the original as
|
||||
`_cpu_unpacked_conv_weight` for the torch spec-decode path, so both must produce
|
||||
identical math.
|
||||
"""
|
||||
seq_lens = [7, 64, 65]
|
||||
x, weight, bias = _conv_inputs(sum(seq_lens))
|
||||
out_unpacked, _ = _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=False)
|
||||
out_packed, _ = _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=True)
|
||||
torch.testing.assert_close(out_packed, out_unpacked, atol=1e-2, rtol=1e-2)
|
||||
out_unpacked, state_unpacked = _run_prefill_cpp(
|
||||
x, weight, bias, seq_lens, is_weight_packed=False
|
||||
)
|
||||
out_packed, state_packed = _run_prefill_cpp(
|
||||
x, weight, bias, seq_lens, is_weight_packed=True
|
||||
)
|
||||
torch.testing.assert_close(out_packed, out_unpacked, atol=0, rtol=0)
|
||||
torch.testing.assert_close(state_packed, state_unpacked, atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16"
|
||||
not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()),
|
||||
reason="C++ causal_conv1d requires AVX-512BF16 or AArch64 BF16",
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_spec_decode_unpacked_conv_weight_stash():
|
||||
"""Spec-decode correctness: AVX-512BF16 CPUs VNNI-pack ``conv1d.weight`` in place
|
||||
at load time and stash the original ``(dim, width)`` tensor as
|
||||
"""Spec-decode correctness: AVX-512BF16 and AArch64 BF16 CPUs pack ``conv1d.weight``
|
||||
in place at load time and stash the original ``(dim, width)`` tensor as
|
||||
``_cpu_unpacked_conv_weight``; the spec-decode path must use that stash since
|
||||
reading the packed weight directly produces garbage. Verifies the recovered
|
||||
weight equals the original and that torch F.conv1d agrees with the C++ conv
|
||||
@@ -1460,7 +1512,7 @@ def test_spec_decode_unpacked_conv_weight_stash():
|
||||
# 1. The stash must exist and equal the original (dim, width) weight.
|
||||
assert hasattr(conv, "_cpu_unpacked_conv_weight"), (
|
||||
"dispatch_cpu_unquantized_gemm did not stash _cpu_unpacked_conv_weight "
|
||||
"on an AVX-512BF16 CPU"
|
||||
"on an AArch64-BF16 or AVX-512BF16 CPU"
|
||||
)
|
||||
stash = conv._cpu_unpacked_conv_weight
|
||||
torch.testing.assert_close(stash, orig_2d, atol=0, rtol=0)
|
||||
@@ -1469,10 +1521,10 @@ def test_spec_decode_unpacked_conv_weight_stash():
|
||||
recovered = getattr(conv, "_cpu_unpacked_conv_weight", None)
|
||||
assert recovered is not None
|
||||
# conv.weight is now packed; using it directly (the bug) would differ.
|
||||
packed_weight = conv.weight # [dim, 1, width], VNNI-packed contents
|
||||
packed_weight = conv.weight # [dim, 1, width], packed contents
|
||||
|
||||
# 3. Spec-path torch conv with the recovered unpacked weight must match the
|
||||
# nonspec C++ conv with the packed weight (is_vnni=True).
|
||||
# nonspec C++ conv with the packed weight (is_weight_packed=True).
|
||||
seq_lens = [7, 65]
|
||||
total = sum(seq_lens)
|
||||
x = tensor_cache(total * CONV_DIM, torch.bfloat16).view(total, CONV_DIM)
|
||||
@@ -1494,7 +1546,7 @@ def test_spec_decode_unpacked_conv_weight_stash():
|
||||
cache_indices=torch.arange(num_seqs, dtype=torch.int32),
|
||||
has_initial_state=torch.zeros(num_seqs, dtype=torch.bool),
|
||||
silu_activation=True,
|
||||
is_vnni=True,
|
||||
is_weight_packed=True,
|
||||
)
|
||||
.transpose(0, 1)
|
||||
.contiguous()
|
||||
|
||||
@@ -88,11 +88,6 @@ def test_short_conv_forward_native_prefill(vllm_config):
|
||||
state_indices_tensor_d=torch.empty((0, 1), dtype=torch.int32),
|
||||
num_accepted_tokens=None,
|
||||
query_start_loc_d=None,
|
||||
block_idx_last_scheduled_token=None,
|
||||
block_idx_first_scheduled_token_p=None,
|
||||
block_idx_last_computed_token=None,
|
||||
block_idx_last_scheduled_token_prev_step=None,
|
||||
num_computed_tokens_p=None,
|
||||
seq_lens=torch.tensor([5]),
|
||||
)
|
||||
|
||||
@@ -145,11 +140,6 @@ def test_short_conv_forward_native_decode(vllm_config):
|
||||
state_indices_tensor_d=state_indices_tensor_d,
|
||||
num_accepted_tokens=None,
|
||||
query_start_loc_d=torch.tensor([0, 1, 2], dtype=torch.int32),
|
||||
block_idx_last_scheduled_token=None,
|
||||
block_idx_first_scheduled_token_p=None,
|
||||
block_idx_last_computed_token=None,
|
||||
block_idx_last_scheduled_token_prev_step=None,
|
||||
num_computed_tokens_p=None,
|
||||
seq_lens=torch.tensor([1, 1]),
|
||||
)
|
||||
|
||||
|
||||
@@ -176,7 +176,7 @@ def test_forward_core_split_matches_unified(
|
||||
builder = GDNAttentionMetadataBuilder(
|
||||
# GDNAttentionMetadataBuilder declares kv_cache_spec as AttentionSpec but
|
||||
# asserts isinstance(kv_cache_spec, MambaSpec). MambaSpec is correct here.
|
||||
kv_cache_spec=MambaSpec( # type: ignore[arg-type]
|
||||
kv_cache_spec=MambaSpec(
|
||||
block_size=BLOCK_SIZE, shapes=((16, 64),), dtypes=(torch.float16,)
|
||||
),
|
||||
layer_names=[PREFIX],
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.mamba import MambaBackendEnum, MambaConfig, MambaSSUAlgorithm
|
||||
from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateShapeCalculator
|
||||
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
|
||||
FlashInferSSUBackend,
|
||||
TritonSSUBackend,
|
||||
commit_replayssm_ring_trackers,
|
||||
get_mamba_ssu_backend,
|
||||
initialize_mamba_ssu_backend,
|
||||
reset_replayssm_ring_trackers,
|
||||
selective_state_update,
|
||||
selective_state_update_replayssm_flashinfer,
|
||||
update_replayssm_ring_trackers,
|
||||
)
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadata
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
@@ -32,6 +38,13 @@ try:
|
||||
except ImportError:
|
||||
HAS_FLASHINFER = False
|
||||
|
||||
try:
|
||||
from flashinfer.mamba.checkpointing_ssu import CheckpointingSSURunner
|
||||
|
||||
HAS_FLASHINFER_CHECKPOINTING_SSU = CheckpointingSSURunner is not None
|
||||
except ImportError:
|
||||
HAS_FLASHINFER_CHECKPOINTING_SSU = False
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def restore_backend_state():
|
||||
@@ -47,6 +60,7 @@ def restore_backend_state():
|
||||
def test_flashinfer_replayssm_ring_tracker_lifecycle():
|
||||
ring_start = torch.zeros(2, dtype=torch.int32, device="cuda")
|
||||
prev_num_accepted = torch.zeros(2, dtype=torch.int32, device="cuda")
|
||||
prev_query_len = torch.zeros(2, dtype=torch.int32, device="cuda")
|
||||
state_batch_indices = torch.tensor([1], dtype=torch.int32, device="cuda")
|
||||
|
||||
observed = []
|
||||
@@ -54,6 +68,7 @@ def test_flashinfer_replayssm_ring_tracker_lifecycle():
|
||||
update_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_batch_indices,
|
||||
logical_window=16,
|
||||
ring_buffer_len=17,
|
||||
@@ -69,9 +84,205 @@ def test_flashinfer_replayssm_ring_tracker_lifecycle():
|
||||
reset_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_batch_indices,
|
||||
)
|
||||
assert (ring_start[1].item(), prev_num_accepted[1].item()) == (0, 0)
|
||||
assert (
|
||||
ring_start[1].item(),
|
||||
prev_num_accepted[1].item(),
|
||||
prev_query_len[1].item(),
|
||||
) == (0, 0, 0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("accepted_sequence", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
[4] * 22,
|
||||
[
|
||||
(0, 0, 4),
|
||||
(0, 4, 4),
|
||||
(0, 8, 4),
|
||||
(0, 12, 4),
|
||||
(0, 16, 4),
|
||||
(16, 4, 4),
|
||||
(16, 8, 4),
|
||||
(16, 12, 4),
|
||||
(16, 16, 4),
|
||||
(12, 4, 4),
|
||||
(12, 8, 4),
|
||||
(12, 12, 4),
|
||||
(12, 16, 4),
|
||||
(8, 4, 4),
|
||||
(8, 8, 4),
|
||||
(8, 12, 4),
|
||||
(8, 16, 4),
|
||||
(4, 4, 4),
|
||||
(4, 8, 4),
|
||||
(4, 12, 4),
|
||||
(4, 16, 4),
|
||||
(0, 4, 4),
|
||||
],
|
||||
id="all-accepted",
|
||||
),
|
||||
pytest.param(
|
||||
[4, 4, 0, 3, 4, 2, 4, 1],
|
||||
[
|
||||
(0, 0, 4),
|
||||
(0, 4, 4),
|
||||
(0, 4, 4),
|
||||
(0, 7, 4),
|
||||
(0, 11, 4),
|
||||
(0, 13, 4),
|
||||
(13, 4, 4),
|
||||
(13, 5, 4),
|
||||
],
|
||||
id="mixed",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_replayssm_commit_tracker_acceptance_sequence(accepted_sequence, expected):
|
||||
logical_window = 16
|
||||
num_speculative_tokens = 3
|
||||
query_len = 1 + num_speculative_tokens
|
||||
ring_buffer_len = logical_window + 1 + num_speculative_tokens
|
||||
ring_start = torch.zeros(2, dtype=torch.int32, device="cuda")
|
||||
prev_num_accepted = torch.zeros(2, dtype=torch.int32, device="cuda")
|
||||
prev_query_len = torch.zeros(2, dtype=torch.int32, device="cuda")
|
||||
state_batch_indices = torch.tensor([1], dtype=torch.int32, device="cuda")
|
||||
query_start_loc = torch.tensor([0, query_len], dtype=torch.int32, device="cuda")
|
||||
|
||||
observed = []
|
||||
for accepted in accepted_sequence:
|
||||
commit_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_batch_indices,
|
||||
torch.tensor([accepted], dtype=torch.int32, device="cuda"),
|
||||
query_start_loc,
|
||||
logical_window,
|
||||
ring_buffer_len,
|
||||
)
|
||||
snapshot = (
|
||||
ring_start[1].item(),
|
||||
prev_num_accepted[1].item(),
|
||||
prev_query_len[1].item(),
|
||||
)
|
||||
observed.append(snapshot)
|
||||
assert snapshot[1] + snapshot[2] <= ring_buffer_len
|
||||
|
||||
assert observed == expected
|
||||
|
||||
|
||||
def test_replayssm_resume_resets_commit_history():
|
||||
ring_start = torch.tensor([0, 13], dtype=torch.int32, device="cuda")
|
||||
prev_num_accepted = torch.tensor([0, 13], dtype=torch.int32, device="cuda")
|
||||
prev_query_len = torch.tensor([0, 4], dtype=torch.int32, device="cuda")
|
||||
state_batch_indices = torch.tensor([1], dtype=torch.int32, device="cuda")
|
||||
|
||||
reset_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_batch_indices,
|
||||
)
|
||||
assert (
|
||||
ring_start[1].item(),
|
||||
prev_num_accepted[1].item(),
|
||||
prev_query_len[1].item(),
|
||||
) == (0, 0, 0)
|
||||
|
||||
commit_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_batch_indices,
|
||||
torch.tensor([3], dtype=torch.int32, device="cuda"),
|
||||
torch.tensor([0, 4], dtype=torch.int32, device="cuda"),
|
||||
logical_window=16,
|
||||
ring_buffer_len=20,
|
||||
)
|
||||
assert (
|
||||
ring_start[1].item(),
|
||||
prev_num_accepted[1].item(),
|
||||
prev_query_len[1].item(),
|
||||
) == (0, 0, 4)
|
||||
|
||||
|
||||
def test_replayssm_commit_tracker_ragged_query_lengths():
|
||||
ring_start = torch.zeros(3, dtype=torch.int32, device="cuda")
|
||||
prev_num_accepted = torch.zeros(3, dtype=torch.int32, device="cuda")
|
||||
prev_query_len = torch.zeros(3, dtype=torch.int32, device="cuda")
|
||||
state_batch_indices = torch.tensor([1, 2], dtype=torch.int32, device="cuda")
|
||||
query_start_loc = torch.tensor([0, 4, 6], dtype=torch.int32, device="cuda")
|
||||
|
||||
observed = []
|
||||
for accepted in ([4, 2], [3, 1]):
|
||||
commit_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_batch_indices,
|
||||
torch.tensor(accepted, dtype=torch.int32, device="cuda"),
|
||||
query_start_loc,
|
||||
logical_window=16,
|
||||
ring_buffer_len=20,
|
||||
)
|
||||
observed.append(
|
||||
[
|
||||
(
|
||||
ring_start[slot].item(),
|
||||
prev_num_accepted[slot].item(),
|
||||
prev_query_len[slot].item(),
|
||||
)
|
||||
for slot in (1, 2)
|
||||
]
|
||||
)
|
||||
|
||||
assert observed == [[(0, 0, 4), (0, 0, 2)], [(0, 3, 4), (0, 1, 2)]]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("operation", ["commit", "reset"])
|
||||
def test_replayssm_tracker_kernels_mask_invalid_slots(operation):
|
||||
num_states = 3
|
||||
ring_start = torch.tensor([11, 2, 33], dtype=torch.int32, device="cuda")
|
||||
prev_num_accepted = torch.tensor([11, 3, 33], dtype=torch.int32, device="cuda")
|
||||
prev_query_len = torch.tensor([11, 4, 33], dtype=torch.int32, device="cuda")
|
||||
state_batch_indices = torch.tensor(
|
||||
[-1, num_states, NULL_BLOCK_ID, 1], dtype=torch.int32, device="cuda"
|
||||
)
|
||||
|
||||
if operation == "commit":
|
||||
commit_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_batch_indices,
|
||||
torch.tensor([4, 4, 4, 2], dtype=torch.int32, device="cuda"),
|
||||
torch.tensor([0, 4, 8, 12, 16], dtype=torch.int32, device="cuda"),
|
||||
logical_window=16,
|
||||
ring_buffer_len=20,
|
||||
)
|
||||
expected_valid = (2, 5, 4)
|
||||
else:
|
||||
reset_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_batch_indices,
|
||||
)
|
||||
expected_valid = (0, 0, 0)
|
||||
|
||||
assert (
|
||||
ring_start.tolist(),
|
||||
prev_num_accepted.tolist(),
|
||||
prev_query_len.tolist(),
|
||||
) == (
|
||||
[11, expected_valid[0], 33],
|
||||
[11, expected_valid[1], 33],
|
||||
[11, expected_valid[2], 33],
|
||||
)
|
||||
|
||||
|
||||
def _kv_cache_config_with_ssu(
|
||||
@@ -231,14 +442,185 @@ def test_triton_basic_call():
|
||||
assert not torch.isnan(out).any()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("layout", ["packed", "dense"])
|
||||
def test_replayssm_flashinfer_call_forwards_mtp_layout(monkeypatch, layout):
|
||||
import vllm.model_executor.layers.mamba.ops.ssu_dispatch as mod
|
||||
|
||||
kernel = Mock(return_value=torch.empty(0))
|
||||
monkeypatch.setattr(mod, "_flashinfer_replayssm_kernel", kernel)
|
||||
|
||||
batch, max_seqlen, nheads, dim, dstate, ngroups = 2, 4, 2, 4, 8, 1
|
||||
state = torch.empty(2, nheads, dim, dstate)
|
||||
x_shape: tuple[int, ...]
|
||||
B_shape: tuple[int, ...]
|
||||
expected_x_shape: tuple[int, ...]
|
||||
expected_B_shape: tuple[int, ...]
|
||||
if layout == "packed":
|
||||
x_shape = (6, nheads, dim)
|
||||
B_shape = (6, ngroups, dstate)
|
||||
expected_x_shape = (1, 6, nheads, dim)
|
||||
expected_B_shape = (1, 6, ngroups, dstate)
|
||||
cu_seqlens = torch.tensor([0, 4, 6], dtype=torch.int32)
|
||||
kernel_max_seqlen = max_seqlen
|
||||
else:
|
||||
x_shape = (batch, max_seqlen, nheads, dim)
|
||||
B_shape = (batch, max_seqlen, ngroups, dstate)
|
||||
expected_x_shape = x_shape
|
||||
expected_B_shape = B_shape
|
||||
cu_seqlens = None
|
||||
kernel_max_seqlen = None
|
||||
x = torch.empty(x_shape)
|
||||
dt = torch.empty_like(x)
|
||||
A = torch.empty(nheads, dim, dstate)
|
||||
B = torch.empty(B_shape)
|
||||
C = torch.empty_like(B)
|
||||
out = torch.empty_like(x)
|
||||
x_cache = torch.empty(2, nheads, 20, dim)
|
||||
dt_cache = torch.empty(2, nheads, 20)
|
||||
B_cache = torch.empty(2, ngroups, 20, dstate)
|
||||
ring_start = torch.zeros(2, dtype=torch.int32)
|
||||
prev_num_accepted = torch.zeros(2, dtype=torch.int32)
|
||||
prev_query_len = torch.zeros(2, dtype=torch.int32)
|
||||
selective_state_update_replayssm_flashinfer(
|
||||
state,
|
||||
x,
|
||||
dt,
|
||||
A,
|
||||
B,
|
||||
C,
|
||||
out,
|
||||
x_cache,
|
||||
B_cache,
|
||||
dt_cache,
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
logical_window=16,
|
||||
state_batch_indices=torch.tensor([0, 1], dtype=torch.int32),
|
||||
cu_seqlens=cu_seqlens,
|
||||
max_seqlen=kernel_max_seqlen,
|
||||
update_trackers=False,
|
||||
)
|
||||
|
||||
args = kernel.call_args.args
|
||||
assert args[6].shape == expected_x_shape
|
||||
assert args[7].shape == expected_x_shape
|
||||
assert args[9].shape == expected_B_shape
|
||||
assert args[10].shape == expected_B_shape
|
||||
assert args[11].shape == expected_x_shape
|
||||
assert kernel.call_args.kwargs["cu_seqlens"] is cu_seqlens
|
||||
assert kernel.call_args.kwargs["max_seqlen"] == kernel_max_seqlen
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("backend", "expected_ring_len"),
|
||||
("query_start_loc", "expected_shape", "expected_max_seqlen"),
|
||||
[
|
||||
(MambaBackendEnum.TRITON, 16),
|
||||
(MambaBackendEnum.FLASHINFER, 17),
|
||||
pytest.param([0, 4, 8], (2, 4, 2, 4), None, id="dense"),
|
||||
pytest.param([0, 4, 6], (6, 2, 4), 4, id="packed"),
|
||||
],
|
||||
)
|
||||
def test_replayssm_physical_ring_shape(backend, expected_ring_len):
|
||||
def test_replayssm_mixer_selects_mtp_layout(
|
||||
monkeypatch, query_start_loc, expected_shape, expected_max_seqlen
|
||||
):
|
||||
import vllm.model_executor.layers.mamba.mamba_mixer2 as mod
|
||||
|
||||
mixer = MambaMixer2.__new__(MambaMixer2)
|
||||
torch.nn.Module.__init__(mixer)
|
||||
mixer.prefix = "mixer"
|
||||
mixer.tped_intermediate_size = 0
|
||||
mixer.tped_conv_size = 1
|
||||
mixer.tped_dt_size = 2
|
||||
mixer.num_heads = 2
|
||||
mixer.head_dim = 4
|
||||
mixer.n_groups = mixer.tp_size = 1
|
||||
mixer.ssm_state_size = 8
|
||||
mixer.num_spec = 3
|
||||
mixer.use_replayssm = True
|
||||
mixer.replayssm_buffer_len = 16
|
||||
mixer._commits_replayssm_trackers = True
|
||||
mixer._updates_replayssm_trackers = False
|
||||
mixer.mamba_config = MambaConfig(backend=MambaBackendEnum.FLASHINFER)
|
||||
mixer.cache_config = SimpleNamespace(mamba_block_size=16, mamba_cache_mode="none")
|
||||
mixer.conv_weights = torch.empty(0)
|
||||
mixer.conv1d = SimpleNamespace(bias=None)
|
||||
mixer.activation = "silu"
|
||||
mixer.A = torch.empty(2)
|
||||
mixer.dt_bias = torch.empty(2)
|
||||
mixer.D = torch.empty(2)
|
||||
mixer._replayssm_ring_start = torch.zeros(3, dtype=torch.int32)
|
||||
mixer._replayssm_prev_num_accepted = torch.zeros(3, dtype=torch.int32)
|
||||
mixer._replayssm_prev_query_len = torch.zeros(3, dtype=torch.int32)
|
||||
mixer.kv_cache = (
|
||||
torch.empty(3, 1),
|
||||
torch.empty(3, 2, 4, 8),
|
||||
torch.empty(3, 2, 20, 4),
|
||||
torch.empty(3, 2, 20),
|
||||
torch.empty(3, 1, 20, 8),
|
||||
)
|
||||
|
||||
num_decode_tokens = query_start_loc[-1]
|
||||
query_start_loc_d = torch.tensor(query_start_loc, dtype=torch.int32)
|
||||
metadata = Mamba2AttentionMetadata(
|
||||
num_prefills=0,
|
||||
num_prefill_tokens=0,
|
||||
num_decodes=2,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
num_reqs=2,
|
||||
has_initial_states_p=None,
|
||||
query_start_loc_p=None,
|
||||
state_indices_tensor_p=None,
|
||||
state_indices_tensor_d=torch.tensor([[1], [2]], dtype=torch.int32),
|
||||
query_start_loc_d=query_start_loc_d,
|
||||
num_accepted_tokens=torch.tensor([4, 2], dtype=torch.int32),
|
||||
seq_lens=torch.tensor([104, 102], dtype=torch.int32),
|
||||
replayssm_scratch=(torch.empty(0), torch.empty(0), torch.empty(0)),
|
||||
replayssm_state_indices_d=torch.tensor([1, 2], dtype=torch.int32),
|
||||
)
|
||||
|
||||
def split_hidden_states_B_C(values):
|
||||
tokens = values.size(0)
|
||||
return (
|
||||
torch.empty(tokens, 8),
|
||||
torch.empty(tokens, 8),
|
||||
torch.empty(tokens, 8),
|
||||
)
|
||||
|
||||
mixer.split_hidden_states_B_C_fn = split_hidden_states_B_C
|
||||
kernel = Mock()
|
||||
monkeypatch.setattr(
|
||||
mod,
|
||||
"get_forward_context",
|
||||
lambda: SimpleNamespace(attn_metadata={mixer.prefix: metadata}),
|
||||
)
|
||||
monkeypatch.setattr(mod, "commit_replayssm_ring_trackers", Mock())
|
||||
monkeypatch.setattr(
|
||||
mod, "causal_conv1d_update", lambda values, *args, **kwargs: values
|
||||
)
|
||||
monkeypatch.setattr(mod, "selective_state_update_replayssm_flashinfer", kernel)
|
||||
|
||||
mixer.conv_ssm_forward(
|
||||
torch.empty(num_decode_tokens, 3), torch.empty(num_decode_tokens, 8)
|
||||
)
|
||||
|
||||
assert kernel.call_args.args[1].shape == expected_shape
|
||||
if expected_max_seqlen is None:
|
||||
assert kernel.call_args.kwargs["cu_seqlens"] is None
|
||||
else:
|
||||
assert kernel.call_args.kwargs["cu_seqlens"] is query_start_loc_d
|
||||
assert kernel.call_args.kwargs["max_seqlen"] == expected_max_seqlen
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("backend", "num_speculative_tokens", "expected_ring_len"),
|
||||
[
|
||||
(MambaBackendEnum.TRITON, 0, 16),
|
||||
(MambaBackendEnum.FLASHINFER, 0, 17),
|
||||
(MambaBackendEnum.FLASHINFER, 3, 20),
|
||||
],
|
||||
)
|
||||
def test_replayssm_physical_ring_shape(
|
||||
backend, num_speculative_tokens, expected_ring_len
|
||||
):
|
||||
base_shapes = ((64, 3), (8, 4, 16))
|
||||
|
||||
shapes = MambaStateShapeCalculator.append_replayssm_ring(
|
||||
@@ -247,6 +629,7 @@ def test_replayssm_physical_ring_shape(backend, expected_ring_len):
|
||||
tp_world_size=2,
|
||||
logical_window=16,
|
||||
backend=backend,
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
)
|
||||
|
||||
assert shapes[2:] == (
|
||||
|
||||
Reference in New Issue
Block a user