[CPU] Conv1d optimised kernel for aarch64 (#54093)

Signed-off-by: Anna Mayne <anna.mayne@arm.com>
Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
almayne
2026-09-30 20:23:42 +08:00
committed by GitHub
co-authored by Li, Jiang <jiang1.li@intel.com>
parent df5668e5c9
commit 93dd80f636
9 changed files with 460 additions and 205 deletions
+154 -102
View File
@@ -494,8 +494,10 @@ def _sd_conv_states(
return storage.transpose(1, 2)
def _maybe_pack_conv_weight(weight: torch.Tensor, is_vnni: bool) -> torch.Tensor:
return ops.causal_conv1d_weight_pack(weight) if is_vnni else weight
def _maybe_pack_conv_weight(
weight: torch.Tensor, is_weight_packed: bool
) -> torch.Tensor:
return ops.causal_conv1d_weight_pack(weight) if is_weight_packed else weight
@torch.inference_mode()
@@ -523,7 +525,7 @@ def test_causal_conv1d_update_cpu_preserves_sd_state(
bias=None,
silu_activation=False,
conv_state_indices=state_indices,
is_vnni=False,
is_weight_packed=False,
)
assert out is x
@@ -666,7 +668,10 @@ def test_spec_aware_nonspec_materializes_state_indices(
@torch.inference_mode()
@pytest.mark.parametrize(("has_amx", "has_arm_bf16"), [(True, False), (False, True)])
def test_spec_forward_prepares_native_conv_metadata(
has_amx: bool,
has_arm_bf16: bool,
monkeypatch: pytest.MonkeyPatch,
) -> None:
block_table = torch.tensor([[0, 1], [4, 5]], dtype=torch.int32)
@@ -687,14 +692,17 @@ def test_spec_forward_prepares_native_conv_metadata(
)
forwarded_indices = None
forwarded_counts = None
forwarded_is_weight_packed = None
def causal_conv1d_update_cpu(**kwargs):
nonlocal forwarded_counts, forwarded_indices
nonlocal forwarded_counts, forwarded_indices, forwarded_is_weight_packed
forwarded_indices = kwargs["conv_state_indices"]
forwarded_counts = kwargs["num_accepted_tokens"]
forwarded_is_weight_packed = kwargs["is_weight_packed"]
return kwargs["x"]
monkeypatch.setattr(torch.cpu, "_is_amx_tile_supported", lambda: True)
monkeypatch.setattr(torch.cpu, "_is_amx_tile_supported", lambda: has_amx)
monkeypatch.setattr(gdn_attention, "is_arm_bf16", lambda: has_arm_bf16)
monkeypatch.setattr(gdn_attention, "is_conv_state_dim_first", lambda: False)
monkeypatch.setattr(
gdn_attention.ops, "causal_conv1d_update_cpu", causal_conv1d_update_cpu
@@ -732,6 +740,7 @@ def test_spec_forward_prepares_native_conv_metadata(
assert actual is not None
assert actual.is_contiguous()
assert actual.dtype == torch.int32
assert forwarded_is_weight_packed is True
torch.testing.assert_close(actual, reference)
@@ -792,15 +801,15 @@ def test_causal_conv1d_torch_two_call_split(total_tokens: int, split: int) -> No
@pytest.mark.skipif(
not torch.cpu._is_amx_tile_supported(),
reason="requires AMX support",
not (torch.cpu._is_amx_tile_supported() or gdn_attention.is_arm_bf16()),
reason="native causal conv1d requires AMX or Arm BF16 support",
)
@torch.inference_mode()
def test_causal_conv1d_update_cpu_accepts_wide_state() -> None:
state_len = CONV_KERNEL - 1
wide_state_len = state_len + 5
batch_size = 3
is_vnni = True
is_weight_packed = True
x, weight, bias = _conv_inputs(batch_size)
conv_state_indices = torch.tensor([2, 0, 1], dtype=torch.int32)
@@ -813,7 +822,7 @@ def test_causal_conv1d_update_cpu_accepts_wide_state() -> None:
wide_state[:, :, state_len:].fill_(7)
wide_tail = wide_state[:, :, state_len:].clone()
conv_weight = _maybe_pack_conv_weight(weight, is_vnni)
conv_weight = _maybe_pack_conv_weight(weight, is_weight_packed)
out_narrow = ops.causal_conv1d_update_cpu(
x=x,
conv_states=narrow_state,
@@ -821,7 +830,7 @@ def test_causal_conv1d_update_cpu_accepts_wide_state() -> None:
bias=bias,
silu_activation=True,
conv_state_indices=conv_state_indices,
is_vnni=is_vnni,
is_weight_packed=is_weight_packed,
)
out_wide = ops.causal_conv1d_update_cpu(
x=x,
@@ -830,7 +839,7 @@ def test_causal_conv1d_update_cpu_accepts_wide_state() -> None:
bias=bias,
silu_activation=True,
conv_state_indices=conv_state_indices,
is_vnni=is_vnni,
is_weight_packed=is_weight_packed,
)
torch.testing.assert_close(out_wide, out_narrow, atol=1e-2, rtol=1e-2)
@@ -872,11 +881,18 @@ def _ref_causal_conv1d_update_cpu_multi(
@pytest.mark.skipif(
not torch.cpu._is_amx_tile_supported(),
reason="requires AMX support",
not (torch.cpu._is_amx_tile_supported() or gdn_attention.is_arm_bf16()),
reason="native causal conv1d requires AMX or Arm BF16 support",
)
@pytest.mark.parametrize(
("batch_size, seq_len, accepted_counts, has_bias, silu_activation, is_vnni"),
(
"batch_size, "
"seq_len, "
"accepted_counts, "
"has_bias, "
"silu_activation, "
"is_weight_packed"
),
[
(1, 1, [1], False, False, False),
(1, 1, [1], True, True, True),
@@ -893,7 +909,7 @@ def test_causal_conv1d_update_cpu_multi_token_matches_python(
accepted_counts: list[int],
has_bias: bool,
silu_activation: bool,
is_vnni: bool,
is_weight_packed: bool,
) -> None:
dim = 96
state_len = seq_len + 2
@@ -911,7 +927,7 @@ def test_causal_conv1d_update_cpu_multi_token_matches_python(
)
conv_states = conv_states_ref.clone()
conv_weight = _maybe_pack_conv_weight(weight, is_vnni)
conv_weight = _maybe_pack_conv_weight(weight, is_weight_packed)
out = ops.causal_conv1d_update_cpu(
x=x,
conv_states=conv_states,
@@ -919,7 +935,7 @@ def test_causal_conv1d_update_cpu_multi_token_matches_python(
bias=bias,
silu_activation=silu_activation,
conv_state_indices=conv_state_indices,
is_vnni=is_vnni,
is_weight_packed=is_weight_packed,
num_accepted_tokens=num_accepted_tokens,
)
ref_out = _ref_causal_conv1d_update_cpu_multi(
@@ -937,8 +953,8 @@ def test_causal_conv1d_update_cpu_multi_token_matches_python(
@pytest.mark.skipif(
not torch.cpu._is_amx_tile_supported(),
reason="requires AMX support",
not (torch.cpu._is_amx_tile_supported() or gdn_attention.is_arm_bf16()),
reason="native causal conv1d requires AMX or Arm BF16 support",
)
@pytest.mark.parametrize("num_accepted", [0, 17])
@torch.inference_mode()
@@ -961,14 +977,15 @@ def test_causal_conv1d_update_cpu_rejects_invalid_accepted_count(
bias=None,
silu_activation=True,
conv_state_indices=torch.tensor([0], dtype=torch.int32),
is_vnni=False,
is_weight_packed=False,
num_accepted_tokens=torch.tensor([num_accepted], dtype=torch.int32),
)
@pytest.mark.skipif(
not torch.cpu._is_avx512_bf16_supported(),
reason="causal_conv1d_fwd_cpu requires AVX-512BF16 (Intel Xeon or AMD EPYC)",
not (torch.cpu._is_avx512_bf16_supported() or gdn_attention.is_arm_bf16()),
reason="causal_conv1d_fwd_cpu requires AVX-512BF16"
" (Intel Xeon or AMD EPYC) or Arm BF16 support",
)
@pytest.mark.parametrize("total_tokens, split", TWO_CALL_SPLITS)
@torch.inference_mode()
@@ -977,13 +994,13 @@ def test_causal_conv1d_fwd_cpu_two_call_split(total_tokens: int, split: int) ->
matches the single-call result.
Regression test for ``causal_conv1d_fwd_varlen_kernel_impl`` (``conv.cpp``)
ignoring the carried conv state on continued chunks. Runs on any
AVX-512BF16 CPU since conv.cpp uses VDPBF16PS, not AMX tiles.
ignoring the carried conv state on continued chunks. Runs on aarch64 cpus with
bf16 support or any AVX-512BF16 CPU since conv.cpp uses VDPBF16PS, not AMX tiles.
"""
state_len = CONV_KERNEL - 1
x, weight, bias = _conv_inputs(total_tokens)
def amx(x_seg, conv_states, has_init):
def native(x_seg, conv_states, has_init):
seq = x_seg.shape[0]
return ops.causal_conv1d_fwd_cpu(
x=x_seg.transpose(0, 1), # [dim, seq]; stride(-2)==1 (view of [seq,dim])
@@ -994,30 +1011,30 @@ def test_causal_conv1d_fwd_cpu_two_call_split(total_tokens: int, split: int) ->
cache_indices=torch.tensor([0], dtype=torch.int32),
has_initial_state=torch.tensor([has_init]),
silu_activation=True,
is_vnni=False,
is_weight_packed=False,
).contiguous()
# conv_state layout passed by the AMX branch: [num_slots, dim, state_len].
cs_full = torch.zeros(1, CONV_DIM, state_len, dtype=x.dtype)
out_full = amx(x, cs_full, False)
out_full = native(x, cs_full, False)
cs_split = torch.zeros(1, CONV_DIM, state_len, dtype=x.dtype)
out1 = amx(x[:split], cs_split, False)
out2 = amx(x[split:], cs_split, True)
out1 = native(x[:split], cs_split, False)
out2 = native(x[split:], cs_split, True)
out_split = torch.cat([out1, out2], dim=1)
torch.testing.assert_close(out_split, out_full, atol=1e-2, rtol=1e-2)
@pytest.mark.skipif(
not torch.cpu._is_amx_tile_supported(),
reason="requires AMX support",
not (torch.cpu._is_amx_tile_supported() or gdn_attention.is_arm_bf16()),
reason="native causal conv1d requires AMX or Arm BF16 support",
)
@torch.inference_mode()
def test_causal_conv1d_fwd_cpu_accepts_wide_state() -> None:
state_len = CONV_KERNEL - 1
wide_state_len = state_len + 5
is_vnni = True
is_weight_packed = True
seq_lens = [CHUNK_SIZE - 1, CHUNK_SIZE + 5]
total_tokens = sum(seq_lens)
x, weight, bias = _conv_inputs(total_tokens)
@@ -1034,7 +1051,7 @@ def test_causal_conv1d_fwd_cpu_accepts_wide_state() -> None:
wide_state[:, :, state_len:].fill_(7)
wide_tail = wide_state[:, :, state_len:].clone()
conv_weight = _maybe_pack_conv_weight(weight, is_vnni)
conv_weight = _maybe_pack_conv_weight(weight, is_weight_packed)
out_narrow = ops.causal_conv1d_fwd_cpu(
x=x.transpose(0, 1),
weight=conv_weight,
@@ -1044,7 +1061,7 @@ def test_causal_conv1d_fwd_cpu_accepts_wide_state() -> None:
cache_indices=cache_indices,
has_initial_state=has_initial_state,
silu_activation=True,
is_vnni=is_vnni,
is_weight_packed=is_weight_packed,
)
out_wide = ops.causal_conv1d_fwd_cpu(
x=x.transpose(0, 1),
@@ -1055,7 +1072,7 @@ def test_causal_conv1d_fwd_cpu_accepts_wide_state() -> None:
cache_indices=cache_indices,
has_initial_state=has_initial_state,
silu_activation=True,
is_vnni=is_vnni,
is_weight_packed=is_weight_packed,
)
torch.testing.assert_close(out_wide, out_narrow, atol=1e-2, rtol=1e-2)
@@ -1100,15 +1117,23 @@ def test_batch_memcpy_cpu_fallback() -> None:
# ---------------------------------------------------------------------------
# C++ conv (conv.cpp) uses VDPBF16PS, not AMX tiles, so it runs on any
# AVX-512BF16 CPU; weight is VNNI-packed on this same predicate at load time.
# C++ conv (conv.cpp) uses AArch64BF16 or VDPBF16PS, not AMX tiles, so it can
# run on any AVX-512BF16 CPU; weight is packed on this same predicate at load time.
# ---------------------------------------------------------------------------
_HAS_AVX512_BF16 = torch.cpu._is_avx512_bf16_supported()
_STATE_LEN = CONV_KERNEL - 1
CONV_EQUIV_SEQ_LENS = [[1], [7], [64], [65], [1, 2, 3], [63, 64, 65], [128, 129]]
CONV_EQUIV_SEQ_LENS = [
[1],
[7],
[64],
[65],
[1, 2, 3],
[63, 64, 65],
[128, 129],
]
def _conv_fp32_oracle(x, weight, bias, seq_lens, activation="silu"):
@@ -1153,10 +1178,10 @@ def _run_prefill_torch(x, weight, bias, seq_lens):
return out.transpose(0, 1).contiguous(), conv_states
def _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=False):
def _run_prefill_cpp(x, weight, bias, seq_lens, is_weight_packed=False):
num_seqs = len(seq_lens)
packed_w = ops.causal_conv1d_weight_pack(weight) if is_vnni else weight
if is_vnni:
packed_w = ops.causal_conv1d_weight_pack(weight) if is_weight_packed else weight
if is_weight_packed:
# C++-branch layout: kv-cache "SD" [slots, state_len, dim] transposed to
# [slots, dim, state_len] (a non-contiguous view).
conv_state = torch.zeros(
@@ -1176,7 +1201,7 @@ def _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=False):
cache_indices=torch.arange(num_seqs, dtype=torch.int32),
has_initial_state=torch.zeros(num_seqs, dtype=torch.bool),
silu_activation=True,
is_vnni=is_vnni,
is_weight_packed=is_weight_packed,
)
return out.transpose(0, 1).contiguous(), conv_state
@@ -1245,7 +1270,7 @@ def test_causal_conv1d_fwd_cpu_ds_adapter_varlen_mixed_initial_state(
cache_indices=state_indices,
has_initial_state=has_initial_state,
silu_activation=True,
is_vnni=False,
is_weight_packed=False,
)
monkeypatch.setattr(gdn_attention, "is_conv_state_dim_first", lambda: True)
out_ds = gdn_attention._causal_conv1d_fwd_cpu(
@@ -1257,7 +1282,7 @@ def test_causal_conv1d_fwd_cpu_ds_adapter_varlen_mixed_initial_state(
cache_indices=state_indices,
has_initial_state=has_initial_state,
silu_activation=True,
is_vnni=False,
is_weight_packed=False,
)
torch.testing.assert_close(out_ds, out_sd)
@@ -1302,7 +1327,7 @@ def test_causal_conv1d_update_cpu_ds_adapter_wide_multi_token_copyback(
bias=None,
silu_activation=False,
conv_state_indices=state_indices,
is_vnni=False,
is_weight_packed=False,
num_accepted_tokens=num_accepted_tokens,
)
@@ -1321,7 +1346,8 @@ def test_causal_conv1d_update_cpu_ds_adapter_wide_multi_token_copyback(
@pytest.mark.skipif(
not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16"
not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()),
reason="C++ causal_conv1d requires AVX-512BF16 or Arm BF16 support",
)
@pytest.mark.parametrize("seq_lens", CONV_EQUIV_SEQ_LENS)
@torch.inference_mode()
@@ -1335,7 +1361,8 @@ def test_conv_cpp_matches_torch(seq_lens):
@pytest.mark.skipif(
not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16"
not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()),
reason="C++ causal_conv1d requires AVX-512BF16 or Arm BF16 support",
)
@pytest.mark.parametrize("seq_lens", CONV_EQUIV_SEQ_LENS)
@torch.inference_mode()
@@ -1354,89 +1381,114 @@ def test_conv_cpp_no_worse_than_torch_vs_fp32(seq_lens):
@pytest.mark.skipif(
not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16"
not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()),
reason="C++ causal_conv1d requires AVX-512BF16 or Arm BF16 support",
)
@pytest.mark.parametrize("seq_lens", CONV_EQUIV_SEQ_LENS)
@torch.inference_mode()
def test_conv_cpp_vnni_packed_matches_torch(seq_lens):
"""The exact runtime prefill sequence (VNNI-packed weight + SD-layout
conv_state view + is_vnni=True) must match the torch fallback. Validates
the packing + layout handoff on any AVX-512BF16 CPU."""
def test_conv_cpp_weight_packed_matches_torch(seq_lens):
"""The exact runtime prefill sequence (packed weight + SD-layout
conv_state view + is_weight_packed=True) must match the torch fallback. Validates
the native packing + layout handoff."""
x, weight, bias = _conv_inputs(sum(seq_lens))
out_torch, _ = _run_prefill_torch(x, weight, bias, seq_lens)
out_vnni, _ = _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=True)
torch.testing.assert_close(out_vnni, out_torch, atol=1e-2, rtol=1e-2)
out_weight_packed, _ = _run_prefill_cpp(
x, weight, bias, seq_lens, is_weight_packed=True
)
torch.testing.assert_close(out_weight_packed, out_torch, atol=1e-2, rtol=1e-2)
@pytest.mark.skipif(
not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16"
not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()),
reason="C++ causal_conv1d requires AVX-512BF16 or Arm BF16 support",
)
@pytest.mark.parametrize("batch", DECODE_BATCH_SIZES)
@pytest.mark.parametrize(
("dim", "has_bias", "silu_activation", "is_weight_packed"),
[
(32, False, False, False),
(32, True, True, True),
(96, False, True, True),
(96, True, False, False),
(8192, False, True, False),
(8192, True, True, True),
],
)
@torch.inference_mode()
def test_conv_update_cpp_matches_torch(batch):
"""Decode conv: causal_conv1d_update_cpu matches causal_conv1d_update_torch,
including the in-place conv_state update (the next-step handoff)."""
def test_conv_update_cpp_matches_torch(
batch, dim, has_bias, silu_activation, is_weight_packed
):
"""Decode preserves indexed, padded state across successive signed inputs."""
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
causal_conv1d_update_torch,
)
x = tensor_cache(batch * CONV_DIM, torch.bfloat16).view(batch, CONV_DIM)
weight = tensor_cache(CONV_DIM * CONV_KERNEL, torch.bfloat16).view(
CONV_DIM, CONV_KERNEL
)
bias = tensor_cache(CONV_DIM, torch.bfloat16)
conv_state = tensor_cache(batch * CONV_DIM * _STATE_LEN, torch.bfloat16).view(
batch, CONV_DIM, _STATE_LEN
)
torch.manual_seed(0)
weight = torch.randn(dim, CONV_KERNEL, dtype=torch.bfloat16)
bias = torch.randn(dim, dtype=torch.bfloat16) if has_bias else None
conv_weight = _maybe_pack_conv_weight(weight, is_weight_packed)
indices = torch.arange(batch, 0, -1, dtype=torch.int32)
storage_ref = torch.randn(batch + 2, _STATE_LEN + 2, dim, dtype=torch.bfloat16)
storage_cpp = storage_ref.clone()
cs_cpp = storage_cpp[:, :_STATE_LEN].transpose(1, 2)
cs_torch = conv_state.clone()
out_torch = causal_conv1d_update_torch(
x=x.unsqueeze(-1),
conv_state=cs_torch,
weight=weight,
bias=bias,
activation="silu",
).squeeze(-1)
cs_cpp = conv_state.clone()
out_cpp = ops.causal_conv1d_update_cpu(
x=x.contiguous(),
conv_states=cs_cpp,
weight=weight,
bias=bias,
silu_activation=True,
conv_state_indices=torch.arange(batch, dtype=torch.int32),
is_vnni=False,
)
torch.testing.assert_close(out_cpp, out_torch, atol=1e-2, rtol=1e-2)
torch.testing.assert_close(cs_cpp, cs_torch, atol=1e-2, rtol=1e-2)
for _ in range(3):
x = torch.randn(batch, dim, dtype=torch.bfloat16)
cs_torch = storage_ref[indices, :_STATE_LEN].transpose(1, 2).contiguous()
out_torch = causal_conv1d_update_torch(
x=x.unsqueeze(-1),
conv_state=cs_torch,
weight=weight,
bias=bias,
activation="silu" if silu_activation else None,
).squeeze(-1)
storage_ref[indices, :_STATE_LEN] = cs_torch.transpose(1, 2)
out_cpp = ops.causal_conv1d_update_cpu(
x=x,
conv_states=cs_cpp,
weight=conv_weight,
bias=bias,
silu_activation=silu_activation,
conv_state_indices=indices,
is_weight_packed=is_weight_packed,
)
torch.testing.assert_close(out_cpp, out_torch, atol=1e-2, rtol=1e-2)
torch.testing.assert_close(storage_cpp, storage_ref, atol=0, rtol=0)
@pytest.mark.skipif(
not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16"
not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()),
reason="C++ causal_conv1d requires AVX-512BF16 or Arm BF16 support",
)
@torch.inference_mode()
def test_conv_weight_pack_roundtrip_unpacked_matches():
"""`causal_conv1d_weight_pack` repacks the (dim, width) weight, so the C++ conv op
with packed weight + is_vnni=True must equal unpacked weight + is_vnni=False.
Guards the spec-decode contract: the runtime VNNI-packs `layer.conv1d.weight`
in place and stashes the original as `_cpu_unpacked_conv_weight` for the torch
spec-decode path, so both must produce identical math.
with packed weight + is_weight_packed=True must equal unpacked weight
+ is_weight_packed=False. Guards the spec-decode contract: the runtime packs
`layer.conv1d.weight` in place and stashes the original as
`_cpu_unpacked_conv_weight` for the torch spec-decode path, so both must produce
identical math.
"""
seq_lens = [7, 64, 65]
x, weight, bias = _conv_inputs(sum(seq_lens))
out_unpacked, _ = _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=False)
out_packed, _ = _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=True)
torch.testing.assert_close(out_packed, out_unpacked, atol=1e-2, rtol=1e-2)
out_unpacked, state_unpacked = _run_prefill_cpp(
x, weight, bias, seq_lens, is_weight_packed=False
)
out_packed, state_packed = _run_prefill_cpp(
x, weight, bias, seq_lens, is_weight_packed=True
)
torch.testing.assert_close(out_packed, out_unpacked, atol=0, rtol=0)
torch.testing.assert_close(state_packed, state_unpacked, atol=0, rtol=0)
@pytest.mark.skipif(
not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16"
not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()),
reason="C++ causal_conv1d requires AVX-512BF16 or AArch64 BF16",
)
@torch.inference_mode()
def test_spec_decode_unpacked_conv_weight_stash():
"""Spec-decode correctness: AVX-512BF16 CPUs VNNI-pack ``conv1d.weight`` in place
at load time and stash the original ``(dim, width)`` tensor as
"""Spec-decode correctness: AVX-512BF16 and AArch64 BF16 CPUs pack ``conv1d.weight``
in place at load time and stash the original ``(dim, width)`` tensor as
``_cpu_unpacked_conv_weight``; the spec-decode path must use that stash since
reading the packed weight directly produces garbage. Verifies the recovered
weight equals the original and that torch F.conv1d agrees with the C++ conv
@@ -1460,7 +1512,7 @@ def test_spec_decode_unpacked_conv_weight_stash():
# 1. The stash must exist and equal the original (dim, width) weight.
assert hasattr(conv, "_cpu_unpacked_conv_weight"), (
"dispatch_cpu_unquantized_gemm did not stash _cpu_unpacked_conv_weight "
"on an AVX-512BF16 CPU"
"on an AArch64-BF16 or AVX-512BF16 CPU"
)
stash = conv._cpu_unpacked_conv_weight
torch.testing.assert_close(stash, orig_2d, atol=0, rtol=0)
@@ -1469,10 +1521,10 @@ def test_spec_decode_unpacked_conv_weight_stash():
recovered = getattr(conv, "_cpu_unpacked_conv_weight", None)
assert recovered is not None
# conv.weight is now packed; using it directly (the bug) would differ.
packed_weight = conv.weight # [dim, 1, width], VNNI-packed contents
packed_weight = conv.weight # [dim, 1, width], packed contents
# 3. Spec-path torch conv with the recovered unpacked weight must match the
# nonspec C++ conv with the packed weight (is_vnni=True).
# nonspec C++ conv with the packed weight (is_weight_packed=True).
seq_lens = [7, 65]
total = sum(seq_lens)
x = tensor_cache(total * CONV_DIM, torch.bfloat16).view(total, CONV_DIM)
@@ -1494,7 +1546,7 @@ def test_spec_decode_unpacked_conv_weight_stash():
cache_indices=torch.arange(num_seqs, dtype=torch.int32),
has_initial_state=torch.zeros(num_seqs, dtype=torch.bool),
silu_activation=True,
is_vnni=True,
is_weight_packed=True,
)
.transpose(0, 1)
.contiguous()
@@ -6,10 +6,61 @@ import pytest
import torch
from vllm.model_executor.layers import utils
from vllm.platforms import current_platform
from vllm.platforms import CpuArchEnum, current_platform
from vllm.utils.torch_utils import set_default_torch_dtype
def test_dispatch_prepacks_arm_bf16_causal_conv(monkeypatch):
monkeypatch.setattr(
current_platform, "get_cpu_architecture", lambda: CpuArchEnum.ARM
)
monkeypatch.setattr(torch.cpu, "get_capabilities", lambda: {"bf16": True})
monkeypatch.setattr(torch.cpu, "_is_avx512_bf16_supported", lambda: False)
packed = torch.randn(32, 4, dtype=torch.bfloat16)
pack_inputs = []
def pack(weight):
pack_inputs.append(weight.clone())
return packed
monkeypatch.setattr(utils.ops, "causal_conv1d_weight_pack", pack)
original = torch.randn(32, 1, 4, dtype=torch.bfloat16)
layer = torch.nn.Module()
layer.weight = torch.nn.Parameter(original.clone(), requires_grad=False)
utils.dispatch_cpu_unquantized_gemm(layer, remove_weight=False)
assert len(pack_inputs) == 1
# Check causal_conv1d_weight_pack received the expected weights
torch.testing.assert_close(pack_inputs[0], original.view(32, 4))
# Check we have correctly stashed the unpacked weights
torch.testing.assert_close(layer._cpu_unpacked_conv_weight, original.view(32, 4))
# Check we have correctly stored the packed weights
assert layer.weight.data_ptr() == packed.data_ptr()
def test_dispatch_does_not_pack_3d_expert_weight(monkeypatch):
monkeypatch.setattr(
current_platform, "get_cpu_architecture", lambda: CpuArchEnum.ARM
)
monkeypatch.setattr(torch.cpu, "get_capabilities", lambda: {"bf16": True})
monkeypatch.setattr(torch.cpu, "_is_avx512_bf16_supported", lambda: False)
pack_calls = []
monkeypatch.setattr(
utils.ops,
"causal_conv1d_weight_pack",
lambda weight: pack_calls.append(weight),
)
layer = torch.nn.Module()
layer.weight = torch.nn.Parameter(
torch.randn(8, 32, 4, dtype=torch.bfloat16), requires_grad=False
)
utils.dispatch_cpu_unquantized_gemm(layer, remove_weight=False)
assert pack_calls == []
@pytest.fixture(scope="module")
def _mock_zentorch_linear_unary():
"""Register a mock zentorch_linear_unary op when zentorch is not installed.