[CPU][Spec Decode] Optimize GDN conv path for speculative decoding (#48577)
Signed-off-by: Li, Tianmu <tianmu.li@intel.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
co-authored by
Codex
Li, Jiang <jiang1.li@intel.com>
parent
c314af1abf
commit
544cb724c8
+173
-12
@@ -451,6 +451,90 @@ void causal_conv1d_update_kernel_impl(
|
||||
});
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
void causal_conv1d_update_multi_kernel_impl(
|
||||
scalar_t* __restrict__ out,
|
||||
const scalar_t* __restrict__ input,
|
||||
scalar_t* __restrict__ conv_states,
|
||||
const scalar_t* __restrict__ weight,
|
||||
const scalar_t* __restrict__ bias,
|
||||
const int32_t* __restrict__ num_accepted_tokens,
|
||||
const int32_t* __restrict__ conv_indices,
|
||||
bool silu_activation,
|
||||
int64_t batch,
|
||||
int64_t dim,
|
||||
int64_t seqlen,
|
||||
int64_t width,
|
||||
int64_t state_len,
|
||||
int64_t conv_state_slot_stride) {
|
||||
constexpr int64_t BLOCK_N = block_size_n() * 2;
|
||||
const int64_t NB = div_up(dim, BLOCK_N);
|
||||
|
||||
AT_DISPATCH_BOOL2(bias != nullptr, has_bias, silu_activation, has_silu, [&] {
|
||||
at::parallel_for(0, batch * NB, 0, [&](int64_t begin, int64_t end) {
|
||||
int64_t bs{0}, nb{0};
|
||||
data_index_init(begin, bs, batch, nb, NB);
|
||||
|
||||
for (int64_t i = begin; i < end; ++i) {
|
||||
const int64_t nb_start = nb * BLOCK_N;
|
||||
const int64_t nb_size = std::min(dim - nb_start, BLOCK_N);
|
||||
const int32_t conv_state_index = conv_indices[bs];
|
||||
const int32_t history_offset = num_accepted_tokens[bs] - 1;
|
||||
|
||||
switch (width << 4 | nb_size >> 4) {
|
||||
case 0x42:
|
||||
tinygemm_kernel<scalar_t, 4, 32, has_bias, has_silu>::apply(
|
||||
input + bs * seqlen * dim + nb_start,
|
||||
weight + nb_start * width,
|
||||
out + bs * seqlen * dim + nb_start,
|
||||
has_bias ? bias + nb_start : nullptr,
|
||||
conv_states + conv_state_index * conv_state_slot_stride +
|
||||
history_offset * dim + nb_start,
|
||||
true,
|
||||
seqlen,
|
||||
dim,
|
||||
true);
|
||||
break;
|
||||
case 0x44:
|
||||
tinygemm_kernel<scalar_t, 4, 64, has_bias, has_silu>::apply(
|
||||
input + bs * seqlen * dim + nb_start,
|
||||
weight + nb_start * width,
|
||||
out + bs * seqlen * dim + nb_start,
|
||||
has_bias ? bias + nb_start : nullptr,
|
||||
conv_states + conv_state_index * conv_state_slot_stride +
|
||||
history_offset * dim + nb_start,
|
||||
true,
|
||||
seqlen,
|
||||
dim,
|
||||
true);
|
||||
break;
|
||||
default:
|
||||
TORCH_CHECK(false, "Unexpected block size, ", width, " x ", nb_size);
|
||||
}
|
||||
|
||||
data_index_step(bs, batch, nb, NB);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
at::parallel_for(0, batch, 0, [&](int64_t begin, int64_t end) {
|
||||
for (int64_t bs = begin; bs < end; ++bs) {
|
||||
const int32_t conv_state_index = conv_indices[bs];
|
||||
const int32_t num_accepted = num_accepted_tokens[bs];
|
||||
scalar_t* state = conv_states + conv_state_index * conv_state_slot_stride;
|
||||
|
||||
std::memmove(
|
||||
state,
|
||||
state + num_accepted * dim,
|
||||
(state_len - seqlen) * dim * sizeof(scalar_t));
|
||||
std::memcpy(
|
||||
state + (state_len - seqlen) * dim,
|
||||
input + bs * seqlen * dim,
|
||||
seqlen * dim * sizeof(scalar_t));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
} // anonymous namespace
|
||||
|
||||
// from [dim, width] or [N, K]
|
||||
@@ -545,7 +629,7 @@ at::Tensor get_block_indices(const std::optional<at::Tensor>& offsets, int64_t n
|
||||
// query_start_loc: (batch + 1) int32
|
||||
// cache_indices: (batch) int32
|
||||
// has_initial_state: (batch) bool
|
||||
// conv_states: (..., dim, width - 1) itype
|
||||
// conv_states: (..., dim, state_len) itype, where state_len >= width - 1
|
||||
// activation: either None or "silu" or "swish"
|
||||
// pad_slot_id: int
|
||||
//
|
||||
@@ -586,11 +670,14 @@ at::Tensor causal_conv1d_fwd_cpu(
|
||||
CHECK_EQ(conv_states_val.scalar_type(), scalar_type);
|
||||
CHECK_GE(padded_batch, batch);
|
||||
CHECK_EQ(conv_states_val.size(1), dim);
|
||||
CHECK_EQ(conv_states_val.size(2), width - 1);
|
||||
const int64_t state_len = conv_states_val.size(2);
|
||||
CHECK_GE(state_len, width - 1);
|
||||
|
||||
// adjust `conv_states` to be contiguous on `dim`
|
||||
// should happen only once
|
||||
if (conv_states_val.stride(-2) != 1) {
|
||||
TORCH_CHECK(state_len == width - 1,
|
||||
"causal_conv1d_fwd_cpu: wide conv_states must be contiguous on dim.");
|
||||
auto conv_states_copy = conv_states_val.clone();
|
||||
conv_states_val.as_strided_({padded_batch, dim, width - 1}, {(width - 1) * dim, 1, dim});
|
||||
conv_states_val.copy_(conv_states_copy);
|
||||
@@ -651,14 +738,14 @@ at::Tensor causal_conv1d_fwd_cpu(
|
||||
|
||||
// API aligned with GPUs
|
||||
//
|
||||
// x: (batch, dim) or (batch, dim, seqlen)
|
||||
// x: (batch, dim) or (batch, seqlen, dim)
|
||||
// conv_state: (..., dim, state_len), where state_len >= width - 1
|
||||
// weight: (dim, width)
|
||||
// bias: (dim,)
|
||||
// cache_seqlens: (batch,), dtype int32.
|
||||
// num_accepted_tokens: (batch,), dtype int32.
|
||||
// conv_state_indices: (batch,), dtype int32
|
||||
// pad_slot_id: int
|
||||
// out: (batch, dim) or (batch, dim, seqlen)
|
||||
// out: (batch, dim) or (batch, seqlen, dim)
|
||||
//
|
||||
at::Tensor causal_conv1d_update_cpu(
|
||||
const at::Tensor& x,
|
||||
@@ -666,7 +753,7 @@ at::Tensor causal_conv1d_update_cpu(
|
||||
const at::Tensor& weight,
|
||||
const std::optional<at::Tensor>& bias,
|
||||
bool silu_activation,
|
||||
const std::optional<at::Tensor>& cache_seqlens,
|
||||
const std::optional<at::Tensor>& num_accepted_tokens,
|
||||
const std::optional<at::Tensor>& conv_state_indices,
|
||||
int64_t pad_slot_id,
|
||||
bool is_vnni) {
|
||||
@@ -674,13 +761,13 @@ at::Tensor causal_conv1d_update_cpu(
|
||||
CHECK_CONTIGUOUS(weight);
|
||||
auto packed_w = is_vnni ? weight : causal_conv1d_weight_pack(weight);
|
||||
|
||||
// TODO: add multi-token prediction support
|
||||
TORCH_CHECK(x.dim() == 2, "causal_conv1d_update_cpu: expect x to be 2D tensor.");
|
||||
TORCH_CHECK(!cache_seqlens.has_value(), "causal_conv1d_update_cpu: don't support cache_seqlens.");
|
||||
TORCH_CHECK(
|
||||
x.dim() == 2 || x.dim() == 3,
|
||||
"causal_conv1d_update_cpu: expect x to be 2D or 3D tensor.");
|
||||
|
||||
int64_t batch = x.size(0);
|
||||
int64_t dim = x.size(1);
|
||||
int64_t seqlen = 1;
|
||||
int64_t dim = x.dim() == 2 ? x.size(1) : x.size(2);
|
||||
int64_t seqlen = x.dim() == 2 ? 1 : x.size(1);
|
||||
int64_t width = weight.size(-1);
|
||||
|
||||
const auto scalar_type = x.scalar_type();
|
||||
@@ -690,10 +777,84 @@ at::Tensor causal_conv1d_update_cpu(
|
||||
|
||||
CHECK_EQ(conv_states.scalar_type(), scalar_type);
|
||||
CHECK_EQ(conv_states.size(1), dim);
|
||||
CHECK_EQ(conv_states.size(2), width - 1);
|
||||
const int64_t state_len = conv_states.size(2);
|
||||
CHECK_GE(state_len, width - 1);
|
||||
|
||||
if (x.dim() == 3) {
|
||||
TORCH_CHECK(
|
||||
num_accepted_tokens.has_value(),
|
||||
"causal_conv1d_update_cpu: num_accepted_tokens is required for 3D x.");
|
||||
TORCH_CHECK(
|
||||
conv_state_indices.has_value(),
|
||||
"causal_conv1d_update_cpu: conv_state_indices is required for 3D x.");
|
||||
CHECK_OPTIONAL_SHAPE_DTYPE(num_accepted_tokens, batch, at::kInt);
|
||||
TORCH_CHECK(
|
||||
width == 4,
|
||||
"causal_conv1d_update_cpu: support only width of 4 for 3D x.");
|
||||
TORCH_CHECK(
|
||||
seqlen > 0,
|
||||
"causal_conv1d_update_cpu: expect non-empty sequence for 3D x.");
|
||||
TORCH_CHECK(
|
||||
state_len >= seqlen,
|
||||
"causal_conv1d_update_cpu: state_len must be >= seqlen for 3D x.");
|
||||
TORCH_CHECK(
|
||||
conv_states.stride(-2) == 1 && conv_states.stride(-1) == dim,
|
||||
"causal_conv1d_update_cpu: 3D x requires SD conv_states layout.");
|
||||
|
||||
const int32_t* accepted_counts =
|
||||
num_accepted_tokens.value().data_ptr<int32_t>();
|
||||
const int32_t* indices = conv_state_indices.value().data_ptr<int32_t>();
|
||||
const int64_t num_slots = conv_states.size(0);
|
||||
for (int64_t bs = 0; bs < batch; ++bs) {
|
||||
const int32_t num_accepted = accepted_counts[bs];
|
||||
const int32_t conv_state_index = indices[bs];
|
||||
TORCH_CHECK(
|
||||
conv_state_index != pad_slot_id,
|
||||
"causal_conv1d_update_cpu: 3D x does not support pad slots.");
|
||||
TORCH_CHECK(
|
||||
conv_state_index >= 0 && conv_state_index < num_slots,
|
||||
"causal_conv1d_update_cpu: conv_state_indices out of range.");
|
||||
TORCH_CHECK(
|
||||
num_accepted >= 1 && num_accepted <= seqlen,
|
||||
"causal_conv1d_update_cpu: num_accepted_tokens must be in [1, "
|
||||
"seqlen].");
|
||||
TORCH_CHECK(
|
||||
num_accepted - 1 + width - 1 <= state_len,
|
||||
"causal_conv1d_update_cpu: history window exceeds conv_states.");
|
||||
}
|
||||
|
||||
int64_t conv_state_slot_stride = conv_states.stride(0);
|
||||
at::Tensor out = at::empty_like(x);
|
||||
AT_DISPATCH_REDUCED_FLOATING_TYPES(
|
||||
scalar_type, "causal_conv1d_update_multi_kernel_impl", [&] {
|
||||
causal_conv1d_update_multi_kernel_impl<scalar_t>(
|
||||
out.data_ptr<scalar_t>(),
|
||||
x.data_ptr<scalar_t>(),
|
||||
conv_states.data_ptr<scalar_t>(),
|
||||
packed_w.data_ptr<scalar_t>(),
|
||||
conditional_data_ptr<scalar_t>(bias),
|
||||
accepted_counts,
|
||||
indices,
|
||||
silu_activation,
|
||||
batch,
|
||||
dim,
|
||||
seqlen,
|
||||
width,
|
||||
state_len,
|
||||
conv_state_slot_stride);
|
||||
});
|
||||
return out;
|
||||
}
|
||||
|
||||
TORCH_CHECK(
|
||||
!num_accepted_tokens.has_value(),
|
||||
"causal_conv1d_update_cpu: num_accepted_tokens is only supported for 3D "
|
||||
"x.");
|
||||
|
||||
// adjust `conv_states` to be contiguous on `dim`
|
||||
if (conv_states.stride(-2) != 1) {
|
||||
TORCH_CHECK(state_len == width - 1,
|
||||
"causal_conv1d_update_cpu: wide conv_states must be contiguous on dim.");
|
||||
int64_t num_cache_lines = conv_states.size(0);
|
||||
auto conv_states_copy = conv_states.clone();
|
||||
conv_states.as_strided_({num_cache_lines, dim, width - 1}, {(width - 1) * dim, 1, dim});
|
||||
|
||||
@@ -147,7 +147,7 @@ at::Tensor causal_conv1d_fwd_cpu(
|
||||
at::Tensor causal_conv1d_update_cpu(
|
||||
const at::Tensor& x, const at::Tensor& conv_states,
|
||||
const at::Tensor& weight, const std::optional<at::Tensor>& bias,
|
||||
bool silu_activation, const std::optional<at::Tensor>& cache_seqlens,
|
||||
bool silu_activation, const std::optional<at::Tensor>& num_accepted_tokens,
|
||||
const std::optional<at::Tensor>& conv_state_indices, int64_t pad_slot_id,
|
||||
bool is_vnni);
|
||||
|
||||
@@ -516,7 +516,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
|
||||
ops.def(
|
||||
"causal_conv1d_update_cpu(Tensor x, Tensor(a!) conv_states, Tensor "
|
||||
"weight, Tensor? bias, bool silu_activation,"
|
||||
"Tensor? cache_seqlens, Tensor? conv_state_indices, int pad_slot_id, "
|
||||
"Tensor? num_accepted_tokens, Tensor? conv_state_indices, int "
|
||||
"pad_slot_id, "
|
||||
"bool is_vnni) -> Tensor");
|
||||
ops.impl("causal_conv1d_update_cpu", torch::kCPU, &causal_conv1d_update_cpu);
|
||||
#endif
|
||||
|
||||
@@ -2,12 +2,14 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import functools
|
||||
import types
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.model_executor.layers.mamba.ops.cpu import gdn_attention
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
@@ -417,6 +419,207 @@ def _conv_inputs(total_tokens: int):
|
||||
return x, weight, bias
|
||||
|
||||
|
||||
def _sd_conv_states(
|
||||
num_slots: int, state_len: int, dim: int = CONV_DIM
|
||||
) -> torch.Tensor:
|
||||
storage = torch.zeros(num_slots, state_len, dim, dtype=torch.bfloat16)
|
||||
return storage.transpose(1, 2)
|
||||
|
||||
|
||||
def _maybe_pack_conv_weight(weight: torch.Tensor, is_vnni: bool) -> torch.Tensor:
|
||||
return ops.causal_conv1d_weight_pack(weight) if is_vnni else weight
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_spec_aware_mixed_routing_preserves_token_order(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
num_tokens = 4
|
||||
projection = torch.arange(num_tokens * 16, dtype=torch.float32).view(num_tokens, 16)
|
||||
mixed_qkv, b, a = projection[:, :4], projection[:, 4:6], projection[:, 6:8]
|
||||
assert all(not tensor.is_contiguous() for tensor in (mixed_qkv, b, a))
|
||||
|
||||
spec_indices = torch.tensor([0, 2])
|
||||
nonspec_indices = torch.tensor([1, 3])
|
||||
metadata = types.SimpleNamespace(
|
||||
spec_sequence_masks=torch.ones(1, dtype=torch.bool),
|
||||
spec_token_indx=spec_indices,
|
||||
non_spec_token_indx=nonspec_indices,
|
||||
num_prefills=1,
|
||||
num_decodes=0,
|
||||
)
|
||||
routed = []
|
||||
|
||||
def record(*args):
|
||||
routed.append(args[2:5])
|
||||
return args[2]
|
||||
|
||||
monkeypatch.setattr(gdn_attention, "is_conv_state_dim_first", lambda: True)
|
||||
monkeypatch.setattr(gdn_attention, "_spec_forward", record)
|
||||
monkeypatch.setattr(gdn_attention, "_spec_aware_nonspec_subset", record)
|
||||
|
||||
layer = types.SimpleNamespace(
|
||||
kv_cache=[torch.empty(1, 4, 6), torch.empty(1, 1, 1, 1)]
|
||||
)
|
||||
core_attn_out = torch.empty_like(mixed_qkv)
|
||||
gdn_attention._cpu_gdn_attention_spec_aware(
|
||||
layer=layer,
|
||||
attn_metadata_i=metadata,
|
||||
mixed_qkv=mixed_qkv,
|
||||
b=b,
|
||||
a=a,
|
||||
core_attn_out=core_attn_out,
|
||||
width=CONV_KERNEL,
|
||||
state_len=6,
|
||||
)
|
||||
|
||||
expected_inputs = (mixed_qkv, b, a)
|
||||
assert len(routed) == 2
|
||||
for actual_inputs, indices in zip(routed, (spec_indices, nonspec_indices)):
|
||||
for actual, expected in zip(actual_inputs, expected_inputs):
|
||||
assert actual.is_contiguous()
|
||||
torch.testing.assert_close(actual, expected.index_select(0, indices))
|
||||
torch.testing.assert_close(core_attn_out, mixed_qkv)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_spec_aware_nonspec_materializes_state_indices(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
block_table = torch.arange(8, dtype=torch.int32).view(2, 4)
|
||||
state_indices = block_table[:, 0]
|
||||
assert not state_indices.is_contiguous()
|
||||
|
||||
metadata = types.SimpleNamespace(
|
||||
non_spec_state_indices_tensor=state_indices,
|
||||
non_spec_query_start_loc=torch.tensor([0, 2, 4], dtype=torch.int32),
|
||||
num_decodes=0,
|
||||
num_decode_tokens=0,
|
||||
num_prefills=2,
|
||||
num_prefill_tokens=4,
|
||||
has_initial_state=torch.tensor([False, False]),
|
||||
)
|
||||
|
||||
recorded_indices = None
|
||||
|
||||
def causal_conv1d_fwd_cpu(**kwargs):
|
||||
nonlocal recorded_indices
|
||||
recorded_indices = kwargs["cache_indices"]
|
||||
return kwargs["x"]
|
||||
|
||||
def fused_gdn_gating_cpu(**kwargs):
|
||||
return kwargs["a"], kwargs["b"]
|
||||
|
||||
def chunk_gated_delta_rule_cpu(**kwargs):
|
||||
out = torch.zeros(1, 4, 1, 1)
|
||||
return out, kwargs["initial_state"]
|
||||
|
||||
monkeypatch.setattr(torch.cpu, "_is_amx_tile_supported", lambda: True)
|
||||
monkeypatch.setattr(gdn_attention, "is_conv_state_dim_first", lambda: False)
|
||||
monkeypatch.setattr(
|
||||
gdn_attention.ops, "causal_conv1d_fwd_cpu", causal_conv1d_fwd_cpu
|
||||
)
|
||||
monkeypatch.setattr(gdn_attention.ops, "fused_gdn_gating_cpu", fused_gdn_gating_cpu)
|
||||
monkeypatch.setattr(
|
||||
gdn_attention.ops,
|
||||
"chunk_gated_delta_rule_cpu",
|
||||
chunk_gated_delta_rule_cpu,
|
||||
)
|
||||
|
||||
layer = types.SimpleNamespace(
|
||||
activation="silu",
|
||||
conv1d=types.SimpleNamespace(weight=torch.empty(0), bias=None),
|
||||
A_log=torch.empty(0),
|
||||
dt_bias=torch.empty(0),
|
||||
rearrange_mixed_qkv=lambda x: (
|
||||
x[:, :1].view(1, 4, 1, 1),
|
||||
x[:, :1].view(1, 4, 1, 1),
|
||||
x[:, :1].view(1, 4, 1, 1),
|
||||
),
|
||||
)
|
||||
gdn_attention._spec_aware_nonspec(
|
||||
layer=layer,
|
||||
attn_metadata_i=metadata,
|
||||
mixed_qkv=torch.zeros(4, 4),
|
||||
b=torch.zeros(4, 1),
|
||||
a=torch.zeros(4, 1),
|
||||
core_attn_out=torch.zeros(4, 1, 1),
|
||||
conv_buf=torch.zeros(8, 1, 6),
|
||||
ssm_state=torch.zeros(8, 1, 1, 1),
|
||||
width=4,
|
||||
)
|
||||
|
||||
assert recorded_indices is not None
|
||||
assert recorded_indices.is_contiguous()
|
||||
torch.testing.assert_close(
|
||||
recorded_indices, torch.tensor([0, 4], dtype=torch.int32)
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_spec_forward_prepares_native_conv_metadata(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
block_table = torch.tensor([[0, 1], [4, 5]], dtype=torch.int32)
|
||||
state_indices = block_table[:, 0]
|
||||
accepted_counts = torch.tensor([1, 4], dtype=torch.int32)
|
||||
assert not state_indices.is_contiguous()
|
||||
metadata = types.SimpleNamespace(
|
||||
num_spec_decodes=2,
|
||||
spec_state_indices_tensor=block_table,
|
||||
spec_query_start_loc=torch.tensor([0, 4, 8], dtype=torch.int32),
|
||||
num_accepted_tokens=accepted_counts,
|
||||
)
|
||||
forwarded_indices = None
|
||||
forwarded_counts = None
|
||||
|
||||
def causal_conv1d_update_cpu(**kwargs):
|
||||
nonlocal forwarded_counts, forwarded_indices
|
||||
forwarded_indices = kwargs["conv_state_indices"]
|
||||
forwarded_counts = kwargs["num_accepted_tokens"]
|
||||
return kwargs["x"]
|
||||
|
||||
monkeypatch.setattr(torch.cpu, "_is_amx_tile_supported", lambda: True)
|
||||
monkeypatch.setattr(gdn_attention, "is_conv_state_dim_first", lambda: False)
|
||||
monkeypatch.setattr(
|
||||
gdn_attention.ops, "causal_conv1d_update_cpu", causal_conv1d_update_cpu
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
gdn_attention.ops,
|
||||
"fused_sigmoid_gating_delta_rule_update_spec_cpu",
|
||||
lambda **kwargs: kwargs["q"],
|
||||
)
|
||||
|
||||
layer = types.SimpleNamespace(
|
||||
activation="silu",
|
||||
conv1d=types.SimpleNamespace(weight=torch.empty(1, CONV_KERNEL), bias=None),
|
||||
A_log=None,
|
||||
dt_bias=None,
|
||||
rearrange_mixed_qkv=lambda x: (x.unsqueeze(0),) * 3,
|
||||
)
|
||||
gdn_attention._spec_forward(
|
||||
layer=layer,
|
||||
attn_metadata_i=metadata,
|
||||
mixed_qkv_spec=torch.zeros(8, 1, dtype=torch.bfloat16),
|
||||
b_spec=torch.empty(0),
|
||||
a_spec=torch.empty(0),
|
||||
conv_buf=torch.empty(0),
|
||||
ssm_state=torch.empty(0),
|
||||
width=CONV_KERNEL,
|
||||
state_len=0,
|
||||
)
|
||||
|
||||
expected = (
|
||||
(forwarded_indices, torch.tensor([0, 4], dtype=torch.int32)),
|
||||
(forwarded_counts, torch.tensor([1, 4], dtype=torch.int32)),
|
||||
)
|
||||
for actual, reference in expected:
|
||||
assert actual is not None
|
||||
assert actual.is_contiguous()
|
||||
assert actual.dtype == torch.int32
|
||||
torch.testing.assert_close(actual, reference)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("total_tokens, split", TWO_CALL_SPLITS)
|
||||
@torch.inference_mode()
|
||||
def test_causal_conv1d_torch_two_call_split(total_tokens: int, split: int) -> None:
|
||||
@@ -475,7 +678,182 @@ def test_causal_conv1d_torch_two_call_split(total_tokens: int, split: int) -> No
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cpu._is_amx_tile_supported(),
|
||||
reason="causal_conv1d_fwd_cpu requires AMX/AVX512",
|
||||
reason="requires AMX support",
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_causal_conv1d_update_cpu_accepts_wide_state() -> None:
|
||||
state_len = CONV_KERNEL - 1
|
||||
wide_state_len = state_len + 5
|
||||
batch_size = 3
|
||||
is_vnni = True
|
||||
x, weight, bias = _conv_inputs(batch_size)
|
||||
conv_state_indices = torch.tensor([2, 0, 1], dtype=torch.int32)
|
||||
|
||||
narrow_state = _sd_conv_states(batch_size, state_len)
|
||||
narrow_state.copy_(
|
||||
tensor_cache(narrow_state.numel(), torch.bfloat16).view_as(narrow_state)
|
||||
)
|
||||
wide_state = _sd_conv_states(batch_size, wide_state_len)
|
||||
wide_state[:, :, :state_len].copy_(narrow_state)
|
||||
wide_state[:, :, state_len:].fill_(7)
|
||||
wide_tail = wide_state[:, :, state_len:].clone()
|
||||
|
||||
conv_weight = _maybe_pack_conv_weight(weight, is_vnni)
|
||||
out_narrow = ops.causal_conv1d_update_cpu(
|
||||
x=x,
|
||||
conv_states=narrow_state,
|
||||
weight=conv_weight,
|
||||
bias=bias,
|
||||
silu_activation=True,
|
||||
conv_state_indices=conv_state_indices,
|
||||
is_vnni=is_vnni,
|
||||
)
|
||||
out_wide = ops.causal_conv1d_update_cpu(
|
||||
x=x,
|
||||
conv_states=wide_state,
|
||||
weight=conv_weight,
|
||||
bias=bias,
|
||||
silu_activation=True,
|
||||
conv_state_indices=conv_state_indices,
|
||||
is_vnni=is_vnni,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(out_wide, out_narrow, atol=1e-2, rtol=1e-2)
|
||||
torch.testing.assert_close(
|
||||
wide_state[:, :, :state_len], narrow_state, atol=0, rtol=0
|
||||
)
|
||||
torch.testing.assert_close(wide_state[:, :, state_len:], wide_tail, atol=0, rtol=0)
|
||||
|
||||
|
||||
def _ref_causal_conv1d_update_cpu_multi(
|
||||
x: torch.Tensor,
|
||||
conv_states: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
silu_activation: bool,
|
||||
conv_state_indices: torch.Tensor,
|
||||
num_accepted_tokens: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
batch_size, seq_len, dim = x.shape
|
||||
state_len = conv_states.size(2)
|
||||
conv_out = torch.empty_like(x)
|
||||
conv_weight = weight.unsqueeze(1)
|
||||
|
||||
for i in range(batch_size):
|
||||
slot = int(conv_state_indices[i].item())
|
||||
offset = int(num_accepted_tokens[i].item()) - 1
|
||||
state = conv_states[slot]
|
||||
x_seq = x[i].transpose(0, 1).to(state.dtype)
|
||||
prior = state[:, offset : offset + CONV_KERNEL - 1]
|
||||
conv_in = torch.cat([prior, x_seq], dim=-1).unsqueeze(0)
|
||||
out = F.conv1d(conv_in, conv_weight, bias, groups=dim)[0]
|
||||
if silu_activation:
|
||||
out = F.silu(out)
|
||||
conv_out[i] = out.transpose(0, 1).to(conv_out.dtype)
|
||||
keep = state[:, offset + 1 : offset + 1 + (state_len - seq_len)]
|
||||
state.copy_(torch.cat([keep, x_seq], dim=-1))
|
||||
|
||||
return conv_out
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cpu._is_amx_tile_supported(),
|
||||
reason="requires AMX support",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("batch_size, seq_len, accepted_counts, has_bias, silu_activation, is_vnni"),
|
||||
[
|
||||
(1, 1, [1], False, False, False),
|
||||
(1, 1, [1], True, True, True),
|
||||
(4, 4, [1, 2, 3, 4], False, True, False),
|
||||
(4, 4, [4, 3, 2, 1], True, False, True),
|
||||
(4, 16, [1, 5, 10, 16], False, False, True),
|
||||
(4, 16, [16, 10, 5, 1], True, True, False),
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_causal_conv1d_update_cpu_multi_token_matches_python(
|
||||
batch_size: int,
|
||||
seq_len: int,
|
||||
accepted_counts: list[int],
|
||||
has_bias: bool,
|
||||
silu_activation: bool,
|
||||
is_vnni: bool,
|
||||
) -> None:
|
||||
dim = 96
|
||||
state_len = seq_len + 2
|
||||
x = tensor_cache(batch_size * seq_len * dim, torch.bfloat16).view(
|
||||
batch_size, seq_len, dim
|
||||
)
|
||||
weight = tensor_cache(dim * CONV_KERNEL, torch.bfloat16).view(dim, CONV_KERNEL)
|
||||
bias = tensor_cache(dim, torch.bfloat16) if has_bias else None
|
||||
conv_state_indices = torch.arange(batch_size - 1, -1, -1, dtype=torch.int32)
|
||||
num_accepted_tokens = torch.tensor(accepted_counts, dtype=torch.int32)
|
||||
|
||||
conv_states_ref = _sd_conv_states(batch_size, state_len, dim)
|
||||
conv_states_ref.copy_(
|
||||
tensor_cache(conv_states_ref.numel(), torch.bfloat16).view_as(conv_states_ref)
|
||||
)
|
||||
conv_states = conv_states_ref.clone()
|
||||
|
||||
conv_weight = _maybe_pack_conv_weight(weight, is_vnni)
|
||||
out = ops.causal_conv1d_update_cpu(
|
||||
x=x,
|
||||
conv_states=conv_states,
|
||||
weight=conv_weight,
|
||||
bias=bias,
|
||||
silu_activation=silu_activation,
|
||||
conv_state_indices=conv_state_indices,
|
||||
is_vnni=is_vnni,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
)
|
||||
ref_out = _ref_causal_conv1d_update_cpu_multi(
|
||||
x=x,
|
||||
conv_states=conv_states_ref,
|
||||
weight=weight,
|
||||
bias=bias,
|
||||
silu_activation=silu_activation,
|
||||
conv_state_indices=conv_state_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(out, ref_out, atol=1e-2, rtol=1e-2)
|
||||
torch.testing.assert_close(conv_states, conv_states_ref, atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cpu._is_amx_tile_supported(),
|
||||
reason="requires AMX support",
|
||||
)
|
||||
@pytest.mark.parametrize("num_accepted", [0, 17])
|
||||
@torch.inference_mode()
|
||||
def test_causal_conv1d_update_cpu_rejects_invalid_accepted_count(
|
||||
num_accepted: int,
|
||||
) -> None:
|
||||
batch_size = 1
|
||||
seq_len = 16
|
||||
dim = 96
|
||||
state_len = seq_len + 2
|
||||
x = torch.zeros(batch_size, seq_len, dim, dtype=torch.bfloat16)
|
||||
weight = torch.zeros(dim, CONV_KERNEL, dtype=torch.bfloat16)
|
||||
conv_states = _sd_conv_states(batch_size, state_len, dim)
|
||||
|
||||
with pytest.raises(RuntimeError, match="num_accepted_tokens must be in.*seqlen"):
|
||||
ops.causal_conv1d_update_cpu(
|
||||
x=x,
|
||||
conv_states=conv_states,
|
||||
weight=weight,
|
||||
bias=None,
|
||||
silu_activation=True,
|
||||
conv_state_indices=torch.tensor([0], dtype=torch.int32),
|
||||
is_vnni=False,
|
||||
num_accepted_tokens=torch.tensor([num_accepted], dtype=torch.int32),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cpu._is_amx_tile_supported(),
|
||||
reason="requires AMX support",
|
||||
)
|
||||
@pytest.mark.parametrize("total_tokens, split", TWO_CALL_SPLITS)
|
||||
@torch.inference_mode()
|
||||
@@ -515,6 +893,62 @@ def test_causal_conv1d_fwd_cpu_two_call_split(total_tokens: int, split: int) ->
|
||||
torch.testing.assert_close(out_split, out_full, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cpu._is_amx_tile_supported(),
|
||||
reason="requires AMX support",
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_causal_conv1d_fwd_cpu_accepts_wide_state() -> None:
|
||||
state_len = CONV_KERNEL - 1
|
||||
wide_state_len = state_len + 5
|
||||
is_vnni = True
|
||||
seq_lens = [CHUNK_SIZE - 1, CHUNK_SIZE + 5]
|
||||
total_tokens = sum(seq_lens)
|
||||
x, weight, bias = _conv_inputs(total_tokens)
|
||||
query_start_loc = torch.tensor([0, seq_lens[0], total_tokens], dtype=torch.int32)
|
||||
cache_indices = torch.tensor([2, 0], dtype=torch.int32)
|
||||
has_initial_state = torch.tensor([True, False])
|
||||
|
||||
narrow_state = _sd_conv_states(3, state_len)
|
||||
narrow_state.copy_(
|
||||
tensor_cache(narrow_state.numel(), torch.bfloat16).view_as(narrow_state)
|
||||
)
|
||||
wide_state = _sd_conv_states(3, wide_state_len)
|
||||
wide_state[:, :, :state_len].copy_(narrow_state)
|
||||
wide_state[:, :, state_len:].fill_(7)
|
||||
wide_tail = wide_state[:, :, state_len:].clone()
|
||||
|
||||
conv_weight = _maybe_pack_conv_weight(weight, is_vnni)
|
||||
out_narrow = ops.causal_conv1d_fwd_cpu(
|
||||
x=x.transpose(0, 1),
|
||||
weight=conv_weight,
|
||||
bias=bias,
|
||||
conv_states=narrow_state,
|
||||
query_start_loc=query_start_loc,
|
||||
cache_indices=cache_indices,
|
||||
has_initial_state=has_initial_state,
|
||||
silu_activation=True,
|
||||
is_vnni=is_vnni,
|
||||
)
|
||||
out_wide = ops.causal_conv1d_fwd_cpu(
|
||||
x=x.transpose(0, 1),
|
||||
weight=conv_weight,
|
||||
bias=bias,
|
||||
conv_states=wide_state,
|
||||
query_start_loc=query_start_loc,
|
||||
cache_indices=cache_indices,
|
||||
has_initial_state=has_initial_state,
|
||||
silu_activation=True,
|
||||
is_vnni=is_vnni,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(out_wide, out_narrow, atol=1e-2, rtol=1e-2)
|
||||
torch.testing.assert_close(
|
||||
wide_state[:, :, :state_len], narrow_state, atol=0, rtol=0
|
||||
)
|
||||
torch.testing.assert_close(wide_state[:, :, state_len:], wide_tail, atol=0, rtol=0)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_batch_memcpy_cpu_fallback() -> None:
|
||||
"""The ctypes batch_memcpy fallback (used when triton-cpu is absent) must
|
||||
|
||||
+2
-1
@@ -3505,6 +3505,7 @@ def causal_conv1d_update_cpu(
|
||||
silu_activation: bool,
|
||||
conv_state_indices: torch.Tensor | None,
|
||||
is_vnni: bool,
|
||||
num_accepted_tokens: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return torch.ops._C.causal_conv1d_update_cpu(
|
||||
x,
|
||||
@@ -3512,7 +3513,7 @@ def causal_conv1d_update_cpu(
|
||||
weight,
|
||||
bias,
|
||||
silu_activation,
|
||||
None,
|
||||
num_accepted_tokens,
|
||||
conv_state_indices,
|
||||
-1,
|
||||
is_vnni,
|
||||
|
||||
@@ -306,25 +306,21 @@ def _cpu_gdn_attention_spec_aware(
|
||||
width: int,
|
||||
state_len: int,
|
||||
) -> None:
|
||||
mixed_qkv = mixed_qkv.contiguous()
|
||||
a = a.contiguous()
|
||||
b = b.contiguous()
|
||||
|
||||
spec_sequence_masks = attn_metadata_i.spec_sequence_masks
|
||||
conv_buf = _conv_buffer_view(layer) # (num_slots, dim, state_len)
|
||||
ssm_state = _ssm_state_view(layer)
|
||||
|
||||
if spec_sequence_masks is None:
|
||||
# No spec sequences in this batch (e.g. the prompt prefill step while
|
||||
# speculative decoding is configured). Process as prefill/decode using
|
||||
# torch conv (which only touches the first ``width-1`` columns of the
|
||||
# wide buffer, leaving the rolling history untouched).
|
||||
# speculative decoding is configured). Process as ordinary
|
||||
# prefill/decode while touching only the first ``width-1`` columns of
|
||||
# the wide buffer, leaving the rolling history untouched.
|
||||
_spec_aware_nonspec(
|
||||
layer,
|
||||
attn_metadata_i,
|
||||
mixed_qkv,
|
||||
b,
|
||||
a,
|
||||
mixed_qkv.contiguous(),
|
||||
b.contiguous(),
|
||||
a.contiguous(),
|
||||
core_attn_out,
|
||||
conv_buf,
|
||||
ssm_state,
|
||||
@@ -339,9 +335,9 @@ def _cpu_gdn_attention_spec_aware(
|
||||
num_decodes = attn_metadata_i.num_decodes
|
||||
|
||||
if num_prefills == 0 and num_decodes == 0:
|
||||
mixed_qkv_spec = mixed_qkv
|
||||
b_spec = b
|
||||
a_spec = a
|
||||
mixed_qkv_spec = mixed_qkv.contiguous()
|
||||
b_spec = b.contiguous()
|
||||
a_spec = a.contiguous()
|
||||
spec_out_indx = None
|
||||
else:
|
||||
assert spec_token_indx is not None
|
||||
@@ -405,50 +401,71 @@ def _spec_forward(
|
||||
assert spec_qsl is not None
|
||||
assert num_accepted is not None
|
||||
|
||||
spec_qsl_cpu = spec_qsl[: num_spec_decodes + 1].to("cpu", torch.int64)
|
||||
num_acc_cpu = num_accepted[:num_spec_decodes].to("cpu", torch.int64)
|
||||
spec_qsl_cpu = spec_qsl[: num_spec_decodes + 1]
|
||||
seq_starts = spec_qsl_cpu[:-1]
|
||||
seq_lens = spec_qsl_cpu[1:] - spec_qsl_cpu[:-1]
|
||||
|
||||
# ---- 1. Convolution (per-sequence rolling buffer) ----
|
||||
w2d = _unpacked_conv_weight(layer) # (dim, width)
|
||||
dim = w2d.size(0)
|
||||
w = w2d.unsqueeze(1) # (dim, 1, width) for F.conv1d depthwise
|
||||
dim = mixed_qkv_spec.size(-1)
|
||||
bias = layer.conv1d.bias
|
||||
silu = layer.activation == "silu"
|
||||
|
||||
conv_out = torch.empty_like(mixed_qkv_spec)
|
||||
col0 = spec_state_indices[:, 0].to("cpu", torch.int64)
|
||||
for i in range(num_spec_decodes):
|
||||
q_i = int(seq_lens[i].item())
|
||||
if q_i == 0:
|
||||
continue
|
||||
start = int(seq_starts[i].item())
|
||||
slot0 = int(col0[i].item())
|
||||
a_prev = int(num_acc_cpu[i].item())
|
||||
offset = a_prev - 1
|
||||
B = conv_buf[slot0] # (dim, state_len)
|
||||
x_seq = mixed_qkv_spec[start : start + q_i].transpose(0, 1).to(B.dtype)
|
||||
prior = B[:, offset : offset + (width - 1)]
|
||||
conv_in = torch.cat([prior, x_seq], dim=-1).unsqueeze(0) # (1, dim, w-1+q)
|
||||
out = F.conv1d(conv_in, w, bias, groups=dim)[0] # (dim, q_i)
|
||||
if silu:
|
||||
out = F.silu(out)
|
||||
conv_out[start : start + q_i] = out.transpose(0, 1).to(conv_out.dtype)
|
||||
# Roll the buffer: drop ``a_prev`` from the front, append the new
|
||||
# draft tokens, keep total length == state_len.
|
||||
keep = B[:, offset + 1 : offset + 1 + (state_len - q_i)]
|
||||
new_B = torch.cat([keep, x_seq], dim=-1)
|
||||
B.copy_(new_B)
|
||||
can_use_native_conv = (
|
||||
torch.cpu._is_amx_tile_supported()
|
||||
and not is_conv_state_dim_first()
|
||||
and width == 4
|
||||
and num_spec_decodes > 0
|
||||
and bool(torch.all(seq_lens == seq_lens[0]).item())
|
||||
and int(seq_lens[0].item()) > 0
|
||||
)
|
||||
if can_use_native_conv:
|
||||
q_i = int(seq_lens[0].item())
|
||||
conv_out = ops.causal_conv1d_update_cpu(
|
||||
x=mixed_qkv_spec.view(num_spec_decodes, q_i, dim),
|
||||
conv_states=conv_buf,
|
||||
weight=layer.conv1d.weight,
|
||||
bias=bias,
|
||||
silu_activation=silu,
|
||||
conv_state_indices=spec_state_indices[:num_spec_decodes, 0]
|
||||
.to("cpu", torch.int32)
|
||||
.contiguous(),
|
||||
is_vnni=True,
|
||||
num_accepted_tokens=num_accepted[:num_spec_decodes].to("cpu", torch.int32),
|
||||
).view_as(mixed_qkv_spec)
|
||||
else:
|
||||
w = _unpacked_conv_weight(layer).unsqueeze(1)
|
||||
col0 = spec_state_indices[:num_spec_decodes, 0]
|
||||
num_acc_cpu = num_accepted[:num_spec_decodes]
|
||||
conv_out = torch.empty_like(mixed_qkv_spec)
|
||||
for i in range(num_spec_decodes):
|
||||
q_i = int(seq_lens[i].item())
|
||||
if q_i == 0:
|
||||
continue
|
||||
start = int(seq_starts[i].item())
|
||||
slot0 = int(col0[i].item())
|
||||
offset = int(num_acc_cpu[i].item()) - 1
|
||||
B = conv_buf[slot0] # (dim, state_len)
|
||||
x_seq = mixed_qkv_spec[start : start + q_i].transpose(0, 1).to(B.dtype)
|
||||
prior = B[:, offset : offset + (width - 1)]
|
||||
conv_in = torch.cat([prior, x_seq], dim=-1).unsqueeze(0)
|
||||
out = F.conv1d(conv_in, w, bias, groups=dim)[0] # (dim, q_i)
|
||||
if silu:
|
||||
out = F.silu(out)
|
||||
conv_out[start : start + q_i] = out.transpose(0, 1).to(conv_out.dtype)
|
||||
# Roll the buffer: drop the accepted history from the front, append
|
||||
# the new draft tokens, keep total length == state_len.
|
||||
keep = B[:, offset + 1 : offset + 1 + (state_len - q_i)]
|
||||
new_B = torch.cat([keep, x_seq], dim=-1)
|
||||
B.copy_(new_B)
|
||||
|
||||
# ---- 2. Recurrent (multi-slot SSM state) ----
|
||||
# Single fused kernel call: it runs the recurrence over each sequence's
|
||||
# draft tokens internally, resumes from slot ``num_accepted-1`` and stores
|
||||
# the state after token ``t`` into slot ``t`` (rollback for the next step).
|
||||
query, key, value = layer.rearrange_mixed_qkv(conv_out)
|
||||
query = query.squeeze(0).contiguous()
|
||||
key = key.squeeze(0).contiguous()
|
||||
value = value.squeeze(0).contiguous()
|
||||
query = query.squeeze(0)
|
||||
key = key.squeeze(0)
|
||||
value = value.squeeze(0)
|
||||
spec_idx = spec_state_indices[:num_spec_decodes].to(torch.int32).contiguous()
|
||||
num_acc = num_accepted[:num_spec_decodes].to(torch.int32).contiguous()
|
||||
cu = spec_qsl[: num_spec_decodes + 1].to(torch.int32).contiguous()
|
||||
@@ -458,8 +475,8 @@ def _spec_forward(
|
||||
q=query,
|
||||
k=key,
|
||||
v=value,
|
||||
a=a_spec.contiguous(),
|
||||
b=b_spec.contiguous(),
|
||||
a=a_spec,
|
||||
b=b_spec,
|
||||
initial_state_source=ssm_state,
|
||||
spec_state_indices=spec_idx,
|
||||
num_accepted_tokens=num_acc,
|
||||
@@ -469,15 +486,6 @@ def _spec_forward(
|
||||
return out_spec
|
||||
|
||||
|
||||
def core_attn_out_like(layer, mixed_qkv_spec: torch.Tensor) -> torch.Tensor:
|
||||
num_tokens = mixed_qkv_spec.size(0)
|
||||
return torch.zeros(
|
||||
(num_tokens, layer.num_v_heads // layer.tp_size, layer.head_v_dim),
|
||||
dtype=mixed_qkv_spec.dtype,
|
||||
device=mixed_qkv_spec.device,
|
||||
)
|
||||
|
||||
|
||||
def _spec_aware_nonspec(
|
||||
layer,
|
||||
attn_metadata_i: GDNAttentionMetadata,
|
||||
@@ -489,13 +497,19 @@ def _spec_aware_nonspec(
|
||||
ssm_state: torch.Tensor,
|
||||
width: int,
|
||||
) -> None:
|
||||
"""Non-spec prefill/decode with a wide conv buffer (torch path)."""
|
||||
"""Non-spec prefill/decode with a wide conv buffer."""
|
||||
state_indices_tensor = attn_metadata_i.non_spec_state_indices_tensor
|
||||
query_start_loc = attn_metadata_i.non_spec_query_start_loc
|
||||
assert state_indices_tensor is not None
|
||||
assert query_start_loc is not None
|
||||
state_indices_tensor = state_indices_tensor.contiguous()
|
||||
|
||||
conv_weights = _unpacked_conv_weight(layer)
|
||||
is_amx = torch.cpu._is_amx_tile_supported()
|
||||
if is_amx and is_conv_state_dim_first():
|
||||
raise RuntimeError("AMX GDN attention requires `SD` conv_state layout.")
|
||||
|
||||
if not is_amx:
|
||||
conv_weights = _unpacked_conv_weight(layer)
|
||||
|
||||
num_decodes = attn_metadata_i.num_decodes
|
||||
num_decode_tokens = attn_metadata_i.num_decode_tokens
|
||||
@@ -507,42 +521,48 @@ def _spec_aware_nonspec(
|
||||
decode_b = b[:num_decode_tokens]
|
||||
decode_a = a[:num_decode_tokens]
|
||||
decode_state_indices = state_indices_tensor[:num_decodes]
|
||||
# Only the first ``width-1`` columns hold the real conv state.
|
||||
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
|
||||
conv_state_view = conv_buf[:, :, : width - 1]
|
||||
decode_conv_state = conv_state_view[decode_state_indices].contiguous()
|
||||
decode_mixed_qkv = causal_conv1d_update_torch(
|
||||
x=decode_mixed_qkv.unsqueeze(-1),
|
||||
conv_state=decode_conv_state,
|
||||
weight=conv_weights,
|
||||
bias=layer.conv1d.bias,
|
||||
activation=layer.activation,
|
||||
).squeeze(-1)
|
||||
conv_state_view[decode_state_indices] = decode_conv_state
|
||||
else:
|
||||
decode_mixed_qkv = causal_conv1d_update_cpu(
|
||||
if is_amx:
|
||||
decode_mixed_qkv = ops.causal_conv1d_update_cpu(
|
||||
x=decode_mixed_qkv,
|
||||
conv_state=conv_buf[:, :, : width - 1],
|
||||
weight=conv_weights,
|
||||
conv_states=conv_buf,
|
||||
weight=layer.conv1d.weight,
|
||||
bias=layer.conv1d.bias,
|
||||
activation=layer.activation,
|
||||
silu_activation=layer.activation == "silu",
|
||||
conv_state_indices=decode_state_indices,
|
||||
is_vnni=True,
|
||||
)
|
||||
else:
|
||||
# Only the first ``width-1`` columns hold the real conv state.
|
||||
conv_state_view = conv_buf[:, :, : width - 1]
|
||||
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
|
||||
decode_conv_state = conv_state_view[decode_state_indices].contiguous()
|
||||
decode_mixed_qkv = causal_conv1d_update_torch(
|
||||
x=decode_mixed_qkv.unsqueeze(-1),
|
||||
conv_state=decode_conv_state,
|
||||
weight=conv_weights,
|
||||
bias=layer.conv1d.bias,
|
||||
activation=layer.activation,
|
||||
).squeeze(-1)
|
||||
conv_state_view[decode_state_indices] = decode_conv_state
|
||||
else:
|
||||
decode_mixed_qkv = causal_conv1d_update_cpu(
|
||||
x=decode_mixed_qkv,
|
||||
conv_state=conv_state_view,
|
||||
weight=conv_weights,
|
||||
bias=layer.conv1d.bias,
|
||||
activation=layer.activation,
|
||||
conv_state_indices=decode_state_indices,
|
||||
)
|
||||
|
||||
query, key, value = layer.rearrange_mixed_qkv(decode_mixed_qkv)
|
||||
# rearrange_mixed_qkv can return views whose last dim is not
|
||||
# contiguous; the fused CPU kernel requires a contiguous last dim.
|
||||
query = query.contiguous()
|
||||
key = key.contiguous()
|
||||
value = value.contiguous()
|
||||
attn_out = ops.fused_sigmoid_gating_delta_rule_update_cpu(
|
||||
A_log=layer.A_log,
|
||||
dt_bias=layer.dt_bias,
|
||||
q=query,
|
||||
k=key,
|
||||
v=value,
|
||||
a=decode_a.contiguous(),
|
||||
b=decode_b.contiguous(),
|
||||
a=decode_a,
|
||||
b=decode_b,
|
||||
initial_state_source=ssm_state,
|
||||
initial_state_indices=decode_state_indices,
|
||||
cu_seqlens=query_start_loc[: num_decodes + 1],
|
||||
@@ -568,17 +588,29 @@ def _spec_aware_nonspec(
|
||||
prefill_has_initial_state = has_initial_state[
|
||||
num_decodes : num_decodes + num_prefills
|
||||
]
|
||||
# ``causal_conv1d_torch`` only touches columns [:width-1] of the buffer.
|
||||
prefill_mixed_qkv = causal_conv1d_torch(
|
||||
x=prefill_mixed_qkv.transpose(0, 1),
|
||||
weight=conv_weights,
|
||||
bias=layer.conv1d.bias,
|
||||
conv_states=conv_buf,
|
||||
query_start_loc=prefill_query_start_loc,
|
||||
cache_indices=prefill_state_indices,
|
||||
has_initial_state=prefill_has_initial_state,
|
||||
activation=layer.activation,
|
||||
).transpose(0, 1)
|
||||
if is_amx:
|
||||
prefill_mixed_qkv = ops.causal_conv1d_fwd_cpu(
|
||||
x=prefill_mixed_qkv.transpose(0, 1),
|
||||
weight=layer.conv1d.weight,
|
||||
bias=layer.conv1d.bias,
|
||||
conv_states=conv_buf,
|
||||
query_start_loc=prefill_query_start_loc,
|
||||
cache_indices=prefill_state_indices,
|
||||
has_initial_state=prefill_has_initial_state,
|
||||
silu_activation=layer.activation == "silu",
|
||||
is_vnni=True,
|
||||
).transpose(0, 1)
|
||||
else:
|
||||
prefill_mixed_qkv = causal_conv1d_torch(
|
||||
x=prefill_mixed_qkv.transpose(0, 1),
|
||||
weight=conv_weights,
|
||||
bias=layer.conv1d.bias,
|
||||
conv_states=conv_buf,
|
||||
query_start_loc=prefill_query_start_loc,
|
||||
cache_indices=prefill_state_indices,
|
||||
has_initial_state=prefill_has_initial_state,
|
||||
activation=layer.activation,
|
||||
).transpose(0, 1)
|
||||
|
||||
query, key, value = layer.rearrange_mixed_qkv(prefill_mixed_qkv)
|
||||
g, beta = ops.fused_gdn_gating_cpu(
|
||||
@@ -618,24 +650,41 @@ def _spec_aware_nonspec_subset(
|
||||
|
||||
Returns outputs ordered like ``non_spec_token_indx``.
|
||||
"""
|
||||
out = core_attn_out_like(layer, mixed_qkv)
|
||||
has_initial_state = attn_metadata_i.has_initial_state
|
||||
prefill_state_indices = attn_metadata_i.prefill_state_indices
|
||||
prefill_qsl = attn_metadata_i.prefill_query_start_loc
|
||||
assert prefill_state_indices is not None and prefill_qsl is not None
|
||||
assert has_initial_state is not None
|
||||
prefill_state_indices = prefill_state_indices.contiguous()
|
||||
|
||||
conv_weights = _unpacked_conv_weight(layer)
|
||||
conv_out = causal_conv1d_torch(
|
||||
x=mixed_qkv.transpose(0, 1),
|
||||
weight=conv_weights,
|
||||
bias=layer.conv1d.bias,
|
||||
conv_states=conv_buf,
|
||||
query_start_loc=prefill_qsl,
|
||||
cache_indices=prefill_state_indices,
|
||||
has_initial_state=has_initial_state,
|
||||
activation=layer.activation,
|
||||
).transpose(0, 1)
|
||||
is_amx = torch.cpu._is_amx_tile_supported()
|
||||
if is_amx and is_conv_state_dim_first():
|
||||
raise RuntimeError("AMX GDN attention requires `SD` conv_state layout.")
|
||||
|
||||
if is_amx:
|
||||
conv_out = ops.causal_conv1d_fwd_cpu(
|
||||
x=mixed_qkv.transpose(0, 1),
|
||||
weight=layer.conv1d.weight,
|
||||
bias=layer.conv1d.bias,
|
||||
conv_states=conv_buf,
|
||||
query_start_loc=prefill_qsl,
|
||||
cache_indices=prefill_state_indices,
|
||||
has_initial_state=has_initial_state,
|
||||
silu_activation=layer.activation == "silu",
|
||||
is_vnni=True,
|
||||
).transpose(0, 1)
|
||||
else:
|
||||
conv_weights = _unpacked_conv_weight(layer)
|
||||
conv_out = causal_conv1d_torch(
|
||||
x=mixed_qkv.transpose(0, 1),
|
||||
weight=conv_weights,
|
||||
bias=layer.conv1d.bias,
|
||||
conv_states=conv_buf,
|
||||
query_start_loc=prefill_qsl,
|
||||
cache_indices=prefill_state_indices,
|
||||
has_initial_state=has_initial_state,
|
||||
activation=layer.activation,
|
||||
).transpose(0, 1)
|
||||
|
||||
query, key, value = layer.rearrange_mixed_qkv(conv_out)
|
||||
g, beta = ops.fused_gdn_gating_cpu(
|
||||
@@ -658,8 +707,7 @@ def _spec_aware_nonspec_subset(
|
||||
ssm_state[prefill_state_indices] = last_recurrent_state.to(
|
||||
ssm_state.dtype, copy=False
|
||||
)
|
||||
out[:] = attn_out.squeeze(0)
|
||||
return out
|
||||
return attn_out.squeeze(0)
|
||||
|
||||
|
||||
def cpu_gdn_attention_core_fake(
|
||||
|
||||
Reference in New Issue
Block a user