[Kernel] Mamba support different layout for Conv state (#37416)
This commit is contained in:
@@ -31,7 +31,11 @@ from .linear import (
|
||||
RowParallelLinear,
|
||||
)
|
||||
from .mamba.abstract import MambaBase
|
||||
from .mamba.mamba_utils import MambaStateDtypeCalculator, MambaStateShapeCalculator
|
||||
from .mamba.mamba_utils import (
|
||||
MambaStateDtypeCalculator,
|
||||
MambaStateShapeCalculator,
|
||||
is_conv_state_dim_first,
|
||||
)
|
||||
from .mamba.ops.causal_conv1d import causal_conv1d_fn, causal_conv1d_update
|
||||
from .quantization.base_config import QuantizationConfig
|
||||
|
||||
@@ -315,10 +319,12 @@ class KimiDeltaAttention(nn.Module, MambaBase):
|
||||
beta = beta[:num_actual_tokens]
|
||||
|
||||
(conv_state_q, conv_state_k, conv_state_v, recurrent_state) = constant_caches
|
||||
# deal with strides
|
||||
conv_state_q = conv_state_q.transpose(-1, -2)
|
||||
conv_state_k = conv_state_k.transpose(-1, -2)
|
||||
conv_state_v = conv_state_v.transpose(-1, -2)
|
||||
# conv_state must be (..., dim, width-1) for the conv kernels.
|
||||
# DS layout stores it that way directly; SD layout needs a transpose.
|
||||
if not is_conv_state_dim_first():
|
||||
conv_state_q = conv_state_q.transpose(-1, -2)
|
||||
conv_state_k = conv_state_k.transpose(-1, -2)
|
||||
conv_state_v = conv_state_v.transpose(-1, -2)
|
||||
|
||||
q_conv_weights = self.q_conv1d.weight.view(
|
||||
self.q_conv1d.weight.size(0), self.q_conv1d.weight.size(2)
|
||||
|
||||
Reference in New Issue
Block a user