Refactor sliding window configuration to Transformers best practice (#21927)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -116,13 +116,8 @@ class SambaYAttention(nn.Module):
|
||||
self.Wqkv = nn.Linear(self.hidden_size, op_size, bias=True)
|
||||
|
||||
# disable sliding window for the second half of the model
|
||||
sliding_window = config.interleaved_sliding_window[layer_idx]
|
||||
if layer_idx >= config.num_hidden_layers // 2:
|
||||
assert sliding_window is None, \
|
||||
"sliding_window must be none for the second decoder"
|
||||
else:
|
||||
assert sliding_window is not None, \
|
||||
"sliding_window must be set for the first decoder"
|
||||
is_sliding = config.layer_types[layer_idx] == "sliding_attention"
|
||||
sliding_window = config.sliding_window if is_sliding else None
|
||||
|
||||
assert self.num_heads % 2 == 0, 'num_heads should be even'
|
||||
assert self.num_key_value_heads % 2 == 0, 'num_heads should be even'
|
||||
|
||||
Reference in New Issue
Block a user