[CI] change spell checker from codespell to typos (#18711)
Signed-off-by: Andy Xie <andy.xning@gmail.com>
This commit is contained in:
@@ -145,7 +145,7 @@ class Phi3SmallSelfAttention(nn.Module):
|
||||
self.num_q_per_kv = self.num_heads // self.num_key_value_heads
|
||||
if self.tp_size > 1:
|
||||
assert self.num_key_value_heads % self.tp_size == 0
|
||||
self.num_kv_heads_per_partion = max(
|
||||
self.num_kv_heads_per_partition = max(
|
||||
1, self.num_key_value_heads // self.tp_size)
|
||||
self.num_heads_per_partition = self.num_heads // self.tp_size
|
||||
|
||||
@@ -212,7 +212,7 @@ class Phi3SmallSelfAttention(nn.Module):
|
||||
bs_params = {
|
||||
'max_seqlen': self.max_position_embeddings,
|
||||
'num_heads': self.num_heads_per_partition,
|
||||
"num_kv_heads": self.num_kv_heads_per_partion,
|
||||
"num_kv_heads": self.num_kv_heads_per_partition,
|
||||
"block_size": self.sparse_block_size,
|
||||
"local_blocks": self.local_blocks,
|
||||
"vert_stride": self.vert_stride,
|
||||
@@ -222,7 +222,7 @@ class Phi3SmallSelfAttention(nn.Module):
|
||||
self.attn = Attention(self.num_heads_per_partition,
|
||||
self.head_dim,
|
||||
self.scale,
|
||||
num_kv_heads=self.num_kv_heads_per_partion,
|
||||
num_kv_heads=self.num_kv_heads_per_partition,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
blocksparse_params=bs_params,
|
||||
@@ -243,8 +243,8 @@ class Phi3SmallSelfAttention(nn.Module):
|
||||
# NOTE: this is required by RotaryEmbed, which indeed does not have to
|
||||
# TODO: allow 3D QK for rotary forward
|
||||
q = q.reshape(-1, self.head_dim * self.num_heads_per_partition)
|
||||
k = k.reshape(-1, self.head_dim * self.num_kv_heads_per_partion)
|
||||
v = v.reshape(-1, self.head_dim * self.num_kv_heads_per_partion)
|
||||
k = k.reshape(-1, self.head_dim * self.num_kv_heads_per_partition)
|
||||
v = v.reshape(-1, self.head_dim * self.num_kv_heads_per_partition)
|
||||
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
attn_output = self.attn(q, k, v)
|
||||
|
||||
Reference in New Issue
Block a user