[Model] Align nemotron config with final HF state and fix lm-eval-small (#7611)
This commit is contained in:
@@ -774,7 +774,7 @@ def get_rope(
|
||||
is_neox_style: bool = True,
|
||||
rope_scaling: Optional[Dict[str, Any]] = None,
|
||||
dtype: Optional[torch.dtype] = None,
|
||||
rotary_percent: float = 1.0,
|
||||
partial_rotary_factor: float = 1.0,
|
||||
) -> RotaryEmbedding:
|
||||
if dtype is None:
|
||||
dtype = torch.get_default_dtype()
|
||||
@@ -787,8 +787,8 @@ def get_rope(
|
||||
rope_scaling_args = tuple(rope_scaling_tuple.items())
|
||||
else:
|
||||
rope_scaling_args = None
|
||||
if rotary_percent < 1.0:
|
||||
rotary_dim = int(rotary_dim * rotary_percent)
|
||||
if partial_rotary_factor < 1.0:
|
||||
rotary_dim = int(rotary_dim * partial_rotary_factor)
|
||||
key = (head_size, rotary_dim, max_position, base, is_neox_style,
|
||||
rope_scaling_args, dtype)
|
||||
if key in _ROPE_DICT:
|
||||
|
||||
@@ -53,7 +53,7 @@ from .utils import PPMissingLayer, is_pp_missing_parameter, make_layers
|
||||
# - There is no gate_proj, just up_proj
|
||||
# - Normal LayerNorm (with a +1 to the weights) instead of RMSNorm
|
||||
# - Squared ReLU instead of SwiGLU
|
||||
# - Adds a rotary_percent to RoPE
|
||||
# - Adds a partial_rotary_factor to RoPE
|
||||
|
||||
|
||||
def _cast_if_autocast_enabled(*args):
|
||||
@@ -161,7 +161,7 @@ class NemotronAttention(nn.Module):
|
||||
self.kv_size = self.num_kv_heads * self.head_dim
|
||||
self.scaling = self.head_dim**-0.5
|
||||
self.rope_theta = rope_theta
|
||||
self.rotary_percent = config.rope_percent
|
||||
self.partial_rotary_factor = config.partial_rotary_factor
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
@@ -187,7 +187,7 @@ class NemotronAttention(nn.Module):
|
||||
max_position=max_position_embeddings,
|
||||
base=rope_theta,
|
||||
rope_scaling=rope_scaling,
|
||||
rotary_percent=self.rotary_percent,
|
||||
partial_rotary_factor=self.partial_rotary_factor,
|
||||
)
|
||||
self.attn = Attention(self.num_heads,
|
||||
self.head_dim,
|
||||
|
||||
Reference in New Issue
Block a user