@@ -63,8 +63,8 @@ def _cast_if_autocast_enabled(*args):
|
||||
if not torch.is_autocast_enabled():
|
||||
return args
|
||||
else:
|
||||
return torch.cuda.amp.autocast_mode._cast(
|
||||
args, torch.get_autocast_gpu_dtype())
|
||||
return torch.amp.autocast_mode._cast(
|
||||
args, device_type="cuda", dtype=torch.get_autocast_gpu_dtype())
|
||||
|
||||
|
||||
class NemotronLayerNorm1P(nn.LayerNorm):
|
||||
@@ -89,7 +89,7 @@ class NemotronLayerNorm1P(nn.LayerNorm):
|
||||
residual = x
|
||||
args = _cast_if_autocast_enabled(x, self.normalized_shape,
|
||||
self.weight + 1, self.bias, self.eps)
|
||||
with torch.cuda.amp.autocast(enabled=False):
|
||||
with torch.amp.autocast("cuda", enabled=False):
|
||||
x = torch.nn.functional.layer_norm(*args)
|
||||
return x if residual is None else (x, residual)
|
||||
|
||||
|
||||
@@ -1766,9 +1766,12 @@ class MultiHeadedAttention(nn.Module):
|
||||
if mask.dtype != q.dtype:
|
||||
attn_mask = attn_mask.to(q.dtype)
|
||||
|
||||
with torch.backends.cuda.sdp_kernel(enable_flash=True,
|
||||
enable_math=True,
|
||||
enable_mem_efficient=True):
|
||||
with torch.nn.attention.sdpa_kernel([
|
||||
torch.nn.attention.SDPBackend.FLASH_ATTENTION,
|
||||
torch.nn.attention.SDPBackend.EFFICIENT_ATTENTION,
|
||||
torch.nn.attention.SDPBackend.MATH,
|
||||
torch.nn.attention.SDPBackend.CUDNN_ATTENTION,
|
||||
]):
|
||||
x = torch.nn.functional.scaled_dot_product_attention(
|
||||
q,
|
||||
k,
|
||||
|
||||
Reference in New Issue
Block a user