fix activations

Author: ayasyrev

model_constructor/activations.py

@@ -3,11 +3,12 @@
 from torch import nn as nn
 from torch.nn import functional as F
 
+
 __all__ = ['mish', 'Mish', 'mish_jit', 'MishJit', 'mish_jit_fwd', 'mish_jit_bwd', 'MishJitAutoFn', 'mish_me', 'MishMe',
            'hard_mish_jit', 'HardMishJit', 'hard_mish_jit_fwd', 'hard_mish_jit_bwd', 'HardMishJitAutoFn',
            'hard_mish_me', 'HardMishMe']
 
-# Cell
+
 def mish(x, inplace: bool = False):
     """Mish: A Self Regularized Non-Monotonic Neural Activation Function - https://arxiv.org/abs/1908.08681
     NOTE: I don't have a working inplace variant
@@ -24,7 +25,7 @@ def __init__(self, inplace: bool = False):
     def forward(self, x):
         return mish(x)
 
-# Cell
+
 @torch.jit.script
 def mish_jit(x, _inplace: bool = False):
     """Jit version of Mish.
@@ -42,7 +43,7 @@ def __init__(self, inplace: bool = False):
     def forward(self, x):
         return mish_jit(x)
 
-# Cell
+
 @torch.jit.script
 def mish_jit_fwd(x):
     # return x.mul(torch.tanh(F.softplus(x)))
@@ -83,7 +84,7 @@ def __init__(self, inplace: bool = False):
     def forward(self, x):
         return MishJitAutoFn.apply(x)
 
-# Cell
+
 @torch.jit.script
 def hard_mish_jit(x, inplace: bool = False):
     """ Hard Mish
@@ -104,7 +105,7 @@ def __init__(self, inplace: bool = False):
     def forward(self, x):
         return hard_mish_jit(x)
 
-# Cell
+
 @torch.jit.script
 def hard_mish_jit_fwd(x):
     return 0.5 * x * (x + 2).clamp(min=0, max=2)
@@ -146,4 +147,4 @@ def __init__(self, inplace: bool = False):
         super(HardMishMe, self).__init__()
 
     def forward(self, x):
-        return HardMishJitAutoFn.apply(x)
+        return HardMishJitAutoFn.apply(x)