Adds GELU Activation

neat/activations.py

@@ -96,6 +96,18 @@ def cube_activation(z):
     return z ** 3
 
 
+def gelu_activation(z):
+    return 0.5 * z * (1 + math.erf(z / math.sqrt(2)))
+
+
+def gelu_approximation_activation(z):
+    return 0.5 * z * (
+        1 + math.tanh(
+            math.sqrt(2 / math.pi) * (z + 0.044715 * z**3)
+        )
+    )
+
+
 class InvalidActivationFunction(TypeError):
     pass
 
@@ -151,6 +163,8 @@ def __init__(self):
         self.add('hat', hat_activation)
         self.add('square', square_activation)
         self.add('cube', cube_activation)
+        self.add('gelu', gelu_activation)
+        self.add('gelu_approximation', gelu_approximation_activation)
 
     def add(self, name, function):
         validate_activation(function)

tests/test_activation.py

@@ -105,6 +105,21 @@ def test_cube():
     assert activations.cube_activation(0.5) == 0.125
     assert activations.cube_activation(1.0) == 1.0
 
+def test_gelu():
+    assert_almost_equal(activations.gelu_activation(-1.0), -0.15865525393145707)
+    assert_almost_equal(activations.gelu_activation(-0.5), -0.15426876936299347)
+    assert activations.gelu_activation(0.0) == 0.0
+    assert_almost_equal(activations.gelu_activation(0.5), 0.3457312306370065)
+    assert_almost_equal(activations.gelu_activation(1.0), 0.8413447460685429)
+
+
+def test_gelu_approximation():
+    assert_almost_equal(activations.gelu_approximation_activation(-1.0), -0.1588080093917233)
+    assert_almost_equal(activations.gelu_approximation_activation(-0.5), -0.15428599017485606)
+    assert activations.gelu_approximation_activation(0.0) == 0.0
+    assert_almost_equal(activations.gelu_approximation_activation(0.5), 0.34571400982514394)
+    assert_almost_equal(activations.gelu_approximation_activation(1.0), 0.8411919906082768)
+
 
 def plus_activation(x):
     """ Not useful - just a check. """
@@ -145,6 +160,8 @@ def test_function_set():
     assert s.get('hat') is not None
     assert s.get('square') is not None
     assert s.get('cube') is not None
+    assert s.get('gelu') is not None
+    assert s.get('gelu_approximation') is not None
 
     assert s.is_valid('sigmoid')
     assert s.is_valid('tanh')
@@ -163,6 +180,8 @@ def test_function_set():
     assert s.is_valid('hat')
     assert s.is_valid('square')
     assert s.is_valid('cube')
+    assert s.is_valid('gelu')
+    assert s.is_valid('gelu_approximation') 
 
     assert not s.is_valid('foo')
 
@@ -213,3 +232,5 @@ def test_bad_add2():
     test_hat()
     test_square()
     test_cube()
+    test_gelu()
+    test_gelu_approximation()