machine-learning.functions: Add softmax, softmin, relu, etc.

Add some unit-tests.

extra/machine-learning/functions/authors.txt

@@ -0,0 +1 @@
+Doug Coleman

extra/machine-learning/functions/functions-tests.factor

@@ -0,0 +1,60 @@
+! Copyright (C) 2017 Doug Coleman.
+! See http://factorcode.org/license.txt for BSD license.
+USING: tools.test machine-learning.functions ;
+IN: machine-learning.functions.tests
+
+{ 0 } [ 0 relu ] unit-test
+{ .5 .000001 } [ .5 relu ] unit-test~
+{ 1 } [ 1 relu ] unit-test
+{ 0 } [ -.5 relu ] unit-test
+{ 0 } [ -1 relu ] unit-test
+
+{ 0 } [ -1 relu6 ] unit-test
+{ 6 } [ 10 relu6 ] unit-test
+
+{ -.01 .00001 } [ -1 .01 leaky-relu ] unit-test~
+{ 0 } [ 0 .01 leaky-relu ] unit-test
+{ .5 .000001 } [ .5 .01 leaky-relu ] unit-test~
+{ 1 } [ 1 .01 leaky-relu ] unit-test
+{ -.005 .1 } [ -.5 .01 leaky-relu ] unit-test~
+
+{ 0 } [ 0 default-leaky-relu ] unit-test
+{ 1 } [ 1 default-leaky-relu ] unit-test
+{ .5 .000001 } [ .5 default-leaky-relu ] unit-test~
+{ -.005 .1 } [ -.5 default-leaky-relu ] unit-test~
+
+{
+    {
+        0.327201948676532
+        0.1203708700293295
+        0.04428196839971006
+        0.0162904257888568
+        0.327201948676532
+        0.1203708700293295
+        0.04428196839971006
+    } .00001
+} [ { 1. 2. 3. 4. 1. 2. 3. } softmin ] unit-test-v~
+
+{
+    {
+        0.02364054302159138
+        0.06426165851049616
+        0.1746812985957223
+        0.4748329997443803
+        0.02364054302159138
+        0.06426165851049616
+        0.1746812985957223
+    } .00001
+} [ { 1. 2. 3. 4. 1. 2. 3. } softmax ] unit-test-v~
+
+{
+    {
+        -7.463073944710613
+        -12.13384821518222
+        -24.83032903943923
+        -59.3429421493958
+        -7.463073944710613
+        -12.13384821518222
+        -24.83032903943923
+    } .00001
+} [ { 1. 2. 3. 4. 1. 2. 3. } log-softmax ] unit-test-v~

extra/machine-learning/functions/functions.factor

@@ -0,0 +1,27 @@
+! Copyright (C) 2017 Doug Coleman.
+! See http://factorcode.org/license.txt for BSD license.
+USING: fry kernel math math.functions math.order math.vectors
+sequences ;
+IN: machine-learning.functions
+
+: relu ( x -- x' ) 0 max ; inline
+
+: relu6 ( x -- x' ) 0 6 clamp ; inline
+
+: leaky-relu ( x a -- x' )
+    over 0 < [ * ] [ drop ] if ; inline
+
+: default-leaky-relu ( x -- x' )
+    .01 leaky-relu ; inline
+
+: vexp-sum ( seq -- seq' sum )
+    [ e^ ] map dup sum ; inline
+
+: softmax ( seq -- softmax )
+    vexp-sum '[ _ /f ] map ; inline
+
+: log-softmax ( seq -- softmax )
+    vexp-sum '[ e^ _ * recip log ] map ;
+
+: softmin ( seq -- softmin )
+    vneg softmax ; inline