diff --git a/src/fann.c b/src/fann.c index 9af7e388..d2e5f181 100644 --- a/src/fann.c +++ b/src/fann.c @@ -769,6 +769,12 @@ FANN_EXTERNAL fann_type *FANN_API fann_run(struct fann * ann, fann_type * input) case FANN_LINEAR_PIECE_SYMMETRIC: neuron_it->value = (fann_type)((neuron_sum < -multiplier) ? -multiplier : (neuron_sum > multiplier) ? multiplier : neuron_sum); break; + case FANN_RELU: + neuron_it->value = (fann_type)(neuron_sum > 0 ? neuron_sum : 0); + break; + case FANN_LEAKY_RELU: + neuron_it->value = (fann_type)(neuron_sum > 0 ? neuron_sum : neuron_sum / 100); + break; case FANN_ELLIOT: case FANN_ELLIOT_SYMMETRIC: case FANN_GAUSSIAN: diff --git a/src/fann_train.c b/src/fann_train.c index 049e6de9..856bcc6c 100644 --- a/src/fann_train.c +++ b/src/fann_train.c @@ -72,6 +72,10 @@ fann_type fann_activation_derived(unsigned int activation_function, return (fann_type) fann_cos_derive(steepness, sum); case FANN_THRESHOLD: fann_error(NULL, FANN_E_CANT_TRAIN_ACTIVATION); + case FANN_RELU: + return (fann_type) fann_relu_derive(steepness, sum); + case FANN_LEAKY_RELU: + return (fann_type) fann_leaky_relu_derive(steepness, sum); } return 0; } @@ -133,6 +137,8 @@ fann_type fann_update_MSE(struct fann *ann, struct fann_neuron* neuron, fann_typ case FANN_LINEAR_PIECE: case FANN_SIN: case FANN_COS: + case FANN_RELU: + case FANN_LEAKY_RELU: break; } diff --git a/src/include/config.h b/src/include/config.h index f2fb1c8a..8e1c98e9 100644 --- a/src/include/config.h +++ b/src/include/config.h @@ -2,7 +2,7 @@ /* #undef PACKAGE */ /* Version number of package */ -#define VERSION "2.2.0" +/* #undef VERSION */ /* Define for the x86_64 CPU famyly */ /* #undef X86_64 */ diff --git a/src/include/fann_activation.h b/src/include/fann_activation.h index 3ab2193a..236d7370 100644 --- a/src/include/fann_activation.h +++ b/src/include/fann_activation.h @@ -82,6 +82,14 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA #define fann_cos_real(sum) (cos(sum)/2.0f+0.5f) #define fann_cos_derive(steepness, sum) (steepness*-sin(steepness*sum)/2.0f) + +/* FANN_RELU */ +#define fann_relu_real(sum) (sum > 0.0 ? sum : 0.0) +#define fann_relu_derive(steepness, sum) (sum > 0.0 ? steepness : 0.0) + +#define fann_leaky_relu_real(sum) (sum > 0.0 ? sum : (sum / 100.0)) +#define fann_leaky_relu_derive(steepness, sum) (sum > 0.0 ? steepness : (steepness / 100.0)) + #define fann_activation_switch(activation_function, value, result) \ switch(activation_function) \ { \ @@ -137,8 +145,14 @@ switch(activation_function) \ result = (fann_type)fann_cos_real(value); \ break; \ case FANN_GAUSSIAN_STEPWISE: \ - result = 0; \ - break; \ + result = 0; \ + break; \ + case FANN_RELU: \ + result = (fann_type)fann_relu_real(value); \ + break; \ + case FANN_LEAKY_RELU: \ + result = (fann_type)fann_leaky_relu_real(value); \ + break; \ } #endif diff --git a/src/include/fann_data.h b/src/include/fann_data.h index 99f42c76..a7318284 100644 --- a/src/include/fann_data.h +++ b/src/include/fann_data.h @@ -226,7 +226,9 @@ enum fann_activationfunc_enum FANN_SIN_SYMMETRIC, FANN_COS_SYMMETRIC, FANN_SIN, - FANN_COS + FANN_COS, + FANN_RELU, + FANN_LEAKY_RELU }; /* Constant: FANN_ACTIVATIONFUNC_NAMES @@ -258,7 +260,9 @@ static char const *const FANN_ACTIVATIONFUNC_NAMES[] = { "FANN_SIN_SYMMETRIC", "FANN_COS_SYMMETRIC", "FANN_SIN", - "FANN_COS" + "FANN_COS", + "FANN_RELU", + "FANN_LEAKY_RELU" }; /* Enum: fann_errorfunc_enum