From 588151befcf193df4c69a6c6a6afec2421042518 Mon Sep 17 00:00:00 2001 From: hatsunearu Date: Fri, 24 Jun 2016 10:40:25 -0700 Subject: [PATCH 1/2] Added ReLU and LReLU functionality --- src/fann.c | 6 ++++++ src/fann_train.c | 8 +++++++- src/include/config.h | 2 +- src/include/fann_activation.h | 14 ++++++++++++++ src/include/fann_data.h | 8 ++++++-- 5 files changed, 34 insertions(+), 4 deletions(-) diff --git a/src/fann.c b/src/fann.c index 9af7e388..6aedf93f 100644 --- a/src/fann.c +++ b/src/fann.c @@ -769,6 +769,12 @@ FANN_EXTERNAL fann_type *FANN_API fann_run(struct fann * ann, fann_type * input) case FANN_LINEAR_PIECE_SYMMETRIC: neuron_it->value = (fann_type)((neuron_sum < -multiplier) ? -multiplier : (neuron_sum > multiplier) ? multiplier : neuron_sum); break; + case FANN_RELU: + neuron_it->value = (fann_type)(neuron_sum > 0 ? neuron_sum : 0); + break; + case FANN_LEAKY_RELU: + neuron_it->value = (fann_type)(neuron_sum > 0 ? neuron_sum : neuron_sum / 100); + break; case FANN_ELLIOT: case FANN_ELLIOT_SYMMETRIC: case FANN_GAUSSIAN: diff --git a/src/fann_train.c b/src/fann_train.c index 049e6de9..4fd2bdef 100644 --- a/src/fann_train.c +++ b/src/fann_train.c @@ -72,6 +72,10 @@ fann_type fann_activation_derived(unsigned int activation_function, return (fann_type) fann_cos_derive(steepness, sum); case FANN_THRESHOLD: fann_error(NULL, FANN_E_CANT_TRAIN_ACTIVATION); + case FANN_RELU: + return (fann_type) fann_relu_derive(steepness, sum); + case FANN_LEAKY_RELU: + return (fann_type) fann_leaky_relu_derive(steepness, sum); } return 0; } @@ -132,7 +136,9 @@ fann_type fann_update_MSE(struct fann *ann, struct fann_neuron* neuron, fann_typ case FANN_ELLIOT: case FANN_LINEAR_PIECE: case FANN_SIN: - case FANN_COS: + case FANN_COS: + case FANN_RELU: + case FANN_LEAKY_RELU: break; } diff --git a/src/include/config.h b/src/include/config.h index f2fb1c8a..8e1c98e9 100644 --- a/src/include/config.h +++ b/src/include/config.h @@ -2,7 +2,7 @@ /* #undef PACKAGE */ /* Version number of package */ -#define VERSION "2.2.0" +/* #undef VERSION */ /* Define for the x86_64 CPU famyly */ /* #undef X86_64 */ diff --git a/src/include/fann_activation.h b/src/include/fann_activation.h index 3ab2193a..d89a1cb1 100644 --- a/src/include/fann_activation.h +++ b/src/include/fann_activation.h @@ -82,6 +82,14 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA #define fann_cos_real(sum) (cos(sum)/2.0f+0.5f) #define fann_cos_derive(steepness, sum) (steepness*-sin(steepness*sum)/2.0f) + +/* FANN_RELU */ +#define fann_relu_real(sum) (sum > 0.0 ? sum : 0.0) +#define fann_relu_derive(steepness, sum) (sum > 0.0 ? steepness : 0.0) + +#define fann_leaky_relu_real(sum) (sum > 0.0 ? sum : (sum / 100.0)) +#define fann_leaky_relu_derive(steepness, sum) (sum > 0.0 ? steepness : (steepness / 100.0)) + #define fann_activation_switch(activation_function, value, result) \ switch(activation_function) \ { \ @@ -139,6 +147,12 @@ switch(activation_function) \ case FANN_GAUSSIAN_STEPWISE: \ result = 0; \ break; \ + case FANN_RELU: \ + result = (fann_type)fann_relu_real(value); \ + break; \ + case FANN_LEAKY_RELU: \ + result = (fann_type)fann_leaky_relu_real(value); \ + break; \ } #endif diff --git a/src/include/fann_data.h b/src/include/fann_data.h index 99f42c76..fdea61ef 100644 --- a/src/include/fann_data.h +++ b/src/include/fann_data.h @@ -226,7 +226,9 @@ enum fann_activationfunc_enum FANN_SIN_SYMMETRIC, FANN_COS_SYMMETRIC, FANN_SIN, - FANN_COS + FANN_COS, + FANN_RELU, + FANN_LEAKY_RELU }; /* Constant: FANN_ACTIVATIONFUNC_NAMES @@ -258,7 +260,9 @@ static char const *const FANN_ACTIVATIONFUNC_NAMES[] = { "FANN_SIN_SYMMETRIC", "FANN_COS_SYMMETRIC", "FANN_SIN", - "FANN_COS" + "FANN_COS", + "FANN_RELU", + "FANN_LEAKY_RELU" }; /* Enum: fann_errorfunc_enum From 14f7b01905ca9066503d94fcd8351e0a1c5e21c4 Mon Sep 17 00:00:00 2001 From: hatsunearu Date: Sat, 20 Aug 2016 13:02:09 -0400 Subject: [PATCH 2/2] fix indentation issues --- src/fann.c | 12 ++++++------ src/fann_train.c | 14 +++++++------- src/include/fann_activation.h | 16 ++++++++-------- src/include/fann_data.h | 8 ++++---- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/src/fann.c b/src/fann.c index 6aedf93f..d2e5f181 100644 --- a/src/fann.c +++ b/src/fann.c @@ -769,12 +769,12 @@ FANN_EXTERNAL fann_type *FANN_API fann_run(struct fann * ann, fann_type * input) case FANN_LINEAR_PIECE_SYMMETRIC: neuron_it->value = (fann_type)((neuron_sum < -multiplier) ? -multiplier : (neuron_sum > multiplier) ? multiplier : neuron_sum); break; - case FANN_RELU: - neuron_it->value = (fann_type)(neuron_sum > 0 ? neuron_sum : 0); - break; - case FANN_LEAKY_RELU: - neuron_it->value = (fann_type)(neuron_sum > 0 ? neuron_sum : neuron_sum / 100); - break; + case FANN_RELU: + neuron_it->value = (fann_type)(neuron_sum > 0 ? neuron_sum : 0); + break; + case FANN_LEAKY_RELU: + neuron_it->value = (fann_type)(neuron_sum > 0 ? neuron_sum : neuron_sum / 100); + break; case FANN_ELLIOT: case FANN_ELLIOT_SYMMETRIC: case FANN_GAUSSIAN: diff --git a/src/fann_train.c b/src/fann_train.c index 4fd2bdef..856bcc6c 100644 --- a/src/fann_train.c +++ b/src/fann_train.c @@ -72,10 +72,10 @@ fann_type fann_activation_derived(unsigned int activation_function, return (fann_type) fann_cos_derive(steepness, sum); case FANN_THRESHOLD: fann_error(NULL, FANN_E_CANT_TRAIN_ACTIVATION); - case FANN_RELU: - return (fann_type) fann_relu_derive(steepness, sum); - case FANN_LEAKY_RELU: - return (fann_type) fann_leaky_relu_derive(steepness, sum); + case FANN_RELU: + return (fann_type) fann_relu_derive(steepness, sum); + case FANN_LEAKY_RELU: + return (fann_type) fann_leaky_relu_derive(steepness, sum); } return 0; } @@ -136,9 +136,9 @@ fann_type fann_update_MSE(struct fann *ann, struct fann_neuron* neuron, fann_typ case FANN_ELLIOT: case FANN_LINEAR_PIECE: case FANN_SIN: - case FANN_COS: - case FANN_RELU: - case FANN_LEAKY_RELU: + case FANN_COS: + case FANN_RELU: + case FANN_LEAKY_RELU: break; } diff --git a/src/include/fann_activation.h b/src/include/fann_activation.h index d89a1cb1..236d7370 100644 --- a/src/include/fann_activation.h +++ b/src/include/fann_activation.h @@ -145,14 +145,14 @@ switch(activation_function) \ result = (fann_type)fann_cos_real(value); \ break; \ case FANN_GAUSSIAN_STEPWISE: \ - result = 0; \ - break; \ - case FANN_RELU: \ - result = (fann_type)fann_relu_real(value); \ - break; \ - case FANN_LEAKY_RELU: \ - result = (fann_type)fann_leaky_relu_real(value); \ - break; \ + result = 0; \ + break; \ + case FANN_RELU: \ + result = (fann_type)fann_relu_real(value); \ + break; \ + case FANN_LEAKY_RELU: \ + result = (fann_type)fann_leaky_relu_real(value); \ + break; \ } #endif diff --git a/src/include/fann_data.h b/src/include/fann_data.h index fdea61ef..a7318284 100644 --- a/src/include/fann_data.h +++ b/src/include/fann_data.h @@ -227,8 +227,8 @@ enum fann_activationfunc_enum FANN_COS_SYMMETRIC, FANN_SIN, FANN_COS, - FANN_RELU, - FANN_LEAKY_RELU + FANN_RELU, + FANN_LEAKY_RELU }; /* Constant: FANN_ACTIVATIONFUNC_NAMES @@ -261,8 +261,8 @@ static char const *const FANN_ACTIVATIONFUNC_NAMES[] = { "FANN_COS_SYMMETRIC", "FANN_SIN", "FANN_COS", - "FANN_RELU", - "FANN_LEAKY_RELU" + "FANN_RELU", + "FANN_LEAKY_RELU" }; /* Enum: fann_errorfunc_enum