initial commit of pix_recNN

svn path=/trunk/externals/grh/; revision=3320

1 files changed, 636 insertions, 0 deletions

diff --git a/pix_recNN/NNet.h b/pix_recNN/NNet.h
new file mode 100755
index 0000000..349688f
--- /dev/null
+++ b/pix_recNN/NNet.h
@@ -0,0 +1,636 @@
+/////////////////////////////////////////////////////////////////////////////
+//
+// class NNet
+//
+//   this is a template for all the nets
+//   (see NeuralNet documentations for more information)
+//
+//   header file
+//
+//   Copyright (c) 2005 Georg Holzmann <grh@gmx.at>
+//
+//   
+//   This program is free software; you can redistribute it and/or
+//   modify it under the terms of the GNU General Public License
+//   as published by the Free Software Foundation; either version 2
+//   of the License, or (at your option) any later version.
+//
+/////////////////////////////////////////////////////////////////////////////
+
+
+#ifndef _INCLUDE_NEURAL_TEMPLATE_NET__
+#define _INCLUDE_NEURAL_TEMPLATE_NET__
+
+#include "NNActivation.h"
+#include "NNException.h"
+
+namespace TheBrain
+{
+
+template <class HiddNeuronType,class OutNeuronType>
+class NNet
+{
+ protected:
+
+  /* the number of output values
+   * this is automatically also the 
+   * number of output neurons !
+   */
+  int output_val_;
+
+  /* the number of hidden neurons
+   * per one output neuron
+   * (this net has one hidden layer,
+   * so this is the number of hidden
+   * neurons is hidden_val_*output_val_) 
+   */
+  int hidden_val_;
+
+  /* nr of input values per one output neuron
+   * (so the number of input values are
+   * input_val_*output_val_)
+   */
+  int input_val_;
+
+  /* the memory of the output layer
+   * if you use a recurrent neuron, this
+   * determines how much output values the 
+   * recurrent neurons can remeber
+   * these values are fed back as new input
+   */
+  int memory_out_;
+
+  /* the memory of the hidden layer
+   * if you use a recurrent neuron, this
+   * determines how much output values the 
+   * recurrent neurons can remeber
+   * these values are fed back as new input
+   */
+  int memory_hidden_;
+
+  /* these are the output neurons
+   */
+  OutNeuronType *out_neurons_;
+
+  /* these are the hidden neurons
+   */
+  HiddNeuronType *hidden_neurons_;
+
+  /* function pointer to the activation
+   * function of the output neurons
+   */
+  float (*output_act_f)(float value);
+
+  /* function pointer to the activation
+   * function of the hidden neurons
+   */
+  float (*hidden_act_f)(float value);
+
+  /* function pointer to the derivation of the
+   * activation function of the hidden neurons
+   */
+  float (*hidden_act_f_d)(float value);
+
+
+ public:
+
+  /* Constructor
+   */
+  NNet(int input_val=1, int hidden_val=1, int output_val=1, int memory_out=0,
+       int memory_hidden=1, int HIDDEN_ACT_FUNC=0, int OUT_ACT_FUNC=0);
+
+  /* Destructor
+   */
+  virtual ~NNet();
+
+
+  //-----------------------------------------------------
+
+  /* Set/Get learning rate
+   */
+  virtual void setLearningRate(float learn_rate);
+  virtual float getLearningRate() const;
+
+  /* Set/Get range
+   * (see Neuron.h)
+   */
+  virtual void setRange(float range);
+  virtual float getRange() const;
+
+  /* some more get/set methods
+   */
+  virtual void setOutputVal(int output_val)
+    throw();
+  virtual int getOutputVal() const;
+
+  virtual void setHiddenVal(int hidden_val)
+    throw();
+  virtual int getHiddenVal() const;
+
+  virtual void setInputVal(int input_val)
+    throw();
+  virtual int getInputVal() const;
+
+  virtual void setMemoryOut(int memory)
+    throw();
+  virtual int getMemoryOut() const;
+
+  virtual void setMemoryHidden(int memory)
+    throw();
+  virtual int getMemoryHidden() const;
+
+
+  //-----------------------------------------------------
+
+  /* creates the network
+   */
+  virtual void create()
+    throw(NNExcept);
+
+  /* inits the weight matrix and the bias vector of
+   * the network with random values between [min|max]
+   */
+  virtual void initRand(const int &min, const int &max)
+    throw(NNExcept);
+
+  /* calculates the output with the current Net and writes
+   * it in the array output_data
+   * ATTENTION: array input_data must be a matrix in the form:
+   *              float[output_val_][input_val_]
+   *            array output_data must be in size output_val_
+   *            (there is no checking !!!)
+   */
+  virtual void calculate(float **input_data, float *output_data);
+
+  /* this method trains the network:
+   * input_data is, as above, the input data, output_data is the 
+   * output of the current net with input_data, target_output is 
+   * the desired output data
+   * (this is the a truncated backpropagation through time
+   * algorithm to train the network)
+   * ATTENTION: array input_data must be a matrix in the form:
+   *              float[output_val_][input_val_]
+   *            array output_data must be in size output_val_
+   *            array target_output  must be in size output_val_
+   *            (there is no checking !!!)
+   */
+  virtual void trainBTT(float **input_data, float *output_data, 
+		     float *target_output);
+
+
+  //-----------------------------------------------------
+
+  /* saves the contents of the current net to file
+   */
+  virtual void save(string filename)
+    throw(NNExcept);
+
+  /* loads the parameters of the net from file
+   */
+  virtual void load(string filename)
+    throw(NNExcept);
+
+
+  //-----------------------------------------------------
+ private:
+
+  /* output of the hidden layer with activation function
+   */
+  float *hidden_a_;
+
+  /* output of the hidden layer without activation function
+   */
+  float *hidden_s_;
+
+  /* error signal of the neurons in the hidden layer
+   */
+  float *hidden_error_;
+
+  /* out signal without activation function
+   */
+  float out_s_;
+
+  /* error signal of the output layer
+   */
+  float out_error_;
+
+  /* Copy Construction is not allowed
+   */
+  NNet(const NNet<HiddNeuronType,OutNeuronType> &src)
+    { }
+
+  /* assignement operator is not allowed
+   */
+  const NNet<HiddNeuronType,OutNeuronType>& operator= 
+    (const NNet<HiddNeuronType,OutNeuronType>& src)
+    { return *this; }
+};
+
+
+//--------------------------------------------------
+/* Constructor
+ */
+template <class HiddNeuronType, class OutNeuronType>
+NNet<HiddNeuronType,OutNeuronType>
+  ::NNet(int input_val, int hidden_val, int output_val, int memory_out, 
+	 int memory_hidden, int HIDDEN_ACT_FUNC, int OUT_ACT_FUNC)
+  : out_neurons_(NULL), hidden_neurons_(NULL), hidden_a_(NULL),
+  hidden_s_(NULL), hidden_error_(NULL)
+{
+  output_val_ = (output_val<1) ? 1 : output_val;
+  hidden_val_ = (hidden_val<0) ? 0 : hidden_val;
+  input_val_ = (input_val<1) ? 1 : input_val;
+  memory_out_ = (memory_out<0) ? 0 : memory_out;
+  memory_hidden_ = (memory_hidden<0) ? 0 : memory_hidden;
+
+  // choose hidden activation function:
+  switch(HIDDEN_ACT_FUNC)
+    {
+    case SIGMOID:
+      hidden_act_f = act_sigmoid;
+      hidden_act_f_d = act_sigmoid_derive;
+      break;
+    case TANH:
+      hidden_act_f = act_tanh;
+      hidden_act_f_d = act_tanh_derive;
+      break;
+    default:
+    case LINEAR:
+      hidden_act_f = act_linear;
+      hidden_act_f_d = act_linear_derive;
+      break;
+    }
+
+  // choose out function:
+  switch(OUT_ACT_FUNC)
+    {
+    case SIGMOID:
+      output_act_f = act_sigmoid;
+      break;
+    case TANH:
+      output_act_f = act_tanh;
+      break;
+    default:
+    case LINEAR:
+      output_act_f = act_linear;
+      break;
+    }
+}
+
+//--------------------------------------------------
+/* Destructor
+ */
+template <class HiddNeuronType, class OutNeuronType>
+NNet<HiddNeuronType, OutNeuronType>::~NNet()
+{
+  if(hidden_neurons_)
+    delete[] hidden_neurons_;
+
+  if(out_neurons_)
+    delete[] out_neurons_;
+
+  if(hidden_a_)
+    delete[] hidden_a_;
+
+  if(hidden_s_)
+    delete[] hidden_s_;
+
+  if(hidden_error_)
+    delete[] hidden_error_;
+}
+
+//--------------------------------------------------
+/* creates the network
+ */
+template <class HiddNeuronType, class OutNeuronType>
+void NNet<HiddNeuronType,OutNeuronType>::create()
+  throw(NNExcept)
+{
+  // delete if they exist
+  if(out_neurons_)
+    delete[] out_neurons_;
+  if(hidden_neurons_)
+    delete[] hidden_neurons_;
+  if(hidden_a_)
+    delete[] hidden_a_;
+  if(hidden_s_)
+    delete[] hidden_s_;
+  if(hidden_error_)
+    delete[] hidden_error_;
+
+
+  out_neurons_ = new OutNeuronType[output_val_](input_val_,memory_out_);
+  hidden_neurons_ = new HiddNeuronType[hidden_val_*output_val_](input_val_,memory_hidden_);
+
+  if(!out_neurons_ || !hidden_neurons_)
+    throw NNExcept("No memory for Neurons!");
+
+  // create the temporary storage
+  hidden_a_ = new float[hidden_val_];
+  hidden_s_ = new float[hidden_val_];
+  hidden_error_ = new float[hidden_val_];
+
+  if(!hidden_a_ || !hidden_s_ || !hidden_error_)
+    throw NNExcept("No memory for Neurons!");
+
+
+  // create all the neurons
+  for(int i=0; i<output_val_; i++)
+    out_neurons_[i].create();
+  for(int i=0; i<hidden_val_*output_val_; i++)
+    hidden_neurons_[i].create();
+}
+
+//--------------------------------------------------
+/* inits the weight matrix and the bias vector of
+ * the network with random values between [min|max]
+ */
+template <class HiddNeuronType, class OutNeuronType>
+void NNet<HiddNeuronType,OutNeuronType>::initRand(const int &min, const int &max)
+    throw(NNExcept)
+{
+  if(!out_neurons_)
+    throw NNExcept("You must first create the Net!");
+
+  // init all the neurons
+  for(int i=0; i<output_val_; i++)
+    out_neurons_[i].initRand(min,max);
+  for(int i=0; i<hidden_val_*output_val_; i++)
+    hidden_neurons_[i].initRand(min,max);
+}
+
+//--------------------------------------------------
+/* calculates the output with the current Net and writes
+ * it in the array output_data
+ * ATTENTION: array input_data must be a matrix in the form:
+ *              float[output_val_][input_val_]
+ *            array output_data must be in size output_val_
+ *            (there is no checking !!!)
+ */
+template <class HiddNeuronType, class OutNeuronType>
+void NNet<HiddNeuronType,OutNeuronType>::calculate(float **input_data, float *output_data)
+{
+  for(int i=0; i<output_val_; i++)
+    {
+
+      // 1.: calculation of the hidden layer
+      for(int j=0; j<hidden_val_; j++)
+	{
+	  hidden_a_[j] = hidden_act_f( 
+	    hidden_neurons_[i*hidden_val_+j].calculate(input_data[i]) );
+	}
+
+      // 2.: calculation of the output layer
+      *output_data++ = output_act_f( out_neurons_[i].calculate(hidden_a_) );
+    }
+}
+
+//--------------------------------------------------
+/* this method trains the network:
+ * input_data is, as above, the input data, output_data is the 
+ * output of the current net with input_data, target_output is 
+ * the desired output data
+ * (this is the a truncated backpropagation through time
+ * algorithm to train the network)
+ * ATTENTION: array input_data must be a matrix in the form:
+ *              float[output_val_][input_val_]
+ *            array output_data must be in size output_val_
+ *            array target_output  must be in size output_val_
+ *            (there is no checking !!!)
+ */
+template <class HiddNeuronType, class OutNeuronType>
+void NNet<HiddNeuronType,OutNeuronType>::trainBTT(float **input_data, float *output_data, 
+			 float *target_output)
+{
+  post("train");
+
+  for(int i=0; i<output_val_; i++)
+    {
+
+
+      //---------------------------------------------------------
+      // 1. Forward - Pass:
+      // 
+      // the output of the hidden and the output-layer
+      // are calculated and saved (before and after
+      // the activation function)
+
+      // calculation of the hidden layer
+      for(int j=0; j<hidden_val_; j++)
+	{
+	  hidden_s_[j] = hidden_neurons_[i*hidden_val_+j].calculate(input_data[i]);
+	  hidden_a_[j] = hidden_act_f(hidden_s_[j]);
+	}
+
+      // calculation of the output layer
+      out_s_ = out_neurons_[i].calculate(hidden_a_);
+      output_data[i] = output_act_f(out_s_);
+ 
+
+      //---------------------------------------------------------
+      // 2. Backward - Pass:
+      // 
+      // calculation of the error signals
+      // (they are also stored)
+
+      // output layer
+      out_error_ = output_data[i] - target_output[i];
+      
+      // hidden layer:
+      for(int j=0; j<hidden_val_; j++)
+	{
+	  hidden_error_[j] = hidden_act_f_d( hidden_s_[j]+0.1 ) *
+	    ( out_error_ * out_neurons_[i].getIW(j) );
+	}
+
+
+      //---------------------------------------------------------
+      // 3. Modification of the weights:
+
+      for(int j=0; j<hidden_val_; j++)
+      {
+	// output layer:
+	out_neurons_[i].setIW(j, 
+	       out_neurons_[i].getIW(j) - 
+	       getLearningRate() * out_error_ 
+	       * hidden_a_[j] );
+
+	// hidden layer:
+	for(int k=0; k<input_val_; k++)
+	{
+	  hidden_neurons_[i*hidden_val_+j].setIW(k,
+	       hidden_neurons_[i*hidden_val_+j].getIW(k) -
+	       getLearningRate() * hidden_error_[j]
+	       * input_data[i][k]/hidden_neurons_[0].getRange() );
+	}
+
+
+	// recurrent part of the hidden layer:
+	float delta = getLearningRate() * hidden_error_[j] * hidden_a_[j];
+	for(int k=0; k<memory_hidden_; k++)
+	  {
+	    hidden_neurons_[i*hidden_val_+j].setLW(k,
+	       hidden_neurons_[i*hidden_val_+j].getLW(k) - delta);
+	  }
+      }
+
+      // recurrent part of the output layer:
+      float delta = getLearningRate() * out_error_ * output_data[i];
+      for(int j=0; j<memory_out_; j++)
+	{
+	  out_neurons_[i].setLW(j,
+	     out_neurons_[i].getLW(j) - delta);
+	}
+    
+
+    }
+}
+
+//--------------------------------------------------
+/* saves the contents of the current net to file
+ */
+template <class HiddNeuronType, class OutNeuronType>
+void NNet<HiddNeuronType,OutNeuronType>::save(string filename)
+  throw(NNExcept)
+{
+
+}
+
+//--------------------------------------------------
+  /* loads the parameters of the net from file
+   */
+template <class HiddNeuronType, class OutNeuronType>
+void NNet<HiddNeuronType,OutNeuronType>::load(string filename)
+  throw(NNExcept)
+{
+
+}
+
+//-----------------------------------------------------
+/* Set/Get learning rate
+ * (see Neuron.h)
+ */
+template <class HiddNeuronType, class OutNeuronType>
+void NNet<HiddNeuronType,OutNeuronType>::setLearningRate(float learn_rate)
+{
+  learn_rate = (learn_rate<0) ? 0 : learn_rate;
+
+  for(int i=0; i<output_val_; i++)
+    out_neurons_[i].setLearningRate(learn_rate);
+  for(int i=0; i<hidden_val_*output_val_; i++)
+    hidden_neurons_[i].setLearningRate(learn_rate);
+}
+template <class HiddNeuronType, class OutNeuronType>
+float NNet<HiddNeuronType, OutNeuronType>::getLearningRate() const
+{
+  return out_neurons_[0].getLearningRate();
+}
+
+//-----------------------------------------------------
+/* Set/Get range
+ * (see Neuron.h)
+ */
+template <class HiddNeuronType, class OutNeuronType>
+void NNet<HiddNeuronType,OutNeuronType>::setRange(float range)
+{
+  for(int i=0; i<output_val_; i++)
+    out_neurons_[i].setRange(1);
+
+  for(int i=0; i<hidden_val_*output_val_; i++)
+    hidden_neurons_[i].setRange(range);
+}
+template <class HiddNeuronType, class OutNeuronType>
+float NNet<HiddNeuronType, OutNeuronType>::getRange() const
+{
+  return hidden_neurons_[0].getRange();
+}
+
+//-----------------------------------------------------
+/* get/set output_val_
+ */
+template <class HiddNeuronType, class OutNeuronType>
+void NNet<HiddNeuronType,OutNeuronType>::setOutputVal(int output_val)
+  throw()
+{
+  output_val_ = (output_val<1) ? 1 : output_val;
+
+  create();
+}
+template <class HiddNeuronType, class OutNeuronType>
+int NNet<HiddNeuronType,OutNeuronType>::getOutputVal() const
+{
+  return output_val_;
+}
+
+//-----------------------------------------------------
+/* get/set hidden_val_
+ */
+template <class HiddNeuronType, class OutNeuronType>
+void NNet<HiddNeuronType,OutNeuronType>::setHiddenVal(int hidden_val)
+  throw()
+{
+  hidden_val_ = (hidden_val<1) ? 1 : hidden_val;
+
+  create();
+}
+template <class HiddNeuronType, class OutNeuronType>
+int NNet<HiddNeuronType,OutNeuronType>::getHiddenVal() const
+{
+  return hidden_val_;
+}
+
+//-----------------------------------------------------
+/* get/set input_val_
+ */
+template <class HiddNeuronType, class OutNeuronType>
+void NNet<HiddNeuronType,OutNeuronType>::setInputVal(int input_val)
+    throw()
+{
+  input_val_ = (input_val<1) ? 1 : input_val;
+
+  create();
+}
+template <class HiddNeuronType, class OutNeuronType>
+int NNet<HiddNeuronType,OutNeuronType>::getInputVal() const
+{
+  return input_val_;
+}
+
+//-----------------------------------------------------
+/* get/set memory of the output layer
+ */
+template <class HiddNeuronType, class OutNeuronType>
+void NNet<HiddNeuronType,OutNeuronType>::setMemoryOut(int memory)
+    throw()
+{
+  memory_out_ = (memory<0) ? 0 : memory;
+
+  create();
+}
+template <class HiddNeuronType, class OutNeuronType>
+int NNet<HiddNeuronType,OutNeuronType>::getMemoryOut() const
+{
+  return memory_out_;
+}
+
+//-----------------------------------------------------
+/* get/set memory of the hidden layer
+ */
+template <class HiddNeuronType, class OutNeuronType>
+void NNet<HiddNeuronType,OutNeuronType>::setMemoryHidden(int memory)
+    throw()
+{
+  memory_hidden_ = (memory<0) ? 0 : memory;
+
+  create();
+}
+template <class HiddNeuronType, class OutNeuronType>
+int NNet<HiddNeuronType,OutNeuronType>::getMemoryHidden() const
+{
+  return memory_hidden_;
+}
+
+
+} // end of namespace
+
+#endif //_INCLUDE_LIN_NEURAL_NET__