Dense.hpp

#pragma once
 
#include <cereal/access.hpp>
#include <cereal/cereal.hpp>
#include <cereal/types/base_class.hpp>
#include <cereal/types/polymorphic.hpp>
 
#include "Layer.hpp"
 
namespace NeuralNet {
class Dense : public Layer {
 public:
  Dense(int nNeurons, ACTIVATION activation = ACTIVATION::SIGMOID,
        WEIGHT_INIT weightInit = WEIGHT_INIT::RANDOM, int bias = 0) {
    type = LayerType::DENSE;
    this->bias = bias;
    this->nNeurons = nNeurons;
    this->weightInit = weightInit;
    this->activation = activation;
    this->setActivation(activation);
  };

  Eigen::MatrixXd getWeights() const { return weights; };

  Eigen::MatrixXd getBiases() const { return biases; };

  Eigen::MatrixXd getOutputs() const { return outputs; };

  void printWeights() {
    std::cout << this->weights << "\n";
    return;
  };

  std::string getSlug() const override {
    return slug + std::to_string(nNeurons) + activationSlug;
  }

  virtual Eigen::MatrixXd feedInputs(Eigen::MatrixXd inputs,
                                     bool training = false) override {
    // Dense layer positioned as input layer
    if (weights.rows() == 0 && weights.cols() == 0) {
      setOutputs(inputs);
      return inputs;
    }
 
    if (inputs.cols() != weights.rows()) {
      Eigen::MatrixXd transposedMat = inputs.transpose();
      inputs = transposedMat;
    }
 
    assert(inputs.cols() == weights.rows());
    return this->computeOutputs(inputs, training);
  };
 
  ~Dense(){};
 
 private:
  // non-public serialization
  friend class cereal::access;
  friend class Network;
 
  double bias;
  std::string slug = "dns";
  std::string activationSlug = "";
  Eigen::MatrixXd biases;
  WEIGHT_INIT weightInit;
  Eigen::MatrixXd weights;
  Eigen::MatrixXd cachedWeights;
  Eigen::MatrixXd cachedBiases;
  ACTIVATION activation;
  Eigen::MatrixXd (*activate)(const Eigen::MatrixXd &);
  Eigen::MatrixXd (*diff)(const Eigen::MatrixXd &);
 
  template <class Archive>
  void save(Archive &ar) const {
    ar(cereal::base_class<Layer>(this), nNeurons, biases, weights, activation);
  }
 
  template <class Archive>
  void load(Archive &ar) {
    ar(cereal::base_class<Layer>(this), nNeurons, biases, weights, activation);
    setActivation(activation);
  }

  void init(int numRows) override {
    // First and foremost init the biases and the outputs
    double mean = 0, stddev = 0;
    this->weights = Eigen::MatrixXd::Zero(numRows, nNeurons);
 
    // This is going to be used for testing
    if (this->weightInit == WEIGHT_INIT::CONSTANT) {
      this->weights = Eigen::MatrixXd::Constant(numRows, nNeurons, 1);
      return;
    }
 
    // calculate mean and stddev based on init algo
    switch (this->weightInit) {
      case WEIGHT_INIT::GLOROT:
        // sqrt(fan_avg)
        stddev = sqrt(static_cast<double>(2) / (numRows + nNeurons));
        break;
      case WEIGHT_INIT::HE:
        // sqrt(2/fan_in)
        stddev = sqrt(2.0 / numRows);
        break;
      case WEIGHT_INIT::LECUN:
        // sqrt(1/fan_in)
        stddev = sqrt(1.0 / numRows);
        break;
      default:
        break;
    }
 
    // Init the weights
    this->weightInit == WEIGHT_INIT::RANDOM
        ? randomWeightInit(&(this->weights), -1, 1)
        : randomDistMatrixInit(&(this->weights), mean, stddev);
  }

  Eigen::MatrixXd computeOutputs(Eigen::MatrixXd inputs,
                                 bool training) override {
    // Initialize the biases based on the input's size
    if (biases.rows() == 0 && biases.cols() == 0) {
      biases = Eigen::MatrixXd::Constant(1, nNeurons, bias);
    }
 
    // Weighted sum
    Eigen::MatrixXd wSum = inputs * weights;
 
    wSum.rowwise() += biases.row(0);
 
    Eigen::MatrixXd a = activate(wSum);
 
    // Caching outputs for training
    if (training) outputs = a;
 
    return a;
  };

  void setActivation(ACTIVATION activation) {
    if (type == LayerType::FLATTEN) {
      this->activate = Activation::activate;
      this->diff = Activation::diff;
      return;
    }
 
    switch (activation) {
      case ACTIVATION::SIGMOID:
        this->activate = Sigmoid::activate;
        this->diff = Sigmoid::diff;
        this->activationSlug = Sigmoid::slug;
        break;
      case ACTIVATION::RELU:
        this->activate = Relu::activate;
        this->diff = Relu::diff;
        this->activationSlug = Relu::slug;
        break;
      case ACTIVATION::SOFTMAX:
        this->activate = Softmax::activate;
        this->diff = Softmax::diff;
        this->activationSlug = Softmax::slug;
        break;
      default:
        assert(false && "Activation not defined");
    }
 
    return;
  };
 
  Dense(){};  // Required for serialization
};
}  // namespace NeuralNet
 
CEREAL_REGISTER_TYPE(NeuralNet::Dense);
 
CEREAL_REGISTER_POLYMORPHIC_RELATION(NeuralNet::Layer, NeuralNet::Dense);