Dropout.hpp

#pragma once
 
#include <algorithm>
#include <cereal/access.hpp>
#include <cereal/archives/binary.hpp>
#include <cereal/archives/portable_binary.hpp>
#include <cereal/cereal.hpp>
#include <cereal/types/base_class.hpp>
#include <cereal/types/polymorphic.hpp>
#include <iterator>
#include <random>
#include <tuple>
 
#include "Layer.hpp"
 
namespace NeuralNet {
class Dropout : public Layer {
 public:
  float rate, scaleRate;
  unsigned int seed;
  Eigen::MatrixXd mask;
 
  Dropout(float rate, unsigned int seed = 0) : rate(rate), seed(seed) {
    assert(rate < 1 && rate > 0);
    this->type = LayerType::DROPOUT;
    this->trainingOnly = true;  // Training only layer
    this->scaleRate = 1.0 / (1.0 - rate);
  };
 
  std::string getSlug() const override {
    return slug + removeTrailingZeros(std::to_string(rate));
  }
 
  Eigen::MatrixXd feedInputs(Eigen::MatrixXd inputs,
                             bool training = false) override {
    return this->computeOutputs(inputs, training);
  };
 
 private:
  std::vector<std::tuple<int, int>> coordinates;
  std::string slug = "do";
 
  // non-public serialization
  friend class cereal::access;
 
  Dropout(){};  // Necessary for serialization
 
  template <class Archive>
  void serialize(Archive& ar) {
    ar(cereal::base_class<Layer>(this), seed, rate);
  }
 
  unsigned int getSeed() {
    if (seed != 0) return seed;
    std::random_device rd;
    return rd();
  };
 
 protected:
  void init(int numNeurons) override { this->nNeurons = numNeurons; };
 
  Eigen::MatrixXd computeOutputs(Eigen::MatrixXd inputs,
                                 bool training) override {
    int rows = inputs.rows();
    int cols = inputs.cols();
    int numCoord = rows * cols;  // Number of coordinates
    mask = Eigen::MatrixXd::Constant(rows, cols, 1);
 
    seed = getSeed();
    std::mt19937 gen(seed);
    const int num_zeros = static_cast<int>(numCoord * (1.0 - rate));
 
    std::vector<std::tuple<int, int>> randCoordinates;
 
    randCoordinates.reserve(num_zeros);
 
    if (!coordinates.size() || coordinates.size() != (numCoord)) {
      coordinates.clear();
      coordinates.reserve(numCoord);
 
      for (int i = 0; i < rows; i++) {
        for (int j = 0; j < cols; j++) {
          coordinates.emplace_back(std::make_tuple(i, j));
        }
      }
    }
 
    // Randomly select tuples from coordinates
    std::sample(coordinates.begin(), coordinates.end(),
                std::back_inserter(randCoordinates), num_zeros, gen);
 
    for (std::tuple<int, int>& coord : randCoordinates) {
      mask(std::get<0>(coord), std::get<1>(coord)) = 0;
    }
 
    Eigen::MatrixXd dO = (inputs.array() * mask.array()) * scaleRate;
 
    // Caching outputs for training
    if (training) outputs = dO;
 
    return dO;
  };
};
}  // namespace NeuralNet
 
namespace cereal {
template <class Archive>
struct specialize<Archive, NeuralNet::Dropout,
                  cereal::specialization::member_serialize> {};
}  // namespace cereal
 
CEREAL_REGISTER_TYPE(NeuralNet::Dropout);
 
CEREAL_REGISTER_POLYMORPHIC_RELATION(NeuralNet::Layer, NeuralNet::Dropout);