#include <stdio.h>
#include <stdlib.h>
#include <math.h>
#include <string.h>
#include <time.h>
#include <gsl/gsl_rng.h>
#include <gsl/gsl_randist.h>
#include <gsl/gsl_cblas.h>
#include <gsl/gsl_interp.h>

#include "allvars.h"
#include "proto.h"

#define SNR (120.0)
int n_con_sim=301, n_line_sim=301;
double sim_error=10.0*1.0/SNR, sim_rel_error=1.0/SNR;

double *PQmat, *PSmat, *Peigens, *Peigens_mat, *Peigens_vecs;

gsl_rng_type const * gsl_T_sim;
gsl_rng * gsl_r_sim;
gsl_interp_accel *gsl_acc, *gsl_acc_error;
gsl_interp  *gsl_linear, *gsl_linear_error;

void simulate_con(double sigma, double tau, double alpha, double ave_con)
{
  FILE *fscon_out;

  double *Py, *Prandvec;
  int i, info;

  Py = array_malloc(n_con_sim);
  Prandvec = array_malloc(n_con_sim);

  simulate_con_init();

  set_covar_Simmat(sigma, tau, alpha);
  memcpy(PQmat, PSmat, n_con_sim*n_con_sim*sizeof(double));

/*  eigen_sym_mat(PQmat, n_con_sim, Peigens, &info);
  memcpy(Peigens_vecs, PQmat, n_con_sim*n_con_sim*sizeof(double));

  for(i=0; i<n_con_sim; i++)
  {
    for(j=0; j<n_con_sim; j++)
      Peigens_mat[i*n_con_sim+j] = 0.0;
    Peigens_mat[i*n_con_sim+i] = sqrt(Peigens[i]);
  }*/

  for(i=0;i<n_con_sim; i++)
  {
    Prandvec[i]  = gsl_ran_gaussian(gsl_r_sim, 1.0);
  }

  Chol_decomp_U(PQmat, n_con_sim, &info);
  multiply_matvec(PQmat, Prandvec, n_con_sim, Py);

//   multiply_matvec(Peigens_mat, Prandvec, n_con_sim, Pybuf);
//   multiply_matvec(Peigens_vecs, Pybuf, n_con_sim, Py);

  gsl_rng_set(gsl_r_sim, time(NULL));
  for(i=0;i<n_con_sim;i++)
  {
    Fcon[i] = Py[i] + ave_con + sim_error * gsl_ran_gaussian(gsl_r_sim, 1.0);
    Fcerrs[i] = sim_error;
  }

  fscon_out = fopen("data/sim_con.txt", "w");

  for(i=0;i<n_con_sim;i++)
  {
    if(Tcon[i]>150.0)
      fprintf(fscon_out, "%f\t%f\t%f\n", Tcon[i], Fcon[i], Fcerrs[i]);
  }

  gsl_interp_init(gsl_linear, Tcon, Fcon, n_con_sim);
  gsl_interp_init(gsl_linear_error, Tcon, Fcerrs, n_con_sim);

  fclose(fscon_out);
}

void simulate_con_init()
{
  int i;
  double Tcon_min, Tcon_max, Tline_min, Tline_max, dTs;

  PSmat = array_malloc(n_con_sim*n_con_sim);
  PQmat = array_malloc(n_con_sim*n_con_sim);
  Peigens = array_malloc(n_con_sim);
  Peigens_vecs = array_malloc(n_con_sim*n_con_sim);
  Peigens_mat = array_malloc(n_con_sim*n_con_sim);

  Tcon = array_malloc(n_con_sim);
  Fcon = array_malloc(n_con_sim);
  Fcerrs = array_malloc(n_con_sim);

  Tcon_min = 0.0;
  Tcon_max = 300.0;
  dTs = (Tcon_max - Tcon_min) / (n_con_sim-1.0);
  for(i=0; i<n_con_sim; i++)
  {
    Tcon[i] = dTs * i + Tcon_min;
  }

  Tline = array_malloc(n_line_sim);
  Fline = array_malloc(n_line_sim);
  Flerrs = array_malloc(n_line_sim);

  Tline_min = 0.0;
  Tline_max = 300.0;
  dTs = (Tline_max - Tline_min) / (n_line_sim-1.0);
  for(i=0; i<n_line_sim; i++)
  {
    Tline[i] = dTs * i + Tline_min;
  }

  gsl_acc = gsl_interp_accel_alloc();
  gsl_acc_error = gsl_interp_accel_alloc();

  gsl_linear = gsl_interp_alloc(gsl_interp_linear, n_con_sim);
  gsl_linear_error = gsl_interp_alloc(gsl_interp_linear, n_con_sim);

  gsl_T_sim = gsl_rng_default;
  gsl_r_sim = gsl_rng_alloc (gsl_T_sim);
  //gsl_rng_set(gsl_r_sim, 12);
  gsl_rng_set(gsl_r_sim, time(NULL));

  return;
}

void set_covar_Simmat(double sigma, double tau, double alpha)
{
  double t1, t2, nerr;
  int i, j;

  for(i=0; i<n_con_sim; i++)
  {
    t1 = Tcon[i];
    for(j=0; j<=i; j++)
    {
      t2 = Tcon[j];
      PSmat[i*n_con_sim+j] = sigma*sigma* exp (- pow (fabs(t1-t2) / tau, alpha));
      PSmat[j*n_con_sim+i] = PSmat[i*n_con_sim+j];
    }
    nerr = 0.0; //sim_error;
    PSmat[i*n_con_sim+i] += nerr * nerr;
  }
  return;
}

void simulate_line()
{
  int i, j;
  double flux, err, tline, tcon, taup, norm, fcon, fcon_err;
  double dtau, sig;
  int type=1;

  dtau = 30.0/(ntau-1.0);
  for(i=0; i<ntau; i++)
  {
    TF_tau[i] =  dtau * i;
    TF[i] = 0.0;
  }
// top-hat transfer function
  FILE *ftf;
  ftf=fopen("data/transfer_sim.txt", "w");
  norm = 0.0;
  for(i=0; i<ntau; i++)
  {
    if(type==0)  // two top-hats
    {
      if(TF_tau[i]<=5.0)
        TF[i] = 0.0;
      else if(TF_tau[i]<10.0)
        TF[i] = 1.0;
      else if(TF_tau[i]<15.0)
        TF[i] = 0.0;
      else if(TF_tau[i]<20.0)
        TF[i] = 2.0;
      else
        TF[i] = 0.0;

      norm += TF[i] * dtau;
    }
    if(type==1) // two Gaussians
    {
      sig = 2.5;
      TF[i] = exp(-0.5*pow(TF_tau[i]-10.0, 2.0)/sig/sig) + exp(-0.5*pow(TF_tau[i]-18.0, 2.0)/sig/sig);
      norm += TF[i] * dtau;
    }
  }
  for(i=0; i<ntau; i++)
  {
    TF[i] /= norm;
    fprintf(ftf, "%f %f\n", TF_tau[i], TF[i]);
  }
  fclose(ftf);

  FILE *fp;
  fp = fopen("data/sim_line.txt", "w");

  for(i=0; i<n_line_sim; i++)
  {
    flux = 0.0;
    err = 0.0;
    tline = Tline[i]; //4540.0 + 70.0/(nline_data-1.0) * i;
    for(j=0; j<ntau; j++)
    {
      taup = TF_tau[j];
      tcon = tline - taup;
      if(tcon >=Tcon[0] && tcon <= Tcon[n_con_sim-1])
      {
        fcon = gsl_interp_eval(gsl_linear, Tcon, Fcon, tcon, gsl_acc);
        fcon_err = gsl_interp_eval(gsl_linear_error, Tcon, Fcerrs, tcon, gsl_acc_error);
        flux += fcon * TF[j];
        err += pow(fcon_err *TF[j] * dtau, 2);
      }
    }
    flux *= dtau;
    err = sqrt(err);
    Fline[i] = flux;
    Flerrs[i] = sim_error;
  }

  gsl_rng_set(gsl_r_sim, time(NULL)); // reset the seed of the random generator
  for(i=0; i<n_line_sim; i++)
  {
    if(Tline[i]>170.0)
      fprintf(fp, "%f %f %f\n", Tline[i], Fline[i] + Flerrs[i]*gsl_ran_gaussian(gsl_r_sim, 1.0), Flerrs[i]);
  }
  fclose(fp);
}


/*
 * Module that implements in c a Bayesian network trained with probabilistic
 * backpropagation (PBP).
 *
 * Author: Jose Miguel Hernandez Lobato
 * Date: 10 March 2015
 *
 */

#include <stdio.h>
#include <stdlib.h>
#include <math.h>
#include <malloc.h>
#include <cblas.h>

#include "network.h"
#include "pnorm.h"

/* Prototipes of private functions */

void refine_prior(NETWORK *network);
void do_ADF_update(NETWORK * network, double *x, double y);
double randn(double mu, double sigma);

/**
 * Generates a sample from a Gaussian distribution
 * @param mu    Mean of the Gaussian.
 * @param sigma Standard deviation of the Gaussian
 *
 */

double randn(double mu, double sigma) {

  double U1, U2, W, mult;
  static double X1, X2;
  static int call = 0;

  if (call == 1)
    {
      call = !call;
      return (mu + sigma * (double) X2);
    }

  do
    {
      U1 = -1 + ((double) rand () / RAND_MAX) * 2;
      U2 = -1 + ((double) rand () / RAND_MAX) * 2;
      W = pow (U1, 2) + pow (U2, 2);
    }
  while (W >= 1 || W == 0);

  mult = sqrt ((-2 * log (W)) / W);
  X1 = U1 * mult;
  X2 = U2 * mult;

  call = !call;

  return (mu + sigma * (double) X1);
}

/**
 * Constructor for a network.
 *
 * @param size_hidden_layers    The size of each hidden layer.
 * @param n_hidden_layers       The number of hidden layers.
 * @param d                     The input dimensionality.
 * @param random_noise          Random noise for the initialization of the
 *                              posterior means.
 *
 */

NETWORK *init_network(int *size_hidden_layers, int n_hidden_layers, int d,double *random_noise) {

    NETWORK *ret;
    int n, i, j, k;

    /* We save memory for the structure */

    ret = malloc(sizeof(NETWORK));

    /* We initialize the number of hidden layers */

    ret->n_hidden_layers = n_hidden_layers;

    /* We refine the prior on the noise variance and on the prior variance */

    ret->a_noise = 2.0 * 3;
    ret->b_noise = 2.0 * 3;
    ret->a_prior = 2.0 * 3;
    ret->b_prior = 2.0 * 3;

    /* We save memory for the array with the number of neurons per layer,
     * the activation type, the scaling of the activation per layer,
       and he starting position for the different arrays */

    ret->neurons_per_layer = malloc(sizeof(int) * (n_hidden_layers + 3));
    ret->linear = malloc(sizeof(int) * (n_hidden_layers + 3));
    ret->scaling_a = malloc(sizeof(double) * (n_hidden_layers + 3));
    ret->start_w = malloc(sizeof(int) * (n_hidden_layers + 3));
    ret->start_z = malloc(sizeof(int) * (n_hidden_layers + 3));
    ret->start_a = malloc(sizeof(int) * (n_hidden_layers + 3));

    /* We initialize the number of neurons per layer, the activation type and
     * compute the size of the auxiliary variables */

    /* The input layer */

    ret->start_w[ 0 ] = -1;
    ret->start_z[ 0 ] = 0;
    ret->start_a[ 0 ] = 0;

    ret->neurons_per_layer[ 0 ] = d;
    ret->linear[ 0 ] = 0;
    ret->scaling_a[ 0 ] = 1.0;

    ret->size_w = 0;
    ret->size_a = d;
    ret->size_z = d + 1;

    /* The hidden layers */

    for (i = 1 ; i < n_hidden_layers + 1 ; i ++) {
        ret->start_w[ i ] = ret->size_w;
        ret->start_z[ i ] = ret->size_z;
        ret->start_a[ i ] = ret->size_a;

        ret->neurons_per_layer[ i ] = size_hidden_layers[ i - 1 ];
        ret->linear[ i ] = 0;
        ret->scaling_a[ i ] = 1.0 / (ret->neurons_per_layer[ i - 1 ] + 1);

        ret->size_w += (ret->neurons_per_layer[ i - 1 ] + 1) * ret->neurons_per_layer[ i ];
        ret->size_a += ret->neurons_per_layer[ i ];
        ret->size_z += ret->neurons_per_layer[ i ] + 1;
    }

    /* The output layer */

    ret->start_w[ i ] = ret->size_w;
    ret->start_z[ i ] = ret->size_z;
    ret->start_a[ i ] = ret->size_a;

    ret->neurons_per_layer[ i ] = 1;
    ret->linear[ i ] = 1;
    ret->scaling_a[ i ] = 1.0 / (ret->neurons_per_layer[ i - 1 ] + 1);

    ret->size_w += (ret->neurons_per_layer[ i - 1 ] + 1) * ret->neurons_per_layer[ i ];
    ret->size_a += ret->neurons_per_layer[ i ];
    ret->size_z += ret->neurons_per_layer[ i ] + 1;

    /* The fake output layer */

    i++;
    ret->start_w[ i ] = ret->size_w;
    ret->start_z[ i ] = ret->size_z;
    ret->start_a[ i ] = ret->size_a;

    ret->neurons_per_layer[ i ] = 1;
    ret->linear[ i ] = 1;
    ret->scaling_a[ i ] = 1.0;

    ret->size_w += (ret->neurons_per_layer[ i - 1 ] + 1) * ret->neurons_per_layer[ i ];
    ret->size_a += ret->neurons_per_layer[ i ];
    ret->size_z += ret->neurons_per_layer[ i ] + 1;

    /* We save memory for the auxiliary variables */

    ret->sample_w = (double *) malloc(sizeof(double) * ret->size_w);
    ret->m_w = (double *) malloc(sizeof(double) * ret->size_w);
    ret->m_w_new = (double *) malloc(sizeof(double) * ret->size_w);
    ret->m_w_squared = (double *)  malloc(sizeof(double) * ret->size_w);
    ret->v_w = (double *) malloc(sizeof(double) * ret->size_w);
    ret->v_w_new = (double *) malloc(sizeof(double) * ret->size_w);
    ret->m_z = (double *) malloc(sizeof(double) * ret->size_z);
    ret->m_z_scaled = (double *) malloc(sizeof(double) * ret->size_z);
    ret->m_z_squared = (double *) malloc(sizeof(double) * ret->size_z);
    ret->v_z = (double *) malloc(sizeof(double) * ret->size_z);
    ret->v_z_scaled = (double *) malloc(sizeof(double) * ret->size_z);
    ret->m_a = (double *) malloc(sizeof(double) * ret->size_a);
    ret->v_a = (double *) malloc(sizeof(double) * ret->size_a);
    ret->alpha = (double *) malloc(sizeof(double) * ret->size_a);
    ret->gamma = (double *) malloc(sizeof(double) * ret->size_a);
    ret->delta_m = (double *) malloc(sizeof(double) * ret->size_z);
    ret->delta_v = (double *) malloc(sizeof(double) * ret->size_z);

    ret->dm_z_d_m_a = (double *) malloc(sizeof(double) * ret->size_z);
    ret->dv_z_d_m_a = (double *) malloc(sizeof(double) * ret->size_z);
    ret->dm_z_d_v_a = (double *) malloc(sizeof(double) * ret->size_z);
    ret->dv_z_d_v_a = (double *) malloc(sizeof(double) * ret->size_z);
    ret->dm_a_d_m_a = (double *) malloc(sizeof(double) * ret->size_w);
    ret->dv_a_d_m_a = (double *) malloc(sizeof(double) * ret->size_w);
    ret->dm_a_d_v_a = (double *) malloc(sizeof(double) * ret->size_w);
    ret->dv_a_d_v_a = (double *) malloc(sizeof(double) * ret->size_w);
    ret->dm_a_d_m_z = (double *) malloc(sizeof(double) * ret->size_w);
    ret->dv_a_d_m_z = (double *) malloc(sizeof(double) * ret->size_w);
    ret->dm_a_d_v_z = (double *) malloc(sizeof(double) * ret->size_w);
    ret->dv_a_d_v_z = (double *) malloc(sizeof(double) * ret->size_w);
    ret->dm_a_d_m_w = (double *) malloc(sizeof(double) * ret->size_w);
    ret->dv_a_d_m_w = (double *) malloc(sizeof(double) * ret->size_w);
    ret->dm_a_d_v_w = (double *) malloc(sizeof(double) * ret->size_w);
    ret->dv_a_d_v_w = (double *) malloc(sizeof(double) * ret->size_w);
    ret->grad_m_w = (double *) malloc(sizeof(double) * ret->size_w);
    ret->grad_v_w = (double *) malloc(sizeof(double) * ret->size_w);

    ret->m_w_hat_nat = (double *) malloc(sizeof(double) * ret->size_w);
    ret->v_w_hat_nat = (double *) malloc(sizeof(double) * ret->size_w);
    ret->a_w_hat_nat = (double *) malloc(sizeof(double) * ret->size_w);
    ret->b_w_hat_nat = (double *) malloc(sizeof(double) * ret->size_w);

    ret->m_w_old = (double *) malloc(sizeof(double) * ret->size_w);
    ret->v_w_old = (double *) malloc(sizeof(double) * ret->size_w);

    /* We initialize the posterior approximation */

    k = 0;
    for (i = 1 ; i < n_hidden_layers + 3 ; i ++) {
        n = (ret->neurons_per_layer[ i - 1 ] + 1) * ret->neurons_per_layer[ i ];
        for (j = 0 ; j < n ; j++) {
            ret->m_w[ k ] = 1.0 / sqrt(ret->neurons_per_layer[ i - 1 ] + 1) * random_noise[ k ];
            ret->v_w[ k ] = ret->b_prior / (ret->a_prior - 1);
            ret->m_w_hat_nat[ k ] = 0;
            ret->v_w_hat_nat[ k ] = (ret->a_prior - 1) / ret->b_prior;
            ret->a_w_hat_nat[ k ] = 0;
            ret->b_w_hat_nat[ k ] = 0;
            k++;
        }
    }

    /* We initialize the weights of the fake ouput layer */

    ret->m_w[ ret->start_w[ n_hidden_layers + 2 ] ] = 1.0;
    ret->m_w[ ret->start_w[ n_hidden_layers + 2 ] + 1 ] = 0.0;
    ret->v_w[ ret->start_w[ n_hidden_layers + 2 ] ] = 0.0;
    ret->v_w[ ret->start_w[ n_hidden_layers + 2 ] + 1 ] = 0.0;

    return ret;
}

/**
 * Constructor for a network.
 *
 * @param network   Pointer to the network to destroy.
 *
 */

void destroy_network(NETWORK *n) {

    free(n->neurons_per_layer);
    free(n->linear);
    free(n->scaling_a);
    free(n->start_w);
    free(n->start_a);
    free(n->start_z);
    free(n->sample_w);
    free(n->m_w);
    free(n->m_w_new);
    free(n->m_w_squared);
    free(n->v_w);
    free(n->v_w_new);
    free(n->m_z);
    free(n->m_z_squared);
    free(n->m_z_scaled);
    free(n->v_z);
    free(n->m_a);
    free(n->v_a);
    free(n->alpha);
    free(n->gamma);
    free(n->delta_m);
    free(n->delta_v);
    free(n->dm_z_d_m_a);
    free(n->dv_z_d_m_a);
    free(n->dm_z_d_v_a);
    free(n->dv_z_d_v_a);
    free(n->dm_a_d_m_a);
    free(n->dv_a_d_m_a);
    free(n->dm_a_d_v_a);
    free(n->dv_a_d_v_a);
    free(n->dm_a_d_m_z);
    free(n->dv_a_d_m_z);
    free(n->dm_a_d_v_z);
    free(n->dv_a_d_v_z);
    free(n->dm_a_d_m_w);
    free(n->dv_a_d_m_w);
    free(n->dm_a_d_v_w);
    free(n->dv_a_d_v_w);
    free(n->grad_m_w);
    free(n->grad_v_w);
    free(n->m_w_hat_nat);
    free(n->v_w_hat_nat);
    free(n->a_w_hat_nat);
    free(n->b_w_hat_nat);
    free(n->m_w_old);
    free(n->v_w_old);

    return;
}

/**
 * Deterministic forward propagation of the data.
 *
 */

void deterministc_forward_PBP(NETWORK *n, int index_layer) {

    int i;
    double sqrt_aux;

    /* Auxiliary pointers */

    double * __restrict__ sample_w;
    double * __restrict__ m_z;
    double * __restrict__ m_z_previous_layer;
    double * __restrict__ m_a;

    /* Auxiliary variables */

    double scaling_a;
    int n_neurons;
    int input_size;
    int linear;

    /* We initialize the pointers */

    sample_w = n->sample_w + n->start_w[ index_layer ];
    m_z = n->m_z + n->start_z[ index_layer ];
    m_z_previous_layer = n->m_z + n->start_z[ index_layer - 1 ];
    m_a = n->m_a + n->start_a[ index_layer ];

    /* We update the number of neurons, input size, type of activation and scaling constant */

    n_neurons = n->neurons_per_layer[ index_layer ];
    input_size = n->neurons_per_layer[ index_layer - 1 ] + 1;
    linear = n->linear[ index_layer ];
    scaling_a = n->scaling_a[ index_layer ];

    /* We compute the scaled activation mean: sqrt_aux * sample_w x m_z */

    sqrt_aux = sqrt(scaling_a);
    cblas_dgemv(CblasRowMajor, CblasNoTrans, n_neurons, input_size, sqrt_aux, sample_w, input_size, m_z_previous_layer, 1, 0.0, m_a, 1);

    /* We apply the non-linearity or compute a linear activation function */

    if (linear) {
        for (i = 0 ; i < n_neurons ; i++) {
            m_z[ i ] = m_a[ i ];
        }
    } else {
        for (i = 0 ; i < n_neurons ; i++) {
            m_z[ i ] = (m_a[ i ] > 0) ? m_a[ i ] : 0;
        }
    }

    /* We add the bias if we are not in the two last layers */

    if (index_layer < n->n_hidden_layers + 1) {
        m_z[ n_neurons ] = 1.0;
    }
}

/**
 * Forward propagation of probabilities.
 *
 */

void forward_PBP(NETWORK *n, int index_layer) {

    int i, limit;
    double a, a2, p, g, sqrt_aux;

    /* Auxiliary pointers */

    double * __restrict__ m_w;
    double * __restrict__ m_w_squared;
    double * __restrict__ v_w;
    double * __restrict__ m_z;
    double * __restrict__ v_z;
    double * __restrict__ m_a;
    double * __restrict__ v_a;
    double * __restrict__ m_z_previous_layer;
    double * __restrict__ m_z_previous_layer_squared;
    double * __restrict__ v_z_previous_layer;
    double * __restrict__ alpha;
    double * __restrict__ gamma;

    /* Auxiliary variables */

    double scaling_a;
    int n_neurons;
    int input_size;
    int linear;

    /* We initialize the pointers */

    m_w = n->m_w + n->start_w[ index_layer ];
    m_w_squared = n->m_w_squared + n->start_w[ index_layer ];
    v_w = n->v_w + n->start_w[ index_layer ];
    m_z = n->m_z + n->start_z[ index_layer ];
    v_z = n->v_z + n->start_z[ index_layer ];
    m_a = n->m_a + n->start_a[ index_layer ];
    v_a = n->v_a + n->start_a[ index_layer ];
    m_z_previous_layer = n->m_z + n->start_z[ index_layer - 1 ];
    v_z_previous_layer = n->v_z + n->start_z[ index_layer - 1 ];
    m_z_previous_layer_squared = n->m_z_squared + n->start_z[ index_layer - 1 ];
    alpha = n->alpha + n->start_a[ index_layer ];
    gamma = n->gamma + n->start_a[ index_layer ];

    /* We update the number of neurons, input size, type of activation and scaling constant */

    n_neurons = n->neurons_per_layer[ index_layer ];
    input_size = n->neurons_per_layer[ index_layer - 1 ] + 1;
    linear = n->linear[ index_layer ];
    scaling_a = n->scaling_a[ index_layer ];

    /* We compute the m_z_previous_layer_squared and the m_w_squared */

    limit = n_neurons * input_size;
    for (i = 0 ; i < limit ; i++)
        m_w_squared[ i ] = m_w[ i ] * m_w[ i ];
    for (i = 0 ; i < input_size ; i++)
        m_z_previous_layer_squared[ i ] = m_z_previous_layer[ i ] * m_z_previous_layer[ i ];

    /* We compute the scaled activation mean: sqrt_aux * m_w x m_z */

    sqrt_aux = sqrt(scaling_a);
    cblas_dgemv(CblasRowMajor, CblasNoTrans, n_neurons, input_size, sqrt_aux, m_w, input_size, m_z_previous_layer, 1, 0.0, m_a, 1);

    /* We compute the scaled activation variance: (m_w_squared x v_z_previous_layer +
       v_w x m_z_previous_layer_squared + v_w x v_z_previous_layer) */

    cblas_dgemv(CblasRowMajor, CblasNoTrans, n_neurons, input_size, scaling_a, m_w_squared, input_size, v_z_previous_layer, 1, 0, v_a, 1);
    cblas_dgemv(CblasRowMajor, CblasNoTrans, n_neurons, input_size, scaling_a, v_w, input_size, m_z_previous_layer_squared, 1, 1.0, v_a, 1);
    cblas_dgemv(CblasRowMajor, CblasNoTrans, n_neurons, input_size, scaling_a, v_w, input_size, v_z_previous_layer, 1, 1.0, v_a, 1);

    /* We apply the non-linearity or compute a linear activation function */

    if (linear) {
        for (i = 0 ; i < n_neurons ; i++) {
            m_z[ i ] = m_a[ i ];
            v_z[ i ] = v_a[ i ];
        }
    } else {
        for (i = 0 ; i < n_neurons ; i++) {

            sqrt_aux = sqrt(v_a[ i ]);
            a = m_a[ i ] / sqrt_aux;
            a2 = a * a;
            p = pnorm(a);
            g = (a < -30) ? -a - 1.0 / a + 2.0 / (a * a2) : 0.398942280401432677939946059 * exp(-0.5 * a2) / p;

            /* We compute the output mean and variance */

            m_z[ i ] = p * (m_a[ i ] + sqrt_aux * g);
            v_z[ i ] = m_z[ i ] * (m_a[ i ] + sqrt_aux * g) * (1 - p) + p * v_a[ i ] * (1 - g * g - g * a);

            alpha[ i ] = a;
            gamma[ i ] = g;
        }
    }

    /* We add the bias if we are not in the two last layers */

    if (index_layer < n->n_hidden_layers + 1) {
        m_z[ n_neurons ] = 1.0;
        v_z[ n_neurons ] = 0.0;
    }
}

/**
 * Backward computation of gradients.
 *
 */

void backward_PBP(NETWORK *n, int index_layer) {

    double g, a, da_dm_a, da_dv_a, dg_dm_a, dg_dv_a, d, p;
    int i, j, k, limit_1, limit_2;

    /* Auxiliary pointers */

    double * __restrict__ m_w;
    double * __restrict__ m_w_layer_above;
    double * __restrict__ v_w_layer_above;
    double * __restrict__ m_z;
    double * __restrict__ m_z_scaled;
    double * __restrict__ m_a;
    double * __restrict__ v_a;
    double * __restrict__ m_z_layer_below;
    double * __restrict__ m_z_layer_below_scaled;
    double * __restrict__ v_z_layer_below;
    double * __restrict__ v_z_layer_below_scaled;
    double * __restrict__ alpha;
    double * __restrict__ gamma;
    double * __restrict__ delta_m_layer_above;
    double * __restrict__ delta_v_layer_above;
    double * __restrict__ delta_m;
    double * __restrict__ delta_v;
    double * __restrict__ dm_z_d_m_a;
    double * __restrict__ dv_z_d_m_a;
    double * __restrict__ dm_z_d_v_a;
    double * __restrict__ dv_z_d_v_a;
    double * __restrict__ dm_a_d_m_a;
    double * __restrict__ dv_a_d_m_a;
    double * __restrict__ dm_a_d_v_a;
    double * __restrict__ dv_a_d_v_a;
    double * __restrict__ dm_a_d_m_z;
    double * __restrict__ dv_a_d_m_z;
    double * __restrict__ dm_a_d_v_z;
    double * __restrict__ dv_a_d_v_z;
    double * __restrict__ dm_a_d_m_w;
    double * __restrict__ dv_a_d_m_w;
    double * __restrict__ dm_a_d_v_w;
    double * __restrict__ dv_a_d_v_w;
    double * __restrict__ grad_m_w;
    double * __restrict__ grad_v_w;

    /* Auxiliary variables */

    double scaling_a, scaling_a_layer_above, sqrt_aux, sqrt_v_a;
    double *p_aux_1, *p_aux_2, *p_aux_3, *p_aux_4;
    double *p_aux_1_2, *p_aux_2_2, *p_aux_3_2, *p_aux_4_2;
    int n_neurons;
    int linear;

    /* We initialize the auxiliary pointers */

    m_w = n->m_w + n->start_w[ index_layer ];
    m_w_layer_above = n->m_w + n->start_w[ index_layer + 1 ];
    v_w_layer_above = n->v_w + n->start_w[ index_layer + 1 ];
    m_z = n->m_z + n->start_z[ index_layer ];
    m_z_scaled = n->m_z_scaled + n->start_z[ index_layer ];
    m_a = n->m_a + n->start_a[ index_layer ];
    v_a = n->v_a + n->start_a[ index_layer ];
    m_z_layer_below = n->m_z + n->start_z[ index_layer - 1 ];
    v_z_layer_below = n->v_z + n->start_z[ index_layer - 1 ];
    m_z_layer_below_scaled = n->m_z_scaled + n->start_z[ index_layer - 1 ];
    v_z_layer_below_scaled = n->v_z_scaled + n->start_z[ index_layer - 1 ];
    alpha = n->alpha + n->start_a[ index_layer ];
    gamma = n->gamma + n->start_a[ index_layer ];
    delta_m_layer_above = n->delta_m + n->start_z[ index_layer + 1 ];
    delta_v_layer_above = n->delta_v + n->start_z[ index_layer + 1 ];
    delta_m = n->delta_m + n->start_z[ index_layer ];
    delta_v = n->delta_v + n->start_z[ index_layer ];

    dm_z_d_m_a = n->dm_z_d_m_a + n->start_z[ index_layer ];
    dv_z_d_m_a = n->dv_z_d_m_a + n->start_z[ index_layer ];
    dm_z_d_v_a = n->dm_z_d_v_a + n->start_z[ index_layer ];
    dv_z_d_v_a = n->dv_z_d_v_a + n->start_z[ index_layer ];
    dm_a_d_m_a = n->dm_a_d_m_a + n->start_w[ index_layer ];
    dv_a_d_m_a = n->dv_a_d_m_a + n->start_w[ index_layer ];
    dm_a_d_v_a = n->dm_a_d_v_a + n->start_w[ index_layer ];
    dv_a_d_v_a = n->dv_a_d_v_a + n->start_w[ index_layer ];
    dm_a_d_m_z = n->dm_a_d_m_z + n->start_w[ index_layer ];
    dv_a_d_m_z = n->dv_a_d_m_z + n->start_w[ index_layer ];
    dm_a_d_v_z = n->dm_a_d_v_z + n->start_w[ index_layer ];
    dv_a_d_v_z = n->dv_a_d_v_z + n->start_w[ index_layer ];
    dm_a_d_m_w = n->dm_a_d_m_w + n->start_w[ index_layer ];
    dv_a_d_m_w = n->dv_a_d_m_w + n->start_w[ index_layer ];
    dm_a_d_v_w = n->dm_a_d_v_w + n->start_w[ index_layer ];
    dv_a_d_v_w = n->dv_a_d_v_w + n->start_w[ index_layer ];
    grad_m_w = n->grad_m_w + n->start_w[ index_layer ];
    grad_v_w = n->grad_v_w + n->start_w[ index_layer ];

    /* We update the number of neurons, input size, type of activation and scaling constant */

    n_neurons = n->neurons_per_layer[ index_layer ];
    linear = n->linear[ index_layer ];
    scaling_a = n->scaling_a[ index_layer ];
    scaling_a_layer_above = n->scaling_a[ index_layer + 1 ];

    /* We compute the gradient of the non-linear activations with respect to the activations */

    if (linear) {
        for (i = 0 ; i < n_neurons ; i++) {
            dm_z_d_m_a[ i ] = 1.0;
            dm_z_d_v_a[ i ] = 0.0;
            dv_z_d_m_a[ i ] = 0.0;
            dv_z_d_v_a[ i ] = 1.0;

        }
    } else {
        for (i = 0 ; i < n_neurons ; i++) {
            g = gamma[ i ];
            a = alpha[ i ];
            da_dm_a = 1.0 / sqrt(v_a[ i ]);
            da_dv_a = m_a[ i ] / (2 * v_a[ i ]) * da_dm_a;
            if (a < -30) {
                g = -a - 1.0 / a + 2.0 / (a * a * a);
                dg_dm_a = -da_dm_a + 1.0 / (a * a) * da_dm_a - 6.0 / (a * a * a * a) * da_dm_a;
                dg_dv_a = -da_dv_a + 1.0 / (a * a) * da_dv_a - 6.0 / (a * a * a * a) * da_dv_a;
            } else {
                dg_dm_a = -(g * a + g * g) * da_dm_a;
                dg_dv_a = -(g * a + g * g) * da_dv_a;
            }
            d = 0.398942280401432677939946059 * exp(-0.5 * a * a);
            p = pnorm(a);
            sqrt_v_a = sqrt(v_a[ i ]);
            dm_z_d_m_a[ i ] = da_dm_a * d * (m_a[ i ] + sqrt_v_a * g) + p * (1 + sqrt_v_a * dg_dm_a);
            dm_z_d_v_a[ i ] = da_dv_a * d * (m_a[ i ] + sqrt_v_a * g) + p * (g / (2 * sqrt_v_a) + sqrt_v_a * dg_dv_a);
            dv_z_d_m_a[ i ] = dm_z_d_m_a[ i ] * (m_a[ i ] + sqrt_v_a * g) * (1 - p) + m_z[ i ] * ((1 + sqrt_v_a * dg_dm_a) * (1 - p) -
                (m_a[ i ] + sqrt_v_a * g) * d * da_dm_a) + d * da_dm_a * v_a[ i ] * (1 - g * g - g * a) - p * v_a[ i ] *
                (2 * g * dg_dm_a + dg_dm_a * a + g * da_dm_a);
            dv_z_d_v_a[ i ] = dm_z_d_v_a[ i ] * (m_a[ i ] + sqrt_v_a * g) * (1 - p) + m_z[ i ] * ((0.5 / sqrt_v_a * g + dg_dv_a * sqrt_v_a) * (1 - p) -
                (m_a[ i ]+ sqrt_v_a * g) * d * da_dv_a) + d * da_dv_a * v_a[ i ] * (1 - g * g - g * a) + p * ((1 - g * g - g * a) + v_a[ i ] *
                (-2 * g * dg_dv_a - dg_dv_a * a - g * da_dv_a));
        }
    }

    /* We scale m_z */

    limit_1 = n_neurons + 1;
    for (i = 0 ; i < limit_1 ; i++) {
        m_z_scaled[ i ] = m_z[ i ] * scaling_a_layer_above;
    }

    /* We initialize the rows of dv_a_dm_z to m_z_scaled */

    limit_1 = n->neurons_per_layer[ index_layer + 1 ];
    limit_2 = n_neurons + 1;
    for (i = 0 ; i < limit_1 ; i++) {
        p_aux_1 = dv_a_d_m_z + i * limit_2;
        for (j = 0 ; j < limit_2 ; j++) {
            p_aux_1[ j ] = m_z_scaled[ j ];
        }
    }

    /* We compute the gradient of the activation of the layer above with respect to the non-linear activations in this layer */

    sqrt_aux = sqrt(scaling_a_layer_above);
    limit_1 = n->neurons_per_layer[ index_layer + 1 ] * (n_neurons + 1);
    for (i = 0 ; i < limit_1 ; i++) {
        dm_a_d_m_z[ i ] = m_w_layer_above[ i ] * sqrt_aux;
        dm_a_d_v_z[ i ] = 0;
        dv_a_d_m_z[ i ] = dv_a_d_m_z[ i ] * 2 * v_w_layer_above[ i ];
        dv_a_d_v_z[ i ] = (m_w_layer_above[ i ] * m_w_layer_above[ i ] + v_w_layer_above[ i ]) * scaling_a_layer_above;
    }

    /* We scale the activatons from the layer below */

    sqrt_aux = sqrt(scaling_a);
    limit_1 = n->neurons_per_layer[ index_layer - 1 ] + 1;
    for (i = 0 ; i < limit_1 ; i++) {
        m_z_layer_below_scaled[ i ] = m_z_layer_below[ i ] * sqrt_aux;
        v_z_layer_below_scaled[ i ] = v_z_layer_below[ i ] * scaling_a;
    }

    /* We initialize dm_a_dm_w, dv_a_dm_w and dv_a_dv_w */

    limit_1 = n_neurons;
    limit_2 = n->neurons_per_layer[ index_layer - 1 ] + 1;
    for (i = 0 ; i < limit_1 ; i++) {
        p_aux_1 = dm_a_d_m_w + i * limit_2;
        p_aux_2 = dv_a_d_m_w + i * limit_2;
        p_aux_3 = dv_a_d_v_w + i * limit_2;
        for (j = 0 ; j < limit_2 ; j++) {
            p_aux_1[ j ] = m_z_layer_below_scaled[ j ];
            p_aux_2[ j ] = v_z_layer_below_scaled[ j ];
            p_aux_3[ j ] = (m_z_layer_below_scaled[ j ] * m_z_layer_below_scaled[ j ] + v_z_layer_below_scaled[ j ]);
        }
    }

    limit_1 = n_neurons * (n->neurons_per_layer[ index_layer - 1 ] + 1);
    for (i = 0 ; i < limit_1 ; i++) {
         dm_a_d_v_w[ i ] = 0;
         dv_a_d_m_w[ i ] = 2 * dv_a_d_m_w[ i ] * m_w[ i ];
    }

    /* We compute the gradient of the activations of the top layer with respect to the activations of the current layer */

    limit_1 = n->neurons_per_layer[ index_layer + 1 ];
    limit_2 = n_neurons + 1;
    k = 0;
    for (i = 0 ; i < limit_1 ; i++) {
        p_aux_1 = dm_a_d_m_a + i * limit_2;
        p_aux_2 = dv_a_d_m_a + i * limit_2;
        p_aux_3 = dm_a_d_v_a + i * limit_2;
        p_aux_4 = dv_a_d_v_a + i * limit_2;
        p_aux_1_2 = dm_a_d_m_z + i * limit_2;
        p_aux_2_2 = dm_a_d_v_z + i * limit_2;
        p_aux_3_2 = dv_a_d_m_z + i * limit_2;
        p_aux_4_2 = dv_a_d_v_z + i * limit_2;
        for (j = 0 ; j < limit_2 - 1 ; j++) {
            p_aux_1[ j ] = p_aux_1_2[ j ] * dm_z_d_m_a[ j ] + p_aux_2_2[ j ] * dv_z_d_m_a[ j ];
            p_aux_2[ j ] = p_aux_3_2[ j ] * dm_z_d_m_a[ j ] + p_aux_4_2[ j ] * dv_z_d_m_a[ j ];
            p_aux_3[ j ] = p_aux_1_2[ j ] * dm_z_d_v_a[ j ] + p_aux_2_2[ j ] * dv_z_d_v_a[ j ];
            p_aux_4[ j ] = p_aux_3_2[ j ] * dm_z_d_v_a[ j ] + p_aux_4_2[ j ] * dv_z_d_v_a[ j ];
        }
    }

    /* We compute the deltas */

    limit_1 = n->neurons_per_layer[ index_layer + 1 ];
    cblas_dgemv(CblasRowMajor, CblasTrans, limit_1, n_neurons + 1, 1.0, dm_a_d_m_a, n_neurons + 1, delta_m_layer_above, 1, 0.0, delta_m, 1);
    cblas_dgemv(CblasRowMajor, CblasTrans, limit_1, n_neurons + 1, 1.0, dv_a_d_m_a, n_neurons + 1, delta_v_layer_above, 1, 1.0, delta_m, 1);
    cblas_dgemv(CblasRowMajor, CblasTrans, limit_1, n_neurons + 1, 1.0, dm_a_d_v_a, n_neurons + 1, delta_m_layer_above, 1, 0.0, delta_v, 1);
    cblas_dgemv(CblasRowMajor, CblasTrans, limit_1, n_neurons + 1, 1.0, dv_a_d_v_a, n_neurons + 1, delta_v_layer_above, 1, 1.0, delta_v, 1);

    /* We compute the gradients */

    limit_1 = n_neurons;
    limit_2 = n->neurons_per_layer[ index_layer - 1 ] + 1;
    k = 0;
    for (i = 0 ; i < limit_1 ; i++) {
        for (j = 0 ; j < limit_2 ; j++) {
            grad_m_w[ k ] = delta_m[ i ] * dm_a_d_m_w[ k ] + delta_v[ i ] * dv_a_d_m_w[ k ];
            grad_v_w[ k ] = delta_m[ i ] * dm_a_d_v_w[ k ] + delta_v[ i ] * dv_a_d_v_w[ k ];
            k++;
        }
    }
}

/**
 * Function that performs an ADF update.
 *
 * @param network   Pointer to the network.
 * @param x         Pointer to the input features.
 * @param y         Pointer to the target.
 *
 */

void do_ADF_update(NETWORK * n, double *x, double y) {

    int i, limit_1;

    double * __restrict__ m_z;
    double * __restrict__ v_z;
    double * __restrict__ delta_m;
    double * __restrict__ delta_v;

    double * __restrict__ m_w;
    double * __restrict__ v_w;
    double * __restrict__ grad_m_w;
    double * __restrict__ grad_v_w;

    double m, v, v1, v2, logZ, logZ1, logZ2, d_logZ_d_m, d_logZ_d_v, a_new, b_new, m_aux, v_aux;

    /* We initialize the non-linear activations of the input layer with the data */

    limit_1 = n->start_a[ 1 ];
    m_z = n->m_z;
    v_z = n->v_z;
    for (i = 0 ; i < limit_1 ; i++) {
        m_z[ i ] = x[ i ];
        v_z[ i ] = 0.0;
    }
    m_z[ i ] = 1.0;
    v_z[ i ] = 0.0;

    /* We do a forward pass */

    limit_1 = n->n_hidden_layers + 2;
    for (i = 1 ; i < limit_1 ; i++)
        forward_PBP(n, i);

    /* We obtain logZ, logZ1 and logZ2 and the updates for a and b */

    m_z = n->m_z + n->start_z[ n->n_hidden_layers + 1 ];
    v_z = n->v_z + n->start_z[ n->n_hidden_layers + 1 ];

    m = m_z[ 0 ];
    v = v_z[ 0 ] + n->b_noise / (n->a_noise - 1);
    v1 = v_z[ 0 ] + n->b_noise / (n->a_noise - 0);
    v2 = v_z[ 0 ] + n->b_noise / (n->a_noise + 1);
    logZ = -0.5 * (log(v) + (m - y) * (m - y) / v);
    logZ1 = -0.5 * (log(v1) + (m - y) * (m - y) / v1);
    logZ2 = -0.5 * (log(v2) + (m - y) * (m - y) / v2);

    a_new = 1.0 / (exp(logZ2 - 2 * logZ1 + logZ) * (n->a_noise + 1.0) / n->a_noise - 1.0);
    b_new = 1.0 / (exp(logZ2 - logZ1) * (n->a_noise + 1) / n->b_noise - exp(logZ1 - logZ) * n->a_noise / n->b_noise);

    /* We initialize the deltas for the output layer */

    d_logZ_d_m = -(m - y) / v;
    d_logZ_d_v = -0.5 / v + 0.5 * (m - y) * (m - y) / (v * v);
    delta_m = n->delta_m + n->start_z[ n->n_hidden_layers + 2 ];
    delta_v = n->delta_v + n->start_z[ n->n_hidden_layers + 2 ];
    delta_m[ 0 ] = d_logZ_d_m;
    delta_v[ 0 ] = d_logZ_d_v;

    /* We do a backward pass */

    limit_1 = n->n_hidden_layers + 1;
    for (i = limit_1 ; i >= 1 ; i--)
        backward_PBP(n, i);

    /* We update the mean and variance parameters, except those in the fake output layer */

    m_w = n->m_w;
    v_w = n->v_w;
    grad_m_w = n->grad_m_w;
    grad_v_w = n->grad_v_w;

    limit_1 = n->size_w - 2;
    for (i = 0 ; i < limit_1 ; i++) {
        v_aux = v_w[ i ] - v_w[ i ] * v_w[ i ] * (grad_m_w[ i ] * grad_m_w[ i ] - 2 * grad_v_w[ i ]);
        m_aux = m_w[ i ] + v_w[ i ] * grad_m_w[ i ];
        if (v_aux > 1e-100 && m_aux != NAN && v_aux != NAN) {
            m_w[ i ] = m_aux;
            v_w[ i ] = v_aux;
        }
    }

    /* We update the noise variables */

    n->b_noise = b_new;
    n->a_noise = a_new;
}

/**
 * Function that refines the approximate factor for the prior.
 *
 * @param network   Pointer to the network.
 *
 */

void refine_prior(NETWORK *n) {

    int i;

    double v_w_nat, m_w_nat, v_w_cav_nat, m_w_cav_nat, v_w_cav, m_w_cav,
        a_w_nat, b_w_nat, a_w_cav_nat, b_w_cav_nat, a_w_cav, b_w_cav,
        v, v1, v2, logZ, logZ1, logZ2, d_logZ_d_m_w_cav, d_logZ_d_v_w_cav,
        m_w_new, v_w_new, a_w_new, b_w_new, v_w_new_nat, m_w_new_nat,
        a_w_new_nat, b_w_new_nat;

    /* We iterate over the layers refining the prior */

    for (i = 0 ; i < n->size_w - 2 ; i++) {
        v_w_nat = 1.0 / n->v_w[ i ];
        m_w_nat = n->m_w[ i ] / n->v_w[ i ];
        v_w_cav_nat = v_w_nat - n->v_w_hat_nat[ i ];
        m_w_cav_nat = m_w_nat - n->m_w_hat_nat[ i ];

        v_w_cav = 1.0 / v_w_cav_nat;
        m_w_cav = m_w_cav_nat / v_w_cav_nat;
        a_w_nat = n->a_prior - 1;
        b_w_nat = -n->b_prior;
        a_w_cav_nat = a_w_nat - n->a_w_hat_nat[ i ];
        b_w_cav_nat = b_w_nat - n->b_w_hat_nat[ i ];
        a_w_cav = a_w_cav_nat + 1;
        b_w_cav = -b_w_cav_nat;

        if (v_w_cav > 0 && b_w_cav > 0 && a_w_cav > 1 && v_w_cav < 1e6) {

            v = v_w_cav + b_w_cav / (a_w_cav - 1);
            v1  = v_w_cav + b_w_cav / a_w_cav;
            v2  = v_w_cav + b_w_cav / (a_w_cav + 1);
            logZ = -0.5 * log(v) - 0.5 * m_w_cav * m_w_cav / v;
            logZ1 = -0.5 * log(v1) - 0.5 * m_w_cav * m_w_cav / v1;
            logZ2 = -0.5 * log(v2) - 0.5 * m_w_cav * m_w_cav / v2;
            d_logZ_d_m_w_cav = -m_w_cav / v;
            d_logZ_d_v_w_cav = -0.5 / v + 0.5 * m_w_cav * m_w_cav / (v * v);
            m_w_new = m_w_cav + v_w_cav * d_logZ_d_m_w_cav;
            v_w_new = v_w_cav - v_w_cav * v_w_cav * (d_logZ_d_m_w_cav * d_logZ_d_m_w_cav - 2 * d_logZ_d_v_w_cav) ;
            a_w_new = 1.0 / (exp(logZ2 - 2 * logZ1 + logZ) * (a_w_cav + 1) / a_w_cav - 1.0);
            b_w_new = 1.0 / (exp(logZ2 - logZ1) * (a_w_cav + 1) / (b_w_cav) - exp(logZ1 - logZ) * a_w_cav / b_w_cav);
            v_w_new_nat = 1.0 / v_w_new;
            m_w_new_nat = m_w_new / v_w_new;
            a_w_new_nat = a_w_new - 1;
            b_w_new_nat = -b_w_new;

            n->m_w_hat_nat[ i ] = m_w_new_nat - m_w_cav_nat;
            n->v_w_hat_nat[ i ] = v_w_new_nat - v_w_cav_nat;
            n->a_w_hat_nat[ i ] = a_w_new_nat - a_w_cav_nat;
            n->b_w_hat_nat[ i ] = b_w_new_nat - b_w_cav_nat;

            n->m_w[ i ] = m_w_new;
            n->v_w[ i ] = v_w_new;

            n->a_prior = a_w_new;
            n->b_prior = b_w_new;
        }
    }
}

/**
 * Function that performs one learning epoch.
 *
 * @param network       Pointer to the network.
 * @param x             Pointer to the input feature matrix.
 * @param y             Pointer to the target vector.
 * @param n_datapoints  Number of data points in the training set.
 * @param d             Dimensionality of the training data.
 * @param permutation   A random permuation of the input data.
 *
 */

NETWORK *one_learning_epoch(NETWORK *n, double *x, double *y, int n_datapoints, int d, int *permutation) {

    int i, index;

    /* We do one ADF upte for each datapoint */

    for (i = 0 ; i < n_datapoints ; i++) {
        index = permutation[ i ] * d;
        do_ADF_update(n, x + index, y[ permutation[ i ] ]);

        if (i % 1000 == 0) {
            printf(".");
            fflush(stdout);
        }
    }
    printf("\n");
    fflush(stdout);

    /* We refine the prior */

    refine_prior(n);

    /* We are done */

    return n;
}

/**
 * Function that predicts deterministically.
 *
 * @param network       Pointer to the network.
 * @param x_test        Pointer to the input feature matrix.
 * @param y_test        Pointer to store the predictions.
 * @param n_datapoints  Number of data points in the test set.
 * @param d             Dimensionality of the test data.
 *
 */

NETWORK *predict_deterministic(NETWORK *n, double *x_test, double *y_test, int n_datapoints, int d) {

    int i, j, index, limit_1;

    double *m_z;

    for (i = 0 ; i < n_datapoints ; i++) {

        /* We initialize the non-linear activations of the input layer with the data */

        limit_1 = n->start_a[ 1 ];
        m_z = n->m_z;
        for (j = 0 ; j < limit_1 ; j++) {
            index = i * d + j;
            m_z[ j ] = x_test[ index ];
        }
        m_z[ j ] = 1.0;

        /* We do a forward pass */

        limit_1 = n->n_hidden_layers + 2;
        for (j = 1 ; j < limit_1 ; j++)
            deterministc_forward_PBP(n, j);

        /* We store the results */

        m_z = n->m_z + n->start_z[ n->n_hidden_layers + 1 ];

        y_test[ i ] = m_z[ 0 ];
    }

    /* We are done */

    return n;
}

/**
 * Function that predicts.
 *
 * @param network       Pointer to the network.
 * @param x_test        Pointer to the input feature matrix.
 * @param m_test        Pointer to store the predictive mean.
 * @param v_test        Pointer to store the predictive variance.
 * @param v_noise       Pointer to store the noise variance.
 * @param n_datapoints  Number of data points in the test set.
 * @param d             Dimensionality of the test data.
 *
 */

NETWORK *predict(NETWORK *n, double *x_test, double *m_test, double *v_test, double *v_noise, int n_datapoints, int d) {

    int i, j, index, limit_1;

    double *m_z, *v_z;

    for (i = 0 ; i < n_datapoints ; i++) {

        /* We initialize the non-linear activations of the input layer with the data */

        limit_1 = n->start_a[ 1 ];
        m_z = n->m_z;
        v_z = n->v_z;
        for (j = 0 ; j < limit_1 ; j++) {
            index = i * d + j;
            m_z[ j ] = x_test[ index ];
            v_z[ j ] = 0.0;
        }
        m_z[ j ] = 1.0;
        v_z[ j ] = 0.0;

        /* We do a forward pass */

        limit_1 = n->n_hidden_layers + 2;
        for (j = 1 ; j < limit_1 ; j++)
            forward_PBP(n, j);

        /* We store the results */

        m_z = n->m_z + n->start_z[ n->n_hidden_layers + 1 ];
        v_z = n->v_z + n->start_z[ n->n_hidden_layers + 1 ];

        m_test[ i ] = m_z[ 0 ];
        v_test[ i ] = v_z[ 0 ];
    }

    /* We store the noise value */

    *v_noise = n->b_noise / (n->a_noise - 1);

    /* We are done */

    return n;
}

/* Function that returns the parameters of the network */

void get_params(NETWORK *n, double *sample_w_out, double *m_w_out, double *v_w_out, double *m_w_hat_nat_out, double *v_w_hat_nat_out,
    double *a_w_hat_nat_out, double *b_w_hat_nat_out, double *a_noise_out, double *b_noise_out, double *a_prior_out,
    double *b_prior_out, int *neurons_per_layer_out) {

    int i, limit_1;

    /* We copy the parameters */

    limit_1 = n->size_w - 2;
    for (i = 0 ; i < limit_1 ; i++) {
        sample_w_out[ i ] = n->sample_w[ i ];
        m_w_out[ i ] = n->m_w[ i ];
        v_w_out[ i ] = n->v_w[ i ];
        m_w_hat_nat_out[ i ] = n->m_w_hat_nat[ i ];
        v_w_hat_nat_out[ i ] = n->v_w_hat_nat[ i ];
        a_w_hat_nat_out[ i ] = n->a_w_hat_nat[ i ];
        b_w_hat_nat_out[ i ] = n->b_w_hat_nat[ i ];
    }

    *a_noise_out = n->a_noise;
    *b_noise_out = n->b_noise;
    *a_prior_out = n->a_prior;
    *b_prior_out = n->b_prior;

    limit_1 = n->n_hidden_layers + 2;
    for (i = 0 ; i < limit_1 ; i++) {
        neurons_per_layer_out[ i ] = n->neurons_per_layer[ i ];
    }
}

/* Function that returns the sizes of the parameters of the network */

void get_size_params(NETWORK *n, int *size_weihts, int *n_layers) {

    *size_weihts = n->size_w - 2;
    *n_layers = n->n_hidden_layers + 2;
}

/* Function that sets the parameters of the network */

void set_params(NETWORK *n, double *sample_w_in, double *m_w_in, double *v_w_in, double *m_w_hat_nat_in, double *v_w_hat_nat_in,
    double *a_w_hat_nat_in, double *b_w_hat_nat_in, double a_noise_in, double b_noise_in, double a_prior_in, double b_prior_in) {

    int i, limit_1;

    /* We copy the parameters */

    limit_1 = n->size_w - 2;
    for (i = 0 ; i < limit_1 ; i++) {
        n->sample_w[ i ] = sample_w_in[ i ];
        n->m_w[ i ] = m_w_in[ i ];
        n->v_w[ i ] = v_w_in[ i ];
        n->m_w_hat_nat[ i ] = m_w_hat_nat_in[ i ];
        n->v_w_hat_nat[ i ] = v_w_hat_nat_in[ i ];
        n->a_w_hat_nat[ i ] = a_w_hat_nat_in[ i ];
        n->b_w_hat_nat[ i ] = b_w_hat_nat_in[ i ];
    }

    n->a_noise = a_noise_in;
    n->b_noise = b_noise_in;
    n->a_prior = a_prior_in;
    n->b_prior = b_prior_in;
}


#define MAIN_FILE

#include <stdio.h>
#include <stdlib.h>
#include <math.h>
#include <gsl/gsl_matrix.h>
#include <gsl/gsl_spline.h>
#include "global.h"
#include "eos_pres.h"
#include "eos_rho.h"
#include "param.h"

int
main (void)
{
  spline = NULL;
  acc = NULL;
  int i, status;
  double r, r1, y[2];
  // bounds for making grid of m(R) vs. rho(0)
  const double rho_min = 1.0e+13, rho_max = 9.0e+14;
  const int MAX = 100;
  // pressure/density arrays from tabulated EOS data
  int n_eos_pts;
  FILE *fp = NULL;
  double (*pres) (double, void *);	// EOS pressure pointer
  double (*rho) (double, void *);	// EOS density pointer
  double tmp;
  // pointers to tabulated EOS data
  double *eos_tab_pres = NULL, *eos_tab_dens = NULL;

  /* this is the "third arm" of the EOS inversion routine. GSL calls
     the function whose root is being found several times during
     initialization and if we leave the pressure here uninitialized,
     it might be 0 or something crazy, which will be out of bounds of
     the "forward" EOS, and since the forward EOS uses interpolation,
     an out-of-bounds value of pressure will make it crash. */
  tmp_pres = 1.0e+25;

  pres = &eos_pres;
  rho = &eos_rho;

  //  fp = fopen("bck.eos", "r");
  //  fp = fopen("eosC", "r");
  //  fp = fopen("timmes.eos", "r");
  fp = fopen ("../EOS/helmholtz.eos", "r");

  if (fp == NULL)
    {
      fprintf (stderr, "Can't open file!\n");
      exit (1);
    }

  // read in # of EOS data points
  fscanf (fp, "%i", &n_eos_pts);

  eos_tab_dens = (double *) malloc (n_eos_pts * sizeof (double));
  eos_tab_pres = (double *) malloc (n_eos_pts * sizeof (double));

  // read in density and pressure data
  for (i = 0; i < n_eos_pts; i++)
    {
      fscanf (fp, "%le %le", &eos_tab_dens[i], &eos_tab_pres[i]);
    }
  fclose (fp);

  // set up interpolation machinery for EOS
  acc = gsl_interp_accel_alloc ();
  spline = gsl_spline_alloc (gsl_interp_cspline, n_eos_pts);
  gsl_spline_init (spline, eos_tab_dens, eos_tab_pres, n_eos_pts);

  // now make grid
  r = 1.0;			// the integrator will go nuts if we start right at r=0
  r1 = 1.0e+10;			// some final 'radius' (much larger than actual radius)

  printf ("%12s %12s %12s\n", "R", "M(r=R)", "rho(r=0)");

  params.single_star = 1;
  for (i = 0; i < MAX; i++)
    {
      // rho(0) evenly spaced in log
      params.rho_init = log10 (rho_min) +
	(double) i *((log10 (rho_max) - log10 (rho_min)) / (double) MAX);
      params.rho_init = pow (10.0, params.rho_init);

      y[1] = pres (params.rho_init, &params);
      y[0] = (4.0 / 3.0) * M_PI * pow (r, 3.0) * rho (y[1], &params);

      params.pinit = y[1];
      // This function is useful if you want to plot, e.g., central
      // pressure vs. total mass. You can also hang onto the run of
      // pressure with radius, which can be interesting when compared to
      // the Newtonian case.
      status = make_grid (params, r, r1, y);
    }

/*
  params.single_star = 0;
  printf("\nnow for a single star!\n");
  printf("%12s %12s %12s %12s\n", "r", "M(r)", "P(r)", "rho(r)");
  // print P and M vs r for Chandrasekhar-mass star
  params.rho_init = 1.0e+13;

  y[1] = pres(params.rho_init, &params); // rho(0) (roughly) for maximum mass
  y[0] = (4.0/3.0) * M_PI * pow(r, 3.0) * rho(y[1], &params);

  params.pinit = y[1];
  status = make_grid(params, r, r1, y);
*/

  gsl_spline_free (spline);
  gsl_interp_accel_free (acc);

  free (eos_tab_pres);
  free (eos_tab_dens);

  eos_tab_dens = NULL;
  eos_tab_pres = NULL;
  fp = NULL;
  pres = NULL;
  rho = NULL;
  return 0;
}


#include <iostream>
#include <math.h>
#include <gsl/gsl_sort.h>
#include <gsl/gsl_wavelet2d.h>

#include <cgi/mosaic/Mosaic.hpp>

void normalizeBySum(double* data, int n)
{
  int sum = 0;
  for(int i=0; i<n; ++i)
    sum += data[i];

  for(int i=0; i<n; ++i)
    data[i] /= sum;
}

int
main (int argc, char **argv)
{

  const int n = 1024;

  if(argc < 2)
    {
      std::cout << "ERROR" << std::endl;
      return 1;
    }

  cgi::Mosaic m;
  if(cgi::NoError != m.open(argv[1]))
    {
      std::cout << "Failed to open" << std::endl;
      return 2;
    }

  m.setTileSize(cgi::core::Size2d<int>(n,n));

  double* data = new double [n * n];
  if(cgi::NoError != m.getTile<double>(0, 0, data))
    {
      std::cout << "getTile failed" << std::endl;
      return 3;
    }

  gsl_wavelet *w;
  gsl_wavelet_workspace *work;

  w = gsl_wavelet_alloc (gsl_wavelet_daubechies, 4);
  work = gsl_wavelet_workspace_alloc (n);

  if(GSL_SUCCESS != gsl_wavelet2d_transform_forward (w, data, n, n, n, work))
    {
      std::cout << "Transform failed" << std::endl;
      return 4;
    }

  /////////////////////////////////////////
  cgi::Mosaic moDwt;
  moDwt.create("dwt.tif", "GTiff", cgi::core::Size2d<int>(n,n), 1, cgi::Depth64F);
  moDwt.setTileSize(cgi::core::Size2d<int>(n,n));
  moDwt.putTile<double>(data, 0, 0);
  moDwt.close();

  /////////////////////////////////////////
  // normalization
  normalizeBySum(data, n);

  /////////////////////////////////////////
  // do the inverse and reconstruct the image

  if(GSL_SUCCESS != gsl_wavelet2d_transform_inverse (w, data, n, n, n, work))
    {
      std::cout << "Transform failed" << std::endl;
      return 5;
    }

  /////////////////////////////////////////
  // convert the reconstructed image to unsigned short
  unsigned short* recData = new unsigned short [n * n];
  int i;
  for(i=0; i<(n*n); ++i)
    {
      recData[i] = static_cast<unsigned short>(std::max(std::min(data[i], 65535.0), 0.0));
    }

  cgi::Mosaic moRec;
  moRec.create("rec.tif", "GTiff", cgi::core::Size2d<int>(n,n), 1, cgi::Depth16U);
  moRec.setTileSize(cgi::core::Size2d<int>(n,n));
  moRec.putTile<unsigned short>(recData, 0, 0);
  moRec.close();

  /////////////////////////////////////////
  gsl_wavelet_free (w);
  gsl_wavelet_workspace_free (work);

  delete [] recData;
  delete [] data;

  return 0;
}


/* linalg/lu.c
 *
 * Copyright (C) 1996, 1997, 1998, 1999, 2000, 2007, 2009 Gerard Jungman, Brian Gough
 * Copyright (C) 2019 Patrick Alken
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or (at
 * your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 */

#include <config.h>
#include <stdlib.h>
#include <string.h>
#include <gsl/gsl_math.h>
#include <gsl/gsl_vector.h>
#include <gsl/gsl_matrix.h>
#include <gsl/gsl_permute_vector.h>
#include <gsl/gsl_blas.h>
#include <gsl/gsl_linalg.h>

#include "recurse.h"

static int LU_decomp_L2 (gsl_matrix * A, gsl_vector_uint * ipiv);
static int LU_decomp_L3 (gsl_matrix * A, gsl_vector_uint * ipiv);
static int singular (const gsl_matrix * LU);
static int apply_pivots(gsl_matrix * A, const gsl_vector_uint * ipiv);

/* Factorise a general N x N matrix A into,
 *
 *   P A = L U
 *
 * where P is a permutation matrix, L is unit lower triangular and U
 * is upper triangular.
 *
 * L is stored in the strict lower triangular part of the input
 * matrix. The diagonal elements of L are unity and are not stored.
 *
 * U is stored in the diagonal and upper triangular part of the
 * input matrix.
 *
 * P is stored in the permutation p. Column j of P is column k of the
 * identity matrix, where k = permutation->data[j]
 *
 * signum gives the sign of the permutation, (-1)^n, where n is the
 * number of interchanges in the permutation.
 *
 * See Golub & Van Loan, Matrix Computations, Algorithm 3.4.1 (Gauss
 * Elimination with Partial Pivoting).
 */

int
gsl_linalg_LU_decomp (gsl_matrix * A, gsl_permutation * p, int *signum)
{
  const size_t M = A->size1;

  if (p->size != M)
    {
      GSL_ERROR ("permutation length must match matrix size1", GSL_EBADLEN);
    }
  else
    {
      int status;
      const size_t N = A->size2;
      const size_t minMN = GSL_MIN(M, N);
      gsl_vector_uint * ipiv = gsl_vector_uint_alloc(minMN);
      gsl_matrix_view AL = gsl_matrix_submatrix(A, 0, 0, M, minMN);
      size_t i;

      status = LU_decomp_L3 (&AL.matrix, ipiv);

      /* process remaining right matrix */
      if (M < N)
        {
          gsl_matrix_view AR = gsl_matrix_submatrix(A, 0, M, M, N - M);

          /* apply pivots to AR */
          apply_pivots(&AR.matrix, ipiv);

          /* AR = AL^{-1} AR */
          gsl_blas_dtrsm(CblasLeft, CblasLower, CblasNoTrans, CblasUnit, 1.0, &AL.matrix, &AR.matrix);
        }

      /* convert ipiv array to permutation */

      gsl_permutation_init(p);
      *signum = 1;

      for (i = 0; i < minMN; ++i)
        {
          unsigned int pivi = gsl_vector_uint_get(ipiv, i);

          if (p->data[pivi] != p->data[i])
            {
              size_t tmp = p->data[pivi];
              p->data[pivi] = p->data[i];
              p->data[i] = tmp;
              *signum = -(*signum);
            }
        }

      gsl_vector_uint_free(ipiv);

      return status;
    }
}

/*
LU_decomp_L2
  LU decomposition with partial pivoting using Level 2 BLAS

Inputs: A    - on input, matrix to be factored; on output, L and U factors
        ipiv - (output) array containing row swaps

Notes:
1) Based on LAPACK DGETF2
*/

static int
LU_decomp_L2 (gsl_matrix * A, gsl_vector_uint * ipiv)
{
  const size_t M = A->size1;
  const size_t N = A->size2;
  const size_t minMN = GSL_MIN(M, N);

  if (ipiv->size != minMN)
    {
      GSL_ERROR ("ipiv length must equal MIN(M,N)", GSL_EBADLEN);
    }
  else
    {
      size_t i, j;

      for (j = 0; j < minMN; ++j)
        {
          /* find maximum in the j-th column */
          gsl_vector_view v = gsl_matrix_subcolumn(A, j, j, M - j);
          size_t j_pivot = j + gsl_blas_idamax(&v.vector);
          gsl_vector_view v1, v2;

          gsl_vector_uint_set(ipiv, j, j_pivot);

          if (j_pivot != j)
            {
              /* swap rows j and j_pivot */
              v1 = gsl_matrix_row(A, j);
              v2 = gsl_matrix_row(A, j_pivot);
              gsl_blas_dswap(&v1.vector, &v2.vector);
            }

          if (j < M - 1)
            {
              double Ajj = gsl_matrix_get(A, j, j);

              if (fabs(Ajj) >= GSL_DBL_MIN)
                {
                  v1 = gsl_matrix_subcolumn(A, j, j + 1, M - j - 1);
                  gsl_blas_dscal(1.0 / Ajj, &v1.vector);
                }
              else
                {
                  for (i = 1; i < M - j; ++i)
                    {
                      double * ptr = gsl_matrix_ptr(A, j + i, j);
                      *ptr /= Ajj;
                    }
                }
            }

          if (j < minMN - 1)
            {
              gsl_matrix_view A22 = gsl_matrix_submatrix(A, j + 1, j + 1, M - j - 1, N - j - 1);
              v1 = gsl_matrix_subcolumn(A, j, j + 1, M - j - 1);
              v2 = gsl_matrix_subrow(A, j, j + 1, N - j - 1);

              gsl_blas_dger(-1.0, &v1.vector, &v2.vector, &A22.matrix);
            }
        }

      return GSL_SUCCESS;
    }
}

/*
LU_decomp_L3
  LU decomposition with partial pivoting using Level 3 BLAS

Inputs: A    - on input, matrix to be factored; on output, L and U factors
        ipiv - (output) array containing row swaps

Notes:
1) Based on ReLAPACK DGETRF
*/

static int
LU_decomp_L3 (gsl_matrix * A, gsl_vector_uint * ipiv)
{
  const size_t M = A->size1;
  const size_t N = A->size2;

  if (M < N)
    {
      GSL_ERROR ("matrix must have M >= N", GSL_EBADLEN);
    }
  else if (ipiv->size != GSL_MIN(M, N))
    {
      GSL_ERROR ("ipiv length must equal MIN(M,N)", GSL_EBADLEN);
    }
  else if (N <= CROSSOVER_LU)
    {
      /* use Level 2 algorithm */
      return LU_decomp_L2(A, ipiv);
    }
  else
    {
      /*
       * partition matrix:
       *
       *       N1  N2
       * N1  [ A11 A12 ]
       * M2  [ A21 A22 ]
       *
       * and
       *      N1  N2
       * M  [ AL  AR  ]
       */
      int status;
      const size_t N1 = GSL_LINALG_SPLIT(N);
      const size_t N2 = N - N1;
      const size_t M2 = M - N1;
      gsl_matrix_view A11 = gsl_matrix_submatrix(A, 0, 0, N1, N1);
      gsl_matrix_view A12 = gsl_matrix_submatrix(A, 0, N1, N1, N2);
      gsl_matrix_view A21 = gsl_matrix_submatrix(A, N1, 0, M2, N1);
      gsl_matrix_view A22 = gsl_matrix_submatrix(A, N1, N1, M2, N2);

      gsl_matrix_view AL = gsl_matrix_submatrix(A, 0, 0, M, N1);
      gsl_matrix_view AR = gsl_matrix_submatrix(A, 0, N1, M, N2);

      /*
       * partition ipiv = [ ipiv1 ] N1
       *                  [ ipiv2 ] N2
       */
      gsl_vector_uint_view ipiv1 = gsl_vector_uint_subvector(ipiv, 0, N1);
      gsl_vector_uint_view ipiv2 = gsl_vector_uint_subvector(ipiv, N1, N2);

      size_t i;

      /* recursion on (AL, ipiv1) */
      status = LU_decomp_L3(&AL.matrix, &ipiv1.vector);
      if (status)
        return status;

      /* apply ipiv1 to AR */
      apply_pivots(&AR.matrix, &ipiv1.vector);

      /* A12 = A11^{-1} A12 */
      gsl_blas_dtrsm(CblasLeft, CblasLower, CblasNoTrans, CblasUnit, 1.0, &A11.matrix, &A12.matrix);

      /* A22 = A22 - A21 * A12 */
      gsl_blas_dgemm(CblasNoTrans, CblasNoTrans, -1.0, &A21.matrix, &A12.matrix, 1.0, &A22.matrix);

      /* recursion on (A22, ipiv2) */
      status = LU_decomp_L3(&A22.matrix, &ipiv2.vector);
      if (status)
        return status;

      /* apply pivots to A21 */
      apply_pivots(&A21.matrix, &ipiv2.vector);

      /* shift pivots */
      for (i = 0; i < N2; ++i)
        {
          unsigned int * ptr = gsl_vector_uint_ptr(&ipiv2.vector, i);
          *ptr += N1;
        }

      return GSL_SUCCESS;
    }
}

int
gsl_linalg_LU_solve (const gsl_matrix * LU, const gsl_permutation * p, const gsl_vector * b, gsl_vector * x)
{
  if (LU->size1 != LU->size2)
    {
      GSL_ERROR ("LU matrix must be square", GSL_ENOTSQR);
    }
  else if (LU->size1 != p->size)
    {
      GSL_ERROR ("permutation length must match matrix size", GSL_EBADLEN);
    }
  else if (LU->size1 != b->size)
    {
      GSL_ERROR ("matrix size must match b size", GSL_EBADLEN);
    }
  else if (LU->size2 != x->size)
    {
      GSL_ERROR ("matrix size must match solution size", GSL_EBADLEN);
    }
  else if (singular (LU))
    {
      GSL_ERROR ("matrix is singular", GSL_EDOM);
    }
  else
    {
      int status;

      /* copy x <- b */
      gsl_vector_memcpy (x, b);

      /* solve for x */
      status = gsl_linalg_LU_svx (LU, p, x);

      return status;
    }
}

int
gsl_linalg_LU_svx (const gsl_matrix * LU, const gsl_permutation * p, gsl_vector * x)
{
  if (LU->size1 != LU->size2)
    {
      GSL_ERROR ("LU matrix must be square", GSL_ENOTSQR);
    }
  else if (LU->size1 != p->size)
    {
      GSL_ERROR ("permutation length must match matrix size", GSL_EBADLEN);
    }
  else if (LU->size1 != x->size)
    {
      GSL_ERROR ("matrix size must match solution/rhs size", GSL_EBADLEN);
    }
  else if (singular (LU))
    {
      GSL_ERROR ("matrix is singular", GSL_EDOM);
    }
  else
    {
      /* apply permutation to RHS */
      gsl_permute_vector (p, x);

      /* solve for c using forward-substitution, L c = P b */
      gsl_blas_dtrsv (CblasLower, CblasNoTrans, CblasUnit, LU, x);

      /* perform back-substitution, U x = c */
      gsl_blas_dtrsv (CblasUpper, CblasNoTrans, CblasNonUnit, LU, x);

      return GSL_SUCCESS;
    }
}

int
gsl_linalg_LU_refine (const gsl_matrix * A, const gsl_matrix * LU, const gsl_permutation * p, const gsl_vector * b, gsl_vector * x, gsl_vector * work)
{
  if (A->size1 != A->size2)
    {
      GSL_ERROR ("matrix a must be square", GSL_ENOTSQR);
    }
  else if (LU->size1 != LU->size2)
    {
      GSL_ERROR ("LU matrix must be square", GSL_ENOTSQR);
    }
  else if (A->size1 != LU->size2)
    {
      GSL_ERROR ("LU matrix must be decomposition of a", GSL_ENOTSQR);
    }
  else if (LU->size1 != p->size)
    {
      GSL_ERROR ("permutation length must match matrix size", GSL_EBADLEN);
    }
  else if (LU->size1 != b->size)
    {
      GSL_ERROR ("matrix size must match b size", GSL_EBADLEN);
    }
  else if (LU->size1 != x->size)
    {
      GSL_ERROR ("matrix size must match solution size", GSL_EBADLEN);
    }
  else if (LU->size1 != work->size)
    {
      GSL_ERROR ("matrix size must match workspace size", GSL_EBADLEN);
    }
  else if (singular (LU))
    {
      GSL_ERROR ("matrix is singular", GSL_EDOM);
    }
  else
    {
      int status;

      /* compute residual = (A * x  - b) */
      gsl_vector_memcpy (work, b);
      gsl_blas_dgemv (CblasNoTrans, 1.0, A, x, -1.0, work);

      /* find correction, delta = - (A^-1) * residual, and apply it */
      status = gsl_linalg_LU_svx (LU, p, work);
      gsl_blas_daxpy (-1.0, work, x);

      return status;
    }
}

int
gsl_linalg_LU_invert (const gsl_matrix * LU, const gsl_permutation * p, gsl_matrix * inverse)
{
  if (LU->size1 != LU->size2)
    {
      GSL_ERROR ("LU matrix must be square", GSL_ENOTSQR);
    }
  else if (LU->size1 != p->size)
    {
      GSL_ERROR ("permutation length must match matrix size", GSL_EBADLEN);
    }
  else if (inverse->size1 != LU->size1 || inverse->size2 != LU->size2)
    {
      GSL_ERROR ("inverse matrix must match LU matrix dimensions", GSL_EBADLEN);
    }
  else
    {
      gsl_matrix_memcpy(inverse, LU);
      return gsl_linalg_LU_invx (inverse, p);
    }
}

int
gsl_linalg_LU_invx (gsl_matrix * LU, const gsl_permutation * p)
{
  if (LU->size1 != LU->size2)
    {
      GSL_ERROR ("LU matrix must be square", GSL_ENOTSQR);
    }
  else if (LU->size1 != p->size)
    {
      GSL_ERROR ("permutation length must match matrix size", GSL_EBADLEN);
    }
  else if (singular (LU))
    {
      GSL_ERROR ("matrix is singular", GSL_EDOM);
    }
  else
    {
      int status;
      const size_t N = LU->size1;
      size_t i;

      /* compute U^{-1} */
      status = gsl_linalg_tri_invert(CblasUpper, CblasNonUnit, LU);
      if (status)
        return status;

      /* compute L^{-1} */
      status = gsl_linalg_tri_invert(CblasLower, CblasUnit, LU);
      if (status)
        return status;

      /* compute U^{-1} L^{-1} */
      status = gsl_linalg_tri_UL(LU);
      if (status)
        return status;

      /* apply permutation to columns of A^{-1} */
      for (i = 0; i < N; ++i)
        {
          gsl_vector_view v = gsl_matrix_row(LU, i);
          gsl_permute_vector_inverse(p, &v.vector);
        }

      return GSL_SUCCESS;
    }
}

double
gsl_linalg_LU_det (gsl_matrix * LU, int signum)
{
  size_t i, n = LU->size1;

  double det = (double) signum;

  for (i = 0; i < n; i++)
    {
      det *= gsl_matrix_get (LU, i, i);
    }

  return det;
}

double
gsl_linalg_LU_lndet (gsl_matrix * LU)
{
  size_t i, n = LU->size1;

  double lndet = 0.0;

  for (i = 0; i < n; i++)
    {
      lndet += log (fabs (gsl_matrix_get (LU, i, i)));
    }

  return lndet;
}

int
gsl_linalg_LU_sgndet (gsl_matrix * LU, int signum)
{
  size_t i, n = LU->size1;

  int s = signum;

  for (i = 0; i < n; i++)
    {
      double u = gsl_matrix_get (LU, i, i);

      if (u < 0)
        {
          s *= -1;
        }
      else if (u == 0)
        {
          s = 0;
          break;
        }
    }

  return s;
}

static int
singular (const gsl_matrix * LU)
{
  size_t i, n = LU->size1;

  for (i = 0; i < n; i++)
    {
      double u = gsl_matrix_get (LU, i, i);
      if (u == 0) return 1;
    }

  return 0;
}

static int
apply_pivots(gsl_matrix * A, const gsl_vector_uint * ipiv)
{
  if (A->size1 < ipiv->size)
    {
      GSL_ERROR("matrix does not match pivot vector", GSL_EBADLEN);
    }
  else
    {
      size_t i;

      for (i = 0; i < ipiv->size; ++i)
        {
          size_t pi = gsl_vector_uint_get(ipiv, i);

          if (i != pi)
            {
              /* swap rows i and pi */
              gsl_vector_view v1 = gsl_matrix_row(A, i);
              gsl_vector_view v2 = gsl_matrix_row(A, pi);
              gsl_blas_dswap(&v1.vector, &v2.vector);
            }
        }

      return GSL_SUCCESS;
    }
}


/* ————————————————————————————————————————————————————————————————————————————————————————————————
Backwards elimination algorithm using Bidiag2.
———————————————————————————————————————————————————————————————————————————————————————————————————
This is a MEX function which uses the Bidiag2 PLS algorithm to estimate the cross-validated root-
mean-squared-error, RMSEcv, of an initial design matrix X, regressed upon y, together with every
possible single inactivation of a column. The function returns a matrix with RMSEcv as a function
of variable selection and PLS component where the first row corresponds to the initial variable
selection and every following row corresponds to the inactivation of variable 1 to n.

The function takes 5 inputs:
* Input 1: a [m x n] design matrix with observations as rows and variables as columns.
* Input 2: a [m x 1] vector with response values (multiple responses is not supported).
* Input 3: a [1 x 1] scalar 'A' specifying the maximum number of PLS components to calculate.
* Input 4: a [1 x 1] scalar 'MaxIter' specifying how many MonteCarlo CV iterations should be done.
* Input 5: a [1 x 1] scalar 'NoValObs' specifying how many unique responses should be in one validation fold.

The function outputs 1 variable:
* Output 1: a [(n+1) x A] matrix with RMSEcv. Variable selection along rows and components
along columns.

Example on how to compile and run from Matlab:
% Compile .C to .mexw64
>> mex -largeArrayDims -lmwblas BackwardsSelectionBidag2.c

% Run from Matlab when compiled:
>> X = rand(10000, 256);
>> y = rand(10000, 1);
>> A = 10;
>> MaxIters = 10;
>> Valobs = round(length(unique(y))/4);

>> [ RMSEcv ] = BackwardsSelectionBidag2( X , y, A, MaxIters, Valobs );

Example of compatible C compilers:
* Microsoft Visual C++ 2013 Professional (C)
* Microsoft Visual C++ 2015 Professional (C)
* Intel Parallel Studio XE 2017

Written 2017-08-14 by
petter.stefansson@nmbu.no
———————————————————————————————————————————————————————————————————————————————————————————————— */

#include <mex.h>	// Needed to communicate with matlab
#include <blas.h>	// Needed for blas functions
#include <string.h> // Needed to avoid compiler warning due to memcpy when using old compilers
#include <math.h>   // Needed to take the square-root (sqrt)
#include <time.h>   // Needed for counting CPU clock cycle which is used to set seed for rand()

/* ——————————————————————————————————— Function declarations ——————————————————————————————————— */
void PLS(const double *X, const double *y, int A, size_t m, size_t n, size_t p, double *beta,
	double *B, double *w, double *wn, double *W, double *rho, double *rhoi, double *d, double *tty,
	double *Xt, double *Ww, double *WWw, double *theta, double *thetai, double *Tt);

int randr(unsigned int min, unsigned int max);

void MarkAsVal(bool *IsVal, int NoValObs, size_t m, size_t *valm, size_t *trainm, const double *y);

void ExtractXandY(const double *X, const double *y, double *Xtrain, double *Xval, double *ytrain,
	double *yval, size_t m, size_t n, bool *IsVal);

void Pred(double *Xval, double *beta, double *yval, size_t n, size_t valm, int A, double *yhat,
	double *RMSEcv, int cvIter, int MaxIters, int ShavingIndex);

void SwapCols(double *Xtrain, double *Xval, size_t trainm, size_t valm, size_t n, int ColToShave);

/* ——————————————————————————————————— Matlab gateway start ———————————————————————————————————— */
void mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[])
{
	/* —————————————————————————— Variable type and name declaration ——————————————————————————— */
	const double *X, *y;
	double *Xtrain, *Xval, *ytrain, *yval, *beta, *yhat, *B, *w, *wn, *W, *rho, *rhoi, *d, *tty,
		   *Xt, *Ww, *WWw, *theta, *thetai, *Tt;

	int cvIter, NoValObs, MaxIters, A, ColToShave;
	bool *IsVal;
	size_t m, n, p, trainm, valm;

	/* ————————————————————  Get pointers to the input variables from Matlab ——————————————————— */
	X = mxGetPr(prhs[0]);			        // First input (X matrix).
	y = mxGetPr(prhs[1]);			        // Second input (Y vector).

	/* ————————————————————  Get input scalar values containing PLS settings ——————————————————— */
	A        = (int)mxGetScalar(prhs[2]);   // Third input (max number of components to calculate)
	MaxIters = (int)mxGetScalar(prhs[3]);   // Fourth input (Max cv-iters)
	NoValObs = (int)mxGetScalar(prhs[4]);   // Fifth input (Number of validation observations)

	/* ——————————————————————— Get the dimensions of the input variables ——————————————————————— */
	m = mxGetM(prhs[0]);                    // Number of rows in    X.
	n = mxGetN(prhs[0]);                    // Number of columns in X.
	p = mxGetN(prhs[1]);                    // Number of columns in y.

	/* ——————————————————————————— If input A is larger than n, let A = n —————————————————————— */
	if ((A > n) || (A < 1)) { A = n; }

	/* ——————————————————————————————— Specify Matlab outputs —————————————————————————————————— */
	double *RMSEcv;
	plhs[0] = mxCreateDoubleMatrix(n + 1, A, mxREAL);
	RMSEcv = mxGetPr(plhs[0]);

	/* ———— Allocate space for variables that are used multiple times and can be overwritten ——— */
	IsVal     = (bool*)malloc(sizeof(bool)     * m                  ); // [m-by-1]
	Xtrain    = (double*)malloc(sizeof(double) * m    *   n         ); // [m-by-n] (overallocated)
	Xval      = (double*)malloc(sizeof(double) * m    *   n         ); // [m-by-n] (overallocated)
	ytrain    = (double*)malloc(sizeof(double) * m                  ); // [m-by-1] (overallocated)
	yval      = (double*)malloc(sizeof(double) * m                  ); // [m-by-1] (overallocated)
	beta      = (double*)malloc(sizeof(double) * n    *   A         ); // [n-by-A]
	yhat      = (double*)malloc(sizeof(double) * m    *   A         ); // [m-by-A] (overallocated)
	B         = (double*)malloc(sizeof(double) * A    *   2         ); // [A-by-2]
	w         = (double*)malloc(sizeof(double) * n                  ); // [n-by-1]
	wn        = (double*)malloc(sizeof(double)                      ); // [1-by-1]
	W         = (double*)malloc(sizeof(double) * n    * A           ); // [n-by-A]
    rho       = (double*)malloc(sizeof(double)                      ); // [1-by-1]
	rhoi      = (double*)malloc(sizeof(double)                      ); // [1-by-1]
	d         = (double*)malloc(sizeof(double) * n                  ); // [n-by-1]
	tty       = (double*)malloc(sizeof(double)                      ); // [1-by-1]
	Xt        = (double*)malloc(sizeof(double) * n                  ); // [n-by-1]
	Ww        = (double*)malloc(sizeof(double) * A                  ); // [A-by-1] (overallocated)
	WWw       = (double*)malloc(sizeof(double) * n                  ); // [n-by-1]
	theta     = (double*)malloc(sizeof(double)                      ); // [1-by-1]
	thetai    = (double*)malloc(sizeof(double)                      ); // [1-by-1]
	Tt        = (double*)malloc(sizeof(double) * A                  ); // [A-by-1] (overallocated)

	/* Before starting set the seed of the RNG to the number of clock cycles since start.		 */
	srand(clock());

	/* CV-loop starts here																	     */
	for (cvIter = 0; cvIter < MaxIters; cvIter++) {

		/* ————————————— Mark rows of X as either validation rows or training rows ————————————— */
		MarkAsVal(IsVal, NoValObs, m, &valm, &trainm, y);

		/* —————————— Extract data from X and y and place in Xtrain/Xval/ytrain/yval ——————————— */
		ExtractXandY(X, y, Xtrain, Xval, ytrain, yval, m, n, IsVal);

		/* ————————————————————————————————————————————————————————————————————————————————————— */
		/* Shaving step 1. Evaluate matrix with all variables included.							 */
		/* ————————— Call PLS function to estimate Beta using Xtrain & ytrain —————————————————— */
		PLS( Xtrain, ytrain, A, trainm, n, p, beta, B, w, wn, W, rho, rhoi, d, tty, Xt, Ww, WWw, theta, thetai, Tt);
		/* ——————— Use beta to estimate error when predicting yval using Xval —————————————————— */
		Pred( Xval, beta, yval, n, valm, A, yhat, RMSEcv, cvIter, MaxIters, 0);

		/* ————————————————————————————————————————————————————————————————————————————————————— */
		/* Shaving step 2. Evaluate matrix with all except the last variable included.			 */
		/* ————————— Call PLS function to estimate Beta using Xtrain & ytrain —————————————————— */
		PLS(Xtrain, ytrain, A, trainm, n - 1, p, beta, B, w, wn, W, rho, rhoi, d, tty, Xt, Ww, WWw, theta, thetai, Tt);
		/* ——————— Use beta to estimate error when predicting yval using Xval —————————————————— */
		Pred(Xval, beta, yval, n - 1, valm, A, yhat, RMSEcv, cvIter, MaxIters, n);
		/* ————————————————————————————————————————————————————————————————————————————————————— */

		/* ————————————————————————————————————————————————————————————————————————————————————— */
		/* Shaving step 3. Keep last column inactive and swap it one by one with cols 0 : n-1    */
		for (ColToShave = 0; ColToShave < (n - 1); ColToShave++) {

			/* Place ColToShave at the end, effectecly inactivating it.							 */
			SwapCols(Xtrain, Xval, trainm, valm, n, ColToShave + 1);
			/* ———————— Call PLS function to estimate Beta using Xtrain & ytrain ——————————————— */
			PLS(Xtrain, ytrain, A, trainm, n - 1, p, beta, B, w, wn, W, rho, rhoi, d, tty, Xt, Ww, WWw, theta, thetai, Tt);
			/* —————— Use beta to estimate error when predicting yval using Xval ——————————————— */
			Pred(Xval, beta, yval, n - 1, valm, A, yhat, RMSEcv, cvIter, MaxIters, ColToShave + 1);
		}
	}

	/* —————————————————————————————————— Free allocated memory ———————————————————————————————— */
	free(IsVal);
	free(Xtrain);
	free(Xval);
	free(ytrain);
	free(yval);
	free(beta);
	free(yhat);
	free(B);
	free(w);
	free(wn);
	free(W);
	free(rho);
	free(rhoi);
	free(d);
	free(tty);
	free(Xt);
	free(Ww);
	free(WWw);
	free(theta);
	free(thetai);
	free(Tt);
}

/* ————————————————————————————————————————————————————————————————————————————————————————————— */
/* Function for estimating Beta using the bidag2 algorithm										 */
void PLS(const double *X, const double *y, int A, size_t m, size_t n, size_t p, double *beta,
	double *B, double *w, double *wn, double *W, double *rho, double *rhoi, double *d,
	double *tty, double *Xt, double *Ww, double *WWw, double *theta, double *thetai, double *Tt){

	/* ——————————————————————————  Variable type and name declaration —————————————————————————— */
	double *t;
	double *T;
	double *Xw;
	double *TTt;

	int a, row;
	size_t st_a;

	mwSignedIndex IntConstOne = 1;
	double one = 1.0, zero = 0.0, none = -1.0;

	/* ————————————————— Allocate memory for variables used in the calculation ————————————————— */
	t      = (double*)malloc(sizeof(double)  * m                  ); // [m-by-1]
	T      = (double*)malloc(sizeof(double)  * m        * A       ); // [m-by-A]
	Xw     = (double*)malloc(sizeof(double)  * m                  ); // [m-by-1]
	TTt    = (double*)malloc(sizeof(double)  * m                  ); // [m-by-1]

	/* ————————————————————————————————————————————————————————————————————————————————————————— */
	/* ——————————————————————————————— Start of calculation part ——————————————————————————————— */
	/* ——————————————————————————————————————————————————————————————————————————————————————————
	Step 1: w = X'*y; 																             */
	/// X         = [m-by-n]
	/// y         = [m-by-1]
	/// w         = [n-by-1]

	// Matrix-Vector multiplication -> dgemv.
	dgemv("T", &m, &n, &one, X, &m, y, &IntConstOne, &zero, w,&IntConstOne);

	/*—————————————————————————————————————————————————————————————————————————————————————————————
	Step 2: w = w / norm(w); 																     */
	/// w         = [n-by-1]
	/// wn        = [1-by-1]

	// Vector norm -> dnrm2.
	*wn = dnrm2(&n, w, &IntConstOne);
	// Inverse resulting scalar to enable multiplication instead of division.
	*wn = 1 / (*wn);
	// Vector-scalar multiplication -> dscal.
	dscal(&n, wn, w, &IntConstOne);

	/*—————————————————————————————————————————————————————————————————————————————————————————————
	Step 3: W(:,CurrentComp) = w;																 */
	/// W         = [n-by-A]
	/// w         = [n-by-1]

	// Copy n doubles from w to W.
	memcpy(W, w, sizeof(double) * n);

	/*—————————————————————————————————————————————————————————————————————————————————————————————
	Step 4: t = X*w;																             */
	/// X         = [m-by-n]
	/// w         = [n-by-1]
	/// t         = [m-by-1]

	// Matrix-Vector multiplication -> dgemv.
	dgemv("N", &m, &n, &one, X, &m, w, &IntConstOne, &zero, t, &IntConstOne);

	/*—————————————————————————————————————————————————————————————————————————————————————————————
	Step 5: rho = norm(t);																         */
	/// t         = [m-by-1]
	/// rho       = [1-by-1]

	// Vector norm->dnrm2.
	*rho = dnrm2(&m, t, &IntConstOne);

	/*—————————————————————————————————————————————————————————————————————————————————————————————
	Step 6: t = t / rho;																         */
	/// t         = [m-by-1]
	/// rho       = [1-by-1]
	/// rhoi      = [1-by-1]

	// Create inverse rho (rhoi) scalar to enable multiplication instead of division.
	*rhoi = 1 / (*rho);
	// Vector-scalar multiplication ->dscal.
	dscal(&m, rhoi, t, &IntConstOne);

	/*—————————————————————————————————————————————————————————————————————————————————————————————
	Step 7: T(:,CurrentComp) = t;														         */
	/// T         = [m-by-A]
	/// t         = [m-by-p]

	// Copy m doubles from t to T.
	memcpy(T, t, sizeof(double) * m);

	/*—————————————————————————————————————————————————————————————————————————————————————————————
	Step 8: B(1, 1) = rho;														                 */
	/// B         = [A-by-2]
	/// rho       = [1-by-1]

	B[0] = *rho;

	/*—————————————————————————————————————————————————————————————————————————————————————————————
	Step 9: d = w / rho;													                     */
	/// w         = [n-by-1]
	/// rho       = [1-by-1]
	/// rhoi      = [1-by-1]
	/// d         = [n-by-1]

	// Make d = w
	memcpy(d, w, sizeof(double) * n);

	// Then scale it with the inversed version of rho to make it w/rho.
	dscal(&n, rhoi, d, &IntConstOne);

	/*—————————————————————————————————————————————————————————————————————————————————————————————
	Step 9: beta(:, 1) = (t'*y)*d;													             */
	/// t         = [m-by-1]
	/// d         = [n-by-1]
	/// y         = [m-by-1]
	/// tty       = [1-by-1]
	/// beta(:,1) = [n-by-1]

	// (t'*y) vector-vector product -> ddot.
	*tty = ddot(&m, t, &IntConstOne, y, &IntConstOne);

	// (d*tty) vector-scalar product. (Here treated as matrix-matrix).
	dgemm("N", "N", &n, &p, &p, &one, d, &n, tty, &p, &zero, beta, &n);

	//* Start of component loop.															     */
	for (a = 1; a < A; a++) {
		/* size_t version of a (preferred by BLAS instead of int). 								 */
		st_a = (size_t)a;

		/*—————————————————————————————————————————————————————————————————————————————————————————
		Step 10: w = X'*t - rho*w;													             */
		/// w         = [n-by-1]
		/// rho       = [1-by-1]
		/// X         = [m-by-n]
		/// t         = [m-by-1]
		/// Xt        = [n-by-1]

		// Xt = X'*t matrix-vector multiplication -> dgemv.
		dgemv("T", &m, &n, &one, X, &m, t, &IntConstOne, &zero, Xt, &IntConstOne);

		// The value of rho wont be used again, overwrite with -1*rho.
		*rho = -1 * (*rho);

		// w = rho*w scalar-vector multiplication -> dscal.
		dscal(&n, rho, w, &IntConstOne);

		// w = 1*Xt + w vector times constant plus vector -> daxpy.
		daxpy(&n, &one, Xt, &IntConstOne, w, &IntConstOne);

		/*—————————————————————————————————————————————————————————————————————————————————————————
		Step 11: w = w - W(:,1:a)*(W(:,1:a)'*w);											     */
		/// W(:,1:a)   = [n-by-a]
		/// w          = [n-by-1]
		/// Ww         = [a-by-1]
		/// WWw        = [n-by-1]

		// Ww = (W(:,1:a)'*w) matrix-vector multiplication -> dgemm.
		dgemv("T", &n, &st_a, &one, W, &n, w, &IntConstOne, &zero, Ww, &IntConstOne);

		// WWw = W(:,1:a)*(W(:,1:a)'*w)  matrix-vector multiplication -> dgemm.
		dgemv("N", &n, &st_a, &one, W, &n, Ww, &IntConstOne, &zero, WWw, &IntConstOne);

		//  w ←  w + -1 * WWw constant times a vector plus a vector -> daxpy.
		daxpy(&n, &none, WWw, &IntConstOne, w, &IntConstOne);

		/*—————————————————————————————————————————————————————————————————————————————————————————
		Step 12: theta = norm(w);													             */
		/// w         = [n-by-1]
		/// theta     = [1-by-1]

		// Vector norm -> dnrm2.
		*theta = dnrm2(&n, w, &IntConstOne);

		/*—————————————————————————————————————————————————————————————————————————————————————————
		Step 13: w = w / theta;													                 */
		/// w         = [n-by-1]
		/// theta     = [1-by-1]

		// w = w * (1/theta) scalar-vector multiplication -> dscal.
		*thetai = 1 / (*theta);
		dscal(&n, thetai, w, &IntConstOne);

		/*—————————————————————————————————————————————————————————————————————————————————————————
		Step 14: W(:,a) = w;													                 */
		/// w         = [n-by-1]
		/// W         = [n-by-A]

		// Copy n elemments from w into W at offset n * a.
		memcpy(W + n * a, w, sizeof(double) * n);

		/*—————————————————————————————————————————————————————————————————————————————————————————
		Step 15: t = X*w - theta*t;													             */
		/// X         = [m-by-n]
		/// w         = [n-by-1]
		/// theta     = [1-by-1]
		/// t         = [m-by-1]
		/// Xw        = [m-by-1]

		// Xw = X*w = matrix vector multiplication -> dgemv.
		dgemv("N", &m, &n, &one, X, &m, w, &IntConstOne, &zero, Xw, &IntConstOne);

		// t = Xw - theta*t Better to solve with daxpy?
		//for (row = 0; row < m; row++){
	    //	t[row] = Xw[row] - (*theta) * t[row];
		//}
		dscal(&m, theta, t, &IntConstOne);
		dscal(&m, &none, t, &IntConstOne);
		daxpy(&m, &one, Xw, &IntConstOne, t, &IntConstOne);

		/*—————————————————————————————————————————————————————————————————————————————————————————
		Step 16: t = t - T(:,1:a)*(T(:,1:a)'*t);												 */
		/// t         = [m-by-1]
		/// T(:,1:a)  = [m-by-a]
		/// Tt        = [a-by-1]
		/// TTt       = [m-by-1]

		// Tt = (T(:,1:a)'*t) matrix-vector multiplication -> dgemv.
		dgemv("T", &m, &st_a, &one, T, &m, t, &IntConstOne, &zero, Tt, &IntConstOne);

		// TTt = T(:,1:a)*(T(:,1:a)'*t) matrix-vector multiplication -> dgemv.
		dgemv("N", &m, &st_a, &one, T, &m, Tt, &IntConstOne, &zero, TTt, &IntConstOne);

		//  t ←  t + -1 * TTt constant times a vector plus a vector -> daxpy.
		daxpy(&m, &none, TTt, &IntConstOne, t, &IntConstOne);

		/*—————————————————————————————————————————————————————————————————————————————————————————
		Step 17: rho = norm(t);													                 */
		/// t         = [m-by-1]
		/// rho       = [1-by-1]

		// Vector norm -> dnrm2.
		*rho = dnrm2(&m, t, &IntConstOne);

		/*—————————————————————————————————————————————————————————————————————————————————————————
		Step 18: t = t/rho;													                     */
		/// t         = [m-by-1]
		/// rho       = [1-by-1]

		// Update inverse version of rho.
		*rhoi = 1 / (*rho);

		// t = t * (1/rho) scalar-vector multiplication -> dscal
		dscal(&m, rhoi, t, &IntConstOne);

		/*—————————————————————————————————————————————————————————————————————————————————————————
		Step 19: T(:, a) = t;													                 */
		/// T         = [m-by-A]
		/// t         = [m-by-1]

		// Copy m elemments from t into T at offset m * a
		memcpy(T + m * a, t, sizeof(double) * m);

		/*—————————————————————————————————————————————————————————————————————————————————————————
		Step 20: B(a, 1) = rho;													                 */
		/// B         = [A-by-2]
		/// rho       = [1-by-1]

		B[a] = *rho;

		/*—————————————————————————————————————————————————————————————————————————————————————————
		Step 21: B(a - 1, 2) = theta;													         */
		/// B         = [A-by-2]
		/// theta     = [1-by-1]

		B[(a-1) + A] = *theta;

		/*—————————————————————————————————————————————————————————————————————————————————————————
		Step 22: d = (w - theta*d) / rho;													     */
		/// d         = [n-by-1]
		/// rho       = [1-by-1]
		/// theta     = [1-by-1]
		/// w         = [n-by-1]

		for (row = 0; row < n; row++){
			 d[row] = ( w[row] - (*theta) * d[row] ) / (*rho);
		}

		/*—————————————————————————————————————————————————————————————————————————————————————————
		Step 23: beta(:, a) = beta(:, a - 1) + (t'*y)*d;									     */
		/// beta      = [n-by-A]
		/// t         = [m-by-1]
		/// y         = [m-by-1]
		/// d         = [n-by-1]

		// (t'*y) vector-vector product -> ddot
		*tty = ddot(&m, t, &IntConstOne, y, &IntConstOne);

		// (tty*d) = scalar-vector product. (Here treated as matrix-matrix).
		dgemm("N", "N", &n, &p, &p, &one, d, &n, tty, &p, &zero, beta + n * a , &n);

		//  beta(:, a) ←  beta(:, a) + 1 * beta(:, a - 1) -> daxpy
		daxpy(&n, &one, beta + n * (a-1), &IntConstOne, beta + n * a, &IntConstOne);

		//for (row = 0; row < n; row++) {
			//beta[row + n * a] = beta[row + n * a] + beta[row + n * (a - 1)];
			//beta[row + n * a] = beta[row + n * (a - 1)];
		//}

    /* End of component loop.																	 */
	}

	free(t);
	free(T);
	free(Xw);
	free(TTt);
}
/* ————————————————————————————————————————————————————————————————————————————————————————————— */

/* ————————————————————————————————————————————————————————————————————————————————————————————— */
/* Function for drawing a random integer that lies within range.								 */
int randr(unsigned int min, unsigned int max) {
	return min + rand() / (RAND_MAX / (max - min + 1) + 1);
}
/* ————————————————————————————————————————————————————————————————————————————————————————————— */

/* ————————————————————————————————————————————————————————————————————————————————————————————— */
/* Function for marking rows in the data as validation or not during CV							 */
void MarkAsVal(bool *IsVal, int NoValObs, size_t m, size_t *valm, size_t *trainm, const double *y) {
	int activeobs, row, index;
	bool alreadyactive;

	/* Initialize all IsVal elements to false */
	for (row = 0; row < m; row++) {
		IsVal[row] = false;
	}

	activeobs = 0;
	while (activeobs < NoValObs) { // NOTE! this needs a check aginst input NoValObs larger than unique(y)!
								   /* Draw random rownumber between 0 and m */
		index = randr(0, m);
		alreadyactive = false;
		/* Mark all the rows with the same response as validation to keep replicates together */
		for (row = 0; row < m; row++) {
			if (y[row] == y[index]) {
				/* If IsVal was 0 it means this is a newly found observation. */
				if (IsVal[row] == false & alreadyactive == false) {
					activeobs += 1;
					alreadyactive = true;
				}
				/* If IsVal is already 1 stop looping and pick a new observation to save time */
				else if (IsVal[row] == true) {
					break;
				}
				IsVal[row] = true;
			}
		}
	}

	/* Calculate the number of rows in Xval/yval and Xtrain/ytrain so they can be allocated */
	*valm = 0;
	for (row = 0; row < m; row++) {
		if (IsVal[row] == true) {
			*valm += 1;
		}
	}
	*trainm = m - (*valm);
}
/* ————————————————————————————————————————————————————————————————————————————————————————————— */

/* ————————————————————————————————————————————————————————————————————————————————————————————— */
/* Function for populating Xtrain Xval ytrain yval given known IsVal vector					     */
void ExtractXandY(const double *X, const double *y, double *Xtrain, double *Xval, double *ytrain,
	double *yval, size_t m, size_t n, bool *IsVal) {

	int col, row, xte, xve, yve, yte;

	xte = xve = 0;
	yte = yve = 0;

	/* Outer loop over all columns of X */
	for (col = 0; col < n; col++) {
		/* Inner loop over all rows of X */
		for (row = 0; row < m; row++) {

			if (IsVal[row] == true) {
				Xval[xve] = X[row + col * m];
				xve += 1;
				if (col == 0) {
					yval[yve] = y[row];
					yve += 1;
				}
			}
			else {
				Xtrain[xte] = X[row + col * m];
				xte += 1;
				if (col == 0) {
					ytrain[yte] = y[row];
					yte += 1;
				}
			}
		}
	}
}
/* ————————————————————————————————————————————————————————————————————————————————————————————— */

/* ————————————————————————————————————————————————————————————————————————————————————————————— */
/* Function for calculating validation error using Xval and beta								 */
void Pred(double *Xval, double *beta, double *yval, size_t n, size_t valm, int A, double *yhat, double *RMSEcv, int cvIter, int MaxIters, int ShavingIndex) {

	size_t st_A;
	int col;
	double one = 1.0, zero = 0.0, none = -1.0;
	mwSignedIndex IntConstOne = 1;
	st_A = (size_t)A;

	/* yhat = Xval*beta														                     */
	dgemm("N", "N", &valm, &st_A, &n, &one, Xval, &valm, beta, &n, &zero, yhat, &valm);

	/* Loop the columns of yhat and calulate the error for each component.						 */
	for (col = 0; col < A; col++) {

		/* yhat is here overwritten and becomes epsilon	where epsilon = yhat + (-Yval).	         */
		daxpy(&valm, &none, yval, &IntConstOne, yhat + col * valm, &IntConstOne);

		/* Calculate the Mean Squared Error by taking the dot product of each error column with
		itself and then dividing by the number of observations (valm). At the same time, add it
		to the	existing value of the RMSE vector, thereby performing a summation of the errors
		of each cross-validation iteration at the same time (prevents the need of a 2D matrix).	 */
		if (ShavingIndex == 0) {
			RMSEcv[ShavingIndex + col*(n + 1)] += ddot(&valm, yhat + col * valm, &IntConstOne, yhat + col * valm, &IntConstOne) / valm;
		}
		else {
			RMSEcv[ShavingIndex + col*(n + 2)] += ddot(&valm, yhat + col * valm, &IntConstOne, yhat + col * valm, &IntConstOne) / valm;
		}
		/* If its the last cross-validation iteration, finalize RMSEcv (which right now
		represents sum(MSE,1) [folds-by-components] ) by dividing by the number of cv iterations
		and taking the square-root of the currently squared errors.								 */
		if (cvIter == MaxIters - 1) {
			if (ShavingIndex == 0) {
				RMSEcv[ShavingIndex + col*(n + 1)] = sqrt(RMSEcv[ShavingIndex + col*(n + 1)] / MaxIters);
			}
			else {
				RMSEcv[ShavingIndex + col*(n + 2)] = sqrt(RMSEcv[ShavingIndex + col*(n + 2)] / MaxIters);
			}
		}

	}

}
/* ————————————————————————————————————————————————————————————————————————————————————————————— */

/* ————————————————————————————————————————————————————————————————————————————————————————————— */
/* Function for placing a column at the and of a matrix (swapping it with the last column)       */
void SwapCols(double *Xtrain, double *Xval, size_t trainm, size_t valm, size_t n, int ColToShave) {

	mwSignedIndex IntConstOne = 1;

	/* Use dswap to swap location of two column vectors in Xtrain.								 */
	dswap(&trainm, Xtrain + (n - 1) * trainm, &IntConstOne,	Xtrain + ColToShave * trainm, &IntConstOne);

	/* Use dswap to swap location of two column vectors in Xval.								 */
	dswap(&valm, Xval + (n - 1) * valm, &IntConstOne, Xval + ColToShave * valm, &IntConstOne);
}
/* ————————————————————————————————————————————————————————————————————————————————————————————— */


/* specfunc/clausen.c
 *
 * Copyright (C) 1996, 1997, 1998, 1999, 2000 Gerard Jungman
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or (at
 * your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 */

/* Author:  G. Jungman */

#include <config.h>
#include <gsl/gsl_math.h>
#include <gsl/gsl_errno.h>
#include <gsl/gsl_sf_trig.h>
#include <gsl/gsl_sf_clausen.h>

#include "chebyshev.h"
#include "cheb_eval.c"

static double aclaus_data[15] = {
  2.142694363766688447e+00,
  0.723324281221257925e-01,
  0.101642475021151164e-02,
  0.3245250328531645e-04,
  0.133315187571472e-05,
  0.6213240591653e-07,
  0.313004135337e-08,
  0.16635723056e-09,
  0.919659293e-11,
  0.52400462e-12,
  0.3058040e-13,
  0.18197e-14,
  0.1100e-15,
  0.68e-17,
  0.4e-18
};
static cheb_series aclaus_cs = {
  aclaus_data,
  14,
  -1, 1,
  8  /* FIXME:  this is a guess, correct value needed here BJG */
};

/*-*-*-*-*-*-*-*-*-*-*-* Functions with Error Codes *-*-*-*-*-*-*-*-*-*-*-*/

int gsl_sf_clausen_e(double x, gsl_sf_result *result)
{
  const double x_cut = M_PI * GSL_SQRT_DBL_EPSILON;

  double sgn = 1.0;
  int status_red;

  if(x < 0.0) {
    x   = -x;
    sgn = -1.0;
  }

  /* Argument reduction to [0, 2pi) */
  status_red = gsl_sf_angle_restrict_pos_e(&x);

  /* Further reduction to [0,pi) */
  if(x > M_PI) {
    /* simulated extra precision: 2PI = p0 + p1 */
    const double p0 = 6.28125;
    const double p1 = 0.19353071795864769253e-02;
    x = (p0 - x) + p1;
    sgn = -sgn;
  }

  if(x == 0.0) {
    result->val = 0.0;
    result->err = 0.0;
  }
  else if(x < x_cut) {
    result->val = x * (1.0 - log(x));
    result->err = x * GSL_DBL_EPSILON;
  }
  else {
    const double t = 2.0*(x*x / (M_PI*M_PI) - 0.5);
    gsl_sf_result result_c;
    cheb_eval_e(&aclaus_cs, t, &result_c);
    result->val = x * (result_c.val - log(x));
    result->err = x * (result_c.err + GSL_DBL_EPSILON);
  }

  result->val *= sgn;

  return status_red;
}

/*-*-*-*-*-*-*-*-*-* Functions w/ Natural Prototypes *-*-*-*-*-*-*-*-*-*-*/

#include "eval.h"

double gsl_sf_clausen(const double x)
{
  EVAL_RESULT(gsl_sf_clausen_e(x, &result));
}