/**
 * Set of routines to handle absolute flux calibration of a spectrum
 * structure.
 * The response curves can be in ASCII or (preferably) in FITS
 * binary format (first extension). ASCI format is simply three
 * space separated colums (WAVELENGTH, THROUGHPUT, ERROR). The FITS
 * binary table must contain three columns with names WAVELENGTH,
 * SENSITIVITY, ERROR. The throughput should be DN/s per Erg/cm^2/s/A.
 */

#include <gsl/gsl_matrix.h>
#include <gsl/gsl_vector.h>
#include <gsl/gsl_interp.h>
#include <gsl/gsl_roots.h>

#include "aXe_grism.h"
#include "disp_conf.h"
#include "fringe_conf.h"
#include "spc_wl_calib.h"
#include "spc_resp.h"

#define MIN(x,y) (((x)<(y))?(x):(y))
#define MAX(x,y) (((x)>(y))?(x):(y))

/**
 * Function: get_response_function_from_FITS
 * This function reads a FITS table, reads the WAVELENGTH,
 * SENSITIVITY, and ERROR columns and returns a spectrum
 * structure which has been populated with this throughput.
 *
 * Parameters:
 * @param filename - a pointer to a char array containing the
 *                   name of an existing FITS file
 * @param hdunum   - the extension number to read (2 for STSDAS tables)
 *
 * Returns:
 * @return res     - a pointer to a newly allocated spectrum structure
 */
spectrum *get_response_function_from_FITS(char filename[], int hdunum)
{
  fitsfile *input;
  int f_status = 0;
  int hdutype, anynul;
  double *lambda, *resp, *error;
  double tmp;
  long numrows;
  int colnum, i;
  spectrum *res;

  //  Open the file for reading
  fits_open_file (&input, filename, READONLY, &f_status);
  if (f_status)
    {
      ffrprt (stderr, f_status);
      aXe_message (aXe_M_FATAL, __FILE__, __LINE__,
                   "get_response_function_fromFITS: "
                   "Could not open" " file: %s",
                   filename);
    }
  /* Move to first hdu */
  fits_movabs_hdu (input, hdunum, &hdutype, &f_status);
  if (f_status)
    {
      ffrprt (stderr, f_status);
      aXe_message (aXe_M_FATAL, __FILE__, __LINE__,
                   "get_response_function_fromFITS: "
                   "Could not read extention %d from file: %s",
                   hdunum, filename);
    }

  /* Get number of rows */
  fits_get_num_rows (input, &numrows, &f_status);
  if (f_status) {
    ffrprt (stderr, f_status);
    aXe_message (aXe_M_FATAL, __FILE__, __LINE__,
                 "get_response_function_fromFITS: "
                 "Could not determine the number of rows in"
                 " table %s",filename);
  }

  /* Allocate temporary memory space */
  lambda = (double *) malloc(numrows*sizeof(double));
  if (!lambda) {
    aXe_message (aXe_M_ERROR, __FILE__, __LINE__,
                 "Memory allocation failed");
  }
  resp = (double *) malloc(numrows*sizeof(double));
  if (!resp) {
    aXe_message (aXe_M_ERROR, __FILE__, __LINE__,
                 "Memory allocation failed");
  }
  error = (double *) malloc(numrows*sizeof(double));
  if (!error) {
    aXe_message (aXe_M_ERROR, __FILE__, __LINE__,
                 "Memory allocation failed");
  }

  /**************************/
  /* Read the WAVELENGTH column */
  /**************************/
  /* Get column number */
  fits_get_colnum (input, CASEINSEN, "WAVELENGTH", &colnum, &f_status);
  if (f_status)
    {
      ffrprt (stderr, f_status);
      aXe_message (aXe_M_FATAL, __FILE__, __LINE__,
                   "get_response_function_fromFITS: "
                   "Could not determine WAVELENGTH column number in "
                   " table %s",filename);
    }

  /* Read the data */
  fits_read_col (input, TDOUBLE, colnum, 1, 1, numrows, NULL, lambda,
                    &anynul, &f_status);
  if (f_status)
    {
      ffrprt (stderr, f_status);
      aXe_message (aXe_M_FATAL, __FILE__, __LINE__,
                   "get_response_function_fromFITS: "
                   "Could not read content of WAVELENGTH column "
                   " from BINARY table %s",filename);
    }

  /**************************/
  /* Read the SENSITIVITY column */
  /**************************/
  /* Get column number */
  fits_get_colnum (input, CASEINSEN, "SENSITIVITY", &colnum, &f_status);
  if (f_status)
    {
      ffrprt (stderr, f_status);
      aXe_message (aXe_M_FATAL, __FILE__, __LINE__,
                   "get_response_function_fromFITS: "
                   "Could not determine SENSITIVITY column number in "
                   " table %s",filename);
    }
  /* Read the data */
  fits_read_col (input, TDOUBLE, colnum, 1, 1, numrows, NULL, resp,
                    &anynul, &f_status);
  if (f_status)
    {
      ffrprt (stderr, f_status);
      aXe_message (aXe_M_FATAL, __FILE__, __LINE__,
                   "get_response_function_fromFITS: "
                   "Could not read content of SENSITIVITY column "
                   " from BINARY table %s",filename);
    }
  /**************************/
  /* Read the ERROR column */
  /**************************/
  /* Get column number */
  fits_get_colnum (input, CASEINSEN, "ERROR", &colnum, &f_status);
  if (f_status)
    {
      ffrprt (stderr, f_status);
      aXe_message (aXe_M_FATAL, __FILE__, __LINE__,
                   "get_response_function_fromFITS: "
                   "Could not determine ERROR column number in "
                   " table %s",filename);
    }
  /* Read the data */
  fits_read_col (input, TDOUBLE, colnum, 1, 1, numrows, NULL, error,
                    &anynul, &f_status);
  if (f_status)
    {
      ffrprt (stderr, f_status);
      aXe_message (aXe_M_FATAL, __FILE__, __LINE__,
                   "get_response_function_fromFITS: "
                   "Could not read content of ERROR column "
                   " from BINARY table %s",filename);
    }

  /* Allocate memory */
  res = (spectrum *) malloc(sizeof(spectrum));
  if(!res) {
      aXe_message (aXe_M_ERROR, __FILE__, __LINE__,
                   "Memory allocation failed");
  }
  res->spec = (spc_entry *) malloc(numrows*sizeof(spc_entry));
  if(!(res->spec)) {
      aXe_message (aXe_M_ERROR, __FILE__, __LINE__,
                   "Memory allocation failed");
  }

  for (i=0;i<numrows;i++) {
    res->spec[i].lambda_mean = lambda[i];
    res->spec[i].flux = resp[i];
    res->spec[i].ferror = error[i];
  }

  res->spec_len=i;

  // define the minimum and maximum wavelength
  res->lambdamin=res->spec[0].lambda_mean;
  res->lambdamax=res->spec[numrows-1].lambda_mean;

  // check whether the wavelength is raising.
  // switch minimum and maximum for falling spectrum
  if (res->lambdamin > res->lambdamax)
    {
      tmp = res->lambdamin;
      res->lambdamin = res->lambdamax;
      res->lambdamax = tmp;
    }

  fits_close_file(input,&f_status);
  if (f_status) {
      aXe_message (aXe_M_ERROR, __FILE__, __LINE__,
                   "Could not close %s",filename);
  }

  free(lambda);
  free(resp);
  free(error);

  return res;
}

/**
 * Function: apply_response_function
 * This function applies a thoughput curve to a given spectrum
 * by dividing the count attribute of the spectrum. The flux
 * and ferror attributes of the spectrum are populated.
 * gsl spline interpolation is used to compute the throughput
 * curve at the needed spectral wavelengths.
 *
 * Parameters:
 * @param spec    - a pointer to an existing spectrum array
 * @param resp    - a pointer to a spectrum structure containing
 *                  a throughput curve previously loaded
 * @param exptime - the exposure time
 * @param gain    - the gain
 *
 * Returns
 * @return -
 */
void
apply_response_function(spectrum *spec, spectrum *resp, const int quant_cont)
{
  gsl_interp_accel *acc1 = gsl_interp_accel_alloc();
  gsl_spline *spline1 = gsl_spline_alloc (gsl_interp_cspline, resp->spec_len);

  gsl_interp_accel *acc2 = gsl_interp_accel_alloc();
  gsl_spline *spline2 = gsl_spline_alloc (gsl_interp_cspline, resp->spec_len);

  double *x1,*y1, *x2, *y2;
  int i,j;
  double r1, r2, fr, fr1, fr2;
  double min_l, max_l;
  // test
  // int nguess;
  // double tval=0.0;

  if ((int) resp->spec_len <= 0){
    aXe_message(aXe_M_ERROR, __FILE__, __LINE__,
                "apply_response_function: response spec length invalid: %i", resp->spec_len);
      gsl_spline_free(spline1);
      gsl_interp_accel_free(acc1);
      gsl_spline_free(spline2);
      gsl_interp_accel_free(acc2);
      return;
  }
  if ( spec==NULL)
    {
      aXe_message (aXe_M_WARN4, __FILE__, __LINE__,
                   "apply_response_function: spectra empty.");
      gsl_spline_free(spline1);
      gsl_interp_accel_free(acc1);
      gsl_spline_free(spline2);
      gsl_interp_accel_free(acc2);
      return;
    }

  x1 = malloc((double)(resp->spec_len) *sizeof(double));
  y1 = malloc((double)(resp->spec_len) *sizeof(double));
  x2 = malloc((double)(resp->spec_len) *sizeof(double));
  y2 = malloc((double)(resp->spec_len) *sizeof(double));

  min_l = 1e32;
  max_l = -1e32;

  for (i=0;i<resp->spec_len; i++) {
    if (resp->spec[i].lambda_mean > max_l) {
     max_l = resp->spec[i].lambda_mean;
    }
    if (resp->spec[i].lambda_mean < min_l) {
     min_l = resp->spec[i].lambda_mean;
    }

    x1[i] = resp->spec[i].lambda_mean;
    y1[i] = resp->spec[i].flux - resp->spec[i].ferror;
    x2[i] = resp->spec[i].lambda_mean;
    y2[i] = resp->spec[i].flux + resp->spec[i].ferror;
  }

  gsl_spline_init (spline1, x1, y1, resp->spec_len);
  gsl_spline_init (spline2, x2, y2, resp->spec_len);
  // test
  // nguess = 0;

  for (j=0; j<spec->spec_len; j++) {
    fprintf(stderr,"%f %f %f\n",spec->spec[j].lambda_mean, min_l, max_l);
    if ((spec->spec[j].lambda_mean>=min_l) && (spec->spec[j].lambda_mean <= max_l)) {
    //     fprintf(stderr,"lambda mean between min and max\n\t%f : %f %f\n", spec->spec[j].lambda_mean, min_l, max_l);
    r1 = gsl_spline_eval(spline1, spec->spec[j].lambda_mean, acc1);
    r2 = gsl_spline_eval(spline2, spec->spec[j].lambda_mean, acc2);
   }
   else {
     r1 = 0;
     r2 = 0;
   }

    /* Need to divide by the pixel width of the
       wavelength calibration at the given lambda */

    if ((r2+r1)!=0) {

      // test
      // spec->spec[j].flux = spec->spec[j].count / tval;
      spec->spec[j].flux = spec->spec[j].count / ((r2+r1)/2.);
      spec->spec[j].flux = spec->spec[j].flux/spec->spec[j].dlambda;
      // fprintf(stdout, "# %f %f %f\n", spec->spec[j].lambda_mean, spec->spec[j].dlambda, spec->spec[j].weight);

      if (quant_cont && (int)spec->spec[j].contam != -1)
        {
          // test
          //      spec->spec[j].contam = spec->spec[j].contam / tval;
          spec->spec[j].contam = spec->spec[j].contam / ((r2+r1)/2.);
          spec->spec[j].contam = spec->spec[j].contam/spec->spec[j].dlambda;
        }
    } else {
      spec->spec[j].flux = GSL_NAN;
    }

    fr1 = spec->spec[j].error / spec->spec[j].count; /* frac. error */
    fr2 = fabs((r2-r1)/((r1+r2)/2.0)); /* frac. error in resp. */
    fr = sqrt(fr1*fr1+fr2*fr2);
    spec->spec[j].ferror = fabs(spec->spec[j].flux) * fr;
  }

  gsl_spline_free(spline1);
  gsl_interp_accel_free(acc1);
  gsl_spline_free(spline2);
  gsl_interp_accel_free(acc2);
  free(x1);
  free(x2);
  free(y1);
  free(y2);

}

/**
 * Function: apply_response_functionII
 * This function applies a thoughput curve to a given spectrum
 * by dividing the count attribute of the spectrum. The flux
 * and ferror attributes of the spectrum are populated.
 *
 * Parameters:
 * @param spec       - a pointer to an existing spectrum array
 * @param resp_func  - a pointer to a spectrum structure containing
 *                     a throughput curve previously loaded
 * @param quant_cont - the exposure time
 *
 * Returns
 * @return -
 */
void
apply_response_functionII(spectrum *spec, response_function *resp_func, const int quant_cont)
{
  long j;
  double fr, fr1, fr2;

  double *resp_vals = malloc( 2 * sizeof(double));

  // immediately return
  // an empty spectrum
  if ( spec==NULL)
    {
      aXe_message (aXe_M_WARN4, __FILE__, __LINE__,
                   "apply_response_function: spectra empty.");
      return;
    }

  // go over all elements
  for (j=0;j<spec->spec_len;j++)
    {
      // get the sensitivity value plus error
      get_response_values(resp_func, spec->spec[j].lambda_mean, resp_vals);

      // make sure you can compute the flux value
      if (resp_vals[0] != 0.0)
        {
          // compute the flux value
          spec->spec[j].flux = spec->spec[j].count / resp_vals[0];
          spec->spec[j].flux = spec->spec[j].flux/spec->spec[j].dlambda;

          // compute the flux error
          fr1 = spec->spec[j].error / spec->spec[j].count; /* frac. error */
          fr2 = fabs(resp_vals[1] / resp_vals[0]); /* frac. error in resp. */
          fr = sqrt(fr1*fr1+fr2*fr2);
          spec->spec[j].ferror = fabs(spec->spec[j].flux) * fr;

          // check whether the contamination value is necessary
          if (quant_cont && (int)spec->spec[j].contam != -1)
            {
              // compute the contamination value
              spec->spec[j].contam = spec->spec[j].contam / resp_vals[0];
              spec->spec[j].contam = spec->spec[j].contam/spec->spec[j].dlambda;
            }
        }
      else
        {
          // set the flux and the error to NaN
          spec->spec[j].flux = GSL_NAN;
          spec->spec[j].ferror = GSL_NAN;
        }
    }

  // free the memory
  free(resp_vals);
}

/**
 * Function: get_smooth_pars_for_beam
 * The function determines the smoothing parameters for an object beam
 * and a given indicator. If smoothed calibration is requested, the source
 * size (in dispersion direction) is compared to the poin-like object size.
 * For non poin-like objects, the adjustment factor is extracted and
 * the smoothing size from the object is derived.
 *
 * Parameters:
 * @param conf        - the configuarion file structure
 * @param smooth_conv - boolean for smooth conversion
 * @param actbeam     - the beam to be calibrated
 *
 * Returns
 * @return smooth_pars - the paramters used in the smoothing
 */
d_point
get_smooth_pars_for_beam(const aperture_conf *conf, const int smooth_conv, beam actbeam)
  {
    d_point smooth_pars;

    // initialize the return
    smooth_pars.x = -1.0;
    smooth_pars.y = -1.0;

    // if there is nothing to
    // do, return the default
    if (!smooth_conv)
      return smooth_pars;

    // check whether the relevant information
    // does exist
    if (smooth_conv && actbeam.slitgeom[2] < 0.0)
      aXe_message (aXe_M_FATAL, __FILE__, __LINE__,
          "Smoothed flux conversion is impossible"
          " since the OAF\ndoes not contain the slit width!\n");

    //fprintf(stdout, "smoothing factor: %f, point-like size: %f\n", conf->smfactor, conf->pobjsize);
    // check whether the objects is smaller than a psf-object
    // in dispersion direction
    if (conf->pobjsize < actbeam.slitgeom[2])
      {
        // compute the smoothing size from the objects;
        // transfer the adjustment factor
        smooth_pars.x = pow((actbeam.slitgeom[2]* actbeam.slitgeom[2]) - (conf->pobjsize*conf->pobjsize), 0.5);
        smooth_pars.y = conf->smfactor;
      }

    // return the
    // smoothing parameters
    return smooth_pars;
  }

/**
 * Function: check_conf_for_smoothing
 * The functions checks whether a smoothed flux conversion
 * is possible or not. In case that keywords in the configuration
 * files are missing, it is NOT possible, and 0 is returned.
 *
 * Parameters:
 * @param conf        - the configuarion file structure
 * @param smooth_conv - boolean for smooth conversion
 *
 * Returns
 * @return is_possible - the paramters used in the smoothing
 */
int
check_conf_for_smoothing(const aperture_conf *conf, const int smooth_conv)
  {
    int is_possible=1;

    // make sure that, if smoothed covnersion is requested,
    // a point-like objects size and an adjustment factor is defined
    if (smooth_conv && (conf->pobjsize < 0.0 || conf->smfactor < 0.0))
      // change the switch
      is_possible = 0;

    // return the pointer
    return is_possible;
  }

/**
 * Function: apply_smoothed_response
 * Convert a spectrum from e/s to flux units in f_lambda. The conversion is done
 * using smoothed sensitivity values that take the object width into account
 * by smoothing the sensitivity function. Gaussian smoothing is applied, taking
 * into account that the original calibration function was derived for a point
 * like object with a finite size.
 *
 * Parameters:
 * @param spec           - a pointer to an existing spectrum array
 * @param resp_func      - a pointer to a spectrum structure containing
 *                         a throughput curve
 * @param quant_cont     - indicator for quantitative contamination
 * @param bwidth         - the B_IMAGE value of the beam
 * @param psf_ext        - the width of a point-like object
 * @param wl_calibration - the wavelength calibration structure
 *
 * Returns
 * @return -
 */
void
apply_smoothed_response(const calib_function *wl_calibration, const int for_grism,
                        const int quant_cont, response_function *resp_func,
                        const d_point smooth_pars, spectrum *spec)
{
  long j;
  double fr, fr1, fr2;
  //double sigma_wav=10.0;

  double *resp_vals = malloc( 2 * sizeof(double));
  //double *tmp_vals  = malloc( 2 * sizeof(double));

  gsl_vector *weights;

  // allocate memory for the vectors
  weights = gsl_vector_alloc(2 * RESP_SMOOTH_LENGTH + 1);

  // immediately return
  // if an empty spectrum
  if ( (spec==NULL) || (spec->spec_len == 0))
    {
      aXe_message (aXe_M_WARN4, __FILE__, __LINE__,
                   "apply_response_function: spectra empty.");
      return;
    }

  // fill the weight vector
  // for smoothing
  fill_weight(weights);

  // output to screen
  fprintf(stdout, "aXe_PET2SPC: Gaussian smoothing in tracelength: %.2f * %.2f = %.2f pix\n", smooth_pars.x, smooth_pars.y, smooth_pars.x * smooth_pars.y);

  // go over all elements
  for (j=0;j<spec->spec_len;j++)
    {
      // get the smoothed sensitivity value plus error
      get_smoothed_response(spec->spec[j].lambda_mean, smooth_pars, wl_calibration,
                            for_grism, weights, resp_func, resp_vals);
      // make sure you can compute the flux value
      if (resp_vals[0] != 0.0)
        {
          // compute the flux value
          spec->spec[j].flux = spec->spec[j].count / resp_vals[0];
          spec->spec[j].flux = spec->spec[j].flux/spec->spec[j].dlambda;

          // compute the flux error
          fr1 = spec->spec[j].error / spec->spec[j].count; /* frac. error */
          fr2 = fabs(resp_vals[1] / resp_vals[0]); /* frac. error in resp. */
          fr = sqrt(fr1*fr1+fr2*fr2);
          spec->spec[j].ferror = fabs(spec->spec[j].flux) * fr;

          // check whether the contamination value is necessary
          if (quant_cont && (int)spec->spec[j].contam != -1)
            {
              // compute the contamination value
              spec->spec[j].contam = spec->spec[j].contam / resp_vals[0];
              spec->spec[j].contam = spec->spec[j].contam/spec->spec[j].dlambda;
            }
        }
      else
        {
          // set the flux and the error to NaN
          spec->spec[j].flux = GSL_NAN;
          spec->spec[j].ferror = GSL_NAN;
        }
    }

  // free the memory
  free(resp_vals);
  gsl_vector_free(weights);
}

/**
 * Function: find_wavelength
 * For a given trace-length value, the function returns the difference
 * between the actual and the targeted wavelength.
 * The function is designed t be called by a gsl root-solving procedure.
 *
 * Parameters:
 * @param x        - guess value for the tracelength
 * @param params   - structure with input parameters
 *
 * Returns
 * @return wav_zero - difference between actual and targeted wavelength
 */
double
find_wavelength(double x, void *params)
  {
    double wav_zero;

    // extract the components from the parameter-structure
    const calib_function *wl_calib = ((trlength_search *) params)->wl_calibration;
    double wavelength        = ((trlength_search *) params)->wavelength;

    // compute the function value
    wav_zero = wl_calib->func (x, wl_calib->order, wl_calib->coeffs) - wavelength;

    // return the function value
    return wav_zero;
  }

/**
 * Function: get_tlength_prismwav
 * The function determines the tracelength value for a given wavelength
 * value and dispersion function. Should be called only for a prism dispersion.
 * The gsl root solver is used.
 *
 * Parameters:
 * @param wavelength     - the wavelength value
 * @param wl_calibration - the dispersion solution
  *
 * Returns
 * @return wav_zero - difference between actual and targeted wavelength
 */
double
get_tlength_prismwav(const double wavelength, const calib_function *wl_calibration)
{
  int iter=0;
  int status;

  d_point x_interv;

  double tr_length=0.0;

  // define and initialize the solver
  const gsl_root_fsolver_type *T = gsl_root_fsolver_brent;
  gsl_root_fsolver            *s = gsl_root_fsolver_alloc (T);

  gsl_function F;
  trlength_search *tr_pars;

  // allocate and fill the parameters
  tr_pars = (trlength_search *) malloc(sizeof(trlength_search));
  tr_pars->wl_calibration = wl_calibration;
  tr_pars->wavelength     = wavelength;

  // fille the GSL-function
  F.function = &find_wavelength;
  F.params =   tr_pars;

  // initialize the intervall variable
  x_interv.x = gsl_vector_get(wl_calibration->pr_range, 0) + wl_calibration->coeffs[0];
  x_interv.y = gsl_vector_get(wl_calibration->pr_range, 1) + wl_calibration->coeffs[0];

  // set the boundaries for the solver
  gsl_root_fsolver_set (s, &F, x_interv.x, x_interv.y);

  // iterate to find the zeropoint
  do
    {
      // increment the iteration counter
      iter++;

      // iterate on the solver
      status = gsl_root_fsolver_iterate (s);

      // get a new guess from the solver
      tr_length = gsl_root_fsolver_root (s);

      // derive and set new boundaries
      x_interv.x = gsl_root_fsolver_x_lower (s);
      x_interv.y = gsl_root_fsolver_x_upper (s);

      // check the accuracy
      status = gsl_root_test_interval (x_interv.x, x_interv.y,
                                       0, 0.0001);
    }
  // check for the break condition
  while (status == GSL_CONTINUE && iter < MAX_ITER_TL);

  // free the memory
  free(tr_pars);

  // free the memory
  gsl_root_fsolver_free (s);

  // return the result
  return tr_length;
}

/**
 * Function: get_tlength_prismwav
 * The function determines the tracelength value for a given wavelength
 * value and dispersion function. Should be called only for a prism dispersion.
 * The gsl root solver is used.
 *
 * Parameters:
 * @param wavelength     - the wavelength value
 * @param wl_calibration - the dispersion solution
 * @param for_grism      - boolean to indicate grism/prism dispersion solution
 *
 * Returns
 * @return wav_zero - difference between actual and targeted wavelength
 */
double
get_central_tracelength(const double wavelength, const calib_function *wl_calibration,
                        const int for_grism)
  {
    double tl_central=0.0;
    double a, b, c;

    // checc whether it's a grism
    if (for_grism)
      {
        // solve a linear dispersion
        if (wl_calibration->order == 1)
          {
            tl_central = (wavelength - wl_calibration->coeffs[0]) / wl_calibration->coeffs[1];
          }
        // solve a quadratic dispersion
        else if (wl_calibration->order == 2)
          {
            // make the quantities
            a = wl_calibration->coeffs[2];
            b = wl_calibration->coeffs[1];
            c = wl_calibration->coeffs[0] - wavelength;

            // compute the central tracelength
            //tl_central = (-b + sqrt(b*b - 4.0*a*c)) / (2.0*c);
            tl_central = (-b + sqrt(b*b - 4.0*a*c)) / (2.0*a);
            //fprintf(stdout, "I should be here... %g %g %g %g\n", tl_central, a, b, c);
          }
        else
          {
            // give error message
            aXe_message (aXe_M_FATAL, __FILE__, __LINE__,
                "Smoothed flux conversion is impossible for grism "
                "data with dispersion solution of order: %i", wl_calibration->order);
          }
      }
    else
      {
        // use the root finder to get the tace-length value
        // for the given wavelength
        if (isnan(wavelength))
          tl_central = GSL_NAN;
        else
          tl_central = get_tlength_prismwav(wavelength, wl_calibration);
      }

    // return the central
    // tracelength value
    return tl_central;
  }

/**
 * Function: get_smoothed_response
 * The function determine a smoothe sensitivity value and its error.
 * Gaussian smoothing is applied, but the Gaussian weights are delivered
 * as a function parameter (for perfomrance reasons). The function detemines
 * the response values over the weight window and then combines these values
 * with the weights to get one weighted sensitivity value at the desired
 * wavelength.
 *
 * Parameters:
 * @param wavelength     - the central wavelength
 * @param smooth_pars    - the smoothing parameters
 * @param calib_function - the wavelength calibration function
 * @param for_grism      - boolean to indicate grism/prism solution
 * @param weights        - the weight vector
 * @param resp_func      - the response function
 * @param resp_vals      - vector for the response values
 *
 * Returns
 * @return -
 */
void
get_smoothed_response(const double wavelength, const d_point smooth_pars,
                           const calib_function *wl_calibration, const int for_grism,
                           const gsl_vector *weights, response_function *resp_func, double *resp_vals)
{
  int j;

  double tl_incr;
  double tl_act;
  double tl_central;
  double lambda_act;

  gsl_vector *pixvalues;
  gsl_vector *errvalues;
  gsl_vector *pmask;

  double *tmp_vals = malloc( 2 * sizeof(double));

  // allocate memory for the vectors
  pixvalues = gsl_vector_alloc(2 * RESP_SMOOTH_LENGTH + 1);
  errvalues = gsl_vector_alloc(2 * RESP_SMOOTH_LENGTH + 1);
  pmask     = gsl_vector_alloc(2 * RESP_SMOOTH_LENGTH + 1);

  // set all mask points to zero
  gsl_vector_set_all(pmask, 0.0);

  // determine the tracelength value
  tl_central = get_central_tracelength(wavelength, wl_calibration, for_grism);

  // determine the wavelength increments
  //wav_incr = (double)RESP_SMOOTH_NSIG * sigma_wav / (double)RESP_SMOOTH_LENGTH;
  tl_incr = smooth_pars.x * smooth_pars.y * (double)RESP_SMOOTH_NSIG / (double)RESP_SMOOTH_LENGTH;

  // set the initial wavelength value
  //value = wavelength - sigma_wav * (double)RESP_SMOOTH_NSIG;
  tl_act = tl_central - 1.0 * smooth_pars.x * smooth_pars.y * (double)RESP_SMOOTH_NSIG;

  // print the starting and central wavelength
  //fprintf(stdout, " %.5g < %.5g",
  //    wl_calibration->func(tl_act, wl_calibration->order, wl_calibration->coeffs),
  //    wl_calibration->func(tl_central, wl_calibration->order, wl_calibration->coeffs));

  // go over the array
  for (j=0; j < (int)pixvalues->size; j++)
    {
      // determine the wavelength value
      lambda_act = wl_calibration->func(tl_act, wl_calibration->order, wl_calibration->coeffs);

      // check whether the actual wavelength is inside
      // the defined sensitivity interval
      if  (lambda_act < resp_func->xmin || lambda_act > resp_func->xmax)
        {
          // for outside values:
          // set everything to zero
          gsl_vector_set(pixvalues, j, 0.0);
          gsl_vector_set(errvalues, j, 0.0);
          gsl_vector_set(pmask, j, 0.0);
        }
      else
        {
          // for inside values:
          // compute and set the response
          get_response_values(resp_func, lambda_act, tmp_vals);
          gsl_vector_set(pixvalues, j, tmp_vals[0]);
          gsl_vector_set(errvalues, j, tmp_vals[1]);
          gsl_vector_set(pmask, j, 1.0);
        }

      // increment trace length
      tl_act += tl_incr;
    }
  // print the ending wavelength
  //fprintf(stdout, " > %.5g;", wl_calibration->func(tl_act, wl_calibration->order, wl_calibration->coeffs));

  // get the weighted response values
  get_weighted_sensitivity(pixvalues, errvalues, weights, pmask, resp_vals);

  // free the memory
  free(tmp_vals);
  gsl_vector_free(pixvalues);
  gsl_vector_free(errvalues);
  gsl_vector_free(pmask);
}

/**
 * Function: get_weighted_sensitivity
 * Combines the weigh values, the sensitivity and error at the weight point
 * to a single sensitivity value plus error. A mask array defines at which
 * points values do exist.
 *
 * Parameters:
 * @param pixvalues - the central wavelength
 * @param errvalues - the Gaussian sigma used in smoothing
 * @param weights   - the weight vector
 * @param pmask     - the response function
 * @param resp_vals - vector for the response values
 *
 * Returns
 * @return
 */
void
get_weighted_sensitivity(const gsl_vector *pixvalues, const gsl_vector *errvalues, const gsl_vector *weights,
                         const gsl_vector *pmask, double *resp_vals)
{
  int index;

  // initialize the total
  // sum and weight
  double val_sum = 0.0;
  double err_sum = 0.0;
  double www=0.0;

  // go over all indices of an array
  for (index=0; index < (int)pixvalues->size; index ++)
    {
      // check whether the index is masked
      if (gsl_vector_get(pmask, index) != 0.0)
        {
          // enhance the total sum
          val_sum += gsl_vector_get(pixvalues, index) * gsl_vector_get(weights, index);
          err_sum += gsl_vector_get(errvalues, index) * gsl_vector_get(errvalues, index)
            * gsl_vector_get(weights, index) * gsl_vector_get(weights, index);

          // enhance the total weight
          www += gsl_vector_get(weights, index);
        }
    }

  // check whether there
  // were weights at all
  if (www)
    {
      // return the total sum,
      // divided by the total weight
      resp_vals[0] =  val_sum / www;
      resp_vals[1] =  sqrt(err_sum) / www;
    }
  else
    {
      // set zero sensitivity
      resp_vals[0] = 0.0;
      resp_vals[1] = 0.0;
    }
}

/**
 * Function: fill_weight
 * The function fills an array with Gaussian values.
 * The number of points and the extend are all defined in
 * global variables in the h-file.
 *
 * Parameters:
 * @param weights   - the weight vector
 *
 * Returns
 * @return -
 */
void
fill_weight(gsl_vector *weights)
{
  int j;

  double d_incr;
  double value;

  // determine the increments
  d_incr = (double)RESP_SMOOTH_NSIG / (double)RESP_SMOOTH_LENGTH;

  // compute the initial exponent
  // go over the weights vector
  value = -(double)RESP_SMOOTH_NSIG;
  for (j=0; j < (int)weights->size; j++)
    {
      // compute and set the weight
      gsl_vector_set(weights, j, exp(-1.0*value * value));

      // increment the exponent
      value += d_incr;
    }
}

/**
 * Function: get_troughput_table_name
 * Parses a configuration file and get the name of the SENSITIVITY
 * curve defined for a wanted beamID
 *
 * Parameters:
 * @param filename   - a pointer to a string containing the name of the
 *                     configuration file
 * @param beamID     - the beam ID number (see BEAM())
 * @param table_name - a pointer pointing to a string to contain the name
 *                     of the file.
 *
 * Returns:
 * @return -
 */
void
get_troughput_table_name(char *filename, int beamID, char *table_name)
{
  char beam[MAXCHAR];
  char name[MAXCHAR];
  int i=0;

  struct CfgStrings ThrConfig[] = {
    {NULL, NULL},
    {NULL,NULL}
  };
  ThrConfig[0].name = beam;
  sprintf (beam, "SENSITIVITY_%c", BEAM(beamID));

  CfgRead (filename, ThrConfig);
  if ((ThrConfig[0].name == beam) && (ThrConfig[0].data != NULL))
    {
      strcpy(name,ThrConfig[0].data);
    } else {
      aXe_message (aXe_M_FATAL, __FILE__, __LINE__,
                   "get_throughput_table_name: %s was not found in "
                   "%s",beam,filename);
    }

  // release memory
  i=0;
  while(ThrConfig[i].name!=NULL)
    free(ThrConfig[i++].data);

  strcpy(table_name,name);
}

/**
 * Function: get_response_value_plus
 * The function determines the sensitivity at a given wavelength
 * in a sensitivity curve. A guess value helps to start searching
 * around the estimated position of the wavelength in the vector
 * of the sensitifity curve. This should considerably speed up the
 * code. the guess value is updated in case that a sensitivity
 * value could be determined. The return value is interpolated
 * between the adjacent tabulated values.
 *
 * Parameters:
 * @param  resp       - the sensitivity spectrum
 * @param  wavelength - the wavelength to determine the sensitivity for
 * @param  nguess     - the guess value for the array to start searching from
 *
 * Returns:
 * @return ret        - the sensitivity value at the input wavelength
 */
double
get_response_value_plus(const spectrum *resp, const double wavelength,
                        int *nguess)
{
  double ret;
  double factor;

  int nact;

  // check whether the wavelength is within the range of
  //the sensitivity curve return GSL_NAN if not
  if (wavelength < resp->lambdamin || wavelength > resp->lambdamax)
    return GSL_NAN;

  // check whether you have to search upwards or downwards
  if (wavelength >= resp->spec[*nguess].lambda_mean)
    {

      // in case that you search upwards, go up
      // the spectrum until you find the right interval
      nact = *nguess + 1;
      while(wavelength > resp->spec[nact].lambda_mean)
        nact++;
    }
  else
    {

      // in case that you search upwards, go up
      // the spectrum until you find the right interval
      nact = *nguess;
      while(wavelength < resp->spec[nact-1].lambda_mean)
        nact--;
    }

  // interpolate within the interval to calculate the
  // sensitivity
  factor = (wavelength - resp->spec[nact-1].lambda_mean) /
    (resp->spec[nact].lambda_mean - resp->spec[nact-1].lambda_mean);
  ret = resp->spec[nact-1].flux + factor * (resp->spec[nact].flux - resp->spec[nact-1].flux);

  // update the guess value
  *nguess = nact;

  // return the sensitivity value
  return ret;
}

/**
 * Function: get_response_values
 * The function evaluates the response function at the desired
 * independent value. The sensitivity and it error are returned
 * in an array parameter.
 *
 * Parameters:
 * @param resp_func  - the response function
 * @param wavelength - the independent value
 * @param resp_vals  - the dependent value and its error
 *
 * Returns:
 * @return -
 */
void
get_response_values(response_function *resp_func, double wavelength, double* resp_vals)
{
  // check whether the independent value is outside the deifned range
  if (wavelength < resp_func->xmin || wavelength > resp_func->xmax)
    {
      // give the default
      // outside values
      resp_vals[0] = 0.0;
      resp_vals[1] = 0.0;
    }
  else
    {
      // get the response value and its error
      resp_vals[0] = eval_interp(resp_func->resp_values, wavelength);
      resp_vals[1] = eval_interp(resp_func->resp_errors, wavelength);
    }
}

/**
 * Function: create_response_function
 * Load the sensitivity file and creates a new response
 * function. Relies heavily on other subroutines
 *
 * Parameters:
 * @param filename - sensitivity file name
 *
 * Returns:
 * @return resp_func - the response function created
 */
response_function *
create_response_function(char *filename)
{
  response_function *resp_func;

  // allocate space for the return structure;
  // complain if this fails
  resp_func = (response_function *)malloc (sizeof (response_function));
  if (resp_func == NULL)
    aXe_message (aXe_M_FATAL, __FILE__, __LINE__,
                 "Could not allocate memory for response function");

  // load the two intgerpolators
  resp_func->resp_values = create_interp_ftable(filename, 2, "WAVELENGTH", "SENSITIVITY", RESP_FUNC_INTERP_TYPE);
  resp_func->resp_errors = create_interp_ftable(filename, 2, "WAVELENGTH", "ERROR", RESP_FUNC_INTERP_TYPE);

  // transfer the min and max values from the interpolator
  resp_func->xmin = resp_func->resp_values->xmin;
  resp_func->xmax = resp_func->resp_values->xmax;

  // return the respons function
  return resp_func;
}

/**
 * Function: free_response_function
 * Releases the memory allocated in a response function
 *
 * Parameters:
 * @param resp_func - the response function
 *
 * Returns:
 * @return -
  */
void
free_response_function(response_function *resp_func)
{
  // free the two interpolators
  free_interp(resp_func->resp_values);
  free_interp(resp_func->resp_errors);

  // free the rest
  free(resp_func);

  // set it to NULL
  resp_func = NULL;

}

/**
 * Function: get_calfunc_for_beam
 * The subroutine determines and returns the calibration function
 * for a certain beam and configuration file. The coefficients of
 * the wavelength calibration are evaluated at the position of the
 * beam object.
 *
 * Parameters:
 * @param actbeam   - the current beam
 * @param for_grism - indicator for grism or prism dispersion
 * @param CONF_file - full pathname to configuration file
 * @param conf      - the configuration structure
 *
 * Returns:
 * @return wl_calibration - the calibration function
  */
calib_function *
get_calfunc_for_beam(const beam actbeam, const int for_grism, char CONF_file[],
                     const aperture_conf * conf)
{
  calib_function *wl_calibration;
  dispstruct     *disp;
  d_point         pixel;

  // get the reference point right
  pixel.x = actbeam.refpoint.x - conf->refx;
  pixel.y = actbeam.refpoint.y - conf->refy;

  // get the dispersion structure
  disp = get_dispstruct_at_pos(CONF_file, for_grism, actbeam.ID, pixel);

  // transform the dispersion structure into the
  // wavelength calibration
  wl_calibration = create_calib_from_gsl_vector(for_grism, disp->pol);
  if (!for_grism)
    wl_calibration->pr_range = get_prange (CONF_file, actbeam.ID);

  // free the whole structure
  free_dispstruct(disp);

  // return the wavelength calibration
  return wl_calibration;
}


/**
 *
 * @file core_dgbtype1cb.c
 *
 *  PLASMA core_blas kernel
 *  PLASMA is a software package provided by Univ. of Tennessee,
 *  Univ. of California Berkeley and Univ. of Colorado Denver
 *
 * @version 2.6.0
 * @author Azzam Haidar
 * @date 2012-12-15
 * @generated d Tue Jan  7 11:44:49 2014
 *
 **/
#include <lapacke.h>
#include "common.h"

//#define AU(m,n) &(A[(m) + LDA*(n)])
//#define AL(m,n) &(A[(m) + LDA*(n)])
#define AL(m_, n_) (A + NB + LDA * (n_) + ((m_)-(n_)))
#define AU(m_, n_) (A + NB + LDA * (n_) + ((m_)-(n_)+NB))
#define VQ(m)     (VQ + (m))
#define VP(m)     (VP + (m))
#define TAUQ(m)   (TAUQ + (m))
#define TAUP(m)   (TAUP + (m))

/***************************************************************************//**
 *
 * @ingroup CORE_double
 *
 *  CORE_dgbtype1cb is a kernel that will operate on a region (triangle) of data
 *  bounded by st and ed. This kernel eliminate a column by an column-wise
 *  annihiliation, then it apply a left+right update on the hermitian triangle.
 *  Note that the column to be eliminated is located at st-1.
 *
 *  All detail are available on technical report or SC11 paper.
 *  Azzam Haidar, Hatem Ltaief, and Jack Dongarra. 2011.
 *  Parallel reduction to condensed forms for symmetric eigenvalue problems
 *  using aggregated fine-grained and memory-aware kernels. In Proceedings
 *  of 2011 International Conference for High Performance Computing,
 *  Networking, Storage and Analysis (SC '11). ACM, New York, NY, USA, ,
 *  Article 8 , 11 pages.
 *  http://doi.acm.org/10.1145/2063384.2063394
 *
 *******************************************************************************
 *
 * @param[in] N
 *          The order of the matrix A.
 *
 * @param[in] NB
 *          The size of the band.
 *
 * @param[in, out] A
 *          A pointer to the matrix A of size (3*NB+1)-by-N.
 *
 * @param[in] LDA
 *          The leading dimension of the matrix A. LDA >= max(1,3*NB+1)
 *
 * @param[out] V
 *          double array, dimension N if eigenvalue only
 *          requested or (LDV*blkcnt*Vblksiz) if Eigenvectors requested
 *          The Householder reflectors are stored in this array.
 *
 * @param[out] TAU
 *          double array, dimension (N).
 *          The scalar factors of the Householder reflectors are stored
 *          in this array.
 *
 * @param[in] st
 *          A pointer to the start index where this kernel will operate.
 *
 * @param[in] ed
 *          A pointer to the end index where this kernel will operate.
 *
 * @param[in] sweep
 *          The sweep number that is eliminated. it serve to calculate the
 *          pointer to the position where to store the Vs and Ts.
 *
 * @param[in] Vblksiz
 *          constant which correspond to the blocking used when applying the Vs.
 *          it serve to calculate the pointer to the position where to store the
 *          Vs and Ts.
 *
 * @param[in] WANTZ
 *          constant which indicate if Eigenvalue are requested or both
 *          Eigenvalue/Eigenvectors.
 *
 * @param[in] WORK
 *          Workspace of size nb.
 *
 *******************************************************************************
 *
 * @return
 *          \retval PLASMA_SUCCESS successful exit
 *          \retval <0 if -i, the i-th argument had an illegal value
 *
 ******************************************************************************/
/***************************************************************************
 *          TYPE 1-BAND-bidiag Lower/Upper columnwise-Householder
 ***************************************************************************/
void
CORE_dgbtype1cb(PLASMA_enum uplo, int N, int NB,
                double *A, int LDA,
                double *VQ, double *TAUQ,
                double *VP, double *TAUP,
                int st, int ed, int sweep, int Vblksiz, int WANTZ,
                double *WORK)
{
    double ctmp;
    int i, len, LDX, lenj;
    int blkid, vpos, taupos, tpos;
    /* find the pointer to the Vs and Ts as stored by the bulgechasing
     * note that in case no eigenvector required V and T are stored
     * on a vector of size N
     * */
     if( WANTZ == 0 ) {
         vpos   = ((sweep+1)%2)*N + st;
         taupos = ((sweep+1)%2)*N + st;
     } else {
         findVTpos(N, NB, Vblksiz, sweep, st,
                   &vpos, &taupos, &tpos, &blkid);
     }

    LDX = LDA-1;
    len = ed-st+1;

    if( uplo == PlasmaUpper ) {
        /* ========================
         *       UPPER CASE
         * ========================*/
        /* Eliminate the row  at st-1 */
        *VP(vpos) = 1.;
        for(i=1; i<len; i++){
            *VP(vpos+i)     = (*AU(st-1, st+i));
            *AU(st-1, st+i) = 0.;
        }
        ctmp = (*AU(st-1, st));
        LAPACKE_dlarfg_work(len, &ctmp, VP(vpos+1), 1, TAUP(taupos) );
        *AU(st-1, st) = ctmp;
        /* Apply right on A(st:ed,st:ed) */
        ctmp = *TAUP(taupos);
        LAPACKE_dlarfx_work(LAPACK_COL_MAJOR, lapack_const(PlasmaRight),
                            len, len, VP(vpos), ctmp, AU(st, st), LDX, WORK);

        /* Eliminate the created col at st */
        *VQ(vpos) = 1.;
        memcpy( VQ(vpos+1), AU(st+1, st), (len-1)*sizeof(double) );
        memset( AU(st+1, st), 0, (len-1)*sizeof(double) );
        LAPACKE_dlarfg_work(len, AU(st, st), VQ(vpos+1), 1, TAUQ(taupos) );
        lenj = len-1;
        ctmp = (*TAUQ(taupos));
        LAPACKE_dlarfx_work(LAPACK_COL_MAJOR, lapack_const(PlasmaLeft),
                            len, lenj, VQ(vpos), ctmp, AU(st, st+1), LDX, WORK);
    }else{
        /* ========================
         *       LOWER CASE
         * ========================*/
        /* Eliminate the col  at st-1 */
        *VQ(vpos) = 1.;
        memcpy( VQ(vpos+1), AL(st+1, st-1), (len-1)*sizeof(double) );
        memset( AL(st+1, st-1), 0, (len-1)*sizeof(double) );
        LAPACKE_dlarfg_work(len, AL(st, st-1), VQ(vpos+1), 1, TAUQ(taupos) );
        /* Apply left on A(st:ed,st:ed) */
        ctmp = (*TAUQ(taupos));
        LAPACKE_dlarfx_work(LAPACK_COL_MAJOR, lapack_const(PlasmaLeft),
                            len, len, VQ(vpos), ctmp, AL(st, st), LDX, WORK);
        /* Eliminate the created row at st */
        *VP(vpos) = 1.;
        for(i=1; i<len; i++){
            *VP(vpos+i)     = (*AL(st, st+i));
            *AL(st, st+i)   = 0.;
        }
        ctmp = (*AL(st, st));
        LAPACKE_dlarfg_work(len, &ctmp, VP(vpos+1), 1, TAUP(taupos) );
        *AL(st, st) = ctmp;
        lenj = len-1;
        ctmp = (*TAUP(taupos));
        LAPACKE_dlarfx_work(LAPACK_COL_MAJOR, lapack_const(PlasmaRight),
                            lenj, len, VP(vpos), ctmp, AL(st+1, st), LDX, WORK);
    }
    /* end of uplo case */
    return;
}
/***************************************************************************/
#undef AU
#undef AL
#undef VQ
#undef VP
#undef TAUQ
#undef TAUP


// Authors: David Blei (blei@cs.princeton.edu)
//          Sean Gerrish (sgerrish@cs.princeton.edu)
//
// Copyright 2011 Sean Gerrish and David Blei
// All Rights Reserved.
//
// See the README for this package for details about modifying or
// distributing this software.

#include "lda-seq.h"

#include "gflags.h"
#include <gsl/gsl_linalg.h>
#include <gsl/gsl_blas.h>
#include <gsl/gsl_vector_double.h>

extern int LDA_INFERENCE_MAX_ITER;

static double* scaled_influence = NULL;

// const int TIME = 61;
const int TIME = -3;
const double PI = 3.141592654;

DEFINE_int32(lda_sequence_max_iter,
	     20,
	     "The maximum number of iterations.");
DEFINE_int32(lda_sequence_min_iter,
	     1,
	     "The maximum number of iterations.");

DEFINE_int32(forward_window,
	     1,
	     "The forward window for deltas. "
	     "If negative, we use a beta with mean "
	     "5.");

DEFINE_string(normalize_docs,
	      "normalize",
	      "Describes how documents's wordcounts "
	      "are considered for finding influence. "
	      "Options are \"normalize\", \"none\", "
	      "\"occurrence\", \"log\", or \"log_norm\".");

DEFINE_int32(save_time,
	     1e20,
	     "Save a specific time.  If -1, save all times.");

DEFINE_int32(fix_topics,
	     0,
	     "Fix a set of this many topics. This amounts "
	     "to fixing these topics' variance at 1e-10.");

DECLARE_string(model);
DECLARE_int32(max_number_time_points);
DECLARE_double(sigma_d);
DECLARE_double(sigma_l);
DECLARE_double(sigma_c);
DECLARE_double(sigma_cv);

/*
 * populate an LDA model at a particular time point
 *
 */

inf_var* inf_var_alloc(int number_topics,
		       corpus_seq_t* corpus_seq) {
  // Hate to do this, but I had trouble using it before.  This should
  // be the first place we use it; otherwise we'll get a sigsev nil.
  if (scaled_influence == NULL) {
    scaled_influence = NewScaledInfluence(FLAGS_max_number_time_points);
  }

  inf_var* inf_var_ptr = (inf_var*) malloc(sizeof(inf_var));
  inf_var_ptr->doc_weights = (gsl_matrix**) malloc(sizeof(gsl_matrix*)
						   * corpus_seq->len);
  inf_var_ptr->renormalized_doc_weights = (gsl_matrix**) malloc(
    sizeof(gsl_matrix*)
    * corpus_seq->len);
  inf_var_ptr->ntime = corpus_seq->len;
  int i=0;
  for (i=0; i < corpus_seq->len; ++i) {
    corpus_t* corpus = corpus_seq->corpus[i];
    outlog("creating matrix. %d %d", corpus->ndocs, number_topics);
    if (corpus->ndocs == 0) {
      inf_var_ptr->doc_weights[i] = (gsl_matrix*) malloc(sizeof(gsl_matrix));
      inf_var_ptr->doc_weights[i]->size1 = 0;
      inf_var_ptr->doc_weights[i]->size2 = number_topics;
      inf_var_ptr->renormalized_doc_weights[i] = (gsl_matrix*) malloc(sizeof(gsl_matrix));
      inf_var_ptr->renormalized_doc_weights[i]->size1 = 0;
      inf_var_ptr->renormalized_doc_weights[i]->size2 = number_topics;
    } else {
      inf_var_ptr->doc_weights[i] = gsl_matrix_calloc(corpus->ndocs,
						      number_topics);
      inf_var_ptr->renormalized_doc_weights[i] = gsl_matrix_calloc(corpus->ndocs,
								   number_topics);
    }
  }
  return inf_var_ptr;
}

void inf_var_free(inf_var* ptr) {
  // TODO.
}

// Solves the linear system Ax = b for x.
// Assumes that x is already allocated.
void Solve(gsl_matrix* A,
	   const gsl_vector* b,
	   gsl_vector* x) {
    int permutation_sign;
    gsl_permutation* permutation = gsl_permutation_alloc(b->size);
    gsl_linalg_LU_decomp(A, permutation, &permutation_sign);
    gsl_linalg_LU_solve(A, permutation, b, x);
    gsl_permutation_free(permutation);
}

// Find the sums of influence of all documents in advance.
// g has scores for everything *up to but not including* g.
void InfluenceTotalFixed(lda_seq* seq,
		    const corpus_seq_t* data) {
  gsl_vector* exp_tmp = gsl_vector_alloc(seq->nterms);

  for (int k=0; k < seq->ntopics; ++k) {

    for (int s=0; s < seq->nseq; ++s) {
      // Pull out elements of g, and make sure to set them to 0!
      gsl_vector_view g = gsl_matrix_column(
	  seq->influence_sum_lgl[k], s);
      gsl_vector_set_zero(&g.vector);

      for (int t=0; t <= s; ++t) {
	gsl_vector_view w_phi_l =
	  gsl_matrix_column(seq->topic[k]->w_phi_l, t);
	gsl_vector_memcpy(exp_tmp, &w_phi_l.vector);
	gsl_vector_scale(exp_tmp, scaled_influence[s - t]);
	gsl_vector_add(&g.vector, exp_tmp);
      }
    }
  }
  gsl_vector_free(exp_tmp);
}

void DumpTimeDocTopicStats(const char* root,
			   size_t t,
			   corpus_t* corpus,
			   gsl_matrix** phi) {
  // For all documents, dump the top topics.

  char name[400];
  // Dump the top topics for each word.
  sprintf(name, "%s%ld_doc_term_topics.dat", root, t);
  FILE* f = fopen(name, "w");
  for (unsigned int d=0; d < corpus->ndocs; ++d) {
    gsl_matrix* phi_d = phi[d];
    doc_t* doc = corpus->doc[d];
    for (unsigned int n=0; n < doc->nterms; ++n) {

      unsigned int w = doc->word[n];

      // First, find the max topic weight.
      unsigned int max_topic_index = 0;
      double max_topic_weight = gsl_matrix_get(phi_d, n, 0);
      for (unsigned int k=0; k < phi_d->size2; ++k) {
	double phi_d_n_k = gsl_matrix_get(phi_d, n, k);
	if (phi_d_n_k > max_topic_weight) {
	  max_topic_weight = phi_d_n_k;
	  max_topic_index = k;
	}
      }

      fprintf(f, "%d:%d:%.3f", w, max_topic_index, max_topic_weight);

      if (n < doc->nterms - 1) {
	fprintf(f, " ");
      }
    }
    fprintf(f, "\n");
  }
  fclose(f);
}

void PrepareRegressionComponents(
    corpus_t* corpus,
    lda_seq* seq,
    unsigned int k,
    gsl_matrix* W_phi,
    gsl_matrix* W_phi_var,
    gsl_vector* d_phi_var_tmp,
    gsl_vector* response,
    gsl_vector* d_tmp,
    gsl_matrix* dd_tmp,
    gsl_vector* document_weights,
    gsl_matrix* W_phi_tmp,
    gsl_vector* exp_h_tmp) {

  // Then left-multiply this by W_phi:
  gsl_blas_dgemv(CblasTrans,
		 1.0,
		 W_phi,
		 response,
		 0.0,
		 d_tmp);

  // Set up the transformation matrix.
  // First, set up W_phi^T \Lambda W_phi.
  gsl_matrix_memcpy(W_phi_tmp, W_phi);
  for (unsigned int d=0; d < corpus->ndocs; ++d) {
    gsl_vector_view col = gsl_matrix_column(W_phi_tmp, d);
    gsl_vector_mul(&col.vector, exp_h_tmp);
  }

  // Yuck.  Maybe we should do this sparsely?
  // Probably won't be too bad, at least for now.
  gsl_blas_dgemm(CblasTrans,
		 CblasNoTrans,
		 1.0,
		 W_phi_tmp,
		 W_phi,
		 0.0,
		 dd_tmp);

  gsl_blas_dgemv(CblasTrans,
		 1.0,
		 W_phi_var,
		 exp_h_tmp,
		 0.0,
		 d_phi_var_tmp);

  // Next, add elements to the diagonal of dd_tmp.
  for (unsigned int d=0; d < corpus->ndocs; ++d) {
    double value = gsl_matrix_get(dd_tmp, d, d);
    value += (seq->topic[k]->chain_variance
	      / (FLAGS_sigma_d * FLAGS_sigma_d));
    // sgerrish: Is this supposed to be multiplied by anything?
    value += gsl_vector_get(d_phi_var_tmp, d);
    gsl_matrix_set(dd_tmp, d, d, value);
  }
}

void SetExpHTmp(lda_seq* seq,
		const corpus_seq_t* data,
		unsigned int t,
		unsigned int k,
		unsigned int n,
		gsl_vector* exp_h_tmp,
		gsl_vector* zw_tmp,
		gsl_vector** exp_i_tmp) {
  gsl_vector_set_zero(exp_h_tmp);
  for (int i = t; i < seq->nseq; ++i) {
    gsl_vector_view mean_i_current =
      gsl_matrix_column(seq->topic[k]->e_log_prob, i);
    gsl_vector_view var_i_current =
      gsl_matrix_column(seq->topic[k]->variance, i + 1);

    // Set up exp_h_tmp.
    gsl_vector_memcpy(zw_tmp, &var_i_current.vector);
    gsl_vector_sub(zw_tmp, &mean_i_current.vector);
    gsl_vector_scale(zw_tmp, 2.0);
    for (n=0; n < data->nterms; ++n) {
      gsl_vector_set(zw_tmp, n,
		     exp(gsl_vector_get(zw_tmp, n)));
    }
    gsl_vector_scale(zw_tmp,
		     scaled_influence[i - t]
		     * scaled_influence[i - t]);
    gsl_vector_add(exp_h_tmp, zw_tmp);

    // Set up exp_i_tmp.
    gsl_vector_memcpy(exp_i_tmp[i], &var_i_current.vector);
    gsl_vector_scale(exp_i_tmp[i], 0.5);
    gsl_vector_sub(exp_i_tmp[i], &mean_i_current.vector);
    for (n=0; n < data->nterms; ++n) {
      gsl_vector_set(exp_i_tmp[i], n,
		     exp(gsl_vector_get(exp_i_tmp[i], n)));
    }
  }
}

double update_inf_var_fixed(lda_seq* seq,
			    const corpus_seq_t* data,
			    gsl_matrix** phi,
			    size_t t,
			    const char* root,
			    int dump_doc_stats) {
  double lhood = 0.0;
  // Note that we're missing a suspicious factor of -2 on the document
  // weights currently.  We won't worry about that for now (since
  // we're still experimenting), but it should soon be fixed.
  corpus_t* corpus = data->corpus[t];
  if (!corpus->ndocs) {
    return lhood;
  }
  inf_var* influence = seq->influence;
  if (t != TIME && 0) {
    return lhood;
  }

  // We need access to the following:
  gsl_matrix* documents_topics_t = influence->doc_weights[t];
  gsl_matrix* renormalized_documents_topics_t = influence->renormalized_doc_weights[t];

  size_t k;

  gsl_matrix* W_phi = gsl_matrix_calloc(
    seq->nterms, corpus->ndocs);
  gsl_matrix* W2_phi2 = gsl_matrix_calloc(
    seq->nterms, corpus->ndocs);
  gsl_matrix* W_phi_var =
    gsl_matrix_calloc(seq->nterms, corpus->ndocs);
  gsl_matrix* W_phi_tmp = gsl_matrix_alloc(
    seq->nterms, corpus->ndocs);
  gsl_matrix* dd_tmp = gsl_matrix_alloc(
    corpus->ndocs, corpus->ndocs);
  gsl_vector* xd_tmp = gsl_vector_calloc(
    corpus->ndocs);
  gsl_vector* yw_tmp = gsl_vector_calloc(
    seq->nterms);
  gsl_vector* zw_tmp = gsl_vector_calloc(
    seq->nterms);
  gsl_vector* terms_inc_tmp = gsl_vector_calloc(
    seq->nterms);
  gsl_vector** exp_i_tmp = new gsl_vector*[seq->nseq];
  for (int i=0; i < seq->nseq; ++i) {
    exp_i_tmp[i] = gsl_vector_calloc(seq->nterms);
  }

  int n;

  gsl_vector* response = gsl_vector_calloc(seq->nterms);
  gsl_vector* exp_h_tmp = gsl_vector_calloc(seq->nterms);
  gsl_vector* d_tmp = gsl_vector_calloc(corpus->ndocs);
  gsl_vector* d_phi_var_tmp = gsl_vector_calloc(corpus->ndocs);

  //  assert(post->phi->size2 == documents_topics_t->size2);
  double* total_terms = (double*) malloc(sizeof(double) * corpus->ndocs);
  double* renormalization_totals = (double*) malloc(sizeof(double) * corpus->ndocs);
  for (k=0; k < documents_topics_t->size2; ++k) {
    // Set up W_phi and W_phi_var.
    for (int d=0; d < corpus->ndocs; ++d) {
      doc_t* doc = corpus->doc[d];
      total_terms[d] = 0.0;
      renormalization_totals[d] = 0.0;
      double log_norm_sum = 0.0;
      for (n=0; n < doc->nterms; ++n) {
	total_terms[d] += doc->count[n];
	if (FLAGS_normalize_docs == "normalize") {
	  renormalization_totals[d] = 1.0;
	} else if (FLAGS_normalize_docs == "log") {
	  renormalization_totals[d] += log(doc->count[n] + 1);
	} else if (FLAGS_normalize_docs == "log_norm") {
	  double weight = log(doc->count[n] / doc->total);
	  renormalization_totals[d] += weight;
	} else if (FLAGS_normalize_docs == "identity") {
	  renormalization_totals[d] += doc->count[n];
	} else if (FLAGS_normalize_docs == "occurrence") {
	  renormalization_totals[d] = 1.0;
	}
      }
      assert(doc->total == total_terms[d]);

      for (n=0; n < doc->nterms; ++n) {
	// Phi, for the doc's term n and topic k.
	double phi_d_n_k = gsl_matrix_get(phi[d], n, k);

	// This cast should happen automatically because total_terms is a double,
	// but we make the cast here to be explicit and to avoid bugs later.
	double number_terms;
	if (FLAGS_normalize_docs == "normalize") {
	  number_terms = ((double) doc->count[n]
			  / (double) total_terms[d]);
	} else if (FLAGS_normalize_docs == "log") {
	  number_terms = log(doc->count[n] + 1.0);
	} else if (FLAGS_normalize_docs == "log_norm") {
	  number_terms = log(doc->count[n] / total_terms[d]);
	  renormalization_totals[d] += log(doc->count[n] / total_terms[d]);
	} else if (FLAGS_normalize_docs == "identity") {
	  number_terms = doc->count[n];
	} else if (FLAGS_normalize_docs == "occurrence") {
	  number_terms = ((double) doc->count[n] / (double) total_terms[d]);
	  assert(doc->count[n] == 1);
	} else {
	  assert(0);
	}

	gsl_matrix_set(W_phi, doc->word[n], d,
		       number_terms * phi_d_n_k);
	gsl_matrix_set(W2_phi2, doc->word[n], d,
		       number_terms * number_terms
		       * phi_d_n_k * phi_d_n_k);
	gsl_matrix_set(W_phi_var, doc->word[n], d,
		       number_terms * number_terms
		       * (phi_d_n_k - phi_d_n_k * phi_d_n_k));
      }
    }

    gsl_vector_view document_weights = gsl_matrix_column(
      documents_topics_t,
      k);
    gsl_vector_view renormalized_document_weights = gsl_matrix_column(
      renormalized_documents_topics_t,
      k);
    assert(seq->topic[k]->e_log_prob->size2 == data->len);

    // Now, with w_phi_var, etc. set, determine
    // \sum_{i=t}^{T-1} r(...) h(t, i)
    SetExpHTmp(seq,
	       data,
	       t,
	       k,
	       n,
	       exp_h_tmp,
	       zw_tmp,
	       exp_i_tmp);

    // Next, set up the weighted response,
    // \exp_{-m + v / 2) \circ (m_{t+1} - m_t + v_t}).
    // Here we also subtract the current l's contribution to influence_sum_lgl.
    gsl_vector_view w_phi_l_t = gsl_matrix_column(seq->topic[k]->w_phi_l, t);
    gsl_vector_set_zero(response);
    for (int i = t; i < seq->nseq - 1; ++i) {
      gsl_vector_view total_influence_time_i =
	gsl_matrix_column(seq->influence_sum_lgl[k], i);
      gsl_vector_memcpy(zw_tmp, &w_phi_l_t.vector);
      gsl_vector_scale(zw_tmp, scaled_influence[i - t]);
      gsl_vector_sub(&total_influence_time_i.vector, zw_tmp);

      // Now, copy this total influence at time i back into zw_tmp.
      gsl_vector_memcpy(zw_tmp, &total_influence_time_i.vector);
      gsl_vector_mul(zw_tmp, exp_i_tmp[i]);

      gsl_vector_view mean_i_current =
	gsl_matrix_column(seq->topic[k]->e_log_prob, i);
      gsl_vector_view mean_i_next =
	gsl_matrix_column(seq->topic[k]->e_log_prob, i + 1);
      gsl_vector_view var_i_current =
	gsl_matrix_column(seq->topic[k]->variance, i + 1);
      gsl_vector_memcpy(terms_inc_tmp, &mean_i_next.vector);
      gsl_vector_sub(terms_inc_tmp, &mean_i_current.vector);
      gsl_vector_add(terms_inc_tmp, &var_i_current.vector);
      gsl_vector_sub(terms_inc_tmp, zw_tmp);
      assert(data->nterms == terms_inc_tmp->size);
      gsl_vector_mul(terms_inc_tmp, exp_i_tmp[i]);
      gsl_vector_scale(terms_inc_tmp, scaled_influence[i - t]);

      gsl_vector_add(response, terms_inc_tmp);
    }

    PrepareRegressionComponents(corpus,
				seq,
				k,
				W_phi,
				W_phi_var,
				d_phi_var_tmp,
				response,
				d_tmp,
				dd_tmp,
				&document_weights.vector,
				W_phi_tmp,
				exp_h_tmp);
    // Finally, solve for the document weights d!
    Solve(dd_tmp,
	  d_tmp,
	  &document_weights.vector);

    // Keep track of the iteration so we can dump certain stats
    // occasionally (but not always).
    static int dump_count = 0;
    ++dump_count;

    if (FLAGS_save_time == t) {
      outlog("Updating topic %ld, time %ld.", k, t);
      char name[400];
      sprintf(name, "%s%ld_%ld_weighted_document_terms.dat", root, k, t);
      FILE* f = fopen(name, "w");
      params_write_sparse_gsl_matrix(f, "W_phi", W_phi);
      fclose(f);

      sprintf(name, "%s%ld_%ld_weighted_document_terms_var.dat", root, k, t);
      f = fopen(name, "w");
      params_write_sparse_gsl_matrix(f, "W_phi_var", W_phi_var);
      fclose(f);

      sprintf(name, "%s%ld_%ld_phi.dat", root, k, t);
      f = fopen(name, "w");
      params_write_gsl_matrix(f, "phi 0", phi[0]);
      //     params_write_gsl_matrix(f, "phi 2", phi[1]);
      // params_write_gsl_matrix(f, "phi 2", phi[7]);
      fclose(f);

      sprintf(name, "%s%ld_%ld_weighted_document_terms_sq.dat", root, k, t);
      f = fopen(name, "w");
      params_write_sparse_gsl_matrix(f, "W2_phi2", W2_phi2);
      fclose(f);

      sprintf(name, "%s%ld_%ld_weighted_response.dat", root, k, t);
      f = fopen(name, "w");
      params_write_gsl_vector_multiline(f, "weighted_response", d_tmp);
      fclose(f);

      sprintf(name, "%s%ld_%ld_response.dat", root, k, t);
      f = fopen(name, "w");
      params_write_gsl_vector_multiline(f, "response", response);
      fclose(f);

      sprintf(name, "%s%ld_%ld_exp_h.dat", root, k, t);
      f = fopen(name, "w");
      params_write_gsl_vector_multiline(f, "exp_h", exp_h_tmp);
      fclose(f);

      if (dump_doc_stats || dump_count % 4 == 0) {
	sprintf(name, "%s%ld_%ld_document_document_matrix.dat", root, k, t);
	f = fopen(name, "w");
	params_write_gsl_matrix(f, "document_document_matrix", dd_tmp);
	fclose(f);

	sprintf(name, "%s%ld_%ld_exp_h.dat", root, k, t);
	f = fopen(name, "w");
	params_write_gsl_vector_multiline(f, "exp_h", exp_h_tmp);
	fclose(f);
      }
    }

    if (FLAGS_save_time == -1) {
      // First, dump phi's for the top topics.
      DumpTimeDocTopicStats(root, t, corpus, phi);
    }
    outlog("Done updating topic %ld, time %ld.", k, t);

    for (int d = 0; d < document_weights.vector.size; ++d) {
      gsl_vector_set(&renormalized_document_weights.vector, d,
		     vget(&document_weights.vector, d)
		     * renormalization_totals[d]);
    }

    // Now copy this and several products to
    // the sslm_var object.
    gsl_vector_view w_phi_l =
      gsl_matrix_column(seq->topic[k]->w_phi_l, t);
    gsl_blas_dgemv(CblasNoTrans,
		   1.0,
		   W_phi,
		   &document_weights.vector,
		   0.0,
		   &w_phi_l.vector);

    // Copy this value back into lgl.
    for (int i=t; i < seq->nseq - 1; ++i) {
      gsl_vector_view total_influence_time_i =
	gsl_matrix_column(seq->influence_sum_lgl[k], i);
      gsl_vector_memcpy(zw_tmp, &w_phi_l.vector);
      gsl_vector_scale(zw_tmp, scaled_influence[i - t]);
      gsl_vector_add(&total_influence_time_i.vector, zw_tmp);
    }

     // Keep track of the term we need to add to m_update_coeff.
     gsl_vector_memcpy(terms_inc_tmp, &w_phi_l.vector);
     gsl_vector_mul(terms_inc_tmp, &w_phi_l.vector);

     // Copy and square the document weights vector.
     for (int i = 0; i < xd_tmp->size; ++i) {
       double value = gsl_vector_get(&document_weights.vector, i);
       value = value * value + FLAGS_sigma_l * FLAGS_sigma_l;
       gsl_vector_set(xd_tmp, i, value);
     }
     gsl_blas_dgemv(CblasNoTrans,
		    1.0,
		    W_phi_var,
		    xd_tmp,
		    0.0,
		    yw_tmp);
     gsl_vector_add(terms_inc_tmp, yw_tmp);

     for (int i = 0; i < xd_tmp->size; ++i) {
       gsl_vector_set(xd_tmp, i, FLAGS_sigma_l * FLAGS_sigma_l);
     }
     gsl_blas_dgemv(CblasNoTrans,
		    1.0,
		    W2_phi2,
		    xd_tmp,
		    0.0,
		    yw_tmp);
     gsl_vector_add(terms_inc_tmp, yw_tmp);

     // Store an update coefficient for the beta updates.
     for (int i = t; i < seq->nseq; ++i) {
       gsl_vector_view m_update_coeff_h =
	 gsl_matrix_column(seq->topic[k]->m_update_coeff, i);
       if (t == 0) {
	 gsl_vector_set_zero(&m_update_coeff_h.vector);
       }
       gsl_vector_memcpy(yw_tmp, terms_inc_tmp);
       gsl_vector_scale(yw_tmp, scaled_influence[i - t]);
       gsl_vector_add(&m_update_coeff_h.vector, yw_tmp);
     }

     for (int i = t; i < seq->nseq; ++i) {
       gsl_vector_view m_update_coeff_g =
	 gsl_matrix_column(seq->topic[k]->m_update_coeff_g, i);
       if (t == 0) {
	 gsl_vector_set_zero(&m_update_coeff_g.vector);
       }
       gsl_vector_memcpy(yw_tmp, &w_phi_l.vector);
       gsl_vector_scale(yw_tmp, scaled_influence[i - t]);
       gsl_vector_add(&m_update_coeff_g.vector, yw_tmp);
     }

     for (int i = 0; i < corpus->ndocs; ++i) {
       double value = gsl_vector_get(&document_weights.vector, i);

       // While we're here, increment the likelihood.
       lhood += (-(value * value + FLAGS_sigma_l * FLAGS_sigma_l)
		 / (2.0 * FLAGS_sigma_d * FLAGS_sigma_d)
		 - 0.5 * log(2 * PI)
		 - log(FLAGS_sigma_d * FLAGS_sigma_d));
     }
  }
  free(total_terms);
  free(renormalization_totals);
  gsl_matrix_free(W_phi);
  gsl_matrix_free(W_phi_tmp);
  gsl_matrix_free(W2_phi2);
  gsl_matrix_free(W_phi_var);
  gsl_matrix_free(dd_tmp);
  gsl_vector_free(exp_h_tmp);
  gsl_vector_free(response);
  gsl_vector_free(terms_inc_tmp);
  for (int i=0; i < seq->nseq; ++i) {
    gsl_vector_free(exp_i_tmp[i]);
  }
  delete[] exp_i_tmp;
  gsl_vector_free(d_tmp);
  gsl_vector_free(d_phi_var_tmp);
  gsl_vector_free(xd_tmp);
  gsl_vector_free(yw_tmp);
  gsl_vector_free(zw_tmp);

  return lhood;
}

void make_lda_from_seq_slice(lda* lda_m,
			     lda_seq* lda_seq_m,
			     int time) {
     // set lda model topics
     // !!! note: we should be able to point to the view...

     int k;
     for (k = 0; k < lda_seq_m->ntopics; k++)
     {
	 // get topic
	 gsl_vector s =
	     gsl_matrix_column(lda_seq_m->topic[k]->e_log_prob,
			       time).vector;
	 gsl_vector d =
	     gsl_matrix_column(lda_m->topics, k).vector;
	 gsl_blas_dcopy(&s, &d);
     }
     gsl_blas_dcopy(lda_seq_m->alpha, lda_m->alpha);
 }

static gsl_matrix* g_alloc(lda_seq* model,
			   const corpus_seq_t* data,
			   int time) {
     gsl_matrix* g = gsl_matrix_calloc(model->nterms,
				       model->ntopics);
     double exp_m, m, m_next;
     for (int k = 0; k < model->ntopics; ++k) {
       for (int w=0; w < model->nterms; ++w) {
	 double variance_first = mget(model->topic[k]->variance, w, time);
	 double m = mget(model->topic[k]->e_log_prob, w, time);
	 double m_next;
	 exp_m = exp(-m + variance_first / 2.0);
	 gsl_matrix_set(g, w, k,
			(scaled_influence[0]
			 * -variance_first
			 * exp_m));

	 for (int i=time; i < model->nseq - 1; ++i) {
	   // This loop is kind of going overboard, but at least we
	   // do this once per E-M iteration.
	   double influence_other_times = 0.0;
	   for (int j = 0; j < i; ++j) {
	     exp_m = exp(-mget(model->topic[k]->e_log_prob, w, j)
			 + mget(model->topic[k]->variance, w, j) / 2.0);
	     // Note that we skip the other docs in this time period.
	     // Those get special treatment below.
	     if (j != time) {
	       influence_other_times += (
	          mget(model->topic[k]->w_phi_l, w, j)
		  * scaled_influence[i - j]
		  * exp_m);
	      }
	   }

	   m = mget(model->topic[k]->e_log_prob, w, i);
	   m_next = mget(model->topic[k]->e_log_prob, w, i + 1);
	   // Increment the current count by this value.
	   gsl_matrix_set(g, w, k,
			  mget(g, w, k) +
			  (scaled_influence[i - time]
			   * (m_next - m - influence_other_times)));
	 }
	 exp_m = exp(-m
		     + mget(model->topic[k]->variance, w, time) / 2.0);
	 gsl_matrix_set(g, w, k, mget(g, w, k) * exp_m);
       }
     }
     return g;
}

static gsl_matrix* g3_alloc(lda_seq* model,
			    const corpus_seq_t* data,
			    int time) {
     gsl_matrix* g = gsl_matrix_calloc(model->nterms,
				       model->ntopics);
     double exp_m, m, m_next, variance, total;
     for (int k = 0; k < model->ntopics; ++k) {
       for (int w=0; w < model->nterms; ++w) {
	 total = 0.0;
	 for (int i=time; i < model->nseq - 1; ++i) {
	   // This loop is kind of going overboard, but at least we
	   // do this once per E-M iteration.
	   variance = mget(model->topic[k]->variance, w, i + 1);
	   m = mget(model->topic[k]->e_log_prob, w, i);
	   m_next = mget(model->topic[k]->e_log_prob, w, i + 1);
	   exp_m = exp(-m + variance / 2.0);
	   total += (scaled_influence[i - time]
		     * exp_m
		     * (m_next - m + variance));
	 }
	 gsl_matrix_set(g, w, k, total);
       }
     }
     return g;
}

static gsl_matrix* g4_alloc(lda_seq* model,
		    const corpus_seq_t* data,
		    int time) {
     gsl_matrix* g = gsl_matrix_calloc(model->nterms,
				       model->ntopics);
     double exp_m, exp_m_scaled, m, total, variance, w_phi_l;
     for (int k = 0; k < model->ntopics; ++k) {
       for (int w=0; w < model->nterms; ++w) {
	 total = 0.0;
	 for (int i=time; i < model->nseq - 1; ++i) {
	   // This loop is kind of going overboard, but at least we
	   // do this once per E-M iteration.
	   variance = mget(model->topic[k]->variance, w, i + 1);
	   m = mget(model->topic[k]->e_log_prob, w, i);
	   exp_m = exp(-2.0 * m + 2.0 * variance);
	   exp_m_scaled = exp_m * scaled_influence[i - time];
	   for (int j=0; j <= i; ++j) {
	     w_phi_l = mget(model->topic[k]->w_phi_l, w, j);
	     total += exp_m_scaled * w_phi_l * scaled_influence[i - j];
	   }
	 }
	 gsl_matrix_set(g, w, k, total);
       }
     }
     return g;
}

static gsl_matrix* g5_alloc(lda_seq* model,
			    const corpus_seq_t* data,
			    int time) {
     gsl_matrix* g = gsl_matrix_calloc(model->nterms,
				       model->ntopics);
     double exp_m, m, total, variance;
     for (int k = 0; k < model->ntopics; ++k) {
       for (int w=0; w < model->nterms; ++w) {
	 total = 0.0;
	 for (int i=time; i < model->nseq - 1; ++i) {
	   // This loop is kind of going overboard, but at least we
	   // do this once per E-M iteration.
	   variance = mget(model->topic[k]->variance, w, i + 1);
	   m = mget(model->topic[k]->e_log_prob, w, i);
	   exp_m = exp(-2.0 * m + 2.0 * variance);
	   total += exp_m * (scaled_influence[i - time]
			     * scaled_influence[i - time]);
	 }
	 gsl_matrix_set(g, w, k, total);
       }
     }
     return g;
}

 /*
  * compute the likelihood of a sequential corpus under an LDA seq
  * model.  return the likelihood bound.
  *
  */

static void InferDTMSeq(const int K,
			unsigned int iter,
			unsigned int last_iter,
			const corpus_seq_t* data,
			gsl_matrix* gammas,
			gsl_matrix* lhoods,
			lda* lda_model,
			lda_post* post,
			lda_seq* model,
			gsl_matrix** suffstats,
			double* bound) {
  int doc_index = 0;
  for (int t = 0; t < data->len; t++) {
    // Prepare coefficients for the phi updates.  This change is
    // relatively painless.
    make_lda_from_seq_slice(lda_model, model, t);
    int ndocs = data->corpus[t]->ndocs;
    for (int d = 0; d < ndocs; d++) {
      gsl_vector gam   = gsl_matrix_row(gammas, doc_index).vector;
      gsl_vector lhood = gsl_matrix_row(lhoods, doc_index).vector;
      post->gamma = &gam;
      post->doc   = data->corpus[t]->doc[d];
      post->lhood = &lhood;
      double doc_lhood;
      // For now, only do the standard, phi-based update.
      if (iter == 0) {
	doc_lhood = fit_lda_post(d, t,
				 post, NULL,
				 NULL,
				 NULL,
				 NULL,
				 NULL);
      } else {
	doc_lhood = fit_lda_post(d, t, post, model, NULL,
				 NULL, NULL, NULL);
      }
      if (suffstats != NULL) {
	update_lda_seq_ss(t,
			  data->corpus[t]->doc[d],
			  post,
			  suffstats);
      }
      *bound += doc_lhood;
      doc_index++;
    }
  }
}

static void InferDIMSeq(const int K,
			unsigned int iter,
			unsigned int last_iter,
			const char* file_root,
			const corpus_seq_t* data,
			gsl_matrix* gammas,
			gsl_matrix* lhoods,
			lda* lda_model,
			lda_post* post,
			lda_seq* model,
			gsl_matrix** suffstats,
			double* bound) {
  int doc_index = 0;
  for (int t = 0; t < data->len; t++) {
    // Prepare coefficients for the phi updates.  This change is
    // relatively painless.
    gsl_matrix* g = g_alloc(model, data, t);

    gsl_matrix* g3_matrix = g3_alloc(model, data, t);
    gsl_matrix* g4_matrix = g4_alloc(model, data, t);
    gsl_matrix* g5_matrix = g5_alloc(model, data, t);

    make_lda_from_seq_slice(lda_model, model, t);
    int ndocs = data->corpus[t]->ndocs;
    gsl_matrix** phi_t = (gsl_matrix**) malloc(ndocs
					       * sizeof(gsl_matrix*));
    for (int d = 0; d < ndocs; d++) {
      gsl_vector gam   = gsl_matrix_row(gammas, doc_index).vector;
      gsl_vector lhood = gsl_matrix_row(lhoods, doc_index).vector;
      post->gamma = &gam;
      post->doc   = data->corpus[t]->doc[d];
      post->lhood = &lhood;
      double doc_lhood;
      // For now, only do the standard, phi-based update.
      if (iter == 0) {
	doc_lhood = fit_lda_post(d, t,
				 post, NULL,
				 NULL,
				 NULL,
				 NULL,
				 NULL);
      } else {
	doc_lhood = fit_lda_post(d, t, post, model, g,
				 g3_matrix, g4_matrix, g5_matrix);
      }
      if (suffstats != NULL) {
	update_lda_seq_ss(t,
			  data->corpus[t]->doc[d],
			  post,
			  suffstats);
      }
      phi_t[d] = gsl_matrix_alloc(post->doc->nterms, K);
      gsl_matrix_view phi_view = gsl_matrix_submatrix(
        post->phi,
	0, 0, post->doc->nterms, K);
      gsl_matrix_memcpy(phi_t[d], &phi_view.matrix);
      *bound += doc_lhood;
      doc_index++;
    }

    if (t < data->len - 1) {
      if (FLAGS_model == "fixed") {
	double l_bound = update_inf_var_fixed(
          model,
	  data,
	  // Also want a copy of phi for each doc.
	  // Can keep this as a vector for now.
	  phi_t,
	  t,
	  file_root,
	  last_iter || iter >= FLAGS_lda_sequence_min_iter);
	*bound += l_bound;
      }
    }

    for (int d=0; d < ndocs; ++d) {
      gsl_matrix_free(phi_t[d]);
    }
    free(phi_t);
    gsl_matrix_free(g);
    gsl_matrix_free(g3_matrix);
    gsl_matrix_free(g4_matrix);
    gsl_matrix_free(g5_matrix);
  }
}

double lda_seq_infer(lda_seq* model,
		     const corpus_seq_t* data,
		     gsl_matrix** suffstats,
		     gsl_matrix* gammas,
		     gsl_matrix* lhoods,
		     unsigned int iter,
		     unsigned int last_iter,
		     const char* file_root) {
  int K = model->ntopics; int W = model->nterms;
  double bound = 0.0;
  lda* lda_model = new_lda_model(K, W);
  lda_post post;
  post.phi = gsl_matrix_calloc(data->max_nterms, K);
  post.log_phi = gsl_matrix_calloc(data->max_nterms, K);
  post.model = lda_model;

  if (FLAGS_model == "fixed") {
    // First, pre-compute the functions f and g.
    InfluenceTotalFixed(model, data);
    InferDIMSeq(K,
		iter,
		last_iter,
		file_root,
		data,
		gammas,
		lhoods,
		lda_model,
		&post,
		model,
		suffstats,
		&bound);
  } else if (FLAGS_model == "dtm") {
    InferDTMSeq(K,
		iter,
		last_iter,
		data,
		gammas,
		lhoods,
		lda_model,
		&post,
		model,
		suffstats,
		&bound);
  } else {
    printf("Error.  Unknown model.\n");
    exit(1);
  }

  gsl_matrix_free(post.phi);
  gsl_matrix_free(post.log_phi);
  free_lda_model(lda_model);
  return(bound);
}

 /*
  * fit an lda sequence model:
  *
  * . for each time period
  * .     set up lda model with E[log p(w|z)] and \alpha
  * .     for each document
  * .         perform posterior inference
  * .         update sufficient statistics/likelihood
  * .
  * . maximize topics
  *
  */

double fit_lda_seq(lda_seq* m, const corpus_seq_t* data,
		   const corpus_seq_t* heldout, const char* file_root) {
     int K = m->ntopics, W = m->nterms;
     int k;

     // initialize sufficient statistics

     gsl_matrix* topic_suffstats[K];
     for (k = 0; k < K; k++) {
	 topic_suffstats[k] = gsl_matrix_calloc(W, data->len);
     }

     // set up variables

     char name[400];
     gsl_matrix* gammas = gsl_matrix_calloc(data->ndocs, K);
     gsl_matrix* lhoods = gsl_matrix_calloc(data->ndocs, K+1);
     gsl_matrix* heldout_gammas = NULL;
     gsl_matrix* heldout_lhoods = NULL;
     if (heldout != NULL)
     {
	 heldout_gammas = gsl_matrix_calloc(heldout->ndocs, K);
	 heldout_lhoods = gsl_matrix_calloc(heldout->ndocs, K+1);
     }

     double bound = 0, heldout_bound = 0, old_bound;
     double convergence = LDA_SEQ_EM_THRESH + 1;

     char root[400];
     sprintf(root, "%s/lda-seq/", file_root);
     make_directory(root);

     char em_log_filename[400];
     sprintf(em_log_filename, "%s/em_log.dat", file_root);
     FILE* em_log = fopen(em_log_filename, "w");

     // run EM

     int iter = 0;
     // LDA_INFERENCE_MAX_ITER = 1;
     short final_iters_flag = 0;
     unsigned int last_iter = 0;
     while (iter < FLAGS_lda_sequence_min_iter ||
	    ((final_iters_flag == 0 || convergence > LDA_SEQ_EM_THRESH)
	     && iter <= FLAGS_lda_sequence_max_iter)
	    && !last_iter) {

	 if (!(iter < FLAGS_lda_sequence_min_iter ||
	       ((final_iters_flag == 0 || convergence > LDA_SEQ_EM_THRESH)
		&& iter <= FLAGS_lda_sequence_max_iter))) {
	   last_iter = 1;
	 }
	 outlog("\nEM iter %3d\n", iter);
	 outlog("%s", "E step\n");
	 fprintf(em_log, "%17.14e %5.3e\n", bound, convergence);

	 old_bound = bound;
	 gsl_matrix_set_zero(gammas);
	 gsl_matrix_set_zero(lhoods);
	 if (heldout != NULL) {
	   gsl_matrix_set_zero(heldout_gammas);
	   gsl_matrix_set_zero(heldout_lhoods);
	 }

	 for (k = 0; k < K; k++) {
	     gsl_matrix_set_zero(topic_suffstats[k]);
	 }

	 // compute the likelihood of a sequential corpus under an LDA
	 // seq model and find the evidence lower bound.
	 bound = lda_seq_infer(m,
			       data,
			       topic_suffstats,
			       gammas,
			       lhoods,
			       iter,
			       last_iter,
			       file_root);
	 if (heldout != NULL) {
	   heldout_bound = lda_seq_infer(m,
					 heldout,
					 NULL,
					 heldout_gammas,
					 heldout_lhoods,
					 iter,
					 last_iter,
					 file_root);
	 }

	// print out the gammas and likelihoods.
        sprintf(name, "%s/gam.dat", root);
        mtx_fprintf(name, gammas);
        sprintf(name, "%s/lhoods.dat", root);
        mtx_fprintf(name, lhoods);
        if (heldout != NULL)
        {
            sprintf(name, "%s/heldout_lhoods.dat", root);
            mtx_fprintf(name, heldout_lhoods);
            sprintf(name, "%s/heldout_gam.dat", root);
            mtx_fprintf(name, heldout_gammas);
        }
        outlog("%s", "\nM step");

	// fit the variational distribution
        double topic_bnd = fit_lda_seq_topics(m, topic_suffstats);
        bound += topic_bnd;

        write_lda_seq(m, root);
        if ((bound - old_bound) < 0)
        {
            if (LDA_INFERENCE_MAX_ITER == 1)
                LDA_INFERENCE_MAX_ITER = 2;
            if (LDA_INFERENCE_MAX_ITER == 2)
                LDA_INFERENCE_MAX_ITER = 5;
            if (LDA_INFERENCE_MAX_ITER == 5)
                LDA_INFERENCE_MAX_ITER = 10;
            if (LDA_INFERENCE_MAX_ITER == 10)
                LDA_INFERENCE_MAX_ITER = 20;

            outlog( "\nWARNING: bound went down %18.14f; "
		    "increasing var iter to %d\n",
                    bound-old_bound, LDA_INFERENCE_MAX_ITER);
        }

	// check for convergence
        convergence = fabs((bound - old_bound) / old_bound);
        if (convergence < LDA_SEQ_EM_THRESH) {
            final_iters_flag = 1;
            LDA_INFERENCE_MAX_ITER = 500;
            outlog("starting final iterations : max iter = %d\n",
                   LDA_INFERENCE_MAX_ITER);
            convergence = 1.0;
        }
        outlog("\n(%02d) lda seq bound=% 15.7f; "
	       "heldout bound=% 15.7f, conv=% 15.7e\n",
               iter, bound, heldout_bound, convergence);
        iter++;
    }
    return(bound);
}

/*
 * read and write lda sequence variational distribution
 *
 */

void write_lda_seq(const lda_seq* model, const char* root) {
  char name[400];
  sprintf(name, "%sinfo.dat", root);
  FILE* f = fopen(name, "w");

  params_write_int(f, "NUM_TOPICS", model->ntopics);
  params_write_int(f, "NUM_TERMS", model->nterms);
  params_write_int(f, "SEQ_LENGTH", model->nseq);
  params_write_gsl_vector(f, "ALPHA", model->alpha);

  fclose(f);

  int k;
  for (k = 0; k < model->ntopics; k++) {
    const int tmp = k;
    outlog("\nwriting topic %03d", tmp);
    sprintf(name, "%stopic-%03d", root, tmp);
    write_sslm_var(model->topic[tmp], name);
  }

  if (FLAGS_model == "fixed") {
    for (int t=0; t < model->influence->ntime; ++t) {
      sprintf(name, "%sinfluence_time-%03d", root, t);
      outlog("\nwriting influence weights for time %d to %s", t, name);
      gsl_matrix* influence_t = model->influence->doc_weights[t];
      assert(model->ntopics == influence_t->size2);
      mtx_fprintf(name, influence_t);

      sprintf(name, "%srenormalized_influence_time-%03d", root, t);
      outlog("\nwriting influence weights for time %d to %s", t, name);
      influence_t = model->influence->renormalized_doc_weights[t];
      assert(model->ntopics == influence_t->size2);
      mtx_fprintf(name, influence_t);
    }
  }
}

// Read information about a particular model.
// This model should be named "{root}info.dat"
// and should contain the following rows:
// number_topics
// number_times
// alpha, as a gsl vector
lda_seq* read_lda_seq(const char* root, corpus_seq_t* data) {
  char name[400];
  lda_seq* model = (lda_seq*) malloc(sizeof(lda_seq));

  sprintf(name, "%sinfo.dat", root);
  FILE* f = fopen(name, "r");
  if (f == NULL) {
    outlog("Unable to open file %s.  Failing.", name);
    exit(1);
  }
  params_read_int(f, "NUM_TOPICS", &(model->ntopics));
  params_read_int(f, "NUM_TERMS", &(model->nterms));
  params_read_int(f, "SEQ_LENGTH", &(model->nseq));
  params_read_gsl_vector(f, "ALPHA", &(model->alpha));
  fclose(f);

  model->topic = (sslm_var**) malloc(sizeof(sslm_var*) * model->ntopics);

  for (int k = 0; k < model->ntopics; k++) {
    outlog( "reading topic %d", k);
    sprintf(name, "%stopic-%03d", root, k);
    model->topic[k] = read_sslm_var(name);

    model->topic[k]->w_phi_l =
      gsl_matrix_alloc(model->nterms, model->nseq);
    model->topic[k]->w_phi_sum =
      gsl_matrix_alloc(model->nterms, model->nseq);
    model->topic[k]->w_phi_l_sq =
      gsl_matrix_alloc(model->nterms, model->nseq);

    if (FLAGS_model == "dim"
	|| FLAGS_model == "regression") {
      sprintf(name, "%sw_phi_l-%d", root, k);
      mtx_fscanf(name, model->topic[k]->w_phi_l);

      sprintf(name, "%sw_phi_sum-%d", root, k);
      mtx_fscanf(name, model->topic[k]->w_phi_sum);

      sprintf(name, "%sw_phi_l_sq-%d", root, k);
      mtx_fscanf(name, model->topic[k]->w_phi_l_sq);
    }
  }

  if (FLAGS_model == "dim"
      || FLAGS_model == "regression"
      && data != NULL) {
    model->influence = (inf_var*) malloc(sizeof(inf_var));
    model->influence->doc_weights =
      (gsl_matrix**) malloc(sizeof(gsl_matrix*));
    int t;
    model->influence->ntime = model->nseq;
    for (t=0; t < model->nseq; ++t) {
      //	outlog("%d %d", t, model->influence->ntime);
      sprintf(name, "%sinfluence_time-%03d", root, t);
      outlog("\n reading influence weights for time %d from %s", t, name);
      model->influence->doc_weights[t] =
	gsl_matrix_alloc(data->corpus[t]->ndocs,
			 model->ntopics);
      mtx_fscanf(name, model->influence->doc_weights[t]);
    }
  } else {
    model->influence = NULL;
  }

  return(model);
}

/*
 * update lda sequence sufficient statistics from an lda posterior
 *
 */

void update_lda_seq_ss(int time,
                       const doc_t* doc,
                       const lda_post* post,
                       gsl_matrix** ss) {
    int K = post->phi->size2, N = doc->nterms;
    int k, n;

    for (k = 0; k < K; k++)
    {
        gsl_matrix* topic_ss = ss[k];
        for (n = 0; n < N; n++)
        {
            int w = doc->word[n];
            int c = doc->count[n];
            minc(topic_ss, w, time, c * mget(post->phi, n, k));
        }
    }
}

/*
 * fit lda sequence
 *
 */

double fit_lda_seq_topics(lda_seq* model,
                          gsl_matrix** ss) {
    double lhood = 0, lhood_term = 0;
    int k;

    for (k = 0; k < model->ntopics; k++)
    {
        outlog( "\nfitting topic %02d", k);
        lhood_term = fit_sslm(model->topic[k], ss[k]);
        lhood += lhood_term;
    }
    return(lhood);
}

/*
 * allocate lda seq
 *
 */

lda_seq* new_lda_seq(corpus_seq_t* data,
		     int W, int T, int K) {
    lda_seq* model = (lda_seq*) malloc(sizeof(lda_seq));

    model->ntopics = K;
    model->nterms = W;
    model->nseq = T;

    model->alpha = gsl_vector_alloc(K);
    model->topic = (sslm_var**) malloc(sizeof(sslm_var*) * K);

    // Create the vectors of total counts for each time.
    model->influence_sum_lgl = (gsl_matrix**) malloc(sizeof(gsl_matrix*) * K);

    for (int k = 0; k < K; k++) {
      //      model->w_phi_l = (gsl_matrix*) malloc(sizeof(gsl_matrix));
      // model->w_phi_l_sq = (gsl_matrix*) malloc(sizeof(gsl_matrix*));
      model->influence_sum_lgl[k] = gsl_matrix_calloc(W, T);

      model->topic[k] = sslm_var_alloc(W, T);
      if (k < FLAGS_fix_topics) {
	model->topic[k]->chain_variance = 1e-10;
      }
      model->topic[k]->w_phi_l = gsl_matrix_calloc(W, T);
      model->topic[k]->w_phi_sum = gsl_matrix_calloc(W, T);
      model->topic[k]->w_phi_l_sq = gsl_matrix_calloc(W, T);
    }
    model->influence = inf_var_alloc(K, data);

    return(model);
}

/*
 * initialize from sufficient statistics (expected counts).
 *
 */

void init_lda_seq_from_ss(lda_seq* model,
                          double topic_chain_variance,
                          double topic_obs_variance,
                          double alpha,
                          gsl_matrix* init_suffstats) {
    gsl_vector_set_all(model->alpha, alpha);

    for (int k = 0; k < model->ntopics; k++)
    {
        gsl_vector slice = gsl_matrix_column(init_suffstats, k).vector;
        sslm_counts_init(model->topic[k],
                         topic_obs_variance,
                         topic_chain_variance,
                         &slice);
	if (k < FLAGS_fix_topics) {
	  model->topic[k]->chain_variance = 1e-10;
	}
	model->topic[k]->w_phi_l = gsl_matrix_calloc(model->nterms,
						     model->nseq);
	model->topic[k]->w_phi_sum = gsl_matrix_calloc(model->nterms,
						       model->nseq);
	model->topic[k]->w_phi_l_sq = gsl_matrix_calloc(model->nterms,
							model->nseq);
    }

}