/*
# Program to run a Monte Carlo radiation transfer through the 2D
# simulations of GRB jets.
#
# Python code written by D. Lazzati at Oregonstate, C code written by Tyler Parsotan @ Oregon State
# ver 0.1 July 8, 2015
# ver 1.1 July 20, 2015: added record of number of scatterings, included
# 	all terms in weight. Should now give correct light curves.
# ver 1.2 July 21, 2015: added parameter file to keep track of input
# 	params of each simulation

# ver 2.0 July 22, 2015: corrected the problem that arises when there is
# 	no scattering in the time span of one frame. Fixed output arrays dimension.

# ver 2.1 July 25, 2015: fixed bug that did not make the number of
# 	scattering grow with the number of photons.

# ver 3.0 July 28, 2015: using scipy nearest neighbor interpolation to
# 	speed things up. Gained about factor 2

# ver 3.1 July 29, 2015: added radial spread of photon injection points
# ver 3.2 July 31, 2015: added Gamma to the weight of photons!!!

# ver 4.0 Aug 5, 2015: try to speed up by inverting cycle
# ver 4.1 Aug 8, 2015: add spherical test as an option
# ver 4.2 Aug 9, 2015: saving files appending rather than re-writing
# ver 4.3 Aug 11, 2015: corrected error in the calculation of the local temperature
# ver 4.4 Aug 13, 2015: added cylindrical test
# ver 4.5 Aug 18, 2015: fixd various problems pointed by the cylindrical test
# ver 4.6 Aug 21, 2015: corrected mean free path for large radii

# ver 5.0 Aug 25, 2015: corrected problem with high-T electrons and excess scatterings
# ver 5.1 Aug 25, 2015: cleaned-up coding
# ver 5.2 Sept 3, 2015: fixed problem with number of scatterings for multiple injections
 *
 * ver 6.0 Dec 28, 2016: rewrote the code in C, added checkpoint file so if the code is interrupted all the progress wont be lost, made the code only need to be compiled once for a given MC_XXX directory path
                                            so you just need to supply the sub directory of MC_XXX as a command line argument
*/

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <dirent.h>
#include <math.h>
#include <gsl/gsl_rng.h>
//#include "mclib.h"
#include "mclib_3d.h"
#include <omp.h>

#define THISRUN "Science"
#define FILEPATH "/Users/Tylerparsotan/Documents/Box\ Sync/RIKEN_HYDRO_DATA/JP_HYDRODATA/"
#define FILEROOT "u"
#define MC_PATH "PHOTON_TEST/"
//#define MC_PATH "MC_16OI/Single_Photon_Cy_mc_total/"
#define MCPAR "riken_mc.par"

int main(int argc, char **argv)
{
    //compile each time a macro is changed

	// Define variables
	char flash_prefix[200]="";
	char mc_file[200]="" ;
    char this_run[200]=THISRUN;
    char *cyl="Cylindrical";
    char *sph="Spherical";
    char spect;//type of spectrum
    char restrt;//restart or not
    double fps, theta_jmin, theta_jmax ;//frames per second of sim, min opening angle of jet, max opening angle of jet in radians
    double inj_radius_small, inj_radius_large,  ph_weight_suggest ;//radius at chich photons are injected into sim
    int frm0_small, frm0_large,last_frm, frm2_small, frm2_large, j=0, min_photons, max_photons ;//frame starting from, last frame of sim, frame of last injection

    int num_thread=0, angle_count=0;
    int half_threads=floor((num_thread/2));
    int num_angles=0;
    int dim_switch=0;
    double *thread_theta=NULL; //saves ranges of thetas for each thread to go through
    double delta_theta=0;

    //new muliple threads injecting and propagating photons
    const gsl_rng_type *rng_t;
    gsl_rng **rng;
    gsl_rng_env_setup();
    rng_t = gsl_rng_ranlxs0;

    //want to break up simulation by angle and injection frame & have each thread save data in its own folder
    //have each thread check if its directory is made and if its restarting (delete evrything) or if its continuing with a previous simulation
    //the angle and the injection frames will be the names of mc_dir, therefore read mc.par first in MC_XXX directory

    //make strings of proper directories etc.
	snprintf(flash_prefix,sizeof(flash_prefix),"%s%s",FILEPATH,FILEROOT );
	//snprintf(mc_dir,sizeof(flash_prefix),"%s%s",FILEPATH,MC_PATH);
    //snprintf(mc_file,sizeof(flash_prefix),"%s%s",mc_dir, MCPAR);
    snprintf(mc_file,sizeof(flash_prefix),"%s%s%s",FILEPATH, MC_PATH,MCPAR);

    //printf(">> mc.py:  Reading mc.par\n");

    readMcPar(mc_file, &fps, &theta_jmin, &theta_jmax, &delta_theta, &inj_radius_small,&inj_radius_large, &frm0_small , &frm0_large ,&last_frm ,&frm2_small, &frm2_large, &ph_weight_suggest, &min_photons, &max_photons, &spect, &restrt, &num_thread, &dim_switch); //thetas that comes out is in degrees

    //printf("small: %d large: %d weights: %e, %c\n", min_photons, max_photons, ph_weight_suggest, restrt);
    if (num_thread==0)
    {
        num_thread=omp_get_max_threads(); //if user specifies 0 in mc.par, default to max number of threads possible
    }

    half_threads=floor((num_thread/2));
    //thread_theta=malloc(((half_threads)+1)*sizeof(double) );

    rng = (gsl_rng **) malloc((num_thread ) * sizeof(gsl_rng *));
     rng[0] = gsl_rng_alloc (rng_t); //initalize first random number generator to seed the others with random numbers

    for(j=1;j<num_thread;j++)
    {
        rng[j] = gsl_rng_alloc (rng_t);
        gsl_rng_set(rng[j],gsl_rng_get(rng[0]));
    }

    //divide up angles and frame injections among threads DONT WANT NUMBER OF THREADS TO BE ODD
    //assign ranges to array that hold them
    /*
    delta_theta=(theta_jmax-theta_jmin)/(half_threads);
    *(thread_theta+0)=theta_jmin;
    //printf("%e\n", *(thread_theta+0));

    for (j=1;j<(half_threads+1); j++)
    {
        *(thread_theta+j)=*(thread_theta+(j-1))+delta_theta;
        //printf("%e\n", *(thread_theta+j)*180/M_PI);
    }
    */

     //leave angles in degrees here
    num_angles=(int) (((theta_jmax-theta_jmin)/delta_theta)) ;//*(180/M_PI));
    thread_theta=malloc( num_angles *sizeof(double) );
    *(thread_theta+0)=theta_jmin;//*(180/M_PI);
    printf("%e\n", *(thread_theta+0));

    for (j=1;j<(num_angles); j++)
    {
        *(thread_theta+j)=*(thread_theta+(j-1))+delta_theta;
        printf("%e\n", *(thread_theta+j));
    }

    //printf("half_threads: %d\n", half_threads);
    //start parallel section with half the threads possible and assign each thread to its value of theta by rewriting its private value of theta min and max that was read from mcrat
    omp_set_nested(1); //allow for nested parallelization
    //omp_set_dynamic(0);
    //omp_set_num_threads(num_thread);
    //#pragma omp parallel firstprivate(theta_jmin_thread, theta_jmax_thread, mc_dir, phPtr, framestart, scatt_framestart, num_ph, restrt, time_now, st, dirp, file_count, mc_filename, mc_operation, dt_max)
    //printf("%d\n", omp_get_num_threads() );
    //    for (angle_count=(int) (theta_jmin); angle_count< (int) (theta_jmax+1)  ;angle_count=angle_count+delta_theta )

    #pragma omp parallel for num_threads(num_thread) private(angle_count)
    for (angle_count=0; angle_count< num_angles ;angle_count++ )
    {

        printf("%d\t%lf\n", omp_get_thread_num(), delta_theta );
        double inj_radius;
        int frm2, frm0;
        char mc_filename[200]="";
        char mc_filename_2[200]="";
        char mc_operation[200]="";
        char mc_dir[200]="" ;
        int file_count = 0;
        DIR * dirp;
        struct dirent * entry;
        struct stat st = {0};
        double theta_jmin_thread=0, theta_jmax_thread=0;

        theta_jmin_thread= (*(thread_theta+angle_count))*(M_PI/180);//(*(thread_theta+omp_get_thread_num() ));
        theta_jmax_thread= ((*(thread_theta+angle_count))+delta_theta)*(M_PI/180);//(*(thread_theta+omp_get_thread_num()+1 ));
        printf("Thread %d: %0.1lf, %0.1lf \n %d %d\n", omp_get_thread_num(),  theta_jmin_thread*180/M_PI,  theta_jmax_thread*180/M_PI, frm2_small, frm2_large );

        snprintf(mc_dir,sizeof(flash_prefix),"%s%s%0.1lf-%0.1lf/",FILEPATH,MC_PATH, theta_jmin_thread*180/M_PI, theta_jmax_thread*180/M_PI ); //have to add angle into this

        printf(">> Thread %d in  MCRaT: I am working on path: %s \n",omp_get_thread_num(), mc_dir  );

        if ((theta_jmin_thread >= 0) &&  (theta_jmax_thread <= (2*M_PI/180) )) //if within small angle (0-2 degrees) use _small inj_radius and frm2
        {
            inj_radius=inj_radius_small;
            frm2=frm2_small;
            frm0=frm0_small;
        }
        else
        {
            inj_radius=inj_radius_large;
            frm2=frm2_large;
            frm0=frm0_large;
        }
        printf("Thread %d: %0.1lf, %0.1lf \n %d %e %d\n", omp_get_thread_num(),  theta_jmin_thread*180/M_PI,  theta_jmax_thread*180/M_PI, frm2, inj_radius, frm0 );

        //want to also have another set of threads that each has differing ranges of frame injections therefore do nested parallelism here so each thread can read its own checkpoint file
        //#pragma omp parallel num_threads(2) firstprivate(restrt)
        {
            char flash_file[200]="";
            char log_file[200]="";
            FILE *fPtr=NULL; //pointer to log file for each thread
            double *xPtr=NULL,  *yPtr=NULL,  *rPtr=NULL,  *thetaPtr=NULL,  *velxPtr=NULL,  *velyPtr=NULL,  *densPtr=NULL,  *presPtr=NULL,  *gammaPtr=NULL,  *dens_labPtr=NULL;
            double *phiPtr=NULL, *velzPtr=NULL, *zPtr=NULL;
            double *szxPtr=NULL,*szyPtr=NULL, *tempPtr=NULL; //pointers to hold data from FLASH files
            int num_ph=0, array_num=0, ph_scatt_index=0, max_scatt=0, min_scatt=0,i=0; //number of photons produced in injection algorithm, number of array elleemnts from reading FLASH file, index of photon whch does scattering, generic counter
            double dt_max=0, thescatt=0, accum_time=0;
            double  gamma_infinity=0, time_now=0, time_step=0, avg_scatt=0; //gamma_infinity not used?
            double ph_dens_labPtr=0, ph_vxPtr=0, ph_vyPtr=0, ph_tempPtr=0, ph_vzPtr=0;// *ph_cosanglePtr=NULL ;
            double min_r=0, max_r=0;
            int frame=0, scatt_frame=0, frame_scatt_cnt=0, scatt_framestart=0, framestart=0;
            struct photon *phPtr=NULL; //pointer to array of photons

            //if (omp_get_thread_num()==0)
            //{
                //printf( "A:%d Im Thread: %d with ancestor %d working in %s\n", omp_get_num_threads(),  omp_get_thread_num(), omp_get_ancestor_thread_num(1), mc_dir);
            //}

            if (restrt=='c')
            {
                printf(">> mc.py:  Reading checkpoint\n");
                //#pragma omp critical
                {
                    readCheckpoint(mc_dir, &phPtr, frm0, &framestart, &scatt_framestart, &num_ph, &restrt, &time_now);

                /*
                for (i=0;i<num_ph;i++)
                {
                    printf("%e,%e,%e, %e,%e,%e, %e, %e\n",(phPtr+i)->p0, (phPtr+i)->p1, (phPtr+i)->p2, (phPtr+i)->p3, (phPtr+i)->r0, (phPtr+i)->r1, (phPtr+i)->r2, (phPtr+i)->num_scatt );
                }
                */
                if (restrt=='c')
                {
                    printf(">> Thread %d with ancestor %d: Starting from photons injected at frame: %d out of %d\n", omp_get_thread_num(), omp_get_ancestor_thread_num(1),framestart, frm2);
                    printf(">> Thread %d with ancestor %d: Continuing scattering %d photons from frame: %d\n", omp_get_thread_num(), omp_get_ancestor_thread_num(1),num_ph, scatt_framestart);
                    printf(">> Thread %d with ancestor %d: The time now is: %e\n", omp_get_thread_num(), omp_get_ancestor_thread_num(1),time_now);
                }
                else
                {
                    printf(">> Thread %d with ancestor %d: Continuing simulation by injecting photons at frame: %d out of %d\n", omp_get_thread_num(), omp_get_ancestor_thread_num(1),framestart, frm2); //starting with new photon injection is same as restarting sim
                }
                }
            }
            else if (stat(mc_dir, &st) == -1)
            {
                mkdir(mc_dir, 0777); //make the directory with full permissions

                framestart=frm0; //if restarting then start from parameters given in mc.par file
                scatt_framestart=frm0;
            }
            else
            {
                //remove everything from MC directory to ensure no corruption of data if theres other files there besides the mc.par file
                //for a checkpoint implementation, need to find the latest checkpoint file and read it and not delete the files
                #pragma omp critical
                {
                    printf(">> Thread %d with ancestor %d:  Cleaning directory \n",omp_get_thread_num(), omp_get_ancestor_thread_num(1));
                    dirp = opendir(mc_dir);
                    while ((entry = readdir(dirp)) != NULL)
                    {
                        if (entry->d_type == DT_REG) { /* If the entry is a regular file */
                             file_count++; //count how many files are in dorectory
                        }
                    }
                    printf("File count %d\n", file_count);

                    if (file_count>0)
                    {
                        for (i=0;i<=last_frm;i++)
                        {

                            snprintf(mc_filename,sizeof(mc_filename),"%s%s%d%s", mc_dir,"mcdata_",i,"_P0.dat");
                            if(( access( mc_filename, F_OK ) != -1 ) )
                            {
                                snprintf(mc_operation,sizeof(flash_prefix),"%s%s%s%d%s","exec rm ", mc_dir,"mcdata_",i,"_*.dat"); //prepares string to remove *.dat in mc_dir
                                printf("%s\n",mc_operation);
                                system(mc_operation);

                                //snprintf(mc_operation,sizeof(flash_prefix),"%s%s%s%d%s","exec rm ", mc_dir,"mcdata_",i,"_*");
                                //system(mc_operation);
                            }
                        }
                        //snprintf(mc_operation,sizeof(flash_prefix),"%s%s%s","exec rm ", mc_dir,"mcdata_PW_*.dat"); //prepares string to remove *.dat in mc_dir
                        //system(mc_operation);

                        snprintf(mc_operation,sizeof(flash_prefix),"%s%s%s","exec rm ", mc_dir,"mcdata_PW.dat"); //prepares string to remove *.dat in mc_dir
                        system(mc_operation);

                        //snprintf(mc_operation,sizeof(flash_prefix),"%s%s%s","exec rm ", mc_dir,"mc_chkpt_*.dat"); //prepares string to remove *.dat in mc_dir
                        //system(mc_operation);

                        snprintf(mc_operation,sizeof(flash_prefix),"%s%s%s","exec rm ", mc_dir,"mc_output_*.log"); //prepares string to remove *.log in mc_dir
                        system(mc_operation);

                    }
                }
                framestart=frm0; //if restarting then start from parameters given in mc.par file
                scatt_framestart=frm0;

            }

            dt_max=1.0/fps;
            //#pragma omp barrier

            snprintf(log_file,sizeof(log_file),"%s%s",mc_dir,"mc_output.log" );
            printf("%s\n",log_file);
            fPtr=fopen(log_file, "w");

            fprintf(fPtr, "%d Im Thread: %d with ancestor %d Starting on Frame: %d scatt_framestart: %d\n", omp_get_num_threads(),  omp_get_thread_num(), omp_get_ancestor_thread_num(1), framestart, scatt_framestart);
            fflush(fPtr);
            //fclose(fPtr);
            //#pragma omp barrier
            //exit(0);
            //loop over frames
            //for a checkpoint implementation, start from the last saved "frame" value and go to the saved "frm2" value

            //#pragma omp for
            for (frame=framestart;frame<=frm2;frame++)
            {
                 if (restrt=='r')
                 {
                    time_now=frame/fps; //for a checkpoint implmentation, load the saved "time_now" value when reading the ckeckpoint file otherwise calculate it normally
                 }

                //printf(">> mc.py: Working on Frame %d\n", frame);
                fprintf(fPtr,"%d Im Thread: %d with ancestor %d Working on Frame: %d\n", omp_get_num_threads(),  omp_get_thread_num(), omp_get_ancestor_thread_num(1), frame);
                fflush(fPtr);

                if (restrt=='r')
                {

                    //exit(0);
                    //read in FLASH file
                    //for a checkpoint implmentation, dont need to read the file yet
                    if (dim_switch==0)
                    {
                        //put proper number at the end of the flash file
                        modifyFlashName(flash_file, flash_prefix, frame, dim_switch);
                        fprintf(fPtr,">> Im Thread: %d with ancestor %d: Opening FLASH file %s\n",omp_get_thread_num(), omp_get_ancestor_thread_num(1), flash_file);
                        fflush(fPtr);

                        #pragma omp critical
                        {
                            readAndDecimate(flash_file, inj_radius, fps, &xPtr,  &yPtr,  &szxPtr, &szyPtr, &rPtr,\
                                &thetaPtr, &velxPtr,  &velyPtr,  &densPtr,  &presPtr,  &gammaPtr,  &dens_labPtr, &tempPtr, &array_num, 1, min_r, max_r, fPtr);
                        }
                    }
                    else
                    {
                        read_hydro(FILEPATH, frame, inj_radius, &xPtr,  &yPtr, &zPtr,  &szxPtr, &szyPtr, &rPtr,\
                                &thetaPtr, &phiPtr, &velxPtr,  &velyPtr, &velzPtr,  &densPtr,  &presPtr,  &gammaPtr,  &dens_labPtr, &tempPtr, &array_num, 1, min_r, max_r, fps, fPtr);
                    }

                    //check for run type
                    if(strcmp(cyl, this_run)==0)
                    {
                        //printf("In cylindrical prep\n");
                        cylindricalPrep(gammaPtr, velxPtr, velyPtr, densPtr, dens_labPtr, presPtr, tempPtr, array_num);
                    }
                    else if (strcmp(sph, this_run)==0)
                    {
                        sphericalPrep(rPtr, xPtr, yPtr,gammaPtr, velxPtr, velyPtr, densPtr, dens_labPtr, presPtr, tempPtr, array_num );
                    }

                    //determine where to place photons and how many should go in a given place
                    //for a checkpoint implmentation, dont need to inject photons, need to load photons' last saved data
                    fprintf(fPtr,">>  Thread: %d with ancestor %d: Injecting photons\n",omp_get_thread_num(), omp_get_ancestor_thread_num(1));
                    fflush(fPtr);

                    if (dim_switch==0)
                    {
                        photonInjection(&phPtr, &num_ph, inj_radius, ph_weight_suggest, min_photons, max_photons,spect, array_num, fps, theta_jmin_thread, theta_jmax_thread, xPtr, yPtr, szxPtr, szyPtr,rPtr,thetaPtr, tempPtr, velxPtr, velyPtr,rng[omp_get_thread_num()] );
                    }
                    else
                    {
                        photonInjection3D(&phPtr, &num_ph, inj_radius, ph_weight_suggest, min_photons, max_photons,spect, array_num, fps, theta_jmin_thread, theta_jmax_thread, xPtr, yPtr, zPtr, szxPtr, szyPtr,rPtr,thetaPtr, phiPtr, tempPtr, velxPtr, velyPtr, velzPtr, rng[omp_get_thread_num()] );
                    }

                    //printf("%d\n",num_ph); //num_ph is one more photon than i actually have
                    /*
                    for (i=0;i<num_ph;i++)
                        printf("%e,%e,%e \n",(phPtr+i)->r0, (phPtr+i)->r1, (phPtr+i)->r2 );
                    */
                }

                //scatter photons all the way thoughout the jet
                //for a checkpoint implmentation, start from the last saved "scatt_frame" value eh start_frame=frame or start_frame=cont_frame
                if (restrt=='r')
                {
                    scatt_framestart=frame; //have to make sure that once the inner loop is done and the outer loop is incrememnted by one the inner loop starts at that new value and not the one read by readCheckpoint()
                }

                for (scatt_frame=scatt_framestart;scatt_frame<=last_frm;scatt_frame++)
                {
                    fprintf(fPtr,">>\n");
                    fprintf(fPtr,">> Thread %d with ancestor %d : Working on photons injected at frame: %d out of %d\n", omp_get_thread_num(), omp_get_ancestor_thread_num(1),frame, frm2);
                    fprintf(fPtr,">> Thread %d with ancestor %d: %s - Working on frame %d\n",omp_get_thread_num(), omp_get_ancestor_thread_num(1), THISRUN, scatt_frame);
                    fprintf(fPtr,">> Thread %d with ancestor %d: Opening file...\n", omp_get_thread_num(), omp_get_ancestor_thread_num(1));
                    fflush(fPtr);

                    if (dim_switch==0)
                    {
                        //put proper number at the end of the flash file
                        modifyFlashName(flash_file, flash_prefix, scatt_frame, dim_switch);
                        #pragma omp critical
                        {
                            phMinMax(phPtr, num_ph, &min_r, &max_r);
                            readAndDecimate(flash_file, inj_radius, fps, &xPtr,  &yPtr,  &szxPtr, &szyPtr, &rPtr,\
                                &thetaPtr, &velxPtr,  &velyPtr,  &densPtr,  &presPtr,  &gammaPtr,  &dens_labPtr, &tempPtr, &array_num, 0, min_r, max_r, fPtr);
                        }
                    }
                    else
                    {
                        phMinMax(phPtr, num_ph, &min_r, &max_r);

                        read_hydro(FILEPATH, scatt_frame, inj_radius, &xPtr,  &yPtr, &zPtr,  &szxPtr, &szyPtr, &rPtr,\
                                &thetaPtr, &phiPtr, &velxPtr,  &velyPtr, &velzPtr,  &densPtr,  &presPtr,  &gammaPtr,  &dens_labPtr, &tempPtr, &array_num, 0, min_r, max_r, fps, fPtr);
                    }

                    //check for run type
                    if(strcmp(cyl, this_run)==0)
                    {
                        //printf("In cylindrical prep\n");
                        cylindricalPrep(gammaPtr, velxPtr, velyPtr, densPtr, dens_labPtr, presPtr, tempPtr, array_num);
                    }
                    else if (strcmp(sph, this_run)==0)
                    {
                        sphericalPrep(rPtr, xPtr, yPtr,gammaPtr, velxPtr, velyPtr, densPtr, dens_labPtr, presPtr, tempPtr, array_num );
                    }
                        //printf("The result of read and decimate are arrays with %d elements\n", array_num);

                    fprintf(fPtr,">> Thread %d with ancestor %d: propagating and scattering %d photons\n", omp_get_thread_num(), omp_get_ancestor_thread_num(1),num_ph);
                    fflush(fPtr);

                    frame_scatt_cnt=0;
                    while (time_now<((scatt_frame+1)/fps))
                    {
                        //if simulation time is less than the simulation time of the next frame, keep scattering in this frame
                        //go through each photon and find blocks closest to each photon and properties of those blocks to calulate mean free path
                        //and choose the photon with the smallest mfp and calculate the timestep

                        ph_scatt_index=findNearestPropertiesAndMinMFP(phPtr, num_ph, array_num, &time_step, xPtr,  yPtr, zPtr, velxPtr,  velyPtr,  velzPtr, dens_labPtr, tempPtr,\
                            &ph_dens_labPtr, &ph_vxPtr, &ph_vyPtr, &ph_vzPtr, &ph_tempPtr, rng[omp_get_thread_num()], dim_switch);

                        //printf("In main: %e, %d, %e, %e\n", *(ph_num_scatt+ph_scatt_index), ph_scatt_index, time_step, time_now);
                        printf("In main: %e, %d, %e, %e\n",((phPtr+ph_scatt_index)->num_scatt), ph_scatt_index, time_step, time_now);

                         if (time_step<dt_max)
                        {

                            //update number of scatterings and time
                            //(*(ph_num_scatt+ph_scatt_index))+=1;
                            ((phPtr+ph_scatt_index)->num_scatt)+=1;
                            frame_scatt_cnt+=1;
                            time_now+=time_step;

                            updatePhotonPosition(phPtr, num_ph, time_step);

                            //scatter the photon
                            //printf("Passed Parameters: %e, %e, %e\n", (ph_vxPtr), (ph_vyPtr), (ph_tempPtr));

                            photonScatter( (phPtr+ph_scatt_index), (ph_vxPtr), (ph_vyPtr), ph_vzPtr, (ph_tempPtr), rng[omp_get_thread_num()] , dim_switch, fPtr);

                            //if (frame_scatt_cnt%1000 == 0)
                            {
                                fprintf(fPtr,"Scattering Number: %d\n", frame_scatt_cnt);
                                fprintf(fPtr,"The local temp is: %e\n", (ph_tempPtr));
                                fprintf(fPtr,"Average photon energy is: %e\n", averagePhotonEnergy(phPtr, num_ph)); //write function to average over the photons p0 and then do (*3e10/1.6e-9)
                                fflush(fPtr);
                            }

                        }
                        else
                        {
                            time_now+=dt_max;

                            //for each photon update its position based on its momentum

                            updatePhotonPosition(phPtr, num_ph, dt_max);
                        }

                        //printf("In main 2: %e, %d, %e, %e\n", ((phPtr+ph_scatt_index)->num_scatt), ph_scatt_index, time_step, time_now);

                    }

                //get scattering statistics
                phScattStats(phPtr, num_ph, &max_scatt, &min_scatt, &avg_scatt);

                fprintf(fPtr,"The number of scatterings in this frame is: %d\n", frame_scatt_cnt);
                fprintf(fPtr,"The last time step was: %lf.\nThe time now is: %lf\n", time_step,time_now);
                fprintf(fPtr,"The maximum number of scatterings for a photon is: %d\nThe minimum number of scattering for a photon is: %d\n", max_scatt, min_scatt);
                fprintf(fPtr,"The average number of scatterings thus far is: %lf\n", avg_scatt);
                fflush(fPtr);

                printPhotons(phPtr, num_ph,  scatt_frame , frame, mc_dir);
                exit(0);
                //for a checkpoint implmentation,save the checkpoint file here after every 5 frames or something
                //save the photons data, the scattering number data, the scatt_frame value, and the frame value
                //WHAT IF THE PROGRAM STOPS AFTER THE LAST SCATT_FRAME, DURING THE FIRST SCATT_FRAME OF NEW FRAME VARIABLE - save restrt variable as 'r'
                fprintf(fPtr, ">> Thread %d with ancestor %d: Making checkpoint file\n", omp_get_thread_num(), omp_get_ancestor_thread_num(1));
                fflush(fPtr);
                saveCheckpoint(mc_dir, frame, scatt_frame, num_ph, time_now, phPtr, last_frm);

                free(xPtr);free(yPtr);free(szxPtr);free(szyPtr);free(rPtr);free(thetaPtr);free(velxPtr);free(velyPtr);free(densPtr);free(presPtr);
                free(gammaPtr);free(dens_labPtr);free(tempPtr);
                xPtr=NULL; yPtr=NULL;  rPtr=NULL;thetaPtr=NULL;velxPtr=NULL;velyPtr=NULL;densPtr=NULL;presPtr=NULL;gammaPtr=NULL;dens_labPtr=NULL;
                szxPtr=NULL; szyPtr=NULL; tempPtr=NULL;
                }
                restrt='r';//set this to make sure that the next iteration of propogating photons doesnt use the values from the last reading of the checkpoint file
                free(phPtr);
                phPtr=NULL;

            }

        }//end omp parallel inner section

        //merge files from each worker thread within a directory
        //dirFileMerge(mc_dir, frm0, last_frm);

    } //end omp parallel section

    //gsl_rng_free (rand);
    //free rand number generator
    for (j=0;j<num_thread;j++)
    {
        gsl_rng_free(rng[j]);
    }
    free(rng);
    free(thread_theta);

	return 0;

}


/*
 * SPDX-License-Identifier:  BSD-3-Clause
 *
 * sph_linked_list.c :
 *     Header containing the declarions for several
 *     cell linked list operations, including hash
 *     calculations, setup hash tables and neighbour finding.
 *
 * (C) Copyright 2021 José Hugo Elsas
 * Author: José Hugo Elsas <jhelsas@gmail.com>
 *
 */

#include <math.h>
#include <ctype.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <limits.h>
#include <unistd.h>
#include <stdbool.h>
#include <sys/time.h>
#include <inttypes.h>

#include <omp.h>
#include <gsl/gsl_math.h>
#include <gsl/gsl_rng.h>
#include <gsl/gsl_randist.h>
#include <gsl/gsl_heapsort.h>

#include "MZC3D64.h"
#include "MZC2D64.h"
#include "sph_data_types.h"
#include "sph_linked_list.h"

#define dbg false

int safe_free_box(linkedListBox *box){
	kh_destroy(0, box->hbegin);
	kh_destroy(1, box->hend);
	free(box);

	return 0;
}

int compare_int64_t(const void *p,const void *q){
	int64_t *data1,*data2;
	data1 = (int64_t*)p;
	data2 = (int64_t*)q;

	if(data1[0] < data2[0])         // data[0] is the hash value,
		return -1;                    // data[1] is the corresponding
	else if(data1[0] == data2[0])   // in the unsorted array
		return 0;
	else
		return 1;
}

#define safe_check_alloc(ptr,N,dtype) {                                          \
                                       (ptr) = (dtype*)malloc((N)*sizeof(dtype));\
                                       if((ptr)==NULL){                          \
                                         success=0;                              \
                                         goto finishlabel;                       \
                                       }                                         \
                                      }

#define safe_check__aligned_alloc(ptr,alignment,N,dtype) {                                        \
                                       (ptr) = (dtype*)aligned_alloc(alignment,(N)*sizeof(dtype));\
                                       if((ptr)==NULL){                                           \
                                         success=0;                                               \
                                         goto finishlabel;                                        \
                                       }                                                          \
                                      }

#define safe_free(ptr) {               \
                        if(ptr != NULL)\
                          free(ptr);   \
                       }

/*
 *  Function SPHparticle_SoA_malloc:
 *    Easily allocate the SPHparticle array
 *
 *    Arguments:
 *       N          <int>     : Number of Particles
 *     lsph <SPHparticle**>   : SoA Particle struct reference
 */
int SPHparticle_SoA_malloc(int N,SPHparticle **lsph){
  int success=1;
  //const int alignment = 32;
  (*lsph) = (SPHparticle*)malloc(1*sizeof(SPHparticle));
  if(lsph==NULL){
    success = 0;
    goto finishlabel;
  }
  (*lsph)->x  = NULL; (*lsph)->y  = NULL; (*lsph)->z  = NULL;
  (*lsph)->ux = NULL; (*lsph)->uy = NULL; (*lsph)->uz = NULL;
  (*lsph)->Fx = NULL; (*lsph)->Fy = NULL; (*lsph)->Fz = NULL;
  (*lsph)->nu = NULL; (*lsph)->rho= NULL;
  (*lsph)->id = NULL; (*lsph)->hash= NULL;

  safe_check_alloc((*lsph)->x   , N ,double);
  safe_check_alloc((*lsph)->y   , N ,double);
  safe_check_alloc((*lsph)->z   , N ,double);
  safe_check_alloc((*lsph)->ux  , N ,double);
  safe_check_alloc((*lsph)->uy  , N ,double);
  safe_check_alloc((*lsph)->uz  , N ,double);
  safe_check_alloc((*lsph)->Fx  , N ,double);
  safe_check_alloc((*lsph)->Fy  , N ,double);
  safe_check_alloc((*lsph)->Fz  , N ,double);
  safe_check_alloc((*lsph)->nu  , N ,double);
  safe_check_alloc((*lsph)->rho , N ,double);
  safe_check_alloc((*lsph)->id  , N ,int64_t);
  safe_check_alloc((*lsph)->hash,2*N,int64_t);

  // SPHparticle_SoA_malloc((*lsph)->x   , 32, N ,double);
  // SPHparticle_SoA_malloc((*lsph)->y   , 32, N ,double);
  // SPHparticle_SoA_malloc((*lsph)->z   , 32, N ,double);
  // SPHparticle_SoA_malloc((*lsph)->ux  , 32, N ,double);
  // SPHparticle_SoA_malloc((*lsph)->uy  , 32, N ,double);
  // SPHparticle_SoA_malloc((*lsph)->uz  , 32, N ,double);
  // SPHparticle_SoA_malloc((*lsph)->Fx  , 32, N ,double);
  // SPHparticle_SoA_malloc((*lsph)->Fy  , 32, N ,double);
  // SPHparticle_SoA_malloc((*lsph)->Fz  , 32, N ,double);
  // SPHparticle_SoA_malloc((*lsph)->nu  , 32, N ,double);
  // SPHparticle_SoA_malloc((*lsph)->rho , 32, N ,double);
  // SPHparticle_SoA_malloc((*lsph)->id  , 32, N ,int64_t);
  // SPHparticle_SoA_malloc((*lsph)->hash, 32,2*N,int64_t);

finishlabel:

  if(success)
    return 0;
  else{
    if(*lsph==NULL)
      return 1;

    safe_free((*lsph)->x);  safe_free((*lsph)->y);  safe_free((*lsph)->z);
    safe_free((*lsph)->ux); safe_free((*lsph)->uy); safe_free((*lsph)->uz);
    safe_free((*lsph)->Fx); safe_free((*lsph)->Fy); safe_free((*lsph)->Fz);
    safe_free((*lsph)->nu); safe_free((*lsph)->rho);
    safe_free((*lsph)->id); safe_free((*lsph)->hash);

    return 1;
  }
}

/*
 *  Function SPHparticleSOA_safe_free:
 *    Easily free the SPHparticle array
 *
 *    Arguments:
 *       N          <int>     : Number of Particles
 *     lsph <SPHparticle**>   : SoA Particle struct reference
 */
int SPHparticleSOA_safe_free(int N,SPHparticle **lsph){

  if(*lsph==NULL)
    return 1;

  safe_free((*lsph)->x);  safe_free((*lsph)->y);  safe_free((*lsph)->z);
  safe_free((*lsph)->ux); safe_free((*lsph)->uy); safe_free((*lsph)->uz);
  safe_free((*lsph)->Fx); safe_free((*lsph)->Fy); safe_free((*lsph)->Fz);
  safe_free((*lsph)->nu); safe_free((*lsph)->rho);
  safe_free((*lsph)->id); safe_free((*lsph)->hash);

  free((*lsph));

  return 0;
}

/*
 *  Function gen_unif_rdn_pos:
 *    Generate particles positions uniformely in a [0,1]^3 box
 *
 *    Arguments:
 *       N          <int>     : Number of Particles
 *     lsph <SPHparticle**>   : SoA Particles array
 *     seed         <int>     : seed for the PRNG
 */
int gen_unif_rdn_pos(int64_t N, int seed, SPHparticle *lsph){

	const gsl_rng_type *T=NULL;
	gsl_rng *r=NULL;

	if(lsph==NULL)
		return 1;

	gsl_rng_env_setup();                      // Initialize environment for PRNG generator

	T = gsl_rng_default;                      // Set generator type as default type
	r = gsl_rng_alloc(T);                     // allocate state for it
	gsl_rng_set(r,seed);                      // set seed accordingly

	for(int64_t i=0;i<N;i+=1){
		lsph->x[i] = gsl_rng_uniform(r);        // Generate a uniform value for X component between 0 and 1
		lsph->y[i] = gsl_rng_uniform(r);        // Generate a uniform value for Y component between 0 and 1
		lsph->z[i] = gsl_rng_uniform(r);        // Generate a uniform value for Z component between 0 and 1

		lsph->ux[i] = 0.0; lsph->Fx[i] = 0.0;   // set velocity and force components as zero
		lsph->uy[i] = 0.0; lsph->Fy[i] = 0.0;   // as they will not be needed
    lsph->uz[i] = 0.0; lsph->Fz[i] = 0.0;

		lsph->nu[i]   = 1.0/N;                  // set each particle mass as to the total mass be 1
		lsph->rho[i]  = 0.0;                    // initialize density to zero
		lsph->id[i]   = (int64_t) i;            // set the particle's id
		lsph->hash[2*i+0] = (int64_t) 0;        // initialize particle's hash value as zero
		lsph->hash[2*i+1] = (int64_t) i;        // set particle's index position by the position in current array
	}

	gsl_rng_free(r);                          // release the PRNG

	return 0;
}

/*
 *  Function gen_unif_rdn_pos_box:
 *    Generate particles positions uniformely in a
 *    [Xmin,Xmax]x[Ymin,Ymax]x[Zmin,Zmax] box
 *
 *    Arguments:
 *       N          <int>     : Number of Particles
 *     lsph <SPHparticle**>   : SoA Particles array
 *     seed         <int>     : seed for the PRNG
 *      box  <linkedListBox*> : Cell Linked List box
 */
int gen_unif_rdn_pos_box(int64_t N, int seed, linkedListBox *box,SPHparticle *lsph){

  const gsl_rng_type *T=NULL;
  gsl_rng *r=NULL;

  if(lsph==NULL)
    return 1;

	gsl_rng_env_setup();                      // Initialize environment for PRNG generator

	T = gsl_rng_default;                      // Set generator type as default type
	r = gsl_rng_alloc(T);                     // allocate state for it
	gsl_rng_set(r,seed);                      // set seed accordingly

  for(int64_t i=0;i<N;i+=1){
    lsph->x[i] = gsl_rng_uniform(r)*(box->Xmax-box->Xmin) // Generate a uniform value for X
                     + box->Xmin;                         // component between Xmin and Xmax
    lsph->y[i] = gsl_rng_uniform(r)*(box->Ymax-box->Ymin) // Generate a uniform value for Y
                     + box->Ymin;                         // component between Ymin and Ymax
    lsph->z[i] = gsl_rng_uniform(r)*(box->Zmax-box->Zmin) // Generate a uniform value for Z
                     + box->Zmin;                         // component between Zmin and Zmax

		lsph->ux[i] = 0.0; lsph->Fx[i] = 0.0;   // set velocity and force components as zero
		lsph->uy[i] = 0.0; lsph->Fy[i] = 0.0;   // as they will not be needed
    lsph->uz[i] = 0.0; lsph->Fz[i] = 0.0;

		lsph->nu[i]   = 1.0/N;                  // set each particle mass as to the total mass be 1
		lsph->rho[i]  = 0.0;                    // initialize density to zero
		lsph->id[i]   = (int64_t) i;            // set the particle's id
		lsph->hash[2*i+0] = (int64_t) 0;        // initialize particle's hash value as zero
		lsph->hash[2*i+1] = (int64_t) i;        // set particle's index position by the position in current array
  }

  gsl_rng_free(r);

  return 0;
}

/*
 *  Function compute_hash_MC3D:
 *    Compute the Morton Z hashes for all particles based on
 *    the parameters given by the Cell Linked List box
 *
 *    Arguments:
 *       N          <int>     : Number of Particles
 *     lsph <SPHparticle**>   : SoA Particles array
 *     box  <linkedListBox*>  : Box of cell linked lists
 */
int compute_hash_MC3D(int64_t N, SPHparticle *lsph, linkedListBox *box){

	if(lsph==NULL)
		return 1;

	if(box==NULL)
		return 1;

	const double etax = box->Nx/(box->Xmax - box->Xmin);
	const double etay = box->Ny/(box->Ymax - box->Ymin);
	const double etaz = box->Nz/(box->Zmax - box->Zmin);

	for(int64_t i=0;i<N;i+=1){
		uint32_t kx,ky,kz;
		kx = (uint32_t)(etax*(lsph->x[i] - box->Xmin));      // Compute cell index in the X direction
		ky = (uint32_t)(etay*(lsph->y[i] - box->Ymin));      // Compute cell index in the Y direction
		kz = (uint32_t)(etaz*(lsph->z[i] - box->Zmin));      // Compute cell index in the Z direction

		if((kx<0)||(ky<0)||(kz<0))                           // The can't be negative indexes
			return 1;
		else if((kx>=box->Nx)||(ky>=box->Nx)||(kz>=box->Nx)) // Nor indexes greater than the upper bounds
			return 1;
		else{
			lsph->hash[2*i+0] = ullMC3Dencode(kx,ky,kz);         // If ok, compute the corresponding Morton Z hash
      lsph->hash[2*i+1] = i;
		}
	}

	return 0;
}

/*
 * To swap two arrays, copy data from the original array into
 * a swap buffer in the right order, and then overrite the original array.
 */
#define swap_loop(N,lsph,temp_swap,member,type) for(int64_t i=0;i<(N);i+=1)                            \
																	 	              (temp_swap)[i] = (lsph)->member[(lsph)->hash[2*i+1]];\
																		            memcpy((lsph)->member,temp_swap,(N)*sizeof(type))

/*
 *  Function reorder_lsph_SoA:
 *    reorder the array according to the indexes reordered according to
 *    their hashes.
 *
 *    Arguments:
 *       N          <int>     : Number of Particles
 *     lsph <SPHparticle**>   : SoA Particles array
 *     swap_arr  <void*>      : buffer for reordering data in the array
 */
int reorder_lsph_SoA(int N, SPHparticle *lsph, void *swap_arr){

	int64_t *int64_temp_swap = (int64_t *)swap_arr;
	swap_loop(N,lsph,int64_temp_swap,id ,int64_t);
	double *double_temp_swap = (double *)swap_arr;
	swap_loop(N,lsph,double_temp_swap,nu ,double);
	swap_loop(N,lsph,double_temp_swap,rho,double);
	swap_loop(N,lsph,double_temp_swap,x  ,double);
	swap_loop(N,lsph,double_temp_swap,y  ,double);
	swap_loop(N,lsph,double_temp_swap,z  ,double);
	swap_loop(N,lsph,double_temp_swap,ux ,double);
	swap_loop(N,lsph,double_temp_swap,uy ,double);
	swap_loop(N,lsph,double_temp_swap,uz ,double);
	swap_loop(N,lsph,double_temp_swap,Fx ,double);
	swap_loop(N,lsph,double_temp_swap,Fy ,double);
	swap_loop(N,lsph,double_temp_swap,Fz ,double);

	return 0;
}

/*
 *  Function setup_interval_hashtables:
 *    Setup the information where each cell begins and ends in the SPH
 *    Array in a pair of hash tables for quick consultation.
 *    This effectively completes the creation of the implicit
 *    cell linked list structure initiated by sorting the array
 *    according to hashes.
 *
 *    Arguments:
 *       N          <int>     : Number of Particles
 *     lsph <SPHparticle**>   : SoA Particles array
 *     box  <linkedListBox*>  : Box of cell linked lists
 */
int setup_interval_hashtables(int64_t N,SPHparticle *lsph,linkedListBox *box){

	int ret;
	int64_t hash0 = lsph->hash[2*0];                             // Store the first hash in hash0
	khiter_t kbegin,kend;

	if(lsph==NULL)
		return 1;

	if(box==NULL)
		return 1;

	kbegin = kh_put(0, box->hbegin, lsph->hash[2*0], &ret);      // Insert start hash as first value
	kh_value(box->hbegin, kbegin) = (int64_t)0;                  // Set the start index of first cell as 0
	for(int64_t i=0;i<N;i+=1){
		lsph->hash[i] = lsph->hash[2*i];                           // Clean the first half of the array only to contain hashes
		if(lsph->hash[i] == hash0)                                 // If the the particle hash is the same as the previous
			continue;                                                // skip because they are in the same cell
		hash0 = lsph->hash[i];                                     // otherwise, store the new hash in hash0

		kend   = kh_put(1, box->hend  , lsph->hash[i-1], &ret);    // If so, insert the previous hash as an cell end
		kh_value(box->hend  , kend)   = i;                         // and set this index as the upper index of that cell

		kbegin = kh_put(0, box->hbegin, lsph->hash[i  ], &ret);    // and then insert this new hash as the begining
		kh_value(box->hbegin, kbegin) = i;                         // of a new cell in with the lower index as this index
	}
	kend   = kh_put(1, box->hend  , lsph->hash[2*(N-1)], &ret);  // finally set the last hash for the end of the last cell,
	kh_value(box->hend  , kend)   = N;                           // and the number of particles as the index of the last cell

	return 0;
}

/*
 *  Function neighbour_hash_3d:
 *    Find the hashes of all valid cells that neighbor cell with
 *    int hash.
 *
 *    Arguments:
 *       hash    <int64_t>    : Hash of the central cell
 *       nblist <int64_t*>    : Array for hashes of the neighbor cells
 *       width     <int>      : Width of the neighborhood
 *     box  <linkedListBox*>  : Box of cell linked lists
 */
int neighbour_hash_3d(int64_t hash,int64_t *nblist,int width, linkedListBox *box){
	int idx=0,kx=0,ky=0,kz=0;

	kx = ullMC3DdecodeX(hash);   // To find a given cell's neighbors, start by first
	ky = ullMC3DdecodeY(hash);   // recovering the integer index of that cell in
	kz = ullMC3DdecodeZ(hash);   // the overall box domain

	for(int ix=-width;ix<=width;ix+=1){            // iterate over all cells surrounding the original cell
		for(int iy=-width;iy<=width;iy+=1){          // as long as they are within width of the original cell
			for(int iz=-width;iz<=width;iz+=1){
				if((kx+ix<0)||(ky+iy<0)||(kz+iz<0))                              // if the cell is off bounds for having lower index then 0
					nblist[idx++] = -1;                                            // annotate the cell as invalid
				else if( (kx+ix>=box->Nx)||(ky+iy>=box->Ny)||(kz+iz>=box->Nz) )  // or if it is off bounds for having higher index then
					nblist[idx++] = -1;                                            // the maximum, also annotate as invalid
				else if( kh_get(0, box->hbegin, ullMC3Dencode(kx+ix,ky+iy,kz+iz)) == kh_end(box->hbegin) ) // Also, if no corresponding hash
					nblist[idx++] = -1;                                            // is not found in the hash table, also annotate as invalid
				else                                                             // at last, if it has no problems
					nblist[idx++] = ullMC3Dencode(kx+ix,ky+iy,kz+iz);              // annotate the hash of the corresponding cell
			}
		}
	}

	return 0;
}

/*
 *  Function count_box_pairs:
 *    Count the number of valid cell pairs
 *
 *    Arguments:
 *     box  <linkedListBox*>  : Box of cell linked lists
 */
int count_box_pairs(linkedListBox *box){
  int64_t pair_count = 0;                                                     // initialize the number of pairs as zero

  for (khint32_t kbegin = kh_begin(box->hbegin); kbegin != kh_end(box->hbegin); kbegin++){ // iterate over the cells
    int64_t node_hash=-1;
    int64_t nblist[(2*box->width+1)*(2*box->width+1)*(2*box->width+1)];

    if (kh_exist(box->hbegin, kbegin)){                                       // check if the cell is valid
      node_hash = kh_key(box->hbegin, kbegin);                                // get the hash of that cell

      neighbour_hash_3d(node_hash,nblist,box->width,box);                     // fetch the list of neighbors
      for(int j=0;j<(2*box->width+1)*(2*box->width+1)*(2*box->width+1);j+=1){ // and iterate over those neighbors
        if(nblist[j]>=0){                                                     // if the neighbor is valid
          pair_count += 1;                                                    // count it as a valid for a pair
        }
      }
    }
  }

  return pair_count;
}

/*
 *  Function setup_box_pairs:
 *    Pre-compute indexes for all valid cell pairs.
 *
 *  Arguments:
 *     box  <linkedListBox*>  : Box of cell linked lists
 *  Returns:
 *     node_begin <int64_t*>  : Array for receiver cell begin indexes
 *     node_end   <int64_t*>  : Array for receiver cell end indexes
 *     nb_begin   <int64_t*>  : Array for sender cell begin indexes
 *     nb_end     <int64_t*>  : Array for sender cell end indexes
 */
int setup_box_pairs(linkedListBox *box,
                    int64_t *node_begin,int64_t *node_end,
                    int64_t *nb_begin,int64_t *nb_end)
{
  int64_t pair_count = 0,particle_pair_count = 0;

  for (khint32_t kbegin = kh_begin(box->hbegin); kbegin != kh_end(box->hbegin); kbegin++){    // iterate over the cells
    int64_t node_hash=-1;
    int64_t nblist[(2*box->width+1)*(2*box->width+1)*(2*box->width+1)];

    if (kh_exist(box->hbegin, kbegin)){                                                       // check if the cell is valid
      khint32_t kend = kh_get(1, box->hend, kh_key(box->hbegin, kbegin));                     // get the iterator for the end index

      node_hash = kh_key(box->hbegin, kbegin);                                                // and also get the corresponding hash

      neighbour_hash_3d(node_hash,nblist,box->width,box);                                     // fetch the list of neighbors
      for(int j=0;j<(2*box->width+1)*(2*box->width+1)*(2*box->width+1);j+=1){                 // and iterate over those neighbors
        if(nblist[j]>=0){                                                                     // check if the neighbor is valid
          node_begin[pair_count] = kh_value(box->hbegin, kbegin);                             // get the cell's begin index
          node_end[pair_count]   = kh_value(box->hend, kend);                                 // get the cell's end index
          nb_begin[pair_count]   = kh_value(box->hbegin, kh_get(0, box->hbegin, nblist[j]) ); // get the neighbor's begin index
          nb_end[pair_count]     = kh_value(box->hend  , kh_get(1, box->hend  , nblist[j]) ); // get the neighbor's end index

          particle_pair_count += (nb_end[pair_count]-nb_begin[pair_count])*                   // just for bookeeping/profiling,
                                       (node_end[pair_count]-node_begin[pair_count]);         // annotate the number of particle pairs
          pair_count += 1;                                                                    // that are part of the computation
        }
      }
    }
  }

  if(dbg)
    printf("particle_pair_count = %ld\n",particle_pair_count);

  return pair_count;
}

/*
 *  Function setup_unique_box_pairs:
 *    Pre-compute indexes for all valid unique cell pairs.
 *    by unique means that if [node_begin,node_end)x[nb_begin,nb_end)
 *    is in the list, [nb_begin,nb_end)x[node_begin,node_end) is not.
 *
 *  Arguments:
 *     box  <linkedListBox*>  : Box of cell linked lists
 *  Returns:
 *     node_begin <int64_t*>  : Array for receiver cell begin indexes
 *     node_end   <int64_t*>  : Array for receiver cell end indexes
 *     nb_begin   <int64_t*>  : Array for sender cell begin indexes
 *     nb_end     <int64_t*>  : Array for sender cell end indexes
 */
int setup_unique_box_pairs(linkedListBox *box,
                           int64_t *node_begin,int64_t *node_end,
                           int64_t *nb_begin,int64_t *nb_end)
{
  int64_t pair_count = 0, particle_pair_count = 0;

  for (khint32_t kbegin = kh_begin(box->hbegin); kbegin != kh_end(box->hbegin); kbegin++){      // iterate over the cells
    int64_t node_hash=-1;
    int64_t nblist[(2*box->width+1)*(2*box->width+1)*(2*box->width+1)];

    if (kh_exist(box->hbegin, kbegin)){                                                         // check if the cell is valid
      khint32_t kend = kh_get(1, box->hend, kh_key(box->hbegin, kbegin));                       // get the iterator for the end index

      node_hash = kh_key(box->hbegin, kbegin);                                                  // and also get the corresponding hash

      neighbour_hash_3d(node_hash,nblist,box->width,box);                                       // fetch the list of neighbors
      for(int j=0;j<(2*box->width+1)*(2*box->width+1)*(2*box->width+1);j+=1){                   // and iterate over those neighbors
        if(nblist[j]>=0){                                                                       // check if the neighbor is valid
          if(kh_value(box->hbegin, kbegin) <=                                                   // and if the node_index is smaller
                     kh_value(box->hbegin, kh_get(0, box->hbegin, nblist[j]) ))                 // then the neighbor index
          {
            node_begin[pair_count] = kh_value(box->hbegin, kbegin);                             // get the cell's begin index
            node_end[pair_count]   = kh_value(box->hend, kend);                                 // get the cell's end index
            nb_begin[pair_count]   = kh_value(box->hbegin, kh_get(0, box->hbegin, nblist[j]) ); // get the neighbor's begin index
            nb_end[pair_count]     = kh_value(box->hend  , kh_get(1, box->hend  , nblist[j]) ); // get the neighbor's end index

            particle_pair_count += (nb_end[pair_count]-nb_begin[pair_count])*                   // just for bookeeping/profiling,
                                         (node_end[pair_count]-node_begin[pair_count]);         // annotate the number of particle pairs
            pair_count += 1;                                                                    // that are part of the computation
          }
        }
      }
    }
  }

  if(dbg)
    printf("particle_pair_count = %ld\n",particle_pair_count);

  return pair_count;
}

/*
 *  Function print_sph_particles_density:
 *    Write in disk the density values for the computed SPH particles.
 *
 *  Arguments:
 *     prefix  <const char*>  : prefix for the file name
 *     is_cll    <bool>       : boolean defined naive or cell linked list case
 *       N      <int64_t>     : Number of SPH particles
 *       h      <double>      : smoothing length
 *     seed    <long int>     : seed utilized for the PRNG
 *     runs     <int>         : Number of runs executed
 *     lsph  <SPHparticle*>   : Array of SPH particles
 *     box  <linkedListBox*>  : Box of cell linked lists
 *  Returns:
 *     0  <int>   : Error code
 *    fp <FILE*>  : Written file with particle densities
 */
int print_sph_particles_density(const char *prefix, bool is_cll, int64_t N, double h,
																long int seed, int runs, SPHparticle *lsph, linkedListBox *box){
	FILE *fp;
	char filename[2048+1];

	if(is_cll){
		sprintf(filename,
						"data/cd3d(%s,runs=%d)-P(seed=%ld,N=%ld,h=%lg)-B(Nx=%d,Ny=%d,Nz=%d)-D(%lg,%lg,%lg,%lg,%lg,%lg).csv",
						prefix,runs,seed,N,h,box->Nx,box->Ny,box->Nz,box->Xmin,box->Ymin,box->Zmin,box->Xmax,box->Ymax,box->Zmax);

		fp = fopen(filename,"w");
		fprintf(fp,"id,x,y,z,rho\n");
		for(int64_t i=0;i<N;i+=1)
			fprintf(fp,"%ld,%lf,%lf,%lf,%lf\n",lsph->id[i],lsph->x[i],lsph->y[i],lsph->z[i],lsph->rho[i]);
		fclose(fp);
	}
	else{
		sprintf(filename,
						"data/cd3d(%s,runs=%d)-P(seed=%ld,N=%ld,h=%lg)-B(Nx=%d,Ny=%d,Nz=%d)-D(%lg,%lg,%lg,%lg,%lg,%lg).csv",
						prefix,runs,seed,N,h,box->Nx,box->Ny,box->Nz,box->Xmin,box->Ymin,box->Zmin,box->Xmax,box->Ymax,box->Zmax);

		fp = fopen(filename,"w");
		fprintf(fp,"id,x,y,z,rho\n");
		for(int64_t i=0;i<N;i+=1)
			fprintf(fp,"%ld,%lf,%lf,%lf,%lf\n",lsph->id[i],lsph->x[i],lsph->y[i],lsph->z[i],lsph->rho[i]);
		fclose(fp);
	}

	return 0;
}

/*
 *  Function print_sph_particles_density:
 *    Write in disk the density values for the computed SPH particles.
 *
 *  Arguments:
 *     prefix  <const char*>  : prefix for the file name
 *     is_cll    <bool>       : boolean defined naive or cell linked list case
 *       N      <int64_t>     : Number of SPH particles
 *       h      <double>      : smoothing length
 *     seed    <long int>     : seed utilized for the PRNG
 *     runs     <int>         : Number of runs executed
 *     lsph  <SPHparticle*>   : Array of SPH particles
 *     box  <linkedListBox*>  : Box of cell linked lists
 *    times   <double*>       : timings for each of the processing steps
 *  Returns:
 *     0  <int>   : Error code
 *    fp <FILE*>  : Written file with particle densities
 */
int print_time_stats(const char *prefix, bool is_cll, int64_t N, double h,
										 long int seed, int runs, SPHparticle *lsph, linkedListBox *box,double *times){
  FILE *fp;
	char filename[1024+1];

	if(is_cll){
    const int COMPUTE_BLOCKS = 5;
    double t[COMPUTE_BLOCKS], dt[COMPUTE_BLOCKS], total_time, dtotal_time;
  	sprintf(filename,
						"data/times-(%s,runs=%d)-P(seed=%ld,N=%ld,h=%lg)-B(Nx=%d,Ny=%d,Nz=%d)-D(%lg,%lg,%lg,%lg,%lg,%lg).csv",
						prefix,runs,seed,N,h,box->Nx,box->Ny,box->Nz,box->Xmin,box->Ymin,box->Zmin,box->Xmax,box->Ymax,box->Zmax);

  	fp = fopen(filename,"w");
		fprintf(fp,"id,compute_hash_MC3D,sorting,reorder_lsph_SoA,setup_interval_hashtables,compute_density\n");
		for(int run=0;run<runs;run+=1)
			fprintf(fp,"%d,%lf,%lf,%lf,%lf,%lf\n",run,times[COMPUTE_BLOCKS*run+0],times[COMPUTE_BLOCKS*run+1],times[COMPUTE_BLOCKS*run+2],
                                                times[COMPUTE_BLOCKS*run+3],times[COMPUTE_BLOCKS*run+4]);
		fclose(fp);

  	total_time = 0.;
  	for(int k=0;k<COMPUTE_BLOCKS;k+=1){
    	t[k]=0.; dt[k]=0.;
    	for(int run=0;run<runs;run+=1)
      	t[k] += times[COMPUTE_BLOCKS*run+k];
    	t[k] /= runs;
    	for(int run=0;run<runs;run+=1)
      	dt[k] += (times[COMPUTE_BLOCKS*run+k]-t[k])*(times[COMPUTE_BLOCKS*run+k]-t[k]);
    	dt[k] /= runs;
    	dt[k] = sqrt(dt[k]);

    	total_time += t[k];
  	}

  	dtotal_time = 0.;
  	for(int run=0;run<runs;run+=1){
	    double rgm = 0.;
  	  for(int k=0;k<COMPUTE_BLOCKS;k+=1)
    	  rgm += times[COMPUTE_BLOCKS*run+k];

    	dtotal_time += (rgm-total_time)*(rgm-total_time);
  	}
  	dtotal_time /= runs;
  	dtotal_time = sqrt(dtotal_time);

  	if(dbg){
      printf("compute_hash_MC3D calc time                 : %.5lf +- %.6lf s : %.3lg%% +- %.3lg%%\n",t[0],dt[0],100*t[0]/total_time,100*dt[0]/total_time);
  	  printf("qsort calc time                             : %.5lf +- %.6lf s : %.3lg%% +- %.3lg%%\n",t[1],dt[1],100*t[1]/total_time,100*dt[1]/total_time);
  	  printf("reorder_lsph_SoA calc time                  : %.5lf +- %.6lf s : %.3lg%% +- %.3lg%%\n",t[2],dt[2],100*t[2]/total_time,100*dt[2]/total_time);
  	  printf("setup_interval_hashtables calc time         : %.5lf +- %.6lf s : %.3lg%% +- %.3lg%%\n",t[3],dt[3],100*t[3]/total_time,100*dt[3]/total_time);
  	  printf("compute_density_3d load balanced calc time  : %.5lf +- %.6lf s : %.3lg%% +- %.3lg%%\n",t[4],dt[4],100*t[4]/total_time,100*dt[4]/total_time);
  	  printf("compute_density_3d load balanced total time : %.5lf +- %.6lf s : %.3lf%%\n",total_time,dtotal_time,100.);
    }
	}
	else{
    const int COMPUTE_BLOCKS = 1;
    double t[COMPUTE_BLOCKS], dt[COMPUTE_BLOCKS], total_time, dtotal_time;
		sprintf(filename,
						"data/times-(%s,runs=%d)-P(seed=%ld,N=%ld,h=%lg)-B(Nx=%d,Ny=%d,Nz=%d)-D(%lg,%lg,%lg,%lg,%lg,%lg).csv",
						prefix,runs,seed,N,h,box->Nx,box->Ny,box->Nz,box->Xmin,box->Ymin,box->Zmin,box->Xmax,box->Ymax,box->Zmax);

  	fp = fopen(filename,"w");
		fprintf(fp,"id,compute_density\n");
		for(int run=0;run<runs;run+=1)
			fprintf(fp,"%d,%lf\n",run,times[COMPUTE_BLOCKS*run+0]);
		fclose(fp);

  	total_time = 0.;
  	for(int k=0;k<COMPUTE_BLOCKS;k+=1){
    	t[k]=0.; dt[k]=0.;
    	for(int run=0;run<runs;run+=1)
      	t[k] += times[COMPUTE_BLOCKS*run+k];
    	t[k] /= runs;
    	for(int run=0;run<runs;run+=1)
      	dt[k] += (times[COMPUTE_BLOCKS*run+k]-t[k])*(times[COMPUTE_BLOCKS*run+k]-t[k]);
    	dt[k] /= runs;
    	dt[k] = sqrt(dt[k]);

    	total_time += t[k];
  	}

  	dtotal_time = 0.;
  	for(int run=0;run<runs;run+=1){
	    double rgm = 0.;
  	  for(int k=0;k<1;k+=1)
    	  rgm += times[COMPUTE_BLOCKS*run+k];

    	dtotal_time += (rgm-total_time)*(rgm-total_time);
  	}
  	dtotal_time /= runs;
  	dtotal_time = sqrt(dtotal_time);

  	if(dbg)
      printf("compute_density_3d naive %s : %.5lf +- %.6lf s : %.3lf%%\n",prefix,total_time,dtotal_time,100.);
	}

  return 0;
}


#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdbool.h>
#include <unistd.h>
#include <math.h>
#include <float.h>
#include <iniparser.h>
#include "tdsearch_gridsearch.h"
#ifdef TDSEARCH_USE_INTEL
#include <mkl_cblas.h>
#else
#include <cblas.h>
#endif
#include "tdsearch_greens.h"
#include "sacio.h"
#include "parmt_utils.h"
#include "iscl/array/array.h"
#include "iscl/memory/memory.h"
#include "iscl/os/os.h"
#include "iscl/signal/convolve.h"

#define NTSTAR 12   /*!< Default number of t*'s in the grid search. */
#define TSTAR0 0.1  /*!< Default first t* in grid search. */
#define TSTAR1 1.2  /*!< Default last t* in grid search. */
#define NDEPTH 12   /*!< Default number of depths in grid search. */
#define DEPTH0 0.0  /*!< First depth in grid search (km). */
#define DEPTH1 3.0  /*!< Last depth in grid search (km). */
#define DOUBLE_PAGE_SIZE sysconf(_SC_PAGE_SIZE)/sizeof(double)
#define PAGE_SIZE sysconf(_SC_PAGE_SIZE)
#define LDG 6

/*!
 * @brief Sets the default grid search parameters.
 *
 * @param[out] tds   On exit contains the default grid search settings such
 *                   as the t* range, depth range, and toggles off the max
 *                   time shift.
 *
 * @result 0 indicates success.
 *
 * @ingroup tdsearch_gridsearch
 *
 * @author Ben Baker, ISTI
 *
 */
int tdsearch_gridSearch_setDefaults(struct tdSearch_struct *tds)
{
    int ierr;
    memset(tds, 0, sizeof(struct tdSearch_struct));
    tds->maxShiftTime = DBL_MAX;
    ierr = tdsearch_gridSearch_defineTstarGrid(NTSTAR, TSTAR0, TSTAR1, tds);
    if (ierr != 0)
    {
        fprintf(stderr, "%s: Failed to set t* grid\n", __func__);
        return -1;
    }
    ierr = tdsearch_gridSearch_defineDepthGrid(NDEPTH, DEPTH0, DEPTH1, tds);
    if (ierr != 0)
    {
        fprintf(stderr, "%s: Failed to set depth grid\n", __func__);
        return -1;
    }
    return ierr;
}
//============================================================================//
/*!
 * @brief Frees memory on the grid-search structure.
 *
 * @param[out] tds    On exit all memory has been freed and parameters set to
 *                    0 or NULL.
 *
 * @result 0 indicates success.
 *
 * @ingroup tdsearch_gridsearch
 *
 * @author Ben Baker, ISTI
 *
 */
int tdsearch_gridSearch_free(struct tdSearch_struct *tds)
{
    memory_free64f(&tds->G);
    memory_free64f(&tds->Gxc);
    memory_free64f(&tds->tstar);
    memory_free64f(&tds->depths);
    memory_free64f(&tds->synthetic);
    memory_free64f(&tds->xcorr);
    memory_free32i(&tds->lags);
    memory_free32i(&tds->grnsMatrixPtr);
    memory_free32i(&tds->grnsXCMatrixPtr);
    memset(tds, 0, sizeof(struct tdSearch_struct));
    return 0;
}
//============================================================================//
/*!
 * @brief Defines the t* grid in grid search.
 *
 * @param[in] ntstars    Number of t*'s in grid search (> 0).
 * @param[in] tstar0     First t* in grid search (>= 0).
 * @param[in] tstar1     Final t* in grid search (>= tstar0).
 *
 * @param[out] tds       Grid search structure with t* grid.
 *
 * @result 0 indicates success.
 *
 * @ingroup tdsearch_gridsearch
 *
 * @author Ben Baker, ISTI
 *
 */
int tdsearch_gridSearch_defineTstarGrid(const int ntstars, const double tstar0,
                                        const double tstar1,
                                        struct tdSearch_struct *tds)
{
    int ierr;
    tds->ntstar = 0;
    if (ntstars < 1 || tstar0 < 0.0 || tstar0 > tstar1)
    {
        if (ntstars < 1)
        {
            fprintf(stderr, "%s: Error no t*'s %d in search\n", __func__, ntstars);
        }
        if (tstar0 < 0.0)
        {
            fprintf(stderr, "%s: Error tstar0 %f must be non-negative\n",
                    __func__, tstar0);
        }
        if (tstar0 > tstar1)
        {
            fprintf(stderr, "%s: Error tstar0 %f > tstar1 %f\n",
                    __func__, tstar0, tstar1);
        }
        return -1;
    }
    tds->ntstar = ntstars;
    memory_free64f(&tds->tstar);
    tds->tstar = array_linspace64f(tstar0, tstar1, ntstars, &ierr);
    return ierr;
}
//============================================================================//
/*!
 * @brief Defines the depth grid in grid search.
 *
 * @param[in] ndepths    Number of depths in grid search (> 0).
 * @param[in] depth0     First depth (km) in grid search (>= 0).
 * @param[in] depth1     Final depth (km) in grid search (>= depth0).
 *
 * @param[out] tds       Grid search structure with depth grid.
 *
 * @result 0 indicates success.
 *
 * @ingroup tdsearch_gridsearch
 *
 * @author Ben Baker, ISTI
 *
 */
int tdsearch_gridSearch_defineDepthGrid(const int ndepths, const double depth0,
                                        const double depth1,
                                        struct tdSearch_struct *tds)
{
    int ierr;
    tds->ndepth = 0;
    if (ndepths < 1 || depth0 < 0.0 || depth0 > depth1)
    {
        if (ndepths < 1)
        {
            fprintf(stderr, "%s: Error no depthss %d in search\n",
                    __func__, ndepths);
        }
        if (depth0 < 0.0)
        {
            fprintf(stderr, "%s: Error depth0 %f must be non-negative\n",
                    __func__, depth0);
        }
        if (depth0 > depth1)
        {
            fprintf(stderr, "%s: Error depth0 %f > depth1 %f\n",
                    __func__, depth0, depth1);
        }
        return -1;
    }
    tds->ndepth = ndepths;
    memory_free64f(&tds->depths);
    tds->depths = array_linspace64f(depth0, depth1, ndepths, &ierr);
    return ierr;
}
//============================================================================//
/*!
 * @brief Reads the grid search parameters from the ini file.
 *
 * @param[in] iniFile    Name of ini file with parameters.
 *
 * @param[out] tds       On successful exit contains the grid search parameters.
 *
 * @result 0 indicates success.
 *
 * @ingroup tdsearch_gridsearch
 *
 * @author Ben Baker, ISTI
 *
 */
int tdsearch_gridSearch_initializeFromFile(const char *iniFile,
                                           struct tdSearch_struct *tds)
{
    double depth0, depth1, tstar0, tstar1;
    dictionary *ini;
    int ierr, ndepth, ntstar;
    // Set the defaults
    ierr = 0;
    tdsearch_gridSearch_setDefaults(tds);
    // Verify the file exists
    if (!os_path_isfile(iniFile))
    {
        fprintf(stderr, "%s: Error ini file %s doesn't exist\n", __func__, iniFile);
        return -1;
    }
    ini = iniparser_load(iniFile);
    // Unpack the parameter file
    tds->maxShiftTime
      = iniparser_getdouble(ini, "tdSearch:gridSearch:maxShiftTime\0", DBL_MAX);
    if (tds->maxShiftTime < DBL_MAX)
    {
        if (tds->maxShiftTime <= 0.0)
        {
            fprintf(stderr, "%s: Max shift time must be positive\n", __func__);
            ierr = 1;
            goto ERROR;
        }
    }
    // Number of t*'s
    ntstar = iniparser_getint(ini, "tdSearch:gridSearch:ntstar\0", NTSTAR);
    if (ntstar < 1)
    {
        fprintf(stderr, "%s: number of t*'s must be positive\n", __func__);
        ierr = 1;
        goto ERROR;
    }
    // Number of depths
    ndepth = iniparser_getint(ini, "tdSearch:gridSearch:ndepths\0", NDEPTH);
    if (ndepth < 1)
    {
        fprintf(stderr, "%s: number of depths must be positive\n", __func__);
        ierr = 1;
        goto ERROR;
    }
    // Depth grid search range
    depth0 = iniparser_getdouble(ini, "tdSearch:gridSearch:depth0\0", DEPTH0);
    depth1 = iniparser_getdouble(ini, "tdSearch:gridSearch:depth1\0", DEPTH1);
    if (depth0 >= depth1)
    {
        fprintf(stderr, "%s: depth0 >= depth1 is invalid\n", __func__);
        ierr = 1;
        goto ERROR;
    }
    // t* grid search range
    tstar0 = iniparser_getdouble(ini, "tdSearch:gridSearch:tstar0\0", TSTAR0);
    tstar1 = iniparser_getdouble(ini, "tdSearch:gridSearch:tstar1\0", TSTAR1);
    if (tstar0 >= tstar1)
    {
        fprintf(stderr, "%s: tstar0 >= tstar1 is invalid\n", __func__);
        ierr = 1;
        goto ERROR;
    }
    // Create the grid
    ierr = tdsearch_gridSearch_defineTstarGrid(ntstar,
                                               tstar0, tstar1,
                                               tds);
    if (ierr != 0)
    {
        fprintf(stderr, "%s: Error defining t* grid\n", __func__);
        goto ERROR;
    }
    ierr = tdsearch_gridSearch_defineDepthGrid(ndepth,
                                               depth0, depth1,
                                               tds);
    if (ierr != 0)
    {
        fprintf(stderr, "%s: Error defining depth grid\n", __func__);
        goto ERROR;
    }
ERROR:;
    iniparser_freedict(ini);
    return ierr;
}
//============================================================================//
/*!
 * @brief Returns the number of depths in the grid search.
 *
 * @param[in] tds     Holds the number of t*'s.
 *
 * @result The number of t*'s in the grid search.
 *
 * @ingroup tdsearch_gridsearch
 *
 * @author Ben Baker, ISTI
 *
 */
int tdsearch_gridSearch_getNumberOfTstars(const struct tdSearch_struct tds)
{
    return tds.ntstar;
}
//============================================================================//
/*!
 * @brief Returns the number of depths in the grid search.
 *
 * @param[in] tds     Holds the number of depths.
 *
 * @result The number of depths in the grid search.
 *
 * @author Ben Baker, ISTI
 *
 */
int tdsearch_gridSearch_getNumberOfDepths(const struct tdSearch_struct tds)
{
    return tds.ndepth;
}
//============================================================================//
/*!
 * @brief Convenience utility for computing the index corresponding to the
 *        2D row major depth x t* grid.
 *
 * @param[in] it      The t* index [0,ntstar-1].
 * @param[in] id      The depth indx [0,ndepth-1].
 * @param[in] ntstar  The number of t*'s in the grid search.
 *
 * @result The grid search index (= id*ntstar + it).
 *
 * @ingroup tdsearch_gridsearch
 *
 * @author Ben Baker, ISTI
 *
 */
int tdsearch_gridSearch_gridToIndex2(const int it, const int id,
                                     const int ntstar)
{
    int idt;
    idt = id*ntstar + it;
    return idt;
}
//============================================================================//
/*!
 * @brief Convenience utility for computing the index corresponding to the
 *        2D row major depth x t* grid.
 *
 * @param[in] it      The t* index [0,ntstar-1].
 * @param[in] id      The depth indx [0,ndepth-1].
 * @param[in] tds     Grid search structure holding the number of ntstar's
 *                    in the grid search.
 *
 * @result The grid search index (= id*tds.ntstar + it).
 *
 * @ingroup tdsearch_gridsearch
 *
 * @author Ben Baker, ISTI
 *
 */
int tdsearch_gridSearch_gridToIndex(const int it, const int id,
                                    const struct tdSearch_struct tds)
{
    int idt, ntstar;
    ntstar = tdsearch_gridSearch_getNumberOfTstars(tds);
    idt = tdsearch_gridSearch_gridToIndex2(it, id, ntstar);
    return idt;
}
//============================================================================//
/*!
 * @brief Computes the forward modeling matrix, G, so that estimates, e, can
 *        can be efficiently generated via \f$ G m = e \f$ for any moment
 *        tensor, m.  The columns of G represent cross-correlations of the
 *        Green's functions corresponding to the \f$ m_{ij} \f$ moment tensor
 *        component so that the estimate, e, is a cross-correlogram.
 *
 * @param[in] iobs     Observation index in the range [0,nobs-1].
 * @param[in] data     Structure containing the data.
 * @param[in] grns     Structure containing the Green's functions.
 *
 * @param[in,out] tds  On input contains the initialized tds structure.
 * @param[in,out] tds  On exit the forward modeling matrix G has been set.
 *
 * @result 0 indicates success.
 *
 * @ingroup tdsearch_gridsearch
 *
 * @author Ben Baker, ISTI
 *
 */
int tdsearch_gridSearch_setForwardModelingMatrices(
    const int iobs,
    const struct tdSearchData_struct data,
    const struct tdSearchGreens_struct grns,
    struct tdSearch_struct *tds)
{
    double *G, *Gxc, *work;
    int *grnsMatrixPtr, *grnsXCMatrixPtr, indices[6],
        i, idep, idt, ierr, indx, it, jndx, kndx,
        lxc, lxcMax, lxc0, ntstar, ndepth, npgrns, npts;
    // Reset pointers
    memory_free64f(&tds->G);
    memory_free64f(&tds->Gxc);
    memory_free32i(&tds->grnsMatrixPtr);
    memory_free32i(&tds->grnsXCMatrixPtr);
    // Check sizes
    ndepth = grns.ndepth;
    ntstar = grns.ntstar;
    if (ndepth != tds->ndepth || ntstar != tds->ntstar)
    {
        if (ndepth != tds->ndepth)
        {
            fprintf(stderr, "%s: Error ndepth %d != tds->ndepth %d\n",
                    __func__, ndepth, tds->ndepth);
        }
        if (ntstar != tds->ntstar)
        {
            fprintf(stderr, "%s: Error ntstar %d != tds->ntstar %d\n",
                    __func__, ntstar, tds->ntstar);
        }
        return -1;
    }
    npts = data.obs[iobs].npts;
    if (npts < 1)
    {
        fprintf(stderr, "%s: Error data has no points\n", __func__);
        return -1;
    }
    tds->dnorm = cblas_dnrm2(npts, data.obs[iobs].data, 1);
    if (fabs(tds->dnorm) < DBL_EPSILON)
    {
        fprintf(stderr, "%s: Error - data is identically zero\n", __func__);
        return -1;
    }
    tds->npts = npts;
    sacio_getFloatHeader(SAC_FLOAT_DELTA, data.obs[iobs].header, &tds->dt);
    // Set the Green's functions forward modeling matrix size
    lxcMax = 0;
    grnsMatrixPtr = memory_calloc32i(ndepth*ntstar+1);
    grnsXCMatrixPtr = memory_calloc32i(ndepth*ntstar+1);
    grnsMatrixPtr[0] = 0;
    grnsXCMatrixPtr[0] = 0;
    for (idep=0; idep<ndepth; idep++)
    {
        for (it=0; it<ntstar; it++)
        {
            // Get the Green's functions indices
            ierr = tdsearch_greens_getGreensFunctionsIndices(iobs, it,
                                                             idep, grns,
                                                             indices);
            if (ierr != 0)
            {
                fprintf(stderr, "%s: Failed to get indices\n", __func__);
                return -1;
            }
            // Compute the cross-correlation length at (dep, t*)
            idt = tdsearch_gridSearch_gridToIndex2(it, idep, ntstar);
            lxc0 = 0;
            for (i=0; i<6; i++)
            {
                indx = indices[i];
                npgrns = grns.grns[indx].npts;
                if (npgrns < 1)
                {
                    fprintf(stderr, "%s: Error - no points in grns fn\n", __func__);
                    return -1;
                }
                lxc = npgrns + npts - 1;
                lxcMax = MAX(lxcMax, lxc);
                if (i == 0)
                {
                    lxc0 = lxc;
                    grnsMatrixPtr[idt+1] = grnsMatrixPtr[idt] + npgrns;
                    grnsXCMatrixPtr[idt+1] = grnsXCMatrixPtr[idt] + lxc;
                }
                if (lxc != lxc0)
                {
                    fprintf(stderr, "%s: Inconsistent column sizes\n", __func__);
                    return -1;
                }
            }
        }
    }
    if (lxcMax < 1)
    {
        fprintf(stderr, "%s: Error - cross-correlations are empty\n", __func__);
        return -1;
    }
    // Set the forward modeling matrices
    tds->nptsGrns = grnsMatrixPtr[ndepth*ntstar];
    G = memory_calloc64f(LDG*grnsMatrixPtr[ndepth*ntstar]);
    Gxc = memory_calloc64f(LDG*grnsXCMatrixPtr[ndepth*ntstar]);
    work = memory_calloc64f(lxcMax);
    for (idep=0; idep<ndepth; idep++)
    {
        for (it=0; it<ntstar; it++)
        {
            idt = tdsearch_gridSearch_gridToIndex2(it, idep, ntstar);
            // Get the Green's functions indices
            ierr = tdsearch_greens_getGreensFunctionsIndices(iobs, it,
                                                             idep, grns,
                                                             indices);
            // Compute the cross-correlation length at (dep, t*)
            idt = tdsearch_gridSearch_gridToIndex2(it, idep, ntstar);
            jndx = LDG*grnsMatrixPtr[idt];
            kndx = LDG*grnsXCMatrixPtr[idt];
            for (i=0; i<6; i++)
            {
                indx = indices[i];
                npgrns = grns.grns[indx].npts;
                signal_convolve_correlate64f_work(npts, data.obs[iobs].data,
                                                  npgrns, grns.grns[indx].data,
                                                  CONVCOR_FULL,
                                                  lxc, work);
                //printf("%e %e\n",cblas_dnrm2(npgrns, grns.grns[indx].data, 1),
                //               cblas_dnrm2(lxc, work, 1));
                cblas_dcopy(npgrns, grns.grns[indx].data, 1, &G[jndx+i],   LDG);
                cblas_dcopy(lxc,    work,                 1, &Gxc[kndx+i], LDG);
            }
            //printf("%d\n", grnsMatrixPtr[idt+1]);
        }
    }
    tds->G = G;
    tds->Gxc = Gxc;
    tds->grnsMatrixPtr = grnsMatrixPtr;
    tds->grnsXCMatrixPtr = grnsXCMatrixPtr;
    memory_free64f(&work);
    return 0;
}
//============================================================================//
/*!
 * @brief Sets the moment tensor on the tdSearch structure.
 *
 * @param[in] m6    moment tensor in NED system packed:
 *                   \f$ \{ m_{xx}, m_{yy}, m_{zz},
 *                         m_{xy}, m_{xz}, m_{yz} \} \f$.
 *
 * @param[out] tds  On successful exit contains the moment tensor.
 *
 * @result 0 indicates success.
 *
 * @ingroup tdsearch_gridsearch
 *
 * @author Ben Baker, ISTI
 *
 */
int tdSearch_gridSearch_setMomentTensor(const double *__restrict__ m6,
                                        struct tdSearch_struct *tds)
{
    tds->lhaveMT = false;
    array_zeros64f_work(8, tds->mt); // NOTE - this is padded
    if (m6 == NULL)
    {
        fprintf(stderr, "%s: Error moment tensor cannot be NULL\n", __func__);
        return -1;
    }
    array_copy64f_work(6, m6, tds->mt);
    tds->lhaveMT = true;
    return 0;
}
//============================================================================//
/*!
 * @brief Sets the moment tensor elements on the grid search structure.
 *
 * @param[in] m11      m11 moment tensor in Newton meters.
 * @param[in] m22      m22 moment tensor in Newton meters.
 * @param[in] m33      m33 moment tensor in Newton meters.
 * @param[in] m12      m12 moment tensor in Newton meters.
 * @param[in] m13      m13 moment tensor in Newton meters.
 * @param[in] m23      m23 moment tensor in Newton meters.
 * @param[in] basis    The appropriate moment tensor basis.
 * @param[in] basis    CE_USE indicates the moment tensor is in up, south
 *                     east coordinates like what one would obtain from
 *                     the GCMT catalog.
 * @param[in] basis    CE_NED indicates the moment tensor is in north, east,
 *                     down coordinates and is consistent with Programs in
 *                     Seismology.
 *
 * @param[in,out] tds  On input contains the initialized grid search
 *                     structure.
 * @param[in,out] tds  On exit contains the input moment tensor.
 *
 * @result 0 indicates success.
 *
 * @ingroup tdsearch_gridsearch
 *
 */
int tdSearch_gridSearch_setMomentTensorFromElements(
    const double m11, const double m22, const double m33,
    const double m12, const double m13, const double m23,
    const enum compearthCoordSystem_enum basis,
    struct tdSearch_struct *tds)
{
    double mt[6], mtNED[6];
    int ierr;
    tds->lhaveMT = false;
    mt[0] = m11;
    mt[1] = m22;
    mt[2] = m33;
    mt[3] = m12;
    mt[4] = m13;
    mt[5] = m23;
    ierr = compearth_convertMT(1, basis, CE_NED, mt, mtNED);
    if (ierr != 0)
    {
        printf("%s: Failed to set moment tensor\n", __func__);
        return -1;
    }
    ierr = tdSearch_gridSearch_setMomentTensor(mtNED, tds);
    if (ierr != 0)
    {
        printf("%s: Failed to set moment tensor\n", __func__);
        return -1;
    }
    return 0;
}
//============================================================================//
/*!
 * @brief Sets the moment tensor information from an event structure.
 *
 * @param[in] event    Contains the event on which the moment tensor is defined.
 * @param[in,out] tds  On input this is the initialized grid search structure.
 * @param[in,out] tds  On exit contains the given moment tensor.
 *
 * @result 0 indicates success.
 *
 * @ingroup tdsearch_gridsearch
 *
 */
int tdSearch_gridSearch_setMomentTensorFromEvent(
    const struct tdSearchEventParms_struct event,
    struct tdSearch_struct *tds)
{
    int ierr;
    ierr = tdSearch_gridSearch_setMomentTensorFromElements(
                    event.m11, event.m22, event.m33,
                    event.m12, event.m13, event.m23,
                    event.basis, tds);
    if (ierr != 0)
    {
        fprintf(stderr, "%s: Error setting moment tensor\n", __func__);
        return -1;
    }
    return 0;
}
//============================================================================//
/*!
 * @brief Computes the cross-correlations between the data and synthetics
 *        at each (t*,depth) in the grid search.
 *
 * @param[in,out] tds   On input contains the initialized grid search
 *                      structure with the Green's functions and source
 *                      properties.
 * @param[in,out] tds   On exit contains the result of the cross-correlations
 *                      and lags at each (t*,depth) in the grid search.
 *
 * @result 0 indicates success.
 *
 * @ingroup tdsearch_gridsearch
 *
 */
int tdSearch_gridSearch_performGridSearch(struct tdSearch_struct *tds)
{
    double *u, *ud, dnorm, unorm;
    int id, idt, indx, iopt, it, jndx, l1, l2, lxc, maxlags,
        ndepth, ndt, nlag, npgrns, nrows, ntstar;
    const int ncols = 6;
    enum isclError_enum ierr;
    tds->itopt =-1;
    tds->idopt =-1;
    ndepth = tds->ndepth;
    ntstar = tds->ntstar;
    ndt = ntstar*ndepth;
    dnorm = tds->dnorm;
    memory_free64f(&tds->xcorr);
    memory_free64f(&tds->synthetic);
    memory_free32i(&tds->lags);
    // Some easy checks
    if (!tds->lhaveMT)
    {
        fprintf(stderr, "%s: Error moment tensor not set\n", __func__);
        return -1;
    }
    if (tds->G == NULL || tds->Gxc == NULL ||
        tds->grnsMatrixPtr == NULL || tds->grnsXCMatrixPtr == NULL)
    {
        fprintf(stderr, "%s: You need to set the forward modeling matrices\n", __func__);
        return -1;
    }
    maxlags =-1;
    if (tds->maxShiftTime < DBL_MAX)
    {
        maxlags = (int) (tds->maxShiftTime/tds->dt + 0.5);
    }
    // Set output space
    tds->xcorr = memory_calloc64f(ndt);
    tds->lags = memory_calloc32i(ndt);
    // Compute the cross-correlation
    nrows = tds->grnsXCMatrixPtr[ndt];
    ud = memory_calloc64f(nrows);
    cblas_dgemv(CblasRowMajor, CblasNoTrans,
                nrows, ncols, 1.0, tds->Gxc, LDG,
                tds->mt, 1, 0.0, ud, 1);
    // Compute the synthetics
    nrows = tds->grnsMatrixPtr[ndt];
    u = memory_calloc64f(nrows);
    cblas_dgemv(CblasRowMajor, CblasNoTrans,
                nrows, ncols, 1.0, tds->G, LDG,
                tds->mt, 1, 0.0, u, 1);
    // Now do the grid search
    for (id=0; id<ndepth; id++)
    {
        for (it=0; it<ntstar; it++)
        {
            idt = tdsearch_gridSearch_gridToIndex(it, id, *tds);
            indx = tds->grnsMatrixPtr[idt];
            jndx = tds->grnsXCMatrixPtr[idt];
            lxc = tds->grnsXCMatrixPtr[idt+1]
                - tds->grnsXCMatrixPtr[idt];
            npgrns = tds->grnsMatrixPtr[idt+1]
                   - tds->grnsMatrixPtr[idt];
            unorm = cblas_dnrm2(npgrns, &u[indx], 1);
            if (maxlags < 0)
            {
                iopt = array_argmax64f(lxc, &ud[jndx], &ierr);
            }
            else
            {
                l1 = MAX(0,  npgrns - maxlags);
                l2 = MIN(lxc-1, npgrns + maxlags);
                nlag = l2 - l1 + 1;
                iopt = l1 + array_argmax64f(nlag, &ud[jndx+l1], &ierr);
 /*
for (int k=0; k<lxc; k++)
{
printf("%d %e\n",k, ud[jndx+k]);
}
getchar();
*/
            }
            tds->xcorr[idt] = ud[jndx+iopt]/(dnorm*unorm);
            // Compute the lag where the times range from [-npts+1:npgrns-1]
            tds->lags[idt] =-npgrns + 1 + iopt;
            //printf("%f %f %f %d\n", tds->depths[id], tds->tstar[it], xc, idelay);
        }
        //printf("\n");
    }
    tds->synthetic = u;
    memory_free64f(&ud);
    // Compute the optimum indices
    iopt = array_argmax64f(ndt, tds->xcorr, &ierr);
    tds->idopt = iopt/tds->ntstar;
    tds->itopt = iopt - tds->idopt*tds->ntstar;
    if (tdsearch_gridSearch_gridToIndex(tds->itopt, tds->idopt, *tds) != iopt)
    {
        fprintf(stdout, "%s: Switching to slow optimum computation\n", __func__);
        for (id=0; id<ndepth; id++)
        {
            for (it=0; it<ntstar; it++)
            {
                idt = tdsearch_gridSearch_gridToIndex(it, id, *tds);
                if (iopt == idt)
                {
                    tds->idopt = id;
                    tds->itopt = it;
                }
            }
        }
    }
    return 0;
}
//============================================================================//
/*!
 * @brief Writes a map of the max of the cross-correlograms as a function
 *        of t* and depth in a format useful for Gnpulot as well as a
 *        corresponding Gnuplot plotting script.
 *
 * @param[in] dirnm   Directory where the heat map is to be written.
 * @param[in] froot   The root name of the heat map.
 * @param[in] tds     Grid search structure with the tabulated
 *                    cross-correlogram maximums.
 *
 * @result 0 indicates success.
 *
 * @ingroup tdsearch_gridsearch
 *
 */
int tdsearch_gridSearch_writeHeatMap(const char *dirnm, const char *froot,
                                     const struct tdSearch_struct tds)
{
    FILE *ofl;
    char fname[PATH_MAX], fgnu[PATH_MAX], cmd[256];
    int id, idt, it;
    memset(fname, 0, PATH_MAX*sizeof(char));
    if (!tds.xcorr || !tds.lags)
    {
        fprintf(stderr, "%s: Error gridsearch likely not performed\n", __func__);
        return -1;
    }
    if (!tds.tstar || !tds.depths)
    {
        fprintf(stderr, "%s: Error tds likely not initialized\n", __func__);
        return -1;
    }
    if (dirnm != NULL)
    {
        sprintf(fname, "%s/%s.surf_txt", dirnm, froot);
    }
    else
    {
        sprintf(fname, "./%s.surf_txt", froot);
    }
    ofl = fopen(fname, "w");
    for (id=0; id<tds.ndepth; id++)
    {
        for (it=0; it<tds.ntstar; it++)
        {
            idt = tdsearch_gridSearch_gridToIndex(it, id, tds);
            fprintf(ofl, "%f %f %e %d\n",
                    tds.tstar[it], tds.depths[id],
                    tds.xcorr[idt], tds.lags[idt]);
        }
       fprintf(ofl, "\n");
    }
    fclose(ofl);
    // and the gnuplot file
    memset(fgnu, 0, PATH_MAX*sizeof(char));
    if (dirnm != NULL)
    {
        sprintf(fgnu, "%s/%s.gnu", dirnm, froot);
    }
    else
    {
        sprintf(fgnu, "./%s.gnu", froot);
    }
    ofl = fopen(fgnu, "w");
    fprintf(ofl, "#!/usr/bin/gnuplot -persist\n");
    fprintf(ofl, "set pm3d map\n");
    fprintf(ofl, "set xlabel 't*'\n");
    fprintf(ofl, "set ylabel 'Depth (km)'\n");
    fprintf(ofl, "set title 'Cross-Correlation'\n");
    fprintf(ofl, "set grid\n");
    fprintf(ofl, "set xrange [%f:%f]\n", tds.tstar[0],
                                         tds.tstar[MAX(0, tds.ntstar-1)]);
    fprintf(ofl, "set yrange [%f:%f]\n", tds.depths[MAX(0, tds.ndepth-1)],
                                         tds.depths[0]);
    fprintf(ofl, "splot '%s.surf_txt' u 1:2:3\n", froot);
    fclose(ofl);
    // change permissions
    memset(cmd, 0, 256*sizeof(char));
    sprintf(cmd, "chmod 0755 %s", fgnu);
    system(cmd);
    return 0;
}
//============================================================================//
/*!
 * @brief Makes a shifted synthetic seismogram corresponding to the
 *        given observation, t*, and depth.
 *
 * @param[in] iobs    Observation index in the range [0,nobs-1].
 * @param[in] it      t* index in the range [0,tds.ntstar-1].
 * @param[in] id      Depth index in the range [0,tds.ndepth].
 * @param[in] data    Contains the observed seismograms.
 * @param[in] grns    Contains the Green's functions used to make
 *                    the synthetic seisogram.
 * @param[in] tds     Contains the time lags computed in the grid search.
 *
 * @param[out] synth  Structure containing the synthetic seismogram.

 * @result 0 indicates success.
 *
 * @ingroup tdsearch_gridsearch
 */
int tdSearch_gridSearch_makeSACSynthetic(
    const int iobs, const int it, const int id,
    const struct tdSearchData_struct data,
    const struct tdSearchGreens_struct grns,
    const struct tdSearch_struct tds,
    struct sacData_struct *synth)
{
    double epoch;
    int idt, ierr, igrns, npGrns;
    memset(synth, 0, sizeof(struct sacData_struct));
    // Pick off the header
    if (it < 0 || it >= tds.ntstar || id < 0 || id >= tds.ndepth)
    {
        fprintf(stderr, "%s: Invalid (t*,depth) index (%d,%d)\n", __func__, it, id);
        return - 1;
    }
    if (tds.synthetic == NULL || tds.lags == NULL)
    {
        if (tds.synthetic == NULL)
        {
            fprintf(stderr, "%s: Error tds.synethetic is NULL\n", __func__);
        }
        if (tds.lags == NULL)
        {
            fprintf(stderr, "%s: Error tds.lags is NULL\n", __func__);
        }
        return -1;
    }
    idt = tdsearch_gridSearch_gridToIndex(it, id, tds);
    igrns = tdsearch_greens_getGreensFunctionIndex(G11_GRNS, iobs,
                                                   it, id, grns);
    if (idt < 0 || igrns < 0)
    {
        fprintf(stderr, "%s: Failed getting an index\n", __func__);
        return -1;
    }
    // Copy header
    sacio_copyHeader(grns.grns[igrns].header, &synth->header);
    // Copy corresponding synthetic
    synth->npts = grns.grns[igrns].npts;
    synth->data = sacio_malloc64f(synth->npts);
    npGrns = tds.grnsMatrixPtr[idt+1] - tds.grnsMatrixPtr[idt];
    if (npGrns != synth->npts)
    {
        fprintf(stderr, "%s: Size mismatch\n", __func__);
        sacio_free(synth);
        return -1;
    }
    array_copy64f_work(npGrns, &tds.synthetic[tds.grnsMatrixPtr[idt]],
                       synth->data);
    // Get the component inline with the estimate
    sacio_setCharacterHeader(SAC_CHAR_KCMPNM, data.obs[iobs].header.kcmpnm,
                             &synth->header);
    // Fix the timing
    ierr = sacio_getEpochalStartTime(synth->header, &epoch);
    if (ierr != 0)
    {
        fprintf(stderr, "%s: Failed to get start time\n", __func__);
        return -1;
    }
    epoch = epoch + (double) tds.lags[idt]*tds.dt;
    sacio_setEpochalStartTime(epoch, &synth->header);
    return 0;
}
//============================================================================//