#!/usr/bin/env Rscript
library("optparse")

computedher<-function(beta, se, af,N){
#https://journals.plos.org/plosone/article/file?type=supplementary&id=info:doi/10.1371/journal.pone.0120758.s001
maf<-af
maf[!is.na(af) & af>0.5]<- 1 - maf[!is.na(af) & af>0.5]
ba<-!is.na(beta) & !is.na(se) & !is.na(maf) & !is.na(N)
a<-rep(NA, length(beta))
b<-rep(NA, length(beta))
a<-2*(beta[ba]**2)*(maf[ba]*(1-maf[ba]))
b<-2*(se[ba]**2)*N[ba]*maf[ba]*(1-maf[ba])
res<-rep(NA, length(beta))
res[ba]<-a[ba]/(a[ba]+b[ba])
res
}

option_list = list(
  make_option(c("-f", "--file"), type="character", default=NULL,
              help="dataset file name", metavar="character"),
  make_option(c("--chro"), type="character", default=NULL,
              help="dataset file name", metavar="character"),
  make_option(c("--chro_head"), type="character", default=NULL,
              help="dataset file name", metavar="character"),
  make_option(c("--bp_head"), type="character", default=NULL,
              help="dataset file name", metavar="character"),
  make_option(c("--pheno_head"), type="character", default=NULL,
              help="dataset file name", metavar="character"),
  make_option(c("--beta_head"), type="character", default=NULL,
              help="dataset file name", metavar="character"),
  make_option(c("--ci_head"), type="character", default=NULL,
              help="dataset file name", metavar="character"),
  make_option(c("--p_head"), type="character", default=NULL,
              help="dataset file name", metavar="character"),
  make_option(c("--n_head"), type="character", default=NULL,
              help="dataset file name", metavar="character"),
  make_option(c("--freq_head"), type="character", default=NULL,
              help="dataset file name", metavar="character"),
  make_option(c("--rs_head"), type="character", default=NULL,
              help="dataset file name", metavar="character"),
  make_option(c("--riskall_head"), type="character", default=NULL,
              help="dataset file name", metavar="character"),
  make_option(c("--format"), type="character", default="USCS",
              help="dataset file name", metavar="character"),
  make_option(c("--typeformat"), type="character", default="csv",
              help="dataset file name", metavar="character"),
  make_option(c("-o", "--out"), type="character", default="out.txt",
              help="output file name [default= %default]", metavar="character")
);

opt_parser = OptionParser(option_list=option_list);
opt = parse_args(opt_parser);
#bin 	590	smallint(5) unsigned 	range 	Indexing field to speed chromosome range queries.
#chrom 	chr1	varchar(255) 	values 	Reference sequence chromosome or scaffold
#chromStart 	768252	int(10) unsigned 	range 	Start position in chromosome
#chromEnd 	768253	int(10) unsigned 	range 	End position in chromosome
#name 	rs2977608	varchar(255) 	values 	ID of SNP associated with trait
#pubMedID 	31969693	int(10) unsigned 	range 	PubMed ID of publication of the study
#author 	Coleman JRI	varchar(255) 	values 	First author of publication
#pubDate 	2020-01-23	varchar(255) 	values 	Date of publication
#journal 	Mol Psychiatry	varchar(255) 	values 	Journal of publication
#title 	Genome-wide gene-environmen...	varchar(1024) 	values 	Title of publication
#trait 	Major depressive disorder	varchar(255) 	values 	Disease or trait assessed in study
#initSample 	29,475 European ancestry ca...	longblob 	  	Initial sample size
#replSample 	NA	longblob 	  	Replication sample size
#region 	1p36.33	varchar(255) 	values 	Chromosome band / region of SNP
#genes 	NR	longblob 	  	Reported Gene(s)
#riskAllele 	rs2977608-A	longblob 	  	Strongest SNP-Risk Allele
#riskAlFreq 	0.259029	varchar(255) 	values 	Risk Allele Frequency
#pValue 	8E-6	varchar(255) 	values 	p-Value
#pValueDesc 	 	varchar(255) 	values 	p-Value Description
#orOrBeta 	1.0729614	varchar(255) 	values 	Odds ratio or beta
#ci95 	[1.04-1.1]	varchar(255) 	values 	95% Confidence Interval
#platform 	Affymetrix [7791636] (imputed)	varchar(255) 	values 	Platform and [SNPs passing QC]
#cnv 	N	enum('Y', 'N') 	values 	Y if Copy Number Variant

#lisp="Type 2 diabetes";lisc=c("22","21")
format=opt[['format']]
if(format=="USCS"){
Data<-read.csv(opt[['file']], header=F, sep='\t')
chrohead="chrom";phenhead="trait";poshead="chromEnd";OrBetaHead="orOrBeta";headCi95="ci95";pValueHead="pValue";nvalueHead="initSample";freqHead="riskAlFreq";rsHead="name";riskall="riskAllele"
names(Data)<-c("bin", "chrom", "chromStart", "chromEnd", "name", "pubMedID", "author", "pubDate", "journal", "title", "trait", "initSample","replSample","region", "genes", "riskAllele", "riskAlFreq", "pValue", "pValueDesc", "orOrBeta", "ci95","platform", "cnv")
}else{
chrohead=opt[['chro_head']];phenhead=opt[['pheno_head']];poshead=opt[['bp_head']];OrBetaHead=opt[['beta_head']];headCi95=opt[['ci_head']];pValueHead=opt[['p_head']];nvalueHead=opt[['n_head']];freqHead=opt[['freq_head']];rsHead=opt[['rs_head']];riskall=opt[['riskall_head']]
if(opt[['typeformat']]=='csv')Data<-read.csv(opt[['file']])
else Data<-read.table(opt[['file']], header=T, sep='\t')
}

Data[,chrohead]<-gsub('chr', '',as.character(Data[,chrohead]))

Data2Sub<-Data #[, c(chrohead,poshead, OrBetaHead,headCi95,pValueHead,phenhead,nvalueHead,freqHead)]

IC<-t(sapply(strsplit(gsub("[", "", sapply(strsplit(as.character(Data2Sub[,headCi95]), split=']',fixed=T),function(x)x[1]),fixed=T), split="-"),function(x){
if(length(x)==2){return(c(as.numeric(x[1]),as.numeric(x[2])))
}else{
return(c(-as.numeric(x[2]),as.numeric(x[3])))
}
}))
IC<-data.frame(lower.cat=IC[,1], upper.cat=IC[,2])
Data2Sub<-cbind(Data2Sub,IC)
Data2Sub$nsample.cat<-sapply(strsplit(as.character(Data2Sub[,nvalueHead]),split="[ ]"),function(x)sum(as.integer(gsub(",", "",grep("[0-9]", x,value=T)))))
Data2Sub$beta.cat<-as.numeric(as.character(Data2Sub[,OrBetaHead]))
balisebeta<-!is.na(Data2Sub$beta.cat) & Data2Sub$beta.cat>=1
Data2Sub$lower.cat[balisebeta]<-log2(Data2Sub$lower.cat[balisebeta])
Data2Sub$upper.cat[balisebeta]<-log2(Data2Sub$upper.cat[balisebeta])
#Data2Sub<-Data2Sub[!is.na(Data2Sub$beta.cat) & !is.na(Data2Sub$lower.cat) & !is.na(Data2Sub$upper.cat) & !is.na(Data2Sub$nsample.cat) ,]

Data2Sub$beta.cat[balisebeta]<-log2(Data2Sub$beta.cat[balisebeta])
Data2Sub$risk.allele.af[balisebeta]<-as.numeric(as.character(Data2Sub[balisebeta,freqHead]))
Data2Sub$sd.cat<-(Data2Sub$upper.cat - Data2Sub$beta.cat)/1.96
Data2Sub$sd.cat2<-(Data2Sub$upper.cat - Data2Sub$beta.cat)/1.96*sqrt(Data2Sub$nsample.cat)
Data2Sub$z.cat.v1<-Data2Sub$beta.cat/(Data2Sub$sd.cat)
Data2Sub$z.cat<-qnorm(Data2Sub[,pValueHead],lower.tail=FALSE)
Data2Sub$h2.cat=computedher(Data2Sub$beta.cat, Data2Sub$sd.cat, Data2Sub$risk.allele.af,Data2Sub$nsample.cat)
Data2Sub$pvalue<-as.numeric(as.character(Data2Sub[,pValueHead]))
#Data2Sub<-Data2Sub[!is.na(Data2Sub$h2.cat) & !is.na(Data2Sub[,pValueHead]),]
#Data2Sub<-Data2Sub[order(Data2Sub$h2.cat),]
Data2Sub$order<-1:nrow(Data2Sub)
#Good<-aggregate(as.formula(paste("order~",chrohead,"+",poshead,sep="")), data=Data2Sub, min)$order
#Data2Sub<-Data2Sub[Data2Sub$order %in% Good,]

write.csv(Data2Sub, file=paste(opt[['out']], '_all.csv',sep=''), row.names=F)
writeLines(unique(as.character(Data2Sub[,phenhead])), con=paste(opt[['out']], '_pheno.csv',sep=''))
write.csv(as.data.frame(table(Data2Sub[,phenhead])), file=paste(opt[['out']], '_phenocount.csv',sep=''))


#Castigo = 25

fit = function(observed_I, beta0, beta_min, beta_max, tau10, tau1_min, tau1_max,
               tau20, tau2_min, tau2_max,
               N00, N0_min, N0_max, Castigo, ignore_beta_diff, max_eval = 1000 ) {
  "
  Fits the model to observed daily new case counts.

  Parameters
  ----------
  observed_I : numeric vector
    Daily observed number of new cases. (son los incubadores!! o son los infecciosos nuevos)
  beta0 : numeric vector
    Initial value for beta; the expected number of cases (incubadores) stemming from a single
    person in a single day.
  beta_min : numeric
    Beta lower bound.
  beta_max : numeric
    Beta upper bound.
  ignore_beta_diff : numeric vector
    List of beta indices for which differences should be ignored while
    calculating the loss function. This amounts to moments in time in which we
    allow the beta series to be discontinuous.

  Returns
  -------
  beta : numeric vector
    Expected number of cases stemming from a single person in a single day.
  "
  # concatenate initial parameters into a single vector for optimization
  x0 = c(beta0, tau10, tau20, N00)
  steps = length(beta0)
  lb = c(rep(beta_min, steps), tau1_min, tau2_min, N0_min)
  ub = c(rep(beta_max, steps), tau1_max, tau2_max,  N0_max)
  # set optimization parameters
  opts = list("algorithm" = "NLOPT_LN_BOBYQA", "xtol_rel" = 1.0e-7, "maxeval" = max_eval)
  print(opts)
  # define loss function
  loss = function(x) {
    # unpack values
    beta = x[1:steps]
    tau1 = x[steps + 1]
    tau2 = x[steps + 2]
    N0 = x[steps + 3]
    # calculate loss
    expected = get_expected(beta, N0, tau1, tau2) # Obtiene el número de casos infectados
    beta_diff = diff(beta)
    regularization1 =  (beta_diff/beta[1:length(beta_diff)])
    regularization =  (diff(regularization1)) ^ 2
    regularization = regularization[!1:length(regularization) %in% ignore_beta_diff]
    observed_I=round(observed_I,0)
    factSum=function(x){
      return(sum(log(1:round(x,0))))
    }
    loglikelihood=observed_I*log(expected$NN)-expected$NN-sapply(as.matrix(observed_I), FUN=factSum)
    loglikelihood[observed_I==0]=-expected$NN[observed_I==0]

    loss = -mean(loglikelihood) + Castigo * mean(regularization)
    #loss=sum((observed_I-expected$I)^2)+ sum(regularization)
    loss
  }
  result = nloptr(x0=x0, eval_f=loss, eval_grad_f=NULL, lb=lb, ub=ub, opts=opts)
  model = list()
  model$beta = result$solution[1:steps]
  model$tau1 = result$solution[steps + 1]
  model$tau2 = result$solution[steps + 2]
  model$N0 = result$solution[steps + 3]
  model$loss = result$objective
  beta=model$beta
  beta_diff = diff(beta)
  regularization1 =  (beta_diff/beta[1:length(beta_diff)])
  regularization =  (diff(regularization1)) ^ 2
  regularization = regularization[!1:length(regularization) %in% ignore_beta_diff]
  model$likelihood=(model$loss-Castigo*mean(regularization))*length(beta)
  return(model)
}


# purpose: calculate the heterosis (MPH and HPH)
# Jinliang Yang
# 2.1.2012
# updated: 4.12.2012

setwd("/Users/yangjl/Documents/Heterosis_GWAS/pheno2011");
#### Heterosis ######################################################

diallel <- read.csv("cache/pheno_diallel_master_BLUE.csv")
dim(diallel)
# 225 8
p <- read.csv("cache/pheno_namparents_blue.csv")
dim(p)
# 27 8

namid <- read.table("nam_parents", header=TRUE)
namid <- namid[, c(1,3)]

diallel$z1 <- NA;
diallel$z2 <- NA;
diallel$Genotype <- as.character(diallel$Genotype)
for(i in 1:nrow(diallel)){
  tem <- unlist(strsplit(diallel$Genotype[i], split="x"))
  diallel$z1[i] <- tem[1];
  diallel$z2[i] <- tem[2];
}

diallel <- merge(namid, diallel, by.x="pop", by.y="z1")
diallel <- merge(namid, diallel, by.x="pop", by.y="z2")
diallel <- diallel[, c(5, 3,1, 4,2, 6:12)]
names(diallel)[2:5] <- c("z1", "z2", "p1", "p2")
diallel$p1 <- toupper(diallel$p1);
diallel$p2 <- toupper(diallel$p2);

diallel$KRN_HPH <- NA
diallel$KRN_MPH <- NA
diallel$CL_HPH <- NA
diallel$CL_MPH <- NA
diallel$CW_HPH <- NA
diallel$CW_MPH <- NA
diallel$CD_HPH <- NA
diallel$CD_MPH <- NA
diallel$AKW_HPH <- NA
diallel$AKW_MPH <- NA
diallel$TKW_HPH <- NA
diallel$TKW_MPH <- NA
diallel$KC_HPH <- NA
diallel$KC_MPH <- NA

##########################################################################################################
p$Genotype <- as.character(p$Genotype)
for (i in 1:nrow(diallel)){
  p1 <- as.character(diallel$p1[i]);
  p2 <- as.character(diallel$p2[i]);
  diallel$KRN_HPH[i] <- diallel$KRN[i] - max(p[p$Genotype==p1, ]$KRN, p[p$Genotype==p2, ]$KRN);
  diallel$KRN_MPH[i] <- diallel$KRN[i] - mean(c(p[p$Genotype==p1, ]$KRN, p[p$Genotype==p2, ]$KRN), na.rm=T);
  diallel$CL_HPH[i] <- diallel$CL[i] - max(p[p$Genotype==p1, ]$CL, p[p$Genotype==p2, ]$CL);
  diallel$CL_MPH[i] <- diallel$CL[i] - mean(c(p[p$Genotype==p1, ]$CL, p[p$Genotype==p2, ]$CL), na.rm=T);
  diallel$CW_HPH[i] <- diallel$CW[i] - max(p[p$Genotype==p1, ]$CW, p[p$Genotype==p2, ]$CW);
  diallel$CW_MPH[i] <- diallel$CW[i] - mean(c(p[p$Genotype==p1, ]$CW, p[p$Genotype==p2, ]$CW), na.rm=T);
  diallel$CD_HPH[i] <- diallel$CD[i] - max(p[p$Genotype==p1, ]$CD, p[p$Genotype==p2, ]$CD);
  diallel$CD_MPH[i] <- diallel$CD[i] - mean(c(p[p$Genotype==p1, ]$CD, p[p$Genotype==p2, ]$CD), na.rm=T);
  diallel$AKW_HPH[i] <- diallel$AKW[i] - max(p[p$Genotype==p1, ]$AKW, p[p$Genotype==p2, ]$AKW);
  diallel$AKW_MPH[i] <- diallel$AKW[i] - mean(c(p[p$Genotype==p1, ]$AKW, p[p$Genotype==p2, ]$AKW), na.rm=T);
  diallel$TKW_HPH[i] <- diallel$TKW[i] - max(p[p$Genotype==p1, ]$TKW, p[p$Genotype==p2, ]$TKW);
  diallel$TKW_MPH[i] <- diallel$TKW[i] - mean(c(p[p$Genotype==p1, ]$TKW, p[p$Genotype==p2, ]$TKW), na.rm=T);
  diallel$KC_HPH[i] <- diallel$KC[i] - max(p[p$Genotype==p1, ]$KC, p[p$Genotype==p2, ]$KC);
  diallel$KC_MPH[i] <- diallel$KC[i] - mean(c(p[p$Genotype==p1, ]$KC, p[p$Genotype==p2, ]$KC), na.rm=T);
}

write.table(diallel, "cache/pheno_diallel_master_BLUE_heterosis.csv", sep=",", quote=FALSE, row.names=FALSE)


################################### INTERPOLACAO VIA TRIANGULACAO ##################################

#' Triangulacao De \code{curvacolina}
#'
#' Funcao interna executada quando \code{metodo = "triangulacao"} em \code{interpolador}. O
#' argumento \code{tessfunc} permite controlar como o espaco projetado sera tesselado em triangulos.
#' Atualmente ha duas opcoes implementadas no pacote:ang
#'
#' \itemize{
#' \item{\code{\link{tessdelaunay}}}
#' \item{\code{\link{tessradial}}}
#' }
#'
#' A primeira executa a tesselacao pelo metodo de Delaunay atraves da funcao
#' \code{\link[geometry]{delaunayn}}. A segunda e uma versao alternativa quase igual a primeira, com
#' excessao dos triangulos definidos entre a ultima curva e o maximo. Pelo metodo de Delaunay e
#' possivel que acontecam platos, isto e, triangulos cujos tres vertices pertencem a mesma curva.
#' \code{tessradial} interfere apenas nessa regiao, forcando para que todos os triangulos entre a
#' ultima curva e o maximo tenham este ponto como um vertice, deixando os dois restantes como pontos
#' adjacentes na ultima curva.
#'
#' Esta funcao nao deve ser chamada pelo usuario diretamente na maioria dos casos
#'
#' @param colina objeto \code{curvacolina} retornado pelas funcoes de leitura
#' @param tessfunc funcao ou string com nome da funcao pela qual executar a tesselacao do espaco.
#'     Ver Detalhes
#' @param ... nao possui funcao, so existe para compatibilizacao com a chamada generica de
#'     \code{\link{interpolador}}
#'
#' @return objeto da classe \code{triangulacao} contendo a tesselacao da curva colina
#'
#' @export

triangulacao <- function(colina, tessfunc = tessdelaunay, ...) {
    hl <- pot <- NULL

    if(is.character(tessfunc)) tessfunc <- as.name(tessfunc)
    tri <- eval(as.call(list(tessfunc, colina)))

    new_triangulacao(tri, colina)
}

new_triangulacao <- function(tri, colina) {

    obj <- list(triangulos = tri, colina = colina)

    class(obj) <- c("triangulacao", "interpolador")
    attr(obj, "ntri") <- nrow(tri)

    return(obj)
}

#' @export

print.triangulacao <- function(x, ...) {
    cat("* Tesselacao ", "\n")
    cat("Numero de triangulos: ", attr(x, "ntri"), "\n")
    cat("-----\n")
    cat("* Curva colina \n")
    summary(x$colina)
}

# FUNCOES DE TRIANGULACAO --------------------------------------------------------------------------

#' Triangulacao Delauney
#'
#' Realiza triangulacao do espaco pelo metodo de Delauney via \code{geometry::delaunayn}
#'
#' @param colina objeto \code{curvacolina} contendo curva a tesselar
#'
#' @return matriz com tres colunas indicando, em cada linha, o indice em \code{colina$CC} dos pontos
#'     correspondentes aos vertices de cada triangulo gerado

tessdelaunay <- function(colina) {
    hl <- pot <- NULL
    geometry::delaunayn(colina$CC[, list(hl, pot)])
}

#' Triangulacao Radial
#'
#' Define os triangulos sempre usando o maximo como um dos vertices
#'
#' Este e um metodo de triangulacao especifico para uso em torno do maximo. Cada triangulo e
#' definido usando sempre o maximo como um dos vertices. Os dois vertices restantes sao pontos
#' adjacentes entre si na curva de rendimento imediatamente inferior ao maximo, ou seja, o espaco
#' entre ultima curva e ponto maximo e fatiado em triangulos com arestas radiais, para todos os
#' pontos da ultima curva.
#'
#' @param colina um data.frame ou data.table contendo queda liquida, potencia e rendimento
#'
#' @return matriz de tres colunas indicando o indicie em \code{dat} dos pontos correspondentes aos
#'     vertices de cada triangulo. Cada linha corresponde a um triangulo
#'
#' @importFrom utils tail

tessradial <- function(colina) {

    rend <- NULL

    tri <- tessdelaunay(colina)

    # identifica triangulos da ultima curva de rend para dentro
    ultrends <- tail(attr(colina, "rends"), 2)
    innertri <- sapply(seq(ncol(tri)), function(i) colina$CC$rend[tri[, i]] %in% ultrends)
    innertri <- rowMeans(innertri) == 1
    tri <- tri[!innertri, ]

    dat <- copy(colina$CC)
    chl  <- dat[rend == ultrends[2]]$hl
    cpot <- dat[rend == ultrends[2]]$pot
    dat <- dat[rend == ultrends[1]]
    angord <- orderpoly(dat, chl, cpot)

    N1 <- nrow(dat)
    N2 <- nrow(colina$CC[!(rend %in% ultrends)])
    N3 <- nrow(colina$CC)

    out <- cbind(angord, c(angord[-1], angord[1])) + N2
    out <- cbind(out, N3)
    out <- rbind(tri, out)

    return(out)
}

# METODOS ------------------------------------------------------------------------------------------

#' @rdname getcolina

getcolina.triangulacao <- function(object) object$colina

#' Amostragem De Pontos Na Triangulacao
#'
#' Realiza interpolacao baricentrica de \code{pontos} nos triangulos da tesselacao
#'
#' @param object objeto da classe \code{triangulacao} retornado pela funcao homonima
#' @param pontos data.frame ou matriz contendo coordenadas \code{(hl, pot)} dos pontos onde
#'     interpolar
#' @param as.gradecolina booleano -- se \code{FALSE} (padrao) retorna apenas o vetor de rendimentos
#'     interpolados nas coordenadas \code{pontos}; se \code{TRUE} um objeto \code{gradecolina}. Veja
#'     \code{\link{gradecolina}}
#' @param ... existe somente para consistencia de metodos. Nao possui utilidade
#'
#' @return se \code{as.gradecolina = FALSE}, vetor de rendimentos interpolados, do contrario um
#'     objeto \code{\link{gradecolina}}
#'
#' @export

predict.triangulacao <- function(object, pontos, as.gradecolina = FALSE, ...) {

    pontos <- pontos[complete.cases(pontos), ]

    if(nrow(pontos) == 0) return(numeric(0))

    npontos <- nrow(pontos)
    pontos <- data.matrix(pontos)

    triangulos <- object$triangulos
    colina     <- data.matrix(object$colina$CC)

    barycoord <- geometry::tsearchn(colina[, c("hl", "pot")], triangulos, pontos)

    rends <- sapply(seq(npontos), function(i) {
        indtri <- barycoord$idx[i]
        vertices <- triangulos[indtri, ]
        rends  <- colina[vertices, "rend"]
        sum(barycoord$p[i, ] * rends)
    })

    if(as.gradecolina) {
        out <- new_gradecolina(pontos, rends, object)
    } else {
        out <- as.numeric(rends)
    }

    return(out)
}


#' bootstrap.face
#'
#' Apply the bootstrap test for testing a quadratic polynomial covariance
#'
#' @param data  "data frame with three arguments:
#'                  (1) "argvals": observation times;
#'                  (2) "subj": subject indices;
#'                  (3) "y": values of observations;
#'  Note that: we only handle complete data, so missing values are not allowed at this moment
#' @param nbs number of bootstrap samples, default = 1000
#' @param argvals.new  "argvals.new" if we want the estimated covariance function at "argvals.new"; if NULL,
#' then 100 equidistant points in the range of "argvals" in "data"
#' @param semi.iter indicator whether to use semi_iterative for bootstrap
#' @param tune.bs indicator whether to use tunning for bootstrap
#' @param center   "center" means if we want to compute population mean
#' @param knots   number of interior knots for B-spline basis functions to be used;
#' @param p the degrees of B-splines; defaults to 3.
#' @param m the order of differencing penalty; defaults to 2.
#' @param lambda  the value of the smoothing parameter for covariance smoothing; defaults to NULL.
#' @param lambda_mean the value of the smoothing parameter for mean smoothing; defaults to NULL.
#' @param search.length the number of equidistant (log scale) smoothing parameters to search; defaults to 14.
#' @param upper,lower bounds for log smoothing parameter for first step of estimation; defaults are -3 and 10, respectively.
#' @param pve Defaults 0.99. To select the number of eigenvalues by percentage of variance.
#'
#'
#' @import nlme
#' @return an object "bootstrap.face" contain:
#'  fit.alt: Alternative model fit (functional principal components analysis)
#'  fit.null: Null model fit (linear random effects)
#'  C.alt: Covariance matrix under alternative model
#'  C.null: Covariance matrix under null model
#'  tn: Test statistic
#'  p: p-value for test statistic based on the bs.approx
#'  bs.approx: List of values from the null distribution of Tn
#'
#' @references modified from face.sparse.inner from face package
#' and bootstrap.test.R written by Stephanie
bootstrap.face <- function(data, nbs = 1000, argvals.new = NULL,
                          trunc.eig = 1,
                          semi.iter = F, center.bs = F,
                          center = TRUE,
                           knots = 7, knots.option = "equally-spaced",
                           p = 3, m = 2, lambda = NULL, lambda_mean = NULL,
                           search.length = 14,
                           lower = -3, upper = 10,
                           pve = 0.99, off_diag = F, gam.mgcv = T, no.pen = F, dense.grid = 8e5) {
  #########################
  #### step 0: read in data
  #########################
  check.data(data)
  nb <- knots + p

  y <- data$y
  t <- data$argvals
  subj <- data$subj
  tnew <- argvals.new
  if (is.null(tnew)) tnew <- seq(min(t), max(t), length = 100)

  fit_mean <- NULL

  knots.initial <- knots
  #########################
  #### step 1: Test statistics
  #########################

  ###### a. Initialize C, X, Q
  r <- y
  if (center) {
    if (gam.mgcv) {
      fit_mean <- mgcv::gam(as.vector(y) ~ s(t, k = nb))
    } else {
      fit_mean <- pspline(data, argvals.new = tnew, knots = knots.initial, lambda = lambda_mean)
    }
    r <- y - fit_mean$fitted.values
  }

  raw <- raw.construct(data.frame("argvals" = t, "subj" = subj, "y" = as.vector(r)))
  C <- raw$C  # C
  st <- raw$st
  N2 <- raw$N2
  n0 <- raw$n0
  # indicator where ti = tj
  delta <- Matrix((st[, 1] == st[, 2]) * 1) # sparse
  delta <- c(as.matrix(delta))

  knots <- construct.knots(t, knots, knots.option, p)
  ## construct design & penalty on column dimension
  List <- pspline.setting(st[, 1], knots = knots, p, m, type = "simple", knots.option = knots.option)
  B1 <- List$B
  B1 <- Matrix(B1)
  DtD <- List$P
  ## construct design on row dimension
  B2 <- spline.des(knots = knots, x = st[, 2], ord = p + 1, outer.ok = TRUE, sparse = TRUE)$design
  c <- dim(B1)[2]
  c2 <- c * (c + 1) / 2
  ## combine tensor product design
  B <- Matrix(t(KhatriRao(Matrix(t(B2)), Matrix(t(B1)))))
  G <- Matrix(duplication.matrix(c))
  ### double the weight for off-diagonal
  # v_idx <- which(delta == 0)
  # B[v_idx, ] <- sqrt(2) * B[v_idx, ]
  if (off_diag) {
    B <- B * (sqrt(2) * (1 - delta) + delta)
    C <- C * (sqrt(2) * (1 - delta) + delta)
  }

  # BtWB <- matrix(0, nrow = c^2, ncol = c^2)
  # Wdelta <- c()
  # WC <- c()
  # for (i in 1:n0) {
  #   # select combos for object i
  #   seq <- (sum(N2[1:i]) - N2[i] + 1):(sum(N2[1:i]))
  #   B3 <- Matrix(matrix(B[seq, ], nrow = length(seq)))
  #   BtWB <- BtWB + crossprod(B3, B3)
  #   Wdelta <- c(Wdelta, as.matrix(delta[seq]))
  #   WC <- c(WC, as.matrix(C[seq]))
  # }
  BtB <- crossprod(B)
  BG <- B %*% G # sparse

  GtBtBG <- crossprod(G, BtB %*% G)
  detde <- crossprod(delta) # detWde = sum(delta)
  GtBtdelta <- crossprod(BG, delta)
  XtX <- rbind(cbind(GtBtBG, GtBtdelta), cbind(t(GtBtdelta), detde))
  ## design X
  X <- cbind(BG, delta)
  #XtX <- crossprod(X)
  ## penalty
  P <- crossprod(G, Matrix(suppressMessages(kronecker(diag(c), DtD)))) %*% G
  Q <- bdiag(P, 0)

  ###### b. Pre-calculate s, F, B^*, Li, g, f, G (2 eigens)
  eSig <- eigen(XtX, symmetric = TRUE)
  V <- eSig$vectors
  E <- eSig$values
  E <- E + 0.000001 * max(E)
  Sigi_sqrt <- matrix.multiply(V, 1 / sqrt(E)) %*% t(V)

  tUQU <- crossprod(Sigi_sqrt, (Q %*% Sigi_sqrt))
  Esig <- eigen(tUQU, symmetric = TRUE)
  # s, f, F(m_F here)
  U <- Esig$vectors
  s <- Esig$values
  A0 <- Sigi_sqrt %*% U
  m_F <- as.matrix(X %*% A0)

  m_FtF <- crossprod(m_F) # FTF
  f <- crossprod(m_F, C) # f=FTC

  c2 <- c2 + 1
  g <- rep(0, c2)
  G1 <- matrix(0, c2, c2)
  mat_list <- list()
  # G(G1 here), g, Li
  for (i in 1:n0) {
    seq <- (sum(N2[1:i]) - N2[i] + 1):(sum(N2[1:i]))
    Fi <- matrix(m_F[seq, ], nrow = length(seq))
    Li <- crossprod(Fi)#Li <- FitFi

    fi <- crossprod(Fi, C[seq]) # t(Fi)Ci

    g <- g + fi * fi
    G1 <- G1 + Li * (fi %*% t(f))

    LList <- list()
    LList[[1]] <- Li
    #LList[[2]] <- Li
    mat_list[[i]] <- LList
  }
  # B^*
  st.construct <- function(tnew) {
    m1 <- length(tnew)
    if (m1 > 1) {
      st <- cbind(vech(kronecker(tnew, t(rep(1, m1)))),
                  vech(kronecker(rep(1, m1), t(tnew))))
    } else if (m1 == 1) {
      st <- rbind(st, c(tnew, tnew))
    }
    st
  }
  #stnew <- st.construct(tnew)
  #Bnew1 <- spline.des(knots = knots, x = stnew[, 1], ord = p + 1, outer.ok = TRUE, sparse = TRUE)$design
  #Bnew2 <- spline.des(knots = knots, x = stnew[, 2], ord = p + 1, outer.ok = TRUE, sparse = TRUE)$design
  #Bstar <- Matrix(t(KhatriRao(Matrix(t(Bnew2)), Matrix(t(Bnew1)))))

  #Xstar <- Bstar %*% G
  #delta_star <- which(stnew[, 1] != stnew[, 2])
  #Xstar[delta_star, ] <- sqrt(2) * Xstar[delta_star, ]
  dense.t <- seq(min(tnew), max(tnew), length.out = dense.grid)
  Xstar <- spline.des(knots = knots, x = dense.t, ord = p + 1, outer.ok = TRUE, sparse = TRUE)$design
  Xstar <- crossprod(Xstar) / dense.grid
  Eigen1 <- eigen(Xstar)
  Xstar.half <- Eigen1$vectors %*% diag(sqrt(Eigen1$values)) %*% t(Eigen1$vectors)
  Xstar.invhalf <- Eigen1$vectors %*% diag(1 / sqrt(Eigen1$values)) %*% t(Eigen1$vectors)

  Bnew <- spline.des(knots = knots, x = tnew, ord = p + 1, outer.ok = TRUE, sparse = TRUE)$design

  ###### c. Null estimate Ctilde0, f0 (2 eigens)
  fitNull <- function(data) {
     try(nlme::lme(y ~ 1, random = list(subj = pdSymm(~ 1 + argvals)),
          data = data), silent = T)
  }
  fit.null <- fitNull(data.frame("argvals" = t, "subj" = subj, "y" = as.vector(r)))
  if ("try-error" %in% class(fit.null)) {# issue with null fit
    stop(fit.null)
  }
  calc.R0 <- function(fit.null, times) {
    var.mat <- VarCorr(fit.null)
    sigsq0 <- as.numeric(var.mat[1, 1])
    sigsq1 <- as.numeric(var.mat[2, 1])
    cov01 <- as.numeric(var.mat[2, 3]) * sqrt(sigsq0 * sigsq1) # corr->cov
    Rbar0 <- sigsq0 + cov01 * (times[, 1] + times[, 2]) + sigsq1 * (times[, 1] * times[, 2])
    if (off_diag) {
      Rbar0 <- Rbar0 * (sqrt(2) * (1 - delta) + delta)
    }
    list(Rbar0 = Rbar0, coef.null = c(sigsq0, cov01, sigsq1))
  }

  Rbar0.fit <- calc.R0(fit.null, st)
  C0 <- Rbar0.fit$Rbar0

  ###### d. tunning(s, F, F^*, Li, g, f, G) to get lambda^*
  Lambda <- seq(lower, upper, length = search.length)
  Gcv <- 0 * Lambda
  gcv <- function(x) {
    lambda <- exp(x)
    d <- 1 / (1 + lambda * s)
    f_d <- f * d
    cv0 <- -2 * sum(f_d * f)
    cv1 <- sum(f_d * (m_FtF %*% f_d))
    cv2 <- 2 * sum(d * g)
    cv3 <- -4 * sum(d * (G1 %*% d))
    cv4 <- sum(unlist(sapply(mat_list, function(x) {
      a <- x[[1]] %*% f_d
      2 * sum(a * a * d)
    })))
    cv <- cv0 + cv1 + cv2 + cv3 + cv4
    return(cv)
  }
  if (is.null(lambda)) {
    Lambda <- seq(lower, upper, length = search.length) # construct lambda grid
    Length <- length(Lambda)
    Gcv <- rep(0, Length)
    for (i in 1:Length) {
      Gcv[i] <- gcv(Lambda[i])
    }
    i0 <- which.min(Gcv)
    lambda <- exp(Lambda[i0]) # lambda^*
  }

  ###### e. calculate estimated covariance function and test statistics
  Xxstar <- crossprod(Bnew)
  m_est_sigma <- (t(A0[c2, ]) * t(1 / (1 + lambda * s))) %*% t(m_F)
  sigsq <- m_est_sigma %*% C
  if (sigsq <= 0.000001) {
    # warning("error variance cannot be non-positive, reset to 1e-6!")
    sigsq <- 0.000001
  }
  if (no.pen || lambda == 0) {
    m_est <- A0[-c2, ] %*% t(m_F)
  } else {
    m_est <- matrix.multiply(A0[-c2, ], 1 / (1 + lambda * s)) %*% t(m_F)
  }

  trun_mat <- function(C) {
    alpha <- m_est %*% C
    Theta <- G %*% alpha
    Theta <- matrix(Theta, c, c)
    # sigma2 <- alpha[c2]
    # if (sigma2 <= 0.000001) {
    #   # warning("error variance cannot be non-positive, reset to 1e-6!")
    #   sigma2 <- 0.000001
    # }
    if (trunc.eig != 0) {
      # make sure Theta positive definite(2 eigens)
      Eigen <- eigen(Theta, symmetric = TRUE)
      Eigen$values[Eigen$values < 0] <- 0
      npc <- sum(Eigen$values > 0) # which.max(cumsum(Eigen$values)/sum(Eigen$values)>pve)[1]
      if (npc > 1) {
        pc <- Bnew %*% Eigen$vectors[, 1:npc]
        C <- tcrossprod(matrix.multiply(pc, Eigen$values[1:npc]), pc)
      }
      if (npc == 1) {
        C <- Eigen$values[1] * tcrossprod(Bnew %*% Eigen$vectors[, 1])
      }
    } else {
       C <- as.matrix(tcrossprod(Bnew %*% Matrix(Theta), Bnew))
    }
    list(C = C, Theta = Theta)
  }

  trunc.mat.inner <- function(Theta, trunc.eig = 1) {
  if (trunc.eig == 1) {
    eigen.fit <- eigen(Theta, symmetric = T)
    efuncs <- eigen.fit$vectors
  } else if (trunc.eig == 2) {
    Theta <- as.matrix(Xstar.half %*% Matrix(Theta) %*% Xstar.half)
    eigen.fit <- eigen(Theta, symmetric = T)
    efuncs <- Xstar.invhalf %*% eigen.fit$vectors
  }
  evals <- as.numeric(eigen.fit$values)
  evals[evals < 1e-5] <- 0
  return(matrix.multiply(efuncs, evals) %*% t(efuncs))
}

  # sigsq <- l_mat$sigma2
  C.alt <- trun_mat(C)
  C.null <- trun_mat(C0)

  if (trunc.eig == 0) {
    Tn <- C.alt$Theta - C.null$Theta
  } else {
    Theta.alt <- trunc.mat.inner(C.alt$Theta, trunc.eig)
    Theta.null <- trunc.mat.inner(C.null$Theta, trunc.eig)
    Tn <- Theta.alt - Theta.null
  }

  Tn <- Tn %*% Xstar
  Tn <- sqrt(sum(Tn * t(Tn)))

  #########################
  #### step 2: Bootstrap
  #########################

  raw.C <- function(data) {
    y <- data$y
    subj <- data$subj
    subj_unique <- unique(subj)
    n <- length(subj_unique)
    C <- c()
    for (i in 1:n) {
      r1 <- y[subj == subj_unique[i]]
      m1 <- length(r1)
      if (m1 > 1) {
        C <- c(C, vech(tcrossprod(r1)))
      } ## for if(m1>1)
      if (m1 == 1) {
        C <- c(C, r1^2)
      }
    } ## for i
    if (off_diag) {
      C <- C * (sqrt(2) * (1 - delta) + delta)
    }
    C
  }

  bs.stats <- c()
  bs.success <- 0
  if (center.bs) {
      mean.bs <- fit_mean$fitted.values
      this.bs <- data
  } else {
      mean.bs <- 0
  }

  if (semi.iter) {
    C0.bs <- matrix(NA, nrow = length(C), ncol = 0)
    C.bs  <- C0.bs
    y.bs <- resample(data, mean.bs, Rbar0.fit$coef.null, sigsq, L = nbs)

    while (bs.success < nbs) { #(0.15s)

      ###### a. generate Y_ij^(l) (0.01s)## fit_mean$fitted.values
      s_iterator <- bs.success + 1
      if (ncol(y.bs) < s_iterator) {
        y.bs <- cbind(y.bs, resample(data, mean.bs, Rbar0.fit$coef.null, sigsq, L = 5))
      }
      y <- y.bs[, s_iterator]

      ###### b. center Y_ij^(l) (0.01s)
      r <- y
      if (center && center.bs) {
        this.bs$y <- r
        if (gam.mgcv) {
          fit_mean.bs <- mgcv::gam(as.vector(y) ~ s(t, k = nb))
        } else {
          fit_mean.bs <- pspline(this.bs, argvals.new = tnew, knots = knots.initial, lambda = lambda_mean)
        }
        r <- y - fit_mean.bs$fitted.values
      }
      data.demean.bs <- data.frame(
        "argvals" = t,
        "subj" = subj, "y" = as.vector(r)
      )

      ###### c. Initialize C0^(l) (0.08s)
      fit.null.bs <- fitNull(data.demean.bs) # null fit
      if ("try-error" %in% class(fit.null.bs)) { # issue with null fit
        y.bs <- y.bs[, -s_iterator]
        next # if problem
      }
      Rbar0.fit.bs <- calc.R0(fit.null.bs, st)
      # C0.bs <- cbind(C0.bs, Rbar0.fit.bs$Rbar0)

      ###### d. Initialize C^(l) (0.05s)
      C.bs <- cbind(C.bs, Rbar0.fit.bs$Rbar0 - raw.C(data.demean.bs))
      bs.success <- bs.success + 1

    }
# ptm <- proc.time()
    # Compute delta.bs (0.17 s)
    delta.bs <- m_est %*% Matrix(C.bs)
# print(proc.time() - ptm)
    # compute bs statistics (0 s)
    bs.stats <- (Xstar %*% delta.bs) ^ 2
    bs.stats <- sqrt(colSums(bs.stats))

  } else {
    while (bs.success < nbs) { #(0.6s)
#ptm <- proc.time()
      ###### a. generate Y_ij^(l) (0.01s)
      y <- c(resample(data, mean.bs, Rbar0.fit$coef.null, sigsq))

      ###### b. center Y_ij^(l) (0.01s)
      r <- y
      if (center && center.bs) {
        this.bs$y <- r
        if (gam.mgcv) {
          fit_mean.bs <- mgcv::gam(as.vector(y) ~ s(t, k = nb))
        } else {
          fit_mean.bs <- pspline(this.bs, argvals.new = tnew, knots = knots.initial, lambda = lambda_mean)
        }
        r <- y - fit_mean.bs$fitted.values
      }
      data.demean.bs <- data.frame(
        "argvals" = t,
        "subj" = subj, "y" = as.vector(r)
      )

      ###### c. Initialize C0^(l) (0.04s)
      fit.null.bs <- fitNull(data.demean.bs) # null fit
      if ("try-error" %in% class(fit.null.bs)) { # issue with null fit
        next # if problem
      }

      C0.bs <- calc.R0(fit.null.bs, st)$Rbar0

      ###### d. Initialize C^(l) (0.00s)
      C.bs <- raw.C(data.demean.bs) #(0.00s)

      ###### e. calculate estimated covariance function and test statistics
      # (0.00 s)

      if (trunc.eig == 0) {
        Tn.bs <- matrix(G %*% (m_est %*% (C.bs - C0.bs)), c)
      } else {
        Theta.alt <- trunc.mat.inner(matrix(G %*% (m_est %*% C.bs), c), trunc.eig)
        Theta.null <- trunc.mat.inner(matrix(G %*% (m_est %*% C0.bs), c), trunc.eig)
        Tn.bs <- Theta.alt - Theta.null
      }
      Tn.bs <- Tn.bs %*% Xstar
      Tn.bs <- sqrt(sum(Tn.bs * t(Tn.bs)))

      bs.stats <- c(bs.stats, Tn.bs) # save bs stats
      bs.success <- bs.success + 1
#print(proc.time() - ptm)

    }
  }

  #########################
  #### step 3: P-values
  #########################
  p.bs <- function(stat, bs.stat) {
    p <- mean(stat <= bs.stat)
    list(p = p, mean = mean(bs.stat), var = var(bs.stat))
  }
  Tn.stats <- p.bs(Tn, unlist(bs.stats))

  list(
    mu = fit_mean$fitted.values,
    C.alt = C.alt$C,
    C.null = C.null$C,
    sigma2 = sigsq,
    Tn = Tn, p = Tn.stats$p, p.var = Tn.stats$var,
    bs.approx = bs.stats
  )
}

############################
# Written by: Stephanie Chen (stchen3@ncsu.edu)
# Purpose: Generate bootstrap resamples
# Updated: Aug 4, 2018
# Modified on July 30, 2021

resample <- function(data, mu, coef.null, sigsq, L = 1) {
  nsubj <- length(unique(data$subj))
  cov.mat <- matrix(c(coef.null[1], coef.null[2], coef.null[2], coef.null[3]), nrow = 2)
  b.mat <- matrix(rnorm(2 * nsubj * L), ncol = 2) %*% chol(cov.mat) # random effects

  # r.slope & r.int by subject
  par.random <- sapply(seq_len(L), function(i) {
        s_offset <- (i-1) * nsubj
        sapply(seq_len(nsubj), function(x) {
            b.mat[s_offset + x, 1] + b.mat[s_offset + x, 2] * subset(data, subj == x)$argvals
        })
    })

  if (is.list(par.random)) { # if a list
    par.random <- unlist(par.random)
  }
  # mean.null + null.par r.int & r.slope*t + alt.par residual
  matrix(mu + par.random + rnorm(nrow(data) * L, sd = sqrt(sigsq)), ncol = L)
}

# resample(data, 0, c(1,0,1), 2, L = 2)
# sapply(1:2, function(i) {
#         sapply(6:8, function(x) {
#             i*x
#         })
#     })


subroutine delout(delsum,nadj,madj,x,y,ntot,npd,nerror)

# Put a summary of the Delaunay triangles with a vertex at point i,
# for i = 1, ..., npd, into the array delsum.  Do this in the original
# order of the points, not the order into which they have been
# bin-sorted.
# Called by master.

implicit double precision(a-h,o-z)
dimension nadj(-3:ntot,0:madj), x(-3:ntot), y(-3:ntot)
dimension delsum(npd,4)

do i = 1,npd {
        area = 0.   # Initialize area of polygon consisting of triangles
                    # with a vertex at point i.
        # Get the coordinates of the point and the number of
        # (real) triangles emanating from it.
        np = nadj(i,0)
	xi = x(i)
	yi = y(i)
        npt = np
        do k = 1,np {
                kp = k+1
                if(kp>np) kp = 1
                if(nadj(i,k)<=0|nadj(i,kp)<=0) npt = npt-1
        }

        # For each point in the adjacency list of point i, find its
        # successor, and the area of the triangle determined by these
        # three points.
        do j1 = 1,np {
                j = nadj(i,j1)
                if(j<=0) next
		xj = x(j)
		yj = y(j)
                call succ(k,i,j,nadj,madj,ntot,nerror)
		if(nerror > 0) return
                if(k<=0) next
		xk = x(k)
		yk = y(k)
                call triar(xi,yi,xj,yj,xk,yk,tmp)
                # Downweight the area by 1/3, since each
                # triangle eventually appears 3 times over.
                area = area+tmp/3.
        }
	delsum(i,1) = xi
	delsum(i,2) = yi
	delsum(i,3) = npt
	delsum(i,4) = area
}

return
end


library("rgl")

crossdot <- function(a, b, c) {
	return (c[1]*(a[2]*b[3]-a[3]*b[2]) + c[2]*(a[3]*b[1]-a[1]*b[3]) + c[3]*(a[1]*b[2]-a[2]*b[1]) )
}

cross <- function(a, b) {
	return (c( a[2]*b[3]-a[3]*b[2] , a[3]*b[1]-a[1]*b[3] , a[1]*b[2]-a[2]*b[1] ))
}

arrow3d <- function(start, end, cl) {
	x <- (cross(end-start, c(0,0,1))/10)
	lines3d(c(start[1], end[1]), c(start[2], end[2]), c(start[3], end[3]), col=cl, lwd=3)
	head <- start+(end-start)*0.8
	lines3d(c(end[1], head[1]+x[1]), c(end[2], head[2]+x[2]), c(end[3], head[3]+x[3]), col=cl, lwd=3)
	lines3d(c(end[1], head[1]-x[1]), c(end[2], head[2]-x[2]), c(end[3], head[3]-x[3]), col=cl, lwd=3)
	lines3d(c(head[1]-x[1], head[1]+x[1]), c(head[2]-x[2], head[2]+x[2]), c(head[3]-x[3], head[3]+x[3]), col=cl, lwd=3)
}

if (rerun==0) {
	gas <- read.table("halo_gas.txt", head=TRUE)
	#n <- intersect(which(gas$m>2e5), which(sqrt(gas$x^2 + gas$y^2 + gas$z^2)<0.01))
	n <- which(sqrt(gas$x^2 + gas$y^2 + gas$z^2)<0.04)
	gas <- gas[n,]

	star <- read.table("halo_star.txt", head=TRUE)
	n <- which(sqrt(star$x^2 + star$y^2 + star$z^2)<0.04)
	star <- star[n,]
	n <- which(star$m > quantile(star$m, probs=0.8))
	#n <- sample(seq(length(star[,1])), 10000)
	star <- star[n,]

	phi <- read.table("halophi.txt", head=TRUE)
	phi$pot[which(is.nan(phi$pot))] <- 0
	phifunc <- splinefun(phi$r, phi$pot)
	realv <- sqrt(phifunc(phi$r, deriv=1)*phi$r)

	j0 <- c(0,0,0)
	for (i in seq(length(gas$x))) {
	  	j0 <- j0 + cross(c(gas$x[i], gas$y[i], gas$z[i]), c(gas$vx[i], gas$vy[i], gas$vz[i]))
	}
	j0 <- j0/sqrt(sum(j0^2))

	ratio <- numeric(length(gas$x))
	for (i in seq(length(gas$x))) {
		radius <- sqrt(gas$x[i]^2 + gas$y[i]^2 + gas$z[i]^2)
		ratio[i] <- crossdot(c(gas$x[i], gas$y[i], gas$z[i]), c(gas$vx[i], gas$vy[i], gas$vz[i]), j0)/radius
		radbin <- which.min(abs(phi$r - radius))
		ratio[i] <- ratio[i]/realv[radbin]
	}

	pcols <- ratio
	pcols[which(is.infinite(pcols))] = 0
	pcols <- pcols/max(pcols)
	pcols[which(pcols<0)] = 0

	sratio <- numeric(length(star$x))
	for (i in seq(length(star$x))) {
		radius <- sqrt(star$x[i]^2 + star$y[i]^2 + star$z[i]^2)
		sratio[i] <- crossdot(c(star$x[i], star$y[i], star$z[i]), c(star$vx[i], star$vy[i], star$vz[i]), j0)/radius
		radbin <- which.min(abs(phi$r - radius))
		sratio[i] <- sratio[i]/realv[radbin]
	}

	spcols <- sratio
	spcols[which(is.infinite(spcols))] = 0
	spcols <- spcols/max(spcols)
	spcols[which(spcols<0)] = 0

	force <- read.table("forcemap.txt", head=TRUE)
}

if (type==0) {
	n <- which(ratio>=1.0)
	plot3d(gas$x[n], gas$y[n], gas$z[n], col=rgb(pcols[n], 0, 0), xlim=c(-0.01, 0.01), ylim=c(-0.01, 0.01), zlim=c(-0.01, 0.01))
	n <- which(ratio<1.0)
	points3d(gas$x[n], gas$y[n], gas$z[n], col=rgb(0, 0, pcols[n]))
}

if (type==1) {
	n <- which(ratio>=1.0)
	plot3d(gas$x[n], gas$y[n], gas$z[n], col=rgb(pcols[n], 0, 0), xlim=c(-0.01, 0.01), ylim=c(-0.01, 0.01), zlim=c(-0.01, 0.01))
	n <- which(ratio<1.0)
	points3d(gas$x[n], gas$y[n], gas$z[n], col=rgb(0, 0, pcols[n]))
	arrow3d(-j0/500,j0/500, "green")
	for (i in seq(length(force$x))) {
		fscale <- 4e10
		arrow3d(c(force$x[i], force$y[i], force$z[i]), c(force$x[i], force$y[i], force$z[i])+c(force$Fx[i], force$Fy[i], force$Fz[i])/fscale, "black")
	}
}

if (type==2) {
	n <- which(sratio>=1.0)
	plot3d(star$x[n], star$y[n], star$z[n], col=rgb(spcols[n], 0, 0), xlim=c(-0.01, 0.01), ylim=c(-0.01, 0.01), zlim=c(-0.01, 0.01))
	n <- which(sratio<1.0)
	points3d(star$x[n], star$y[n], star$z[n], col=rgb(0, 0, spcols[n]))
	arrow3d(-j0/500,j0/500, "green")
}

if (type==3) {
	n <- which(sratio>=1.0)
	plot3d(star$x[n], star$y[n], star$z[n], col=rgb(spcols[n], 0, 0), xlim=c(-0.01, 0.01), ylim=c(-0.01, 0.01), zlim=c(-0.01, 0.01))
	n <- which(sratio<1.0)
	points3d(star$x[n], star$y[n], star$z[n], col=rgb(0, 0, spcols[n]))
	tr <- read.table("trcurve.txt", head=TRUE)
	theta <- seq(0,2*pi,by=pi/100)
	for (i in seq(length(tr$r)/2)) {
		xhat <- cross(c(tr$Lx[i], tr$Ly[i], tr$Lz[i]), c(1,0,0))
		yhat <- cross(c(tr$Lx[i], tr$Ly[i], tr$Lz[i]), xhat)
		xhat <- (xhat*tr$r[i])/sqrt(xhat[1]^2 + xhat[2]^2 + xhat[3]^2)
		yhat <- (yhat*tr$r[i])/sqrt(yhat[1]^2 + yhat[2]^2 + yhat[3]^2)
		lines3d(xhat[1]*cos(theta)+yhat[1]*sin(theta), xhat[2]*cos(theta)+yhat[2]*sin(theta), xhat[3]*cos(theta)+yhat[3]*sin(theta), col="green", lwd=2)
	}
}


#autor:      Joao Sollari Lopes
#local:      INE, Lisboa
#Rversion:   3.6.1
#criado:     10.07.2017
#modificado: 06.12.2019

#based on https://en.wikipedia.org/wiki/File:Local_maximum.png
f <- function(x,y){
    exp(1)^(-(x^2+y^2))+2*exp(1)^(-((x-1.5)^2+(y-1.5)^2))
}
x <- seq(-2,4,len=50)
y <- seq(-2,4,len=50)
z <- outer(x,y,f)

fnam <- "../images/concept_3.tiff"
tiff(file=fnam,units="in",width=5,height=4,res=300,compression="lzw")
par(mar=c(0.2,0.2,0.2,0.2))
persp(x,y,z,phi=20,theta=30,d=5,expand=0.5,box=FALSE)
dev.off()


### Calculate matrix using Plucinski's Unsupervized Naive Bayes classifier

# calculate allele frequencies

# calculate likelihood of no relation and likelihood of relation

calculate_loglikelihood = function(v1,v2,p1,p2,ploid){
	# p1 is vector of allele frequencies in sample 1
	# p2 is vector of allele frequencies in sample 2
	n1 = length(p1)
	n2 = length(p2)
	loglikelihood0 = sum(log(p1))+sum(log(p2))
	loglikelihood1 = log(sum(sum(sapply(1:n1, function (i) sapply(1:n2,
				function (j) 1/n1/n2 * (v1[i] == v2[j])*exp((sum(log(p1[-i]))+sum(log(p2)))))))))
	if (length(v1) > 1 & length(v2) > 1) {
	pairs1 = combn(v1,2,simplify = FALSE)
	pairs2 = combn(v2,2,simplify = FALSE)
	npairs1 = length(pairs1)
	npairs2 = length(pairs2)

	loglikelihood2 = log(sum(sum(sapply(1:npairs1, function (i) sapply(1:npairs2,
				function (j) 1/npairs1/npairs2 * (sum(sort(pairs1[[i]]) == sort(pairs2[[j]]))==2)*
							exp((sum(log(p1[-match(pairs1[[i]],v1)]))+sum(log(p2)))))))))
	} else { loglikelihood2 = NA}
	if (ploid == 1) {
		loglikelihood2 = loglikelihood1
	}
	c(loglikelihood0,loglikelihood1,loglikelihood2)
}

##### modify epsilon value here

epsilon = 0.3072  #rate of missed alleles, estimated by Joel May 2 2019

calculate_loglikelihood2 = function(v1,v2,p1,p2,ploid){
	# p1 is vector of allele frequencies in sample 1
	# p2 is vector of allele frequencies in sample 2
	n1 = length(p1)
	n2 = length(p2)
	loglikelihood0 = sum(log(p1))+sum(log(p2))
	loglikelihood1 = log(max(sapply(1:n1, function (i) sapply(1:n2,
				function (j)  (v1[i] == v2[j])*exp((sum(log(p1[-i]))+sum(log(p2)))))),na.rm=TRUE))
	if (length(v1) > 1 & length(v2) > 1) {
	pairs1 = combn(v1,2,simplify = FALSE)
	pairs2 = combn(v2,2,simplify = FALSE)
	npairs1 = length(pairs1)
	npairs2 = length(pairs2)

	loglikelihood2 = log(max(sapply(1:npairs1, function (i) sapply(1:npairs2,
				function (j) (sum(sort(pairs1[[i]]) == sort(pairs2[[j]]))==2)*
							exp((sum(log(p1[-match(pairs1[[i]],v1)]))+sum(log(p2)))))),na.rm=TRUE))
	} else { loglikelihood2 = NA}

	if (loglikelihood1 == -Inf) {
		loglikelihood1 = log(epsilon * min(c(p1,p2)))
	}
	if (loglikelihood2 == -Inf & ploid > 1) {
		nsharedalleles = max(sapply(1:npairs1, function (i) sapply(1:npairs2,
						function (j) (length(intersect(pairs1[[i]], pairs2[[j]])))))) ## modified line - used to be:
									                               ## function (j) (sum(sort(pairs1[[i]]) == sort(pairs2[[j]]))))))
		loglikelihood2 = log((epsilon*min(c(p1,p2))) ^ (2-nsharedalleles))
	}
	if (ploid == 1) {
		loglikelihood2 = loglikelihood1
	}
	c(loglikelihood0,loglikelihood1,loglikelihood2)
}

alleles = list()
frequencies = list()

for (j in 1:nloci) {
	locicolumns = grepl(paste(locinames[j],"",sep=""),colnames(data))
	raw_alleles = c(as.matrix(data[,locicolumns]))
	raw_alleles[raw_alleles == "NA"] = NA
	raw_alleles[raw_alleles == 0] = NA
	alleles[[j]] = unique(raw_alleles[!is.na(raw_alleles)])
	frequencies[[j]] = sapply(alleles[[j]], function(x) sum(raw_alleles == x,na.rm=TRUE))
	frequencies[[j]] = frequencies[[j]] / sum(frequencies[[j]])

}

observeddatamatrix = list()
for (j in 1:nloci) {
	locus = locinames[j]
	locicolumns = grepl(paste(locus,"",sep=""),colnames(data))
	oldalleles = as.vector(data[,locicolumns])
	oldalleles [oldalleles == "NA"] = NA
	oldalleles [oldalleles == 0] = NA
	if (length(dim(oldalleles)[2]) == 0) {
		oldalleles = matrix(oldalleles,length(oldalleles),1)
	}
	observeddatamatrix[[j]] = oldalleles
}

pairwisedistance = function(isolate1,isolate2){
	print(((isolate2-1)*nids+isolate1)/ (nids*nids))
	print(isolate1)
	print(isolate2)
	loglik = matrix(NA,nloci,3)
	for (j in 1:nloci) {
		v1 = observeddatamatrix[[j]][isolate1,]
		v1 = v1[!is.na(v1)]
		p1 = frequencies[[j]][match(v1,alleles[[j]])]
		v2 = observeddatamatrix[[j]][isolate2,]
		v2 = v2[!is.na(v2)]
		p2 = frequencies[[j]][match(v2,alleles[[j]])]
		if (length(v1) > 0 & length(v2) > 0) {
			loglik[j,] = calculate_loglikelihood2(v1,v2,p1,p2,ploidy[j])
		} else { loglik[j,] = c(NA,NA,NA)}
	}
	loglik_allloci = (colSums(matrix(loglik,ncol=3),na.rm=TRUE))
	lik_allloci = exp(loglik_allloci) / sum(exp(loglik_allloci))
	sum(lik_allloci*c(0,1,2))
}

allpossiblepairs = expand.grid(1:nids,1:nids)
allpossiblepairs = unique(allpossiblepairs[allpossiblepairs[,1] <= allpossiblepairs[,2],])

# pairwisedistancevector = unlist(lapply(1:dim(allpossiblepairs)[1], function (x) pairwisedistance(allpossiblepairs[x,1],allpossiblepairs[x,2]))) # not parallel

				  ###### MODIFY NUMBER OF CORES BELOW - mc.cores=###

pairwisedistancevector = unlist(mclapply(1:dim(allpossiblepairs)[1], function (x) pairwisedistance(allpossiblepairs[x,1],allpossiblepairs[x,2]),mc.cores=12)) # parallel

pairwisedistancematrix = matrix(NA,nids,nids)
sapply(1:dim(allpossiblepairs)[1], function (x) pairwisedistancematrix[allpossiblepairs[x,1],allpossiblepairs[x,2]] <<- pairwisedistancevector[x])
sapply(1:dim(allpossiblepairs)[1], function (x) pairwisedistancematrix[allpossiblepairs[x,2],allpossiblepairs[x,1]] <<- pairwisedistancevector[x])

colnames(pairwisedistancematrix) = ids
rownames(pairwisedistancematrix) = ids
###write.csv(pairwisedistancematrix,"pairwisedistancematrix_Bayesian.csv")

Bayesian_pairwisedistancematrix = pairwisedistancematrix

pairwisedistancematrix

colv_clustering = rep(rgb(0,0,0), length(ids))

print("Calculation of Bayesian matrix complete")


# --------------------------------------------------------------------------------------
# Programmer: Jason Thorpe
# Date        02/24/2011
# Language:   R (Version 2.6.0)
# Purpose:    This package has functions for application of the PEB algorithm
# Comments:
# --------------------------------------------------------------------------------------

#' Functions that apply the Parametric Empirical Bayes (PEB) algorithm
#'
#' Functions that apply the Parametric Empirical Bayes (PEB) algorithm.
#'
#' \code{zpeb(x,[n,ybar,]...)} returnes the z-score for a marker value after applying the PEB algorithm
#'
#' \code{ppeb(x,[n,ybar,]...)} returnes the p-values for a marker value after applying the PEB algorithm
#'
#' \code{qpeb(p,n,ybar,...,conf.level)} returnes the quantiles for a marker if the PEB is
#' to algorithm with a specificity \code{p}. In particular, if the normality
#' assumptions are satisified, the probability that a marker exceeds
#' \code{threshold = qpeb(p,n,ybar,...)} is \code{p}.
#'
#' @name peb
#' @family peb
#'
#' @param x A vector of marker values to be interpreted by the PEB.  \strong{Note}
#' If n and ybar are not specified, x is interpreted to be a series of observed
#' marker levels from a single individual
#'
#' @param n \code{n[i]} is the number of prior observations from the same
#' individual which preceed the i'th observation (\code{x[i]}).  Note that
#' n and ybar are required paremeters for qpeb and for zpeb and ppeb, must
#' both be present or both be absent.
#'
#' @param ybar \code{n[i]} is the mean of prior observations from the same
#' individual which preceed the i'th observation (\code{x[i]}).  Note that
#' n and ybar are required paremeters for qpeb and for zpeb and ppeb, must
#' both be present or both be absent.
#'
#' @param details If True, a data frame containing resutls from each
#' sub-calculation is returned,otherwise a vector with the z-scores
#' or p-values is returned. (Default = FALSE)
#'
#' @param na.rm If TRUE and the parameters \code{n} and \code{ybar} are not passed, then
#' mising values in X are ignored when calculationg n and ybar from x. \strong{Note}
#' that if \code{n} and \code{ybar} are not passed and all values following the first missing
#' value in x will also be missing.
#'
#' @param conf.level (optional) confidence level for the False Positve Rate (FPR)
#' for the thresholds provided \code{qpeb()}. If conf.level is specified, \code{length(p)}
#' must be 1 and the PEB parameters must be specified by the object returned by
#' \code{PEBparams()} using the \code{'Iterative'} method, as in:
#'
#' \code{params = PEBparams(x,id,method='Iterative',iterations=2);
#' qpeb(p,params,conf.level=0.95)}
#'
#' @inheritParams pebVarArgs
#'
#' @note Note that each function requires just 2 of the 4 parameters v,sigma2,tau2, and beta (=beta1)
#' \strong{Note} also that mu,sigma2,tau2,v, and beta may be substitured by a the object returned by \code{\link{PEBparams}}
#'
#' @examples
#'
#' n = 10
#' m = 200
#' params <- PEBparams(x = rnorm(m*n) + rep(rnorm(m),each=n), id=gl(m,n,m*n))
#' x <- rnorm(10) + 6
#' plot(seq(length(x)),ppeb(x,params))
#'
#' # interesting facts about the PEB...
#' b1 = .7
#'
#' # this should be uniformly distributed by definition
#' hist(ppeb(rnorm(5000)*sqrt(1-b1),mu=0,v=1,beta=b1))
#'
#' # this should be positive on the outset.
#' hist(ppeb(rnorm(5)*sqrt(1-b1) + 3,mu=0,v=1,beta=b1)) # 10/sqrt(1-b1) standard deviations above normal!!
#'
#' # but even this should be uniformly distributed in the long run
#' hist(ppeb(rnorm(5000)*sqrt(1-b1) + 10,mu=0,v=1,beta=b1)) # 10/sqrt(1-b1) standard deviations above normal!!
#'

NULL

#' @export
#' @rdname peb
#' @usage zpeb(x, ..., [n, ybar,] na.rm = FALSE, details = FALSE)
zpeb<- function(x,...,n,ybar,na.rm=FALSE,details=FALSE) {
	if(missing(n) != missing(ybar))
		stop("parameters 'n' and 'ybar' must be either both present or both absent")

	if(missing(n)){
		if(na.rm){
			x.na <- is.na(x)
			ybar <- n <- rep(NA,length(x))
			if(any(!x.na)){
				# personal means
				ybar[!x.na] <- ybarCalc(x[!x.na])
				# number of prior results
				n[!x.na] <- nCalc(x[!x.na])
			}
		}else{
			# personal means
			ybar <- ybarCalc(x)
			# number of prior results
			n <- nCalc(x)
		}
	}

	# DO THE WORK
	out <- with(pebVarArgs(...),{
			bn <- bn(n)
			vn_hat <- sigma2 + (tau2*(1-bn))
			sdn_hat <- sqrt(vn_hat)
			yn_hat <- mu*(1-bn) + ybar*bn
			z <- (x - yn_hat)/sdn_hat
			data.frame(x=x,
				  n=n,
				  ybar=ybar,
				  bn=bn,
				  z=z,
				  mu_n=yn_hat,
				  sd_n=sdn_hat,
				  z=z)
			})
	if(details)
		return(cbind(out
					,p=pnorm(out[,'z'])
					))
	else
		return(out$z)
}

#' @export
#' @usage ppeb(x, ..., [n, ybar,] na.rm = FALSE, details = FALSE)
#' @rdname peb
ppeb<-function(...,details=FALSE)  {
	# determine the probability of a sereies of marker values
	if(details)
		return(zpeb(...,details = T))
	else
		return(pnorm(zpeb(...,details = F)))
}

#-- "
#--
#-- source('r:/Urban_N/UrbanGrp/Data Analysis/JThorpe/RTools/PEB/R/peb-parameters.r')
#-- source('r:/Urban_N/UrbanGrp/Data Analysis/JThorpe/RTools/PEB/R/peb-utils.r')
#-- source('r:/Urban_N/UrbanGrp/Data Analysis/JThorpe/RTools/PEB/R/peb-functions.r')
#--
#-- params_iter <-  PEBparams(x,id,method='iterative',iterations=2)
#-- params_icc  <-  PEBparams(x,id,method='ICC')
#-- params
#--
#-- pebVarArgs(params_iter)
#-- pebVarArgs(params_icc )
#--
#-- qpeb(p=0.95, params, n=3, ybar=.5,conf.level=0.95)
#--
#-- "

#' @export
#' @usage qpeb(p, ..., n, ybar, verbose = FALSE)
#' @rdname peb
#' @importFrom CompQuadForm davies
qpeb<-function(p,...,n,ybar,conf.level,conf.method='davies', verbose = FALSE)  {

	pebArgs  <-  pebVarArgs(...)
	with(pebArgs,{
				bn_fun <<- bn(n)
				bn <- bn(n)
				bn <<- bn
				s2 <<- sigma2
				t2 <<- tau2
				v_n <<- sigma2 + (tau2*(1-bn))
				sd_n <<- sqrt(sigma2 + (tau2*(1-bn)))
				mu_n <<- mu*(1-bn) + ybar*bn
				})
	if(length(p) == 1){
		threshold  <-  qnorm(p,mean = mu_n,sd=sd_n)
		if(missing(conf.level))
			return(threshold)
		params  <-  list(...)[[1]]
		if((!inherits(params,'PEBparams')) || (params$method != 'iterative'))
			stop("If 'conf.level' specified, qpeb must be supplied with a iteragive PEB params object.  (See notes in help file)")

		UN <-  unique(n)
		conf = PEB.conf.int(p,UN,conf.level,params)
		indx  <- match(n,UN)
 		return(cbind(quantile=threshold,
					  p.lower.CI = conf[,'p.lower.CI'][indx],
					  p.upper.CI = conf[,'p.upper.CI'][indx]))
	}

	if(!missing(conf.level))
		stop("'conf.level' not supported when length(p) > 1")

	out <- matrix(NA,length(n),length(p))
	for(i in 1:length(p))
		out[,i] <- qnorm(p[i],mean = mu_n,sd=sd_n)
	dimnames(out) <- list(if(!is.null(names(n)))names(N) else seq(length(n)),
						  paste('p =',p))
	return(out)
}

# just a couple of utility functions so that the 'if(mising(n))){...}' block in zpeb() is not to ugly
ybarCalc <- function(x)
	return(if(length(x) > 1)
				c(0,cumsum(x[-length(x)])/seq(length(x)-1))
			else 0
			)

nCalc <- function(x)
	return(if(length(x) > 1)
				c(0,seq(length(x)-1))
			else 0
			)

#' Confidence intervals for personalized PEB thresholds
#'
#' Confidence intervals for personalized PEB thresholds.
#'
#' @export
#' @family peb
#'
#' @param p probability (specificity)
#'
#' @param n Number of previous observations in the individual's history. If \code{length(n) == 1}, n is expanded to \code{seq(0,n)}.
#'
#' @param conf.level Confidence level of the interval.
#'
#' @param params A PEBparams object returned by \code{PEBparams(...,method='iterative')}
#'
#' @examples
#' PEB.conf.int(p=0.95, n=6, conf.level=0.9, params=params)

PEB.conf.int  <-  function(p,n,conf.level,params){
	if((!inherits(params,'PEBparams')) || (params$method != 'iterative'))
		stop("If 'conf.level' specified, qpeb must be supplied with a iteragive PEB params object.  (See notes in help file)")

	if(length(n) == 1)
		n  <-  seq(0,n)

	with(pebVarArgs(params),
	{
		bn <<- bn <- bn(n)
		sd_n <<- sqrt(sigma2 + (tau2*(1-bn)))
	})

	threshold  <-  qnorm(p,sd=sd_n)
	p.lower.CI <-
	p.upper.CI <- numeric(0)

	with(params,
	{
		alpha <- 1 - conf.level
		cint <- c(alpha/2,1-alpha/2)

		FUN <- function(x,bn,prob){
			CompQuadForm::davies(x,
				   lambda=c(s2/(N-m),t2*(1-bn)/nstar),
				   h = c(N-m  ,nstar),# degrees of freedom

				   # sigma2 *DOES NOT* ACCOUNT FOR THE STANDARD ERROR IN THE ESTIMATE
				   # OF THE INDIVIDUALS ESTIMATED MEAN WITH N PRIOR OBSERVATIONS
				   # B/C THE PEB THRESHOLD IS SET AS THE SUM OF THE MEAN + THE SQRT
				   # OF THE WEIGHTED AVERAGE OF TWO CHI-SQUARED VARIALBLES (S2 AND T2)
				   # WHEREAS, THIS sigma2 PARAMETER IS IS ADDED TO THE WEIGHTED AVERAGE
				   # *WITHIN* THE SQUARE ROOT...
				   # sigma=(1-bn)*mu.se + bn*(sqrt(s2)/n),

				   lim=10000,
				   acc=0.0001)$Qq-prob
		}

		for(i in 1:length(bn)){
			(INT <- (t2*(1-bn[i])*qchisq(p=cint,df = nstar)/(nstar)) +
					(s2*qchisq(p=cint,df = N-m  )/(N-m  )))
			sd.upper.ci <<- sqrt(uniroot(FUN,
										 interval = INT,
										 bn=bn[i],
										 prob=1-alpha/2)$root)
			sd.lower.ci <<- sqrt(uniroot(FUN,
										 interval = INT,
										 bn=bn[i],
										 prob=alpha/2)$root)
			p.lower.CI[i] <<- pnorm(threshold[i], sd=sd.lower.ci)
			p.upper.CI[i] <<- pnorm(threshold[i], sd=sd.upper.ci)
		}
	})
	return(cbind(n,
				 p.lower.CI = p.lower.CI,
				 p.upper.CI = p.upper.CI))
}


envelop <- function(x, np=3) {
  for (k in 1:np) {
#
# Fill minima
#
    dm1 = x[1]
    dz = x[2]
    for (j in 2:(length(x)-1)) {
      dp1 = x[j+1]
      if (dm1 > dz & dp1 > dz)
        x[j] <- 0.5 * (dm1 + dp1)
      dm1 = dz
      dz = dp1
    }
#
# Smooth non-maxima
#
    dm1 = x[1]
    dz = x[2]
    for (j in 2:(length(x)-1)) {
      dp1 = x[j+1]
      if (dm1 >= dz || dp1 >= dz)
        x[j] <- 0.25*dm1 + 0.5*dz + 0.25*dp1
      dm1 = dz
      dz = dp1
    }
  }
  x
}

polevne <- function(x, np=3) {
  for (k in 1:np) {
    #
    # Degrade maxima
    #
    dm1 = x[1]
    dz = x[2]
    for (j in 2:(length(x)-1)) {
      dp1 = x[j+1]
      if (dm1 < dz & dp1 < dz)
        x[j] <- 0.5 * (dm1 + dp1)
      dm1 = dz
      dz = dp1
    }
    #
    # Smooth non-maxima
    #
    dm1 = x[1]
    dz = x[2]
    for (j in 2:(length(x)-1)) {
      dp1 = x[j+1]
      if (dm1 >= dz || dp1 >= dz)
        x[j] <- 0.25*dm1 + 0.5*dz + 0.25*dp1
      dm1 = dz
      dz = dp1
    }
  }
  x
}


### fimd8.R
### R code from Van Buuren, S. (2012).
###		Flexible Imputation of Missing Data.
###		CRC/Chapman & Hall, Boca Raton, FL.
### (c) 2012 Stef van Buuren, www.multiple-imputation.com
### Version 1, 22mar2012
### Version 2, 4nov2015 tested with mice 2.23
### Tested with Mac OS X 10.7.3, R2.14-2, mice 2.12

if (packageVersion("mice")<'2.12') stop("This code requires mice 2.12.")

library("mice")
library("lattice")

### Section 8.1 Correcting for selective drop-out

data <- pops
pred <- pops.pred
if (!is.data.frame(data)) stop("The code for section 8.1 requires access to the POPS data.")

### Section 8.1.4 A degenerate solution (TIME CONSUMING (30 MINUTES))

imp1 <- mice(data, pred=pred, maxit=20, seed=51121)

### Figure 8.2

tp82 <- plot(imp1, c("a10u","a10b","adhd"),col=mdc(5),lty=1:5)
print(tp82)

### Section 8.1.5 A better solution

##  TIME CONSUMING (30 MINUTES)
pred2 <- pred
pred2[61:86,61:86] <- 0
imp2 <- mice(data, pred=pred2, maxit=20, seed=51121)

### Figure 8.3

tp83 <- bwplot(imp2, iq+e_tot+l19_sd+b19_sd+coping+seffz~.imp,
        layout=c(2,3))
print(tp83)

### Section 8.1.6 Results

### Full responders
summary(lm(as.numeric(a10u)~1,data,na.action=na.omit))
summary(lm(as.numeric(a10b)~1,data,na.action=na.omit))
summary(lm(as.numeric(adhd)~1,data,na.action=na.omit))

### All children
summary(pool(with(imp2,lm(as.numeric(a10u)~1))))
summary(pool(with(imp2,lm(as.numeric(a10b)~1))))
summary(pool(with(imp2,lm(as.numeric(adhd)~1))))

### Section 8.2 Correcting for nonresponse

library("mice")
data <- fdgs

### Section 8.2.4 Augmenting the sample

nimp <- c(400, 600, 75, 300, 200, 400)
regcat <- c("North","City","North","East","North","City")
reg <- rep(regcat, nimp)

nimp2 <- floor(rep(nimp, each=2)/2)
nimp2[5:6] <- c(38,37)
sex <- rep(rep(c("boy","girl"),6), nimp2)

minage <- rep(c(0, 0, 10, 10, 14, 14), nimp)
maxage <- rep(c(10, 10, 14, 14, 21, 21), nimp)
set.seed(42444)
age <- runif(length(minage), minage, maxage)

id <- 600001:601975

pad <- data.frame(id, reg, age, sex, hgt=NA, wgt=NA, hgt.z=NA, wgt.z=NA)
data2 <- rbind(data, pad)

### Figure 8.4

## inspect the age by region pattern
means <- aggregate(data$hgt.z, by=list(reg=data$reg,
             age=floor(data$age)), mean, na.rm=TRUE)
tp84 <- xyplot(x~age, means, group=reg, type=c("g","l"),
             lty=1:5,col=mdc(4),
             xlim=c(-1,22), ylim=c(-0.6, 0.8),
             ylab="Height (SDS)", xlab="Age (years)",
             key=list(
               text=list(levels(means$reg)),
               lines=list(lty=1:5, col=mdc(4)),
               x=0.1, y=0.98, background="white",
               columns=3, between.columns=0),
             scales=list(x=list(tck=c(1,0)),
               y=list(tck=c(1,0))))
print(tp84)

### Section 8.2.5 Imputation model

## add interaction terms
na.opt <- options(na.action=na.pass)
int <- model.matrix(~I(age-10)*hgt.z+I(age-10)*wgt.z+age*reg, data=data2)[,-(1:9)]
options(na.opt)
data3 <- cbind(data2, int)

### define the imputation model

ini <- mice(data3, maxit=0)

meth <- ini$meth
meth["hgt"] <- ""
meth["wgt"] <- ""
meth["hgt.z"] <- "norm"
meth["wgt.z"] <- "norm"
meth["I(age - 10):hgt.z"] <- "~I(I(age-10)*hgt.z)"
meth["I(age - 10):wgt.z"] <- "~I(I(age-10)*wgt.z)"

pred <- ini$pred
pred[,c("hgt","wgt")] <- 0
pred["hgt.z", c("id","I(age - 10):hgt.z")] <- 0
pred["wgt.z", c("id","I(age - 10):wgt.z")] <- 0

vis <- ini$vis[c(3,5,4,6)]

### run the imputation model

imp <- mice(data3, meth=meth, pred=pred, vis=vis, m=10, maxit=20, seed=28107)

### Figure 8.5

cda <- complete(imp, "long", include=TRUE)
means2 <- aggregate(cda$hgt.z, by=list(reg=cda$reg, age=floor(cda$age), imp=cda$.imp), mean, na.rm=TRUE)
tp85 <- xyplot(x~age|reg,means2, group=imp, subset=(reg=="North"|reg=="City"),
              type=c("g","l"),lwd=c(4,rep(1,imp$m)),
              lty=1:5,col=c(mdc(4),rep(mdc(6),imp$m)),
              ylab="Height (SDS)", xlab="Age (years)",
              ylim=c(-0.5,0.8), xlim=c(-2,23),
              scales=list(x=list(alternating=FALSE, tck=c(1,0)),
                y=list(tck=c(1,0))),
                strip = strip.custom(bg="grey95"))
print(tp85)


H02BBJ Example Program Results

 *** IP solver

 Parameters
 ----------

 Linear constraints......         3        First integer solution..       OFF
 Variables...............         2        Max depth of the tree...         4

 Feasibility tolerance...  1.05E-08        Print level.............        10
 Infinite bound size.....  1.00E+20        EPS (machine precision).  1.11E-16

 Integer feasibility tol.  1.00E-05        Iteration limit.........        50
 Max number of nodes.....      NONE

 ** Workspace provided with MAXDPT =     4: LRWORK =    84  LIWORK =   137
 ** Workspace required with MAXDPT =     4: LRWORK =    84  LIWORK =   137

   *** Optimum LP solution ***   -17.50000

 Varbl State     Value     Lower Bound   Upper Bound    Lagr Mult   Residual

 V  1    FR    3.92857       0.00000         None        0.000       3.929
 V  2    FR    1.42857       0.00000         None        0.000       1.429

 L Con State     Value     Lower Bound   Upper Bound    Lagr Mult   Residual

 L  1    UL    15.0000         None        15.0000      -1.000       0.000
 L  2    UL    5.00000         None        5.00000     -0.5000     -8.8818E-16
 L  3    FR    14.6429       5.00000         None        0.000       9.643

   *** Start of tree search ***

  Node  Parent   Obj       Varbl  Value      Lower      Upper      Value   Depth
   No    Node   Value      Chosen Before     Bound      Bound      After
    2     1   No Feas Soln    1    3.93       4.00       None       4.00       1
    3     1    -16.2          1    3.93       0.00       3.00       3.00       1
    4     3    -15.5          2    1.80       2.00       None       2.00       2
    5     3    -13.0          2    1.80       0.00       1.00       1.00       2
      *** Integer solution ***

  Node  Parent   Obj       Varbl  Value      Lower      Upper      Value   Depth
   No    Node   Value      Chosen Before     Bound      Bound      After
    6     4   No Feas Soln    1    2.50       3.00       3.00       3.00       3
    7     4    -14.8          1    2.50       0.00       2.00       2.00       3
    8     7    -12.0     CO   2    2.20       3.00       None       3.00       4
    9     7    -14.0          2    2.20       2.00       2.00       2.00       4
      *** Integer solution ***

     *** End of tree search ***

 Total of     9 nodes investigated.

 Exit IP solver - Optimum IP solution found.

 Final IP objective value =   -14.00000

 Varbl State     Value     Lower Bound   Upper Bound    Lagr Mult   Residual

 V  1    UL    2.00000       0.00000       2.00000      -3.000       0.000
 V  2    EQ    2.00000       2.00000       2.00000      -4.000       0.000

 L Con State     Value     Lower Bound   Upper Bound    Lagr Mult   Residual

 L  1    FR    14.0000         None        15.0000       0.000       1.000
 L  2    FR    0.00000         None        5.00000       0.000       5.000
 L  3    FR    10.0000       5.00000         None        0.000       5.000


rmnl <- function(Data,Z,R=1000,keep=1,ss.b=.4){

  ## load libraries
  library(bayesm)

################################################################################

  ## internal functions

  rmultiregG = function (Y, X, Bbar, A, nu, V) {
    n = nrow(Y)
    m = ncol(Y)
    k = ncol(X)
    RA = chol(A)
    W = rbind(X, RA)
    Z = rbind(Y, RA %*% Bbar)
    IR = backsolve(chol(crossprod(W)), diag(k))
    Btilde = crossprod(t(IR)) %*% crossprod(W, Z)
    S = crossprod(Z - W %*% Btilde)
    rwout = rwishart(nu + n, chol2inv(chol(V + S)))
    B = Btilde + IR %*% matrix(rnorm(m * k), ncol = m) %*% t(rwout$CI)
    return(list(B = B, Sigma = rwout$IW))
  }
################################################################################

  ## Unpack Data
  X = Data[[1]]$X[,,1] # Design differes for most respondents respondents: array(K,A,J)
  Y = Data[[1]]$Y  # J*K matrix of ranked respones

################################################################################

  ## define constants
  J = length(Y) #number of choice taskts
  K1 = nrow(X) #number of alternatives per choice set
  A = ncol(X) #total number of attributes
  N = length(Data) #number of respondents
  Sig1 = IMat1 = diag(K1)
  sigi1=chol2inv(chol(Sig1))

  #ss.b = .07
  ssB = double(N)
  ssB = ssB + ss.b

################################################################################

  ## allocate storage space
  betadraw = array(double((R/keep)*N*A),dim=c(R/keep,A,N))
  bbardraw = matrix(double((R/keep)*A),ncol=A)
  vbetakeep = matrix(double(A*A*R/keep),ncol=A*A)
  nacceptB = matrix(double(N),ncol=1)
  lltemp = matrix(double(N),ncol=1)
  llkeep = matrix(double(N*R/keep),ncol=N)

################################################################################
  ## Starting values for beta

  ## Uniform [-.01,.01]
  betaM = betaMnew = matrix(runif(N*A,-.01,.01),ncol=A)

  ## Initialize variables
  beta = c(double(A))
  bbar = c(double(A))
  ztemp = c(1:K1)
  LB = -50
  UB = 50
  iota = matrix(1,ncol=1,nrow=N)

  ## Prior values
  bbar.p = c(double(A)) + 2
  nu = A+3
  A.b = 0.01
  V.b = Sig.b = diag(A)*nu #*.1 pp 74 in text
  rooti.b = backsolve(chol(V.b*5),diag(A))

  ## mcmc variables starting values
  llold.b = 0
  pold.b = lndMvn(beta,bbar,rooti.b)

################################################################################

  ## Estimation Algorithm

  cat(" ", fill = TRUE)
  cat("Starting Estimation Routine for Hierarchical MNL:", fill = TRUE)
  itime = proc.time()[3]
  cat("MCMC Iteration (est time to end - min) ", fill = TRUE)
  flush.console()
  for(rep in 1:R){

  ## Unit Loop
    for(nhh in 1:N){

################################################################################

      ## unpack data for household nhh
      X1.nhh = Data[[nhh]]$X
      Y1 = Data[[nhh]]$Y
      betaold = betaM[nhh,]

      llold = lltemp[nhh,]
      ss.b = ssB[nhh]

################################################################################
      ## Step 1. Draw beta|else - via M-H (evaluate likelihood using GHK)

      if(rep==1){
      ## Compute current value of likelihood & prior
      llold = 0

      ## Compute for first set of choices (4 alt)
      for(ntask in 1:J){
        Y = Data[[nhh]]$Y[ntask]
        X = X1.nhh[,,ntask]
        mu = X%*%betaold
        ltemp = exp(mu)/sum(exp(mu))
        llold = llold + log(ltemp[Y])
      }

      }

      pold.b = lndMvn(betaold,as.vector(bbar),rooti.b)

      ## draw betanew
      betanew = betaold + rnorm(length(betaold),0,ss.b)

      ## Compute likelihood & prior for new values of beta
      llnew = 0
      for(ntask in 1:J){
        Y = Data[[nhh]]$Y[ntask]
        X = X1.nhh[,,ntask]
        mu = X%*%betanew
        ltemp = exp(mu)/sum(exp(mu))
        llnew = llnew + log(ltemp[Y])
      }

      pnew.b = lndMvn(betanew,as.vector(bbar),rooti.b)

      ## metropolis step
      ldiff = llnew + pnew.b - llold - pold.b

      alpha = min(1,exp(ldiff))
      if (alpha<1) {unif=runif(1)} else {unif=0}

      ## update variables and save draws if accepted
      if (unif <= alpha)
      {
        llold = llnew
        betaold = betanew
        nacceptB[nhh,] = nacceptB[nhh,] + 1 } else { }

      betaM[nhh,] = betaold
      betaMnew[nhh,] = betanew

      lltemp[nhh,] = llold

      } ## end unit loop

################################################################################
      ## Step 4. Draw hierarchical priors
      if(N>1){

      ## Distribution of heterogeneity for beta
      rmout = rmultiregG(betaM,iota,bbar.p,A.b,nu,V.b)
      bbar = rmout$B
      Sig.b = rmout$Sigma
      rooti.b = backsolve(chol(rmout$Sigma),diag(A))

      } ## End N>1 condition

################################################################################
  ## Print Time & Parameter Estimates
  if (rep%%100 == 0) {
    ctime = proc.time()[3]
    timetoend = ((ctime-itime)/rep) * (R-rep)
    cat(" ", rep, " ", round(timetoend/60,1), " ", round(bbar,2),
        " ", round(mean(nacceptB/10),2)," ",
        sum(lltemp), " ", fill = TRUE)
    flush.console()
  }

################################################################################
  ## Save Draws
  if (rep%%keep == 0){
    betadraw[rep/keep,,] = t(betaM)
    bbardraw[rep/keep,] = bbar
    vbetakeep[rep/keep,] = as.vector(rmout$Sigma)
    llkeep[rep/keep,] = as.vector(lltemp)
  }

  ## Dynamically update step size

  if(rep%%10 == 0){
    if(rep<R/2){
      for(ii in 1:N){
        if(nacceptB[ii]/10 < .2) ssB[ii] = ssB[ii]*.9
        if(nacceptB[ii]/10 > .4) ssB[ii] = ssB[ii]*1.1
      }
    }
    nacceptB = nacceptB*0
  }

  } ## end MCMC

  return(list(betadraw=betadraw, bbardraw=bbardraw, vbetadraw=vbetakeep,
              llkeep = llkeep))
}


#' Computes estimates and ancillary information for diagonal classifiers
#'
#' Computes the maximum likelihood estimators (MLEs) for each class under the
#' assumption of multivariate normality for each class. Also, computes ancillary
#' information necessary for classifier summary, such as sample size, the number
#' of features, etc.
#'
#' This function computes the common estimates and ancillary information used in
#' all of the diagonal classifiers in the `sparsediscrim` package.
#'
#' The matrix of training observations are given in `x`. The rows of `x`
#' contain the sample observations, and the columns contain the features for each
#' training observation.
#'
#' The vector of class labels given in `y` are coerced to a `factor`.
#' The length of `y` should match the number of rows in `x`.
#'
#' An error is thrown if a given class has less than 2 observations because the
#' variance for each feature within a class cannot be estimated with less than 2
#' observations. If other data have zero variances, these will be removed with
#' a warning.
#'
#' The vector, `prior`, contains the _a priori_ class membership for
#' each class. If `prior` is NULL (default), the class membership
#' probabilities are estimated as the sample proportion of observations belonging
#' to each class. Otherwise, `prior` should be a vector with the same length
#' as the number of classes in `y`. The `prior` probabilities should be
#' nonnegative and sum to one.
#'
#' @inheritParams lda_diag
#' @param pool logical value. If TRUE, calculates the pooled sample variances
#' for each class.
#' @param est_mean the estimator for the class means. By default, we use the
#' maximum likelihood estimator (MLE). To improve the estimation, we provide the
#' option to use a shrunken mean estimator proposed by Tong et al. (2012).
#' @return named list with estimators for each class and necessary ancillary
#' information
#'
#' @references Tong, T., Chen, L., and Zhao, H. (2012), "Improved Mean
#' Estimation and Its Application to Diagonal Discriminant Analysis,"
#' Bioinformatics, 28, 4, 531-537.
#' \url{http://bioinformatics.oxfordjournals.org/content/28/4/531.long}
diag_estimates <- function(x, y, prior = NULL, pool = FALSE,
                           est_mean = c("mle", "tong")) {
  obj <- list()
  obj$labels <- y
  obj$N <- length(y)
  obj$p <- ncol(x)
  obj$groups <- levels(y)
  obj$num_groups <- nlevels(y)

  est_mean <- match.arg(est_mean)

  # Error Checking
  if (!is.null(prior)) {
    if (length(prior) != obj$num_groups) {
      rlang::abort("The number of 'prior' probabilities must match the number of classes in 'y'.")
    }
    if (any(prior <= 0)) {
      rlang::abort("The 'prior' probabilities must be nonnegative.")
    }
    if (sum(prior) != 1) {
      rlang::abort("The 'prior' probabilities must sum to one.")
    }
  }
  if (any(table(y) < 2)) {
    rlang::abort("There must be at least 2 observations in each class.")
  }

  # By default, we estimate the 'a priori' probabilities of class membership with
  # the MLEs (the sample proportions).
  if (is.null(prior)) {
    prior <- as.vector(table(y) / length(y))
  }

  # For each class, we calculate the MLEs (or specified alternative estimators)
  # for each parameter used in the DLDA classifier. The 'est' list contains the
  # estimators for each class.
  obj$est <- tapply(seq_along(y), y, function(i) {
    stats <- list()
    stats$n <- length(i)
    if (est_mean == "mle") {
      stats$xbar <- colMeans(x[i, , drop = FALSE])
    } else if (est_mean == "tong") {
      stats$xbar <- tong_mean_shrinkage(x[i, , drop = FALSE])
    }
    stats$var <- with(stats, (n - 1) / n * apply(x[i, , drop = FALSE], 2, var))
    stats
  })

  # Check to see if any predictors had zero variances
  obj$est <- check_for_zero_vars(obj$est)

  # Calculates the pooled variance across all classes.
  if (pool) {
    obj$var_pool <- Reduce('+', lapply(obj$est, function(x) x$n * x$var)) / obj$N
  }

  # Add each element in 'prior' to the corresponding obj$est$prior
  for(k in seq_len(obj$num_groups)) {
    obj$est[[k]]$prior <- prior[k]
  }
  obj
}

check_for_zero_vars <- function(x, warn = TRUE) {
  var_est <- lapply(x, function(x) x$var == 0)
  is_zv <- do.call("rbind", var_est)
  any_zv <- apply(is_zv, 2, any)
  if (all(any_zv)) {
    rlang::abort("All predictors have zero variance.")
  }
  if (any(any_zv)) {
    if (warn) {
      nms <- paste0(names(any_zv)[any_zv], collapse = ", ")
      nms <- paste("The following predictors had zero variance (possibly within ",
                   "a class) and were removed from the analysis:", nms)
      rlang::warn(nms)
    }
    x <- lapply(x, reduce_elem, retain = names(any_zv)[!any_zv], "xbar")
    x <- lapply(x, reduce_elem, retain = names(any_zv)[!any_zv], "var")
  }
  x
}

reduce_elem <- function(x, retain, col) {
  x[[col]] <- x[[col]][retain]
  x
}

#' Computes estimates and ancillary information for regularized discriminant
#' classifiers
#'
#' Computes the maximum likelihood estimators (MLEs) for each class under the
#' assumption of multivariate normality for each class. Also, computes ancillary
#' information necessary for classifier summary, such as sample size, the number
#' of features, etc.
#'
#' This function computes the common estimates and ancillary information used in
#' all of the regularized discriminant classifiers in the `sparsediscrim`
#' package.
#'
#' The matrix of training observations are given in `x`. The rows of `x`
#' contain the sample observations, and the columns contain the features for each
#' training observation.
#'
#' The vector of class labels given in `y` are coerced to a `factor`.
#' The length of `y` should match the number of rows in `x`.
#'
#' An error is thrown if a given class has less than 2 observations because the
#' variance for each feature within a class cannot be estimated with less than 2
#' observations.
#'
#' The vector, `prior`, contains the _a priori_ class membership for
#' each class. If `prior` is NULL (default), the class membership
#' probabilities are estimated as the sample proportion of observations belonging
#' to each class. Otherwise, `prior` should be a vector with the same length
#' as the number of classes in `y`. The `prior` probabilities should be
#' nonnegative and sum to one.
#'
#' @inheritParams lda_diag
#' @param y vector of class labels for each training observation
#' @param cov logical. Should the sample covariance matrices be computed?
#' (Default: yes)
#' @param prior vector with prior probabilities for each class. If NULL
#' (default), then the sample proportions are used. See details.
#' @return named list with estimators for each class and necessary ancillary
#' information
regdiscrim_estimates <- function(x, y, cov = TRUE, prior = NULL) {
  obj <- list()
  obj$labels <- y
  obj$N <- length(y)
  obj$p <- ncol(x)
  obj$groups <- levels(y)
  obj$num_groups <- nlevels(y)

  # Error Checking
  if (!is.null(prior)) {
    if (length(prior) != obj$num_groups) {
      rlang::abort("The number of 'prior' probabilities must match the number of
            classes in 'y'.")
    }
    if (any(prior <= 0)) {
      rlang::abort("The 'prior' probabilities must be nonnegative.")
    }
    if (sum(prior) != 1) {
      rlang::abort("The 'prior' probabilities must sum to one.")
    }
  }
  if (any(table(y) < 2)) {
    rlang::abort("There must be at least 2 observations in each class.")
  }

  # By default, we estimate the 'a priori' probabilities of class membership with
  # the MLEs (the sample proportions).
  if (is.null(prior)) {
    prior <- as.vector(table(y) / length(y))
  }

  # For each class, we calculate the MLEs for each parameter used in the
  # 'regdiscrim' classifiers. The 'est' list contains the estimators for each
  # class.
  obj$est <- tapply(seq_along(y), y, function(i) {
    stats <- list()
    stats$n <- length(i)
    stats$xbar <- as.vector(colMeans(x[i, , drop = FALSE]))
    if (cov) {
      stats$cov <- with(stats, cov_mle(x[i, , drop = FALSE]))
    }
    stats
  })

  # Calculates the pooled sample covariance matrix.
  if (cov) {
    obj$cov_pool <- Reduce('+', lapply(obj$est, function(x) x$n * x$cov)) / obj$N
  }

  # Add each element in 'prior' to the corresponding obj$est$prior
  for(k in seq_len(obj$num_groups)) {
    obj$est[[k]]$prior <- prior[k]
  }
  obj
}


\name{extreme_deconvolution}
\alias{extreme_deconvolution}
\title{Density estimation using Gaussian mixtures in the presence of noisy, heterogeneous and incomplete data}
\description{We present a general algorithm to infer a d-dimensional distribution function given a set of heterogeneous, noisy observations or samples. This algorithm reconstructs the error-deconvolved or 'underlying' distribution function common to all samples, even when the individual samples have unique error and missing-data properties. The underlying distribution is modeled as a mixture of Gaussians, which is completely general. Model parameters are chosen to optimize a justified, scalar objective function: the logarithm of the probability of the data under the error-convolved model, where the error convolution is different for each data point. Optimization is performed by an Expectation Maximization (EM) algorithm, extended by a regularization technique and 'split-and-merge' procedure. These extensions mitigate problems with singularities and local maxima, which are often encountered when using the EM algorithm to estimate Gaussian density mixtures.}

\usage{
extreme_deconvolution(ydata,ycovar,
                      xamp,xmean,xcovar,
                      projection=NULL,
                      weight=NULL,
                      fixamp=NULL,fixmean=NULL,fixcovar=NULL,
                      tol=1.e-6,maxiter=1e9,w=0,logfile=NULL,
                      splitnmerge=0,maxsnm=FALSE,likeonly=FALSE,
                      logweight=FALSE)
}

\arguments{
  \item{ydata}{[ndata,dy] matrix of observed quantities}
  \item{ycovar}{[ndata,dy] / [ndata,dy,dy] / [dy,dy,ndata] matrix, list or 3D array
    of observational error covariances
    (if [ndata,dy] then the error correlations are assumed to vanish)}
  \item{xamp}{[ngauss] array of initial amplitudes (*not* [1,ngauss])}
  \item{xmean}{[ngauss,dx] matrix of initial means}
  \item{xcovar}{[ngauss,dx,dx] list of matrices of initial covariances}
  \item{projection}{[ndata,dy,dx] list of projection matrices}
  \item{weight}{[ndata] array of weights to be applied to the data points}
  \item{logweight}{(bool, default=False) if True, weight is actually
                   log(weight)}
  \item{fixamp}{(default=None) None, True/False, or list of bools}
  \item{fixmean}{(default=None) None, True/False, or list of bools}
  \item{fixcovar}{(default=None) None, True/False, or list of bools}
  \item{tol}{(double, default=1.e-6) tolerance for convergence}
  \item{maxiter}{(long, default= 10**9) maximum number of iterations to perform}
  \item{w}{(double, default=0.) covariance regularization parameter
            (of the conjugate prior)}
  \item{logfile}{basename for several logfiles (_c.log has output from
                 the c-routine; _loglike.log has the log likelihood path of
                 all the accepted routes, i.e. only parts which increase
                 the likelihood are included, during splitnmerge)}
  \item{splitnmerge}{(int, default=0) depth to go down the splitnmerge path}
  \item{maxsnm}{(Bool, default=False) use the maximum number of split 'n'
                 merge steps, K*(K-1)*(K-2)/2}
  \item{likeonly}{(Bool, default=False) only compute the total log
                   likelihood of the data}
}

\value{
  \item{avgloglikedata}{avgloglikedata after convergence}
  \item{xamp}{updated xamp}
  \item{xmean}{updated xmean}
  \item{xcovar}{updated xcovar}
}

\details{
  ...
}

\author{Jo Bovy, David W. Hogg, & Sam T. Roweis}

\references{
Inferring complete distribution functions from
noisy, heterogeneous and incomplete observations Jo Bovy, David
W. Hogg, & Sam T. Roweis, Submitted to AOAS (2009) [arXiv/0905.2979]
}

\examples{
library("ExtremeDeconvolution")
?extreme_deconvolution
ydata <- c(2.62434536,0.38824359,0.47182825,-0.07296862,1.86540763,-1.30153870,2.74481176,0.23879310,1.31903910,0.75062962,2.46210794,-1.06014071,0.67758280,0.61594565,2.13376944,-0.09989127,0.82757179,0.12214158,1.04221375,1.58281521,-0.10061918,2.14472371,1.90159072,1.50249434,1.90085595,0.31627214,0.87710977,0.06423057,0.73211192,1.53035547,0.30833925,0.60324647,0.31282730,0.15479436,0.32875387,0.98733540,-0.11731035,1.23441570,2.65980218,1.74204416,0.80816445,0.11237104,0.25284171,2.69245460,1.05080775,0.36300435,1.19091548,3.10025514,1.12015895,1.61720311,1.30017032,0.64775015,-0.14251820,0.65065728,0.79110577,1.58662319,1.83898341,1.93110208,1.28558733,1.88514116,0.24560206,2.25286816,1.51292982,0.70190717,1.48851815,0.92442829,2.13162939,2.51981682,3.18557541,-0.39649633,-0.44411380,0.49553414,1.16003707,1.87616892,1.31563495,-1.02220122,0.69379599,1.82797464,1.23009474,1.76201118,0.77767186,0.79924193,1.18656139,1.41005165,1.19829972,1.11900865,0.32933771,1.37756379,1.12182127,2.12948391,2.19891788,1.18515642,0.62471505,0.36126959,1.42349435,1.07734007,0.65614632,1.04359686,0.37999916,1.69803203,0.55287144,2.22450770,1.40349164,1.59357852,-0.09491185,1.16938243,1.74055645,0.04629940,0.73378149,1.03261455,-0.37311732,1.31515939,1.84616065,0.14048406,1.35054598,-0.31228341,0.96130449,-0.61577236,2.12141771,1.40890054,0.97538304,0.22483838,2.27375593,2.96710175,-0.85798186,2.23616403,2.62765075,1.33801170,-0.19926803,1.86334532,0.81907970,0.39607937,-0.23005814,1.55053750,1.79280687,0.37646927,1.52057634,-0.14434139,1.80186103,1.04656730,0.81343023,0.89825413,1.86888616,1.75041164,1.52946532,1.13770121,1.07782113,1.61838026,1.23249456,1.68255141,0.68988323,-1.43483776,2.03882460,3.18697965,1.44136444,0.89984477,0.86355526,0.88094581,1.01740941,-0.12201873,0.48290554,0.00297317,1.24879916,0.70335885,1.49521132,0.82529684,1.98633519,1.21353390,3.19069973,-0.89636092,0.35308331,1.90148689,3.52832571,0.75136522,1.04366899,0.77368576,2.33145711,0.71269214,1.68006984,0.68019840,-0.27255875,1.31354772,1.50318481,2.29322588,0.88955297,0.38263794,1.56276110,1.24073709,1.28066508,0.92688730,2.16033857,1.36949272,2.90465871,2.11105670,1.65904980,-0.62743834,1.60231928,1.42028220,1.81095167,2.04444209)
ydata <- matrix(ydata, length(ydata), 1)
N <- dim(ydata)[1]
ycovar <- ydata * 0 + 0.01
xamp <- c(0.5, 0.5)
xmean <- matrix(c(0.86447943, 0.67078879, 0.322681, 0.45087394), 2, 2)
xcovar <- list(matrix(c(0.03821028, 0.04014796, 0.04108113, 0.03173839), 2, 2),
               matrix(c(0.06219194,0.09738021, 0.04302473, 0.06778009), 2, 2))
projection <- list()
for (i in 1:N) projection[[i]] = matrix(c(i\%\%2,(i+1)\%\%2), 1, 2)
res <- extreme_deconvolution(ydata, ycovar, xamp, xmean, xcovar, projection=projection, logfile="ExDeconDemo")
print(res)
stopifnot((res$avgloglikedata - (-1.3114744655258121)) **2. < 10.**-8)
stopifnot((res$xmean[1,1]-2.30368235)**2. < 10.**-5)
stopifnot((res$xmean[1,2]-1.70701517)**2. < 10.**-5)
stopifnot((res$xmean[2,1]-1.08009397)**2. < 10.**-5)
stopifnot((res$xmean[2,2]-0.8888667)**2. < 10.**-5)
stopifnot((res$xcovar[[1]][1,1]-0.445645987259)**2. < 10.**-5.)
stopifnot((res$xamp[1]-0.11968415)**2. < 10.**-5)
stopifnot((res$xamp[2]-0.880315852981)**2. < 10.**-5)
}


library(openxlsx)
library(matlib)
library(Matrix)

this.dir<- dirname(parent.frame(2)$ofile)
parent.dir <- strsplit(this.dir,'scripts')[[1]]
setwd(parent.dir)

load_file <- 'efa/missing-loadings-f3-f12-sig-2.xlsx' #only statistically signficant loadings included
#To compute LSI using all loadings set load_file to 'efa/missing-loadings-f3-f12.csv'

f12_load_mat <- as.matrix(read.xlsx(load_file, sheet = 1, colNames = FALSE))
f11_load_mat <- as.matrix(read.xlsx(load_file, sheet = 2, colNames = FALSE))
f10_load_mat <- as.matrix(read.xlsx(load_file, sheet = 3, colNames = FALSE))
f9_load_mat <- as.matrix(read.xlsx(load_file, sheet = 4, colNames = FALSE))
f8_load_mat <- as.matrix(read.xlsx(load_file, sheet = 5, colNames = FALSE))
f7_load_mat <- as.matrix(read.xlsx(load_file, sheet = 6, colNames = FALSE))
f6_load_mat <- as.matrix(read.xlsx(load_file, sheet = 7, colNames = FALSE))
f5_load_mat <- as.matrix(read.xlsx(load_file, sheet = 8, colNames = FALSE))
f4_load_mat <- as.matrix(read.xlsx(load_file, sheet = 9, colNames = FALSE))
f3_load_mat <- as.matrix(read.xlsx(load_file, sheet = 10, colNames = FALSE))

eps <- 1.0e-8
w_fun <- function(x) {(x^2 + eps)^(10*x^2)}
"%^%" <- function(x, n) with(eigen(x), vectors %*% (values^n * t(vectors)))
lsi <- function(load_mat) {
  nfac <- ncol(load_mat)
  c_mat <- diag(t(load_mat) %*% load_mat)
  c_mat <- diag(c_mat)
  h_mat <- diag(load_mat %*% inv(c_mat) %*% t(load_mat))
  h_mat <- diag(h_mat)
  b_mat <- (h_mat %^% (-0.5)) %*% load_mat %*% (c_mat %^% (-0.5))
  ww <- sum(sapply( b_mat, w_fun,simplify="array"))*(1/64.0)*(1.0/nfac)
  ee <- ((1.0/nfac) + eps)^(10.0/nfac)
  ind <- (ww-ee)/(1.0-ee)
  return(ind)
}

print(paste0(ncol(f12_load_mat)," factors: LSI =", lsi(f12_load_mat)))
print(paste0(ncol(f11_load_mat)," factors: LSI =", lsi(f11_load_mat)))
print(paste0(ncol(f10_load_mat)," factors: LSI =", lsi(f10_load_mat)))
print(paste0(ncol(f9_load_mat)," factors: LSI =", lsi(f9_load_mat)))
print(paste0(ncol(f8_load_mat)," factors: LSI =", lsi(f8_load_mat)))
print(paste0(ncol(f7_load_mat)," factors: LSI =", lsi(f7_load_mat)))
print(paste0(ncol(f6_load_mat)," factors: LSI =", lsi(f6_load_mat)))
print(paste0(ncol(f5_load_mat)," factors: LSI =", lsi(f5_load_mat)))
print(paste0(ncol(f4_load_mat)," factors: LSI =", lsi(f4_load_mat)))
print(paste0(ncol(f3_load_mat)," factors: LSI =", lsi(f3_load_mat)))


model_conductance <- function (vonKarman = 0.42,
         heightWeatherMeasurements = 2.0,
         zm = 0.13,
         zh = 0.013,
         d = 0.67,
         plantHeight = 0.0,
         wind = 124000.0){
    #'- Name: Conductance -Version: 1.0, -Time step: 1
    #'- Description:
    #'            * Title: Conductance Model
    #'            * Author: Pierre Martre
    #'            * Reference: Modelling energy balance in the wheat crop model SiriusQuality2:
    #'            Evapotranspiration and canopy and soil temperature calculations
    #'
    #'            * Institution: INRA/LEPSE Montpellier
    #'            * Abstract: The boundary layer conductance is expressed as the wind speed profile above the
    #'            canopy and the canopy structure. The approach does not take into account buoyancy
    #'            effects.
    #'
    #'- inputs:
    #'            * name: vonKarman
    #'                          ** description : von Karman constant
    #'                          ** datatype : DOUBLE
    #'                          ** min : 0
    #'                          ** max : 1
    #'                          ** default : 0.42
    #'                          ** unit : dimensionless
    #'                          ** uri : http://www1.clermont.inra.fr/siriusquality/?page_id=547
    #'                          ** inputtype : parameter
    #'                          ** parametercategory : constant
    #'            * name: heightWeatherMeasurements
    #'                          ** description : reference height of wind and humidity measurements
    #'                          ** parametercategory : soil
    #'                          ** datatype : DOUBLE
    #'                          ** min : 0
    #'                          ** max : 10
    #'                          ** default : 2
    #'                          ** unit : m
    #'                          ** uri : http://www1.clermont.inra.fr/siriusquality/?page_id=547
    #'                          ** inputtype : parameter
    #'            * name: zm
    #'                          ** description : roughness length governing momentum transfer, FAO
    #'                          ** parametercategory : constant
    #'                          ** inputtype : parameter
    #'                          ** datatype : DOUBLE
    #'                          ** min : 0
    #'                          ** max : 1
    #'                          ** default : 0.13
    #'                          ** unit : m
    #'                          ** uri : http://www1.clermont.inra.fr/siriusquality/?page_id=547
    #'            * name: zh
    #'                          ** description : roughness length governing transfer of heat and vapour, FAO
    #'                          ** parametercategory : constant
    #'                          ** inputtype : parameter
    #'                          ** datatype : DOUBLE
    #'                          ** min : 0
    #'                          ** max : 1
    #'                          ** default : 0.013
    #'                          ** unit : m
    #'                          ** uri : http://www1.clermont.inra.fr/siriusquality/?page_id=547
    #'            * name: d
    #'                          ** description : corresponding to 2/3. This is multiplied to the crop heigth for calculating the zero plane displacement height, FAO
    #'                          ** inputtype : parameter
    #'                          ** parametercategory : constant
    #'                          ** datatype : DOUBLE
    #'                          ** default : 0.67
    #'                          ** min : 0
    #'                          ** max : 1
    #'                          ** unit : dimensionless
    #'                          ** uri : http://www1.clermont.inra.fr/siriusquality/?page_id=547rl
    #'            * name: plantHeight
    #'                          ** description : plant Height
    #'                          ** datatype : DOUBLE
    #'                          ** default : 0
    #'                          ** min : 0
    #'                          ** max : 1000
    #'                          ** unit : mm
    #'                          ** uri : http://www1.clermont.inra.fr/siriusquality/?page_id=547
    #'                          ** inputtype : variable
    #'                          ** variablecategory : auxiliary
    #'            * name: wind
    #'                          ** description : wind
    #'                          ** variablecategory : auxiliary
    #'                          ** datatype : DOUBLE
    #'                          ** default : 124000
    #'                          ** min : 0
    #'                          ** max : 1000000
    #'                          ** unit : m/d
    #'                          ** uri : http://www1.clermont.inra.fr/siriusquality/?page_id=547
    #'                          ** inputtype : variable
    #'- outputs:
    #'            * name: conductance
    #'                          ** description : the boundary layer conductance
    #'                          ** variablecategory : state
    #'                          ** datatype : DOUBLE
    #'                          ** min : 0
    #'                          ** max : 10000
    #'                          ** unit : m/d
    #'                          ** uri : http://www1.clermont.inra.fr/siriusquality/?page_id=547
    h <- max(10.0, plantHeight) / 100.0
    conductance <- wind * vonKarman ^ 2 / (log((heightWeatherMeasurements - (d * h)) / (zm * h)) * log((heightWeatherMeasurements - (d * h)) / (zh * h)))
    return (list('conductance' = conductance))
}