#autor:      Joao Sollari Lopes
#local:      INE, Lisboa
#Rversion:   3.2.3
#criado:     08.02.2019
#modificado: 08.02.2019

## 1. FUNCTIONS
{
# Key:
# a - auxiliar
# d - dataframe
# f - file name
# i - iterator
# l - list
# n - vector size
# s - selected samples
# t - title
# v - vector
# w - weights

# Perform Partition around medoids clustering (wide format) [allow for specific number of partitions]
# dst1 - distance matrix of variables to be analysed
# kbest - specify number of partitions [if is.null(kbest) then calculate best number of partitions]
# col1 - colors for points
# border - add border to columns
# f1 - output filename
# main - title for plots
plot_pam_v2 = function(dst1,kbest=NULL,col1=NULL,border=1,f1,main=NULL){
    library("cluster")
    kmax <- dim(as.matrix(dst1))[1]-1
    asw <- rep(0,kmax)
    for(k in 2:kmax)
        asw[k] = pam(dst1,k)$silinfo$avg.width
    if(is.null(kbest)){
        kbest = which.max(asw)
    }
    pres1 = pam(dst1,kbest)
    d1 = cbind(1:kmax,asw)
    colnames(d1) = c("K","Average silhouette width")
    d2 = silhouette(pres1)
    f2 = paste(f1,"_1.csv",sep="")
    f3 = paste(f1,"_2.csv",sep="")
    f4 = paste(f1,"_3.csv",sep="")
    write.table(d1,f2,quote=TRUE,sep=",",eol="\n",na="NA",dec=".",
      col.names=TRUE,row.names=FALSE)
    write.table(pres1$clusinfo,f3,quote=TRUE,sep=",",eol="\n",na="NA",dec=".",
      col.names=TRUE,row.names=FALSE)
    write.table(d2,f4,quote=TRUE,sep=",",eol="\n",na="NA",dec=".",col.names=NA)
    for(i1 in c("eps","tiff")){
        if(i1 == "eps"){
            postscript(file=paste(f1,".eps",sep=""),width=7.0,height=3.5,
              colormodel="rgb",horizontal=FALSE,onefile=FALSE,paper="special",
              family="Helvetica")
        }else{
            tiff(file=paste(f1,".tif",sep=""),units="in",width=7.0,height=3.5,
              res=300,compression="lzw",family="sans")
        }
        oldpar = par(mfcol=c(1,2),mar=c(2.2,2.2,1.2,1.2),mgp=c(1.0,0.2,0.0),
          tcl=-0.2,cex=0.8,cex.lab=0.8,cex.axis=0.8,cex.main=0.9)
        plot(d1,type="h",main=main)
        points(kbest,asw[kbest],col="red",type= "h")
        plot(d2,main=main,sub="",xlab="Silhouette width",do.n.k=FALSE,
          border=border,do.clus.stat=FALSE,mgp=c(1.0,0.2,0.0),
          col=col1[rownames(d2)])
        par(oldpar)
        invisible(dev.off())
    }
}

# min-max transformation
# y - vector
# miny - minimum value to scale y
# maxy - maximum value to scale y
minmax = function(y,miny=NULL,maxy=NULL){
    if(is.null(miny)){miny = min(y)}
    if(is.null(maxy)){maxy = max(y)}
    return((y - miny)/(maxy - miny))
}

# Inverse min-max transformation
# y - vector
# miny - minimum value to scale y
# maxy - maximum value to scale y
minmax_inv = function(y,miny,maxy){
    return(y*(maxy - miny) + miny)
}

# Construct network (wide format) [allow for very many nodes]
# dst1 - distance matrix of variables to be analysed
# col1 - colors for points
# vmode - network layout
# edge_thr - threshold for edges to plot
# edge_lwd - scale for edges
# label_cex - size of node labels
# f1 - output filename
# main - title for plots
plot_sna_v2 = function(dst1,col1=NULL,vmode="fruchtermanreingold",edge_thr=0.80,
  edge_lwd=10,label_cex=0.8,f1,main=NULL){
    library("sna")
    m1 = as.matrix(dst1)
    m1 = -m1                            #invert distances
    mind = min(m1)
    maxd = max(m1)
    m1 = minmax(m1,mind,maxd)           #min-max transformation
    m1[m1 < edge_thr] = NA              #remove dist < thresh
    m1 = ((m1 - edge_thr))/(1 - edge_thr) #rescale dist
    edge_col = "lightblue"
    vertex_cex=1.5
    for(i1 in c("eps","tiff")){
        if(i1 == "eps"){
            postscript(file=paste(f1,".eps",sep=""),width=7,height=10,
              colormodel="rgb",horizontal=FALSE,onefile=FALSE,paper="special",
              family="Helvetica")
        }else{
            tiff(file=paste(f1,".tif",sep=""),units="in",width=7,height=10,
              res=300,compression="lzw",family="sans")
        }
        oldpar = par(mar=c(1.1,1.1,2.1,1.1),mgp=c(0,0,0),xpd=TRUE,cex.main=0.9)
        gplot(m1,gmode="graph",displaylabels=TRUE,label.cex=label_cex,
          label.pos=5,mode=vmode,jitter=FALSE,main=paste(main,
          " [minmax(-dist) > ",edge_thr,"]",sep=""),vertex.cex=vertex_cex,
          vertex.col=col1[colnames(m1)],vertex.border=1,vertex.lty=1,
          vertices.last=TRUE,edge.lwd=edge_lwd,edge.lty=1,edge.lty.neg=1,
          edge.col=edge_col)
        par(oldpar)
        invisible(dev.off())
    }
}

# Summary statistics for network
# dst1 - distance matrix of variables to be analysed
# f1 - output filename
analyse_net = function(dst1,f1){
    library("sna")
    m1 = as.matrix(dst1)
    m1 = -m1                            #invert distances
    mind = min(m1)
    maxd = max(m1)
    m1 = (m1 - mind)/(maxd - mind)      #min-max transformation
    f2 = paste(f1,"_degree.csv",sep="")
    f3 = paste(f1,"_eigen.csv",sep="")
    f4 = paste(f1,"_dens.csv",sep="")
    #Degree - number of links incident upon a node
    net_deg = degree(m1,gmode="graph")
    s1 = order(net_deg,decreasing=TRUE)
    d1 = cbind(rownames(m1)[s1],net_deg[s1])
    colnames(d1) = c("Node","Degree")
    write.table(d1,f2,row.names=FALSE,col.names=TRUE,sep=",",dec=".",eol="\n",
      na="NA")
    #Eigenvector centrality - Influence of a node in a network
    net_evc = evcent(m1,gmode="graph",use.eigen=TRUE)
    s1 = order(abs(net_evc),decreasing=TRUE)
    d1 = cbind(rownames(m1)[s1],net_evc[s1])
    colnames(d1) = c("Node","Eigenvector")
    write.table(d1,f3,row.names=FALSE,col.names=TRUE,sep=",",dec=".",eol="\n",
      na="NA")
    #Density - the proportion of direct ties in a network relative to the total number possible
    nvert = nrow(m1)
    m2 = m1; diag(m2) = NA; m2[upper.tri(m2)] = NA
    nedge = sum(m2 != 0 & !is.na(m2))
    tedge = nties(m1,mode="graph")
    dens = gden(m1,mode="graph")
    d1 = data.frame(nvert,nedge,tedge,dens)
    write.table(d1,f4,row.names=FALSE,col.names=TRUE,sep=",",dec=".",eol="\n",
      na="NA")
}

# Reduce number of predictors
# dat1 - data frame of variables to be analysed (only predictors)
# y - response variable
# thr1 - upper threshold for correlation between predictors
# thr2 - lower threshold for correlation between predictors and response variables [if thr2==NULL then reduces till maxsize]
# thr3 - lower threshold for coefficient of variation [if thr3==NULL then reduces till maxsize]
# thr4 - upper threshold for Variance Inflation Factor (VIF)
# maxsize - maximum number of predictors [if maxsize==NULL then reduces till nsubj - 1]
# f1 - output filename
reduce_predictors_v2 = function(dat1,y,thr1=0.95,method="spearman",thr2=NULL,
  thr3=NULL,thr4=10,maxsize=NULL,f1){
# Note: Sqrt(R^2) for continuous variables is the correlation between variables. Sqrt(R^2)
# for continuous and categorical variables is the correlation between observed and predicted
# values.
    if(is.null(maxsize)){
        maxsize = nrow(dat1) - 1
    }
    if(is.null(thr2)){
        doStep2 = TRUE
    }else if(thr2 > 0){
        doStep2 = TRUE
    }else{
        doStep2 = FALSE
    }
    if(is.null(thr3)){
        doStep3 = TRUE
    }else if(thr3 > 0){
        doStep3 = TRUE
    }else{
        doStep3 = FALSE
    }
    write("Step1: Highly correlated between themselves",file=f1,append=FALSE)
    if(thr1 < 1){                       #remove variables highly correlated between themselves
        vars = colnames(dat1)
        rm_vars = c()
        for(i1 in vars){
            for(i2 in vars){
                if(i1 < i2 & !i1 %in% rm_vars){
                    rres = cor(dat1[,i1],dat1[,i2],method=method,use="complete")
                    if(abs(rres) > thr1){
                        write(paste("  Remove ",i2," (r_",i1," = ",round(rres,2),
                          ")",sep=""),file=f1,append=TRUE)
                        rm_vars = c(rm_vars,i2)
                    }
                }
            }
        }
        dat1 = dat1[,!vars %in% rm_vars]
    }else{
        write("<Skip step>",file=f1,append=TRUE)
    }
    write("Step2: Low correlated with response variable",file=f1,append=TRUE)
    if(doStep2){                        #remove variables with low correlation with dependent variable
        vars = colnames(dat1)
        if(is.factor(y)){
            y = as.character(y)
        }
        cors = apply(dat1,2,function(x){sqrt(summary(lm(y~x))$r.squared)})
        if(is.null(thr2)){
            n_rm = ncol(dat1) - maxsize #n_rm, so that, nvars = nsubj - 1
            if(n_rm <= 0){
                thr2 = 0.00
            }else{
                thr2 = sort(cors)[n_rm]
            }
        }
        rm_vars = cors <= thr2
        if(sum(rm_vars) == 0){
            write("<None removed>",file=f1,append=TRUE)
        }else{
            dat1 = dat1[,!rm_vars]
            tmp = cbind(vars[rm_vars],cors[rm_vars])
            tmp = apply(tmp,1,function(x){paste("  Remove ",x[1]," (r = ",
              round(as.numeric(x[2]),2),")",sep="")})
            write(tmp,file=f1,append=TRUE)
        }
    }else{
        write("<Skip step>",file=f1,append=TRUE)
    }
    write("Step3: Variables with low variation",file=f1,append=TRUE)
    if(doStep3){                        #remove variables with low variation
        vars = colnames(dat1)
        sds = apply(dat1,2,sd,na.rm=TRUE)
        means = apply(dat1,2,mean,na.rm=TRUE)
        if(is.null(thr3)){
            n_rm = ncol(dat1) - maxsize #n_rm, so that, nvars = nsubj - 1
            if(n_rm <= 0){
                thr3 = 0.00
            }else{
                thr3 = sort(sds/means)[n_rm]
            }
        }
        rm_vars = sds/means <= thr3
        if(sum(rm_vars) == 0){
            write("<None removed>",file=f1,append=TRUE)
        }else{
            dat1 = dat1[,!rm_vars]
            tmp = cbind(vars[rm_vars],sds/means[rm_vars])
            tmp = apply(tmp,1,function(x){paste("  Remove ",x[1]," (r = ",
              round(as.numeric(x[2]),2),")",sep="")})
            write(tmp,file=f1,append=TRUE)
        }
    }else{
        write("<Skip step>",file=f1,append=TRUE)
    }
    write("Step4: Variables with low VIF",file=f1,append=TRUE)
    if(thr4 < Inf){                     #remove variables using VIF (reduce multicollinearity)
        vars = colnames(dat1)
        keep_vars = stepwise_vif(dat1,thresh=thr4,trace=FALSE)
        if(sum(!keep_vars) == 0){
            write("<None removed>",file=f1,append=TRUE)
        }else{
            dat1 = dat1[,keep_vars]
            tmp = paste("  Remove",vars[!vars %in% keep_vars])
            write(tmp,file=f1,append=TRUE)
        }
    }else{
        write("<Skip step>",file=f1,append=TRUE)
    }
    return(dat1)
}

# Stepwise Variance Inflation Factors (VIF) selection to reduce collinearity
# [From "Marcus W Beck" in https://gist.github.com/fawda123/4717702#file-vif_fun-r]
# in_frame - data frame of variables to be analyzed
# thresh - threshold for VIF
# trace - print output of each iteration
stepwise_vif = function(in_frame,thresh=10,trace=TRUE){
    if(class(in_frame) != 'data.frame'){
        in_frame = data.frame(in_frame)
    }
    #get initial vif value for all comparisons of variables
    var_names = names(in_frame)
    vif_init = vif(in_frame)
    vif_max = max(as.numeric(vif_init),na.rm=TRUE)
    if(vif_max < thresh){
        if(trace==TRUE){
            print(data.frame(vif_init))
            cat('\n')
            cat(paste('All variables have VIF < ', thresh,', max VIF ',
              round(vif_max,2), sep=''),'\n\n')
        }
        return(var_names)
    }else{
        in_dat = in_frame
        #backwards selection of explanatory variables, stops when all VIF values are below 'thresh'
        while(vif_max >= thresh){
            var_names = names(in_dat)
            vif_vals = vif(in_dat)
            imax = which(vif_vals == max(as.numeric(vif_vals),na.rm=TRUE))[1]
            vif_max = as.numeric(vif_vals[imax])
            if(vif_max < thresh){
                break
            }
            if(trace==TRUE){
                print(data.frame(vif_vals))
                cat('\n')
                cat('removed: ',names(vif_vals)[imax],vif_max,'\n\n')
                flush.console()
            }
            in_dat = in_dat[,!names(in_dat) %in% names(vif_vals)[imax]]
        }
        return(names(in_dat))
    }
}

# Calculate Variance Inflation Factors (VIF) at a model-level
# [based on HH::vif()]
# dat1 - data frame of variables to be analysed (only predictors)
vif = function(dat1){
#see https://en.wikipedia.org/wiki/Variance_inflation_factor
#see car::vif() for calculation of VIF at variable-level
#see HH::vif() for calculation of VIF at model-level
#see faraway::vif() for calculation of VIF at model-level
#see fmsb::vif() for calculation of VIF at model-level
# Note: VIF can be defined at variable-level (using the partial R^2) and at the model
# level (using R^2, i.e. coefficient of determination).
    vars = colnames(dat1)
    nvars = length(vars)
    r2 = vector("numeric",length=nvars)
    names(r2) = vars
    for(i1 in 1:nvars){
        tmp = lm(dat1[,i1] ~ data.matrix(dat1[,-i1]),na.action="na.omit")
        r2[i1] = 1/(1 - summary(tmp)$r.squared)
    }
    return(r2)
}

# Automatic model selection for multiple linear regression using General LM [simplified version]
# dat1 - data frame of independet variables (predictors)
# resp - dependent variable (response variable)
# maxs1 -  maximum number of terms in candidate models {"hard"}
# crit - information criteria to use {"aic","aicc","bic","qaic","qaicc"}
# thrs  - threshold for nmods to perform exhaustive search
# popsize - Population size
# mutrate - Per locus (i.e. per term) mutation rate, [0,1]
# sexrate - Sexual reproduction rate, [0,1]
# imm - Immigration rate, [0,1]
# deltaM - Stop Rule: change in mean IC
# deltaB - Stop Rule: change in best IC
# conseq - Stop Rule: times with no improvement
# f1 - output filename
mod_select_lm_v2 = function(dat1,resp,maxs1="hard",crit="aicc",thrs=100000,
  popsize=100,mutrate=0.001,sexrate=0.1,imm=0.3,deltaM=0.05,deltaB=0.05,
  conseq=5,nreps=2,f1){
    library("glmulti")
    d1 = data.frame(resp,dat1)
    colnames(d1) = sapply(colnames(d1),function(x){gsub("[-|.]","_",x)}) #remove symbols used in "formulas"
    nvars = ncol(d1)
    vars = colnames(d1)
    hmax = (nrow(d1) - 1) - 3           #hard max (i.e. nparam = nsubj - 3)
    maxs1 = min(maxs1,hmax)             #prevent maxs1 > hard max
    nmods = sum(sapply(1:maxs1,function(x){choose(nvars-1,x)})) + 1
    if(nmods < thrs){                   #exhaustive search
        res2 = glmulti(y=vars[1],xr=vars[2:nvars],data=d1,level=1,maxsize=maxs1,
          method="h",crit=crit,plotty=FALSE,report=FALSE,includeobjects=TRUE,
          marginality=FALSE,model=TRUE,fitfunction="lm")
    }else{                              #genetic algorithm
        res1 = vector("list",length=nreps)
        for(i1 in 1:nreps){
            f2 = paste(f1,"_rep",i1,"_1",sep="")
            res1[[i1]] = glmulti(y=vars[1],xr=vars[2:nvars],data=d1,level=1,
              maxsize=maxs1,method="g",crit=crit,confsetsize=100,plotty=FALSE,
              report=FALSE,name=f2,includeobjects=TRUE,marginality=FALSE,
              model=TRUE,popsize=popsize,mutrate=mutrate,sexrate=sexrate,
              imm=imm,deltaM=deltaM,deltaB=deltaB,conseq=conseq,
              fitfunction="lm")
        }
        if(nreps > 1){
            res2 = glmulti::consensus(xs=res1,confsetsize=100)
        }else{
            res2 = res1
        }
    }
    t1 = weightable(res2)
    t2 = t1[t1[,crit] <= min(t1[,crit]) + 2,]
    t3 = coef(res2)
    f2 = paste(f1,".log",sep="")
    f3 = paste(f1,"_1.csv",sep="")
    f4 = paste(f1,"_2.csv",sep="")
    capture.output(print(res2),file=f2,append=FALSE,type="output")
    write.table(t2,file=f3,sep=",",row.names=TRUE,col.names=NA)
    write.table(t3,file=f4,sep=",",row.names=TRUE,col.names=NA)
    for(i1 in c("eps","tiff")){
        if(i1 == "eps"){
            postscript(file=paste(f1,".eps",sep=""),width=7.0,height=10,
              colormodel="rgb",horizontal=FALSE,onefile=FALSE,paper="special",
              family="Helvetica")
        }else{
            tiff(file=paste(f1,".tif",sep=""),units="in",width=7.0,height=10,
              res=300,compression="lzw",family="sans")
        }
        oldpar = par(mfcol=c(3,1),mar=c(2.2,2.2,1.2,1.2),mgp=c(1.0,0.2,0.0),
          oma=c(0,0,0,0),tcl=-0.2,cex=0.8,cex.lab=0.8,cex.axis=0.8,cex.main=0.9)
        par(mar=c(2.2,10.2,1.2,1.2))
        plot(res2,type="s",cex.names=0.6)
        par(mar=c(2.2,2.2,1.2,1.2))
        plot(res2,type="p")
        plot(res2,type="w")
        par(oldpar)
        invisible(dev.off())
    }
    a1 = formula(res2@objects[[1]]$terms) #formula
    a2 = res2@objects[[1]]$model        #data
    l1 = list(a1,a2)
    names(l1) = c("formula","data")
    return(l1)
}

# Fit multiple linear regression using General LS
# form1 - formula of the model to fit
# dat1 - data frame of variables to fit
# f1 - output filename
# main - title for plots
fit_lm = function(form1,dat1,f1,main){
#see https://en.wikipedia.org/wiki/General_linear_model
# Note: Multiple linear regression assumptions:
# 1. The error terms need to be independent.
# 2. The error variance is homoskedastic.
# 3. The errors are normally distributed with mean = 0
# 4. The independent variables should be independent from each other (i.e. little or no multicollinearity).
# 5. Linearity between independent variables and dependent variable.
# 6. Large sample sizes (i.e. 10-30 cases per parameter).
    library("car")
    fit1 = lm(form1,data=dat1)
    sfit1 = summary(fit1)
    t1 = Anova(fit1,type="II")
    f2 = paste(f1,".log",sep="")
    f3 = paste(f1,".csv",sep="")
    capture.output(print(sfit1),file=f2,append=FALSE,type="output")
    write.table(t1,file=f3,sep=",",row.names=TRUE,col.names=NA)
    vars = all.vars(form1)
    y = dat1[,vars[1]]
    f = fit1$fitted.values
    r = fit1$residuals
    for(i1 in c("eps","tiff")){
        if(i1 == "eps"){
            postscript(file=paste(f1,".eps",sep=""),width=7.0,height=7.0,
              colormodel="rgb",horizontal=FALSE,onefile=FALSE,paper="special",
              family="Helvetica")
        }else{
            tiff(file=paste(f1,".tif",sep=""),units="in",width=7.0,height=7.0,
              res=300,compression="lzw",family="sans")
        }
        oldpar = par(mfcol=c(2,2),mar=c(2.2,2.2,1.2,1.2),mgp=c(1.0,0.2,0.0),
          tcl=-0.2,cex=0.8,cex.lab=0.8,cex.axis=0.8,cex.main=0.9)
        qqnorm(r,col=4); qqline(r)                        #wg-err_norm: qqplot
        plot(f,sqrt(abs(r)),xlab="Fitted",ylab="sqrt(abs(Residuals))",col=4)
        abline(h=0,lty=2)                                 #wg-err_var: scatterplot
        hist(r,freq=FALSE,main="",col=4,xlab="Residuals") #wg-err_norm: histogram
        plot(f,y,xlab="Fitted values",ylab="Values",main=main,col=4)
        abline(c(0,1),lty=2)                              #goodness-of-fit: scatterplot
        par(oldpar)
        invisible(dev.off())
    }
}

# Fits multiple regression via neural networks
# [adapted from nnet::nnet.formula()]
# formula - formula of the regression
# data - data frame of variables to fit
# weights - weights for each subject
# ... - arguments for "nnet" {e.g. size, decay, maxit, abstol, reltol, Hess, trace}
# na.action - usual model parameter
# contrast - usual model parameter
multivar = function(formula,data,weights,...,subset,na.action,contrasts=NULL){
    m = match.call(expand.dots=FALSE)
    if(is.matrix(eval.parent(m$data))){
        m$data <- as.data.frame(data)
    }
    m$... = m$contrasts = NULL
    m[[1L]] = quote(stats::model.frame)
    m = eval.parent(m)
    Terms = attr(m, "terms")
    x = model.matrix(Terms,m,contrasts)
    cons = attr(x,"contrast")
    xint = match("(Intercept)",colnames(x),nomatch = 0L)
    if(xint > 0L){
        x = x[,-xint,drop=FALSE]
    }
    if(attr(Terms,"intercept") == 0){
        Qr = qr(x)
    }else{
        Qr = qr(cbind(1,x))
        colnames(Qr$qr) = c("(Intercept)",colnames(Qr$qr)[-1])
    }
    w = model.weights(m)
    if(length(w) == 0L){
        w = rep(1,nrow(x))
    }
    y = model.response(m)
    miny = min(y)
    maxy = max(y)
    y_tr = minmax(y,miny,maxy)
    res = nnet.default(x,y_tr,w,linout=TRUE,...)
    res$terms = Terms
    res$coefnames = colnames(x)
    res$call = match.call()
    res$na.action = attr(m,"na.action")
    res$contrasts = cons
    res$call$formula = formula
    res$xlevels = .getXlevels(Terms, m)
    if(0){
        res$tr.fitted.values = matrix(minmax_inv(res$fitted.values[,1],
          miny,maxy))
        res$tr.residuals = matrix(y - res$tr.fitted.values)
    }else{
        res$tr.fitted.values = res$fitted.values
        res$tr.residuals = res$residuals
        res$fitted.values = matrix(minmax_inv(res$fitted.values[,1],miny,maxy))
        res$residuals = matrix(y - res$fitted.values)
    }
    res$deviance = sum(res$residuals^2)
    res$qr = Qr
    res$rank = res$qr$rank
    res$df.residual = length(y) - res$rank
    res$model = m
    class(res) = c("multivar","nnet.formula","nnet")
    res
}

# Create logLik() function for "multivar" objects
# [adapted from stats:::logLik.lm()]
# object - "multivar" object
logLik.multivar = function(object,...){
    res = object$residuals
    p = object$rank
    N = length(res)
    if(is.null(w <- object$weights)){
        w = rep.int(1,N)
    }else{
        excl = w == 0
        if(any(excl)){
            res = res[!excl]
            N = length(res)
            w = w[!excl]
        }
    }
    val = 0.5*(sum(log(w)) - N*(log(2*pi) + 1 - log(N) + log(sum(w*res^2))))
    attr(val, "nobs") = N
    attr(val, "df") = p + 1
    class(val) = "logLik"
    val
}

# Create nobs() function for "multivar" objects
# [adapted from stats:::nobs.lm()]
# object - "multivar" object
nobs.multivar = function(object){
    if(!is.null(w <- object$weights)){
        sum(w != 0)
    }else{
        nrow(object$residuals)
    }
}

# Create summary() function for "multivar" objects
# [adapted from stats:::summary.lm()]
# object - "multivar" object
summary_multivar = function(object){
    z = object
    p = z$rank
    rdf = z$df.residual
    if(p == 0){
        r = z$residuals
        n = length(r)
        w = z$weights
        if(is.null(w)){
            rss = sum(r^2)
        }else{
            rss = sum(w*r^2)
            r = sqrt(w)*r
        }
        resvar = rss/rdf
        ans = z[c("call", "terms", if (!is.null(z$weights)) "weights")]
        ans$residuals = r
        ans$df = n
        ans$sigma = sqrt(resvar)
        ans$r.squared = ans$adj.r.squared <- 0
        return(ans)
    }
    Qr = object$qr
    n = nrow(Qr$qr)
    r = z$residuals
    f = z$fitted.values
    w = z$weights
    if(is.null(w)){
        mss = if(attr(z$terms, "intercept")){sum((f - mean(f))^2)}else{sum(f^2)}
        rss = sum(r^2)
    }else{
        mss = if(attr(z$terms, "intercept")){m = sum(w * f/sum(w));
          sum(w*(f - m)^2)}else{sum(w*f^2)}
        rss = sum(w*r^2)
        r = sqrt(w)*r
    }
    resvar = rss/rdf
    if(is.finite(resvar) && resvar < (mean(f)^2 + var(f))*1e-30)
        warning("essentially perfect fit: summary may be unreliable")
    p1 = 1L:p
    ans = z[c("call", "terms",if(!is.null(z$weights)){"weights"})]
    ans$residuals = r
    ans$sigma = sqrt(resvar)
    ans$df = c(p,rdf,ncol(Qr$qr))
    if(p != attr(z$terms,"intercept")){
        df.int = if(attr(z$terms, "intercept")){1L}else{0L}
        ans$r.squared = mss/(mss + rss)
        ans$adj.r.squared = 1 - (1 - ans$r.squared)*((n - df.int)/rdf)
        ans$fstatistic = c(value=(mss/(p - df.int))/resvar,numdf=p - df.int,
          dendf=rdf)
    }else{
        ans$r.squared = ans$adj.r.squared = 0
    }
    if(!is.null(z$na.action)){
        ans$na.action = z$na.action
    }
    ans
}

# Fit multiple regression using ANN [add finer tune for ANN]
# form1 - formula of the model to fit
# dat1 - data frame of variables to fit
# nn_size - number of units in the hidden layer [ANN]
# nn_decay - parameter for weight decay [ANN]
# nn_maxit - maximum number of iterations [ANN]
# nn_abstol - absolute fit criterion [ANN]
# nn_reltol - relative fit criterion [ANN]
# f1 - output filename
# main - title for plots
fit_nnet_v2 = function(form1,dat1,nn_size=10,nn_decay=0.01,nn_maxit=1000,
  nn_abstol=1e-4,nn_reltol=1e-8,f1,main){
#see https://en.wikipedia.org/wiki/Artificial_neural_network
# Note: Multiple regression assumptions:
# 1. The error terms need to be independent.
# 2. The independent variables should be independent from each other (i.e. little or no multicollinearity).
# 3. Large sample sizes (i.e. 10-30 cases per parameter).
# Note: Neuronal network caveats
# 1. optimal number of hidden units is problem specific (risk of overfitting)
    library("nnet")
    library("caret")
    library("NeuralNetTools")
    vars = all.vars(form1)
    if(is.null(nn_size) || is.null(nn_decay)){
        if(!is.null(nn_size)){
            gr_size = nn_size
        }else{
            gr_size = c(1,3*(1:10))
        }
        if(!is.null(nn_decay)){
            gr_decay = nn_decay
        }else{
            gr_decay = 1*10^(-(1:9))
        }
        y = as.matrix(dat1[,vars[1]])
        miny = min(y)
        maxy = max(y)
        dat2 = dat1
        dat2[,vars[1]] = minmax(y,miny,maxy)
        grid1 = expand.grid(size=gr_size,decay=gr_decay)
        train1 = train(form1,data=dat2,method="nnet",maxit=nn_maxit,
          abstol=nn_abstol,reltol=nn_reltol,linout=TRUE,trace=FALSE,
          trControl=trainControl(method="boot",number=100),tuneGrid=grid1,
          metric="RMSE")
        nn_size = train1$bestTune[1,1]
        nn_decay = train1$bestTune[1,2]
    }
    fit1 = multivar(form1,dat1,size=nn_size,decay=nn_decay,maxit=nn_maxit,
      abstol=nn_abstol,reltol=nn_reltol,trace=FALSE)
    sfit1 = summary(fit1)
    sfit2 = summary_multivar(fit1)
    if(length(vars) > 1){
        t1 = olden(fit1,bar_plot=FALSE)
    }
    f2 = paste(f1,"_1.log",sep="")
    f3 = paste(f1,"_2.log",sep="")
    f4 = paste(f1,"_3.log",sep="")
    f5 = paste(f1,".csv",sep="")
    capture.output(print(sfit1),file=f2,append=FALSE,type="output")
    capture.output(stats:::print.summary.lm(sfit2),file=f3,append=FALSE,
      type="output")
    if(exists("train1")){
        capture.output(train1,file=f4,append=FALSE,type="output")
    }
    if(length(vars) > 1){
        write.table(t1,file=f5,sep=",",row.names=TRUE,col.names=NA)
    }
    f2 = paste(f1,"_1",sep="")
    y = dat1[,vars[1]]
    f = fit1$fitted.values
    r = fit1$residuals
    for(i1 in c("eps","tiff")){
        if(i1 == "eps"){
            postscript(file=paste(f2,".eps",sep=""),width=7.0,height=7.0,
              colormodel="rgb",horizontal=FALSE,onefile=FALSE,paper="special",
              family="Helvetica")
        }else{
            tiff(file=paste(f2,".tif",sep=""),units="in",width=7.0,height=7.0,
              res=300,compression="lzw",family="sans")
        }
        oldpar = par(mfcol=c(2,2),mar=c(2.2,2.2,1.2,1.2),mgp=c(1.0,0.2,0.0),
          tcl=-0.2,cex=0.8,cex.lab=0.8,cex.axis=0.8,cex.main=0.9)

        qqnorm(r,col=4); qqline(r)                        #wg-err_norm: qqplot
        plot(f,sqrt(abs(r)),xlab="Fitted",ylab="sqrt(abs(Residuals))",col=4)
        abline(h=0,lty=2)                                 #wg-err_var: scatterplot
        hist(r,freq=FALSE,main="",col=4,xlab="Residuals") #wg-err_norm: histogram
        plot(f,y,xlab="Fitted values",ylab="Values",main=main,col=4)
        abline(c(0,1),lty=2)                              #goodness-of-fit: scatterplot
        par(oldpar)
        invisible(dev.off())
    }
    if(length(vars) > 1){
        f2 = paste(f1,"_2",sep="")
        for(i1 in c("eps","tiff")){
            if(i1 == "eps"){
                postscript(file=paste(f2,".eps",sep=""),width=7.0,height=7.0,
                  colormodel="rgb",horizontal=FALSE,onefile=FALSE,
                  paper="special",family="Helvetica")
            }else{
                tiff(file=paste(f2,".tif",sep=""),units="in",width=7.0,
                  height=7.0,res=300,compression="lzw",family="sans")
            }
            oldpar = par(mar=c(1.1,1.1,2.1,1.1),mgp=c(0,0,0),xpd=TRUE,
              cex.main=0.9)
            plotnet(fit1,cex_val=0.7,circle_cex=3)
            par(oldpar)
            invisible(dev.off())
        }
    }
}

}


#---
#title: "calculating plant density from survey done in 2020 on transect A"
#author: "Tal Dahan-Meir"
#date: "25/08/2021"
#---

require(geosphere)

csv = read.csv("data/2020_plant_densities.csv",colClasses="character")

#get dists between plants to next plants and add to data.frame
nPlants_a = nrow(csv)
coordinates_next_plants = data.frame("altitude_nextPlant" = rep(NA,nPlants_a),
                                     "latitude_nextPlant" = rep(NA,nPlants_a),
                                     "longitude_nextPlant" = rep(NA,nPlants_a))
for (i in 1:(nPlants_a-1)){
  coordinates_next_plants[i,] = csv[i+1,c("altitude","latitude","longitude")]
}
csv_distances = cbind (csv,coordinates_next_plants)
csv_distances$dist_meters = c( apply(csv_distances[1 : (nPlants_a-1), ], 1, function(tt) { distm(x = c(as.numeric(tt[6]), as.numeric(tt[5]) ), y = c(as.numeric(tt[9]), as.numeric(tt[8]) ) )} ), "NA")
csv_distances$position_meters = as.numeric(c( 0 ,cumsum (csv_distances[1:(nPlants_a-1), "dist_meters"] )))
csv_distances$plantsPerMeter = as.numeric(csv_distances$plants_until_next_point) / as.numeric(csv_distances$dist_meters)
csv_distances$plantsinMM_divided_by_pi = as.numeric(csv_distances$plants_in_1MM2) / pi
csv_distances$plantsinM_combined = as.numeric(csv_distances$plantsinMM_divided_by_pi)+as.numeric(csv_distances$plantsPerMeter)

head(csv_distances)

pdf("transect_a_density_2020.pdf", height=8, width=22)
plot (as.numeric(csv_distances$position_meters),
      as.numeric(csv_distances$plantsinMM_divided_by_pi),
      main = "Density transect A",
      cex.main=2,
      pch = 19,
      cex = 0.2,
      cex.lab = 1.5,
      cex.axis=1.4,
      ylim=c(0,25),
      xlim=c(0,320),
      col="black",
      xlab = "position (M)",
      ylab = "spikes per M^2")

apply(csv_distances,1, function(xx){
  rect(xleft = as.numeric(xx[11]),
      ybottom = 0,
      xright = as.numeric(xx[11])+as.numeric(xx[10]),
      ytop = as.numeric(xx[12])/2,
      col="#00ABA920",
      border= NA)} )
names_height_bonus = 0.5
apply(csv_distances,1, function(xx){
  text(x = as.numeric(xx[11]),
       y = as.numeric(xx[13])+names_height_bonus,
       as.character(xx[1]),
       col="black",
       cex=1)} )
dev.off()

###calculate mean density in m^2###
#(sum of plants around peg divided by pi - to transform from circle to ^2)+(sum of plants until next point divided by 2 - to transform from 2 meters to 1 meter width)/(transect length)
(sum(as.numeric(csv_distances$plantsinMM_divided_by_pi))+(sum(as.numeric(csv_distances$plants_until_next_point),na.rm=T)/2))/max(csv_distances$position_meters)


#  James Rekow

subgroupVecMatcher = function(subgroupVec1, subgroupVec2){

  #  ARGS: subgroupVec1 - a numeric vector whose ith element is the subgroup to which sample i belongs,
  #                       where sample i is an element of a set A
  #        subgroupVec2 - another vector of the same form as subgroupVec1, corresponding to the same set A
  #
  #  RETURNS: subgroupVec2Ordered  - subgroupVec2 with the values permuted so as to minimize the number of
  #                                  differences between subgroupVec1 and subgroupVec2

  #  TODO: add a method argument with two options, Jaccard index, and misallocated elements

  library(gtools)

  #  number of subgroups in subgroupVec2
  numSubgroups2 = max(subgroupVec2)

  #  list which indices correspond to each subgroup
  subgroupIxList = lapply(as.list(1:numSubgroups2), function(ii) as.numeric(subgroupVec2 == ii))

  #  create a matrix whose rows are the permutations of 1:numSubgroups 2
  permutationMat = permutations(n = numSubgroups2, r = numSubgroups2)

  createSubgroupVec = function(perm){

    #  ARGS: perm - numeric vector that is a permutation of 1:numSubgroups2
    #
    #  RETURNS: permSubgroupVec2 - subgroup created by permuting the values of subgroupVec2 according to
    #                              perm

    #  compute the elements of the permutation in each subgroup
    permSubgroupVec2List = lapply(as.list(1:numSubgroups2), function(ii) perm[ii] * subgroupIxList[[ii]])

    #  combine values to create the permuted subgroup vector
    permSubgroupVec2 = Reduce("+", permSubgroupVec2List)

    return(permSubgroupVec2)

  } #  end createSubgroupVec function

  computeSubgroupDiff = function(perm){

    #  ARGS: perm - numeric vector that is a permutation of 1:numSubgroups2
    #
    #  RETURNS: subgroupDiff - number of differences between subgroupVec1 and the given permutation
    #                          of the values of subgroupVec2. All of the values of i in subgroupVec2
    #                          are replaced by perm[i]

    #  create subgroup corresponding to input permutation of values of subgroupVec2
    permSubgroupVec2 = createSubgroupVec(perm)

    #  compute number of differences between the given permutation of values of subgroupVec2 and
    #  subgroupVec1
    subgroupDiff = sum(subgroupVec1 != permSubgroupVec2)

    return(subgroupDiff)

  } #  end computeSubgroupDiff function

  #  compute the number of differences between subgroupVec1 and each permutation of values of subgroupVec2
  subgroupDiffs = apply(permutationMat, 1, computeSubgroupDiff)

  #  find the row number of the permutation that gives the minimum number of subgroup differences
  minDiffIx = which.min(subgroupDiffs)

  #  identify optimal permutation of values of subgroupVec2
  optimalPerm = permutationMat[minDiffIx, ]

  #  create optimal permutation of values of subgroupVec2
  subgroupVec2Ordered = createSubgroupVec(optimalPerm)

  return(subgroupVec2Ordered)

} #  end subgroupVecMatcher function


library(MASS)
library(dplyr)
library(tidyr)
library(ggplot2)
library(ascii)
library(lubridate)
library(splines)
library(mgcv)

## Import datasets needed for chapter 4
PSDS_PATH <- file.path('~', 'statistics-for-data-scientists')

lung <- read.csv(file.path(PSDS_PATH, 'data', 'LungDisease.csv'))

zhvi <- read.csv(file.path(PSDS_PATH, 'data', 'County_Zhvi_AllHomes.csv'))
zhvi <- unlist(zhvi[13,-(1:5)])
dates <- parse_date_time(paste(substr(names(zhvi), start=2, stop=8), "01", sep="."), "Ymd")
zhvi <- data.frame(ym=dates, zhvi_px=zhvi, row.names = NULL) %>%
  mutate(zhvi_idx=zhvi/last(zhvi))

house <- read.csv(file.path(PSDS_PATH, 'data', 'house_sales.csv'), sep='\t')
# house <- house[house$ZipCode > 0, ]
# write.table(house, file.path(PSDS_PATH, 'data', 'house_sales.csv'), sep='\t')
## Code for Figure 1
png(filename=file.path(PSDS_PATH, 'figures', 'psds_0401.png'),  width = 4, height=4, units='in', res=300)
par(mar=c(4,4,0,0)+.1)
plot(lung$Exposure, lung$PEFR, xlab="Exposure", ylab="PEFR")
dev.off()

## Code snippet 4.1
model <- lm(PEFR ~ Exposure, data=lung)
model

## Code for figure 2
png(filename=file.path(PSDS_PATH, 'figures', 'psds_0402.png'), width = 350, height = 350)
par(mar=c(4,4,0,0)+.1)

plot(lung$Exposure, lung$PEFR, xlab="Exposure", ylab="PEFR", ylim=c(300,450), type="n", xaxs="i")
abline(a=model$coefficients[1], b=model$coefficients[2], col="blue", lwd=2)
text(x=.3, y=model$coefficients[1], labels=expression("b"[0]),  adj=0, cex=1.5)
x <- c(7.5, 17.5)
y <- predict(model, newdata=data.frame(Exposure=x))
segments(x[1], y[2], x[2], y[2] , col="red", lwd=2, lty=2)
segments(x[1], y[1], x[1], y[2] , col="red", lwd=2, lty=2)
text(x[1], mean(y), labels=expression(Delta~Y), pos=2, cex=1.5)
text(mean(x), y[2], labels=expression(Delta~X), pos=1, cex=1.5)
text(mean(x), 400, labels=expression(b[1] == frac(Delta ~ Y, Delta ~ X)), cex=1.5)
dev.off()

## Code snippet 4.2
fitted <- predict(model)
resid <- residuals(model)

## Code for figure 3
png(filename=file.path(PSDS_PATH, 'figures', 'psds_0403.png'),  width = 4, height=4, units='in', res=300)
par(mar=c(4,4,0,0)+.1)

lung1 <- lung %>%
  mutate(Fitted=fitted,
         positive = PEFR>Fitted) %>%
  group_by(Exposure, positive) %>%
  summarize(PEFR_max = max(PEFR),
            PEFR_min = min(PEFR),
            Fitted = first(Fitted)) %>%
  ungroup() %>%
  mutate(PEFR = ifelse(positive, PEFR_max, PEFR_min)) %>%
  arrange(Exposure)

plot(lung$Exposure, lung$PEFR, xlab="Exposure", ylab="PEFR")
abline(a=model$coefficients[1], b=model$coefficients[2], col="blue", lwd=2)
segments(lung1$Exposure, lung1$PEFR, lung1$Exposure, lung1$Fitted, col="red", lty=3)
dev.off()

## Code snippet 4.3
head(house[, c("AdjSalePrice", "SqFtTotLiving", "SqFtLot", "Bathrooms",
               "Bedrooms", "BldgGrade")])

## Code snippet 4.4
house_lm <- lm(AdjSalePrice ~ SqFtTotLiving + SqFtLot + Bathrooms +
                 Bedrooms + BldgGrade,
               data=house, na.action=na.omit)

## Code snippet 4.5
house_lm

## Code snippet 4.6
summary(house_lm)

## Code snippet 4.7
house_full <- lm(AdjSalePrice ~ SqFtTotLiving + SqFtLot + Bathrooms +
                   Bedrooms + BldgGrade + PropertyType + NbrLivingUnits +
                   SqFtFinBasement + YrBuilt + YrRenovated + NewConstruction,
                 data=house, na.action=na.omit)

## Code snippet 4.8
step_lm <- stepAIC(house_full, direction="both")
step_lm

lm(AdjSalePrice ~  Bedrooms, data=house)

# WeightedRegression
## Code snippet 4.9
house$Year = year(house$DocumentDate)
house$Weight = house$Year - 2005

## Code snippet 4.10
house_wt <- lm(AdjSalePrice ~ SqFtTotLiving + SqFtLot + Bathrooms +
                 Bedrooms + BldgGrade,
               data=house, weight=Weight, na.action=na.omit)
round(cbind(house_lm=house_lm$coefficients,
            house_wt=house_wt$coefficients), digits=3)

# Factor Variables
## Code snippet 4.11
head(house[, 'PropertyType'])

## Code snippet 4.12
prop_type_dummies <- model.matrix(~PropertyType -1, data=house)
head(prop_type_dummies)

## Code snippet 4.13
lm(AdjSalePrice ~ SqFtTotLiving + SqFtLot + Bathrooms +
     Bedrooms +  BldgGrade + PropertyType, data=house)

## Code snippet 4.14
table(house$ZipCode)

## Code snippet 4.15
zip_groups <- house %>%
  mutate(resid = residuals(house_lm)) %>%
  group_by(ZipCode) %>%
  summarize(med_resid = median(resid),
            cnt = n()) %>%
  # sort the zip codes by the median residual
  arrange(med_resid) %>%
  mutate(cum_cnt = cumsum(cnt),
         ZipGroup = factor(ntile(cum_cnt, 5)))
house <- house %>%
  left_join(select(zip_groups, ZipCode, ZipGroup), by='ZipCode')

# correlated variables
# Code snippet 4.15
step_lm$coefficients

# Code snippet 4.16
update(step_lm, . ~ . -SqFtTotLiving - SqFtFinBasement - Bathrooms)

#  ConfoundingVariables
## Code snippet 4.17
lm(AdjSalePrice ~  SqFtTotLiving + SqFtLot +
     Bathrooms + Bedrooms +
     BldgGrade + PropertyType + ZipGroup,
   data=house, na.action=na.omit)

#  Interactions
## Code snippet 4.18
lm(AdjSalePrice ~  SqFtTotLiving*ZipGroup + SqFtLot +
     Bathrooms + Bedrooms +
     BldgGrade + PropertyType,
   data=house, na.action=na.omit)

head(model.matrix(~C(PropertyType, sum) , data=house))

# outlier anaysis
## Code snippet 4.19
house_98105 <- house[house$ZipCode == 98105,]
lm_98105 <- lm(AdjSalePrice ~ SqFtTotLiving + SqFtLot + Bathrooms +
                 Bedrooms + BldgGrade, data=house_98105)

## Code snippet 4.20
sresid <- rstandard(lm_98105)
idx <- order(sresid, decreasing=FALSE)
sresid[idx[1]]
resid(lm_98105)[idx[1]]

## Code snippet 4.21
house_98105[idx[1], c('AdjSalePrice', 'SqFtTotLiving', 'SqFtLot',
                      'Bathrooms', 'Bedrooms', 'BldgGrade')]

# Figure 4-5: Influential data point in regression
seed <- 11
set.seed(seed)
x <- rnorm(25)
y <- -x/5 + rnorm(25)
x[1] <- 8
y[1] <- 8

png(filename=file.path(PSDS_PATH, 'figures', 'psds_0405.png'),  width = 4, height=4, units='in', res=300)
par(mar=c(3,3,0,0)+.1)
plot(x, y, xlab='', ylab='', pch=16)
model <- lm(y~x)
abline(a=model$coefficients[1], b=model$coefficients[2], col="blue", lwd=3)
model <- lm(y[-1]~x[-1])
abline(a=model$coefficients[1], b=model$coefficients[2], col="red", lwd=3, lty=2)
dev.off()

# influential observations
## Code snippet 4.22
std_resid <- rstandard(lm_98105)
cooks_D <- cooks.distance(lm_98105)
hat_values <- hatvalues(lm_98105)
plot(hat_values, std_resid, cex=10*sqrt(cooks_D))
abline(h=c(-2.5, 2.5), lty=2)

## Code for Figure 4-6
png(filename=file.path(PSDS_PATH, 'figures', 'psds_0406.png'), width = 4, height=4, units='in', res=300)
par(mar=c(4,4,0,0)+.1)
plot(hat_values, std_resid, cex=10*sqrt(cooks_D))
abline(h=c(-2.5, 2.5), lty=2)
dev.off()

## Table 4-2

lm_98105_inf <- lm(AdjSalePrice ~ SqFtTotLiving + SqFtLot +
                     Bathrooms +  Bedrooms + BldgGrade,
                   subset=cooks_D<.08, data=house_98105)

df <- data.frame(lm_98105$coefficients,
                 lm_98105_inf$coefficients)
names(df) <- c('Original', 'Influential Removed')
ascii((df),
      include.rownames=TRUE, include.colnames=TRUE, header=TRUE,
      digits=rep(0, 3), align=c("l", "r", "r") ,
      caption="Comparison of regression coefficients with the full data and with influential data removed")

## heteroskedasticity
## Code snippet 4.23
df <- data.frame(
  resid = residuals(lm_98105),
  pred = predict(lm_98105))
ggplot(df, aes(pred, abs(resid))) +
  geom_point() +
  geom_smooth()

## Code for figure 4-7
png(filename=file.path(PSDS_PATH, 'figures', 'psds_0407.png'), width = 4, height=4, units='in', res=300)

ggplot(df, aes(pred, abs(resid))) +
  geom_point() +
  geom_smooth() +
  theme_bw()

dev.off()

## Code for figure 4-8
png(filename=file.path(PSDS_PATH, 'figures', 'psds_0408.png'), width = 4, height=4, units='in', res=300)
par(mar=c(4,4,0,0)+.1)
hist(std_resid, main='')
dev.off()

## partial residuals plot
## Code snippet 4.24
terms <- predict(lm_98105, type='terms')
partial_resid <- resid(lm_98105) + terms

## Code snippet 4.25
df <- data.frame(SqFtTotLiving = house_98105[, 'SqFtTotLiving'],
                 Terms = terms[, 'SqFtTotLiving'],
                 PartialResid = partial_resid[, 'SqFtTotLiving'])
ggplot(df, aes(SqFtTotLiving, PartialResid)) +
  geom_point(shape=1) + scale_shape(solid = FALSE) +
  geom_smooth(linetype=2) +
  geom_line(aes(SqFtTotLiving, Terms))

## Code for figure 4-9
png(filename=file.path(PSDS_PATH, 'figures', 'psds_0409.png'),  width = 4, height=4, units='in', res=300)

df <- data.frame(SqFtTotLiving = house_98105[, 'SqFtTotLiving'],
                 Terms = terms[, 'SqFtTotLiving'],
                 PartialResid = partial_resid[, 'SqFtTotLiving'])
ggplot(df, aes(SqFtTotLiving, PartialResid)) +
  geom_point(shape=1) + scale_shape(solid = FALSE) +
  geom_smooth(linetype=2) +
  theme_bw() +
  geom_line(aes(SqFtTotLiving, Terms))

dev.off()

## Code snippet 4.26

lm(AdjSalePrice ~ poly(SqFtTotLiving, 2) + SqFtLot +
   BldgGrade + Bathrooms +  Bedrooms,
   data=house_98105)

lm_poly <- lm(AdjSalePrice ~  poly(SqFtTotLiving, 2) + SqFtLot +
                BldgGrade +  Bathrooms +  Bedrooms,
              data=house_98105)
terms <- predict(lm_poly, type='terms')
partial_resid <- resid(lm_poly) + terms

## Code for Figure 4-10
png(filename=file.path(PSDS_PATH, 'figures', 'psds_0410.png'), width = 4, height=4, units='in', res=300)

df <- data.frame(SqFtTotLiving = house_98105[, 'SqFtTotLiving'],
                 Terms = terms[, 1],
                 PartialResid = partial_resid[, 1])
ggplot(df, aes(SqFtTotLiving, PartialResid)) +
  geom_point(shape=1) + scale_shape(solid = FALSE) +
  geom_smooth(linetype=2) +
  geom_line(aes(SqFtTotLiving, Terms))+
  theme_bw()

dev.off()

## Code snippet 4.27
knots <- quantile(house_98105$SqFtTotLiving, p=c(.25, .5, .75))
lm_spline <- lm(AdjSalePrice ~ bs(SqFtTotLiving, knots=knots, degree=3) +  SqFtLot +
                  Bathrooms + Bedrooms + BldgGrade,  data=house_98105)

terms1 <- predict(lm_spline, type='terms')
partial_resid1 <- resid(lm_spline) + terms

## Code for Figure 4-12
png(filename=file.path(PSDS_PATH, 'figures', 'psds_0412.png'), width = 4, height=4, units='in', res=300)

df1 <- data.frame(SqFtTotLiving = house_98105[, 'SqFtTotLiving'],
                 Terms = terms1[, 1],
                 PartialResid = partial_resid1[, 1])
ggplot(df1, aes(SqFtTotLiving, PartialResid)) +
  geom_point(shape=1) + scale_shape(solid = FALSE) +
  geom_smooth(linetype=2) +
  geom_line(aes(SqFtTotLiving, Terms))+
  theme_bw()

dev.off()

## Code snippet 4.27
lm_gam <- gam(AdjSalePrice ~ s(SqFtTotLiving) + SqFtLot +
                Bathrooms +  Bedrooms + BldgGrade,
              data=house_98105)
terms <- predict.gam(lm_gam, type='terms')
partial_resid <- resid(lm_gam) + terms

## Code for Figure 4-13
png(filename=file.path(PSDS_PATH, 'figures', 'psds_0413.png'), width = 4, height=4, units='in', res=300)
df <- data.frame(SqFtTotLiving = house_98105[, 'SqFtTotLiving'],
                 Terms = terms[, 5],
                 PartialResid = partial_resid[, 5])
ggplot(df, aes(SqFtTotLiving, PartialResid)) +
  geom_point(shape=1) + scale_shape(solid = FALSE) +
  geom_smooth(linetype=2) +
  geom_line(aes(SqFtTotLiving, Terms))  +
  theme_bw()
dev.off()


#' Variogram plots for spatial models.
#'
#' Produces variogram plots for checking spatial trends.
#'
#' @param model.obj An `asreml` model object.
#' @param palette A string specifying the colour scheme to use for plotting. The default value (`"default"`) is equivalent to `"rainbow"`. Colour blind friendly palettes can also be provided via options `"colour blind"` (or `"color blind"`, both equivalent to `"viridis"`), `"magma"`, `"inferno"`, `"plasma"` or `"cividis"`. The `"Spectral"` palette from [scales::brewer_pal()] is also possible.
#'
#' @return A ggplot2 object.
#'
#' @importFrom interp interp
#' @importFrom grDevices rainbow
#' @importFrom lattice wireframe
#' @importFrom cowplot plot_grid
#' @importFrom ggplot2 ggplot geom_tile coord_equal geom_contour scale_fill_gradientn theme_bw scale_x_continuous scale_y_continuous theme labs
#'
#' @references S. P. Kaluzny, S. C. Vega, T. P. Cardoso, A. A. Shelly, "S+SpatialStats: User’s Manual for Windows® and UNIX®" _Springer New York_, 2013, p. 68, https://books.google.com.au/books?id=iADkBwAAQBAJ.
#' @references A. R. Gilmour, B. R. Cullis, A. P. Verbyla, "Accounting for Natural and Extraneous Variation in the Analysis of Field Experiments." _Journal of Agricultural, Biological, and Environmental Statistics 2, no. 3_, 1997, pp. 269–93, https://doi.org/10.2307/1400446.
#'
#' @examples
#' \dontrun{
#' library(asreml)
#' oats <- asreml::oats
#' oats <- oats[order(oats$Row, oats$Column),]
#' model.asr <- asreml(yield ~ Nitrogen + Variety + Nitrogen:Variety,
#'                     random = ~ Blocks + Blocks:Wplots,
#'                     residual = ~ ar1(Row):ar1(Column),
#'                     data = oats)
#' variogram(model.asr)
#' }
#' @export

variogram <- function(model.obj, palette = "default") {

    if(!(inherits(model.obj, "asreml"))) {
        stop("model.obj must be an asreml model object")
    }

    aa <- vario_df(model.obj)
    xnam <- names(aa)[2]
    ynam <- names(aa)[1]
    fld <- interp::interp(y = aa[,1], x = aa[,2], z = aa$gamma)
    gdat <- cbind(expand.grid(x = fld$x, y = fld$y), z = as.vector(fld$z))

    a <- ggplot2::ggplot(gdat, ggplot2::aes(x = y, y = x, z = z, fill = z)) +
        ggplot2::geom_tile(alpha = 0.6) +
        ggplot2::coord_equal() +
        ggplot2::geom_contour(color = "white", alpha = 0.5) +
        ggplot2::theme_bw(base_size = 8) +
        ggplot2::scale_y_continuous(expand = c(0, 0), breaks = seq(1, max(gdat$x), 2)) +
        ggplot2::scale_x_continuous(expand = c(0, 0), breaks = seq(1, max(gdat$y), 2)) +
        ggplot2::theme(legend.position = "none", aspect.ratio = 0.3) +
        ggplot2::labs(y = paste(xnam, "Lag", sep = " "), x = paste(ynam, "Lag", sep = " "))

    if(tolower(palette) == "rainbow" | tolower(palette) == "default") {
        a <- a + ggplot2::scale_fill_gradientn(colours = grDevices::rainbow(100))

        b <- lattice::wireframe(z ~ y * x, data = gdat, aspect = c(61/87, 0.4),
                                scales = list(cex = 0.5, arrows = FALSE),
                                shade = TRUE, colorkey = FALSE,
                                par.settings = list(axis.line = list(col = 'transparent')),
                                xlab = list(label = paste(ynam, "Lag", sep = " "), cex = .8, rot = 20),
                                ylab = list(label = paste(xnam, "Lag", sep = " "), cex = .8, rot = -18),
                                zlab = list(label = NULL, cex.axis = 0.5))
    }
    else if(any(grepl("(colou?r([[:punct:]]|[[:space:]]?)blind)|cb|viridis", palette, ignore.case = T))) {
        a <- a + ggplot2::scale_fill_gradientn(colours = scales::viridis_pal(option = "viridis")(50))

        # Create the lattice plot
        b <- lattice::wireframe(z ~ y * x, data = gdat, aspect = c(61/87, 0.4),
                                scales = list(cex = 0.5, arrows = FALSE),
                                drape = TRUE, colorkey = FALSE,
                                par.settings = list(axis.line = list(col = 'transparent')),
                                xlab = list(label = paste(ynam, "Lag", sep = " "), cex = .8, rot = 20),
                                ylab = list(label = paste(xnam, "Lag", sep = " "), cex = .8, rot = -18),
                                zlab = list(label = NULL, cex.axis = 0.5),
                                col.regions = scales::viridis_pal(option = "viridis")(100))
    }
    else if(tolower(palette) %in% c("magma", "inferno", "cividis", "plasma")) {
        a <- a + ggplot2::scale_fill_gradientn(colours = scales::viridis_pal(option = palette)(50))

        # Create the lattice plot
        b <- lattice::wireframe(z ~ y * x, data = gdat, aspect = c(61/87, 0.4),
                                scales = list(cex = 0.5, arrows = FALSE),
                                drape = TRUE, colorkey = FALSE,
                                par.settings = list(axis.line = list(col = 'transparent')),
                                xlab = list(label = paste(ynam, "Lag", sep = " "), cex = .8, rot = 20),
                                ylab = list(label = paste(xnam, "Lag", sep = " "), cex = .8, rot = -18),
                                zlab = list(label = NULL, cex.axis = 0.5),
                                col.regions = scales::viridis_pal(option = palette)(100))
    }

    else if(palette %in% c("Spectral")) {
        a <- a + ggplot2::scale_fill_gradientn(colours = scales::brewer_pal(palette = palette)(11))

        # Create the lattice plot
        b <- lattice::wireframe(z ~ y * x, data = gdat, aspect = c(61/87, 0.4),
                                scales = list(cex = 0.5, arrows = FALSE),
                                drape = TRUE, colorkey = FALSE,
                                par.settings = list(axis.line = list(col = 'transparent')),
                                xlab = list(label = paste(ynam, "Lag", sep = " "), cex = .8, rot = 20),
                                ylab = list(label = paste(xnam, "Lag", sep = " "), cex = .8, rot = -18),
                                zlab = list(label = NULL, cex.axis = 0.5),
                                col.regions = scales::brewer_pal(palette = palette)(11))
    }
    else {
        stop("Invalid value for palette.")
    }

    # if(isTRUE(horizontal)) {
        output <- cowplot::plot_grid(b, a, nrow = 2, scale = c(2, 1))
    # }
    # else if(isFALSE(horizontal)) {
    #     output <- cowplot::plot_grid(b, a, ncol = 2, nrow = 1, scale = c(1, 1))
    # }
    # else {
    #     stop("horizontal must be either TRUE or FALSE")
    # }

    return(output)
}

#' Calculate the variogram data frame for a model
#'
#' @param model.obj An asreml model
#'
#' @return A data frame with the variogram for a model. The data frame contains the spatial coordinates (typically row and column), the $gamma$ for that position and the number of points with the separation.
#' @keywords internal
vario_df <- function(model.obj) {
    # The 'z' value for the variogram is the residuals
    # Need to be able to pull out the x/y from the model object

    dims <- unlist(strsplit(names(model.obj$R.param[1]), ":"))
    Row <- as.numeric(model.obj$mf[[dims[1]]])
    Column <- as.numeric(model.obj$mf[[dims[2]]])
    Resid <- residuals(model.obj)

    nrows <- max(Row)
    ncols <- max(Column)

    vario <- expand.grid(Row = 0:(nrows-1), Column = 0:(ncols-1))

    # Ignore the 0, 0 case (gamma=0, counted row*cols times)
    gammas <- rep(0, nrows*ncols)
    nps <- rep(nrows*ncols, nrows*ncols)

    for (index in 2:nrow(vario)) {
        i <- vario[index, 'Row']
        j <- vario[index, 'Column']

        gamma <- 0
        np <- 0
        for (val_index in 1:nrows) {
            # val <- vals[val_index, ]

            # Deliberate double-counting so that offset handling is easy
            # (so e.g. we compute distance from (1,1)->(2,3), and then again
            # later from (2,3)->(1,1)).
            for (offset in unique(list(c(i, j), c(-i, j), c(i, -j), c(-i, -j)))) {
                row <- Row[val_index] + offset[1]
                col <- Column[val_index] + offset[2]

                if (0 < row && row <= nrows && 0 < col && col <= ncols) {
                    other <- which(Row == row & Column == col)
                    gamma <- gamma + (Resid[val_index]-Resid[other])^2
                    np <- np + 1
                }
            }
        }
        # Since we double-counted precisely, halve to get the correct answer.
        np <- np / 2
        gamma <- gamma / 2

        if (np > 0) {
            gamma <- gamma / (2*np)
        }

        gammas[index] <- gamma
        nps[index] <- np
    }
    vario <- cbind(vario, data.frame(gamma = gammas, np = nps))
    colnames(vario) <- c(dims, "gamma", "np")
    class(vario) <- c("variogram", "data.frame")
    return(vario)
}


#' Compute arithmetic mean with confidence interval.
.get_mean_with_ci <- function (data, key, metric_name) {
    # Key not used but supplied by group map calls.

    # We work on run means for speed.
    runs <- get_run_means (data, metric_name)

    # Do not even try with one run.
    if (length (runs) > 1) {

        # Standard bootstrap computation.
        mean_boot <- boot (runs, function (d, i) mean (d [i]), R = REPLICATES)

        # The computations can fail so fall back to something if they do.
        mean_ci <- tryCatch (boot.ci (mean_boot, type = 'bca', conf = CONFIDENCE), error = function (e) NA)
        if (!is.null (mean_ci) && !any (is.na (mean_ci))) return (tibble (avg = mean_boot $ t0, lo = mean_ci $ bca [1,4], hi = mean_ci $ bca [1,5]))
        mean_ci <- tryCatch (boot.ci (mean_boot, type = 'basic', conf = CONFIDENCE), error = function (e) NA)
        if (!is.null (mean_ci) && !any (is.na (mean_ci))) return (tibble (avg = mean_boot $ t0, lo = mean_ci $ basic [1,4], hi = mean_ci $ basic [1,5]))
    }

    return (tibble (avg = mean (runs), lo = NA, hi = NA))
}

#' Compute arithmetic means with confidence intervals per given group.
comp_mean_with_ci <- function (data, metric_name) {
    return (data %>% group_modify (.get_mean_with_ci, metric_name))
}


s = 0

i = 1
repeat

{

  s = s + i
  i = i + 1

  if(i==9)
    break
}
cat("sum is", s)


#!/usr/bin/env Rscript

# Helper functions to annotate adducts
adductCalculator<-function(mz=NA,charge=NA,mode="pos",adduct=NA, primary = T)
{
  if(is.na(mz))stop("Provide the mz!")
  if(is.na(mode))stop("Provide the polarity!")

  Ion.name.pos=c("M+3H",
             "M+2H+Na",
             "M+H+2Na",
             "M+3Na",
             "M+2H",
             "M+H+NH4",
             "M+H+Na",
             "M+H+K",
             "M+ACN+2H",
             "M+2Na",
             "M+2ACN+2H",
             "M+3ACN+2H",
             "M+H",
             "M+NH4",
             "M+Na",
             "M+CH3OH+H",
             "M+K",
             "M+ACN+H",
             "M+2Na-H",
             "M+IsoProp+H",
             "M+ACN+Na",
             "M+2K-H",
             "M+DMSO+H",
             "M+2ACN+H",
             "M+IsoProp+Na+H",
             "2M+H",
             "2M+NH4",
             "2M+Na",
             "2M+K",
             "2M+ACN+H",
             "2M+ACN+Na")

  Ion.mass.pos=c(
    "M/3 + 1.007276",
    "M/3 + 8.334590",
    "M/3 + 15.7661904",
    "M/3 + 22.989218",
    "M/2 + 1.007276",
    "M/2 + 9.520550",
    "M/2 + 11.998247",
    "M/2 + 19.985217",
    "M/2 + 21.520550",
    "M/2 + 22.989218",
    "M/2 + 42.033823",
    "M/2 + 62.547097",
    "M + 1.007276",
    "M + 18.033823",
    "M + 22.989218",
    "M + 33.033489",
    "M + 38.963158",
    "M + 42.033823",
    "M + 44.971160",
    "M + 61.06534",
    "M + 64.015765",
    "M + 76.919040",
    "M + 79.02122",
    "M + 83.060370",
    "M + 84.05511",
    "2M + 1.007276",
    "2M + 18.033823",
    "2M + 22.989218",
    "2M + 38.963158",
    "2M + 42.033823",
    "2M + 64.015765")

  Charge.pos=c(
    "3+",
    "3+",
    "3+",
    "3+",
    "2+",
    "2+",
    "2+",
    "2+",
    "2+",
    "2+",
    "2+",
    "2+",
    "1+",
    "1+",
    "1+",
    "1+",
    "1+",
    "1+",
    "1+",
    "1+",
    "1+",
    "1+",
    "1+",
    "1+",
    "1+",
    "1+",
    "1+",
    "1+",
    "1+",
    "1+",
    "1+")

  Extended.pos<-c(T,T,T,T,T,T,T,T,T,T,T,T,F,F,F,T,F,T,T,T,T,T,T,T,T,T,T,T,T,T,T)

  Ion.name.neg<-c("M-3H",
                 "M-2H",
                 "M-H2O-H",
                 "M-H",
                 "M+Na-2H",
                 "M+Cl",
                 "M+K-2H",
                 "M+FA-H",
                 "M+Hac-H",
                 "M+Br",
                 "M+TFA-H",
                 "2M-H",
                 "2M+FA-H",
                 "2M+Hac-H",
                 "3M-H")

  Ion.mass.neg<-c("M/3 - 1.007276",
    "M/2 - 1.007276",
    "M - 19.01839",
    "M - 1.007276",
    "M + 20.974666",
    "M + 34.969402",
    "M + 36.948606",
    "M + 44.998201",
    "M + 59.013851",
    "M + 78.918885",
    "M + 112.985586",
    "2M - 1.007276",
    "2M + 44.998201",
    "2M + 59.013851",
    "3M - 1.007276")

  Charge.neg<-c("3-",
    "2-",
    "1-",
    "1-",
    "1-",
    "1-",
    "1-",
    "1-",
    "1-",
    "1-",
    "1-",
    "1-",
    "1-",
    "1-",
    "1-")

  Extended.neg<-c(T,T,T,F,F,F,F,T,T,T,T,T,T,T,T)

  adductsFile.pos<-data.frame(
	Ion.name=Ion.name.pos[!(Extended.pos & primary)],
	Ion.mass=Ion.mass.pos[!(Extended.pos & primary)],
	Charge=Charge.pos[!(Extended.pos & primary)])
  adductsFile.neg<-data.frame(
	Ion.name=Ion.name.neg[!(Extended.neg & primary)],
	Ion.mass=Ion.mass.neg[!(Extended.neg & primary)],
	Charge=Charge.neg[!(Extended.neg & primary)])

  if(tolower(mode)%in%c("pos","positive","p"))
  {
    # select the adducts (primary or extended)
    tmpAdduct<-adductsFile.pos
    if(!is.na(charge) & is.numeric(charge))
    {
      chargeSign<-NA
      chargeSign<-"+"
      tmpAdduct<-tmpAdduct[tmpAdduct[,"Charge"]==paste(charge,chargeSign,sep=""),]
    }
    if(!is.na(adduct))
    {

      tmpAdduct<-tmpAdduct[tmpAdduct[,"Ion.name"]%in%adduct,]
    }

    if(nrow(tmpAdduct)==0){
      warning("Not found! Reporting all possible adducts")
      tmpAdduct<-adductsFile.pos
    }
    signs<-sapply(str_extract(as.character(tmpAdduct[,"Ion.mass"]),"\\+|\\-"),function(x){x[[1]]})
    signs[signs=="+"]=-1
    signs[signs=="-"]=+1
    mzCoefficients<-
      str_extract(sapply(strsplit(as.character(tmpAdduct[,"Ion.mass"]),split = "\\+|\\-"),function(x){x[1]}),
                  "\\d+")

    mzCoefficients[is.na(mzCoefficients)]<-1
    mzCoefficients<-as.numeric(mzCoefficients)
    mzCoefficients[!grepl("/",as.character(tmpAdduct[,"Ion.mass"]),fixed=T)]<-
      1/ mzCoefficients[!grepl("/",as.character(tmpAdduct[,"Ion.mass"]),fixed=T)]

    result<- mz+  (as.numeric(signs)*
            as.numeric(sapply(strsplit(as.character(tmpAdduct[,"Ion.mass"]),"\\+|\\-"),function(x){x[[2]]})))

    result<- result*mzCoefficients
  }else if(tolower(mode)%in%c("neg","negative","n"))
  {
    tmpAdduct<-adductsFile.neg
    if(!is.na(charge) & is.numeric(charge))
    {
      chargeSign<-NA
     chargeSign<-"-"
      tmpAdduct<-tmpAdduct[tmpAdduct[,"Charge"]==paste(charge,chargeSign,sep=""),]
    }
    if(!is.na(adduct))
    {

      tmpAdduct<-tmpAdduct[tmpAdduct[,"Ion.name"]%in%adduct,]
    }

    if(nrow(tmpAdduct)==0){
      warning("Not found! Reporting all possible adducts")
      tmpAdduct<-adductsFile.neg
    }
    signs<-sapply(str_extract(as.character(tmpAdduct[,"Ion.mass"]),"\\+|\\-"),function(x){x[[1]]})
    signs[signs=="+"]=-1
    signs[signs=="-"]=+1
    mzCoefficients<-
      str_extract(sapply(strsplit(as.character(tmpAdduct[,"Ion.mass"]),split = "\\+|\\-"),function(x){x[1]}),
                  "\\d+")

    mzCoefficients[is.na(mzCoefficients)]<-1
    mzCoefficients<-as.numeric(mzCoefficients)
    mzCoefficients[!grepl("/",as.character(tmpAdduct[,"Ion.mass"]),fixed=T)]<-
      1/ mzCoefficients[!grepl("/",as.character(tmpAdduct[,"Ion.mass"]),fixed=T)]

    result<- mz+  (as.numeric(signs)*
                     as.numeric(sapply(strsplit(as.character(tmpAdduct[,"Ion.mass"]),"\\+|\\-"),function(x){x[[2]]})))
    result<- result*mzCoefficients

  }else
  {
    stop("Incorrect mode! Mode has to be either positive or negative!")
  }

return(data.frame(correctedMS=result,adductName=tmpAdduct[,"Ion.name"]))

}