#' Data Fit Assessment
#'
#' Calculates \eqn{R^2} values for correct and incorrect mean reaction times and RMSE values
#' for the proportions of correct answers per reaction times cuantiles.
#' @export
#' @param experim_dat Experimental data in the form produced by \code{\link{experimental_data_processing}}.
#' @param experim_dat Simulated data in the form produced by \code{\link{simulDat}}.
#' @param model_name Optional name for the model to distinguish to which data/model the
#' function was applied to. Default is 'NAME_UNDEFINED'.
#'
#' @return Prints a table with RMSE and \eqn{R^2} values.
fit_quality <- function(experim_dat, simul_dat, model_name='NAME_UNDEFINED'){

    experim_dat$cond <- "Data"

    dat <- rbind(experim_dat, simul_dat$data)

    ##########
    nbins <- 6

    dat_cuants <- quantile(experim_dat$rt, probs = seq(0, 1, length.out = nbins+1))
    dat_cuants[length(dat_cuants)] <- tail(dat_cuants, n=1)+0.1

    sim_cuants <- quantile(simul_dat$data$rt, probs = seq(0, 1, length.out = nbins+1))
    sim_cuants[length(sim_cuants)] <- tail(sim_cuants, n=1)+0.1

    dat$cuants[dat$cond=='Data'] <-  experim_dat %>%
        select(-cond) %>%
        split(experim_dat$suj) %>%  map(c('rt')) %>%
        map(function(x) {cut(x, breaks = dat_cuants, right=FALSE, na.rm = TRUE)}) %>%
        as_tibble() %>% tidyr::pivot_longer(cols=everything(), names_to = 'variable', values_to = 'value') %>%  .$value

    dat$cuants[dat$cond=='Sim'] <-  simul_dat$data %>%
        split(simul_dat$data$suj) %>%  map(c('rt')) %>%
        map(function(x) {cut(x, breaks = sim_cuants, right=FALSE, na.rm = TRUE)}) %>%
        as_tibble() %>% tidyr::pivot_longer(cols=everything(), names_to = 'variable', values_to = 'value') %>%  .$value

    dat$cuants <- unlist(dat$cuants) %>% as.factor()
    dat$cor <- as.numeric(levels(dat$cor))[dat$cor]
    #########

    a <- dat %>% group_by(cond, cuants) %>%
        dplyr::summarise(mean = mean(cor)) %>%
        spread(cond,mean) %>%
        select(-cuants) %>%
        rmse()

    b <- dat %>%
        group_by(suj, cond) %>%
        dplyr::summarize(mean_rt = mean(rt)) %>%
        spread(cond, mean_rt) %>%
        ungroup() %>%
        dplyr::summarise(rmse = rmse(.)) %>%
        pull()

    c <- dat %>%
        group_by(suj, cond) %>%
        dplyr::summarize(mean_rt = mean(rt)) %>%
        spread(cond, mean_rt) %>%
        ungroup() %>%
        dplyr::summarise(r2 = summary(lm(data=., Data~Sim))$adj.r.squared) %>%
        pull()

    d <- dat %>% group_by(cor, cond, suj) %>%
        dplyr::summarise(mean = mean(rt)) %>%
        ungroup() %>%
        spread(cond,mean) %>%
        nest(-cor) %>%
        mutate(data = map(data, ~ rmse(as.data.frame(.x)))) %>%
        mutate(data = unlist(data)) %>%
        pull(data)

    e <- dat %>% group_by(cor, cond, suj) %>%
        dplyr::summarise(mean = mean(rt)) %>%
        ungroup() %>%
        spread(cond,mean) %>%
        nest(-cor) %>%
        mutate(data = map(data, ~ summary(lm(data=as.data.frame(.x), Data~Sim))$adj.r.squared)) %>%
        mutate(data = unlist(data)) %>%
        pull(data)

    stats <- list("RMSE (Correct answers per RT cuantiles)" = a,
                  "RMSE (Overall Means of RTs aggr. by Subj.)" = b,
                  "R2 (Overall Means of RTs aggr. by Subj.)" = c,
                  "RMSE (For Incorrect Mean RTs aggr. by Subj.)" = d[1],
                  "RMSE (For Correct Mean RTs aggr. by Subj.)" = d[2],
                  "R2 (For Incorrect Mean RTs aggr. by Subj.)" = e[1],
                  "R2 (For Correct Mean RTs aggr. by Subj.)" = e[2]
    )

    print(data.frame(stats=unlist(stats)))

}


BoundModwpt<- function(x, wf="la8", n.levels=4, oldtargets)
{

  N <- length(x); storage.mode(N) <- "integer"
  J <- n.levels
  if(2^J > N) stop("Too many depth levels in the wavelet transform")

  dict <- wave.filter(wf)
  L <- dict$length
  storage.mode(L) <- "integer"
  ht <- dict$hpf/sqrt(2)
  storage.mode(ht) <- "double"
  gt <- dict$lpf/sqrt(2)
  storage.mode(gt) <- "double"

  targets=prepareTargets(oldtargets)#encoding targets using gray code

  y <- vector("list", sum(2^(1:J)))
  yn <- length(y)
  crystals1 <- rep(1:J, 2^(1:J))
  crystals2 <- unlist(apply(as.matrix(2^(1:J) - 1), 1, seq, from=0))
  names(y) <- paste("w", crystals1, ".", crystals2, sep="")

  W <- numeric(N);V<- numeric(N)
  storage.mode(W)  <- "double"; storage.mode(V)<- "double"
  for(j in 1:J) {
    index <- 0
    jj <- min((1:yn)[crystals1 == j])
    for(n in 0:(2^j / 2 - 1)) {
      index <- index + 1
      #should filter parent node j-1,n
      sc=shouldCompute(c(j-1,n),targets)
      if(sum(sc)>0){
        if(j > 1)
          x <- y[[(1:yn)[crystals1 == j-1][index]]]
        if(n %% 2 == 0) {
          z <- .C("pmodwpt", as.double(x), N, as.integer(j),as.integer(2), L, ht, gt,
                  W = W, V = V, PACKAGE="RHRV")[8:9]
          y[[jj + 2*n + 1]] <- z$W
          y[[jj + 2*n]] <- z$V
        }
        else {
          z <- .C("pmodwpt", as.double(x), N, as.integer(j),as.integer(2), L, ht, gt,
                  W = W, V = V, PACKAGE="RHRV")[8:9]
          y[[jj + 2*n]] <- z$W
          y[[jj + 2*n + 1 ]] <- z$V
        }
      }

    }
  }

  return(y)
}

shouldCompute2intCode<-function(sc){
  if (codeEquals(sc,c(1,0))){  intCode=0}
  if (codeEquals(sc,c(0,1))) {  intCode=1}
  if (codeEquals(sc,c(1,1))){intCode=2}
  return (intCode)
}

prepareTargets <- function(targets){
  newTargets=list()
  numberTargets=length(targets)/2;
  for (n in 1:numberTargets){
    newTargets[[n]]=getC(targets[[2*n-1]],targets[[2*n]])
  }
  return(newTargets)
}

shouldCompute <- function(node,targets){

  #compute should be c(0,0)(none), c(1,0)(left),c(0,1)(right), c(1,1) (both)
  compute=c(0,0)
  # node c(0,0) must be computed
  if (nodeEquals(node,c(0,0))){
    len=length(targets)
    for (j in 1:len){
      if (targets[[j]][1]==0){
        compute[[1]]=1;
      }else{
        compute[[2]]=1;
      }
      if (codeEquals(compute,c(1,1))) break
    }

  }else{
    len=length(targets)
    nodeCode=getC(node[[1]],node[[2]])
    codeLen=length(nodeCode)
    compute=c(0,0)
    for (j in 1:len){
      if (length(targets[[j]])>codeLen){
        equals=codeEquals(nodeCode,targets[[j]][1:codeLen])
        if (equals){
          ntc=nodeToCompute(targets[[j]][[codeLen+1]])
          compute=compute+ntc
          compute=compute/max(compute)## avoid (2,0) or (0,2)
          if (codeEquals(compute,c(1,1))) break

        }
      }
    }
    return (compute)
  }

  return(compute);
}

nodeToCompute<-function(number){
  if (number==1){##high pass filter
    return (c(0,1))
  }else{## low pass filter
    return (c(1,0))
  }
}

codeEquals<-function(c1,c2){
  if (length(c1)!=length(c2))
  {
    return (FALSE)
  }else{
    return (prod(c1==c2)==1)
  }
}

nodeEquals<-function(n1,n2){
  return ((n1[1]==n2[1])&&(n1[2]==n2[2]));

}


#' Creates a new ARTMAP network.
#'
#' This function creates a new ARTMAP network with the specified number
#' features and classes. The network is created to expand the number of
#' categories as needed. The vigilance parameter defaults to 0.75.
#' The initial number of categories is set to 1. The maximum number
#' of categories defaults to 100. The bias defaults to 0.000001, the
#' number of epochs defaults to 100, and the learning rate defaults
#' to 1.0 (fast-learning).
#' @title ARTMAP_Create_Network
#' @param numFeatures Number of features that the network expects of the input data. Must be a positive integer.
#' @param numClasses Number of classes that exist for the supervisory signal. Must be a positive integer greater than 1.
#' @param maxNumCategories Maximum number of categories that can be activated during the training process. Defaults to 1000.
#' @param vigilance Vigilance parameter that defines the minimum similarity allowed between the input pattern and the weights. Defaults to 0.75.
#' @param bias Constant that is used to differentiate between very similar category activation values. Defaults to 0.000001
#' @param numEpochs Maximum number of training iterations allowed. Defaults to 100.
#' @param learningRate Learning rate. Defaults to 1.
#' @return Structure that holds all of the information for the network. It must be passed into both ARTMAP_LEARN() and
#' ARTMAP_CLASSIFY(). The fields of this structure are numFeatures, numCategories, maxNumCategories, numClasses, weight
#' (an initialized weight vector with the right dimensions), mapField (an initialized map field with the right dimensions),
#' vigilance, bias, numEpochs, neededEpochs (the number of epochs needed in the last training) and learningRate.
#' @export
ARTMAP_Create_Network=function(numFeatures, numClasses,
                               maxNumCategories = 1000, vigilance=0.75,
                                bias=0.000001, numEpochs=100, learningRate=1.0)
{
  if(is.null(numFeatures) || is.null(numClasses))
  {
    stop('You must specify the number of features and the number of classes.');
  }

  # Check the ranges of the input parameters.
  numFeatures = round(numFeatures);
  if(numFeatures < 1)
  {
    stop('The number of features must be a positive integer.');
  }
  numClasses = round(numClasses);
  if(numClasses < 2)
  {
    stop('The number of classes must be a positive integer greater than 1.');
  }

  # Create and initialize the weight matrix.
  weight = matrix(1,0,numFeatures);

  # Create and initialize the map field.
  mapField = matrix(0,0,1);

  # Create the structure and return.
  artmap_network = list("numFeatures"= numFeatures, "numCategories" = 0, "maxNumCategories" = maxNumCategories,
                          "numClasses"=numClasses, "weight"=weight, "mapField"=mapField,
                          "vigilance"=vigilance, "bias"=bias, "numEpochs"=numEpochs,
                          "neededEpochs"=0,"learningRate"=learningRate);

  return(artmap_network);
}


 E04MTJ Example Program Results

++++++++++ Use the Primal-Dual algorithm ++++++++++

 ----------------------------------------------
  E04MT, Interior point method for LP problems
 ----------------------------------------------

 Original Problem Statistics

   Number of variables          7
   Number of constraints        7
   Free variables               0
   Number of nonzeros          41

 Presolved Problem Statistics

   Number of variables         13
   Number of constraints        7
   Free variables               0
   Number of nonzeros          47

 ------------------------------------------------------------------------------
  it|    pobj    |    dobj    |  optim  |  feas   |  compl  |   mu   | mcc | I
 ------------------------------------------------------------------------------
   0 -7.86591E-02  1.71637E-02  1.27E+00  1.06E+00  8.89E-02  1.5E-01
   1  5.74135E-03 -2.24369E-02  6.11E-16  1.75E-01  2.25E-02  2.8E-02   0
   2  1.96803E-02  1.37067E-02  5.06E-16  2.28E-02  2.91E-03  3.4E-03   0
   3  2.15232E-02  1.96162E-02  7.00E-15  9.24E-03  1.44E-03  1.7E-03   0
   4  2.30321E-02  2.28676E-02  1.15E-15  2.21E-03  2.97E-04  3.4E-04   0
   5  2.35658E-02  2.35803E-02  1.32E-15  1.02E-04  8.41E-06  9.6E-06   0
   6  2.35965E-02  2.35965E-02  1.64E-15  7.02E-08  6.35E-09  7.2E-09   0
Iteration 7
     monit() reports good approximate solution (tol = 1.20E-08):
   7  2.35965E-02  2.35965E-02  1.35E-15  3.52E-11  3.18E-12  3.6E-12   0
 ------------------------------------------------------------------------------
 Status: converged, an optimal solution found
 ------------------------------------------------------------------------------
 Final primal objective value         2.359648E-02
 Final dual objective value           2.359648E-02
 Absolute primal infeasibility        4.168797E-15
 Relative primal infeasibility        1.350467E-15
 Absolute dual infeasibility          5.084353E-11
 Relative dual infeasibility          3.518607E-11
 Absolute complementarity gap         2.685778E-11
 Relative complementarity gap         3.175366E-12
 Iterations                                      7

 Primal variables:
   idx   Lower bound        Value      Upper bound
     1  -1.00000E-02   -1.00000E-02    1.00000E-02
     2  -1.00000E-01   -1.00000E-01    1.50000E-01
     3  -1.00000E-02    3.00000E-02    3.00000E-02
     4  -4.00000E-02    2.00000E-02    2.00000E-02
     5  -1.00000E-01   -6.74853E-02    5.00000E-02
     6  -1.00000E-02   -2.28013E-03         inf
     7  -1.00000E-02   -2.34528E-04         inf

 Box bounds dual variables:
   idx   Lower bound        Value      Upper bound        Value
     1  -1.00000E-02    3.30098E-01    1.00000E-02    0.00000E+00
     2  -1.00000E-01    1.43844E-02    1.50000E-01    0.00000E+00
     3  -1.00000E-02    0.00000E+00    3.00000E-02    9.09967E-02
     4  -4.00000E-02    0.00000E+00    2.00000E-02    7.66124E-02
     5  -1.00000E-01    3.51391E-11    5.00000E-02    0.00000E+00
     6  -1.00000E-02    3.42902E-11         inf       0.00000E+00
     7  -1.00000E-02    8.61040E-12         inf       0.00000E+00

 Constraints dual variables:
   idx   Lower bound        Value      Upper bound        Value
     1  -1.30000E-01    0.00000E+00   -1.30000E-01    1.43111E+00
     2       -inf       0.00000E+00   -4.90000E-03    4.00339E-10
     3       -inf       0.00000E+00   -6.40000E-03    1.54305E-08
     4       -inf       0.00000E+00   -3.70000E-03    3.80136E-10
     5       -inf       0.00000E+00   -1.20000E-03    4.72629E-11
     6  -9.92000E-02    1.50098E+00         inf       0.00000E+00
     7  -3.00000E-03    1.51661E+00    2.00000E-03    0.00000E+00

++++++++++ Use the Self-Dual algorithm ++++++++++

 ----------------------------------------------
  E04MT, Interior point method for LP problems
 ----------------------------------------------

 Original Problem Statistics

   Number of variables          7
   Number of constraints        7
   Free variables               0
   Number of nonzeros          41

 Presolved Problem Statistics

   Number of variables         13
   Number of constraints        7
   Free variables               0
   Number of nonzeros          47

 ------------------------------------------------------------------------------
  it|    pobj    |    dobj    |  p.inf  |  d.inf  |  d.gap  |   tau  | mcc | I
 ------------------------------------------------------------------------------
   0 -6.39941E-01  4.94000E-02  1.07E+01  2.69E+00  5.54E+00  1.0E+00
   1 -8.56025E-02 -1.26938E-02  2.07E-01  2.07E-01  2.07E-01  1.7E+00   0
   2  4.09196E-03  1.24373E-02  4.00E-02  4.00E-02  4.00E-02  2.8E+00   0
   3  1.92404E-02  2.03658E-02  6.64E-03  6.64E-03  6.64E-03  3.2E+00   1
   4  1.99631E-02  2.07574E-02  3.23E-03  3.23E-03  3.23E-03  2.3E+00   1
   5  2.03834E-02  2.11141E-02  1.68E-03  1.68E-03  1.68E-03  1.4E+00   0
   6  2.22419E-02  2.25057E-02  5.73E-04  5.73E-04  5.73E-04  1.4E+00   1
   7  2.35051E-02  2.35294E-02  6.58E-05  6.58E-05  6.58E-05  1.4E+00   6
   8  2.35936E-02  2.35941E-02  1.19E-06  1.19E-06  1.19E-06  1.4E+00   0
Iteration 9
     monit() reports good approximate solution (tol = 1.20E-08):
   9  2.35965E-02  2.35965E-02  5.37E-10  5.37E-10  5.37E-10  1.4E+00   0
Iteration 10
     monit() reports good approximate solution (tol = 1.20E-08):
  10  2.35965E-02  2.35965E-02  2.68E-13  2.68E-13  2.68E-13  1.4E+00   0
 ------------------------------------------------------------------------------
 Status: converged, an optimal solution found
 ------------------------------------------------------------------------------
 Final primal objective value         2.359648E-02
 Final dual objective value           2.359648E-02
 Absolute primal infeasibility        2.853383E-12
 Relative primal infeasibility        2.677658E-13
 Absolute dual infeasibility          1.485749E-12
 Relative dual infeasibility          2.679654E-13
 Absolute complementarity gap         7.228861E-13
 Relative complementarity gap         2.683908E-13
 Iterations                                     10

 Primal variables:
   idx   Lower bound        Value      Upper bound
     1  -1.00000E-02   -1.00000E-02    1.00000E-02
     2  -1.00000E-01   -1.00000E-01    1.50000E-01
     3  -1.00000E-02    3.00000E-02    3.00000E-02
     4  -4.00000E-02    2.00000E-02    2.00000E-02
     5  -1.00000E-01   -6.74853E-02    5.00000E-02
     6  -1.00000E-02   -2.28013E-03         inf
     7  -1.00000E-02   -2.34528E-04         inf

 Box bounds dual variables:
   idx   Lower bound        Value      Upper bound        Value
     1  -1.00000E-02    3.30098E-01    1.00000E-02    0.00000E+00
     2  -1.00000E-01    1.43844E-02    1.50000E-01    0.00000E+00
     3  -1.00000E-02    0.00000E+00    3.00000E-02    9.09967E-02
     4  -4.00000E-02    0.00000E+00    2.00000E-02    7.66124E-02
     5  -1.00000E-01    3.66960E-12    5.00000E-02    0.00000E+00
     6  -1.00000E-02    2.47652E-11         inf       0.00000E+00
     7  -1.00000E-02    7.82645E-13         inf       0.00000E+00

 Constraints dual variables:
   idx   Lower bound        Value      Upper bound        Value
     1  -1.30000E-01    0.00000E+00   -1.30000E-01    1.43111E+00
     2       -inf       0.00000E+00   -4.90000E-03    1.07904E-10
     3       -inf       0.00000E+00   -6.40000E-03    1.14799E-09
     4       -inf       0.00000E+00   -3.70000E-03    4.09190E-12
     5       -inf       0.00000E+00   -1.20000E-03    1.52421E-12
     6  -9.92000E-02    1.50098E+00         inf       0.00000E+00
     7  -3.00000E-03    1.51661E+00    2.00000E-03    0.00000E+00


############################################################################################################
############################################################################################################
## Loading necessary libraries
############################################################################################################
############################################################################################################

library(ggplot2)
library(gridExtra)
library(lme4)
library(lmerTest)
library(robustlmm)
library(MCMCglmm)
library(xtable)
library(shinyBS)
library(nlme)
library(combinat)

############################################################################################################
############################################################################################################
## Global functions
############################################################################################################
############################################################################################################

ggCaterpillar=function(re, QQ=TRUE, likeDotplot=TRUE)
{
  require(ggplot2)
  f = function(x) {
    pv=attr(x, "postVar")
    cols=1:(dim(pv)[1])
    se=unlist(lapply(cols, function(i) sqrt(pv[i, i, ])))
    ord=unlist(lapply(x, order)) + rep((0:(ncol(x) - 1)) * nrow(x), each=nrow(x))
    pDf=data.frame(y=unlist(x)[ord],ci=1.96*se[ord],nQQ=rep(qnorm(ppoints(nrow(x))), ncol(x)),
                   ID=factor(rep(rownames(x), ncol(x))[ord], levels=rownames(x)[ord]), ind=gl(ncol(x), nrow(x), labels=names(x)))

    if(QQ) {
      p = ggplot(pDf, aes(nQQ, y))
      p = p + facet_wrap(~ ind, scales="free")
      p = p + xlab("Standard normal quantiles") + ylab("Random effect quantiles")
    } else {
      p = ggplot(pDf, aes(ID, y)) + coord_flip() + ggtitle("Caterpillar plot showing the group-level\nerror terms for each random effect")
      if(likeDotplot) {
        p = p + facet_wrap(~ ind)
      } else {
        p = p + facet_grid(ind ~ ., scales="free_y")
      }
      p = p + xlab("Groups") + ylab("Deviations")
    }

    p = p + theme_bw() + theme(legend.position="none") + geom_hline(yintercept=0) +
      geom_errorbar(aes(ymin=y-ci, ymax=y+ci), width=0, colour="black") + geom_point(aes(size=2), shape=1)

    return(p)
  }
  lapply(re, f)
}

############################################################################################################
############################################################################################################
## Loading sample data sets
############################################################################################################
############################################################################################################

kentucky=read.csv("data/KentuckyMathScores.csv")
music=read.csv("data/musicdata.csv")

############################################################################################################
############################################################################################################
## Shiny server
############################################################################################################
############################################################################################################

shinyServer(function(input, output, session) {

  ############################################################################################################
  ############################################################################################################
  ## Upload Data Panel
  ############################################################################################################
  ############################################################################################################

  data = reactive({
    if(is.null(input$file) & !input$usesample)
    {
      return(NULL)
    } else if(!is.null(input$file) & !input$usesample)
    {
      file = read.csv(input$file$datapath, header=input$header, sep=input$sep, quote=input$quote)
      return(file)
    } else if(input$usesample)
    {
      return(music)
    }
    })

  output$datatable = renderDataTable({
    if(!input$usesample)
    {
      return(data())
    }
  })

  output$sampledata = renderDataTable({
    data()
  })

  output$selectresponse = renderUI({
    selectInput("response","Response variable:",choices=names(data()),selectize=TRUE)
  })

  output$selectlevel1fixedpred = renderUI({
    selectInput("level1.fixed.pred","Level 1 fixed predictors:",choices=names(data()),multiple=TRUE,selectize=TRUE)
  })

  output$selectlevel1randpred = renderUI({
    selectInput("level1.rand.pred","Level 1 random predictors:",choices=names(data()),multiple=TRUE,selectize=TRUE)
  })

  output$selectlevel2id = renderUI({
    selectInput("level2.id","Level 2 observational unit:",choices=names(data()),selectize=TRUE)
  })

  output$selectlevel2pred = renderUI({
    selectInput("level2.pred","Level 2 predictors",choices=names(data()),multiple=TRUE,selectize=TRUE)
  })

  ############################################################################################################
  ############################################################################################################
  ## HLM Study Panel
  ############################################################################################################
  ############################################################################################################

  ############################################################################################################
  ############################################################################################################
  ## Varying-Intercept Only
  ############################################################################################################
  ############################################################################################################

  output$selectresponse1 = renderUI({
    selectInput("response1","Response variable:",choices=names(data()),selectize=TRUE)
  })

  output$selectlevel2id1 = renderUI({
    selectInput("level2.id1","Level 2 observational unit:",choices=names(data()),selectize=TRUE)
  })

  dat.nopred = reactive({
     if(!input$usesample)
     {
       dat = data.frame(response = data()[,paste(input$response1)],
                        level2.id = factor(data()[,paste(input$level2.id1)]))
       return(dat)
     } else if (input$usesample)
    {
      dat = data.frame(response = data()$negative_affect,
                       level2.id = factor(data()$id))
    }
  })

  output$datatab1 = renderDataTable({
    if(!input$usesample) return(dat.nopred())
  })

  label = reactive({
    if(!input$usesample)
    {
      return(c(paste(input$response1),paste(input$level2.id1)))
    }else if(input$usesample)
    {
      return(c("negative affect","musicians"))
    }
  })

  ############################################################################################################
  ## Pooled Regression 1
  ############################################################################################################

  output$poolednopredgraph = renderPlot({
    ggplot(data=dat.nopred()) + geom_histogram(aes(x=response),fill="white",color="navy") +
      xlab(paste(label()[1])) + ylab("Frequency") + ggtitle(paste("Histogram of",label()[1],"with pooled mean imposed")) +
      theme_bw() + geom_vline(aes(xintercept=mean(response)),linetype="dashed",color="navy")
  })

  pooled.mod.nopred = reactive({
    lm(response~1, data=dat.nopred())
  })

  output$pooled.nopred.info = renderUI({
    mean = round(mean(dat.nopred()$response,na.rm=TRUE),digits=3)
    p(HTML("
 y&#773all = pooled mean ="), code(paste0(mean)))
  })

  ############################################################################################################
  ## Unpooled Regression 1
  ############################################################################################################

  output$unpooled.nopred.graph = renderPlot({
    ggplot(data=dat.nopred(),aes(x=level2.id,y=response)) + geom_hline(aes(yintercept=mean(response)),linetype="dashed",color="navy") +
      geom_boxplot(fill="white",color="navy") + xlab(paste(label()[2])) + ylab(paste(label()[1])) +
      ggtitle(paste("Boxplots of",label()[1],"by",label()[2],"with pooled and unpooled means imposed")) +
      theme_bw() + theme(axis.text.x=element_text(angle=90)) +
      stat_summary(fun.y="mean", geom="point", shape=1, size=3, fill="white", color="gold2")
  })

  unpooled.mod.nopred = reactive({
    lm(response~level2.id+0, data=dat.nopred())
  })

  output$unpooled.mod.summary = renderTable({
    anova(unpooled.mod.nopred())
  })

  output$unpooled.nopred.info = renderDataTable({
    mean.dat = data.frame(tapply(dat.nopred()$response,dat.nopred()$level2.id,mean,na.rm=TRUE))
    mean = round(as.numeric(mean.dat[,1]),digits=3)
    var = round(as.numeric(tapply(dat.nopred()$response,dat.nopred()$level2.id,var,na.rm=TRUE)))
    n = as.numeric(tapply(dat.nopred()$response,dat.nopred()$level2.id, function(x) length(!is.na(x))))

    dat = data.frame(ID=rownames(mean.dat),Mean=mean,Variance=var,n=n)
    return(dat)
  })

  ############################################################################################################
  ## Hierarchical Linear Model 1
  ############################################################################################################

  hlm.mod.nopred = reactive({
    lmer(response ~ 1 + (1|level2.id), data=dat.nopred(),
         control=lmerControl(optCtrl=list(maxfun=50000)))
  })

  output$hlm.nopred1 = renderPrint({
    summary(hlm.mod.nopred())
  })

  coefs = reactive({
    data.frame(coef=coef(hlm.mod.nopred())$level2.id,ran=ranef(hlm.mod.nopred())$level2.id)
  })

  output$hlm.nopred2 = renderTable({
    conf = confint(hlm.mod.nopred(),method="boot")
    colnames(conf) = c("95% Lower Bound","95% Upper Bound")
    rownames(conf) = c("Between-group SD","Within-group SD","Intercept")
    return(conf)
  })

  output$intraclass1 = renderUI({
    withMathJax("$$\\hat{\\rho} = \\frac{\\hat{\\sigma}_\\alpha^{2}}{\\hat{\\sigma}_\\alpha^{2}+\\hat{\\sigma}_y^{2}}$$")
  })

  variances = reactive({
    c(round(as.numeric(attr(VarCorr(hlm.mod.nopred())[[1]],"stddev")^2),digits=3),
      round(as.numeric(attr(VarCorr(hlm.mod.nopred()),"sc")^2),digits=3))
  })

  output$calcintraclass1 = renderUI({
    icc = round(variances()[1]/sum(variances()),digits=3)

    p("ICC =",code(paste(variances()[1]))," / (",
      code(paste(variances()[1])),"+",
      code(paste(variances()[2])),") = ",
      code(paste(icc)))
  })

  output$ratio1 = renderUI({
    withMathJax("$$Variance\\,ratio = \\frac{\\hat{\\sigma}_y^{2}}{\\hat{\\sigma}_\\alpha^{2}}$$")
  })

  output$calcratiovariances1 = renderUI({
    ratio = round(variances()[2]/variances()[1],digits=3)

    p("Ratio =",code(paste(variances()[2]))," / ",
      code(paste(variances()[1])), " = ", code(paste(ratio)))
  })

  output$hlmdist1 = renderPlot({
    ggplot(data=coefs(),aes(x=X.Intercept.)) + geom_histogram(color="seagreen2",fill="white") +
      xlab("HLM means") + ylab("Frequency") + ggtitle("Histogram of HLM means") + theme_bw()
  })

  output$hlmdisterror1 = renderPlot({
    ggplot(data=coefs(),aes(x=X.Intercept..1)) + geom_histogram(color="salmon",fill="white") +
      xlab("Group-level intercept errors") + ylab("Frequency") +
      ggtitle("Histogram of group-level intercept errors") + theme_bw()
  })

  output$hlmdistinfo1 = renderUI({
    p(HTML("

"),withMathJax("\\(\\hat{\\mu}_{\\alpha} =\\)"),code(paste(round(as.numeric(fixef(hlm.mod.nopred())),digits=3))),
      HTML("

"),withMathJax("\\(\\hat{\\sigma}_\\alpha^{2} =\\)"),code(paste(variances()[1])),HTML("
"))
  })

  output$hlmtable = renderDataTable({
    mean.dat = data.frame(tapply(dat.nopred()$response,dat.nopred()$level2.id,mean,na.rm=TRUE))
    n = as.numeric(tapply(dat.nopred()$response,dat.nopred()$level2.id, function(x) length(!is.na(x))))
    rancoefs = round(coefs()$X.Intercept..1,digits=3)
    coefs = round(coefs()$X.Intercept.,digits=3)

    dat = matrix(c(rownames(mean.dat),coefs,rancoefs,n),ncol=4)
    colnames(dat) = c("Group","Estimated intercept","Group-level error","Sample size")
    return(dat)
  })

  output$ggcat1 = renderPlot({
    ggCaterpillar(ranef(hlm.mod.nopred(), condVar=TRUE), QQ=FALSE)
  })

  ############################################################################################################
  ## Comparison of Methods 1
  ############################################################################################################

  output$shrinkplot = renderPlot({
    samplesize = tapply(dat.nopred()$response,dat.nopred()$level2.id, function(x) length(x))
    ind = order(samplesize)
    hlmmean = as.vector(unlist(coef(hlm.mod.nopred())))
    hlmmean = hlmmean[ind]
    hlmse = sqrt(attr(ranef(hlm.mod.nopred(), condVar = TRUE)[[1]], "postVar")[1, , ])/samplesize
    hlmse = hlmse[ind]
    unpooledse = tapply(dat.nopred()$response,dat.nopred()$level2.id, function(x) sd(x,na.rm=TRUE)/sqrt(length(x)))
    unpooledse = unpooledse[ind]
    unpooledmean = tapply(dat.nopred()$response,dat.nopred()$level2.id,mean,na.rm=TRUE)
    unpooledmean = unpooledmean[ind]
    groups = levels(factor(dat.nopred()$level2.id))
    groups = groups[ind]
    samplesize1 = samplesize[ind]
    dat = data.frame(hlmse=hlmse,hlmmean=hlmmean,unpooledse=unpooledse,unpooledmean=unpooledmean,groups=groups,
                     n=samplesize1)
    mean = data.frame(x=mean(dat.nopred()$response),na.rm=TRUE)

    g1=ggplot() + theme_bw() + geom_point(data=dat,aes(x=groups,y=unpooledmean,color="Unpooled"),shape=1) +
      geom_point(data=dat,aes(x=groups,y=hlmmean,color="HLM"),shape=16) +
      geom_hline(data=mean,aes(yintercept=x,color="Pooled"),linetype="dashed") +
      geom_errorbar(data=dat,aes(x=groups,y=unpooledmean,ymin=unpooledmean-1.96*unpooledse,ymax=unpooledmean+1.96*unpooledse,color="Unpooled")) +
      geom_errorbar(data=dat,aes(x=groups,y=hlmmean,ymin=hlmmean-1.96*hlmse,ymax=hlmmean+1.96*hlmse,color="HLM")) +
      ylab("Mean") + xlab("Group") + ggtitle("Shrinkage plot: 95% CIs using unpooled method and HLM with pooled mean imposed") +
      scale_color_manual(name="Modelling method",values=c("HLM"="tomato","Unpooled"="gold2","Pooled"="navy")) + theme(legend.position="bottom")

    g2=ggplot() + theme_bw() + geom_histogram(data=dat,aes(x=unpooledmean,y=..density..,fill="Unpooled"),alpha=.5) +
      geom_histogram(data=dat,aes(x=hlmmean,y=..density..,fill="HLM"),alpha=.5) +
      geom_density(data=dat,aes(x=unpooledmean),color="gold2") +
      geom_density(data=dat,aes(x=hlmmean),color="tomato") +
      geom_vline(data=mean,aes(xintercept=x,fill="Pooled"),color="navy",linetype="dashed") +
      scale_fill_manual(name="Modelling method",values=c("HLM"="tomato","Unpooled"="gold2","Pooled"="navy")) +
      theme(legend.position="bottom") + ylab("Relative frequency") + xlab("Mean") + ggtitle("Histogram of unpooled and HLM means with pooled mean imposed")

    g3 = arrangeGrob(g1,g2,nrow=1)
    return(g3)
  },height=500)

  ############################################################################################################
  ## Bayesian Hierarchical Linear Model 1
  ############################################################################################################

  hlmbayes1 = reactive({
    MCMCglmm(response ~ 1, random=~level2.id, data=dat.nopred(), verbose=FALSE, pr=TRUE)
  })

  output$hlm.bayes1 = renderPrint({
    summary(hlmbayes1())
  })

  output$postdist = renderPlot({
    group.error = data.frame(x=as.vector(hlmbayes1()$VCV[,1]))
    indiv.error = data.frame(x=as.vector(hlmbayes1()$VCV[,2]))
    int = data.frame(x=as.vector(hlmbayes1()$Sol[,1]))

    group.plot = ggplot(data=group.error) + geom_density(aes(x=x)) + theme_bw() + ylab("") + xlab("Between-group var in response") +
                   geom_vline(aes(xintercept=mean(x)),color="seagreen2",size=1)
    group.trace = ggplot(data=group.error) + geom_line(aes(x=1:length(x),y=x)) + theme_bw() +
                    geom_hline(aes(yintercept=mean(x)),color="seagreen2",size=1) +
                    xlab("Iteration") + ylab("Between-group\nvar in response")
    indiv.plot = ggplot(data=indiv.error) + geom_density(aes(x=x)) + theme_bw() + ylab("") + xlab("Within-group var in response") +
                   geom_vline(aes(xintercept=mean(x)),color="seagreen2",size=1)
    indiv.trace = ggplot(data=indiv.error) + geom_line(aes(x=1:length(x),y=x)) + theme_bw() +
                    geom_hline(aes(yintercept=mean(x)),color="seagreen2",size=1) +
                    xlab("Iteration") + ylab("Within-group\nvar in response")
    int.plot = ggplot(data=int) + geom_density(aes(x=x)) + theme_bw() + ylab("") + xlab("Intercept") +
                 geom_vline(aes(xintercept=mean(x)),color="seagreen2",size=1)
    int.trace = ggplot(data=int) + geom_line(aes(x=1:length(x),y=x)) + theme_bw() +
                  geom_hline(aes(yintercept=mean(x)),color="seagreen2",size=1) +
                  xlab("Iteration") + ylab("Intercept")
    graph = arrangeGrob(group.trace,group.plot,indiv.trace,indiv.plot,int.trace,int.plot,nrow=3,ncol=2,
                        main=textGrob("Traceplots and Posterior Distributions",gp=gpar(fontsize=15)))
    graph
  },height=500)

  ############################################################################################################
  ############################################################################################################
  ## Varying-intercept and varying-slope
  ############################################################################################################
  ############################################################################################################

  output$selectresponse2 = renderUI({
    selectInput("response2","Response variable:",choices=names(data()),selectize=TRUE)
  })

  output$selectlevel2id2 = renderUI({
    selectInput("level2.id2","Level 2 observational unit:",choices=names(data()),selectize=TRUE)
  })

  output$selectlevel1pred1 = renderUI({
    selectInput("level1.pred1","Level 1 predictor:",choices=names(data()),selectize=TRUE)
  })

  dat.pred1 = reactive({
    if(!input$usesample)
    {
      dat = data.frame(predictor = data()[,paste(input$level1.pred1)],
                       response = data()[,paste(input$response2)],
                       level2.id = factor(data()[,paste(input$level2.id2)]))
    } else if (input$usesample)
    {
      dat = data.frame(predictor = data()$previous,
                       response = data()$negative_affect,
                       level2.id = factor(data()$id))
      return(dat)
    }
  })

  label1 = reactive({
    if(!input$usesample)
    {
      return(c(paste(input$response2),paste(input$level1.pred1),paste(input$level2.id2)))
    }else if(input$usesample)
    {
      return(c("negative affect","previous","musicians"))
    }
  })

  ############################################################################################################
  ## Pooled Regression 2
  ############################################################################################################

  output$pooled.graph = renderPlot({
      ggplot(data=dat.pred1(), aes(x=predictor,y=response)) +
        geom_point(position = "jitter", size=3, shape=1) +
        geom_smooth(method="lm",color="navy",se=FALSE) + xlab(paste(label1()[2])) + ylab(paste(label1()[1])) +
        ggtitle(paste("Scatterplot of",label1()[1],"versus", label1()[2],"with pooled line imposed")) + theme_bw()
  })

  pooled.mod = reactive({
      lm(response~predictor, data=dat.pred1())
  })

  output$pooled.output = renderTable({
    return(summary(pooled.mod()))
  })

  output$pooled.info = renderText({
    paste0("R-squared=",round(summary(pooled.mod())$r.squared,digits=3),"; Residual df=",pooled.mod()$df.residual)
  })

  ############################################################################################################
  ## Unpooled Regression 2
  ############################################################################################################

  output$unpooled.graph = renderPlot({
      ggplot(data=dat.pred1(), aes(x=predictor,y=response)) + geom_point(position = "jitter", size=3, shape=1) +
        geom_smooth(aes(group=level2.id),method="lm",color="gold2",se=FALSE) +
        geom_smooth(aes(group=1),method="lm",color="navy",se=FALSE) + xlab(paste(label1()[2])) + ylab(paste(label1()[1])) +
        ggtitle(paste("Scatterplot of",label1()[1],"versus",label1()[2],"with unpooled lines imposed")) + theme_bw()
  })

  unpooled.mod = reactive({
    lm(response~0+predictor+factor(level2.id)+factor(level2.id)*predictor, data=dat.pred1())
  })

  output$unpooled.output = renderTable({
    return(anova(unpooled.mod()))
  })

  output$unpooled.info = renderText({
    paste0("R-squared=",round(sum(anova(unpooled.mod())$Sum[1:3])/sum(anova(unpooled.mod())$Sum),digits=3))
  })

  slope.intercept = reactive({
    id = unique(dat.pred1()$level2.id)
    slope=NULL
    intercept=NULL
    n=NULL
    i=0
    while(i<length(id))
    {
      i=i+1
      sub.dat = dat.pred1()[which(dat.pred1()$level2.id==id[i]),]
      n[i] = dim(sub.dat[which(complete.cases(sub.dat)),])[1]
      mod = lm(response~predictor,data=sub.dat)
      intercept[i] = mod$coefficients[1]
      slope[i] = mod$coefficients[2]
    }
    intercept = round(intercept,digits=3)
    slope = round(slope,digits=3)

    return(data.frame(intercept=intercept,slope=slope,n=n))
  })

  output$unpooled.est = renderDataTable({
    mat = matrix(c(unique(dat.pred1()$level2.id),slope.intercept()$intercept,
                   slope.intercept()$slope,slope.intercept()$n),ncol=4)
    colnames(mat) = c("ID","Intercept","Slope","n")
    return(mat)
  })

  ############################################################################################################
  ## Hierarchical Linear Model 2
  ############################################################################################################

  hlm = reactive({
    return(lmer(response ~ predictor + (predictor|level2.id), data=dat.pred1(),
                control=lmerControl(optCtrl=list(maxfun=50000))))
  })

  output$hlm.summary = renderPrint({
    summary(hlm())
  })

  output$hlm.confint1 = renderTable({
    conf = confint(hlm(),method="boot")
    colnames(conf) = c("95% Lower Bound","95% Upper Bound")
    rownames(conf) = c("Between-group Intercept SD","Correlation","Between-group Slope SD","Within-group SD",
                       "Intercept","Slope")
    return(conf)
  })

  output$hlm.graph = renderPlot({
    sub.dat = dat.pred1()[which(complete.cases(dat.pred1())),]
    sub.dat$predicted = predict(hlm())

    ggplot(data=sub.dat) + geom_point(aes(x=predictor,y=response), position = "jitter", size=3, shape=1) +
      xlab(paste(label1()[2])) + ylab(paste(label1()[1])) +
      ggtitle(paste("Scatterplot of",label1()[1],"versus\n",label1()[2],"with HLM lines imposed")) +
      geom_smooth(data=sub.dat, aes(x=predictor,y=predicted,group=level2.id),method="lm",color="tomato",se=FALSE) +
      geom_smooth(data=sub.dat, aes(x=predictor,y=response,group=1),method="lm",color="navy",se=FALSE) + theme_bw()
  })

  hlm.slope.intercept = reactive({
    coef = coefficients(hlm())

    id = unique(dat.pred1()$level2.id)
    n=NULL
    i=0
    while(i<length(id))
    {
      i=i+1
      sub.dat = dat.pred1()[which(dat.pred1()$level2.id==id[i]),]
      n[i] = dim(sub.dat[which(complete.cases(sub.dat)),])[1]
    }
    coef[[1]][,1] = round(coef[[1]][,1],digits=3)
    coef[[1]][,2] = round(coef[[1]][,2],digits=3)

    return(data.frame(int.vec=coef[[1]][,1], slope.vec=coef[[1]][,2],n=n))
  })

  output$hlm.est = renderDataTable({
    dat = data.frame(ranef(hlm())$level2.id)
    mat = matrix(c(unique(dat.pred1()$level2.id),hlm.slope.intercept()$int.vec,
                   round(dat$X.Intercept.,digits=3),hlm.slope.intercept()$slope.vec,
                   round(dat$predictor,digits=3),hlm.slope.intercept()$n),ncol=6)
    colnames(mat) = c("ID","Intercept","Group-level Intercept Error","Slope","Group-level Slope Error","n")
    return(mat)
  })

  output$hlmparamdist1 = renderPlot({
    dat = data.frame(coef(hlm())$level2.id)
    plot1 = ggplot(dat) + geom_histogram(aes(x=X.Intercept.), color="seagreen2",fill="white") +
              xlab("HLM intercepts") + ylab("Frequency") + ggtitle("Histogram of HLM intercepts") + theme_bw()
    plot2 = ggplot(dat) + geom_histogram(aes(x=predictor), color="seagreen2",fill="white") +
              xlab("HLM slopes") + ylab("Frequency") + ggtitle("Histogram of HLM slopes") + theme_bw()
    plot3 = arrangeGrob(plot1,plot2,nrow=1)
    return(plot3)
  })

  output$hlmparamdistinfo1 = renderUI({
    p(HTML("

"),withMathJax("\\(\\hat{\\mu}_{\\alpha} =\\)"),code(paste(round(as.numeric(fixef(hlm.mod.nopred())),digits=3))),
      HTML("

"),withMathJax("\\(\\hat{\\sigma}_\\alpha^{2} =\\)"),code(paste(variances()[2])),HTML("
"))
  })

  output$hlmparamdist2 = renderPlot({
    dat = data.frame(ranef(hlm())$level2.id)
    plot1 = ggplot(dat) + geom_histogram(aes(x=X.Intercept.), color="salmon",fill="white") +
              xlab("Group-level intercept errors") + ylab("Frequency") +
              ggtitle("Histogram of group-level\nintercept errors") + theme_bw()
    plot2 = ggplot(dat) + geom_histogram(aes(x=predictor), color="salmon",fill="white") +
              xlab("Group-level slope errors") + ylab("Frequency") +
              ggtitle("Histogram of group-level\nslope errors") + theme_bw()
    plot3 = arrangeGrob(plot1,plot2,nrow=1)
    return(plot3)
  })

  output$ggcat2 = renderPlot({
    ggCaterpillar(ranef(hlm(), condVar=TRUE), QQ=FALSE)
  })

  ############################################################################################################
  ## Comparison of Methods 2
  ############################################################################################################

  output$slopeint.comp = renderPlot({
    dat = dat.pred1()
    mod = lm(response~predictor,data=dat)
    datline = data.frame(x=as.numeric(mod$coef[1]),y=as.numeric(mod$coef[2]))

    intercept = ggplot() + geom_histogram(data=slope.intercept(),aes(x=intercept,y=..density..,fill="Unpooled"),alpha=.5) +
      geom_histogram(data=hlm.slope.intercept(),aes(x=int.vec,y=..density..,fill="HLM"),alpha=.5) +
      geom_density(data=slope.intercept(),aes(x=intercept),color="gold2") +
      geom_vline(data=datline,aes(xintercept=x),color="navy",linetype="dashed") +
      geom_density(data=hlm.slope.intercept(),aes(x=int.vec),color="tomato") +
      ggtitle("Histogram of sample intercepts") + xlab("Sample intercepts") + ylab("Relative frequency") +
      scale_fill_manual(name="Model",values=c("HLM"="tomato","Unpooled"="gold2","Pooled"="navy")) +
      guides(fill=FALSE) + theme_bw()

    slope = ggplot() + geom_histogram(data=slope.intercept(),aes(x=slope,y=..density..,fill="Unpooled"),alpha=.5) +
      geom_histogram(data=hlm.slope.intercept(),aes(x=slope.vec,y=..density..,fill="HLM"),alpha=.5) +
      geom_density(data=slope.intercept(),aes(x=slope),color="gold2") +
      geom_vline(data=datline,aes(xintercept=y),color="navy",linetype="dashed") +
      geom_density(data=hlm.slope.intercept(),aes(x=slope.vec),color="tomato") +
      ggtitle("Histogram of sample slopes") + xlab("Sample slopes") + ylab("Relative frequency") +
      scale_fill_manual(name="Model",values=c("HLM"="tomato","Unpooled"="gold2","Pooled"="navy")) +
      guides(fill=FALSE) + theme_bw()

    graph1 = arrangeGrob(intercept,slope,nrow=1)

    graph2 = ggplot() + geom_vline(data=datline,aes(xintercept=x),color="navy",linetype="dashed") +
      geom_hline(data=datline,aes(yintercept=y),color="navy",linetype="dashed") +
      geom_point(data=hlm.slope.intercept(),aes(x=int.vec,y=slope.vec,color="HLM"),size=3,shape=1) +
      geom_point(data=slope.intercept(),aes(x=intercept,y=slope,color="Unpooled"),size=3,shape=1) +
      geom_smooth(data=hlm.slope.intercept(),aes(x=int.vec,y=slope.vec,color="HLM"),method="lm",se=FALSE) +
      geom_smooth(data=slope.intercept(),aes(x=intercept,y=slope,color="Unpooled"),method="lm",se=FALSE) +
      ggtitle("Scatterplot of sample slopes versus intercepts with pooled estimates imposed") +
      xlab("Intercepts") + ylab("Slopes") + scale_color_manual(name="Modelling method",values=c("HLM"="tomato","Unpooled"="gold2")) +
      theme_bw() + theme(legend.position="bottom")

    graph3 = arrangeGrob(graph2,graph1,ncol=1)

    return(graph3)
  },height=800)

  output$comp.method = renderPlot({
    dat = dat.pred1()
    dat$hlmpred = fitted(hlm())

    mod = lm(response~predictor,data=dat)
    dat$x=as.numeric(mod$coef[1])
    dat$y=as.numeric(mod$coef[2])

    ggplot(data=dat) + geom_point(aes(x=predictor,y=response),shape=1) + facet_wrap(~level2.id) + theme_bw() +
      geom_smooth(aes(x=predictor,y=response, color="Unpooled"), method="lm", se=FALSE) +
      geom_abline(aes(intercept=x,slope=y, color="Pooled"),size=.5) +
      geom_smooth(aes(x=predictor,y=hlmpred,color="HLM"),method="lm",se=FALSE) +
      scale_color_manual(name="Modelling method", values=c("Pooled"="navy","Unpooled"="gold2","HLM"="tomato")) +
      theme(legend.position="bottom", legend.direction="horizontal") + xlab(paste(label1()[2])) + ylab(paste(label1()[1])) +
      ggtitle("Comparison of the three modelling methods")
  },height=800)

  ############################################################################################################
  ## Bayesian Hierarchical Linear Model 2
  ############################################################################################################

  hlmbayes2 = reactive({
    MCMCglmm(response ~ predictor, random=~level2.id+predictor, data=dat.pred1(),
             verbose=FALSE, pr=TRUE)
  })

  output$hlm.bayes2 = renderPrint({
    summary(hlmbayes2())
  })

  output$postdist2 = renderPlot({
    slope.error = data.frame(x=as.vector(hlmbayes2()$VCV[,2]))
    indiv.error = data.frame(x=as.vector(hlmbayes2()$VCV[,3]))
    group.error = data.frame(x=as.vector(hlmbayes2()$VCV[,1]))
    int = data.frame(x=as.vector(hlmbayes2()$Sol[,1]))
    group.slope.plot = ggplot(data=slope.error) + geom_density(aes(x=x)) + theme_bw() + ylab("") + xlab("Between-group var in slopes") +
                         geom_vline(aes(xintercept=mean(x)),color="seagreen2",size=1)
    group.slope.trace = ggplot(data=slope.error) + geom_line(aes(x=1:length(x),y=x)) + theme_bw() +
                          geom_hline(aes(yintercept=mean(x)),color="seagreen2",size=1) +
                          xlab("Iteration") + ylab("Between-group\nvar in slopes")
    group.error.plot = ggplot(data=group.error) + geom_density(aes(x=x)) + theme_bw() + ylab("") + xlab("Between-group var in intercepts") +
                         geom_vline(aes(xintercept=mean(x)),color="seagreen2",size=1)
    group.error.trace = ggplot(data=group.error) + geom_line(aes(x=1:length(x),y=x)) + theme_bw() +
                        geom_hline(aes(yintercept=mean(x)),color="seagreen2",size=1) +
                        xlab("Iteration") + ylab("Between-group\nvar in intercepts")
    indiv.plot = ggplot(data=indiv.error) + geom_density(aes(x=x)) + theme_bw() + ylab("") + xlab("Within-group var in response") +
                   geom_vline(aes(xintercept=mean(x)),color="seagreen2",size=1)
    indiv.trace = ggplot(data=indiv.error) + geom_line(aes(x=1:length(x),y=x)) + theme_bw() +
                    geom_hline(aes(yintercept=mean(x)),color="seagreen2",size=1) +
                    xlab("Iteration") + ylab("Within-group\nvar in response")
    int.plot = ggplot(data=int) + geom_density(aes(x=x)) + theme_bw() + ylab("") + xlab("Intercept") +
                 geom_vline(aes(xintercept=mean(x)),color="seagreen2",size=1)
    int.trace = ggplot(data=int) + geom_line(aes(x=1:length(x),y=x)) + theme_bw() +
                  geom_hline(aes(yintercept=mean(x)),color="seagreen2",size=1) +
                  xlab("Iteration") + ylab("Intercept")
    graph = arrangeGrob(group.error.trace,group.error.plot,group.slope.trace,group.slope.plot,indiv.trace,
                        indiv.plot,int.trace,int.plot,nrow=4,ncol=2,main=textGrob("Traceplots and Posterior Distributions",gp=gpar(fontsize=15)))
    return(graph)
  },height=600)

  ############################################################################################################
  ############################################################################################################
  ## Varying-intercept and varying-slope with level 2 predictor
  ############################################################################################################
  ############################################################################################################

  output$selectresponse3 = renderUI({
    selectInput("response3","Response variable:",choices=names(data()),selectize=TRUE)
  })

  output$selectlevel2id3 = renderUI({
    selectInput("level2.id3","Level 2 observational unit:",choices=names(data()),selectize=TRUE)
  })

  output$selectlevel1pred2 = renderUI({
    selectInput("level1.pred2","Level 1 predictor:",choices=names(data()),selectize=TRUE)
  })

  output$selectlevel2pred1 = renderUI({
    selectInput("level2.pred1","Level 2 predictor:",choices=names(data()),selectize=TRUE)
  })

  dat.pred2 = reactive({
    if(!input$usesample)
    {
      dat = data.frame(predictor = data()[,paste(input$level1.pred2)],
                       response = data()[,paste(input$response3)],
                       level2.id = factor(data()[,paste(input$level2.id3)]),
                       level2.predictor = data()[,paste(input$level2.pred1)])
      return(dat)
    }else if(input$usesample)
    {
      dat = data.frame(predictor = data()$previous,
                       response = data()$negative_affect,
                       level2.id = factor(data()$id),
                       level2.predictor = data()$years_study)
      return(dat)
    }
  })

  label2 = reactive({
    if(!input$usesample)
    {
      return(c(paste(input$response3),paste(input$level1.pred2),paste(input$level2.id3),paste(input$level2.pred1)))
    } else if(input$usesample)
    {
      return(c("negative affect","previous","musicians","years study"))
    }
  })

  ############################################################################################################
  ## Two stage modelling
  ############################################################################################################

  stage2 = reactive({
    dat.pred2()[!duplicated(dat.pred2()$level2.id),]
  })

  output$int.mod = renderTable({
    dat = data.frame(intercept=slope.intercept()$intercept,level2.predictor=stage2()$level2.predictor)

    lm(intercept ~ level2.predictor, data=dat)
  })

  output$slope.mod = renderTable({
    dat = data.frame(slope=slope.intercept()$slope,level2.predictor=stage2()$level2.predictor)

    lm(slope ~ level2.predictor, data=dat)
  })

  output$stage2graph = renderPlot({
    dat = data.frame(int=slope.intercept()$intercept,slope=slope.intercept()$slope,pred=stage2()$level2.predictor)

    g1 = ggplot(data=dat,aes(x=pred,y=int)) + geom_point(shape=1) + geom_smooth(method="lm",se=FALSE,color="gold2") +
      xlab(paste(label2()[4])) + ylab("Unpooled intercepts") + theme_bw() +
      ggtitle(paste("Scatterplot of unpooled\nintercepts versus",label2()[4]))

    g2 = ggplot(data=dat,aes(x=pred,y=slope)) + geom_point(shape=1) + geom_smooth(method="lm",se=FALSE,color="gold2") +
      xlab(paste(label2()[4])) + ylab("Unpooled slopes") + theme_bw() +
      ggtitle(paste("Scatterplot of unpooled\nslopes versus",label2()[4]))

    g3 = arrangeGrob(g1,g2,nrow=1)
    return(g3)
  })

  ############################################################################################################
  ## Hierarchical Linear Model 3
  ############################################################################################################

  hlm2 = reactive({
    return(lmer(response ~ predictor + level2.predictor + predictor:level2.predictor +
                  (1 + predictor|level2.id), data=dat.pred2(),
                control=lmerControl(optCtrl=list(maxfun=50000))))
  })

  output$hlm.summary1 = renderPrint({
    summary(hlm2())
  })

  output$hlm.confint2 = renderTable({
    conf = confint(hlm2(),method="boot")
    colnames(conf) = c("95% Lower Bound","95% Upper Bound")
    rownames(conf) = c("Between-group Intercept SD","Correlation","Between-group Slope SD","Within-group SD",
                       "Intercept","Level 1 Predictor","Level 2 Predictor","Cross-level Interaction")
    return(conf)
  })

  output$hlmhyper1 = renderTable({
    vec = as.vector(fixef(hlm2()))
    tab = matrix(vec,nrow=1)
    colnames(tab) = c("Intercept","Level 1 Predictor","Level 2 Predictor","Cross-level Interaction")
    rownames(tab) = "Estimates"
    return(tab)
  })

  output$hlmhyper2 = renderPrint({
    VarCorr(hlm2())
  })

  output$ggcat3 = renderPlot({
    ggCaterpillar(ranef(hlm2(), condVar=TRUE), QQ=FALSE)
  })

  ############################################################################################################
  ## Comparison of Methods 3
  ############################################################################################################

  output$compgraph = renderPlot({
    dat = data.frame(int=slope.intercept()$intercept,slope=slope.intercept()$slope,pred=stage2()$level2.predictor)
    dat1 = data.frame(int1=fixef(hlm2())[1],slope1=fixef(hlm2())[3],int2=fixef(hlm2())[2],slope2=fixef(hlm2())[4])

    g1 = ggplot() + geom_point(data=dat,aes(x=pred,y=int,color="Unpooled"),shape=1) +
      geom_smooth(data=dat,aes(x=pred,y=int,color="Unpooled"),method="lm",se=FALSE) +
      xlab(paste(label2()[4])) + ylab("Intercepts") + theme_bw() +
      geom_abline(data=dat1,aes(intercept=int1,slope=slope1,color="HLM")) +
      scale_color_manual(name="Modelling method", values=c("Unpooled"="gold2","HLM"="tomato")) +
      theme(legend.position="bottom", legend.direction="horizontal") +
      ggtitle(paste("Scatterplot of intercepts versus",label2()[4],"\ncomparing the unpooled method and HLM"))

    g2 = ggplot() + geom_point(data=dat,aes(x=pred,y=slope,color="Unpooled"),shape=1) +
      geom_smooth(data=dat,aes(x=pred,y=slope,color="Unpooled"),method="lm",se=FALSE) +
      xlab(paste(label2()[4])) + ylab("Slopes") + theme_bw() +
      geom_abline(data=dat1,aes(intercept=int2,slope=slope2,color="HLM")) +
      scale_color_manual(name="Modelling method", values=c("Unpooled"="gold2","HLM"="tomato")) +
      theme(legend.position="bottom", legend.direction="horizontal") +
      ggtitle(paste("Scatterplot of slopes versus",label2()[4],"\ncomparing the unpooled method and HLM"))

    g3 = arrangeGrob(g1,g2,nrow=1)
    return(g3)
  },height=500)

  ############################################################################################################
  ############################################################################################################
  ## Analyze Data Tab
  ############################################################################################################
  ############################################################################################################

  output$selectresponse.analyze = renderUI({
    selectInput("responseanalyze","Response variable:",choices=names(data()),selectize=TRUE)
  })

  output$selectlevel2id.anaylze = renderUI({
    selectInput("level2.idanalyze","Level 2 observational unit:",choices=names(data()),selectize=TRUE)
  })

  output$selectlevel1pred.analyze = renderUI({
    selectizeInput("level1.predanalyze","Level 1 predictor(s):",choices=names(data()),
                multiple=TRUE,options=list(placeholder="level 1 predictor(s)"))
  })

  output$selectlevel1pred.analyze.rand = renderUI({
    selectizeInput("level1.predanalyze.rand","Random predictor(s):",choices=input$level1.predanalyze,
                   multiple=TRUE,options=list(placeholder="random predictor(s)"))
  })

  output$selectlevel2pred.analyze = renderUI({
    selectizeInput("level2.predanalyze","Level 2 predictor(s):",choices=names(data()),
                multiple=TRUE,options=list(placeholder="level 2 predictor(s)"))
  })

  output$selectcrosslevint = renderUI({
    selectizeInput("crosslevelint", "Cross-level interaction(s):",
                   choices=as.vector(outer(input$level1.predanalyze.rand, input$level2.predanalyze, paste, sep=":")),
                   multiple=TRUE,options=list(placeholder="cross-level interaction(s)"))
  })

  dat.analyze = reactive({
    dat.pred1=matrix(nrow=dim(data())[1],ncol=length(input$level1.predanalyze))
    i=0
    formula1 = NULL
    formula1 = input$level1.predanalyze[1]
    while(i<length(input$level1.predanalyze))
    {
      i=i+1
      dat.pred1[,i] = data()[,paste(input$level1.predanalyze[i])]
      if(i>1) formula1 = paste0(formula1,"+",input$level1.predanalyze[i])
    }
    colnames(dat.pred1) = input$level1.predanalyze
    dat.pred1 = data.frame(dat.pred1)

    formula2=NULL
    dat.pred2=matrix(nrow=dim(data())[1],ncol=length(input$responseanalyze))
    if(length(input$level2.predanalyze)>0)
    {
      j=0
      formula2 = paste("+",input$level2.predanalyze[1])
      while(j<length(input$level2.predanalyze))
      {
        j=j+1
        dat.pred2[,j] = data()[,paste(input$level2.predanalyze[j])]
        if(j>1) formula2 = paste0(formula2,"+",input$level2.predanalyze[j])
      }
      colnames(dat.pred2) = input$level2.predanalyze
      dat.pred2 = data.frame(dat.pred2)
    }

    formula3=NULL
    if(length(input$level1.predanalyze.rand)>0)
    {
      k=1
      formula3 = paste0("+",input$level1.predanalyze.rand[1])
      while(k<length(input$level1.predanalyze.rand))
      {
        k=k+1
        formula3 = paste0(formula3,"+",input$level1.predanalyze.rand[k])
      }
    }

    formula4=NULL
    if(length(input$crosslevelint)>0)
    {
      l=1
      formula4 = paste0("+",input$crosslevelint[1])
      while(l<length(input$crosslevelint))
      {
        l=l+1
        formula4 = paste0(formula4,"+",input$crosslevelint[l])
      }
    }

    dat = data.frame(response = data()[,paste(input$responseanalyze)],
                     level2.id = factor(data()[,paste(input$level2.idanalyze)]),
                     dat.pred1,dat.pred2)
    names(dat)[1] = input$responseanalyze
    names(dat)[2] = input$level2.idanalyze

    return(list(dat=dat,formula1=formula1,formula2=formula2,formula3=formula3,formula4=formula4))
  })

  mod.analyze = reactive({
    do.call(paste("lmer"),list(formula=paste0(input$responseanalyze,"~",dat.analyze()[[2]],dat.analyze()[[3]],
                                              dat.analyze()[[5]],"+ (1",dat.analyze()[[4]],"|",input$level2.idanalyze,")"),
                               data=dat.analyze()[[1]],control=lmerControl(optCtrl=list(maxfun=50000))))
  })

  output$summary.analyze = renderPrint({
    input$runmod
    isolate({
      if(input$runmod>0)
      {
        summary = summary(mod.analyze())
        summary$call=NULL
        return(summary)
      }
    })
  })

})


# 3. faza: Vizualizacija podatkov
#Delež ljudi glede na BDP, ki je v zadnjih 3 mesecih opravilo spletni nakup
graf1 <- ggplot(tabela1) + aes(x = BDPpc, y = Value, color = Education) + geom_point() + xlab("BDP na prebivalca") + ylab("Delež ljudi, ki je v zadnjih 3 mesecih opravil spletni nakup") + ggtitle("Delež ljudi glede na BDP na prebivalca, ki je opravil spletni nakup v zadnjih 3 mesecih") + labs(color = "Odstotek prebivalstva s 3. stopnjo izobrazbe")
"Delež ljudi glede na delež ljudi s tretjo st izobrazbe, ki je opravil spletni nakup v zadnjih 3 mesecih"
graf2 <- ggplot(tabela1) + aes(x = Education, y = Value, color = Area ) + geom_point() + xlab("Delež ljudi s tretjo stopnjo izobrazbe") + ylab("Delež ljudi, ki je v zadnjih 3 mesecih opravil spletni nakup") + ggtitle("Delež ljudi glede na delež ljudi s tretjo stopnjo izobrazbe, ki je opravilo spletni nakup v zadnjih 3 mesecih") + scale_color_discrete(name="Predel", labels=c("Živeči v mestih", "Živeči na ruralnih območjih"))

#tabela2 %>%
#  ggplot(
#    mapping = aes(x = tabela2)
#  ) +
#  geom_histogram()
#

graf3 <- tabela1 %>%
  ggplot(
    mapping = aes(group= Country, x = Country, y = Value)
  ) +
  geom_boxplot() +
  xlab("Država") + ylab("Kolikšen delež ljudi je v zadnjih 3 mesecih kupil nekaj preko spleta") + theme(axis.text.x = element_text(angle = 45, vjust = 1, hjust = 1))

n <- tabela1 %>% dplyr::group_by(Country) %>% dplyr::summarise(mean = mean(Value, na.rm = TRUE))
v <- tabela1 %>% dplyr::group_by(Country) %>% dplyr::summarise(BDPmean = mean(BDPpc, na.rm = TRUE))
n$BDP_mean <- v$BDPmean

graf4 <- ggplot(data = tabela1, aes(x = Year, y = Value, color = Country)) +
  geom_line() + xlab("Leto") + ylab("Delež ljudi, ki je v zadnjih 3 mesecih kupil nekaj prek spleta") + ggtitle("Spreminjanje odstotka ljudi po državah, ki so nakupovali prek spleta, čez leta") + labs(color = "Država")

graf5 <- ggplot(data = tabela1, aes(x = Year, y = Value, color = Area)) +
  geom_line() +
  facet_wrap(facets = vars(Country)) + xlab('Leto') + ylab("Delež ljudi, ki je v zadnjih 3 mesecih kupil nekaj prek spleta") + ggtitle("Spreminjanje odstotka ljudi po državah, ki so nakupovali prek spleta, čez leta") + scale_color_discrete(name="Predel", labels=c("Živeči v mestih", "Živeči na ruralnih območjih"))

graf6 <- ggplot(data = tabela2, mapping = aes(x = Year, y = Koliko_vseh_nakupov_je_opravila_ta_skupina, color = Stopnja)) +
  geom_line() +
  facet_wrap(facets =  vars(Country)) + xlab('Leto') + ylab("Kolikšen delež vseh nakupov je opravila določena skupina") + ggtitle("Spreminjanje odstotka ljudi po državah in doseženi izobrazbi, ki so nakupovali prek spleta, čez leta") + scale_color_discrete(name="Dosežena stopnja izobrazbe", labels=c("Posamezniki z visoko formalno izobrazbo", "Posamezniki s srednjo formalno izobrazbo", "Posamezniki brez ali z nizko izobrazbo", "Študenti"))

graf7 <- ggplot(data=n, aes(x=Country, y=mean, fill = BDP_mean)) +
  geom_bar(stat="identity") + coord_flip() + scale_fill_gradient(low="pink", high="magenta")+ xlab('Država') + ylab('Povprečen odstotek ljudi, ki kupuje prek spleta v tej državi') + labs(fill = "Povprečni BDP per capita") + ggtitle("Odstotek ljudi, ki nakupuje prek spleta glede na BDP")

spr <- tabela1
spr["Education"] <- as.integer(unlist(spr["Education"]))
izobrazba <- spr %>% dplyr::group_by(Country) %>% dplyr::summarise(Edmean = mean(Education, na.rm = TRUE))

n$Edmean <- izobrazba$Edmean
graf8 <- ggplot(data=n, aes(x=Country, y=mean, fill = Edmean)) +
  geom_bar(stat="identity") + coord_flip() + scale_fill_gradient(low="yellow", high="red")+ ylab('Povprečen odstotek ljudi, ki nakupuje prek spleta v tej državi') + xlab('Država') + labs(fill = "Povprečen delež ljudi s tretjo stopnjo izobrazbe") + ggtitle("Odstotek ljudi, ki nakupuje prek spleta glede na odstotek ljudi z doseženo tretjo stopnjo izobrazbe")

tabela3_c[,3] <- as.integer(as.character(unlist(tabela3_c[,3])))
tabela3_c[,4] <- as.integer(unlist(tabela3_c[,4]))
tabela3_c[,5] <- as.integer(unlist(tabela3_c[,5]))
tabela3_c[,6] <- as.integer(unlist(tabela3_c[,6]))
tabela3_c[,7] <- as.integer(unlist(tabela3_c[,7]))

ttt <- tabela3_c %>% dplyr::filter(year == 2019, country == 'Slovenia') %>% pivot_longer(c(3, 4, 5, 6, 7), names_to = 'type', values_to =  "value")
graf9 <- ggplot(data = ttt, mapping = aes(x = country, y = value, fill = type)) +
  geom_bar(width = 1, stat = 'identity') + coord_polar("y", start=0) + xlab('') + ylab('') + ggtitle("Kaj kupujejo ljudje v posamezni državi?") + scale_fill_discrete(name = "Vrsta kupljenega izdelka", labels = c("Knjige, revije, učenje,...", "Obleke, šport", "Filmi, glasba", "Gospodinjski pripomočki", "Potovanja, hoteli,..."))

tabela2_c[,4] <- as.integer(unlist(tabela2_c[,4]))
tabela2_c[,5] <- as.integer(unlist(tabela2_c[,5]))
fff1 <- tabela2_c %>% dplyr::filter(year == 2019, country == 'United Kingdom', stopnja == 'Individuals with high formal education') %>% pivot_longer(c(4, 5), names_to = 'type', values_to = 'value')
graf10 <- ggplot(data = fff1, mapping = aes(x = country, y = value, fill = type)) +
  geom_bar(width = 1, stat = 'identity') + coord_polar("y", start=0) + ggtitle("Kaj kupujejo ljudje\nz doseženo visoko izobrazbo\nv Združenem kraljestvu?") + xlab('') + ylab('') + scale_fill_discrete(name = 'Vrsta kupljenega izdelka', labels = c('Knjige, revije, učenje,...', "Obleke, šport")) +
  theme(legend.position = "none", axis.text = element_blank(), axis.ticks = element_blank(), panel.grid = element_blank())

fff2 <- tabela2_c %>% dplyr::filter(year == 2019, country == 'United Kingdom', stopnja == 'Individuals with no or low formal education') %>% pivot_longer(c(4, 5), names_to = 'type', values_to = 'value')
graf11 <- ggplot(data = fff2, mapping = aes(x = country, y = value, fill = type)) +
  geom_bar(width = 1, stat = 'identity') + coord_polar("y", start=0) + ggtitle("Kaj kupujejo ljudje \nbrez izobrazbe ali \nnizko izobrazbo\nv Združenem kraljestvu?") + xlab('') + ylab('') + scale_fill_discrete(name = 'Vrsta kupljenega izdelka', labels = c('Knjige, revije, učenje,...', "Obleke, šport")) +
  theme(axis.text = element_blank(),
      axis.ticks = element_blank(),
      panel.grid  = element_blank())

library(tmap)

world <- ne_countries(scale = "medium", returnclass = "sf")
Europe <- world[which(world$continent == "Europe"),]
ggplot(Europe) +
  geom_sf() +
  coord_sf(xlim = c(-25,50), ylim = c(35,70), expand = FALSE) +
  aes(fill = 'pink')

lvls <-(Europe$sovereignt)
primerjava <- data.frame(Country = lvls) %>% left_join(tabela1, by = "Country")
manjkajoci <- primerjava[is.na(primerjava$BDPpc), ]

df = data.frame(drzava = manjkajoci$Country, Country = c("Albania",
                                                         "Andorra",
                                                         "Bosnia and Herzegovina",
                                                         "Belarus",
                                                         "Czechia",
                                                         "Germany (until 1990 former territory of the FRG)",
                                                         "Kosovo",
                                                         "Liechtenstein",
                                                         "Monaco",
                                                         "Moldova",
                                                         "North Macedonia",
                                                         "Montenegro",
                                                         "Russia",
                                                         "San Marino",
                                                         "Serbia",
                                                         "Ukraine",
                                                         "Vatican"))
tabela1 <- tabela1 %>% left_join(df) %>% dplyr::mutate(Country=ifelse(is.na(drzava), Country, drzava))
tabela1 <- tabela1 %>% dplyr::select(-drzava)
n2 <- tabela1 %>% dplyr::group_by(sovereignt = Country) %>% dplyr::summarise(mean = mean(Value, na.rm = TRUE))
m <- merge(Europe, n2)
z3 <- ggplot(m) +
  geom_sf() +
  coord_sf(xlim = c(-25,50), ylim = c(35,70), expand = FALSE) +
  aes(fill = mean) +
  scale_fill_gradient(low="yellow", high="magenta") + ggtitle('Katera država v povprečju največ spletno nakupuje?') + labs(fill = "Odstotek ljudi,\nki je v zadnjih 3 mesecih\nspletno nakupoval\n(povprečno)")


#Tutorial and Tips for Numerai

#***
#The stock market is the hardest data science problem in the world.
#If it seems easy, you're doing it wrong.
#We don't know how to solve this problem. But here is what we know.
#We need your help. You take it from here.
#***
library(Metrics)
library(gbm)
library(randomForest)

# for automatic download of data and upload of submissions have a look at the package Rnumerai
# for the latest development release use devtools::install_github("Omni-Analytics-Group/Rnumerai") or pak::pkg_install("Omni-Analytics-Group/Rnumerai")
# for the CRAN version use install.packages("Rnumerai")

print("THIS SCRIPT DOWNSAMPLES DATA AND WILL LIKELY UNDERPERFORM THE example_predictions.csv")

#the training data is used to train your model how to predict the targets
training_data<-read.csv("numerai_training_data.csv", header=T)
#the tournament data is the data that Numerai uses to evaluate your model
tournament<-read.csv("numerai_tournament_data.csv", header=T)
#the tournament data contains validation data, test data and live data
#validation is a hold out set out of sample and further in time (higher eras) than the training data
validation<-tournament[tournament$data_type=="validation",]

benchmark<-0.002
consistency<-function(data_with_probs, target_name){
  unique_eras<-unique(data_with_probs$era)
  era_correlation<-vector()
  i<-1
  while(i<=length(unique_eras)){
    this_data<-data_with_probs[data_with_probs$era==unique_eras[i],]
    era_correlation[i]<-cor(this_data[,names(this_data)==target_name], this_data$prediction, method="spearman")
    i<-i+1
  }
  consistency<-sum(era_correlation>benchmark)/length(era_correlation)
  consistency
}

#the training data is large so we reduce it by 10x in order to speed up this script
#NB reducing the training data like this is not recommended for a production ready model
nrow(training_data)
set.seed(10)
#remove the following line to not downsample. Note: it will take much longer to run
sample_rows<-sample(1:nrow(training_data), ceiling(nrow(training_data)/10))
train<-training_data[sample_rows,]

#we remove the id, era, and data_type columns for now
train<-train[,-c(1:3)]
nrow(train)

#there are a large number of features
ncol(train)
#features are a number of feature groups of different sizes (intelligence, charisma, strength, dexterity, constitution, wisdom)
#Numerai does not disclose what the features or feature groups mean; they are abstract and obfuscated
#however, features within feature groups tend to be related in some way
#models which have large feature importance in one group tend to do badly for long stretches of time (eras).
head(names(train),20)

#the target variable has 5 ordinal outcomes (0,0.25,0.5,0.75,1)
#we train a model on the features to predict this target
summary(train$target)

#on this dataset, feature importance analysis is very important
#we build a random forest to understand which features tend to improve the model out of bag
#because stocks within eras are not independent, we use small bag sizes (sampsize=10%) so the out of bag estimate is meaningful
#we use a small subset of features for each tree to build a more feature balanced forest (mtry=10%)
set.seed(10)
forest<-randomForest(target~., data=train, ntree=50, mtry=ceiling(0.1*ncol(train)-1), sampsize=ceiling(0.1*nrow(train)), importance=T, maxnodes=5)
#a good model might drop the bad features according to the forest before training a final model
#if a feature group or feature is too good, it might also be a good idea to drop to improve the feature balance and improve consistency of the model
imp<-importance(forest)
imp<-imp[order(imp[,1], decreasing = FALSE),]
head(imp)

#based on the forest we can generate predictions on the validation set
#the validation set contains eras further in time than the training set
#the validation set is not "representative of the live data"; no one knows how the live data will behave
#usually train performance < validation performance < live performance because live data is many eras into the future from the training data
predictions<-predict(forest, validation)
head(predictions)
#Numerai measures performance based on Rank Correlation between your predictions and the true targets
cor(validation$target, predictions, method="spearman")
val<-validation
val$prediction<-predictions
#consistency is the fraction of months where the model achieves better correlation with the targets than the benchmark
consistency(val, "target")

#we try a gbm model; we also choose a low bag fraction of 10% as a strategy to deal with within-era non-independence (which is a property of this data)
#(if you take a sample from one era and a different sample from the same era that sample is not really out of sample because the observations occured in the same era)
#having small bags also improves the out of bag estimate for the optimal number of trees
set.seed(10)
model<-gbm(target~., data=train, n.trees=50, shrinkage=0.01, interaction.depth=5, train.fraction=1, bag.fraction=0.1, verbose=T)
#looking at the relative importance of the features we can see the model relies more on some features than others
head(summary(model))
best.iter <- gbm.perf(model, method="OOB")
best.iter

#we check performance of the gbm model on the out of sample validation data
predictions<-predict.gbm(model, validation, n.trees=best.iter, type="response")
head(predictions)
cor(validation$target, predictions, method="spearman")
val<-validation
val$prediction<-predictions
consistency(val, "target")

#the gbm model and random forest model have the same consistency (number of eras where correlation > benchmark) even though correlation are different
#improving consistency can be more important than improving standard machine learning metrics like RMSE
#good models might train in such a way as to minimize the error across eras (improve consistency) not just reduce the error on each training example

#so far we have ignored eras for training
#eras are in order of time; for example, era4 is before era5
#the reason to use eras is that cross validation on rows will tend to overfit (rows within eras are not independent)
#so it's much better to cross validate within eras for example: take a subset of eras, build a model and test on the out of sample subset of eras
#this will give a better estimate of how well your model will generalize
#the validation set is not special; it is just an out of sample set of eras greater than the training set
#some users might choose to train on the validation set as well

#to give you a sense of how to use eras we train a model on the first half of the eras and test it on the second half
#we take another 10x smaller sample of data to speed up the script
ordered_eras<-unique(training_data$era)
train<-training_data[sample_rows,]
first_half<-train[train$era%in%ordered_eras[1:ceiling(length(ordered_eras)/2)],]
second_half<-train[!(train$id%in%first_half$id),]

#we remove id, era, data_type column and train a gbm model on the first half of the data
set.seed(10)
model<-gbm(target~., data=first_half[,-c(1:3)], n.trees=100, shrinkage=0.01, interaction.depth=5, train.fraction=1, bag.fraction=0.1, verbose=T)
best.iter <- gbm.perf(model, method="OOB")
best.iter

predictions<-predict.gbm(model, second_half, n.trees=best.iter, type="response")
#our correlation score is good; what we appeared to learn generalizes well on the second half of the eras
0.5+cor(second_half$target, predictions, method="spearman")
sec<-second_half
sec$prediction<-predictions
#consistency is the fraction of months where the model achieves better correlation with the targets than the benchmark
consistency(sec, "target")

#but now we try build a model on the second half of the eras and predict on the first half
set.seed(10)
model<-gbm(target~., data=second_half[,-c(1:3)], n.trees=100, shrinkage=0.01, interaction.depth=5, train.fraction=1, bag.fraction=0.1, verbose=T)
best.iter <- gbm.perf(model, method="OOB")
best.iter

predictions<-predict.gbm(model, first_half, n.trees=best.iter, type="response")
#our correlation score is surprisingly low and surprisingly different from when we trained on the first half
#this means our model is not very consistent, and it's possible that we will see unexpectedly low performance on the test set or live set
#it also shows that our validation performance is likely to greatly overestimate our performance--this era-wise cross validation is more valuable
#a model whose performance when training on the first half is the same as training on the second half would likely be more consistent
#and would likely perform better on the tournament data and the live data
cor(first_half$target, predictions, method="spearman")
fir<-first_half
fir$prediction<-predictions
#consistency is the fraction of months where the model achieves better correlation with the targets than the benchmark
consistency(fir, "target")

#Numerai only pays models with correlations that beat the benchmark on the live portion of the tournament data
#to submit predictions from your model to Numerai, predict on the entire tournament data
#we choose use our original forest model for our final submission
tournament$prediction<-predict(forest, tournament)

#create your submission
submission<-subset(tournament, select=c(id, prediction))
head(submission)

#save your submission and now upload it to https://numer.ai
write.csv(submission, file="submission.csv", row.names=F)


# This function is intended to simulate random drift of 4 alleles in a population, given starting and ending
# frequencies for each allele, by resampling population of given allele frequency.
# Revision of a script by Melissa Hardstone (FourAlleleDrift.r), reviewed and minor edits JCF 2018.
# The MH copy of FourAlleleDrift.r script HAS AN ERROR- in the reporting of results for A2 and can give incorrect results.
# What I changed:
    #
    # Results were previously only kept for current simulation, I routed them to a larger matrix that stores
    #      results for all sims during function. If raw simulation data is desired, default "raw_data=FALSE" can
    #      be changed to true and all data will be returned. (Good for making sure everything is working as intended,
    #      and plotting if desired.)
    # Also quieted down the function a bit, there were unnecessary print commands everywhere.
    # Added comments for clarity

# last edit: 2019-1-8 JCF

# R sessionInfo of last edit.
# > sessionInfo()
# R version 3.5.1 (2018-07-02)
# Platform: x86_64-w64-mingw32/x64 (64-bit)
# Running under: Windows >= 8 x64 (build 9200)
#
# Matrix products: default
#
# locale:
#   [1] LC_COLLATE=English_United States.1252  LC_CTYPE=English_United States.1252    LC_MONETARY=English_United States.1252
# [4] LC_NUMERIC=C                           LC_TIME=English_United States.1252
#
# attached base packages:
#   [1] stats     graphics  grDevices utils     datasets  methods   base
#
# loaded via a namespace (and not attached):
#   [1] compiler_3.5.1 tools_3.5.1

#
# Input:
#
# SIMS is # simulations desired
# gens is # generations between starting & ending allele frequencies
# size_of_pop is # of diploid individuals in the population
# A1-A4 are starting allele frequencies
# A1-A4 are starting allele frequencies

#
# Output:
#
# Vector of 4 p-values (1 for each allele), calculated as the number of simulations where the allele frequency change is as extreme as the
#         observed allele frequency change. Null hypothesis here is that allele frequency change could be due to drift.
# raw_data is set to FALSE as default, but if set to TRUE function will return a matrix

FourAlleleDrift<-function(SIMS, gens, size_of_pop, A1, A2, A3, A4, end_A1, end_A2, end_A3, end_A4, raw_data=FALSE)
{

  # Set up 4 matrices to track allele frequency over gens (dimensions #gens X #sims)
  # Fill first row with the starting allele frequency
	resultA1<-matrix(NA, nrow=gens, ncol=SIMS)                    # Declare results matrix
	resultA1[1,]=A1                                               # Fill first row with the starting allele frequency
	resultA2<-matrix(NA, nrow=gens, ncol=SIMS)
	resultA2[1,]=A2
	resultA3<-matrix(NA, nrow=gens, ncol=SIMS)
	resultA3[1,]=A3
  resultA4<-matrix(NA, nrow=gens, ncol=SIMS)
  resultA4[1,]=A4

  # Now will loop through SIMS (j) & generations (i)
  #
	for (j in 1:SIMS)                                              # For each of j simulations
	{

    gen<-gens                                                    # no. of generations
		pop_size<-size_of_pop                                        # size_of_pop
		i=1                                                          # Initialize for generation

		# starting with the inputed allele frequencies
		A1_allele<-A1                                                # A1 start freq
		A2_allele<-A2                                                # A2 start freq
		A3_allele<-A3                                                # A3 start freq
		A4_allele<-A4                                                # A4 start freq

		# Create a population with the starting allele frequencies (*2 for diploid)
		pop_alleles<-c( rep('A1',2*pop_size*A1_allele), rep('A2',2*pop_size*A2_allele),
		                rep('A3',2*pop_size*A3_allele), rep('A4',2*pop_size*A4_allele) )

		# take a sample with replacement from the whole population to make next generation (panmictic, random mating)
		pop_alleles<-sample( pop_alleles, 2*pop_size, replace=TRUE )

		# Add the simulated allele frequency to results matrices: find # of entries in pop_alleles for each allele & divide by 2*pop_size
		resultA1[i+1,j]<-( length(pop_alleles[pop_alleles=='A1']) )/(2*pop_size)   # freq of A1 allele, add to results matrix
		resultA2[i+1,j]<-( length(pop_alleles[pop_alleles=='A2']) )/(2*pop_size)   # freq of A2 allele
		resultA3[i+1,j]<-( length(pop_alleles[pop_alleles=='A3']) )/(2*pop_size)   # freq of A3 allele
		resultA4[i+1,j]<-( length(pop_alleles[pop_alleles=='A4']) )/(2*pop_size)   # freq of A4 allele

    # each loop, allele freq ending from previous loop used as the start allele freq
		for(i in 2:gen-1)
		{
		  A1_allele<-resultA1[(i+1)-1,j]                                           # uses A1 freq from last generation
			A2_allele<-resultA2[(i+1)-1,j]                                           # uses new A2 freq
			A3_allele<-resultA3[(i+1)-1,j]                                           # uses new A3 freq
			A4_allele<-resultA4[(i+1)-1,j]                                           # uses new A4 freq

      # Simulate population with new allele frequencies
      pop_alleles<-c(rep('A1',2*pop_size*A1_allele),rep('A2',2*pop_size*A2_allele),rep('A3',2*pop_size*A3_allele),rep('A4',2*pop_size*A4_allele));

			# take a sample from the whole population to make next generation
      pop_alleles<-sample(pop_alleles,2*pop_size,replace=TRUE)
      resultA1[i+1,j]<-(length(pop_alleles[pop_alleles=='A1']))/(2*pop_size)   # freq of A1 allele, add to results matrix
      resultA2[i+1,j]<-(length(pop_alleles[pop_alleles=='A2']))/(2*pop_size)   # freq of A2 allele
      resultA3[i+1,j]<-(length(pop_alleles[pop_alleles=='A3']))/(2*pop_size)   # freq of A3 allele
      resultA4[i+1,j]<-(length(pop_alleles[pop_alleles=='A4']))/(2*pop_size)   # freq of A4 allele

	  }

	}

  # Now have full matrix of results for each allele.
  # Now calculate p-values from simulation results.

  # count times A1 allele freq at generation of interest is equal to or more extreme than obs freq
    # need to designate which direction of the extreme values is wanted
    if(end_A1 < A1)
    {
		  A1_p<-length( which( resultA1[gen,] <= end_A1 ) )/SIMS;
		  #cat("probability of getting end A1 freq or smaller is", A1_p, '\n');
	 }else{
	   A1_p<-length( which( resultA1[gen,] >= end_A1 ) )/SIMS;
		  #cat("probability of getting end A1 freq or larger is", A1_p, '\n');
	 }

 	# Count times A2 allele freq at generation of interest is equal to or more extreme than obs freq
	# Need to designate which direction of the extreme values is wanted
  if(end_A2 < A2)
  {
   A2_p<-length( which( resultA2[gen,] <= end_A2 ) )/SIMS;
    # cat("probability of getting end A2 freq or smaller is", A2_p, '\n');
  }else{
    A2_p<-length( which( resultA2[gen,] >= end_A2 ) )/SIMS;
    # cat("probability of getting end A2 freq or larger is", A2_p, '\n');
  }

	# count times A3 allele freq at generation of interest is equal to or more extreme than obs freq
	# need to designate which direction of the extreme values is wanted
  if(end_A3 < A3)
  {
    A3_p<-length( which( resultA3[gen,] <= end_A3 ) )/SIMS;
    #cat("probability of getting end A3 freq or smaller is", A3_p, '\n');
  }else{
    A3_p<-length( which( resultA3[gen,] >= end_A3 ) )/SIMS;
    #cat("probability of getting end A3 freq or larger is", A3_p, '\n');
  }

	# count times A4 allele freq at generation of interest is equal to or more extreme than obs freq
	# need to designate which direction of the extreme values is wanted
  if(end_A4 < A4)
  {
    A4_p<-length( which( resultA4[gen,] <= end_A4 ) )/SIMS;
    #cat("probability of getting end A4 freq or smaller is", A4_p, '\n');
  }else{
    A4_p<-length( which( resultA4[gen,] >= end_A4 ) )/SIMS;
    #cat("probability of getting end A4 freq or larger is", A4_p, '\n');
  }

  # Function by default returns a vector of 4 p-values, 1 for each allele. Null hypothesis here is that allele frequency change could be due to drift.
  # If raw_data==TRUE, will also return a matrix of simulation results (intended for plotting).

  # Create vector of p-values
  p.vals=c(A1_p,A2_p,A3_p,A4_p)

  if( raw_data==TRUE ){
    print( list("A1_results"=resultA1, "A2_results"=resultA2, "A3_results"=resultA3, "A4_results"=resultA4, p.vals) )
  } else { print(p.vals) }

}