#Jenny Smith

#march 30, 2017
#Purpose: Create a list of survival analyses and Given survival package Survfit() object, plot the survival curves and calculate the P values for differences in survival.

SurvObjects <- function(df, colNames, group, rho=0, time=NULL,ref=NULL){
  #df is the dataframe with clinical data.
  #colnames is a character vector (length 2) with the colnames of the time and event, in that order.
  #group is the column name as character vector of the explanatory variable to test
  #time is optional if you need to convert days to years for example.
  #rho is 1 or 0 or inbetween. 0=logrank test and 1=gehan-wilcoxon test

  require(survival)
  require(survMisc)

  #optional time unit conversion
  if (is.null(time)){
    time <- (df[[colNames[1]]])
  }else if (time == "DtoY"){
    time <- (df[,colNames[1]]/365)
  }else if (time == "MtoY"){
    time <- (df[,colNames[1]]/12)
  }

  #Kaplan Meier Estimates
  KM <- Surv(time = time, event = df[[colNames[2]]])

  if (group == 1){
    #model fit
    survFit <- survfit(KM ~ 1)
    return(survFit)

  }else{

    if (!is.null(ref)){
      df[,group] <- relevel(as.factor(df[,group]), ref=ref)
    }

    #model fit
    survFit <- survfit(KM ~ df[,group], data=df)

    #cox proportional Hazards
    cph.test <- coxph(KM ~ df[,group])

    #test proportional hazards assumption
    cph.zph <- cox.zph(cph.test, transform = "km")
    rhoPval <- as.data.frame(cph.zph$table)$p #test the PH assumption

    if (length(rhoPval) > 1){
      rhoPval <- rhoPval[length(rhoPval)]
    }

    # survMisc compare stat tests for two curves
    tne <- ten(eval(survFit$call$formula), data=df)
    capture.output(comp(tne)) -> allTests

    if (rhoPval >= 0.05 ){
      attr(tne, "lrt")[1,] -> pVal #log-rank test
      test <- "log.rank"
    }else if (rhoPval < 0.05){
      attr(tne, "lrt")[2, ] -> pVal #Gehan-Breslow-Wilcoxon
      test <- "Gehan.Breslow.Wilcoxon"
    }

    list <- list(survFit, pVal, cph.test, cph.zph, KM)
    names(list) <- c("survFit", test, "CoxPH", "PH_Test","KaplanMeirerEst")
    return(list)
  }
}

#Survival plot without gridlines.
SurvivalPlot <- function(fit, LegendTitle, timeUnit,include.censored=TRUE, colors,pval=NULL,max.year=NULL){
  #fit the output of the survFit() function.
  #legend title is a character vector which will be the plot legend title.
  #timeUnit is a character vector for the follow-up time in days, years, months etc.
  #colors is a character vector the same length as the number of groups.
  #max.year is the time (singe numeric) in years to end the plot at.
  #pval is string (charcter or numeric) of the p-value

  require(survival)
  library(ggplot2)
  library(GGally)

  if(is.null(pval)){
    pval <- ""
  }else{
    pval <- paste0("p = ",pval)
  }

  if(!is.null(max.year)){
    # if(max(fit$time) < max.year){
    pos.x <- max.year
    # }
  }else{
    pos.x <- max(fit$time)
  }

  if(is.null(fit$strata)){
    lm <- 1 #left margin (lm)
    rm <- 4  #right margin(rm)
    num.cols <- 1
    num.rows <- 1

    #main survival plot
    p <- ggsurv(fit, surv.col = colors,
                cens.col=colors, CI=FALSE,
                lty.est = 1, size.est = 1.0,
                cens.size = 2.0,order.legend=FALSE)

  }else{

    m <- max(nchar(gsub("^.+\\=(.+)", "\\1", names(fit$strata)))) #maximum number of characters
    d <- length(fit$strata) #how many groups
    lm <- case_when(m <= 2 ~ 0.90,
                    m <= 5 ~ 1.25,
                    m > 5 & m <= 9 ~ 1.7,
                    m > 9 & m < 13 ~ 2.6,
                    m >= 13 & m < 17 ~ 3.4,
                    m >= 17 & m < 21 ~ 5.0,
                    m >= 21 & m < 23 ~ 6.0,
                    m >= 23 & m < 25 ~ 6.5,
                    m >= 25 ~ 7.5)#left margin (lm)
    # print(c("km.plot", lm))
    rm <- case_when(d <= 4 ~ 1.5, d > 4 ~ 2.0) #right margin(rm)
    num.cols <- 2
    num.rows <- 2

    if(d > 4 & d < 7){
      num.rows <- 3
    }else if(d > 7 & d <= 18){
      num.cols <- 3
      num.rows <- 6
    }else if(d > 18){
      num.cols <- 5
      num.rows <- 5
    }

    if(num.cols*num.rows < d){
      num.rows <- 10
    }

    #main survival plot
    p <- ggsurv(fit, plot.cens=include.censored,
                surv.col = colors, CI=FALSE,
                lty.est = 1, size.est = 1.0,
                cens.size = 2.0,order.legend=FALSE)
  }

  #customize the plot
  p <- p +
    labs(y= "Fraction Surviving", x = paste(timeUnit, "Follow-up")) +
    scale_y_continuous(limits = c(0,1),
                       breaks = seq(0,1,0.2),
                       minor_breaks = NULL) +
    scale_x_continuous(limits = c(0,pos.x),
                       breaks = seq(0,pos.x, 2)) +

    theme(plot.title = element_text(hjust = 0.5, size=10),
          panel.background = element_rect(fill="white"),
          panel.grid.major = element_blank(),
          panel.grid.minor = element_blank(),
          panel.border = element_rect(colour = "black", fill= NA, size=1.0),

          axis.text = element_text(colour = "black"),
          axis.text.y = element_text(size = 12),
          axis.text.x = element_text(size=12),
          axis.title = element_text(size = 12),
          legend.position = "top",
          legend.direction  = "horizontal",
          legend.margin = margin(0,0,0,0, unit = "pt"),
          legend.text = element_text(size=10),
          legend.title = element_text(size=10),
          legend.key=element_blank()) +

    theme(plot.margin = margin(0, rm,.5,lm,unit="cm")) +
    annotate(geom="text", x=1, y=0.05, label=pval, size=5) +
    guides(color=guide_legend(ncol=num.cols, nrow=num.rows))

  if (length(colors) > 1){
    p$scales$scales[[1]]$name <- LegendTitle
    p$scales$scales[[2]]$name <- LegendTitle
  }

  return(p)
}

#risk table dataframe
risk.table <- function(survFit.obj, f.levels=NULL,
                       times=NULL,col="black",
                       ret.data=FALSE, max.year=NULL){
  #survFit.obj is the results  survfit()
  #f.levels is an optional character vector to relevel the order of the groups, if desired. else levels is alphabetical
  #times would be a seperate numeric vector, with years/x-axis breaks points, if desired.

  if(!is.null(max.year)){
    # if (max(survFit.obj$time) < max.year){
      times <- seq(0,max.year, by= 2)
      xlims <- max.year
    # }
  }else{
    times <- seq(0,max(survFit.obj$time), by= 2)
    xlims <- max(survFit.obj$time)
  }

  #Summary dataframe with the selected time points to get % OS/ EFS for plot x-axis
  summ <- summary(survFit.obj, times=times, extend=TRUE)

  if(is.null(f.levels)){
    f.levels <- unique(as.character(summ$strata)) %>%
      gsub("^.+\\=(.+)", "\\1", .)
  }

  #Create a new dataframe to hold the time,KM estimate and strata for plotting
  risk.df <- bind_cols(list(time=summ$time,
                            surv=summ$surv,
                            n.risk=summ$n.risk,
                            Group=as.character(summ$strata))) %>%
    set_colnames(c("time","surv","N.Risk","Group")) %>%
    mutate(Group=gsub("^.+\\=(.+)", "\\1", Group),
           Group=factor(Group, levels = f.levels))

  if(ret.data){
    return(risk.df)
  }

  #variable for deciding length of margins. longer group names would need more space.
  m <- max(nchar(levels(risk.df$Group))) # print(c("length:", m))
  d <- length(levels(risk.df$Group))
  lm <- case_when(m <= 5 ~ 1.75,
                  m > 5 & m <= 9 ~ 1.00,
                  m > 9 & m <= 10 ~ 1.00,
                  m > 10 & m < 13 ~ 1.10,
                  m >= 13 ~ 1.20)

  rm <- case_when(d <= 5 ~ 1.5,
                  m > 5 ~ 2.0)

  #gglot the risk table (risk.df)
  tbl <- ggplot(risk.df, aes(x = time, y = Group, label=N.Risk)) +
    geom_text(size = 5)  +
    theme_bw()  +
    scale_x_continuous("Number at risk",
                       limits = c(0,xlims),
                       breaks = times) +
    xlab(NULL) +
    ylab(NULL) +
    theme(panel.grid.major = element_blank(),
          panel.grid.minor = element_blank(),
          panel.border = element_blank(),
          legend.position="none",
          axis.line = element_line(size=0.7, color="black"),
          axis.title.x = element_text(size=12),
          axis.ticks.x = element_blank(),
          axis.title.y = element_blank(),
          axis.text.x = element_blank(),
          axis.text.y = element_text(face="bold",
                                     size=12,
                                     color= col[levels(risk.df$Group)],
                                     hjust=1)) +
    #of note: this margin is still not perfect..... the size of window in Rstudio is different than what actually shows up on the tiff...
    theme(plot.margin = margin(-1.5, rm, 0.1, lm, unit="cm"))

  return(tbl)
}

##The function below added 10/2/17
KM.plots <- function(df, group_vars=NULL, type="OS",
                     covariate,
                     cohort,cc=NULL,
                     riskTable=TRUE,
                     f.levels=NULL,
                     max.year=NULL,
                     custom_cols=NULL,
                     include.censored=TRUE){
  library(dplyr)
  library(tibble)
  library(survival)
  library(magrittr)
  library(tidyr)
  suppressPackageStartupMessages(require(gridExtra))

  #df is the cleaned CDE with patient IDs as rownames.
  #group_vars is a character vector of a column name to group_by() in dplyr.
  #type is "OS" for OS/EFS or "EOI" for end of induction time points
  #covariate is a character string for the column in CDE to be the explanatory variable
  #cohort is for "0531" or "1031" (different colnames)

  #Define KM estimates for survival analysis
  if (cohort == "0531" & type=="OS"){
    os.est <- "Surv(Overall.Survival.Time.in.Days/365.25, OS.ID)"
    efs.est <- "Surv(Event.Free.Survival.Time.in.Days/365.25, Event.ID)"
  }else if (cohort == "1031" & type=="OS"){
    os.est <- "Surv(OS.time..days./365.25, OS.ID)"
    efs.est <- "Surv(EFS.time..days./365.25, Event.ID)"
  }else if (cohort == "TCGA" & type=="OS"){
    os.est <- "Surv(OS.months..3.31.12/12, clinData1.vital_status)"
    efs.est <- "Surv(EFS.months....3.31.12/12, first.event)"
  }else if (cohort=="BEAT_AML" & type=="OS"){
    os.est <- "Surv(OverallSurvival_VIZOME/365.25, OS.ID)"
    #No EFS data available
  }else if (type=="EOI"){
    if(is.null(custom_cols)){message("Need custom columnames for time and event for EOI analyses.
                                     Custom calls must a list of 2 names os.est and efs.est containing vector of
                                     time column in days and event.type indicator column as 0,1 in that order.");
      return(custom_cols)}
    os.est <- paste0("Surv(",custom_cols[["os.est"]][1],"/365.25",", ",custom_cols[["os.est"]][2], ")")
    efs.est <- paste0("Surv(",custom_cols[["efs.est"]][1],"/365.25",", ",custom_cols[["efs.est"]][2], ")")
  }

  #function for colors
  colorcodes <- function(df,covariate){
    strata <- unlist(unique(df[,covariate])) #Unique groups (strata)
    strata <- strata[order(strata)] #ensure the alphabetical order
    len <- length(strata)

    if ( ! is.null(cc)){
      return(cc)

    }else{
      colors <- c("dodgerblue4", "brown3", "darkgoldenrod3",
                  "deepskyblue1",
                  "azure4", "darkmagenta",
                  "turquoise4", "green4",
                  "deeppink", "navajowhite2","chartreuse2","lightcoral",
                  "mediumorchid", "saddlebrown", "#466791","#60bf37","#953ada",
                  "#4fbe6c","#ce49d3","#a7b43d","#5a51dc",
                  "#d49f36","#552095","#507f2d","#db37aa",
                  "#84b67c","#a06fda","#df462a","#5b83db",
                  "#c76c2d","#4f49a3","#82702d","#dd6bbb",
                  "#334c22","#d83979","#55baad","#dc4555",
                  "#62aad3","#8c3025","#417d61","#862977",
                  "#bba672","#403367","#da8a6d","#a79cd4",
                  "#71482c","#c689d0","#6b2940","#d593a7","#895c8b","#bd5975")
      cc <- NULL
      for ( i in 1:len){
        c <- colors[i]
        cc <- c(cc,c)
      }
      names(cc) <- strata
      return(cc)
    }
  }

  #Formula for KM curves
  OS.form <- as.formula(paste(os.est,' ~ ',covariate))

  #perform survival analysis for all factor levels
  grouped.df <-  df %>%
      group_by(!!! rlang::syms(group_vars)) %>%
      do(OS.cox=coxph(OS.form, data = .),
         OS.fit=survfit(OS.form, data=.),
         OS.diff=survdiff(OS.form, data=.), #survdiff for log-rank p-value
         OS=SurvivalPlot(survfit(OS.form, data=.), #survival plots
                         LegendTitle = covariate,
                         timeUnit = "Years",
                         max.year = max.year,
                         include.censored=include.censored,
                         colors =  colorcodes(.,covariate = covariate)))

    if(exists("efs.est")){
      #Formula for EFS KM curves
      EFS.form <- as.formula(paste(efs.est,' ~ ', covariate))

      grouped.df.efs <-  df %>%
        group_by(!!! rlang::syms(group_vars)) %>%
        do( EFS.cox=coxph(EFS.form, data = .),
            EFS.fit=survfit(EFS.form, data=.),
            EFS.diff=survdiff(EFS.form, data=.), #survdiff for log-rank p-value
            EFS=SurvivalPlot(survfit(EFS.form, data=.),
                             LegendTitle= covariate,
                             timeUnit= "Years",
                             max.year = max.year,
                             include.censored=include.censored,
                             colors= colorcodes(., covariate = covariate)))

      grouped.df <- bind_cols(grouped.df, grouped.df.efs)
    }

    #Denote the grouping variable in the results dataframe
    if(is.null(group_vars)){
      grouped.df <- add_column(grouped.df, Group="AML",.before = 1)
    }else{
      col.idx <- which(colnames(grouped.df)=="OS.cox")-1
      grouped.df <- unite(grouped.df,
                          "Group",1:all_of(col.idx), remove = FALSE, sep=" in ")
    }

  #function to calculate a p-value
  pvalue <- function(survdiff.res){
    p <- pchisq(survdiff.res$chisq, df=length(survdiff.res$n) - 1, lower=FALSE)
    p <- ifelse(p < 0.001, "p < 0.001", paste0("p = ", round(p, digits = 3)))
    return(p)
  }

  #Function to extract N of cohort.
  groupSize <- function(survdiff.res){paste0("N = ", sum(survdiff.res$n))}

  #Update the survial plots with an informative title and p values, and group N
  types <- c("Relapse", "Failure", "OS", "EFS")
  plots <- which(colnames(grouped.df) %in% types)

  #for each row (nrow > 1, when grouping variable given)
  for ( i in 1:nrow(grouped.df)){

    #for each OS/EFS column
    for (plot.idx in plots){
      fit.idx <- plot.idx-2
      fit <- grouped.df[[fit.idx]][[i]]
      summ <- summary(fit)
      median.surv <- summ$table[,"median"]
      x_pos <- ifelse(min(median.surv,na.rm=T) < 1, max(summ$time,na.rm = T), 1)
      y_pos <- ifelse(min(median.surv, na.rm=T) < 1, 0.95, 0.05)

      idx.logrank <- plot.idx-1
      N <- groupSize(grouped.df[[idx.logrank]][[i]])
      pval <- pvalue(grouped.df[[idx.logrank]][[i]]) %>%
        paste(.,N, sep="\n")

      #Add Main Title that is the column name (eg Relapse,OS, EFS, etc) and grouping factors
      grouped.df[[plot.idx]][[i]]$labels$title <- paste(names(grouped.df[plot.idx]),
                                                        grouped.df[["Group"]][i],
                                                        sep = ": ")

      #Add P value and the N
      grouped.df[[plot.idx]][[i]] <- grouped.df[[plot.idx]][[i]] +
        annotate(geom="text", x=x_pos, y=y_pos, label=pval, size=5) +
        theme(plot.title = element_text(face="bold"))

      #Add a risk table (Number of patients at risk at each time )
      if(riskTable){

        risk.tab <- risk.table(survFit.obj = fit,
                               ret.data = TRUE)
        risk.tab.p <- risk.table(survFit.obj = fit,
                                 col = cc,
                                 f.levels = f.levels,
                                 max.year=max.year)

        # Create a blank plot for place-holding
        blank.pic <- ggplot(risk.tab, aes(time, surv)) +
          geom_blank() +
          theme_bw() +
          theme(axis.text.x = element_blank(),
                axis.text.y = element_blank(),
                axis.title.x = element_blank(),
                axis.title.y = element_blank(),
                axis.ticks = element_blank(),
                panel.grid = element_blank(),
                panel.border = element_blank())

      # grid.arrange(grouped.df[[plot.idx]][[i]], blank.pic, risk.tab.p, clip = FALSE, nrow = 3,
      #                    ncol = 1, heights = unit(c(2, .1, .25), c("null", "null", "null")))

       grouped.df[[plot.idx]][[i]] <-  arrangeGrob(grouped.df[[plot.idx]][[i]],
                                                   blank.pic, risk.tab.p,
                                                   clip = FALSE,
                                                   nrow = 3, ncol = 1,
                                                   heights = unit(c(2, .2, .4),
                                                                  c("null", "null", "null")))

      }
    }
  }

  return(grouped.df)
}

#Function to save the outputs from KM.plots()
saveMultiPlots <- function(KM.plots.res,w=8,h=5){
  #This is for the relapse results from KM.plots()

  N <- nrow(KM.plots.res)

  covar <- function(i){
    names(KM.plots.res[grep("diff",colnames(KM.plots.res))][[i,1]]$n) %>%
      str_split_fixed(.,pattern="=",n=2) %>% .[,1] %>% unique()
  }

  name <- function(i){paste(KM.plots.res[[1]][i],covar(i),col,"KMplot.tiff", sep="_")}
  cols <- grep("diff",colnames(KM.plots.res), value = TRUE, invert = TRUE)[-1]

  for (col in cols){
    lapply(1:N, function(x) ggsave(filename = name(x),
                                   plot = KM.plots.res[[col]][[x]],
                                   device = "tiff",
                                   width = w,
                                   height = 5,
                                   dpi=600))

  }
}

#Added from LSC17  Analysis  on July 12, 2017.
weibull.HR <- function(weibull.mod){
  #weibull.mod is the output of survreg(, dist="wiebull")
  # multivar is a numeric vector idicating the indices of multi varaite analysis to be returned.
  if (is.null(multivar)){
    idx <- 2 #univariate analysis
  }else{
    idx <- 2:length(weibull.mod$coefficients)
  }
  HR <- round(exp(weibull.mod$coefficients*-1*1/weibull.mod$scale)[idx], digits = 3)
}

#Added from LSC17  Analysis  on July 12, 2017.
parametricSumaryTable <- function(survreg.res){
  summ <- summary(survreg.res)
  table <- as.data.frame(summ$table)

  # coef <- summ$coefficients[which(! grepl("Int", names(summ$coefficients)))]
  idx <- which( ! grepl("Int|Log", names(survreg.res$coefficients)))

  UCI_HR2 <- round(exp((survreg.res$coefficients[idx] - 1.96*table$`Std. Error`[idx])*-1*1/survreg.res$scale),
                   digits = 3)
  LCI_HR2 <- round(exp((survreg.res$coefficients[idx]  + 1.96*table$`Std. Error`[idx])*-1*1/survreg.res$scale),
                   digits=3)

  HR <- round(exp(survreg.res$coefficients[idx]*-1*1/survreg.res$scale), digits = 3)
  CI <- paste(LCI_HR2, UCI_HR2, sep="-")
  pVal <- round(table$p[idx], digits=3)

  stats <- data.frame(HR, CI,pVal)
  colnames(stats) <- c("Hazard Ratio", "95% Confidence Interval", "P-value")
  return(stats)
}

#Added from LSC17 Multivariate Analysis  on June 24th, 2017.
coxSummaryTable <- function(coxph.res,Colname="Group"){
  c.mod <- coxph.res
  c.summ <- summary(coxph.res)
  c.table <- as.data.frame(c.summ$coefficients) %>% cbind(.,as.data.frame(c.summ$conf.int))
  # c.table
  c.HR <- round(c.table$`exp(coef)`, digits=3)
  c.CI <- paste(round(c.table$`lower .95`,digits=3),round(c.table$`upper .95`, digits = 3), sep="-")
  c.pVal <- round(c.table$`Pr(>|z|)`, digits=3)

  comp <- gsub("^.+(Yes|No|Unknown|High|Low)", "\\1",names(c.mod$coefficients))
  c.stats <- data.frame(names(c.mod$coefficients) ,comp, c.HR, c.CI, c.pVal)
  colnames(c.stats) <- c("Variable", Colname,"Hazard Ratio", "95% Confidence Interval", "P-value")
  # rownames(c.stats) <- names(c.mod$coefficients)

  return(c.stats)
}

#functions to calculate the p-value for differnces in KM curves from survdiff() function
#Tests if there is a difference between two or more survival curves using the G-rho family of tests, or for a single curve against a known alternative.
calc_KMcurve_pvalues <- function(survdiff, digits=3){
  p <-  pchisq(survdiff$chisq, length(survdiff$n)-1,
              lower.tail = FALSE) %>%
    round(.,  digits = digits)

  return(p)
}

#create a table with OS/EFS percent at specified time point
#uses the summary() function on the survfit() object
#can optionally input p-values manually from survdiff() function
outcome_table <- function(fit, time=5, pvalues=NULL){

  summ <- summary(fit, times=time)
  type <- ifelse(any(grepl("OS", summ$call)),"OS","EFS")
  summ <- summ[!grepl("table|std.chaz|call", names(summ))]
  summ <- summ[sapply(summ,length)==length(summ$strata)]

  surv_col <- paste("Percent", type)
  n <- length(summ$strata)

  tab <- tibble(as.data.frame(summ)) %>%
    mutate_at(vars(surv,lower,upper,std.err), ~round(.*100, digits = 2)) %>%
    mutate(Group=str_split_fixed(strata, pattern = "=", n=2)[,2])

  if(!is.null(pvalues) & length(pvalues) < n){
    pval_ref <- NA #ref is the first position in the strata
    pvalues=c(pval_ref, pvalues) #must be in same order as strata in the survift()!
    tab <- tab %>%
      add_column(pvalue=pvalues)

    tab <- tab %>%
      select(Group,
             "Number Patients per Group"=n,
             "Time (years)"=time,
             !!surv_col := surv,
             "Lower 95% CI"=lower,
             "Upper 95% CI"=upper,
             "Standard Error"=std.err,
             "p-value"=pvalue)
  } else{

    tab <- tab %>%
      select(Group,
             "Number Patients per Group"=n,
             "Time (years)"=time,
             !!surv_col := surv,
             "Lower 95% CI"=lower,
             "Upper 95% CI"=upper,
             "Standard Error"=std.err)

  }

  return(tab)
}

# introduction This entire function was written by Lindsay Keegan based off
# a tutorial by Edwin Thoen posted here:
# http://www.r-statistics.com/2013/07/creating-good-looking-survival-curves-the-ggsurv-function/
# and slightly adapted by Avery McIntosh, 2015.
#See TCGA AML 05May2017.Rmd for example usage.
#S is the fit object
#NEEDS TO BE TESTED AGAIN

ggsurv.m <- function(s, CI = 'def', plot.cens = T, surv.col = 'gg.def',
                     cens.col = 'red', lty.est = 1, lty.ci = 2,
                     cens.shape = " ", back.white = F, xlab = 'Time (days)',
                     ylab = 'Retention Probability', main = '') {

  strata <- length(s$strata)
  n <- s$strata

  #update JS to deal with the super imposed curves to seq by 4 for 24 and split with comma
  groups <- factor(unlist(lapply(strsplit(names(s$strata),split = "=|,"),
                                 function(x) paste(x[2], x[4], sep="."))))

  gr.name <-  unlist(strsplit(names(s$strata), '='))[1] #LSC17
  gr.df <- vector('list', strata)
  ind <- vector('list', strata)
  n.ind <- c(0,n); n.ind <- cumsum(n.ind)

  for(i in 1:strata) ind[[i]] <- (n.ind[i]+1):n.ind[i+1]

  for(i in 1:strata){
    gr.df[[i]] <- data.frame(
      time = c(0, s$time[ ind[[i]] ]),
      surv = c(1, s$surv[ ind[[i]] ]),
      up = c(1, s$upper[ ind[[i]] ]),
      low = c(1, s$lower[ ind[[i]] ]),
      cens = c(0, s$n.censor[ ind[[i]] ]),
      group = rep(groups[i], n[i] + 1))
  }

  dat <- do.call(rbind, gr.df)
  dat.cens <- subset(dat, cens != 0)

  pl <- ggplot(dat, aes(x = time, y = surv, group = group)) +
    xlab(xlab) + ylab(ylab) + ggtitle(main) +
    geom_step(aes(col = group),lty=1, size=1.15) +
    geom_point(data = dat.cens, aes(x=time, y=surv, color=group), shape=3, size=4) +
    labs(y= "Fraction Surviving", x = paste("Follow-up in", "Years")) +
    theme(plot.title = element_text(hjust = 0.5, size=18),
          panel.background = element_rect(fill="white"),
          panel.grid.major = element_blank(),
          panel.grid.minor = element_blank(),
          axis.text.y = element_text(size = 16),
          axis.text.x = element_text(size=16),
          axis.title = element_text(size = 16),
          panel.border = element_rect(colour = "dark grey", fill= NA, size=1.0),
          legend.text = element_text(size=14),
          legend.title = element_text(size=14)) +
    scale_y_continuous(limits = c(0,1), breaks = seq(0,1,0.2), minor_breaks = NULL) +
    scale_x_continuous(limits = c(0,max(dat$time)), breaks = seq(0,max(dat$time), 2))

  # list <- list(n, groups, gr.name)
  # names(list) <- c("n", "groups", "gr.name")
  return(p1)

}

# KM.plots <- function(df, groupBy, type="OS", covariate,cohort,cc=NULL,
#                      riskTable=TRUE,f.levels=NULL, max.year=NULL){
#   library(dplyr)
#   library(survival)
#   library(magrittr)
#   suppressPackageStartupMessages(require(gridExtra))
#
#   #df is the cleaned CDE with patient IDs as rownames. stringsAsFactors=FALSE
#   #type is for relapse due to having strate with 0 members...
#   #covariate is a character string for the column in CDE to be the explanatory variable
#
#   #Formula for survival analysis
#   if (cohort == "0531"){
#     os.est <- "Surv(Overall.Survival.Time.in.Days/365.25, OS.ID)"
#     efs.est <- "Surv(Event.Free.Survival.Time.in.Days/365.25, Event.ID)"
#
#     OS.form <- as.formula(paste(os.est,' ~ ',covariate))
#     EFS.form <- as.formula(paste(efs.est,' ~ ', covariate))
#   }else if (cohort == "1031"){
#     os.est <- "Surv(OS.time..days./365.25, OS.ID)"
#     efs.est <- "Surv(EFS.time..days./365.25, Event.ID)"
#
#     #updated on 6/15/18 to reflect the new CDEs for 1031.
#     OS.form <- as.formula(paste(os.est, ' ~ ', covariate))
#     EFS.form <- as.formula(paste(efs.est, ' ~ ', covariate))
#   }
#
#   #function for colors
#   colorcodes <- function(df){
#     strata <- unlist(unique(df[,covariate])) #Unique groups (strata)
#     strata <- strata[order(strata)] #ensure the alphabetical order
#     len <- length(strata)
#
#     if ( ! is.null(cc)){
#       return(cc)
#
#     }else{
#       colors <- c("dodgerblue4", "firebrick1", "green4",  "darkorange",
#                   "turquoise3","azure4", "chartreuse1", "darkmagenta",
#                   "deeppink", "darkslategray1", "navajowhite2",
#                   "brown3", "darkgoldenrod3", "deepskyblue1", "lightcoral",
#                   "mediumorchid", "saddlebrown")
#       cc <- NULL
#       for ( i in 1:len){
#         c <- colors[i]
#         cc <- c(cc,c)
#       }
#       names(cc) <- strata
#       return(cc)
#     }
#   }
#
#   #perform survival analysis for all factor levels
#   if (type=="OS"){
#     grouped.df <-  df %>%
#       group_by_(groupBy) %>%
#       do(OS.fit=survfit(OS.form, data=.),
#          OS.diff=survdiff(OS.form, data=.), #survdiff for log-rank p-value
#          OS=SurvivalPlot(survfit(OS.form, data=.), #survival plots
#                          LegendTitle = "",
#                          timeUnit = "Years",
#                          max.year = max.year,
#                          colors =  colorcodes(.)),
#
#          EFS.fit=survfit(EFS.form, data=.),
#          EFS.diff=survdiff(EFS.form, data=.),
#          EFS=SurvivalPlot(survfit(EFS.form, data=.),
#                           LegendTitle="",
#                           timeUnit= "Years",
#                           max.year = max.year,
#                           colors= colorcodes(.)))
#   }
#
#   #function to calculate a p-value
#   pvalue <- function(survdiff.res){
#     p <- pchisq(survdiff.res$chisq, df=length(survdiff.res$n) - 1, lower=FALSE)
#     p <- ifelse(p < 0.001, "p < 0.001", paste0("p = ", round(p, digits = 3)))
#     return(p)
#   }
#
#   #Function to extract N of cohort.
#   groupSize <- function(survdiff.res){paste0("N = ", sum(survdiff.res$n))}
#
#   #Update the survial plots with an informative title and p values, and group N
#   types <- c("Relapse", "Failure", "OS", "EFS")
#   plots <- which(colnames(grouped.df) %in% types)
#
#   #for each row (nrow > 1, when grouping variable given)
#   for ( i in 1:nrow(grouped.df)){
#
#     #for each OS/EFS column
#     for (plot.idx in plots){
#       idx.logrank <- plot.idx-1
#       N <- groupSize(grouped.df[[idx.logrank]][[i]])
#       pval <- pvalue(grouped.df[[idx.logrank]][[i]]) %>%
#         paste(., N, sep="\n")
#
#       #Add more informative Legend Title
#       legendTitle <- as.character(grouped.df[i,1]) %>%
#         unique(c(groupBy, .)) %>%
#         paste(., ": ", covariate, sep="")
#
#       for (j in 1:2){
#         grouped.df[[plot.idx]][[i]]$scales$scales[[j]]$name <-  legendTitle
#       }
#
#       #Add Main Title that is the column name (eg Relapse,OS, EFS, etc)
#       grouped.df[[plot.idx]][[i]]$labels$title <- names(grouped.df[plot.idx])
#
#       #Add P value and the N
#       grouped.df[[plot.idx]][[i]] <- grouped.df[[plot.idx]][[i]] +
#         annotate(geom="text", x=1, y=0.05, label=pval, size=5)
#
#       if(riskTable){
#
#         fit.idx <- plot.idx-2
#         fit <- grouped.df[[fit.idx]][[i]]
#         risk.tab <- risk.table(survFit.obj = fit, ret.data = TRUE)
#         risk.tab.p <- risk.table(survFit.obj = fit,col = cc, f.levels = f.levels, max.year=max.year)
#
#         # Create a blank plot for place-holding
#         blank.pic <- ggplot(risk.tab, aes(time, surv)) +
#           geom_blank() +
#           theme_bw() +
#           theme(axis.text.x = element_blank(),
#                 axis.text.y = element_blank(),
#                 axis.title.x = element_blank(),
#                 axis.title.y = element_blank(),
#                 axis.ticks = element_blank(),
#                 panel.grid = element_blank(),
#                 panel.border = element_blank())
#
#
#         # grid.arrange(grouped.df[[plot.idx]][[i]], blank.pic, risk.tab.p, clip = FALSE, nrow = 3,
#         #                    ncol = 1, heights = unit(c(2, .1, .25), c("null", "null", "null")))
#
#         grouped.df[[plot.idx]][[i]] <-  arrangeGrob(grouped.df[[plot.idx]][[i]], blank.pic, risk.tab.p,
#                                                     clip = FALSE, nrow = 3, ncol = 1,
#                                                     heights = unit(c(2, .2, .4),c("null", "null", "null")))
#
#       }
#     }
#   }
#
#   return(grouped.df)
# }

# saveKM <- function(KM.plots.res,dir){
#
#   N <- nrow(KM.plots.res)
#   group <- colnames(KM.plots.res)[1]
#
#   OS <- KM.plots.res$OS
#   EFS <- KM.plots.res$EFS
#
#   saveplot <- function(gg,N){
#     type <- substitute(gg)
#
#     lapply(1:N, function(i)
#       ggsave(filename = paste("TARGET_AML",group,
#                               unlist(KM.plots.res[i,1]),
#                               type,"plot.png", sep="_"),
#              plot = gg[[i]], device = "png", height = 7, width = 12,dpi = 600 ))
#   }
#
#   saveplot(OS,N=N)
#   saveplot(EFS,N=N)
# }

# SurvObjects <- function(df, colNames, group, rho=0, time=NULL){
#   #df is the dataframe with clinical data.
#   #colnames is a character vector (length 2) with the colnames of the time and event, in that order.
#   #group is the column name as character vector of the explanatory variable to test
#   #time is optional if you need to convert days to years for example.
#   #rho is 1 or 0 or inbetween. 0=logrank test and 1=gehan-wilcoxon test
#
#   require(survival)
#   require(survMisc)
#
#   #optional time unit conversion
#   if (is.null(time)){
#     time <- (df[, colNames[1]])
#   }else if (time == "DtoY"){
#     time <- (df[,colNames[1]]/365)
#   }else if (time == "MtoY"){
#     time <- (df[,colNames[1]]/12)
#   }
#
#   #Kaplan Meier Estimates
#   KM <- Surv(time = time, event = df[,colNames[2]])
#
#   if (group == 1){
#     #model fit
#     survFit <- survfit(KM ~ 1)
#     return(survFit)
#
#   }else{
#     #model fit
#     survFit <- survfit(KM ~ df[,group],data=df)
#
#     #cox proportional Hazards
#     cph.test <- coxph(KM ~ df[,group])
#
#     #test proportional hazards assumption
#     cph.zph <- cox.zph(cph.test, transform = "km")
#     rhoPval <- as.data.frame(cph.zph$table)$p #test the PH assumption
#
#     if (length(rhoPval) > 1){
#       rhoPval <- rhoPval[length(rhoPval)]
#     }
#
#     # survMisc compare stat tests for two curves
#     tne <- ten(eval(survFit$call$formula), data=df)
#     capture.output(comp(tne)) -> allTests
#
#     if (rhoPval >= 0.05 ){
#       attr(tne, "lrt")[1,] -> pVal #log-rank test
#       test <- "log.rank"
#     }else if (rhoPval < 0.05){
#       attr(tne, "lrt")[2, ] -> pVal #Gehan-Breslow-Wilcoxon
#       test <- "Gehan.Breslow.Wilcoxon"
#     }
#
#     list <- list(survFit, pVal, cph.test, cph.zph, KM)
#     names(list) <- c("survFit", test, "CoxPH", "PH_Test","KaplanMeirerEst")
#     return(list)
#   }
# }


#' matmult
#'
#' Matrix multiplication.
#'
#' @param x,y
#' fmlmat matrices.
#'
#' @return
#' A matrix of the same type as the inputs.
#'
#' @name matmult
#' @rdname matmult
NULL

#' @rdname matmult
#' @export
setMethod("%*%", signature(x="fmlmat", y="fmlmat"),
  function(x, y)
  {
    ret = linalg_matmult(x=DATA(x), y=DATA(y))
    wrapfml(ret)
  }
)

#' @rdname matmult
#' @export
setMethod("%*%", signature(x="fmlmat", y="matrix"),
  function(x, y)
  {
    y_fml = ytox(x, y)
    ret = linalg_matmult(x=DATA(x), y=DATA(y))
    wrapfml(ret)
  }
)

#' @rdname matmult
#' @export
setMethod("%*%", signature(x="matrix", y="fmlmat"),
  function(x, y)
  {
    x_fml = ytox(y, x)
    ret = linalg_matmult(x=DATA(x), y=DATA(y))
    wrapfml(ret)
  }
)


model_snowdry <- function (Sdry_t1 = 0.0,
         Snowaccu = 0.0,
         Mrf = 0.0,
         M = 0.0){
    #'- Name: SnowDry -Version: 1.0, -Time step: 1
    #'- Description:
    #'            * Title: water in solid state in the snow cover Calculation
    #'            * Author: STICS
    #'            * Reference: doi:http://dx.doi.org/10.1016/j.agrformet.2014.05.002
    #'            * Institution: INRA
    #'            * Abstract: water in solid state in the snow cover
    #'- inputs:
    #'            * name: Sdry_t1
    #'                          ** description : water in solid state in the snow cover in previous day
    #'                          ** inputtype : variable
    #'                          ** variablecategory : state
    #'                          ** datatype : DOUBLE
    #'                          ** default : 0.0
    #'                          ** min : 0.0
    #'                          ** max : 500.0
    #'                          ** unit : mmW
    #'                          ** uri :
    #'            * name: Snowaccu
    #'                          ** description : snowfall accumulation
    #'                          ** inputtype : variable
    #'                          ** variablecategory : state
    #'                          ** datatype : DOUBLE
    #'                          ** default : 0.0
    #'                          ** min :
    #'                          ** max :
    #'                          ** unit : mmW/d
    #'                          ** uri :
    #'            * name: Mrf
    #'                          ** description : liquid water in the snow cover in the process of refreezing
    #'                          ** inputtype : variable
    #'                          ** variablecategory : state
    #'                          ** datatype : DOUBLE
    #'                          ** default : 0.0
    #'                          ** min :
    #'                          ** max :
    #'                          ** unit : mmW/d
    #'                          ** uri :
    #'            * name: M
    #'                          ** description : snow in the process of melting
    #'                          ** inputtype : variable
    #'                          ** variablecategory : state
    #'                          ** datatype : DOUBLE
    #'                          ** default : 0.0
    #'                          ** min :
    #'                          ** max :
    #'                          ** unit : mmW/d
    #'                          ** uri :
    #'- outputs:
    #'            * name: Sdry
    #'                          ** description : water in solid state in the snow cover
    #'                          ** variablecategory : state
    #'                          ** datatype : DOUBLE
    #'                          ** min : 0.0
    #'                          ** max : 500.0
    #'                          ** unit : mmW
    #'                          ** uri :
    Sdry <- 0.0
    if (M <= Sdry_t1)
    {
        tmp_sdry <- Snowaccu + Mrf - M + Sdry_t1
        if (tmp_sdry < 0.0)
        {
            Sdry <- 0.001
        }
        else
        {
            Sdry <- tmp_sdry
        }
    }
    return (list('Sdry' = Sdry))
}


# 4. faza: Napredna analiza podatkov

#RAZVRŠČANJE V SKUPINE

#funkcije iz predavanj in vaj, ki sem jih uporabila v nadaljevanju:
hc.kolena = function(dendrogram, od = 1, do = NULL, eps = 0.5) {
  n = length(dendrogram$height) + 1  # število primerov in nastavitev parametra do
  if (is.null(do)) {
    do = n - 1
  }
  k.visina = tibble(
    k = as.ordered(od:do),
    visina = dendrogram$height[do:od]) %>%
    mutate(dvisina = visina - lag(visina)) %>%         # sprememba višine
    mutate(               # ali se je intenziteta spremembe dovolj spremenila?
      koleno = lead(dvisina) - dvisina > eps)
  k.visina
}

# iz tabele k.visina vrne seznam vrednosti k, pri katerih opazujemo koleno
hc.kolena.k = function(k.visina) {
  k.visina %>%
    filter(koleno) %>%
    dplyr::select(k) %>%
    unlist() %>%
    as.character() %>%
    as.integer()
}

# narišemo diagram višin združevanja
diagram.kolena = function(k.visina) {
  k.visina %>% ggplot() +
    geom_point(
      mapping = aes(x = k, y = visina),
      color = "red") +
    geom_line(
      mapping = aes(x = as.integer(k), y = visina),
      color = "red") +
    geom_point(
      data = k.visina %>% filter(koleno),
      mapping = aes(x = k, y = visina),
      color = "blue", size = 2) +
    ggtitle(paste("Kolena:", paste(hc.kolena.k(k.visina), collapse = ", "))) +
    xlab("število skupin (k)") + ylab("razdalja pri združevanju skupin") +
    theme_classic()
}

diagram.skupine = function(podatki, oznake, skupine, k) {
  podatki = podatki %>%
    bind_cols(skupine) %>%
    rename(skupina = ...4)

  d = podatki %>%
    ggplot(
      mapping = aes(
        x = x, y = y, color = skupina
      )
    ) +
    geom_point() +
    geom_label(label = oznake, size = 2) +
    scale_color_hue() +
    theme_classic()

  for (i in 1:k) {
    d = d + geom_encircle(
      data = podatki %>%
        filter(skupina == i)
    )
  }
  d
}

obrisi = function(podatki, hc = TRUE, od = 2, do = NULL) {
  n = nrow(podatki)
  if (is.null(do)) {
    do = n - 1
  }

  razdalje = dist(podatki)

  k.obrisi = tibble()
  for (k in od:do) {
    if (hc) {
      o.k = hclust(razdalje) %>%
        cutree(k) %>%
        silhouette(razdalje)
    } else {
      set.seed(42) # zato, da so rezultati ponovljivi
      o.k = kmeans(podatki, k)$cluster %>%
        silhouette(razdalje)
    }
    k.obrisi = k.obrisi %>% bind_rows(
      tibble(
        k = rep(k, n),
        obrisi = o.k[, "sil_width"]
      )
    )
  }
  k.obrisi$k = as.ordered(k.obrisi$k)

  k.obrisi
}

obrisi.povprecje = function(k.obrisi) {
  k.obrisi.povprecje = k.obrisi %>%
    group_by(k) %>%
    summarize(obrisi = mean(obrisi))
}

obrisi.k = function(k.obrisi) {
  obrisi.povprecje(k.obrisi) %>%
    filter(obrisi == max(obrisi)) %>%
    summarize(k = min(k)) %>%
    unlist() %>%
    as.character() %>%
    as.integer()
}

diagram.obrisi = function(k.obrisi) {
  ggplot() +
    geom_boxplot(
      data = k.obrisi,
      mapping = aes(x = k, y = obrisi)
    ) +
    geom_point(
      data = obrisi.povprecje(k.obrisi),
      mapping = aes(x = k, y = obrisi),
      color = "red"
    ) +
    geom_line(
      data = obrisi.povprecje(k.obrisi),
      mapping = aes(x = as.integer(k), y = obrisi),
      color = "red"
    ) +
    geom_point(
      data = obrisi.povprecje(k.obrisi) %>%
        filter(obrisi == max(obrisi)) %>%
        filter(k == min(k)),
      mapping = aes(x = k, y = obrisi),
      color = "blue"
    ) +
    xlab("število skupin (k)") +
    ylab("obrisi (povprečje obrisov)") +
    ggtitle(paste("Maksimalno povprečje obrisov pri k =", obrisi.k(k.obrisi))) +
    theme_classic()
}

#________________________________________________________________________________
#uporabila bom podatke tabela2
tabela.skupine1.1 <- tabela2 %>%
  dplyr::select(1, 2, 6) %>%
  filter(regija != "SLOVENIJA") %>%
  group_by(regija, solsko.leto) %>%
  summarise(promil = sum(delez.promil))

tabela.skupine1.2 <- tabela.skupine1.1%>%
  pivot_wider(
    names_from = solsko.leto,
    values_from = promil
  )

#wardove razdalje:
regije <- tabela.skupine1.2[, 1] %>% unlist()
razdalje <- tabela.skupine1.2[, -1] %>% dist()
dendrogram <- razdalje %>% hclust(method = "ward.D")

#tabela, za risanje skupin
regije.x.y <- as.tibble(razdalje %>% cmdscale(k = 2)) %>%
  bind_cols(regije) %>%
  dplyr::select(regija = ...3, x = V1, y = V2)

#DENDROGRAM
#plot(dendrogram, labels = regije, ylab = "višina", main = NULL)

#HIERARHIČNO RAZVRŠČANJE V SKUPINE
#nariše graf in obarva kolena modro
graf.kolena <- dendrogram %>% hc.kolena() %>% diagram.kolena() #dobila sem dva oprimalna k: 2 in 3
#za k=2:
skupine.2 <- dendrogram %>%
  cutree(k = 2) %>%
  as.ordered()
diagram.skupine(regije.x.y, regije.x.y$regija, skupine.2, 2)
#za k=3:
skupine.3 <- dendrogram %>%
  cutree(k = 3) %>%
  as.ordered()
diagram.skupine(regije.x.y, regije.x.y$regija, skupine.3, 3)

#METODA K-TIH VODITELJEV:
#za r.km:
r.km <- tabela.skupine1.2[, -1] %>% obrisi(hc = FALSE)
opt.st.skupin.r.km <- obrisi.k(r.km)     #optimaleno k=2
diagram.obrisi(r.km)

set.seed(123)
skupine <- tabela.skupine1.2[, -1] %>%
  kmeans(centers = 2) %>%
  getElement("cluster") %>%
  as.ordered
#narišem graf in obkroži skupine:
diagram.skupine(regije.x.y, regije.x.y$regija, skupine, opt.st.skupin.r.km)

#za r.hc:
r.hc <- tabela.skupine1.2[, -1] %>% obrisi(hc = TRUE)
opt.st.skupin.r.hc <- obrisi.k(r.hc)     #optimaleno k=2
diagram.obrisi(r.hc)

skupine <- tabela.skupine1.2[, -1] %>%
  dist() %>%
  hclust(method = "ward.D") %>%
  cutree(k = 2) %>%
  as.ordered()
#narišemo graf in obkroži skupine:
diagram.skupine(regije.x.y, regije.x.y$regija, skupine, opt.st.skupin.r.hc)

#funkcija, ki nariše graf za optimalno število kolen glede na nacin dolocanja
narisi <- function(nacin){
  if(nacin == "Hierarhično razvrščanje"){
    graf.kolena <- dendrogram %>% hc.kolena() %>% diagram.kolena()
    graf.kolena
  }else {
    r.km <- tabela.skupine1.2[, -1] %>% obrisi(hc = FALSE)
    diagram.obrisi(r.km)
  }
}

narisi("Hierarhično razvrščanje")
narisi("Metoda k-tih voditeljev")

#zemljevid za razvrščanje po skupinah z metodo HAC
source("lib/uvozi.zemljevid.r")
zemljevid <- uvozi.zemljevid("http://kt.ijs.si/~ljupco/lectures/appr/zemljevidi/si/gadm36_SVN_shp.zip",
                             "gadm36_SVN_1", mapa = "zemljevidi", encoding="UTF-8")

zemljevid <- zemljevid %>%
  spTransform(CRS("+proj=longlat +datum=WGS84")) # pretvorimo v ustrezen format

loc <- locale(encoding="UTF-8")
for (col in names(zemljevid)) {
  if (is.character(zemljevid[[col]])) {
    zemljevid[[col]] <- zemljevid[[col]] %>%
      parse_character(locale=loc)
  }
}

zemljevid@data
zemljevid.regije <- zemljevid
names(zemljevid.regije)
zemljevid.regije$NAME_1

zemljevid.regije$NAME_1 <- factor(zemljevid.regije$NAME_1)

lvls <- levels(zemljevid.regije$NAME_1)

#funkcija, ki nariše zemljevid glede na število izbranih skupin
zemljevid <- function(k){
  skupine.k <- dendrogram %>%
    cutree(k = k) %>%
    as.ordered()

  podatki.k <- regije.x.y %>%
    bind_cols(skupine.k) %>%
    rename(skupina = ...4) %>%
    dplyr::select(1, 4)

  podatki.k$regija <- str_replace_all(
    podatki.k$regija,
    c("Primorsko-notranjska" = "Notranjsko-kraška", "Posavska"= "Spodnjeposavska")
  )

  # spremenim imena regij, da se ujemajo z zemljevidom, kratice spremenim nazaj v dolga imena
  imena.regij.nazaj = tibble(
    regija = c("Gorenjska", "Goriška", "Jugovzhodna Slovenija", "Koroška",
               "Obalno-kraška", "Osrednjeslovenska", "Podravska", "Pomurska",
               "Spodnjeposavska", "Notranjsko-kraška", "Savinjska","Zasavska",
               "SLOVENIJA"),
    oznaka = c(
      "kr", "ng", "nm", "sg", "kp", "lj", "mb", "ms", "kk", "po", "ce", "za", "slo")
  )

  tmap_mode("plot")
  zemljevid1 <- merge(
    x = podatki.k,
    y = imena.regij.nazaj,
    by = "regija", all.x = TRUE
  ) %>%
    dplyr::select(regija, skupina)

  zemljevid1 <- merge(zemljevid.regije,
                      zemljevid1,
                      by.x = "NAME_1",
                      by.y = "regija")

  zem <- tm_shape(zemljevid1) +
    tm_polygons(
      "skupina" ,
      style = "pretty",
      palette="Blues",
      title="Skupina:"
    ) +
    tm_layout(
      "",
      legend.title.size=1,
      legend.text.size = 0.8,
      legend.position = c("right","bottom"),
      legend.bg.color = "white",
      legend.bg.alpha = 1) +
    tm_text("NAME_1", size = 4/5)

  zem

}

#narišem zemljevid za k=2:
zem2 <- zemljevid(2)

#narišem zemljevid za k=3:
zem3 <- zemljevid(3)

#_______________________________________________________________________________
#NAPOVEDNI MODELI
#Linearna regresija
preurejena.4 <- tabela4 %>%
  dplyr::select(2, 4, 5) %>%
  group_by(vrsta.izobrazevanja, solsko.leto) %>%
  summarise(stevilo.diplomantov = sum(stevilo.diplomantov))

tabela.podatki <- tabela3 %>%
  dplyr::select(1, 5, 6) %>%
  group_by(vrsta.izobrazevanja, solsko.leto) %>%
  summarise(stevilo.vpisanih = sum(stevilo.vpisanih)) %>%
  left_join(preurejena.4, by = c("vrsta.izobrazevanja" = "vrsta.izobrazevanja",
                            "solsko.leto" = "solsko.leto")
            ) %>%
  filter(solsko.leto != "2020/21", solsko.leto != "2009/10", solsko.leto != "2010/11")

gg <- ggplot(tabela.podatki, aes(x=stevilo.vpisanih, y=stevilo.diplomantov)) + geom_point()

stevilo.diplomantov ~ stevilo.vpisanih
stevilo.diplomantov ~ stevilo.vpisanih + I(stevilo.vpisanih^2)

podatki <- tabela.podatki
n <- nrow(podatki)
r <- sample(1:n)
formula <- stevilo.diplomantov ~ stevilo.vpisanih

k <- 5
set.seed(123)
razrez <- cut(1:n, k, labels = FALSE)
razbitje <- split(r, razrez)

pp.napovedi <- rep(0, n)
for (i in 1:length(razbitje)){
  train.data <- podatki [-razbitje[[i]], ]           #učni podatki
  test.data <- podatki[razbitje[[i]], ]              #testni podatki
  model <- lm(data = train.data, formula = formula)  #naučimo model
  napovedi <- predict(model, newdata = test.data)    #napovemo za testne podatke
  pp.napovedi[razbitje[[i]]] <- napovedi
}

pp.napovedi

#izračunamo kvadratno napako -MSE
MSE <- mean((pp.napovedi - podatki$stevilo.diplomantov)^2)

gg <- gg +
  geom_smooth(method = "lm",
              formula = y ~ x,
              se = TRUE,
              fullrange = TRUE,
              color = "green") +
  theme_classic() +
  theme(
    plot.title = element_text(hjust = 0.5) #naslov na sredini
  ) +
  labs(
    x = "Število vpisanih",
    y = "Število diplomantov",
    title = "LINEARNA REGRESIJA"
  )


#Tests and Explanations

#Function Library
try(setwd("~/Shared Files/DPM/Latest"))
source(file="ConsensusMechanism.r")

#Declare List
Scenarios <- vector("list")
SLabels <- vector("list")

#[1] Design Case
SLabels$Base <- "Basic Case - 14/24 [58%] Honest"

M1 <-  rbind(
    c(1,1,0,0),
    c(1,0,0,0),
    c(1,1,0,0),
    c(1,1,1,0),
    c(0,0,1,1),
    c(0,0,1,1))

row.names(M1) <- c("True", "Confused 1", "True", "Confused 2", "Liar", "Liar")
colnames(M1) <- c("C1.1","C2.1","C3.0","C4.0")

Scenarios$Base <- M1
#

# [2] Reversed Matrix
SLabels$Reversed <- "Basic Case - 14/24 [58%] Honest, reversed wording"
M2 <- ReverseMatrix(M1)
Scenarios$Reversed <- M2
#

# [3] Deviance: Deranged Nonconformist
SLabels$Deviance <- "Punishment from Deviating from Stable"

M3 <- rbind(M1[c(1,3),],
            M1[c(1,3),],
            "Liar"=c(0,0,1,1))

Scenarios$Deviance <- M3
#

# [4] Coalitional Deviance: Using a group to game the system.
SLabels$CoalitionalDeviance <- "Targeting Contract (#3) with <50% Conspirators (including 1 AntiTruth Diver)"
SLabels$CoalitionalDeviance2 <- "Targeting Contract (#3) with <50% Conspirators (including 1 AntiTeam Diver)"

 M4b <- rbind("True"=c(1,1,0,0),
              "True"=c(1,1,0,0),
              "True"=c(1,1,0,0),
              "True"=c(1,1,0,0),
              "True"=c(1,1,0,0),
              "True"=c(1,1,0,0),
              "Diver"=c(0,0,1,1), #Diver
              "Liar"=c(1,1,1,0),
              "Liar"=c(1,1,1,0),
              "Liar"=c(1,1,1,0), #4 conspirators
              "Liar"=c(1,1,1,0)) # + 1 Diver     = 5 <6

M4c <- M4b
M4c["Diver",3] <- 0 #Diver negatively correlated with his team

M4d <- rbind(M4c,"FailSafe"=c(.5,.5,.5,.5))

Scenarios$CoalitionalDeviance  <- M4b
Scenarios$CoalitionalDeviance2  <- M4c
#

# [5] Clueless: Passing on a Contract - "I have no idea"
SLabels$CluelessControl <- c("Having no idea - 'passing' on a contract [control]")
SLabels$CluelessTest <- c("Having no idea - 'passing' on a contract [test]")

M3a <- rbind(M1[1,],M1[1,],M1[1,],M1[1,],M1[1,],M1[1,],M1[1,]) #bigger reference case
row.names(M3a) <- rep("True",nrow(M3a))

M3m <- M3a
M3m[2,2] <- NA

Scenarios$CluelessControl <- M3a
Scenarios$CluelessTest <- M3m
#

# [6] Inchoerence
SLabels$Incoherence <- c("Punishing Incoherence - I KNOW that this contract is spam/nonsense")
SLabels$Incoherence2 <- c("Punishing Incoherence - I KNOW that this contract is spam/nonsense [2]")

M6 <- M3a
M6[-3,2]  <- .5 #Incoherent

M6b <- M6
M6b[7,2]  <- 0 #Incentive examination

Scenarios$Inchoherence <- M6
Scenarios$Inchoherence2 <- M6b
#

# [7] Unanimous: Perfect Consensus Bug
SLabels$Unanimous <- c("Having everyone agree perfectly (desireable) crashes PCA")

PerCon <- rbind(M1[1,], M1[1,], M1[1,], M1[1,])

Scenarios$PerCon <- PerCon
#

# [8] Contract Gaming
SLabels$Gaming <- c("Gaming the Contracts")

M9 <- cbind(M1,"C0.5"=.5,"C0.5"=.5,"C0.5"=.5,"C0.5"=.5,"C0.5"=.5,"C0.5"=.5,"C0.5"=.5,"C0.5"=.5)
M9[5:6,5:12] <- c(0,1,1,0)
M9 <- rbind(M9,M9,M9,M9)

Scenarios$Gaming <- M9
#

# [9] Handling Missing Values
SLabels$Missing1 <- c("A minority of players give missing values to 1 contract")
SLabels$Missing2 <- c("A majority of players give missing values to a minority of their contracts")
SLabels$Missing3 <- c("All players give missing values to a minority of their contracts")
SLabels$Missing4 <- c("Some players give missing values to a majority of their contracts")
SLabels$Missing5 <- c("All players give missing values to a majority of their contracts")

M10a <- cbind(M1,"C0"=c(0,NA,0,NA,1,1))

M10b <- cbind(M10a, "C1"=c(1,1,1,NA,0,0), "C1"=c(NA,NA,NA,0,1,1))
M10b <- rbind(M10b,M10b)

M10c <- M10b
M10c[5,1] <- NA ; M10c[6,2] <- NA ; M10c[11,1] <- NA ; M10c[12,2] <- NA ;

M10d <- M10b[-11:-12,]
M10d[5,3:6] <- NA ; M10d[6,1:4] <- NA ;
M10d[7:8,1:2] <- NA ; M10d[2,2:3] <- NA;

M10e <- rbind(M1,M1)
M10e <- cbind(M10e[,1],M10e,M10e)

M10e[1,1:5] <- NA
M10e[2,2:6] <- NA
M10e[3,3:7] <- NA
M10e[4,4:8] <- NA
M10e[5,5:9] <- NA
M10e[6,c(6:9,1)] <- NA
M10e[7,c(7:9,1:2)] <- NA
M10e[8,c(8:9,1:3)] <- NA
M10e[9,c(9,1:4)] <- NA
M10e[10,1:5] <- NA
M10e[11,2:6] <- NA
M10e[12,3:7] <- NA

Scenarios$Missing1 <- M10a
Scenarios$Missing2 <- M10b
Scenarios$Missing3 <- M10c
Scenarios$Missing4 <- M10d
Scenarios$Missing5 <- M10e
#

# [10] Riven Judgements
SLabels$Riven <- "Separate but equal subgroups, and their recombination. [1]"
SLabels$Riven2 <- "Separate but equal subgroups, and their recombination. [2]"

Mg <- rbind( cbind(M1,   M1*NA, M1*NA),
             cbind(M1*NA,M1,    M1*NA),
             cbind(M1*NA,M1*NA, M1))

Mg2 <- Mg
Mg2[7,1] <- 1

Scenarios$Riven <- Mg
Scenarios$Riven2 <- Mg2
#

### * Test Results and Commentary * ###

Factory(Scenarios$Base)
CompareIncentives(Scenarios$Base)
#Good.
Chain(X=Scenarios$Base)
#Very good. Conforms quickly to a correct prediction.
#I'm thinking one block per day, assuming we smooth difficulty correctly.

Factory(Scenarios$Reversed)
all.equal(Factory(Scenarios$Reversed)$Agents,Factory(Scenarios$Base)$Agents) #TRUE
#Identical incentive structure, despite reversed inputs and outputs.
#Good.

Factory(Scenarios$Deviance)
#Biggest Deviator gets CRUSHED to zero. High-Stakes!
#Good.

Factory(Scenarios$CoalitionalDeviance)
#Success: An attempted <51% attack which failed.

Factory(Scenarios$CoalitionalDeviance2)
# Oh, no: A Sucessful <51% attack 'Friendly Fire' ...will have to address this.

  #Pre-Analytics
  CompareIncentives(X=Scenarios$CoalitionalDeviance2)

  row.names( Scenarios$CoalitionalDeviance2 )[7] <- "Liar"
  CompareIncentives(X=Scenarios$CoalitionalDeviance2)

  # [1] Success: 'Symmetric Friendly Fire'  (ie Team truth forms a coalition of their own)
  Scenarios$CoalitionalDeviance3 <- Scenarios$CoalitionalDeviance2
  Scenarios$CoalitionalDeviance3[6,] <- c(0,0,1,1)

  CompareIncentives(X=Scenarios$CoalitionalDeviance3)
  Chain(Scenarios$CoalitionalDeviance3,N=100)
  #Team 'True' wins via symmetry-exploitation

  # [2] Success: 'Cold Feet 1' (a single player abandons the coalition)
  Scenarios$CoalitionalDeviance4 <- Scenarios$CoalitionalDeviance2
  Scenarios$CoalitionalDeviance4[8,] <- c(1,1,0,0)

  CompareIncentives(X=Scenarios$CoalitionalDeviance4)
  Ss <- Chain(Scenarios$CoalitionalDeviance4,N=70)[[70]]$Agents
  Ss <- data.frame(NewRep=as.numeric(Ss[,"NewRep"]),Group=row.names(Ss))
  aggregate(.~Group,data=Ss, FUN=sum)

  Scenarios$CoalitionalDeviance5 <- Scenarios$CoalitionalDeviance2
  Scenarios$CoalitionalDeviance5[8,] <- c(1,1,0,0)
  Scenarios$CoalitionalDeviance5[9,] <- c(1,1,0,0)

  CompareIncentives(X=Scenarios$CoalitionalDeviance5)
  Ss <- Chain(Scenarios$CoalitionalDeviance5,N=50)[[50]]$Agents
  Ss <- data.frame(NewRep=as.numeric(Ss[,"NewRep"]),Group=row.names(Ss))
  aggregate(.~Group,data=Ss, FUN=sum)
  #Notice after 50 rounds, the devil [=King of Liars] has actually become the two bottommost liars, as they represent the most significant source of confusion.
  #Team 'True' wins via stoicism

  #[3] Recursive Friendly Fire - a sub-coalition forms to defect, but a sub-coalition of this coalition forms to defect again.
  Scenarios$CoalitionalDeviance6 <- rbind(c(1,1,0,0,1),
                                          c(1,1,0,0,1),
                                          c(1,1,0,0,1),
                                          c(1,1,0,0,1),
                                          c(1,1,0,0,1),
                                          c(1,1,0,0,1),
                                          c(1,1,0,0,1),
                                          c(1,1,0,0,1),
                                          c(1,1,0,0,1),
                                          c(1,1,0,0,1), #10
                                          c(1,1,0,0,1),
                                          c(1,1,0,0,1),
                                          c(1,1,0,0,1),
                                          c(1,1,0,0,1),
                                          c(1,1,0,0,1), #15 (60%)

                                          #Coalition 1, backstabbing Truth to game contract 3
                                          c(0,0,0,1,0), #1 - Friendly Fire
                                          c(1,1,1,0,1),
                                          c(1,1,1,0,1),
                                          c(1,1,1,0,1),
                                          c(1,1,1,0,1),
                                          c(1,1,1,0,1), #6 (24%)

                                          #Coalition 2, 'backstabbing' Coalition 1 to game contract 4
                                          c(0,0,1,0,0), #1 - Friendly Fire
                                          c(1,1,0,1,1),
                                          c(1,1,0,1,1),
                                          c(1,1,0,1,1)) #4 (16%)

  row.names(Scenarios$CoalitionalDeviance6) <- c(rep('Truth',15),rep('Lie 1',6),rep('Lie 2',4))
  Factory(Scenarios$CoalitionalDeviance6)
  CompareIncentives(Scenarios$CoalitionalDeviance6)
  #fantastic sucess...Lie 2 loses less

  #clearly, the Nash Equilibrium
  Scenarios$CoalitionalDeviance7 <- Scenarios$CoalitionalDeviance6[1:21,]
  Scenarios$CoalitionalDeviance7 <- rbind(Scenarios$CoalitionalDeviance7, rbind(
                                                                              'Truth 2'=c(1,1,0,0,1),
                                                                              'Truth 2'=c(1,1,0,0,1),
                                                                              'Truth 2'=c(1,1,0,0,1),
                                                                              'Truth 2'=c(1,1,0,0,1)))
  Factory(Scenarios$CoalitionalDeviance7)
  CompareIncentives(Scenarios$CoalitionalDeviance7)

  # [4] Passive - Sideways expansion by 2 contracts
  Scenarios$CoalitionalDeviance8 <- cbind(Scenarios$CoalitionalDeviance2,Scenarios$CoalitionalDeviance2[,1:2])
  Factory(Scenarios$CoalitionalDeviance8)
  CompareIncentives(Scenarios$CoalitionalDeviance8)
  #Success, larger number of contracts makes this attack improbable.

  Scenarios$CoalitionalDeviance9 <- cbind(Scenarios$CoalitionalDeviance2,
                                          Scenarios$CoalitionalDeviance2,
                                          Scenarios$CoalitionalDeviance2,
                                          Scenarios$CoalitionalDeviance2[,-3])
  Factory(Scenarios$CoalitionalDeviance9)
  CompareIncentives(Scenarios$CoalitionalDeviance9)
  #The attack must expand proportionally.

Factory(Scenarios$CluelessControl)
Factory(Scenarios$CluelessTest)
#Finding: 2 falls from tie at 5th .11 to a tie at 7th with .07; no impact on other results: success.

#Note: Must be a discrete set of options: c(1,0,NA,.5)    ---- !!! by extention, Catch must be implemented in FillNA. Indeed, in this example our lazy character is punished twice.
#otherwise there will likely be pragmatic individuals who rationally deviate to answers like ~.85 or ~.92 or some nonsense. [obviously]

Factory(Scenarios$Inchoherence)
Factory(Scenarios$Inchoherence2)
#interesting behavior, but incentive compatible, particularly given low Schelling Salience
#incentive to switch to the consensus .5

Factory(Scenarios$PerCon)
#No problems.

Factory(Scenarios$Gaming)
CompareIncentives(Scenarios$Gaming)
#more or less what i expected

Factory(Scenarios$Missing1)
Factory(Scenarios$Missing2)
Factory(Scenarios$Missing3)
Factory(Scenarios$Missing4)
Factory(Scenarios$Missing5)
#Works

Factory(Scenarios$Riven)
Factory(Scenarios$Riven2)
#

# !!! Must FillNa with .5 FIRST, then average in, to prevent monopoly voting on brand-new contracts. (Actually, if it will eventually be ruled .5).

#Voting Across Time
#Later Votes should count more
#! ...simple change = ConoutFinal becomes exponentially smoothed result of previous chains.
#! require X number of chains (blocks) before the outcome is officially determined (two weeks?)

# Will need:
# 1] Percent Voted
# 2] Time Dimension of blocks.

#
# Possible solutions:
#   1 - sequential filling of NAs (sequential removal of columns) - pre-processing replace with average?
#   2 - what about the 'expert factor' idea? what happened to that?
#   3 - Completely replace FillNa with Reputations (lots of positives here)

#TO-DO
#Cascading reputation .6,.5,.3.,2., etc =   dexp(nrow(Mg))

#Mysterious behavior - loading only on first factor
#solutions
# 1- ignore. incentives will encourage filling out of contracts on 'obvious' events
# 2 - use later factors. Unknown what behavior could result from this

# [ Additive Reputation ] Is reputation completely additive? - Yes, now.

Mar1 <- M1
r1 <- rep(1/6,6)

Mar2 <- M1[-6,]
r2 <- c(1/6, 1/6, 1/6, 1/6, 2/6)

Mar3 <- M1[c(1,2,4,5),]
r3 <- c( 2/6, 1/6, 1/6, 2/6)

Mar4 <- M1[c(1,2,4,5,6),]
r4 <- c( 2/6, 1/6, 1/6, 1/6, 1/6)

Factory(Mar1,Rep=r1)$Agents
Factory(Mar2,Rep=r2)$Agents
Factory(Mar3,Rep=r3)$Agents
Factory(Mar4,Rep=r4)$Agents

#Is reputation additive? Yes (excluding NA-born effects, we could correct with Rep/mean(Rep) but NA is not part of equilibrium so it shouldnt matter).

Factory(Mg)$Agents[,c("OldRep","ThisRep","SmoothRep")]
Factory(Mg2)$Agents[,c("OldRep","ThisRep","SmoothRep")]
#True 1 of group 2 skyrockets ahead, as desired.

#upon reflection, I dont think this 'problem' is particularly bad.

# [ Scaling - What are the computational limits (on a normal machine)?]
#Many improvements can be made, of course.

TestLimit <- function(n1,n2,AddNa=1) {
  M_huge <- matrix(round(runif(n1*n2)),n1,n2)
  if(AddNa==1) M_huge[sample(1:(n1*n2),size=(n1*n2)/3)] <- NA
  print(Factory(M_huge))
}

system.time(print("1"))
system.time(TestLimit(100,10))
system.time(TestLimit(1000,100))
# user  system elapsed
# 0.66    0.00    0.65

# system.time(TestLimit(10000,1000))
# user  system elapsed
# 135.03    0.67  135.84

# system.time(TestLimit(100000,100))
# user  system elapsed
# 86.28    0.13   86.42

# system.time(TestLimit(100000,10))
# user  system elapsed
# 5.66    0.03    5.66
# registerDoParallel(cores=4)

# Ten million - 10,000,000. That's pretty big
#
# system.time(TestLimit(5000,500))
# user  system elapsed
# 13.53    0.02   13.52

#Solutions

# [1] - cap the number of rows
  # in its current state, it is basically unlimited - unrealistic
  # the first 100,000 votes are probably decentralized enough ...how low should this number go? (can be a f[ncol(Vmatrix)])
  # given that reputation is fully additive, this would discourage the spreading of reputations
  # this would also solve the "dust bust" question (ie suck up any accounts with tiny dust amounts of reputation)
  # can add the slow retarget to make this grow over the next thousand years (every 4 years?)

#after simply listing the specifics of this solution I realize it is the best candidate

#Steps

# [1] [CANCEL - BAD IDEA] Reward later factors (Comp.2, 3)? Seems risky...lets square the loadings or something to truly emphasize the first factor.
# [2] [DONE] Build R (reward matrix) as a proportion.
# [3] [DONE] What if the guy has no idea? NAs
# [4] [DONE] Fix 'perfect consensus' bug.
# [5] Reward function: a% of current reserve pool? [yes], payouts over time ? [this is whats happening, not finished yet]
# [6] Scale Cost By Funds availiable [????]
# [7] Add payment functionality (pay-per-vote)
# [8] Functionality for new arrivals..old contracts missing data.
# [9] [DONE] Remove incentive to create contracts where consensus is @.5 !
# [10.a] [DONE] Univariate fill missing data in an incentive-consistent way.
# [10.b] [part-DONE] Multivariate fill missing data (recursive? bootstrapped?) [chose simultaneous prediction, and ignore]
# [10.c  - FillNa should faithfully reproduce .5s]

# [adding a NA to a contract can wipe out the validations there, ONLY when filling NA, - fix by using exponential smoothing to calc Rewards] [Fixed!]
# [11] Ask - Ask people the outcomes of contracts...how to maximize (contract reward? d(Cr)?)

#

# [20 - Discourage "Obvious Outcome" Contracts by fee for open interest (more disagreement)]

###

# Ask <- function(M, p=.2, epsilon=.001) {
#
#   #M is the target matrix, r the random variable assigned to a player
#   #p is the weight given to NAs versus disputed outcomes
#   #epsilon is the weight given to otherwise weightless contracts.
#
#   Weight1 <- GetWeight( (Factory(M)[[2]][nrow(M)+4,]) + epsilon)  #Na Weight
#   Weight2 <- Factory(M)[[2]][nrow(M)+2,]                          #Contract Reward weight
#   Weight2 <- GetWeight( ((.25)-(Weight2-.5)^2) + epsilon)         #Transformation to find low-weighted
#
#   WeightC <- p*Weight1 + (1-p)*Weight2
#   ConCol <- sample(1:ncol(M),1,prob=WeightC)
#
#   #print(WeightC)
#
#   return(ConCol)
# }
#
# Ask(M1)

# temp <- unlist(lapply(1:100,FUN=function(x) Ask(M10a)))
# hist(temp)

#see Governance
# # [12] Create a New contract...it should cost(?)
# NewContract <- function(M) { #This is going to rely on a time dimension I havent implemented yet
# }
#
# OpenInterest <- function(M,c) {
#
# }

# [11] [Done] Ask
# [12] NewContract
# [13] Open Interest - Two disagreeing parties each pay Ai (i in 1,2 ; Sum(A)=1) to mint two new coins -- eventually redeemable for 0 and 1-FEEepsilon .

# [14] Subsidized Events - Principals paying Agents to influence the outcome of contracts.

# [14a] A market for an event will allow it's occurance to be noted by the Factory after it's occurance.
# [14b] Partition a 'Verification Space' (private information known only to the agent: date, time, colour, etc).
# [14c] Allow private bets within the Verification Space - possibly a second set of Colored Coins "specical coin".
# [14d] At EoC, people collect proportional to their holdings of insider-information coins.
# [14e] Assurance contracts to allow anyone to fund a 'club good' event.

#Plot
PlotJ(M1)


calmod <- function(target,x,sumstat,tol,gwt,rejmethod=T)
{
# this function uses categorical regression to estimate the posterior probability of a particular model
#      under the ABC framework P(Y=y | S = s)
#
# target is the set of target summary stats - i.e. s, what you measured from the data.
# x is the parameter vector, Y (long vector of numbers from the simulations) and is the dependent variable for the regression
#          This is a categorical variable, taking values 1, 2, .. n where there are n categories (models)
# sumstat is an array of simulated summary stats, S (i.e. independent variables).
# tol is the required proportion of points nearest the target values
# gwt is a vector with T/F weights, weighting out any 'bad' values (determined by the simulation program - i.e. nan's etc)
# if rejmethod=T it doesn't bother with the regression, and just does rejection.

# If rejmethod=F it returns a list with the following components:-

# $x1 expected value on a logit scale, with standard errors of the estimate
# $x2 expected value on a natural scale - i.e. p(Y=y | S = s) This is what we would normally report.
# $vals - the Ys in the rejection region. The proportion of the different Ys (different models) is a Pritchard etal-style, rejection-based
#                                         estimate of the posterior probability of the different models. You might get some improvement by
#                                          weighting the frequencies with $wt.
# $wt - the Epanechnikov weight.
# $ss - the sumstats corresponding to the points in $vals.

if(missing(gwt))gwt <- rep(T,length(sumstat[,1]))

nss <- length(sumstat[1,])

# scale everything

    scaled.sumstat <- sumstat

    for(j in 1:nss){

    	scaled.sumstat[,j] <- normalise(sumstat[,j],sumstat[,j][gwt])
    }
    target.s <- target

    for(j in 1:nss){

    	target.s[j] <- normalise(target[j],sumstat[,j][gwt])
    }

# calc euclidean distance

    sum1 <- 0
    for(j in 1:nss){
    	sum1 <- sum1 + (scaled.sumstat[,j]-target.s[j])^2
   }
   dst <- sqrt(sum1)
# includes the effect of gwt in the tolerance
    dst[!gwt] <- floor(max(dst[gwt])+10)

# wt1 defines the region we're interested in
    abstol <- quantile(dst,tol)
    wt1 <- dst < abstol
    nit <- sum(wt1)

    if(rejmethod){
        l1 <- list(x=x[wt1],wt=0)
    }
    else{
        regwt <- 1-dst[wt1]^2/abstol^2

        catx <- as.numeric(names(table(x[wt1])))
        ncat <- length(catx)
        yy <- matrix(nrow=nit,ncol=ncat)
        for(j in 1:ncat){
        	yy[,j] <- as.numeric(x[wt1] == catx[j])
        }

        tr <- list()

        for(j in 1:nss){
        	tr[[j]] <- scaled.sumstat[wt1,j]
        }

        xvar.names <- paste("v",as.character(c(1:nss)),sep="")

        names(tr) <- xvar.names

        fmla <- as.formula(paste("yy ~ ", paste(xvar.names, collapse= "+")))

#        fit1 <- vglm(fmla,data=tr,multinomial) this is the version described in the
#               manuscript,which did not use the Epanechnikov weights.

        fit1 <- vglm(fmla,data=tr,multinomial,weights=regwt)

        target.df <- list()
        for(j in 1:nss){
        	target.df[[j]] <- target.s[j]
        }
        names(target.df) <- xvar.names

#        prediction1 <- predict.vglm(fit1,target.df,se.fit=T)        #commented 2014-12-10
#        prediction2 <- predict.vglm(fit1,target.df,type="response") #commented 2014-12-10
        prediction1 <- predict(fit1,target.df,se.fit=T)              #added 2014-12-10
        prediction2 <- predict(fit1,target.df,type="response")       #added 2014-12-10

        l1 <- list(x1=prediction1,x2=prediction2,vals=x[wt1],wt=regwt,ss=sumstat[wt1,])

    }
    l1
}

normalise <- function(x,y){

if(var(y) == 0)return (x - mean(y))
(x-(mean(y)))/sqrt(var(y))
}


subroutine acchk(i,j,k,anticl,x,y,ntot,eps)
# Check whether vertices i, j, k, are in anti-clockwise order.
# Called by locn, qtest, qtest1.

implicit double precision(a-h,o-z)
dimension x(-3:ntot), y(-3:ntot), xt(3), yt(3)
logical anticl

# Create indicator telling which of i, j, and k are ideal points.
if(i<=0) i1 = 1
else i1 = 0
if(j<=0) j1 = 1
else j1 = 0
if(k<=0) k1 = 1
else k1 = 0
ijk = i1*4+j1*2+k1

# Get the coordinates of vertices i, j, and k. (Pseudo-coordinates for
# any ideal points.)
xt(1) = x(i)
yt(1) = y(i)
xt(2) = x(j)
yt(2) = y(j)
xt(3) = x(k)
yt(3) = y(k)

# Get the ``normalized'' cross product.
call cross(xt,yt,ijk,cprd)

# If cprd is positive then (ij-cross-ik) is directed ***upwards***
# and so i, j, k, are in anti-clockwise order; else not.
if(cprd > eps) anticl = .true.
else anticl = .false.
return
end


#----------------------------------------------------------------------------------------------------------
# Name:         myEVfunctions.r
# Purpose:      External functions called by the EV_RL.r & EV_RP.r scripts
# Author:       Francesco Tonini
# Email: 	    f_tonini@hotmail.com
# Created:      11/1/2009
# Copyright:    (c) 2009 by Francesco Tonini
# License:    	GNU General Public License (GPL)
# Software:     Tested successfully using R-2.14.0 64-bit version(http://www.r-project.org/)
#-------------------------------------------------------------------------------------------

##CALL PACKAGES MODULE
##This module is used to call all required packages
load.packages <- function()
{
	#setRepositories(ind=1:2)
	pkg <- c("ismev","stats","tcltk2","RANN","maptools","evd")
	w <- which(pkg %in% row.names(installed.packages()) == FALSE)
	if (length(w) > 0) install.packages(pkg)[w]
	#install.packages(c("ismev","stats","tcltk2","RANN","maptools","evd")) alternatively

	#Load (call) specific packages from the existing library collection into the current R session
	require(ismev)       #An Introduction to Statistical Modeling of Extreme Values
	require(stats)       #R statistical functions
	require(tcltk2)      #Tkinter objects and classes
	require(RANN)        #spatial functions (e.g. NN interpolation)
	require(maptools)    #reading and writing shapefiles
	require(evd)		 #Functions for extreme value distributions
	#require(raster)     #functions for raster data type
	#require(extRemes)   #only for automatic return-values calculation

	cat('\nAll libraries have been loaded successfully!\n')
}

##GRADIENT FUNCTION MODULE
##This module is used within  the return level module to compute the confidence intervals
##with delta method
gevrlgradient<-function (z, p)
{
    scale <- z$mle[2]
    shape <- z$mle[3]
    if (shape < 0)
        zero.p <- p == 0
    else zero.p <- logical(length(p))
    out <- matrix(NA, nrow = 3, ncol = length(p))
    out[1, ] <- 1
    if (any(zero.p)) {
        out[2, zero.p & !is.na(zero.p)] <- rep(-shape^(-1), sum(zero.p,
            na.rm = TRUE))
        out[3, zero.p & !is.na(zero.p)] <- rep(scale * (shape^(-2)),
            sum(zero.p, na.rm = TRUE))
    }
    if (any(!zero.p)) {
        yp <- -log(1 - p[!zero.p])
        out[2, !zero.p] <- -shape^(-1) * (1 - yp^(-shape))
        out[3, !zero.p] <- scale * (shape^(-2)) * (1 - yp^(-shape)) -
            scale * shape^(-1) * yp^(-shape) * log(yp)
    }
    return(out)
}

##RETURN LEVEL CALCULATION MODULE
##This module computes return levels based on the distibution used (GEV, gumbel, GPD)
return.levels <- function (z, conf = 0.05, rperiods = c(10, 100, 210, 510, 810, 980), make.plot = TRUE)
{
    out <- list()
    out$conf.level <- conf
    eps <- 1e-06
    a <- z$mle

    #prova
    #a1<-z$mle[1] + z$mle[2]*rperiods

    std <- z$se
    mat <- z$cov
    dat <- z$data

    kappa <- qnorm(conf/2, lower.tail = FALSE)
    nx <- length(rperiods)
    cl <- 1 - conf

    if (class(z) == "gev.fit") {

		if (is.null(rperiods)) rperiods <- seq(1.1, 1000, , 200)
        if (any(rperiods <= 1))
            stop("return.level: this function presently only supports return periods >= 1")
        yp <- -log(1 - 1/rperiods)
        if (a[3] < 0)
            zero.p <- yp == 0
        else zero.p <- logical(length(rperiods))
        zero.p[is.na(zero.p)] <- FALSE
        q <- numeric(length(rperiods))
        if (any(zero.p))
            q[zero.p] <- a[1] - a[2]/a[3]
        if (any(!zero.p)) {
            if (a[3] != 0)
                q[!zero.p] <- a[1] - (a[2]/a[3]) * (1 - (yp[!zero.p])^(-a[3]))
            else if (a[3] == 0)
                q[!zero.p] <- a[1] - a[2] * log(yp[!zero.p])
        }

        d <- gevrlgradient(z = z, p = 1/rperiods)
        v <- apply(d, 2, q.form, m = mat)
        yl <- c(min(dat, q, na.rm = TRUE), max(dat, q, na.rm = TRUE))
        if (make.plot) {
            xp <- 1/yp
            plot(xp, q, log = "x", type = "n", xlim = c(0.1,
                1000), ylim = yl, xlab = "Return Period", ylab = "Return Level",
                xaxt = "n")
            axis(1, at = c(0.1, 1, 10, 100, 1000), labels = c("0.1",
                "1", "10", "100", "1000"))
            lines(xp, q)
            lines(xp, (q + kappa * sqrt(v)), col = "blue")
            lines(xp, (q - kappa * sqrt(v)), col = "blue")
            points(-1/log((1:length(dat))/(length(dat) + 1)),
                sort(dat))
        }
        out$return.level <- q
        out$return.period <- rperiods
        conf3 <- cbind(q - kappa * sqrt(v), q + kappa * sqrt(v))
        colnames(conf3) <- c("lower", "upper")
        out$confidence.delta <- conf3

	}

	if (class(z) == "gum.fit") {

        if (is.null(rperiods))
            rperiods <- seq(1.1, 1000,  length=200)
        if (any(rperiods <= 1))
            stop("return.level: this function presently only supports return periods >= 1")
        yp <- -log(1 - 1/rperiods)
        q <- a[1] - a[2] * log(yp)
        vq <- std[1]^2 + ((-log(yp))^2 * std[2]^2)
	      sq <- sqrt(vq)

		yl <- c(min(dat, q, na.rm = TRUE), max(dat, q, na.rm = TRUE))
        if (make.plot) {
            xp <- 1/yp
            plot(xp, q, log = "x", type = "n", xlim = c(0.1,
                1000), ylim = yl, xlab = "Return Period", ylab = "Return Level",
                xaxt = "n")
            axis(1, at = c(0.1, 1, 10, 100, 1000), labels = c("0.1",
                "1", "10", "100", "1000"))
            lines(xp, q)
            lines(xp, (q + kappa * sq), col = "blue")
            lines(xp, (q - kappa * sq), col = "blue")
            points(-1/log((1:length(dat))/(length(dat) + 1)),
                sort(dat))
        }

		out$return.level <- q
        out$return.period <- rperiods
        conf3 <- cbind(q - kappa * sq, q + kappa * sq)
        colnames(conf3) <- c("lower", "upper")
        out$confidence.delta <- conf3
    }

	if (class(z) == "gpd.fit") {

        u <- z$threshold
        la <- z$rate
        a <- c(la, a)
        n <- z$n
        npy <- z$npy
        xdat <- z$xdata
        if (is.null(rperiods)) {
            rperiods <- seq(0.1, 1000, , 200)
        }
        m <- rperiods * npy
        if (a[3] == 0)
            q <- u + a[2] * log(m * la)
        else q <- u + a[2]/a[3] * ((m * la)^(a[3]) - 1)
        d <- gpdrlgradient(z, m)
        mat <- matrix(c((la * (1 - la))/n, 0, 0, 0, mat[1, 1],
            mat[1, 2], 0, mat[2, 1], mat[2, 2]), nc = 3)
        v <- apply(d, 2, q.form, m = mat)
        yl <- c(u, max(xdat, q[q > u - 1] + kappa * sqrt(v)[q >
            u - 1], na.rm = TRUE))
        if (make.plot) {
            if (any(is.na(yl)))
                yl <- range(q, na.rm = TRUE)
            plot(m/npy, q, log = "x", type = "n", xlim = c(0.1,
                max(m)/npy), ylim = yl, xlab = "Return period (years)",
                ylab = "Return level", xaxt = "n")
            axis(1, at = c(0.1, 1, 10, 100, 1000), labels = c("0.1",
                "1", "10", "100", "1000"))
            lines(m[q > u - 1]/npy, q[q > u - 1])
            lines((m[q > u - 1]/npy), (q[q > u - 1] + kappa *
                sqrt(v)[q > u - 1]), col = "blue")
            lines((m[q > u - 1]/npy), (q[q > u - 1] - kappa *
                sqrt(v)[q > u - 1]), col = "blue")
            nl <- n - length(dat) + 1
            sdat <- sort(xdat)
            points((1/(1 - (1:n)/(n + 1))/npy)[sdat > u], sdat[sdat >
                u])
        }
        out$return.level <- q
        out$return.period <- m/npy
        conf3 <- cbind(q[q > u - 1] - kappa * sqrt(v)[q > u -
            1], q[q > u - 1] + kappa * sqrt(v)[q > u - 1])
        colnames(conf3) <- c("lower", "upper")
        out$confidence.delta <- conf3

    }

invisible(out)

}

##NN INTERPOLATION FOR WARNINGS MODULE (FOR RETURN LEVELS)
##This module serves the purpose of replacing pixels, flagged as warning in the main script
##because of convergence issues, with the median of the nearest 4 or 8 neighbors

##NOTE: this works better if the raster resolution is not too coarse and the terrain is
##quite uniform to that of the central pixel. This is not recommended if that is not the case, or
##if you are using a very coarse resolution raster (e.g. 10 Km or more).
median.interp_RL <- function()
{

	Matr_coord <- cbind(dataset[1:npixels,]$x, dataset[1:npixels,]$y)

	for (px in px.idWarning){

		#Rook's neighborhood (using the 'spdep' package)
		NN4 <- knearneigh(Matr_coord,4)
		Matr_NN4 <- NN4$nn

		#Queen's neighborhood (using the 'spdep' package)
		#NN8 <- knearneigh(Matr_coord,8)
		#Matr_NN8 <- NN8$nn

		#take a subset of the NN matrix only for the pixels flagged with a warning
		Sub4 <- Matr_NN4[px,]
		#Sub8 <- Matr_NN8[px,]

		Mat_param <- tab.parameters[Sub4,]
		Mat_rlevels <- tab.rlevels[Sub4,]

		tab.parameters[px,3:length(tab.parameters)] <- round(sapply(Mat_param[,-c(1:2)],median),2)
		tab.rlevels[px,2:length(tab.rlevels)] <- round(sapply(Mat_rlevels[,-1],median),2)
	}

}

##NN INTERPOLATION FOR WARNINGS MODULE (FOR PROBABILITY EXCEEDANCES)
##This module serves the purpose of replacing pixels, flagged as warning in the main script
##because of convergence issues, with the median of the nearest 4 or 8 neighbors

##NOTE: this works better if the raster resolution is not too coarse and the terrain is
##quite uniform to that of the central pixel. This is not recommended if that is not the case, or
##if you are using a very coarse resolution raster (e.g. 10 Km or more).
median.interp_RP <- function()
{

	Matr_coord <- cbind(dataset[1:npixels,]$x, dataset[1:npixels,]$y)

	for (px in px.idWarning){

		#Rook's neighborhood (using the 'spdep' package)
		NN4 <- knearneigh(Matr_coord,4)
		Matr_NN4 <- NN4$nn

		#Queen's neighborhood (using the 'spdep' package)
		#NN8 <- knearneigh(Matr_coord,8)
		#Matr_NN8 <- NN8$nn

		#take a subset of the NN matrix only for the pixels flagged with a warning
		Sub4 <- Matr_NN4[px,]
		#Sub8 <- Matr_NN8[px,]

		Mat_param <- tab.parameters[Sub4,]
		Mat_prob <- tab.prob[Sub4,]

		tab.parameters[px,3:length(tab.parameters)] <- round(sapply(Mat_param[,-c(1:2)],median),2)
		tab.prob[px,2:length(tab.prob)] <- round(sapply(Mat_prob[,-1],median),2)
	}

}

##SAVE SHAPEFILE MODULE (FOR RETURN LEVELS)
##This module is used to save a point dataset to a shapefile (.shp)
save_files_RL <- function()
{

	Matr_coord <- cbind(dataset[1:npixels,]$x, dataset[1:npixels,]$y)

	tab.parameters$X <- Matr_coord[,1]
	tab.parameters$Y <- Matr_coord[,2]
	tab.rlevels$X <- Matr_coord[,1]
	tab.rlevels$Y <- Matr_coord[,2]

	##Turn input point dataset into a SpatialPointsDataFrame
	coordinates(tab.parameters) <- ~X+Y
	coordinates(tab.rlevels) <- ~X+Y

	##Path to folder
	path <- file.path(mainDir, subDir)

	##Write .csv tables
	write.table(tab.parameters, paste(path,'/MLE_Parameters.csv',sep=''), row.names=F, sep=',')
	write.table(tab.rlevels, paste(path,'/ReturnLevels.csv',sep=''), row.names=F, sep=',')

	##Write/Save the shapefile (library 'maptools')
	writeSpatialShape( tab.parameters, paste(path,'/MLE_Parameters',sep='') )
	writeSpatialShape( tab.rlevels, paste(path,'/ReturnLevels',sep='') )

	##Write/Save rasters (library 'raster')

}

##SAVE SHAPEFILE MODULE (FOR PROBABILITY EXCEEDANCES)
##This module is used to save a point dataset to a shapefile (.shp)
save_files_RP <- function()
{

	Matr_coord <- cbind(dataset[1:npixels,]$x, dataset[1:npixels,]$y)

	tab.parameters$X <- Matr_coord[,1]
	tab.parameters$Y <- Matr_coord[,2]
	tab.prob$X <- Matr_coord[,1]
	tab.prob$Y <- Matr_coord[,2]

	##Turn input point dataset into a SpatialPointsDataFrame
	coordinates(tab.parameters) <- ~X+Y
	coordinates(tab.prob) <- ~X+Y

	##Path to folder
	path <- file.path(mainDir, subDir)

	##Write .csv tables
	write.table(tab.parameters, paste(path,'/MLE_Parameters.csv',sep=''), row.names=F, sep=',')
	write.table(tab.prob, paste(path,'/ProbExceedance.csv',sep=''), row.names=F, sep=',')

	##Write/Save the shapefile (library 'maptools')
	writeSpatialShape( tab.parameters, paste(path,'/MLE_Parameters',sep='') )
	writeSpatialShape( tab.prob, paste(path,'/ProbExceedance',sep='') )

	##Write/Save rasters (library 'raster')

}

##PROBABILITY OF EXCEEDANCE MODULE
##This module is used to calculate the probability that the event will be exceeded in any one month
##(or year, or season, depending on what is your maxima/minima time series unit).
##The return period is the inverse of the probability of exceedance
prob_fun <- function(dataset,value,tab.extreme,nome,n=3){

	px.prob <- round(1 - pgev(-return_levels, loc=mu, scale=sig, shape=shp), 3)

	#Use the following line, and not the previous, if you are working with MAXIMA instead of MINIMA
	#px.prob <- round(1 - pgev(-return_levels, loc=mu, scale=sig, shape=shp), 3)

	#NOTE: the return period will then simply be 1/prob (in the time unit used, e.g. months)

	return(px.prob)

}


source('black-scholes.r')

par(family="sans", mar=c(4,4,1,2)+0.1, mgp=c(2,1,0))

dump <- function(filename) {
  fn <- paste("images/", filename, ".pdf", sep="")
  dev.copy2pdf(file=fn, family="sans")
}

plot(function(S){bscall.value(S,100,0.5,0.05,0.20)}, 70, 130, col="#19304f", lwd=3, xlab="", ylab="")
plot(function(S){bsput.value(S,100,0.5,0.05,0.20)}, 70, 130, add=T, col="#d63d00", lwd=3)
title(main='', xlab='Stock Price', ylab='Value')
legend('topleft', c('Call', 'Put'), lwd=3, col=c("#19304f","#d63d00"), bty='n')
abline(v=100,lty=3)
dump('bs-value')

plot(function(S){bscall.delta(S,100,0.5,0.05,0.20)}, 70, 130, col="#19304f", lwd=3, xlab="", ylab="", ylim=c(-1,1))
plot(function(S){bsput.delta(S,100,0.5,0.05,0.20)}, 70, 130, add=T, col="#d63d00", lwd=3)
abline(0,0)
title('', xlab='Stock Price', ylab='Delta')
legend('topleft', c('Call', 'Put'), lwd=3, col=c("#19304f","#d63d00"), bty='n')
dump('bs-delta')

plot(function(S){bscall.delta(S,100,0.5,0.05,0.20)}, 70, 130, col="#19304f", lwd=3, xlab="", ylab="")
plot(function(S){bscall.delta(S,100,0.5,0.05,0.10)}, 70, 130, col="#d63d00", lwd=3, xlab="", ylab="", ylim=c(0,30), add=T)
plot(function(S){bscall.delta(S,100,0.5,0.05,0.05)}, 70, 130, col="#227711", lwd=3, xlab="", ylab="", ylim=c(0,30), add=T)
title('', xlab='Stock Price', ylab='Delta')
legend('topleft', c('sigma = 0.2','sigma = 0.1','sigma = 0.05'), lwd=3, col=c("#19304f","#d63d00","#227711"), bty='n')
dump('bs-delta-sigma')

plot(function(S){bscall.delta(S,100,0.5,0.05,0.20)}, 70, 130, col="#19304f", lwd=3, xlab="", ylab="")
plot(function(S){bscall.delta(S,100,0.2,0.05,0.20)}, 70, 130, col="#d63d00", lwd=3, xlab="", ylab="", ylim=c(0,30), add=T)
plot(function(S){bscall.delta(S,100,0.05,0.05,0.20)}, 70, 130, col="#227711", lwd=3, xlab="", ylab="", ylim=c(0,30), add=T)
title('', xlab='Stock Price', ylab='Delta')
legend('topleft', c('TTM = 0.5','TTM = 0.2','TTM = 0.05'), lwd=3, col=c("#19304f","#d63d00","#227711"), bty='n')
dump('bs-delta-ttm')

plot(function(S){bscall.gamma(S,100,0.5,0.05,0.20)}, 70, 130, col="#19304f", lwd=3, xlab="", ylab="", ylim=c(0,0.03))
title('', xlab='Stock Price', ylab='Gamma')
legend('topleft', c('Call Gamma'), lwd=3, col=c("#19304f","#d63d00"), bty='n')
dump('bs-gamma')

plot(function(S){bscall.value(S,100,0.5,0.05,0.20)}, 70, 130, col="#19304f", lwd=3, xlab="", ylab="")
plot(function(S){bscall.value(S,100,0.2,0.05,0.20)}, 70, 130, col="#d63d00", lwd=3, xlab="", ylab="", ylim=c(0,30), add=T)
plot(function(S){bscall.value(S,100,0.01,0.05,0.20)}, 70, 130, col="#227711", lwd=3, xlab="", ylab="", ylim=c(0,30), add=T)
title('', xlab='Stock Price', ylab='Value')
legend('topleft', c('TTM = 0.5','TTM = 0.2','TTM = 0.01'), lwd=3, col=c("#19304f","#d63d00","#227711"), bty='n')
dump('bs-value-ttm')

plot(function(S){bscall.theta(S,100,0.5,0.05,0.20)}, 70, 130, col="#19304f", lwd=3, xlab="", ylab="", ylim=c(-10,8))
plot(function(S){bsput.theta(S,100,0.5,0.05,0.20)}, 70, 130, add=T, col="#d63d00", lwd=3)
abline(0,0)
title('', xlab='Stock Price', ylab='Theta')
legend('topleft', c('Call', 'Put'), lwd=3, col=c("#19304f","#d63d00"), bty='n')
dump('bs-theta')

plot(function(S){bscall.gamma(S,100,0.5,0.05,0.20)}, 70, 130, col="#19304f", lwd=3, xlab="", ylab="", ylim=c(0,0.07))
plot(function(S){bscall.gamma(S,100,0.2,0.05,0.20)}, 70, 130, col="#d63d00", lwd=3, xlab="", ylab="", add=T)
plot(function(S){bscall.gamma(S,100,0.1,0.05,0.20)}, 70, 130, col="#227711", lwd=3, xlab="", ylab="", add=T)
title('', xlab='Stock Price', ylab='Gamma')
legend('topleft', c('TTM = 0.5','TTM = 0.2','TTM = 0.1'), lwd=3, col=c("#19304f","#d63d00","#227711"), bty='n')
dump('bs-gamma-ttm')

plot(function(S){bscall.theta(S,100,0.5,0.05,0.20)}, 70, 130, col="#19304f", lwd=3, xlab="", ylab="", ylim=c(-15,5))
plot(function(S){bscall.theta(S,100,0.2,0.05,0.20)}, 70, 130, col="#d63d00", lwd=3, xlab="", ylab="", add=T)
plot(function(S){bscall.theta(S,100,0.1,0.05,0.20)}, 70, 130, col="#227711", lwd=3, xlab="", ylab="", add=T)
abline(0,0)
title('', xlab='Stock Price', ylab='Theta')
legend('topleft', c('TTM = 0.5','TTM = 0.2','TTM = 0.1'), lwd=3, col=c("#19304f","#d63d00","#227711"), bty='n')
dump('bs-theta-ttm')

plot(function(S){bscall.vega(S,100,0.5,0.05,0.20)}, 70, 130, col="#19304f", lwd=3, xlab="", ylab="", ylim=c(0,30))
title('', xlab='Stock Price', ylab='Vega')
legend('topleft', c('Call Vega'), lwd=3, col=c("#19304f","#d63d00"), bty='n')
dump('bs-vega')


#Addition
4+4
#Substraction
52-22
#Division
15/3
#Multiplication
6*3
#Division
3^2
#Modulus
100%%10
3^2:

#Assigning values
a<-4
b<-5
a+b # Calculation


# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#' Estimate variance components using EMMA
#'
#' Build date: August 30, 2016
#' Last update: January 27, 2017
#'
#' @author EMMA (Kang et. al. Genetics, 2008), Modified only for speed up by Xiaolei Liu and Lilin Yin
#'
#' @param y phenotype, n * 2
#' @param X covariate matrix, the first column is 1s
#' @param K Kinship matrix
#' @param ngrids parameters for estimating vg and ve
#' @param llim parameters for estimating vg and ve
#' @param ulim parameters for estimating vg and ve
#' @param esp parameters for estimating vg and ve
#'
#' @return
#' Output: REML - maximum log likelihood
#' Output: delta - exp(root)
#' Output: ve - residual variance
#' Output: vg - genetic variance
#'
#' @export
#'
#' @examples
#' phePath <- system.file("extdata", "07_other", "mvp.phe", package = "rMVP")
#' phenotype <- read.table(phePath, header=TRUE)
#' print(dim(phenotype))
#' genoPath <- system.file("extdata", "06_mvp-impute", "mvp.imp.geno.desc", package = "rMVP")
#' genotype <- attach.big.matrix(genoPath)
#' print(dim(genotype))
#'
#' K <- MVP.K.VanRaden(genotype)
#' vc <- MVP.EMMA.Vg.Ve(y=phenotype[,2], X=matrix(1, nrow(phenotype)), K=K)
#' print(vc)
#'
MVP.EMMA.Vg.Ve <-
function(y, X, K, ngrids=100, llim=-10, ulim=10, esp=1e-10) {
    # NA in phenotype
    idx <- !is.na(y)
    y <- y[idx]
    X <- as.matrix(X[idx, ])
    K <- K[idx, idx]

    if(!is.numeric(y))	y <- as.numeric(as.character(y))
    emma.delta.REML.LL.wo.Z <- function(logdelta, lambda, etas)
    {
        nq <- length(etas)
        delta <-  exp(logdelta)
        return( 0.5 * (nq * (log(nq/(2 * pi))-1-log(sum(etas * etas/(lambda + delta))))-sum(log(lambda + delta))) )
    }
    emma.eigen.R.wo.Z=function(K, X) {
        n <- nrow(X)
        q <- ncol(X)

        XX <- crossprod(X)
        iXX <- try(solve(XX), silent = TRUE)
        if(inherits(iXX, "try-error")){
            #library(MASS)
            iXX <- ginv(XX)
        }

        SS1 <- X %*% iXX
        SS2 <- tcrossprod(SS1, X)
        S <- diag(n)-SS2

        eig <- eigen(S %*% (K + diag(1, n)) %*% S, symmetric=TRUE)#S4 error here

        stopifnot(!is.complex(eig$values))
        return(list(values=eig$values[1:(n-q)]-1, vectors=eig$vectors[, 1:(n-q)]))
    }
    eig.R=emma.eigen.R.wo.Z(K=K, X=X)
    n <- length(y)
    t <- nrow(K)
    q <- ncol(X)

    stopifnot(ncol(K) == t)
    stopifnot(nrow(X) == n)

    etas <- crossprod(eig.R$vectors, y)

    logdelta <- (0:ngrids)/ngrids * (ulim-llim) + llim
    m <- length(logdelta)
    delta <- exp(logdelta)
    Lambdas <- matrix(eig.R$values, n-q, m) + matrix(delta, n-q, m, byrow=TRUE)
    Etasq <- matrix(etas * etas, n-q, m)
    LL <- 0.5 * ((n-q) * (log((n-q)/(2 * pi))-1-log(colSums(Etasq/Lambdas)))-colSums(log(Lambdas)))
    dLL <- 0.5 * delta * ((n-q) * colSums(Etasq/(Lambdas * Lambdas))/colSums(Etasq/Lambdas)-colSums(1/Lambdas))
    optlogdelta <- vector(length=0)
    optLL <- vector(length=0)
    if( dLL[1] < esp ) {
        optlogdelta <- append(optlogdelta, llim)
        optLL <- append(optLL, emma.delta.REML.LL.wo.Z(llim, eig.R$values, etas))
    }
    if( dLL[m-1] > 0-esp ) {
        optlogdelta <- append(optlogdelta, ulim)
        optLL <- append(optLL, emma.delta.REML.LL.wo.Z(ulim, eig.R$values, etas))
    }
    for(i in 1:(m-1) ){
        if( ( dLL[i] * dLL[i + 1] < 0 ) && ( dLL[i] > 0 ) && ( dLL[i + 1] < 0 ) ) {
            emma.delta.REML.dLL.wo.Z <- function(logdelta, lambda, etas) {
                nq <- length(etas)
                delta <- exp(logdelta)
                etasq <- etas * etas
                ldelta <- lambda + delta
                return( 0.5 * (nq * sum(etasq/(ldelta * ldelta))/sum(etasq/ldelta)-sum(1/ldelta)) )
            }
            r <- uniroot(emma.delta.REML.dLL.wo.Z, lower=logdelta[i], upper=logdelta[i + 1], lambda=eig.R$values, etas=etas)
            optlogdelta <- append(optlogdelta, r$root)
            emma.delta.REML.LL.wo.Z <- function(logdelta, lambda, etas) {
                nq <- length(etas)
                delta <-  exp(logdelta)
                return( 0.5 * (nq * (log(nq/(2 * pi))-1-log(sum(etas * etas/(lambda + delta))))-sum(log(lambda + delta))) )
            }
            optLL <- append(optLL, emma.delta.REML.LL.wo.Z(r$root, eig.R$values, etas))
        }
    }
    maxdelta <- exp(optlogdelta[which.max(optLL)])
    #handler of grids with NaN log
    replaceNaN<-function(LL) {
        index=(LL == "NaN")
        if(length(index)>0) theMin=min(LL[!index])
        if(length(index)<1) theMin="NaN"
        LL[index]=theMin
        return(LL)
    }
    optLL=replaceNaN(optLL)
    maxLL <- max(optLL)
    maxva <- sum(etas * etas/(eig.R$values + maxdelta))/(n-q)
    maxve <- maxva * maxdelta
    return (list(REML=maxLL, delta=maxdelta, ve=maxve, vg=maxva))
}# end of MVP.EMMA.Vg.Ve function