#####################################################################################################
# Function for FDR corrected hypergeometric enrichment test
#####################################################################################################
set.based.enrichment.test=function(steps, pool, select, DB, nthread=1) {

  ## convert the database and the select and pollt to sorted integer lists
  list_of_all_genes<-unique(c(unlist(DB),pool))

  select <- intersect ( select, pool)

  ############

  DB_names <- names(DB)
  number_of_DB_categories <- length(DB)  # Number of DB categories
  size_pool <- length(pool)
  size_select <- length(select)

  # init empty data frame  with the row-size of DB (number of DB categories) to store the result
  result_df1 <- data.frame(
    DB_names=DB_names,
    DB_in_select = as.integer(NA),
    DB_in_pool = as.integer(NA),
    Genes_in_DB = as.integer(NA),
    P_val = as.double(NA),
    R_obs = as.integer(NA)
  )

   # for every DB entity in the DB list
  for (i in 1:number_of_DB_categories)
  {
    # create a vector of genes connected to the i-th DB category
    current_DB_category=DB[[i]]
    # hypergometric test
    result_df1$DB_in_select[i]=length(intersect(select,current_DB_category)) 	#q: number of common genes between a DBterm and select
    result_df1$DB_in_pool[i]=length(intersect(pool,current_DB_category))	#m: number of common genes between DBterm and BackGround
    result_df1$Genes_in_DB[i]=length(current_DB_category)

  }

  # compute the p values  (raw p values witout any corrections )
  #
  # n: number of non-pool genes among DB
  # k: number of genes in select
  # phyper(q, m, n, k, lower.tail = TRUE, log.p = FALSE)
  result_df1$P_val=1-phyper(result_df1$DB_in_select-1, result_df1$DB_in_pool, size_pool-result_df1$DB_in_pool, size_select)

  P_val_round=round(result_df1$P_val, digits=15) ## can change the digits, this is important for the precision of '0' is R
  result_df1$R_obs=rank(P_val_round, na.last = TRUE,ties.method ="max")

  result_df1$P_adj_Bonf=p.adjust(result_df1$P_val, method="bonferroni")
  result_df1$P_adj_BH=p.adjust(result_df1$P_val, method="BH")

  ################################################
  ############# simualtion
  ######

  if(length(select)==0 )
  {
    result_df1$R_exp=NaN
    result_df1$FDR=NaN

    return(result_df1)
  }

  names(DB) <- NULL

  if( nthread==1)
  {

    # simple call op C++ part of the code

    simulation_result_tbl =  tryCatch(
      enrichment_test_simulation(
        DB,
        list_of_all_genes ,
        pool,
        length(select),
        steps,
        0)
      , error = print)  # RCPP calling
  }else if(nthread>1 && round(nthread)==nthread)
  {

    # paralel call of C++

    vv<-floor(steps / nthread)
    cc<-steps %% nthread
    steps_per_thread <- c( rep(vv , nthread-cc),rep(vv+1 , cc))
    seeds_per_thread <- sample( 2^15,nthread)
    stopifnot(sum(  steps_per_thread)==steps)
    rm(cc,vv)

    library(parallel)
    cl <- makeCluster(spec=nthread, type = "PSOCK",outfile= "log1.txt")

    current_env <- environment()
    clusterExport(cl,"DB", envir = current_env)
    clusterExport(cl,"list_of_all_genes", envir = current_env)
    clusterExport(cl,"pool", envir = current_env)
    clusterExport(cl,"select", envir = current_env)
    # clusterExport(cl,"enrichment_test_simulation") #, envir = current_env
    clusterExport(cl,"seeds_per_thread", envir = current_env)
    clusterExport(cl,"steps_per_thread", envir = current_env)

    clusterEvalQ(cl, library(MulEA))

    # Sys.sleep(10)

    # muleaPkgDir <- find.package("MulEA")
    # cppSourceFile <- paste(muleaPkgDir,"/srcCpp/set-based-enrichment-test.cpp", sep = "")
    # clusterExport(cl,"cppSourceFile", envir = current_env)

    # clusterEvalQ(cl, library(Rcpp))

    # clusterEvalQ(cl, Sys.setenv("PKG_CXXFLAGS"="-std=c++11"))
    # sourceCpp(code='
    #   #include <Rcpp.h>
    #
    #   // [[Rcpp::export]]
    #   int fibonacci(const int x) {
    #     if (x == 0) return(0);
    #     if (x == 1) return(1);
    #     return (fibonacci(x - 1)) + fibonacci(x - 2);
    #   }'
    # )

    # clusterEvalQ(cl, sourceCpp(cppSourceFile))
    # clusterEvalQ(cl, Rcpp::sourceCpp(cppSourceFile))

    result_of_paralel <- clusterApplyLB(cl=cl,1:nthread, function(idx){
      simulation_result_tbl <-  tryCatch(
        enrichment_test_simulation(
          DB,
          list_of_all_genes ,
          pool,
          length(select),
          steps_per_thread[[idx]],
          seeds_per_thread[[idx]])
        , error = print)  # RCPP hívás

      return(simulation_result_tbl)
    })

    stopCluster(cl)
    simulation_result_tbl<-do.call(rbind, result_of_paralel)

  }else{
    stop("nthred parameter must be a positive integer number")
  }

  names(simulation_result_tbl)<-c("DB_in_pool","intersect.size","multiplicity") # set the column names
  simulation_result_tbl$p <- 1-phyper(simulation_result_tbl$intersect.size-1, simulation_result_tbl$DB_in_pool, length(pool)-simulation_result_tbl$DB_in_pool,  length(select))

  # test consitency
  stopifnot(steps*length(DB)==sum(simulation_result_tbl$multiplicity))  # ez nem fontos, de igy kell legyen
  if(! all(is.finite(simulation_result_tbl$p)) ) {stop("ERROR_002")} # TODO handle somehow the exception

  # from the manual of phyper(q, m, n, k)
  #
  # simulation_result_tbl$intersect.size-1  ~ q  vector of quantiles representing the number of white balls drawn without replacement from an urn which contains both black and white balls.
  #
  # simulation_result_tbl$DB_in_pool               ~ m   the number of white balls in the urn.
  # length(pool)-simulation_result_tbl$DB_in_pool  ~ n 	the number of black balls in the urn.
  # length(select)                             ~ k  the number of balls drawn from the urn, hence must be in 0,1,…, m+n.

  #############################
  # group by the equal p-values and summarise the multiplicities

  simulation_result_tbl<-simulation_result_tbl[,c("multiplicity","p")]
  simulation_result_tbl$p<-round(simulation_result_tbl$p, digits=15)
  simulation_result_tbl<-simulation_result_tbl[order(simulation_result_tbl$p),]

  list11<- split(simulation_result_tbl,simulation_result_tbl$p)

  for( i in seq(list11))
  {
    if(nrow( list11[[i]])>1)
    {
      list11[[i]]<-data.frame(multiplicity=sum(list11[[i]]$multiplicity),p=list11[[i]]$p[1])
    }
  }
  simulation_result_tbl<-do.call(rbind, list11)
  rownames(simulation_result_tbl)<-NULL
  rm(list11)

  ##############################################

  ##########

  # some preparation to help the binary search
  if(simulation_result_tbl$p[1]!=0)
  {
    simulation_result_tbl<-rbind(data.frame(multiplicity=0,p=0),simulation_result_tbl)
  }
  if(simulation_result_tbl$p[nrow(simulation_result_tbl)]!=1)
  {
    simulation_result_tbl<-rbind(simulation_result_tbl,data.frame(multiplicity=0,p=1))
  }

  simulation_result_tbl$cum_sum_multiplicity<-cumsum(simulation_result_tbl$multiplicity)

  R_exp=integer(number_of_DB_categories) # init an empty vector. I  will store here the result

  NN<-simulation_result_tbl$cum_sum_multiplicity[nrow(simulation_result_tbl)] # this id the greatest/last in cumsum.multiplicity
  for (i in 1:number_of_DB_categories) {
    if(P_val_round[i]>=1)
    {
      R_exp[i]= NN  # it makes things much faster
    }else{
      target<-P_val_round[i]

      # binary search
      a1 <- 1
      a2 <- nrow(simulation_result_tbl) #-cnt.of.ones
      while(a1+1 < a2)
      {

        a3<-floor((a1+a2)/2)
        current <- simulation_result_tbl$p[a3]
        if( current <= target)
        {
          a1<-a3
        }
        else
        {
          a2<-a3
        }
      }
      # at the end of the binary search the
      # simulation_result_tbl$cum_sum_multiplicity[[a1]] is the count of p where p<=target
      # and a2=a1+1
      #		 		R_exp[l]<-a2
      R_exp[i]<-simulation_result_tbl$cum_sum_multiplicity[[a1]]
    }
  }
  result_df1$R_exp=R_exp/steps
  result_df1$FDR=result_df1$R_exp/result_df1$R_obs

  return(result_df1)
}


#' Generic density function for HMM's emission probability
#'
#' @param x numeric matrix, size T x p, where T is the number of observations, p the dimension of the obs data
#' @param j hidden state index
#' @param model current model as defined in hmmspec.gen(..)
#' @return the emssion density of hidden state j
#' @details see example code for usage
#' @export
generic.density = function( x, j, model, NegInf=-1000 ) {
   require(corpcor)
   p = NCOL(x)
   Ttot = NROW(x)
   D = model$D
   M = model$M
   if ( length(model$mtype) != p )
      stop("no. of columns does not equal length of mtype.")

   # 1 = discrete
   ix1 = which(model$mtype == 1)
   if ( length(ix1) == 1 ) {
     # prob.v is a vector of length D, the number of multinomial outcomes
     prob.v = model$parms.emission$M1$prob.l[[j]]
     # x is coded so that it just take a simple table look-up to get the emission probability
     if ( model$Log ) {
        logb1 = log(prob.v[x])   # table look up
     } else {
        logb1 = prob.v[x]   # table look up
     }
   } else if ( length(ix1) > 1 ) {
     # prob.m is a matrix max(D) x p
     prob.m = model$parms.emission$M1$prob.l[[j]]
     bb = matrix(0, nrow=Ttot, ncol=length(ix1))
     for ( i in 1:length(ix1) ) {
       if ( model$Log ) {
         bb[,i] = log(prob.m[x[, ix1[i]], i])  # <-- make sure usage matches definition
       } else {
         bb[,i] = prob.m[x[, ix1[i]], i]  # <-- make sure usage matches definition
       }
     }
     if ( model$Log ) {
       logb1 = apply(bb, 1, sum)
     } else {
       logb1 = apply(bb, 1, prod)
     }
   } else {
     if ( model$Log ) {
       logb1 = rep(0, Ttot)
     } else {
       logb1 = rep(1, Ttot)
     }
   }

   # 2 = normal/multivariate
   ix2 = which(model$mtype == 2)
   if ( length(ix2) >= 1 ) {
     # for multivariate normal, M2$mu is a list of vector where the ith element
     # of the list is a vector for state i, and each element of the vector
     # is the mu for a gaussian density;
     # M2$sigma is a list of matrices where the ith element is the cov matrix for
     # state i.
     mu.v = model$parms.emission$M2$mu.l[[j]]
     sigma.m = model$parms.emission$M2$sigma.l[[j]]
     #cat("In generic.density, i see sigma :", as.vector(sigma.m), "\n")
     #ev = eigen(sigma.m)$values
     #cat("In generic.density, i see eigenvalues :", ev, "\n")
     if ( model$Log == FALSE & model$Intv == TRUE ) {
       logb2 = pmvnorm.intv(x[,ix2, drop=FALSE], mean=mu.v, sigma=sigma.m)
     } else {
       # Note that Dmvnorm will return a log value if log is set to TRUE.
       logb2 = Dmvnorm(x[,ix2, drop=FALSE], mean=mu.v, sigma=sigma.m, log=model$Log)
     }
   } else {
     if ( model$Log ) {
       logb2 = rep(0, Ttot)
     } else {
       logb2 = rep(1, Ttot)
     }
   }

   # 5 = gaussian mixture
   ix5 = which(model$mtype == 5)
   #cat("ix5", ix5, "\n")
   if ( length(ix5) >= 1 ) {
     # M5$mu is a list of lists where each element of the lists is a list, and
     # each element of this sublist is a vector corresponding to the mu for each
     # component of the mixture gaussian density; likewise, M5$sigma is a list
     # of lists, each list corresponds to a state, and each element of the sublist
     # corresponds to the cov matrix of a mixture component;
     # M5$c.l is a list of vectors, each element is the state i's mixture proportion
     # NOTATION :   x.ll = list of list     x.l = list
     mu.l = model$parms.emission$M5$mu.ll[[j]]
     sigma.l = model$parms.emission$M5$sigma.ll[[j]]
     c.v = model$parms.emission$M5$c.l[[j]]
     Ttot = NROW(x)
     # check for consistency
     #if ( length(c.v) != M ) {
      # cat("c.v length, M :", length(c.v), M, "\n")
      # stop("length mismatch in mixture parameters.")
     #}
     # logbb : each row is a log probability of observing one observation (a row in x[,])
     # conditioned on mu and sigma of the hidden state
     logbb = matrix(NA, nrow=Ttot, ncol=M)
     for ( m in 1:M ) {
       tmp = x[, ix5, drop=FALSE]
       mu.v = mu.l[[m]]
       sigma.m = sigma.l[[m]]
       logbb[,m] = log(c.v[m]) + Dmvnorm(tmp, mean=mu.v, sigma=sigma.m, log=TRUE)
     }
     sum.p = rep(NA, Ttot)
     # since the columns of logbb are the gaussian mixture components, we need to sum them.
     for ( tt in 1:Ttot) {
       sum.p[tt] = Elnsum.vec(logbb[tt,]) # in the non-log version: sum.pr = rowSums(bb)
     }

     if ( model$Log == TRUE ) {
       logb5 = sum.p
     } else {
       logb5 = exp(sum.p)
     }
   } else {
     if ( model$Log ) {
       logb5 = rep(0, Ttot)
     } else {
       logb5 = rep(1, Ttot)
     }
   }

   # 3 = gamma
   ix3 = which(model$mtype == 3)
   if ( length(ix3) >= 1 ) {
     shape.v = model$parms.emission$M3$shape.l[[j]]
     scale.v = model$parms.emission$M3$scale.l[[j]]
     bb = matrix(NA, ncol=length(ix3), nrow=Ttot)
     for ( i in 1:length(ix3) ) {
       tmp = dgamma(x[,ix3[i]], shape=shape.v[i], scale=scale.v[i])
       #iz = which(tmp < 1e-323)
       #tmp[iz] = 1e-323
       #cat("counting small values in dgamma:", length(iz), "\n")
       if ( model$Log ) {
         bb[,i] = log(tmp)
       } else {
         bb[,i] = tmp
       }
     }
     if ( model$Log ) {
       logb3 = apply(bb, 1, sum)
     } else {
       logb3 = apply(bb, 1, prod)
     }
   } else {
     if ( model$Log ) {
       logb3 = rep(0, Ttot)
     } else {
       logb3 = rep(1, Ttot)
     }
   }

   # 4 = beta
   ix4 = which(model$mtype == 4)
   if ( length(ix4) >= 1 ) {
     shape1.v = model$parms.emission$M4$shape1.l[[j]]
     shape2.v = model$parms.emission$M4$shape2.l[[j]]
     bb = matrix(NA, ncol=length(ix4), nrow=Ttot)
     for ( i in 1:length(ix4) ) {
        tmp = dbeta(x[,ix4[i]], shape1=shape1.v[i], shape2=shape2.v[i])
        #iz = which(tmp < 1e-323)
        #tmp[iz] = 1e-323
        #cat("counting small values in dgamma:", length(iz), "\n")
        if ( model$Log ) {
          bb[,i] = log(tmp)
        } else {
          bb[,i] = tmp
        }
     }
     if ( model$Log ) {
       logb4 = apply(bb, 1, sum)
     } else {
       logb4 = apply(bb, 1, prod)
     }
   } else {
     if ( model$Log ) {
       logb4 = rep(0, Ttot)
     } else {
       logb4 = rep(1, Ttot)
     }
   }
   if ( model$Log ) {
     b = logb1 + logb2 + logb3 + logb4 + logb5
     # NOTE : b may have -Inf, which can't be passed into C, so need to make them very small
     b[which(b < NegInf)] = NegInf
   } else {
     b = logb1 * logb2 * logb3 * logb4 * logb5
   }
   return(b)
}

#' Multivariable normal distribution at a point, calculate probability for a small interval around a point
#'
#' @param x numeric matrix
#' @param mean mean vector of the multivariate normal
#' @param sigma covariance matrix
#' @param eps small numeric neighborhood around x
#' @return vector of proabilities, length equal the number of rows in x
#' @details used in generic.density
#' @export
pmvnorm.intv = function (x, mean, sigma, eps=0.1) {
  require(mvtnorm)
  ll = x - eps/2
  uu = x + eps/2
  pp = double(NROW(x))
  for ( i in 1:NROW(x)) {
    pp[i] = pmvnorm(lower=ll[i,], upper=uu[i,], mean=mean, sigma=sigma)
  }
  return(pp)
}

#' Multivariable normal density for a hidden state evaluated at x
#'
#' @param x numeric matrix
#' @param model HMM model
#' @param j hidden state index
#' @param Log boolean whether log is used, default to FALSE
#' @return vector of proabilities, length equal the number of rows in x
#' @details used in generic.density
#' @export
dmvnorm.hext = function (x, j, model, Log=FALSE) {
  mu.v = model$parms.emission$M2$mu.l[[j]]
  sigma.m = model$parms.emission$M2$sigma.l[[j]]
  ans = Dmvnorm(x, mean=mu.v, sigma=sigma.m, log=Log)
  #ans[is.na(ans)] <- 1   # not sure what this is for?
  return(ans)
}

#' Multivariable normal density, similar to the similar named function in mvtnorm
#'
#' @param x numeric matrix
#' @param mean mean vector
#' @param sigma covariance matrix
#' @param log boolean whether log is used, default to FALSE
#' @param num.cutoff numeric cutoff value
#' @return vector of proabilities, length equal the number of rows in x
#' @details use ginv from MASS to compute inverse matrix
#' @export
Dmvnorm = function (x, mean, sigma, log=TRUE, num.cutoff=1e-100)  {
    require(MASS)  # MASS is needed because of ginv
    if (is.vector(x)) {
        x <- matrix(x, ncol = length(x))
    }
    #if (missing(mean)) {
    #    mean <- rep(0, length = NCOL(x))
    #}
    #if (missing(sigma)) {
    #    sigma <- diag(NCOL(x))
    #}
    #if (NCOL(x) != NCOL(sigma)) {
    #    stop("x and sigma have non-conforming size")
    #}
    #if (!isSymmetric(sigma, tol = sqrt(.Machine$double.eps))) {
    #   assign("SIGMA", sigma, envir = .GlobalEnv)
    #   stop("sigma must be a symmetric matrix")
    #}
    #if (length(mean) != NROW(sigma)) {
    #    stop("mean and sigma have non-conforming size")
    #}
    # Use this function from MASS, instead of the default solve()
    inv.sigma = ginv(sigma)

    distval = mahalanobis(x, center=mean, cov=inv.sigma, inverted=TRUE)
    ev = eigen(sigma, symmetric = TRUE, only.values = TRUE)$values
    logdet = sum(log(ev))

    logretval = -(NCOL(x) * log(2 * pi) + logdet + distval)/2
    ix = which(logretval > 0) # because exp(logretval) would be > 1
    #if ( length(ix) > 0 ) {
    #  cat("NCOL(x):", NCOL(x), " ")
    #  cat("logdet:", logdet, " ")
    #  cat("distval[ix]:", distval[ix], "\n")
    #}

    if (log == TRUE) {
      return(logretval)
    } else {
      # put a floor on the probability value
      tmp = exp(logretval)
      ix = which( tmp < num.cutoff )
      if ( length(ix) > 0 ) {
        tmp[ix] = num.cutoff
      }
      return(tmp)
    }
}

#' Beta emission density for hidden state j in an HMM
#'
#' @param x numeric vector
#' @param j hidden state index
#' @param model HMM model
#' @return vector of proabilities, length equal the number of rows in x
#' @export
dbeta.hext = function(x, j, model) {
    b = dbeta(x, shape1=model$parms.emission$shape1[j],
                 shape2=model$parms.emission$shape2[j])
    b[is.na(b)] = 1
    return(b)
}

#' Multivariate beta emission density for hidden state j in an HMM
#'
#' @param x numeric matrix of size T x p, where T is number of obs, p the dimension of x
#' @param j hidden state index
#' @param model HMM model
#' @return vector of proabilities, length equal the number of rows in x
#' @export
dmvbeta.hext = function(x, j, model) {
    p = NCOL(x)
    n = NROW(x)
    Shape1 = model$parms.emission$shape1[[j]]
    Shape2 = model$parms.emission$shape2[[j]]
    # shape and scale should be vector of length p
    if ( length(Shape1) != p | length(Shape2) != p )
      stop("shape1 or shape2 are of the wrong length!")
    b = matrix(NA, nrow=n, ncol=p)
    for ( i in 1:p ) {
       b[,i] = dbeta(x[,i], shape1=Shape1[i], shape2=Shape2[i])
    }
    ans = apply(b, 1, prod)
    return(ans)
}

#' Univariate gamma emission density for hidden state j in an HMM
#'
#' @param x numeric vector
#' @param j hidden state index
#' @param model HMM model
#' @return vector of proabilities, length equal the number of rows in x
#' @export
dgamma.hext = function(x, j, model) {
    b = dgamma(x, shape=model$parms.emission$shape[j],
                  scale=model$parms.emission$scale[j])
    b[is.na(b)] = 1
    return(b)
}

#' Multivariate gamma emission density for hidden state j in an HMM
#'
#' @param x numeric matrix of size T x p, where T is number of obs, p the dimension of x
#' @param j hidden state index
#' @param model HMM model
#' @return vector of proabilities, length equal the number of rows in x
#' @export
dmvgamma.hext = function(x, j, model) {
    p = NCOL(x)
    n = NROW(x)
    Shape = model$parms.emission$shape[[j]]
    Scale = model$parms.emission$scale[[j]]
    # shape and scale should be vector of length p
    if ( length(Shape) != p | length(Scale) != p )
      stop("shape or scale are of the wrong length!")
    b = matrix(NA, nrow=n, ncol=p)
    for ( i in 1:p ) {
       b[,i] = dgamma(x[,i], shape=Shape[i], scale=Scale[i])
    }
    ans = apply(b, 1, prod)
    return(ans)
}

#' Weighted covariance matrices, similar to cov.wt in hsmm
#'
#' @param x numeric matrix of size T x p, where T is number of obs, p the dimension of x
#' @param wt vector of weights for each obs
#' @param cor boolean whether correlation matrix to be returned
#' @param center centers to be used when computing covariances
#' @param method either "unbiased" or "ML
#' @return list with components cov, center, n.obs, wt, cor
#' @export
cov.wt.hext = function (x, wt = rep(1/nrow(x), nrow(x)), cor = FALSE, center = TRUE, method = c("unbiased", "ML")) {
    if (is.data.frame(x))
        x <- as.matrix(x)
    else if (!is.matrix(x))
        stop("'x' must be a matrix or a data frame")
    if (!all(is.finite(x)))
        stop("'x' must contain finite values only")
    n <- nrow(x)
    if (with.wt <- !missing(wt)) {
        if (length(wt) != n)
            stop("length of 'wt' must equal the number of rows in 'x'")
        s = sum(wt)
        if (any(wt < 0) || s == 0)
            stop("weights must be non-negative and not all zero")
        wt <- wt/s

    }
    if (is.logical(center)) {
        center <- if (center)
            colSums(wt * x)
        else 0
    }
    else {
        if (length(center) != ncol(x))
            stop("length of 'center' must equal the number of columns in 'x'")
    }
    x <- sqrt(wt) * sweep(x, 2, center, check.margin = FALSE)
    denom = 1 - sum(wt^2)
    if ( denom < 1e-300 ) {
      stop("1 - sum(wt^2) is very close to zero!\n")
    }
    cov <- switch(match.arg(method), unbiased=crossprod(x)/denom, ML = crossprod(x))
    y <- list(cov = cov, center = center, n.obs = n)
    if (with.wt)
        y$wt <- wt
    if (cor) {
        Is <- 1/sqrt(diag(cov))
        R <- cov
        R[] <- Is * cov * rep(Is, each = nrow(cov))
        y$cor <- R
    }
    y
}


model_refreezing <- function (tavg = 0.0,
         Tmf = 0.0,
         SWrf = 0.0){
    #'- Name: Refreezing -Version: 1.0, -Time step: 1
    #'- Description:
    #'            * Title: snowfall accumulation  calculation
    #'            * Author: STICS
    #'            * Reference: doi:http://dx.doi.org/10.1016/j.agrformet.2014.05.002
    #'            * Institution: INRA
    #'            * Abstract: It simulates the depth of snow cover and recalculate weather data
    #'- inputs:
    #'            * name: tavg
    #'                          ** description : average temperature
    #'                          ** inputtype : variable
    #'                          ** variablecategory : auxiliary
    #'                          ** datatype : DOUBLE
    #'                          ** default : 0.0
    #'                          ** min : 0.0
    #'                          ** max : 100.0
    #'                          ** unit : degC
    #'                          ** uri :
    #'            * name: Tmf
    #'                          ** description : threshold temperature for snow melting
    #'                          ** inputtype : parameter
    #'                          ** parametercategory : constant
    #'                          ** datatype : DOUBLE
    #'                          ** default : 0.0
    #'                          ** min : 0.0
    #'                          ** max : 5000.0
    #'                          ** unit : degC
    #'                          ** uri :
    #'            * name: SWrf
    #'                          ** description : degree-day temperature index for refreezing
    #'                          ** inputtype : parameter
    #'                          ** parametercategory : constant
    #'                          ** datatype : DOUBLE
    #'                          ** default : 0.0
    #'                          ** min : 0.0
    #'                          ** max : 5000.0
    #'                          ** unit : mmW/degC/d
    #'                          ** uri :
    #'- outputs:
    #'            * name: Mrf
    #'                          ** description : liquid water in the snow cover in the process of refreezing
    #'                          ** variablecategory : rate
    #'                          ** datatype : DOUBLE
    #'                          ** min : 0.0
    #'                          ** max : 500.0
    #'                          ** unit : mmW/d
    #'                          ** uri :
    Mrf <- 0.0
    if (tavg < Tmf)
    {
        Mrf <- SWrf * (Tmf - tavg)
    }
    return (list('Mrf' = Mrf))
}


/*
 * File: oarith.r
 *  Contents: arithmetic operators + - * / % ^.  Auxiliary routines
 *   iipow, ripow.
 *
 * The arithmetic operators all follow a canonical conversion
 * protocol encapsulated in the macro ArithOp.
 */

#ifdef DataParallel
int list_add(dptr x, dptr y, dptr z);
#endif					/* DataParallel */

#begdef ArithOp(icon_op, func_name, c_int_op, c_real_op, c_list_op)

   operator{1} icon_op func_name(x, y)
      declare {
#ifdef LargeInts
         tended struct descrip lx, ly;
#endif					/* LargeInts */
	 C_integer irslt;
         }
#ifdef DataParallel
      if is:list(x) then {
         abstract {
            return type(x) ++ type(y)
	    }
	 inline { c_list_op(&x, &y, &result); return result; }
         }
      else
#endif					/* DataParallel */
      arith_case (x, y) of {
         C_integer: {
            abstract {
               return integer
               }
            inline {
               int over_flow = 0;
               c_int_op(x,y);
               }
            }
         integer: { /* large integers only */
            abstract {
               return integer
               }
            inline {
               big_ ## c_int_op(x,y);
               }
            }
         C_double: {
            abstract {
               return real
               }
            inline {
               c_real_op(x, y);
               }
            }
         }
end

#enddef

/*
 * x / y
 */

#begdef big_Divide(x,y)
{
   if ( ( Type ( y ) == T_Integer ) && ( IntVal ( y ) == 0 ) )
      runerr(201);  /* Divide fix */

   if (bigdiv(&x,&y,&result) == RunError) /* alcbignum failed */
      runerr(0);
   return result;
}
#enddef
#begdef Divide(x,y)
{
   if (y == 0)
      runerr(201);  /* divide fix */

   irslt = div3(x,y, &over_flow);
   if (over_flow) {
#ifdef LargeInts
      MakeInt(x,&lx);
      MakeInt(y,&ly);
      if (bigdiv(&lx,&ly,&result) == RunError) /* alcbignum failed */
	 runerr(0);
      return result;
#else                                   /* LargeInts */
      runerr(203);
#endif                                  /* LargeInts */
      }
   else return C_integer irslt;
}
#enddef
#begdef RealDivide(x,y)
{
   double z;

   if (y == 0.0)
      runerr(204);

   z = x / y;
   return C_double z;
}
#enddef

ArithOp( / , divide , Divide , RealDivide, list_add /* bogus */)

/*
 * x - y
 */

#begdef big_Sub(x,y)
{
   if (bigsub(&x,&y,&result) == RunError) /* alcbignum failed */
      runerr(0);
   return result;
}
#enddef

#begdef Sub(x,y)
   irslt = sub(x,y, &over_flow);
   if (over_flow) {
#ifdef LargeInts
      MakeInt(x,&lx);
      MakeInt(y,&ly);
      if (bigsub(&lx,&ly,&result) == RunError) /* alcbignum failed */
         runerr(0);
      return result;
#else					/* LargeInts */
      runerr(203);
#endif					/* LargeInts */
      }
   else return C_integer irslt;
#enddef

#define RealSub(x,y) return C_double (x - y);

ArithOp( - , minus , Sub , RealSub, list_add /* bogus */)

/*
 * x % y
 */

#define Abs(x) ((x) > 0 ? (x) : -(x))

#begdef big_IntMod(x,y)
{
   if ( ( Type ( y ) == T_Integer ) && ( IntVal ( y ) == 0 ) ) {
      irunerr(202,0);
      errorfail;
      }
   if (bigmod(&x,&y,&result) == RunError)
      runerr(0);
   return result;
}
#enddef

#begdef IntMod(x,y)
{
   irslt = mod3(x,y, &over_flow);
   if (over_flow) {
      irunerr(202,y);
      errorfail;
      }
   return C_integer irslt;
}
#enddef

#begdef RealMod(x,y)
{
   double d;

   if (y == 0.0)
      runerr(204);

   d = fmod(x, y);
   /* d must have the same sign as x */
   if (x < 0.0) {
      if (d > 0.0) {
         d -= Abs(y);
         }
      }
   else if (d < 0.0) {
      d += Abs(y);
      }
   return C_double d;
}
#enddef

ArithOp( % , mod , IntMod , RealMod, list_add /* bogus */ )

/*
 * x * y
 */

#begdef big_Mpy(x,y)
{
   if (bigmul(&x,&y,&result) == RunError)
      runerr(0);
   return result;
}
#enddef

#begdef Mpy(x,y)
   irslt = mul(x,y,&over_flow);
   if (over_flow) {
#ifdef LargeInts
      MakeInt(x,&lx);
      MakeInt(y,&ly);
      if (bigmul(&lx,&ly,&result) == RunError) /* alcbignum failed */
         runerr(0);
      return result;
#else					/* LargeInts */
      runerr(203);
#endif					/* LargeInts */
      }
   else return C_integer irslt;
#enddef

#define RealMpy(x,y) return C_double ((long double)x * (long double)y);

ArithOp( * , mult , Mpy , RealMpy, list_add /* bogus */ )

"-x - negate x."

operator{1} - neg(x)
   if cnv:(exact)C_integer(x) then {
      abstract {
         return integer
         }
      inline {
	    C_integer i;
	    int over_flow = 0;

	    i = neg(x, &over_flow);
	    if (over_flow) {
#ifdef LargeInts
	       struct descrip tmp;
	       MakeInt(x,&tmp);
	       if (bigneg(&tmp, &result) == RunError)  /* alcbignum failed */
	          runerr(0);
               return result;
#else					/* LargeInts */
	       irunerr(203,x);
               errorfail;
#endif					/* LargeInts */
               }
         return C_integer i;
         }
      }
#ifdef LargeInts
   else if cnv:(exact) integer(x) then {
      abstract {
         return integer
         }
      inline {
	 if (bigneg(&x, &result) == RunError)  /* alcbignum failed */
	    runerr(0);
	 return result;
         }
      }
#endif					/* LargeInts */
   else {
      if !cnv:C_double(x) then
         runerr(102, x)
      abstract {
         return real
         }
      inline {
         double drslt;
	 drslt = -x;
         return C_double drslt;
         }
      }
end

"+x - convert x to a number."
/*
 *  Operational definition: generate runerr if x is not numeric.
 */
operator{1} + number(x)
   if cnv:(exact)C_integer(x) then {
       abstract {
          return integer
          }
       inline {
          return C_integer x;
          }
      }
#ifdef LargeInts
   else if cnv:(exact) integer(x) then {
       abstract {
          return integer
          }
       inline {
          return x;
          }
      }
#endif					/* LargeInts */
   else if cnv:C_double(x) then {
       abstract {
          return real
          }
       inline {
          return C_double x;
          }
      }
   else
      runerr(102, x)
end

/*
 * x + y
 */

#begdef big_Add(x,y)
{
   if (bigadd(&x,&y,&result) == RunError)
      runerr(0);
   return result;
}
#enddef

#begdef Add(x,y)
   irslt = add(x,y, &over_flow);
   if (over_flow) {
#ifdef LargeInts
      MakeInt(x,&lx);
      MakeInt(y,&ly);
      if (bigadd(&lx, &ly, &result) == RunError)  /* alcbignum failed */
	 runerr(0);
      return result;
#else					/* LargeInts */
      runerr(203);
#endif					/* LargeInts */
      }
   else return C_integer irslt;
#enddef

#define RealAdd(x,y) return C_double (x + y);

ArithOp( + , plus , Add , RealAdd, list_add )

#ifdef DataParallel

int list_add(dptr x, dptr y, dptr z)
{
   tended struct b_list *lp1;
   word size1, size2;
   word i, j, slot;

   if (is:list(*x) && is:list(*y)) {
      size1 = BlkLoc(*x)->List.size;
      size2 = BlkLoc(*y)->List.size;
      if (size1 != size2) return RunError;
#ifdef Arrays
      if ( BlkType(BlkD(*x,List)->listhead)==T_Realarray){
	 double *a, *c;

	 if ( BlkType(BlkD(*y,List)->listhead)==T_Realarray){ /*the two are real arrays*/
	    double *b;
	    if (cprealarray(x, z, (word) 1, size1 + 1) == RunError)
	       return RunError;
	    /* points to the three arrays data and copy! */
	    a = ((struct b_realarray *) BlkLoc(*x)->List.listhead )->a;
	    b = ((struct b_realarray *) BlkLoc(*y)->List.listhead )->a;
	    c = ((struct b_realarray *) BlkLoc(*z)->List.listhead)->a;

	    for(i=0; i<size1; i++)
	       c[i] = a[i] + b[i];
	    } /* two real arrays*/
	 else if ( BlkType(BlkD(*y,List)->listhead)==T_Intarray){ /*first arrays is real, second is int*/
	    word *b;

	    if (cprealarray(x, z, (word) 1, size1 + 1) == RunError)
	       return RunError;

	    a = ((struct b_realarray *) BlkLoc(*x)->List.listhead )->a;
	    b =  ((struct b_intarray *) BlkLoc(*y)->List.listhead )->a;
	    c = ((struct b_realarray *) BlkLoc(*z)->List.listhead)->a;
	    /* a is real, b is int, hopefully the c compiler knows how to do it*/
	    for(i=0; i<size1; i++)
	       c[i] = a[i] + b[i];
	    } /* two real arrays*/
	 else{
	    /* the second list is a List */
	    tended union block *ep;
	    tended struct b_realarray *apa, *apc;
	    word k=0;
	    double f;

	    if (cprealarray(x, z, (word) 1, size1 + 1) == RunError)
	       return RunError;

	    apa = (struct b_realarray *) BlkLoc(*x)->List.listhead;
	    apc = (struct b_realarray *) BlkLoc(*z)->List.listhead;

	    for (ep = BlkD(*y,List)->listhead; BlkType(ep) == T_Lelem;
	       ep = Blk(ep,Lelem)->listnext){
	       for (i = 0; i < Blk(ep,Lelem)->nused; i++) {
		  j = ep->Lelem.first + i;
		  if (j >= ep->Lelem.nslots)
		  j -= ep->Lelem.nslots;

		  if (!cnv:C_double(ep->Lelem.lslots[j], f))
		     return RunError;

		  apc->a[k] = apa->a[k] + f;
		  k++;
		  }
	       }
	    }
	 }
      else if ( BlkType(BlkD(*x,List)->listhead)==T_Intarray){
	 word *a;

	 if ( BlkType(BlkD(*y,List)->listhead)==T_Realarray){ /*first arrays is int, second is real*/
	    double *b, *c;
	    if (cprealarray(y, z, (word) 1, size1 + 1) == RunError)
	       return RunError;
	    /* points to the three arrays data and copy! */
	    a = ((struct b_intarray *) BlkLoc(*x)->List.listhead )->a;
	    b = ((struct b_realarray *) BlkLoc(*y)->List.listhead )->a;
	    c = ((struct b_realarray *) BlkLoc(*z)->List.listhead)->a;

	    for(i=0; i<size1; i++)
	       c[i] = a[i] + b[i];
	    }
	 else if ( BlkType(BlkD(*y,List)->listhead)==T_Intarray){  /*the two are int arrays*/
	    word *b, *c;

	    if (cpintarray(x, z, (word) 1, size1 + 1) == RunError)
	       return RunError;

	    a = ((struct b_intarray *) BlkLoc(*x)->List.listhead )->a;
	    b =  ((struct b_intarray *) BlkLoc(*y)->List.listhead )->a;
	    c = ((struct b_intarray *) BlkLoc(*z)->List.listhead)->a;
	    /* a is real, b is int, hopefully the c compiler knows how to do it*/
	    for(i=0; i<size1; i++)
	       c[i] = a[i] + b[i];
	    } /* two real arrays*/
	 else{
	    /* the second list is a List */
	    tended union block *ep;
	    tended struct b_intarray *apa, *apc;
	    word k=0, d;

	    if (cpintarray(x, z, (word) 1, size1 + 1) == RunError)
	       return RunError;

	    apa = (struct b_intarray *) BlkLoc(*x)->List.listhead;
	    apc = (struct b_intarray *) BlkLoc(*z)->List.listhead;

	    for (ep = BlkD(*y,List)->listhead; BlkType(ep) == T_Lelem;
	       ep = Blk(ep,Lelem)->listnext){
	       for (i = 0; i < Blk(ep,Lelem)->nused; i++) {
		  j = ep->Lelem.first + i;
		  if (j >= ep->Lelem.nslots)
		     j -= ep->Lelem.nslots;

		  /* default : The resutling array is of type int */
		  if (Type(ep->Lelem.lslots[j]) == T_Integer ){
		     if (!cnv:C_integer(ep->Lelem.lslots[j], d))
			return RunError;
		     apc->a[k] = apa->a[k] + d;
		     k++;
		     }
		  else{
		     /* we might be able to continue with real, copy the elements to
		      * a new realarray and continue
		      */
		     tended struct b_realarray *apc2;
		     double f;
		     word ii;
		     if (cpint2realarray(x, z, (word) 1, size1 + 1) == RunError)
			return RunError;

		     apc2 = (struct b_realarray *) BlkLoc(*z)->List.listhead;
		     for (ii=0; ii<k; ii++)
			apc2->a[ii] = apc->a[ii];

		     /* where we stoped in the last list lelem*/
		     ii=i;
		     /* no need to start over since elements were copied already*/
		     for (/*ep = BlkD(*y,List)->listhead*/;
			BlkType(ep) == T_Lelem; ep = Blk(ep,Lelem)->listnext){
			for (i = ii; i < Blk(ep,Lelem)->nused; i++) {
			   j = ep->Lelem.first + i;
			   if (j >= ep->Lelem.nslots)
			      j -= ep->Lelem.nslots;
			   if (!cnv:C_double(ep->Lelem.lslots[j], f))
			      return RunError;
			   apc2->a[k] = apa->a[k] + f;
			   k++;
			   }
			ii=0;
			}
		     return Succeeded;
		     }
		  } /*for i=0 */
	       } /* for ep = */
	    }
	 }
      else if ( BlkType(BlkD(*y,List)->listhead)==T_Realarray){
	 tended union block *ep;
	 tended struct b_realarray *apa, *apc;
	 word k=0;
	 double f;

	 if (cprealarray(y, z, (word) 1, size1 + 1) == RunError)
	    return RunError;

	 apa = (struct b_realarray *) BlkLoc(*y)->List.listhead;
	 apc = (struct b_realarray *) BlkLoc(*z)->List.listhead;

	 for (ep = BlkD(*x,List)->listhead; BlkType(ep) == T_Lelem;
	    ep = Blk(ep,Lelem)->listnext){
	    for (i = 0; i < Blk(ep,Lelem)->nused; i++) {
	       j = ep->Lelem.first + i;
	       if (j >= ep->Lelem.nslots)
		  j -= ep->Lelem.nslots;

	       if (!cnv:C_double(ep->Lelem.lslots[j], f))
		  return RunError;

	       apc->a[k] = apa->a[k] + f;
	       k++;
	       }
	    }
	 }
      else if ( BlkType(BlkD(*y,List)->listhead)==T_Intarray){
	 tended union block *ep;
	 tended struct b_intarray *apa, *apc;
	 word k=0, d;

	 if (cpintarray(y, z, (word) 1, size1 + 1) == RunError)
	    return RunError;

	 apa = (struct b_intarray *) BlkLoc(*y)->List.listhead;
	 apc = (struct b_intarray *) BlkLoc(*z)->List.listhead;

	 for (ep = BlkD(*x,List)->listhead; BlkType(ep) == T_Lelem;
	    ep = Blk(ep,Lelem)->listnext){
	    for (i = 0; i < Blk(ep,Lelem)->nused; i++) {
	       j = ep->Lelem.first + i;
	       if (j >= ep->Lelem.nslots)
		  j -= ep->Lelem.nslots;

	       /* default : The resutling array is of type int */
	       if (Type(ep->Lelem.lslots[j]) == T_Integer ){
		  if (!cnv:C_integer(ep->Lelem.lslots[j], d))
		     return RunError;
		  apc->a[k] = apa->a[k] + d;
		  k++;
		  }
	       else{
		  /* we might be able to continue with real, copy the elements to
		   * a new realarray and continue
		   */
		  tended struct b_realarray *apc2;
		  double f;
		  word ii;
		  if (cpint2realarray(y, z, (word) 1, size1 + 1) == RunError)
		     return RunError;

		  apc2 = (struct b_realarray *) BlkLoc(*z)->List.listhead;
		  for (ii=0; ii<k; ii++)
		     apc2->a[ii] = apc->a[ii];
		   /* where we stoped in the last list lelem*/
		  ii=i;
		  /* no need to start over since elements were copied already*/
		  for (/*ep = BlkD(*x,List)->listhead*/;
		     BlkType(ep) == T_Lelem; ep = Blk(ep,Lelem)->listnext){
		     for (i = ii; i < Blk(ep,Lelem)->nused; i++) {
			j = ep->Lelem.first + i;
			if (j >= ep->Lelem.nslots)
			   j -= ep->Lelem.nslots;
			if (!cnv:C_double(ep->Lelem.lslots[j], f))
			   return RunError;
			apc2->a[k] = apa->a[k] + f;
			k++;

			}
			ii=0;
		     }
		  return Succeeded;

		  }
	       } /*for i=0*/
	    } /* for ep */
	 }
      else{ /* the two lists are of type List  */
#endif					/* Arrays */
	 struct descrip *slotptr;
	 tended struct b_lelem *bp1;
	 if (cplist(x, z, (word)1, size1 + 1) == RunError)
	    return RunError;
	 /* add in values from y */

	 lp1 = (struct b_list *) BlkLoc(*y);
	 bp1 = (struct b_lelem *) lp1->listhead;
	 i = 1;
	 slot = 0;
	 while (size2 > 0) {
	    j = bp1->first + i - 1;
	    if (j >= bp1->nslots)
	       j -= bp1->nslots;
	    slotptr = BlkLoc(*z)->List.listhead->Lelem.lslots + slot++;
	    list_add(slotptr, bp1->lslots+j, slotptr);
	    if (++i > bp1->nused) {
	       i = 1;
	       bp1 = (struct b_lelem *) bp1->listnext;
	       }
	    size2--;
	    }
#ifdef Arrays
	 }
#endif
      }
   else if (is:list(*x)) {
      /* x a list, y a scalar */
#ifdef Arrays
      if ( BlkType(BlkD(*x,List)->listhead)==T_Realarray){
	 double *a, *c, f;
	 size1 = BlkLoc(*x)->List.size;

	 if (cprealarray(x, z, (word) 1, size1 + 1) == RunError)
	    return RunError;
	 if (!cnv:C_double(*y, f))
	    return RunError;

	 a =  ((struct b_realarray *) BlkLoc(*x)->List.listhead )->a;
	 c =  ((struct b_realarray *) BlkLoc(*z)->List.listhead )->a;

	 for(i=0; i<size1; i++)
	    c[i] = a[i] + f ;
	 }
      else if ( BlkType(BlkD(*x,List)->listhead)==T_Intarray){
	 word *a, d;
	 double f;
	 size1 = BlkLoc(*x)->List.size;
	 if ( Type ( *y ) == T_Integer ){
	    word *c;
	    if (!cnv:C_integer(*y, d))
	       return RunError;
	    if (cpintarray(x, z, (word) 1, size1 + 1) == RunError)
	       return RunError;
	    a =  ((struct b_intarray *) BlkLoc(*x)->List.listhead )->a;
	    c =  ((struct b_intarray *) BlkLoc(*z)->List.listhead )->a;
	    for(i=0; i<size1; i++)
	       c[i] = a[i] + d ;
	    }
	 else if (cnv:C_double(*y, f)){
	    double *c;
	    if (cpint2realarray(x, z, (word) 1, size1 + 1, 0 /* no need to copy elements */) == RunError)
	       return RunError;
	    a =  ((struct b_intarray *) BlkLoc(*x)->List.listhead )->a;
	    c =  ((struct b_realarray *) BlkLoc(*z)->List.listhead )->a;
	    for(i=0; i<size1; i++)
	       c[i] = a[i] + f ;
	    }
	 else
	    return RunError;
	 }
      else{
#endif					/* Arrays*/
	 struct descrip *slotptr;
	 size1 = BlkLoc(*x)->List.size;
	 if (cplist(x, z, (word)1, size1 + 1) == RunError)
	    return RunError;
	 for (i=0; i<size1; i++) {
	    slotptr = BlkLoc(*z)->List.listhead->Lelem.lslots + i;
	    list_add(slotptr, y, slotptr);
	    }
#ifdef Arrays
      }
#endif					/* Arrays */
      }
   else if (is:list(*y)) {
      /* y a list, x a scalar */
#ifdef Arrays
      if ( BlkType(BlkD(*y,List)->listhead)==T_Realarray){
	 double *a, *c, f;
	 size1 = BlkLoc(*y)->List.size;

	 if (cprealarray(y, z, (word) 1, size1 + 1) == RunError)
	    return RunError;
	 if (!cnv:C_double(*x, f))
	    return RunError;

	 a =  ((struct b_realarray *) BlkLoc(*y)->List.listhead )->a;
	 c =  ((struct b_realarray *) BlkLoc(*z)->List.listhead )->a;

	 for(i=0; i<size1; i++)
	    c[i] = a[i] + f ;
	 }
      else if ( BlkType(BlkD(*y,List)->listhead)==T_Intarray){
	 word *a, d;
	 double f;
	 size1 = BlkLoc(*y)->List.size;
	 if ( Type ( *x ) == T_Integer ){
	    word *c;
	    if (!cnv:C_integer(*x, d))
	       return RunError;
	    if (cpintarray(y, z, (word) 1, size1 + 1) == RunError)
	       return RunError;
	    a =  ((struct b_intarray *) BlkLoc(*y)->List.listhead )->a;
	    c =  ((struct b_intarray *) BlkLoc(*z)->List.listhead )->a;
	    for(i=0; i<size1; i++)
	       c[i] = a[i] + d ;
	    }
	 else if (cnv:C_double(*x, f)){
	    double *c;
	    if (cpint2realarray(y, z, (word) 1, size1 + 1, 0 /* no need to copy elements */) == RunError)
	       return RunError;
	    a =  ((struct b_intarray *) BlkLoc(*y)->List.listhead )->a;
	    c =  ((struct b_realarray *) BlkLoc(*z)->List.listhead )->a;
	    for(i=0; i<size1; i++)
	       c[i] = a[i] + f ;
	    }
	 else
	    return RunError;
	 }
      else{
#endif					/* Arrays */
	 struct descrip *slotptr;
	 size1 = BlkLoc(*y)->List.size;
	 if (cplist(y, z, (word)1, size1 + 1) == RunError)
	    return RunError;
	 for (i=0; i<size1; i++) {
	    slotptr = BlkLoc(*z)->List.listhead->Lelem.lslots + i;
	 list_add(slotptr, x, slotptr);
	 }
#ifdef Arrays
	 }
#endif					/* Arrays */
      }
   else {
      C_integer tmp, tmp2, irslt;
      double tmp3, tmp4;
#ifdef LargeInts
      tended struct descrip lx, ly;
#endif					/* LargeInts */

      /* x, y must be numeric */
      if (cnv:(exact)C_integer(*x, tmp) && cnv:(exact)C_integer(*y, tmp2)) {
         irslt = add(tmp,tmp2);
	 if (over_flow) {
#ifdef LargeInts
            MakeInt(x,&lx);
            MakeInt(y,&ly);
            if (bigadd(&lx, &ly, z) == RunError)  /* alcbignum failed */
               return RunError;
#endif					/* LargeInts */
	    }
	 else MakeInt(irslt, z);
         }
      else if (cnv:C_double(*x, tmp3) && cnv:C_double(*y, tmp4)) {
         }
      else return RunError;
      }
   return Succeeded;
}
#endif					/* DataParallel */

"x ^ y - raise x to the y power."

operator{1} ^ powr(x, y)
   if cnv:(exact)C_integer(y) then {
      if cnv:(exact)integer(x) then {
	 abstract {
	    return integer
	    }
	 inline {
#ifdef LargeInts
	    tended struct descrip ly;
	    MakeInt ( y, &ly );
	    if (bigpow(&x, &ly, &result) == RunError)  /* alcbignum failed */
	       runerr(0);
	    return result;
#else
	    int over_flow;
	    C_integer r = iipow(IntVal(x), y, &over_flow);
	    if (over_flow)
	       runerr(203);
	    return C_integer r;
#endif
	   }
	 }
      else {
	 if !cnv:C_double(x) then
	    runerr(102, x)
	 abstract {
	    return real
	    }
	 inline {
	    if (ripow( x, y, &result) ==  RunError)
	       runerr(0);
	    return result;
	    }
	 }
      }
#ifdef LargeInts
   else if cnv:(exact)integer(y) then {
      if cnv:(exact)integer(x) then {
	 abstract {
	    return integer
	    }
	 inline {
	    if (bigpow(&x, &y, &result) == RunError)  /* alcbignum failed */
	       runerr(0);
	    return result;
	    }
	 }
      else {
	 if !cnv:C_double(x) then
	    runerr(102, x)
	 abstract {
	    return real
	    }
	 inline {
	    if ( bigpowri ( x, &y, &result ) == RunError )
	       runerr(0);
	    return result;
	    }
	 }
      }
#endif					/* LargeInts */
   else {
      if !cnv:C_double(x) then
	 runerr(102, x)
      if !cnv:C_double(y) then
	 runerr(102, y)
      abstract {
	 return real
	 }
      inline {
	 if (x == 0.0 && y < 0.0)
	     runerr(204);
	 if (x < 0.0)
	    runerr(206);
	 return C_double pow(x,y);
	 }
      }
end

#if COMPILER || !(defined LargeInts)
/*
 * iipow - raise an integer to an integral power.
 */
C_integer iipow(C_integer n1, C_integer n2, int *over_flowp)
   {
   C_integer result;

   /* Handle some special cases first */
   *over_flowp = 0;
   switch ( n1 ) {
      case 1:
	 return 1;
      case -1:
	 /* Result depends on whether n2 is even or odd */
	 return ( n2 & 01 ) ? -1 : 1;
      case 0:
	 if ( n2 <= 0 )
	    *over_flowp = 1;
	 return 0;
      default:
	 if (n2 < 0)
	    return 0;
      }

   result = 1L;
   for ( ; ; ) {
      if (n2 & 01L)
	 {
	 result = mul(result, n1, over_flowp);
	 if (*over_flowp)
	    return 0;
	 }

      if ( ( n2 >>= 1 ) == 0 ) break;
      n1 = mul(n1, n1, over_flowp);
      if (*over_flowp)
	 return 0;
      }
   *over_flowp = 0;
   return result;
   }
#endif					/* COMPILER || !(defined LargeInts) */

/*
 * ripow - raise a real number to an integral power.
 */
int ripow(r, n, drslt)
double r;
C_integer n;
dptr drslt;
   {
   double retval;
   CURTSTATE();

   if (r == 0.0 && n <= 0)
      ReturnErrNum(204, RunError);
   if (n < 0) {
      /*
       * r ^ n = ( 1/r ) * ( ( 1/r ) ^ ( -1 - n ) )
       *
       * (-1) - n never overflows, even when n == MinLong.
       */
      n = (-1) - n;
      r = 1.0 / r;
      retval = r;
      }
   else
      retval = 1.0;

   /* multiply retval by r ^ n */
   while (n > 0) {
      if (n & 01L)
	 retval *= r;
      r *= r;
      n >>= 1;
      }
#ifdef DescriptorDouble
   drslt->vword.realval = retval;
#else					/* DescriptorDouble */
   Protect(BlkLoc(*drslt) = (union block *)alcreal(retval), return RunError);
#endif					/* DescriptorDouble */
   drslt->dword = D_Real;
   return Succeeded;
   }


# Function for generating NOEC distributions for each species
# --- EXAMPLE OF A SPECIAL CASE FOR SILVER
#
# Yields a matrix with dimensions number of species * number of iterations in the simulation
#
# Arguments:   - DP : matrix of data points
#              - UFt : matrix of uncertainty factors for the exposure time
#              - UFdd : matrix of uncertainty factors for the dose-descriptor
#              - SIM : number of iterations in the simulation
#              - CV.DP : coefficient of variation for the interlaboratory variation
#              - CV.UF : coefficient of variation for the use of non-substance-specific
#                uncertainty factors
#
# Date of last modification: 03.07.2019
#
# Associated publication: Systematic consideration of parameter uncertainty and variability in
#                         probabilistic species sensitivity distributions
#
# Authors: Henning Wigger, Delphine Kawecki, Bernd Nowack and Véronique Adam
#
# Institute: Empa, Swiss Federal Laboratories for Materials Science and Technology,
#            Technology and Society Laboratory, Lerchenfeldstrasse 5, 9014 St. Gallen, Switzerland
#
# submitted to Integrated Environmental Assessment and Management in July 2019

# -------------------------------------------------------------------------------------------------

do.pSSD.Ag <- function(DP,
                       UFt,
                       UFdd,
                       SIM,
                       CV.DP,
                       CV.UF){

  # test if there is no data available for one species
  if(any(apply(DP,2,function(x) length(which(!is.na(x)))) == 0)){
    warning("No data is available for one or more species, it/they won't contribute to the PSSD calculation.")
    # find which species has no data
    ind.sp.rem <- which(apply(DP,2,function(x) length(which(!is.na(x)))) == 0)
    # remove those columns
    DP <- DP[,-ind.sp.rem]
    UFt <- UFt[,-ind.sp.rem]
    UFdd <- UFdd[,-ind.sp.rem]
  }

  # Create the step distributions (or triangular or trapezoidal) for each species
  # Create an empty matrix in which step distributions will be compiled
  NOEC_comb <- matrix(NA, ncol(DP), SIM,
                      dimnames = list(colnames(DP), NULL))

  # Fill in the matrix. If there is only one data point, NOEC stays the same. If  there are
  # 2 endpoints, a uniform distribution is produced. If  there are more than 2 endpoints, a step
  # distribution is produced. One line is for one species.
  require(trapezoid)
  require(mc2d)

  # store the corrected endpoints
  corr.endpoints <- DP/(UFdd*UFt)
  sort.endpoints <- apply(corr.endpoints,2, sort)

  for (sp in colnames(DP)){

    # store the indices of the minimal and maximal data point
    ind.min <- which.min(corr.endpoints[,sp])
    ind.max <- which.max(corr.endpoints[,sp])

    # calculate the theoretical minimum and maximum of the distribution we are looking for
    if (UFt[ind.min, sp] == 10){
      sp.min <- corr.endpoints[ind.min,sp]*(1-(sqrt((CV.DP/2.45)^2 + (CV.UF/2.45)^2)))/100
    } else {
      sp.min <- corr.endpoints[ind.min,sp]*(1-(sqrt((CV.DP/2.45)^2 + (CV.UF/2.45)^2 + (CV.UF/2.45)^2)*2.45))
    }
    if (UFt[ind.max, sp] == 10){
      sp.max <- corr.endpoints[ind.max,sp]*(1+(sqrt((CV.DP/2.45)^2) + (CV.UF/2.45)^2))/1
    } else {
      sp.max <- corr.endpoints[ind.max,sp]*(1+(sqrt((CV.DP/2.45)^2 + (CV.UF/2.45)^2 + (CV.UF/2.45)^2)*2.45))
    }

    # For species with one unique data point, NOEC stays the same:
    if(length(unique(sort.endpoints[[sp]])) == 1){
      NOEC_comb[sp,] <- rtrunc("rtriang", min = sp.min,
                               mode = sort.endpoints[[sp]][1],
                               max = sp.max,
                               n = SIM, linf = 0)

      # For species with two endpoints:
    } else if(length(sort.endpoints[[sp]]) == 2){
      # Create a trapezoidal distribution including both endpoints
      NOEC_comb[sp,] <- rtrunc("rtrapezoid", SIM,
                               mode1 = sort.endpoints[[sp]][1],
                               mode2 = sort.endpoints[[sp]][2],
                               min = sp.min, max = sp.max,
                               linf = 0)

      # For species with three endpoints or more:
    } else {

      # Sample from this step distribution for each species
      NOEC_comb[sp,] <- rmore(values = sort.endpoints[[sp]], max = sp.max, min = sp.min, N = SIM, linf = 0)

    }
  }
  # return the whole matrix
  return(NOEC_comb)

}


###############################################################################
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
###############################################################################
# Collection of Utilities to Biuld and Analyze Multiple Factor Model
# Copyright (C) 2012  Michael Kapler
#
# For more information please visit my blog at www.SystematicInvestor.wordpress.com
# or drop me a line at TheSystematicInvestor at gmail
###############################################################################

###############################################################################
# Count Consecutive Changes
#' @export
###############################################################################
consecutive.changes <- function
(
	data, 		# data series
	positive=T	# count positive consecutive changes
)
{
	if(positive) dir = diff(data) > 0 else dir = diff(data) < 0

	temp = cumsum(iif(dir, 1, 0))
	temp - ifna.prev(iif(dir, NA, coredata(temp)))
}

###############################################################################
# Create plot of factors average correlations and returns
###############################################################################
# http://stackoverflow.com/questions/4310727/what-is-rs-multidimensional-equivalent-of-rbind-and-cbind
# apply(temp, 3, rbind)
# http://r.789695.n4.nabble.com/Collapse-an-array-td850008.html
#' @export
factor.avgcor <- function(data, next.month.ret, name) {
	load.packages('abind')
	# create matrix
	temp = abind(data, along = 3)
		temp = abind(next.month.ret, temp, along = 3)
		dimnames(temp)[[3]][1] = 'Ret'

	# plot
	temp = t(compute.avgcor(temp, 'spearman')[,-1])
		temp[] = plota.format(100 * temp, 0, '', '%')
		plot.table(temp, smain=paste(name,'Correlation',sep=' \n '))
}

###############################################################################
# Compute average correlations
#' @export
###############################################################################
compute.avgcor <- function
(
	data, 		# matrix with data: [rows,cols,factors]
	method = c('pearson', 'kendall', 'spearman')
)
{
	nr = dim(data)[1]
	nc = dim(data)[3]
	corm = matrix(NA,nc,nc)
		colnames(corm) = rownames(corm) = dimnames(data)[[3]]

	for( i in 1:(nc-1) ) {
		for( j in (i+1):nc ) {
			corm[i,j] = mean( as.double( sapply(1:nr, function(t)
				try(cor(data[t,,i], data[t,,j], use = 'complete.obs', method[1]),TRUE)
				)), na.rm=T)
		}
	}
	return(corm)
}

###############################################################################
# Compute Market Cap weighted mean
#' @export
###############################################################################
cap.weighted.mean <- function
(
	data, 	# factor
	capitalization	# market capitalization
)
{
	capitalization = capitalization * (!is.na(data))
	weight = capitalization / rowSums(capitalization,na.rm=T)
	rowSums(data * weight,na.rm=T)
}

###############################################################################
# Compute factor mean for each sector
#' @export
###############################################################################
sector.mean <- function
(
	data, 	# factor
	sectors	# sectors
)
{
	out = data * NA
	for(sector in levels(sectors)) {
		index = (sector == sectors)
		out[,index] = ifna(apply(data[,index, drop=F], 1, mean, na.rm=T),NA)
	}
	return(out)
}

###############################################################################
# Create quantiles
# http://en.wikipedia.org/wiki/Quantile
# rank each month stocks according to E/P factor
# create quantiles, and record their performance next month
#' @export
###############################################################################
compute.quantiles <- function
(
	data, 			# factor
	next.month.ret, # future returns
	smain='', 		# title for plot
	n.quantiles=5, 	# number of quantiles
	plot=T			# flag to create plot
)
{
	n = ncol(data)
	nperiods = nrow(data)

	data = coredata(ifna(data,NA))
	next.month.ret = coredata(ifna(next.month.ret,NA))

	temp = matrix(NA, nperiods, n.quantiles)
	hist.factor.quantiles = hist.ret.quantiles = temp

	temp = matrix(NA, nperiods, n)
	quantiles = weights = ranking = temp

	#index = which(rowSums(!is.na(data * next.month.ret)) > n/2)
	#index = which(rowSums(!is.na(data)) > n/2)
	index = which(rowSums(!is.na(data)) >= n.quantiles)
	for(t in index) {
		factor = data[t,]
		ret = next.month.ret[t,]

		ranking[t,] = rank(factor, na.last = 'keep','first')
		t.ranking = ceiling(n.quantiles * ranking[t,] / count(factor))

		quantiles[t,] = t.ranking
		weights[t,] = 1/tapply(rep(1,n), t.ranking, sum)[t.ranking]

		hist.factor.quantiles[t,] = tapply(factor, t.ranking, mean)
		hist.ret.quantiles[t,] = tapply(ret, t.ranking, mean)
	}

	# create plot
	if(plot) {
		par(mar=c(4,4,2,1))
		temp = 100*apply(hist.ret.quantiles,2,mean,na.rm=T)
 		barplot(temp, names.arg=paste(1:n.quantiles), ylab='%',
 			main=paste(smain, ', spread =',round(temp[n.quantiles]-temp[1],2), '%'))
 	}

 	return(list(quantiles=quantiles, weights=weights, ranking=ranking,
 		hist.factor.quantiles = hist.factor.quantiles, hist.ret.quantiles = hist.ret.quantiles))
}

###############################################################################
# Create Average factor
#' @export
###############################################################################
add.avg.factor <- function
(
	data	# factors
)
{
	# compute the overall factor
	temp = abind(data, along = 3)
	data$AVG = data[[1]]
	data$AVG[] = ifna(apply(temp, c(1,2), mean, na.rm=T),NA)
	return(data)
}

###############################################################################
# Convert factor to Z scores, normalize using market capitalization average
#' @export
###############################################################################
normalize.mkval <- function
(
	data,	# factors
	MKVAL	# capitalization
)
{
	# normalize (convert to z scores) cross sectionaly all factors
	for(i in names(data)) {
		#data[[i]] = (data[[i]] - apply(data[[i]], 1, mean, na.rm=T)) / apply(data[[i]], 1, sd, na.rm=T)
		data[[i]] = (data[[i]] - cap.weighted.mean(data[[i]], MKVAL)) /
							apply(data[[i]], 1, sd, na.rm=T)
	}
	return(data)
}

###############################################################################
# Convert factor to Z scores, only keep the ranks
#' @export
###############################################################################
normal.transform <- function(data)
{
	rk=rank(data, na.last='keep', ties.method = 'first')
	n = count(data)
	x = qnorm((1:n) / (n+1))
	return(x[rk])
}

#' @export
normalize.normal <- function
(
	data	# factors
)
{
	# normalize (convert to z scores) cross sectionaly all factors
	for(i in names(data)) {
		data[[i]][] = t(apply(data[[i]], 1, normal.transform))
	}
	return(data)
}

###############################################################################
# Plot Quantiles
#' @export
###############################################################################
plot.quantiles <- function
(
	data, 			# factors
	next.month.ret, # future one month returns
	smain=''		# title
)
{
	layout(matrix(1:(2*ceiling(len(data)/2)), nc=2))
	sapply(1:len(data), function(i)
		compute.quantiles(data[[i]], next.month.ret, paste(names(data)[i],smain))
	)
}

###############################################################################
# Plot Backtest Quantiles and spread (Q5-Q1)
#' @export
###############################################################################
plot.bt.quantiles <- function
(
	factors,		# factors
	next.month.ret, # future one month returns
	smain='',		# title
	data		 	# data
)
{
	out = compute.quantiles(factors, next.month.ret, plot=F)

	prices = data$prices
		prices = bt.apply.matrix(prices, function(x) ifna.prev(x))

	# find month ends
	month.ends = endpoints(prices, 'months')

	# create strategies that invest in each qutile
	models = list()

	for(i in 1:5) {
		data$weight[] = NA
			data$weight[month.ends,] = iif(out$quantiles == i, out$weights, 0)
			capital = 100000
			data$weight[] = (capital / prices) * (data$weight)
		models[[paste('Q',i,sep='')]] = bt.run(data, type='share', capital=capital)
	}

	# spread
	data$weight[] = NA
		data$weight[month.ends,] = iif(out$quantiles == 5, out$weights,
									iif(out$quantiles == 1, -out$weights, 0))
		capital = 100000
		data$weight[] = (capital / prices) * (data$weight)
	models$Q5_Q1 = bt.run(data, type='share', capital=capital)

	#*****************************************************************
	# Create Report
	#******************************************************************
	plotbt(models, plotX = T, log = 'y', LeftMargin = 3, main=smain)
		mtext('Cumulative Performance', side = 2, line = 1)
}

###############################################################################
# Plot Factors details
#' @export
###############################################################################
plot.factors <- function
(
	data, 			# factors
	name, 			# name of factor group
	next.month.ret	# future one month returns
)
{
	x = as.vector(t(data))
	y = as.vector(t(next.month.ret))
		x = ifna(x,NA)
		y = ifna(y,NA)
		index = !is.na(x) & !is.na(y)
		x = x[index]
		y = y[index]

	cor.p = round(100*cor(x, y, use = 'complete.obs', method = 'pearson'),1)
	cor.s = round(100*cor(x, y, use = 'complete.obs', method = 'spearman'),1)

	# Plot
	layout(1:2)
	plot(x, pch=20)

	par(mar=c(4,4,2,1))
	plot(x, y, pch=20, xlab=name, ylab='Next Month Return')
		abline(lm(y ~ x), col='blue', lwd=2)
		plota.legend(paste('Pearson =',cor.p,',Spearman =', cor.s))
}


# Databricks notebook source
# MAGIC %md
# MAGIC ## LASSO Regression model
# MAGIC Lasso stands for Least Absolute Shrinkage and Selection Operator. It makes use of L1 regularization technique in the objective function.
# MAGIC
# MAGIC Lasso regression is a parsimonious model that performs L1 regularization. The L1 regularization adds a penalty equivalent to the absolute magnitude of regression coefficients and tries to minimize them
# MAGIC
# MAGIC Lasso regression can perform in-built variable selection as well as parameter shrinkage. While using ridge regression one may end up getting all the variables but with Shrinked Paramaters.

# COMMAND ----------

# MAGIC %md

# COMMAND ----------

# MAGIC %sh
# MAGIC install.packages("lme4")

# COMMAND ----------

library(lme4)
library(tidyverse)
library(pscl)
library(parameters)
library(gt)
library(gsubfn)
library(proto)
library(sqldf)
library(RSQLite)
library(glmnet)
library(usethis)
library(devtools)
library(visdat)
library(skimr)
library(caret)
library(DataExplorer)

# COMMAND ----------

# MAGIC %sh ls /dbfs/FileStore/tables/

# COMMAND ----------

data<-read.csv("/dbfs/FileStore/tables/all_vars_for_zeroinf_analysis.csv")

# COMMAND ----------

head(data, 10)

# COMMAND ----------

dim(data)

# COMMAND ----------

glimpse(data)

# COMMAND ----------

summary(data)

# COMMAND ----------

str(data)

# COMMAND ----------

colnames(data)

# COMMAND ----------

colnames(data) <- c( "county", "confirmed_cases" , "confirmed_deaths" ,
 "state"  ,"length_of_lockdown" ,"cases" ,"deaths","POP_ESTIMATE_2018" , "total_state_pop",	'Active_Physicians_per_100000_Population_2018_AAMC',	'Total_Active_Patient_Care_Physicians_per_100000_Population_2018_AAMC',	'Active_Primary_Care_Physicians_per_100000_Population_2018_AAMC',	'Active_Patient_Care_Primary_Care_Physicians_per_100000_Population_2018_AAMC',	'Active_General_Surgeons_per_100000_Population_2018_AAMC',	'Active_Patient_Care_General_Surgeons_per_100000_Population_2018_AAMC',	'Percentage_of_Active_Physicians_Who_Are_Female_2018_AAMC',	'Percentage_of_Active_Physicians_Who_Are_Intertiol_Medical_Graduates_IMGs-2018_AAMC',	'Percentage_of_Active_Physicians_Who_Are_Age_60_or_Older_2018_AAMC',	'MD_and_DO_Student_Enrollment_per_100000_Population_AY_2018_2019_AAMC',	'Student_Enrollment_at_Public_MD_and_DO_Schools_per_100000_Population_AY_2018_2019_AAMC',	'Percentage_Change_in_Student_Enrollment_at_MD_and_DO_Schools_2008_2018_AAMC',	'Percentage_of_MD_Students_Matriculating_In_State_AY_2018_2019_AAMC',	'Total_Residents_Fellows_in_ACGME_Programs_per_100000_Population_as_of_December_31_2018_AAMC',	'Total_Residents_Fellows_in_Primary_Care_ACGME_Programs_per_100000_Population_as_of_Dec_31_2018_AAMC',	'Percentage_of_Residents_in_ACGME_Programs_Who_Are_IMGs_as_of_December_31_2018_AAMC',	'Ratio_of_Residents_and_Fellows_GME_to_Medical_Students_UME-AY_2017_2018_AAMC',	'Percent_Change_in_Residents_and_Fellows_in_ACGME_Accredited_Programs_2008_2018_AAMC',	'Percentage_of_Physicians_Retained_in_State_from_Undergraduate_Medical_Education_UME-2018_AAMC',	'All_Specialties_AAMC',	'State_Local_Government_hospital_beds_per_1000_people_2019',	'Non_profit_hospital_beds_per_1000_people_2019',	'For_profit_hospital_beds_per_1000_people_2019',	'Total_hospital_beds_per_1000_people_2019',	'Total_nurse_practitioners_2019',	'Total_physician_assistants_2019',	'Total_Hospitals_2019',	'Total_Primary_Care_Physicians_2019',	'Surgery_specialists_2019',	'Emergency_Medicine_specialists_2019',	'Total_Specialist_Physicians_2019',	'ICU_Beds',	'pop_fraction',	'Length_of_Life_rank',	'Quality_of_Life_rank',	'Health_Behaviors_rank',	'Clinical_Care_rank',	'Social-Economic_Factors_rank',	'Physical_Environment_rank',	'Adult_smoking_percentage',	'Adult_obesity_percentage',	'Excessive_drinking_percentage',	'Population_per_sq_mile',	'House_per_sq_mile',	'Share_of_Tests_with_Positive_COVID_19_Results', 'Number_of_Tests_with_Results_per_1000_Population'
)

# COMMAND ----------

view(data)

# COMMAND ----------

select(data, county, confirmed_cases)

# COMMAND ----------

data %>%
    group_by(county) %>%
    summarise(count = n())

# COMMAND ----------

vis_miss(data)
vis_dat(data)

# COMMAND ----------

skim(data)

# COMMAND ----------

#DataExplorer::create_report(data)

# COMMAND ----------

#colnames(data)

# COMMAND ----------

data %>%
select(county,length_of_lockdown, confirmed_cases,confirmed_deaths,POP_ESTIMATE_2018, POP_ESTIMATE_2018,ICU_Beds, Adult_obesity_percentage, Quality_of_Life_rank, Excessive_drinking_percentage, Population_per_sq_mile,
     Clinical_Care_rank, Adult_smoking_percentage,Total_Specialist_Physicians_2019,Physical_Environment_rank,  Number_of_Tests_with_Results_per_1000_Population) -> final_data

# COMMAND ----------

display(final_data)

# COMMAND ----------

#Creating dependent and independent variables.
final_data %>%
select(confirmed_deaths, ICU_Beds, Adult_obesity_percentage, Quality_of_Life_rank, Excessive_drinking_percentage, Population_per_sq_mile, Clinical_Care_rank, Adult_smoking_percentage, Total_Specialist_Physicians_2019, Physical_Environment_rank, Number_of_Tests_with_Results_per_1000_Population) -> X_variables

final_data %>%
select(confirmed_deaths) -> y_target

# COMMAND ----------

# MAGIC %md
# MAGIC
# MAGIC ### Linear Regression

# COMMAND ----------

set.seed(100)

index = sample(1:nrow(final_data), 0.7*nrow(final_data))

train = final_data[index,] # Create the training data
test = final_data[-index,] # Create the test data

dim(train)

# COMMAND ----------

train

# COMMAND ----------

dim(test)

# COMMAND ----------

lr = lm(confirmed_deaths ~  ICU_Beds + Adult_obesity_percentage + Quality_of_Life_rank + Excessive_drinking_percentage + Population_per_sq_mile + Clinical_Care_rank + Adult_smoking_percentage+ Total_Specialist_Physicians_2019 + Physical_Environment_rank + Number_of_Tests_with_Results_per_1000_Population, data = train)
summary(lr)

# COMMAND ----------

#Step 1 - create the evaluation metrics function

eval_metrics = function(model, df, predictions, target){
    resids = df[,target] - predictions
    resids2 = resids**2
    N = length(predictions)
    r2 = as.character(round(summary(model)$r.squared, 2))
    adj_r2 = as.character(round(summary(model)$adj.r.squared, 2))
    print(adj_r2) #Adjusted R-squared
    print(as.character(round(sqrt(sum(resids2)/N), 2))) #RMSE
}

# Step 2 - predicting and evaluating the model on train data
predictions = predict(lr, newdata = train)
eval_metrics(lr, train, predictions, target = 'confirmed_deaths')

# Step 3 - predicting and evaluating the model on test data
predictions = predict(lr, newdata = test)
eval_metrics(lr, test, predictions, target = 'confirmed_deaths')

# COMMAND ----------

# MAGIC %md
# MAGIC The above output shows that RMSE, one of the two evaluation metrics, is 80.82 for train data and 84.69  for test data. On the other hand, R-squared value is around 25 percent for both train and test data, which indicates bad performance.

# COMMAND ----------

# MAGIC %md
# MAGIC
# MAGIC ### Regularization

# COMMAND ----------

cols_reg = c('county', 'confirmed_deaths', 'ICU_Beds', 'Adult_obesity_percentage', 'Quality_of_Life_rank', 'Excessive_drinking_percentage', 'Population_per_sq_mile', 'Clinical_Care_rank', 'Adult_smoking_percentage', 'Total_Specialist_Physicians_2019', 'Physical_Environment_rank', 'Number_of_Tests_with_Results_per_1000_Population')

dummies <- dummyVars(confirmed_deaths ~ ., data = final_data[,cols_reg])

train_dummies = predict(dummies, newdata = train[,cols_reg])

test_dummies = predict(dummies, newdata = test[,cols_reg])

print(dim(train_dummies)); print(dim(test_dummies))

# COMMAND ----------

# MAGIC %md
# MAGIC
# MAGIC ## Ridge Regression
# MAGIC
# MAGIC Ridge regression is an extension of linear regression where the loss function is modified to minimize the complexity of the model.
# MAGIC
# MAGIC This modification is done by adding a penalty parameter that is equivalent to the square of the magnitude of the coefficients.

# COMMAND ----------

x = as.matrix(train_dummies)
y_train = train$confirmed_deaths

x_test = as.matrix(test_dummies)
y_test = test$confirmed_deaths

lambdas <- 10^seq(2, -3, by = -.1)
ridge_reg = glmnet(x, y_train, nlambda = 25, alpha = 0, family = 'gaussian', lambda = lambdas)

summary(ridge_reg)

# COMMAND ----------

# COMMAND ----------

cv_ridge <- cv.glmnet(x, y_train, alpha = 0, lambda = lambdas)
optimal_lambda <- cv_ridge$lambda.min
optimal_lambda

# COMMAND ----------

# MAGIC %md
# MAGIC The optimal lambda value comes out to be 7.943282 and will be used to build the ridge regression model

# COMMAND ----------

# Compute R^2 from true and predicted values
eval_results <- function(true, predicted, df) {
  SSE <- sum((predicted - true)^2)
  SST <- sum((true - mean(true))^2)
  R_square <- 1 - SSE / SST
  RMSE = sqrt(SSE/nrow(df))

  # Model performance metrics
data.frame(
  RMSE = RMSE,
  Rsquare = R_square
)

}

# Prediction and evaluation on train data
predictions_train <- predict(ridge_reg, s = optimal_lambda, newx = x)
eval_results(y_train, predictions_train, train)

# COMMAND ----------

# Prediction and evaluation on test data
predictions_test <- predict(ridge_reg, s = optimal_lambda, newx = x_test)
eval_results(y_test, predictions_test, test)

# COMMAND ----------

# MAGIC %md
# MAGIC 80.82 for train data and 84.69  for test data.
# MAGIC
# MAGIC The above output shows that the RMSE and R-squared values for the ridge regression model on the training data are 80.89787 and 25 percent, respectively. For the test data, the results for these metrics are 84 and 7 percent, respectively. There is an improvement in the performance compared with linear regression model.

# COMMAND ----------

# MAGIC %md
# MAGIC ### Lasso Regression model
# MAGIC
# MAGIC Lasso regression, or the Least Absolute Shrinkage and Selection Operator, is also a modification of linear regression.
# MAGIC
# MAGIC In lasso, the loss function is modified to minimize the complexity of the model by limiting the sum of the absolute values of the model coefficients (also called the l1-norm).
# MAGIC
# MAGIC The loss function for lasso regression can be expressed as below:
# MAGIC
# MAGIC Loss function = OLS + alpha * summation (absolute values of the magnitude of the coefficients)

# COMMAND ----------

# MAGIC %md
# MAGIC
# MAGIC The first step to build a lasso model is to find the optimal lambda value using the code below.

# COMMAND ----------

lambdas <- 10^seq(2, -3, by = -.1)

# Setting alpha = 1 implements lasso regression
lasso_reg <- cv.glmnet(x, y_train, alpha = 1, lambda = lambdas, standardize = TRUE, nfolds = 5)

# Best
lambda_best <- lasso_reg$lambda.min
lambda_best

# COMMAND ----------

# MAGIC %md
# MAGIC Once we have the optimal lambda value, we train the lasso model in the first line of code below.

# COMMAND ----------

lasso_model <- glmnet(x, y_train, alpha = 1, lambda = lambda_best, standardize = TRUE)

predictions_train <- predict(lasso_model, s = lambda_best, newx = x)
eval_results(y_train, predictions_train, train)

# COMMAND ----------

predictions_test <- predict(lasso_model, s = lambda_best, newx = x_test)
eval_results(y_test, predictions_test, test)

# COMMAND ----------

# MAGIC %md
# MAGIC
# MAGIC ## Elastic Net Regression model
# MAGIC
# MAGIC Elastic net regression combines the properties of ridge and lasso regression.
# MAGIC
# MAGIC It works by penalizing the model using both the 1l2-norm1 and the 1l1-norm1.
# MAGIC
# MAGIC The model can be easily built using the caret package, which automatically selects the optimal value of parameters alpha and lambda.

# COMMAND ----------

# MAGIC %md
# MAGIC
# MAGIC - The first line of code creates the training control object train_cont which specifies how the repeated cross validation will take place.
# MAGIC
# MAGIC - The second line builds the elastic regression model in which a range of possible alpha and lambda values are tested and their optimum value is selected. The argument tuneLength specifies that 10 different combinations of values for alpha and lambda are to be tested.

# COMMAND ----------

# Set training control
train_cont <- trainControl(method = "repeatedcv",
                              number = 10,
                              repeats = 5,
                              search = "random",
                              verboseIter = TRUE)

# Train the model
elastic_reg <- train(confirmed_deaths ~ .,
                           data = train,
                           method = "glmnet",
                           preProcess = c("center", "scale"),
                           tuneLength = 10,
                           trControl = train_cont)

# Best tuning parameter
elastic_reg$bestTune

# COMMAND ----------

# MAGIC %md
# MAGIC Once we have trained the model, we use it to generate the predictions and print the evaluation results for both the training and test datasets

# COMMAND ----------

# Make predictions on training set
predictions_train <- predict(elastic_reg, x)

eval_results(y_train, predictions_train, train)

# COMMAND ----------

# COMMAND ----------

# Make predictions on test set
predictions_test <- predict(elastic_reg, x_test)
eval_results(y_test, predictions_test, test)

# COMMAND ----------

# MAGIC %md
# MAGIC
# MAGIC The above output shows that the RMSE and R-squared values for the elastic net regression model on the training data are and  percent, respectively.
# MAGIC
# MAGIC The results for these metrics on the test data are  and 86 , respectively.
# MAGIC
# MAGIC Conclusion
# MAGIC
# MAGIC The performance of the models is summarized below:
# MAGIC
# MAGIC - Linear Regression Model: Test set RMSE of  and R-square of  percent.
# MAGIC
# MAGIC - Ridge Regression Model: Test set RMSE of and R-square of  percent.
# MAGIC
# MAGIC - Lasso Regression Model: Test set RMSE of  and R-square of percent.
# MAGIC
# MAGIC - ElasticNet Regression Model: Test set RMSE of  and R-square of  percent.

# COMMAND ----------

Y1 <- as.matrix(y_target)
is.matrix(Y1)
X1 <- as.matrix(X_variables)
is.matrix(X_variables)
CV = cv.glmnet(x=X1, y=Y1, family= "gaussian",standardize=FALSE, type.measure = "mae", alpha = 0)

# COMMAND ----------

plot(CV)

# COMMAND ----------

fit = (glmnet(x=X1, y=Y1, family= "gaussian", alpha=0,standardize=FALSE, lambda=CV$lambda.1se))

# COMMAND ----------

fit$beta[,1]
ridge <- as.matrix(fit$beta)

# COMMAND ----------

write.table(ridge,"/databricks/driver/ridge.txt", sep="\t")

# COMMAND ----------

plot(fit, label = TRUE)
plot(fit)
print(fit)

write.table

# COMMAND ----------


model_tempmax <- function (Sdepth_cm = 0.0,
         prof = 0.0,
         tmax = 0.0,
         tminseuil = 0.0,
         tmaxseuil = 0.0){
    #'- Name: TempMax -Version: 1.0, -Time step: 1
    #'- Description:
    #'            * Title: Maximum temperature  recalculation
    #'            * Author: STICS
    #'            * Reference: doi:http://dx.doi.org/10.1016/j.agrformet.2014.05.002
    #'            * Institution: INRA
    #'            * Abstract: recalculation of maximum temperature
    #'- inputs:
    #'            * name: Sdepth_cm
    #'                          ** description : snow depth
    #'                          ** inputtype : variable
    #'                          ** variablecategory : state
    #'                          ** datatype : DOUBLE
    #'                          ** default : 0.0
    #'                          ** min : 0.0
    #'                          ** max : 500.0
    #'                          ** unit : cm
    #'                          ** uri :
    #'            * name: prof
    #'                          ** description : snow cover threshold for snow insulation
    #'                          ** inputtype : parameter
    #'                          ** parametercategory : constant
    #'                          ** datatype : DOUBLE
    #'                          ** default : 0.0
    #'                          ** min : 0.0
    #'                          ** max : 1000
    #'                          ** unit : cm
    #'                          ** uri :
    #'            * name: tmax
    #'                          ** description : current maximum air temperature
    #'                          ** inputtype : variable
    #'                          ** variablecategory : auxiliary
    #'                          ** datatype : DOUBLE
    #'                          ** default : 0.0
    #'                          ** min : 0.0
    #'                          ** max : 100.0
    #'                          ** unit : degC
    #'                          ** uri :
    #'            * name: tminseuil
    #'                          ** description : minimum temperature when snow cover is higher than prof
    #'                          ** inputtype : parameter
    #'                          ** parametercategory : constant
    #'                          ** datatype : DOUBLE
    #'                          ** default : 0.0
    #'                          ** min : 0.0
    #'                          ** max : 5000.0
    #'                          ** unit : degC
    #'                          ** uri :
    #'            * name: tmaxseuil
    #'                          ** description : maximum temperature when snow cover is higher than prof
    #'                          ** inputtype : parameter
    #'                          ** parametercategory : constant
    #'                          ** datatype : DOUBLE
    #'                          ** default : 0.0
    #'                          ** min :
    #'                          ** max :
    #'                          ** unit : degC
    #'                          ** uri :
    #'- outputs:
    #'            * name: tmaxrec
    #'                          ** description : recalculated maximum temperature
    #'                          ** variablecategory : state
    #'                          ** datatype : DOUBLE
    #'                          ** min : 0.0
    #'                          ** max : 500.0
    #'                          ** unit : degC
    #'                          ** uri :
    tmaxrec <- tmax
    if (Sdepth_cm > prof)
    {
        if (tmax < tminseuil)
        {
            tmaxrec <- tminseuil
        }
        else
        {
            if (tmax > tmaxseuil)
            {
                tmaxrec <- tmaxseuil
            }
        }
    }
    else
    {
        if (Sdepth_cm > 0.0)
        {
            if (tmax <= 0.0)
            {
                tmaxrec <- tmaxseuil - ((1 - (Sdepth_cm / prof)) * -tmax)
            }
            else
            {
                tmaxrec <- 0.0
            }
        }
    }
    return (list('tmaxrec' = tmaxrec))
}


# 4. faza: Analiza podatkov

# združevanje v skupine

library(tmap)

zivlj_doba_razv <- data.frame(filter(zivlj_doba, SPOL=='m'))
zivlj_doba_razv <- zivlj_doba_razv[-c(3)]
prorac_razv <- filter(zdravstvo, LETO == '2016')
prorac_razv <- prorac_razv[c(1,3)]
postelje_razv <- filter(zdravstvo, LETO == '2016')
postelje_razv <- postelje_razv[c(1,4)]
tabela_razvrscanje1 <- inner_join(zivlj_doba_razv, prorac_razv, by="DRZAVA")
tabela_razvrscanje2 <- inner_join(tabela_razvrscanje1, postelje_razv, by="DRZAVA")
rownames(tabela_razvrscanje2) =  tabela_razvrscanje2$DRZAVA
tabela_razvrscanje <- tabela_razvrscanje2[-c(1)]

set.seed(1000)
clustri_zdr <- kmeans(scale(tabela_razvrscanje), 4, nstart=100)
tabela1 <- data.frame(DRZAVA = tabela_razvrscanje2$DRZAVA, ZIVLJENJSKA_DOBA = tabela_razvrscanje2$ZIVLJENJSKA_DOBA,
                      PRORACUN = tabela_razvrscanje2$PRORACUN, POSTELJE = tabela_razvrscanje2$POSTELJE,
                      SKUPINA = factor(clustri_zdr$cluster, ordered = TRUE))
skupaj3 <- left_join(drzave, tabela1, by="DRZAVA")
zemljevid_cluster_zdr <- ggplot() + geom_polygon(data=left_join(zemljevid_evrope, skupaj3, by=c("NAME"="DRZAVA")),
                                                aes(x=long, y=lat, group=group, fill=SKUPINA)) +
  geom_line() +
  theme(axis.text.x=element_blank(), axis.ticks.x=element_blank(), axis.text.y=element_blank(),
        axis.ticks.y=element_blank()) +
  scale_fill_manual(values = c('yellow', 'orange', 'orangered', 'red3'),
                    labels = c('1', '2', '3', '4'), na.value="grey") +
  labs(x = " ") +
  labs(y = " ") +
  ggtitle("Razvrstitev držav v skupine glede na zdravstveno stanje")

#print(zemljevid_cluster_zdr)

tabela_razvitost <- data.frame(filter(razvitost, LETO == '2016'))
tabela_razvitost$BREZPOSELNI <- -(tabela_razvitost$BREZPOSELNI)
rownames(tabela_razvitost) =  tabela_razvitost$DRZAVA
tabela_razvitost <- na.omit(tabela_razvitost)[-c(2)]
tabela_razvitost1 <- tabela_razvitost[-c(1)]

set.seed(1000)
clustri_razv <- kmeans(scale(tabela_razvitost1), 4, nstart=100)
tabela2 <- data.frame(DRZAVA = tabela_razvitost$DRZAVA, BDP = tabela_razvitost$BDP,
                      DOHODEK = tabela_razvitost$DOHODEK, BREZPOSELNI = tabela_razvitost$BREZPOSELNI,
                      SKUPINA = factor(clustri_razv$cluster, ordered = TRUE))
skupaj4 <- left_join(drzave, tabela2, by="DRZAVA")
zemljevid_cluster_razv <- ggplot() + geom_polygon(data=left_join(zemljevid_evrope, skupaj4, by=c("NAME"="DRZAVA")),
                                             aes(x=long, y=lat, group=group, fill=SKUPINA)) +
  geom_line() +
  theme(axis.text.x=element_blank(), axis.ticks.x=element_blank(), axis.text.y=element_blank(),
        axis.ticks.y=element_blank()) +
  scale_fill_manual(values = c('yellow', 'orange', 'orangered', 'red3'),
                    labels = c('1', '2', '3', '4'), na.value="grey") +
  labs(x = " ") +
  labs(y = " ") +
  ggtitle("Razvrstitev držav v skupine glede na gospodarsko razvitost")

#print(zemljevid_cluster_razv)

# regresija življenjske dobe

zivlj_doba_evropa <- group_by(zivlj_doba_regresija, LETO) %>% summarise(POVPRECJE=mean(ZIVLJ_DOBA))

prileganje <- lm(data = zivlj_doba_evropa, POVPRECJE ~ LETO)

napoved <- data.frame(LETO=seq(2019, 2025, 1)) %>% mutate(POVPRECJE=predict(prileganje, data.frame(LETO=seq(2019, 2025, 1))))

graf_regresija <- ggplot(zivlj_doba_evropa, aes(x=LETO, y=POVPRECJE)) +
  geom_smooth(method=lm, fullrange = TRUE, color = 'blue') +
  geom_point(data=napoved, aes(x=LETO, y=POVPRECJE), color='red', size=2) +
  geom_point() +
  labs(title='Napoved življenjske dobe za države Evropske unije', y="ŽIVLJENJSKA DOBA")

#print(graf_regresija)